From 7fa7dddc9bc920a3a5078ed8ddfcb5838ff4babd Mon Sep 17 00:00:00 2001 From: david hill Date: Wed, 23 May 2018 14:45:47 -0500 Subject: [PATCH 01/33] bump release --- README | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) diff --git a/README b/README index 8186d1ed7..ca0d9cd19 100644 --- a/README +++ b/README @@ -1,9 +1,9 @@ -This is MariaDB ColumnStore 1.0.12 -MariaDB ColumnStore 1.0.12 is the development version of MariaDB ColumnStore. +This is MariaDB ColumnStore 1.0.14 +MariaDB ColumnStore 1.0.14 is the development version of MariaDB ColumnStore. It is built by porting InfiniDB 4.6.7 on MariaDB 10.1.26 and adding entirely new features not found anywhere else. -MariaDB ColumnStore 1.0.12 is an GA release. This is the first MariaDB +MariaDB ColumnStore 1.0.14 is an GA release. This is the first MariaDB ColumnStore release, not all features planned for the MariaDB ColumnStore 1.0 series are included in this release. From 60eb6516a25bc49a45adb2f4613a885458d9d058 Mon Sep 17 00:00:00 2001 From: david hill Date: Wed, 23 May 2018 14:46:31 -0500 Subject: [PATCH 02/33] bump release --- README.md | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) diff --git a/README.md b/README.md index 40c1e99ec..5cd3824eb 100644 --- a/README.md +++ b/README.md @@ -1,9 +1,9 @@ -#MariaDB ColumnStore Storage/Execution engine 1.0.12 -MariaDB ColumnStore 1.0.12 is the development version of MariaDB ColumnStore. +#MariaDB ColumnStore Storage/Execution engine 1.0.14 +MariaDB ColumnStore 1.0.14 is the development version of MariaDB ColumnStore. It is built by porting InfiniDB 4.6.7 on MariaDB 10.1.26 and adding entirely new features not found anywhere else. -#MariaDB ColumnStore 1.0.12 is an GA release. +#MariaDB ColumnStore 1.0.14 is an GA release. #Building This repository is not meant to be built independently outside of the server. This repository is integrated into http://mariadb-corporation/mariadb-columnstore-server (ie, the *server*) as a git submodule. As such, you can find complete build instructions on *the server* page. From 4ad2c0a8f2cb77554db2f9a7311522e5bfa6c046 Mon Sep 17 00:00:00 2001 From: david hill Date: Wed, 23 May 2018 14:47:05 -0500 Subject: [PATCH 03/33] bump release --- VERSION | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/VERSION b/VERSION index 7053d9518..1e84dad87 100644 --- a/VERSION +++ b/VERSION @@ -1,4 +1,4 @@ COLUMNSTORE_VERSION_MAJOR=1 COLUMNSTORE_VERSION_MINOR=0 -COLUMNSTORE_VERSION_PATCH=13 +COLUMNSTORE_VERSION_PATCH=14 COLUMNSTORE_VERSION_RELEASE=1 From 6664dc832c99ea3c181288d712f5f9cc0896d547 Mon Sep 17 00:00:00 2001 From: David Hill Date: Thu, 16 Aug 2018 08:27:47 -0500 Subject: [PATCH 04/33] update version to 1.0.15 --- README | 8 ++++---- README.md | 8 ++++---- VERSION | 2 +- 3 files changed, 9 insertions(+), 9 deletions(-) diff --git a/README b/README index ca0d9cd19..3eac5c82e 100644 --- a/README +++ b/README @@ -1,9 +1,9 @@ -This is MariaDB ColumnStore 1.0.14 -MariaDB ColumnStore 1.0.14 is the development version of MariaDB ColumnStore. -It is built by porting InfiniDB 4.6.7 on MariaDB 10.1.26 and adding entirely +This is MariaDB ColumnStore 1.0.15 +MariaDB ColumnStore 1.0.15 is the development version of MariaDB ColumnStore. +It is built by porting InfiniDB 4.6.7 on MariaDB 10.1.28 and adding entirely new features not found anywhere else. -MariaDB ColumnStore 1.0.14 is an GA release. This is the first MariaDB +MariaDB ColumnStore 1.0.15 is an GA release. This is the first MariaDB ColumnStore release, not all features planned for the MariaDB ColumnStore 1.0 series are included in this release. diff --git a/README.md b/README.md index 5cd3824eb..aa84a12ea 100644 --- a/README.md +++ b/README.md @@ -1,9 +1,9 @@ -#MariaDB ColumnStore Storage/Execution engine 1.0.14 -MariaDB ColumnStore 1.0.14 is the development version of MariaDB ColumnStore. -It is built by porting InfiniDB 4.6.7 on MariaDB 10.1.26 and adding entirely +#MariaDB ColumnStore Storage/Execution engine 1.0.15 +MariaDB ColumnStore 1.0.15 is the development version of MariaDB ColumnStore. +It is built by porting InfiniDB 4.6.7 on MariaDB 10.1.28 and adding entirely new features not found anywhere else. -#MariaDB ColumnStore 1.0.14 is an GA release. +#MariaDB ColumnStore 1.0.15 is an GA release. #Building This repository is not meant to be built independently outside of the server. This repository is integrated into http://mariadb-corporation/mariadb-columnstore-server (ie, the *server*) as a git submodule. As such, you can find complete build instructions on *the server* page. diff --git a/VERSION b/VERSION index 1e84dad87..50ab911ca 100644 --- a/VERSION +++ b/VERSION @@ -1,4 +1,4 @@ COLUMNSTORE_VERSION_MAJOR=1 COLUMNSTORE_VERSION_MINOR=0 -COLUMNSTORE_VERSION_PATCH=14 +COLUMNSTORE_VERSION_PATCH=15 COLUMNSTORE_VERSION_RELEASE=1 From bccd2acd4001579853a0f68b90c6262b65354012 Mon Sep 17 00:00:00 2001 From: Andrew Hutchings Date: Thu, 16 Aug 2018 15:57:42 +0100 Subject: [PATCH 05/33] Update README.md Fix server version --- README.md | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/README.md b/README.md index aa84a12ea..6a43e516e 100644 --- a/README.md +++ b/README.md @@ -1,6 +1,6 @@ #MariaDB ColumnStore Storage/Execution engine 1.0.15 MariaDB ColumnStore 1.0.15 is the development version of MariaDB ColumnStore. -It is built by porting InfiniDB 4.6.7 on MariaDB 10.1.28 and adding entirely +It is built by porting InfiniDB 4.6.7 on MariaDB 10.1.35 and adding entirely new features not found anywhere else. #MariaDB ColumnStore 1.0.15 is an GA release. From 9cb83957701516eb735bfee89a4f0ecb5e9a2f4a Mon Sep 17 00:00:00 2001 From: Andrew Hutchings Date: Thu, 16 Aug 2018 15:58:01 +0100 Subject: [PATCH 06/33] Update README Fix server version --- README | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/README b/README index 3eac5c82e..6e3bf8d59 100644 --- a/README +++ b/README @@ -1,6 +1,6 @@ This is MariaDB ColumnStore 1.0.15 MariaDB ColumnStore 1.0.15 is the development version of MariaDB ColumnStore. -It is built by porting InfiniDB 4.6.7 on MariaDB 10.1.28 and adding entirely +It is built by porting InfiniDB 4.6.7 on MariaDB 10.1.35 and adding entirely new features not found anywhere else. MariaDB ColumnStore 1.0.15 is an GA release. This is the first MariaDB From 36a2a7e2774280b816ee3ae7d403e22e6fc64cbe Mon Sep 17 00:00:00 2001 From: david hill Date: Mon, 20 Aug 2018 10:50:42 -0500 Subject: [PATCH 07/33] bump version to 1.0.16 --- README | 6 +++--- README.md | 6 +++--- VERSION | 2 +- 3 files changed, 7 insertions(+), 7 deletions(-) diff --git a/README b/README index 6e3bf8d59..ccc95c61b 100644 --- a/README +++ b/README @@ -1,9 +1,9 @@ -This is MariaDB ColumnStore 1.0.15 -MariaDB ColumnStore 1.0.15 is the development version of MariaDB ColumnStore. +This is MariaDB ColumnStore 1.0.16 +MariaDB ColumnStore 1.0.16 is the development version of MariaDB ColumnStore. It is built by porting InfiniDB 4.6.7 on MariaDB 10.1.35 and adding entirely new features not found anywhere else. -MariaDB ColumnStore 1.0.15 is an GA release. This is the first MariaDB +MariaDB ColumnStore 1.0.16 is an GA release. This is the first MariaDB ColumnStore release, not all features planned for the MariaDB ColumnStore 1.0 series are included in this release. diff --git a/README.md b/README.md index 6a43e516e..3ad645174 100644 --- a/README.md +++ b/README.md @@ -1,9 +1,9 @@ -#MariaDB ColumnStore Storage/Execution engine 1.0.15 -MariaDB ColumnStore 1.0.15 is the development version of MariaDB ColumnStore. +#MariaDB ColumnStore Storage/Execution engine 1.0.16 +MariaDB ColumnStore 1.0.16 is the development version of MariaDB ColumnStore. It is built by porting InfiniDB 4.6.7 on MariaDB 10.1.35 and adding entirely new features not found anywhere else. -#MariaDB ColumnStore 1.0.15 is an GA release. +#MariaDB ColumnStore 1.0.16 is an GA release. #Building This repository is not meant to be built independently outside of the server. This repository is integrated into http://mariadb-corporation/mariadb-columnstore-server (ie, the *server*) as a git submodule. As such, you can find complete build instructions on *the server* page. diff --git a/VERSION b/VERSION index 50ab911ca..b13a94d3f 100644 --- a/VERSION +++ b/VERSION @@ -1,4 +1,4 @@ COLUMNSTORE_VERSION_MAJOR=1 COLUMNSTORE_VERSION_MINOR=0 -COLUMNSTORE_VERSION_PATCH=15 +COLUMNSTORE_VERSION_PATCH=16 COLUMNSTORE_VERSION_RELEASE=1 From 60fb620dcb4b213a3a5c81292c0acae925a714cd Mon Sep 17 00:00:00 2001 From: Patrick LeBlanc Date: Mon, 5 Nov 2018 13:33:56 -0600 Subject: [PATCH 08/33] MCOL-1654. Querystats table creation broken. Escaped the ` chars to make the shell happy. Conflicts: dbcon/mysql/install_calpont_mysql.sh --- dbcon/mysql/install_calpont_mysql.sh | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/dbcon/mysql/install_calpont_mysql.sh b/dbcon/mysql/install_calpont_mysql.sh index 9b55f9448..ecea9d503 100755 --- a/dbcon/mysql/install_calpont_mysql.sh +++ b/dbcon/mysql/install_calpont_mysql.sh @@ -99,7 +99,7 @@ CREATE TABLE IF NOT EXISTS infinidb_querystats.querystats query varchar(8000), startTime timestamp NOT NULL, endTime timestamp NOT NULL, - rows bigint, + \`rows\` bigint, errno int, phyIO bigint, cacheIO bigint, From 35662d3258b9329a944477fcddf1e02f212ed84e Mon Sep 17 00:00:00 2001 From: Patrick LeBlanc Date: Wed, 14 Nov 2018 17:15:14 -0600 Subject: [PATCH 09/33] MCOL-1844. Preserve user-added args in 'myCnf-include-args.text' across upgrades. --- oam/install_scripts/pre-uninstall | 1 + oamapps/postConfigure/mycnfUpgrade.cpp | 59 +++++++++++++++++++++++++- 2 files changed, 59 insertions(+), 1 deletion(-) diff --git a/oam/install_scripts/pre-uninstall b/oam/install_scripts/pre-uninstall index 8709d369c..8621cd9ac 100755 --- a/oam/install_scripts/pre-uninstall +++ b/oam/install_scripts/pre-uninstall @@ -153,6 +153,7 @@ if [ $quiet != 1 ]; then #make copy of Columnstore.xml /bin/cp -f $installdir/etc/Columnstore.xml $installdir/etc/Columnstore.xml.rpmsave > /dev/null 2>&1 /bin/cp -f $installdir/mysql/my.cnf $installdir/mysql/my.cnf.rpmsave > /dev/null 2>&1 + cp $installdir/bin/myCnf-include-args.text $installdir/bin/myCnf-include-args.text.rpmsave >& /dev/null rm -f $installdir/etc/AlarmConfig.xml.installSave fi diff --git a/oamapps/postConfigure/mycnfUpgrade.cpp b/oamapps/postConfigure/mycnfUpgrade.cpp index a8808e740..18b606a2b 100644 --- a/oamapps/postConfigure/mycnfUpgrade.cpp +++ b/oamapps/postConfigure/mycnfUpgrade.cpp @@ -54,6 +54,59 @@ using namespace std; using namespace oam; + +/* MCOL-1844. On an upgrade, the user may have customized options in their old + * myCnf-include-args.text file. Merge it with the packaged version, and then process as we + * have before. + */ +string rtrim(const string &in) { + string::const_reverse_iterator rbegin = in.rbegin(); + while (rbegin != in.rend() && isspace(*rbegin)) + ++rbegin; + return string(in.begin(), rbegin.base()); +} + +void mergeMycnfIncludeArgs() +{ + string userArgsFilename = startup::StartUp::installDir() + "/bin/myCnf-include-args.text.rpmsave"; + string packagedArgsFilename = startup::StartUp::installDir() + "/bin/myCnf-include-args.text"; + ifstream userArgs(userArgsFilename.c_str()); + fstream packagedArgs(packagedArgsFilename.c_str(), ios::in); + + if (!userArgs || !packagedArgs) + return; + + // de-dup the args and comments in both files + set argMerger; + set comments; + string line; + while (getline(packagedArgs, line)) { + line = rtrim(line); + if (line[0] == '#') + comments.insert(line); + else if (line.size() > 0) + argMerger.insert(line); + } + while (getline(userArgs, line)) { + line = rtrim(line); + if (line[0] == '#') + comments.insert(line); + else if (line.size() > 0) + argMerger.insert(line); + } + userArgs.close(); + packagedArgs.close(); + + // write the merged version, comments first. They'll get ordered + // alphabetically but, meh. + packagedArgs.open(packagedArgsFilename.c_str(), ios::out | ios::trunc); + for (set::iterator it = comments.begin(); it != comments.end(); it++) + packagedArgs << *it << endl; + for (set::iterator it = argMerger.begin(); it != argMerger.end(); it++) + packagedArgs << *it << endl; + packagedArgs.close(); +} + int main(int argc, char *argv[]) { Oam oam; @@ -84,6 +137,10 @@ int main(int argc, char *argv[]) exit (1); } + // MCOL-1844. The user may have added options to their myCnf-include-args file. Merge + // myCnf-include-args.text with myCnf-include-args.text.rpmsave, save in myCnf-include-args.text + mergeMycnfIncludeArgs(); + //include arguments file string includeFile = startup::StartUp::installDir() + "/bin/myCnf-include-args.text"; ifstream includefile (includeFile.c_str()); @@ -144,7 +201,7 @@ int main(int argc, char *argv[]) ofstream newFile (mycnfFile.c_str()); //create new file - int fd = open(mycnfFile.c_str(), O_RDWR|O_CREAT, 0666); + int fd = open(mycnfFile.c_str(), O_RDWR|O_CREAT, 0644); copy(lines.begin(), lines.end(), ostream_iterator(newFile, "\n")); newFile.close(); From 165a5c76bf75548109602f3415541d5a4157e54f Mon Sep 17 00:00:00 2001 From: Patrick LeBlanc Date: Thu, 15 Nov 2018 09:14:20 -0600 Subject: [PATCH 10/33] MCOL-1844. Removed an unassociated change I made. --- oamapps/postConfigure/mycnfUpgrade.cpp | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/oamapps/postConfigure/mycnfUpgrade.cpp b/oamapps/postConfigure/mycnfUpgrade.cpp index 18b606a2b..75b9a3d3a 100644 --- a/oamapps/postConfigure/mycnfUpgrade.cpp +++ b/oamapps/postConfigure/mycnfUpgrade.cpp @@ -201,7 +201,7 @@ int main(int argc, char *argv[]) ofstream newFile (mycnfFile.c_str()); //create new file - int fd = open(mycnfFile.c_str(), O_RDWR|O_CREAT, 0644); + int fd = open(mycnfFile.c_str(), O_RDWR|O_CREAT, 0666); copy(lines.begin(), lines.end(), ostream_iterator(newFile, "\n")); newFile.close(); From 0f764ed74157f85167b6148d2722570d5f77ccc2 Mon Sep 17 00:00:00 2001 From: Patrick LeBlanc Date: Tue, 20 Nov 2018 10:49:05 -0600 Subject: [PATCH 11/33] MCOL-1607. WIP. Added the new option, no other change yet. --- oamapps/postConfigure/postConfigure.cpp | 171 ++++++++++++------------ 1 file changed, 85 insertions(+), 86 deletions(-) diff --git a/oamapps/postConfigure/postConfigure.cpp b/oamapps/postConfigure/postConfigure.cpp index bb828b5be..bfc630cd6 100644 --- a/oamapps/postConfigure/postConfigure.cpp +++ b/oamapps/postConfigure/postConfigure.cpp @@ -14,7 +14,7 @@ You should have received a copy of the GNU General Public License along with this program; if not, write to the Free Software Foundation, Inc., 51 Franklin Street, Fifth Floor, Boston, - MA 02110-1301, USA. */ + MA 02110-1301, USA. */ /****************************************************************************************** * $Id: postConfigure.cpp 64 2006-10-12 22:21:51Z dhill $ @@ -192,6 +192,7 @@ bool nonDistributeFlag = false; bool single_server_quick_install = false; bool multi_server_quick_install = false; bool amazon_quick_install = false; +bool doNotResolveHostNames = false; string DataFileEnvFile; @@ -275,12 +276,12 @@ int main(int argc, char* argv[]) usergroup = getgid(); string SUDO = ""; - if (user != 0) + if (user != 0) { rootUser = false; SUDO = "sudo "; } - + char* p = getenv("USER"); if (p && *p) @@ -298,7 +299,7 @@ int main(int argc, char* argv[]) for ( int i = 1; i < argc; i++ ) { - if( string("-h") == argv[i] ) + if( string("-h") == argv[i] ) { cout << endl; cout << "This is the MariaDB ColumnStore System Configuration and Installation tool." << endl; @@ -322,13 +323,18 @@ int main(int argc, char* argv[]) cout << " -qs Quick Install - Single Server" << endl; cout << " -qm Quick Install - Multi Server" << endl; cout << " -port MariaDB ColumnStore Port Address" << endl; - cout << " -i Non-root Install directory, Only use for non-root installs" << endl; - cout << " -n Non-distributed install, meaning postConfigure will not install packages on remote nodes" << endl; - cout << " -d Distributed install, meaning postConfigure will install packages on remote nodes" << endl; + cout << " -i Non-root Install directory, Only use for non-root installs" << endl; + cout << " -n Non-distributed install, meaning postConfigure will not install packages on remote nodes" << endl; + cout << " -d Distributed install, meaning postConfigure will install packages on remote nodes" << endl; cout << " -sn System Name" << endl; cout << " -pm-ip-addrs Performance Module IP Addresses xxx.xxx.xxx.xxx,xxx.xxx.xxx.xxx" << endl; cout << " -um-ip-addrs User Module IP Addresses xxx.xxx.xxx.xxx,xxx.xxx.xxx.xxx" << endl; + cout << " -x Do not resolve IP Addresses from host names" << endl; exit (0); + } + else if (string("-x") == argv[i]) + { + doNotResolveHostNames = true; } else if( string("-qs") == argv[i] ) { @@ -349,33 +355,33 @@ int main(int argc, char* argv[]) nodeps = "--nodeps"; else if ( string("-o") == argv[i] ) startOfflinePrompt = true; - else if( string("-c") == argv[i] ) + else if( string("-c") == argv[i] ) { i++; - if (i >= argc ) + if (i >= argc ) { cout << " ERROR: Config File not provided" << endl; exit (1); } oldFileName = argv[i]; - if ( oldFileName.find("Columnstore.xml") == string::npos ) + if ( oldFileName.find("Columnstore.xml") == string::npos ) { cout << " ERROR: Config File is not a Columnstore.xml file name" << endl; exit (1); } } - else if( string("-p") == argv[i] ) + else if( string("-p") == argv[i] ) { i++; - if (i >= argc ) + if (i >= argc ) { cout << " ERROR: Password not provided" << endl; exit (1); } password = argv[i]; - if ( password.find("-") != string::npos ) + if ( password.find("-") != string::npos ) { cout << " ERROR: Valid Password not provided" << endl; exit (1); @@ -394,10 +400,10 @@ int main(int argc, char* argv[]) nonDistribute = false; nonDistributeFlag = true; } - else if( string("-port") == argv[i] ) + else if( string("-port") == argv[i] ) { i++; - if (i >= argc ) + if (i >= argc ) { cout << " ERROR: MariaDB ColumnStore Port ID not supplied" << endl; exit (1); @@ -411,10 +417,10 @@ int main(int argc, char* argv[]) exit (1); } } - else if( string("-i") == argv[i] ) + else if( string("-i") == argv[i] ) { i++; - if (i >= argc ) + if (i >= argc ) { cout << " ERROR: Path not provided" << endl; exit (1); @@ -422,50 +428,50 @@ int main(int argc, char* argv[]) installDir = argv[i]; } - else if( string("-sn") == argv[i] ) + else if( string("-sn") == argv[i] ) { i++; - if (i >= argc ) + if (i >= argc ) { cout << " ERROR: System-name not provided" << endl; exit (1); } systemName = argv[i]; } - else if( string("-pm-ip-addrs") == argv[i] ) + else if( string("-pm-ip-addrs") == argv[i] ) { i++; - if (i >= argc ) + if (i >= argc ) { cout << " ERROR: PM-IP-ADRESSES not provided" << endl; exit (1); } pmIpAddrs = argv[i]; } - else if( string("-um-ip-addrs") == argv[i] ) + else if( string("-um-ip-addrs") == argv[i] ) { i++; - if (i >= argc ) + if (i >= argc ) { cout << " ERROR: UM-IP-ADRESSES not provided" << endl; exit (1); } umIpAddrs = argv[i]; } - else if( string("-pm-count") == argv[i] ) + else if( string("-pm-count") == argv[i] ) { i++; - if (i >= argc ) + if (i >= argc ) { cout << " ERROR: PM-COUNT not provided" << endl; exit (1); } pmNumber = atoi(argv[i]); } - else if( string("-um-count") == argv[i] ) + else if( string("-um-count") == argv[i] ) { i++; - if (i >= argc ) + if (i >= argc ) { cout << " ERROR: UM-COUNT not provided" << endl; exit (1); @@ -479,7 +485,7 @@ int main(int argc, char* argv[]) exit (1); } } - + //check if quick install multi-server has been given ip address if (multi_server_quick_install) { @@ -553,7 +559,7 @@ int main(int argc, char* argv[]) // redirectStandardOutputToFile(postConfigureLog, false ); } } - + //check if MariaDB ColumnStore is up and running if (oam.checkSystemRunning()) { @@ -567,7 +573,7 @@ int main(int argc, char* argv[]) cout << "ERROR: Configuration File not setup" << endl; exit(1); } - + //determine package type string EEPackageType; @@ -605,7 +611,7 @@ int main(int argc, char* argv[]) cout << "ERROR: Failed trying to update MariaDB ColumnStore System Configuration file" << endl; exit(1); } - + //check for local ip address as pm1 ModuleConfig moduleconfig; @@ -789,7 +795,7 @@ int main(int argc, char* argv[]) } cout << endl; - + if (single_server_quick_install) { cout << "===== Quick Install Single-Server Configuration =====" << endl << endl; @@ -797,13 +803,13 @@ int main(int argc, char* argv[]) cout << "Single-Server install is used when there will only be 1 server configured" << endl; cout << "on the system. It can also be used for production systems, if the plan is" << endl; cout << "to stay single-server." << endl; - + singleServerInstall = "1"; } else if (multi_server_quick_install) { cout << "===== Quick Install Multi-Server Configuration =====" << endl << endl; - + cout << "Multi-Server install defaulting to using local storage" << endl; singleServerInstall = "2"; @@ -811,7 +817,7 @@ int main(int argc, char* argv[]) else if (amazon_quick_install) { cout << "===== Quick Install Amazon Configuration =====" << endl << endl; - + cout << "Amazon AMI EC2 install defaulting to using local storage" << endl; singleServerInstall = "2"; @@ -842,7 +848,7 @@ int main(int argc, char* argv[]) else singleServerInstall = "2"; - while(true) + while(true) { string temp = singleServerInstall; prompt = "Select the type of System Server install [1=single, 2=multi] (" + singleServerInstall + ") > "; @@ -889,7 +895,7 @@ int main(int argc, char* argv[]) //setup to Columnstore.xml file for single server singleServerConfigSetup(sysConfig); } - + //module ProcessConfig.xml to setup all apps on the pm if ( !updateProcessConfig() ) cout << "Update ProcessConfig.xml error" << endl; @@ -967,7 +973,7 @@ int main(int argc, char* argv[]) } // perform multi-node install - + try { sysConfig->setConfig(InstallSection, "SingleServerInstall", "n"); @@ -987,7 +993,7 @@ int main(int argc, char* argv[]) // // Multi-server install // - + ModuleIP InputModuleIP; ModuleIpList InputModuleIPList; @@ -1001,7 +1007,7 @@ int main(int argc, char* argv[]) } catch(...) {} - + if (umIpAddrs == "" ) { // set Server Type Installation to combined @@ -1026,10 +1032,10 @@ int main(int argc, char* argv[]) InputModuleIP.moduleName = module; InputModuleIPList.push_back(InputModuleIP); } - + umNumber = id-1; } - + if (pmIpAddrs != "" ) { int id = 1; @@ -1045,16 +1051,16 @@ int main(int argc, char* argv[]) InputModuleIP.moduleName = module; InputModuleIPList.push_back(InputModuleIP); } - + pmNumber = id-1; } - if ( !writeConfig(sysConfig) ) + if ( !writeConfig(sysConfig) ) { cout << "ERROR: Failed trying to update MariaDB ColumnStore System Configuration file" << endl; exit(1); } - + MaxNicID = 1; } else @@ -1067,13 +1073,13 @@ int main(int argc, char* argv[]) } catch(...) {} - + try { sysConfig->setConfig(InstallSection, "Cloud", "amazon-vpc"); } catch(...) {} - + if (umNumber == 0 ) { // set Server Type Installation to combined @@ -1084,12 +1090,12 @@ int main(int argc, char* argv[]) {} } - if ( !writeConfig(sysConfig) ) + if ( !writeConfig(sysConfig) ) { cout << "ERROR: Failed trying to update MariaDB ColumnStore System Configuration file" << endl; exit(1); } - + MaxNicID = 1; } } @@ -1350,7 +1356,7 @@ int main(int argc, char* argv[]) //amazon install setup check bool amazonInstall = false; string cloud = oam::UnassignedName; - + if (!multi_server_quick_install) { string amazonLog = tmpDir + "/amazon.log"; @@ -1367,7 +1373,7 @@ int main(int argc, char* argv[]) // not running on amazon with ec2-api-tools if (amazon_quick_install) { - cout << "ERROR: Amazon Quick Installer was specified, bu the Amazon CLI API packages isnt installed, exiting" << endl; + cout << "ERROR: Amazon Quick Installer was specified, but the Amazon CLI API packages is not installed, exiting" << endl; exit(1); } @@ -1380,7 +1386,7 @@ int main(int argc, char* argv[]) // not running on amazon with ec2-api-tools if (amazon_quick_install) { - cout << "ERROR: Amazon Quick Installer was specified, bu the AMazon CLI API packages isnt installed, exiting" << endl; + cout << "ERROR: Amazon Quick Installer was specified, but the Amazon CLI API packages is not installed, exiting" << endl; exit(1); } @@ -1390,7 +1396,7 @@ int main(int argc, char* argv[]) amazonInstall = true; } } - + try { cloud = sysConfig->getConfig(InstallSection, "Cloud"); @@ -1717,13 +1723,6 @@ int main(int argc, char* argv[]) // are we using settings from previous config file? if ( reuseConfig == "n" ) { - - - - - - - string numBlocksPct; try @@ -1877,14 +1876,14 @@ int main(int argc, char* argv[]) catch (...) {} } - + if ( moduleType == "um") if ( umNumber != 0 ) moduleCount = umNumber; - + if ( moduleType == "pm") if ( pmNumber != 0 ) - moduleCount = pmNumber; + moduleCount = pmNumber; //verify and setup of modules count switch ( IserverTypeInstall ) @@ -2044,7 +2043,7 @@ int main(int argc, char* argv[]) moduleDisableState = oam::ENABLEDSTATE; //setup HostName/IPAddress for each NIC - + string moduleHostName = oam::UnassignedName; string moduleIPAddr = oam::UnassignedIpAddr; @@ -2063,7 +2062,7 @@ int main(int argc, char* argv[]) } } } - + unsigned int nicID=1; for( ; nicID < MaxNicID +1 ; nicID++ ) { @@ -2076,9 +2075,9 @@ int main(int argc, char* argv[]) for( ; listPT != sysModuleTypeConfig.moduletypeconfig[i].ModuleNetworkList.end() ; listPT++) { - if (newModuleName == (*listPT).DeviceName) + if (newModuleName == (*listPT).DeviceName) { - if ( nicID == 1 ) + if ( nicID == 1 ) { moduleDisableState = (*listPT).DisableState; @@ -2091,7 +2090,7 @@ int main(int argc, char* argv[]) for( ; pt1 != (*listPT).hostConfigList.end() ; pt1++) { - if ((*pt1).NicID == nicID) + if ((*pt1).NicID == nicID) { moduleHostName = (*pt1).HostName; moduleIPAddr = (*pt1).IPAddr; @@ -2103,8 +2102,8 @@ int main(int argc, char* argv[]) } } } - - if ( nicID == 1 ) + + if ( nicID == 1 ) { if ( moduleDisableState != oam::ENABLEDSTATE ) { @@ -2948,7 +2947,7 @@ int main(int argc, char* argv[]) } } } - + vector dbroots; string tempdbrootList; @@ -3497,7 +3496,7 @@ int main(int argc, char* argv[]) } if ( IserverTypeInstall != oam::INSTALL_COMBINE_DM_UM_PM || - pmNumber > 1 ) + pmNumber > 1 ) { if ( password.empty() ) { @@ -3531,10 +3530,10 @@ int main(int argc, char* argv[]) if ( p1 == p2 ) { password = p2; break; - } + } else cout << "Password mismatch, please re-enter" << endl; - } + } //add single quote for special characters if ( password != "ssh" ) @@ -3543,7 +3542,7 @@ int main(int argc, char* argv[]) } } - } + } } int thread_id = 0; @@ -3824,7 +3823,7 @@ int main(int argc, char* argv[]) cout << " DONE" << endl; } } - + //configure data redundancy if (DataRedundancy) { @@ -4039,7 +4038,7 @@ int main(int argc, char* argv[]) } } } - + string dbbuilderLog = tmpDir + "/dbbuilder.log"; if (hdfs) @@ -4060,7 +4059,7 @@ int main(int argc, char* argv[]) //send message to procmon's to run upgrade script int status = sendUpgradeRequest(IserverTypeInstall, pmwithum); - + if ( status != 0 ) { cout << endl << "MariaDB Columnstore Install Failed" << endl << endl; exit(1); @@ -4423,7 +4422,7 @@ bool updateProcessConfig() vector oldModule; string newModule = ">pm"; oldModule.push_back(">um"); - + string fileName = installDir + "/etc/ProcessConfig.xml"; //Save a copy of the original version @@ -4564,9 +4563,9 @@ bool uncommentCalpontXml( string entry) */ bool makeRClocal(string moduleType, string moduleName, int IserverTypeInstall) { - + return true; - + vector lines; string mount1; @@ -5147,7 +5146,7 @@ bool storageSetup(bool amazonInstall) //check if hadoop is installed string hadoopLog = tmpDir + "/hadoop.log"; - + string cmd = "which hadoop > " + hadoopLog + " 2>&1"; system(cmd.c_str()); @@ -5732,7 +5731,7 @@ void setSystemName() Oam oam; //setup System Name - + if ( systemName.empty() ) { try { @@ -5743,7 +5742,7 @@ void setSystemName() systemName = oam::UnassignedName; } } - + if ( systemName.empty() ) systemName = "columnstore-1"; @@ -5759,7 +5758,7 @@ void setSystemName() callFree(pcommand); } } - + try { sysConfig->setConfig(SystemSection, "SystemName", systemName); @@ -6926,7 +6925,7 @@ bool glusterSetup(string password) void singleServerConfigSetup(Config* sysConfig) { - + try { sysConfig->setConfig("ExeMgr1", "IPAddr", "127.0.0.1"); From cadfb0be928bf5a9308902a4a2b3dcb074acb686 Mon Sep 17 00:00:00 2001 From: Patrick LeBlanc Date: Tue, 20 Nov 2018 14:44:30 -0600 Subject: [PATCH 12/33] Very speculative change to postConfigure. 99% sure it won't work. --- oamapps/postConfigure/postConfigure.cpp | 12 ++++++------ 1 file changed, 6 insertions(+), 6 deletions(-) diff --git a/oamapps/postConfigure/postConfigure.cpp b/oamapps/postConfigure/postConfigure.cpp index bfc630cd6..0c8d2e0ac 100644 --- a/oamapps/postConfigure/postConfigure.cpp +++ b/oamapps/postConfigure/postConfigure.cpp @@ -129,7 +129,7 @@ void singleServerConfigSetup(Config* sysConfig); void remoteInstallThread(void*); -bool glusterSetup(string password); +bool glusterSetup(string password, bool doNotResolveHostNames); std::string launchInstance(ModuleIP moduleip); @@ -2446,7 +2446,7 @@ int main(int argc, char* argv[]) newModuleIPAddr = oam::UnassignedIpAddr; else { - if (!amazonInstall) + if (!amazonInstall && !doNotResolveHostNames) { if ( moduleIPAddr == oam::UnassignedIpAddr ) { @@ -3896,7 +3896,7 @@ int main(int argc, char* argv[]) { cout << endl << "===== Configuring MariaDB ColumnStore Data Redundancy Functionality =====" << endl << endl; - if (!glusterSetup(password)) + if (!glusterSetup(password, doNotResolveHostNames)) { cout << "ERROR: Problem setting up ColumnStore Data Redundancy" << endl; exit(1); @@ -6274,7 +6274,7 @@ std::string launchInstance(ModuleIP moduleip) return instanceName; } -bool glusterSetup(string password) +bool glusterSetup(string password, bool doNotResolveHostNames) { Oam oam; int dataRedundancyCopies = 0; @@ -6428,7 +6428,7 @@ bool glusterSetup(string password) callFree(pcommand); } - if ( moduleIPAddr == oam::UnassignedIpAddr ) + if ( moduleIPAddr == oam::UnassignedIpAddr) { //get IP Address string IPAddress = oam.getIPAddress( moduleHostName); @@ -6443,7 +6443,7 @@ bool glusterSetup(string password) moduleIPAddr = "unassigned"; //prompt for IP address - while (true) + while (!doNotResolveHostNames) { prompt = "Enter PM #" + oam.itoa(DataRedundancyConfigs[pm].pmID) + " IP Address of " + moduleHostName + " (" + moduleIPAddr + ") > "; pcommand = callReadline(prompt.c_str()); From d75659aa90fc5d310e1fe6c5c9e67e725a58f84e Mon Sep 17 00:00:00 2001 From: Patrick LeBlanc Date: Tue, 20 Nov 2018 15:53:52 -0600 Subject: [PATCH 13/33] Some changes to postConfigure to get hostname put in IPAddr field. --- oamapps/postConfigure/postConfigure.cpp | 22 +++++++++++++++------- 1 file changed, 15 insertions(+), 7 deletions(-) diff --git a/oamapps/postConfigure/postConfigure.cpp b/oamapps/postConfigure/postConfigure.cpp index 0c8d2e0ac..e5680485b 100644 --- a/oamapps/postConfigure/postConfigure.cpp +++ b/oamapps/postConfigure/postConfigure.cpp @@ -2446,12 +2446,16 @@ int main(int argc, char* argv[]) newModuleIPAddr = oam::UnassignedIpAddr; else { - if (!amazonInstall && !doNotResolveHostNames) + if (!amazonInstall) { if ( moduleIPAddr == oam::UnassignedIpAddr ) { //get IP Address - string IPAddress = oam.getIPAddress( newModuleHostName); + string IPAddress; + if (doNotResolveHostNames) + IPAddress = newModuleHostName; + else + IPAddress = oam.getIPAddress( newModuleHostName); if ( !IPAddress.empty() ) newModuleIPAddr = IPAddress; @@ -2467,7 +2471,7 @@ int main(int argc, char* argv[]) //prompt for IP address while (true) { - prompt = "Enter Nic Interface #" + oam.itoa(nicID) + " IP Address of " + newModuleHostName + " (" + newModuleIPAddr + ") > "; + prompt = "Enter Nic Interface #" + oam.itoa(nicID) + " IP Address or hostname of " + newModuleHostName + " (" + newModuleIPAddr + ") > "; pcommand = callReadline(prompt.c_str()); if (pcommand) @@ -2488,7 +2492,7 @@ int main(int argc, char* argv[]) continue; } - if (oam.isValidIP(newModuleIPAddr)) + if (oam.isValidIP(newModuleIPAddr) || doNotResolveHostNames) { //check and see if hostname already used bool matchFound = false; @@ -6431,7 +6435,11 @@ bool glusterSetup(string password, bool doNotResolveHostNames) if ( moduleIPAddr == oam::UnassignedIpAddr) { //get IP Address - string IPAddress = oam.getIPAddress( moduleHostName); + string IPAddress; + if (doNotResolveHostNames) + IPAddress = moduleHostName; + else + IPAddress = oam.getIPAddress( moduleHostName); if ( !IPAddress.empty() ) moduleIPAddr = IPAddress; @@ -6443,7 +6451,7 @@ bool glusterSetup(string password, bool doNotResolveHostNames) moduleIPAddr = "unassigned"; //prompt for IP address - while (!doNotResolveHostNames) + while (true) { prompt = "Enter PM #" + oam.itoa(DataRedundancyConfigs[pm].pmID) + " IP Address of " + moduleHostName + " (" + moduleIPAddr + ") > "; pcommand = callReadline(prompt.c_str()); @@ -6466,7 +6474,7 @@ bool glusterSetup(string password, bool doNotResolveHostNames) continue; } - if (oam.isValidIP(moduleIPAddr)) + if (oam.isValidIP(moduleIPAddr) || doNotResolveHostNames) { // run ping test to validate From 7ad0aa442060fa888322c220aad182b6f329b431 Mon Sep 17 00:00:00 2001 From: Patrick LeBlanc Date: Thu, 29 Nov 2018 15:20:05 -0600 Subject: [PATCH 14/33] MCOL-1607. Tentative change to include addModule. This might or might not let addModule store host names instead of IP addresses. --- oam/etc/ConsoleCmds.xml | 5 +- oam/oamcpp/liboamcpp.cpp | 125 ++++++++++++++++++++++++++++++++-- oam/oamcpp/liboamcpp.h | 9 ++- oamapps/mcsadmin/mcsadmin.cpp | 123 ++++++++++++++++++++------------- procmgr/processmanager.cpp | 17 +++-- procmgr/processmanager.h | 3 +- 6 files changed, 223 insertions(+), 59 deletions(-) diff --git a/oam/etc/ConsoleCmds.xml b/oam/etc/ConsoleCmds.xml index 56a6e678f..9c58d78f3 100644 --- a/oam/etc/ConsoleCmds.xml +++ b/oam/etc/ConsoleCmds.xml @@ -309,8 +309,9 @@ MariaDB Columnstore Packages and setup the module to make it ready to be restored Required: Module-type or Module-name being added Required: Number-of-Modules being added when Module-type is specified - Optional: Server-Hostnames/Amazon-Instance-Names seperated by commas - Optional: Server-root-password + Optional: Store given hostnames in the configuration instead of IP addresses (y/N) + Optional: Server-Hostnames/Amazon-Instance-Names seperated by commas + Optional: Server-root-password removeModule diff --git a/oam/oamcpp/liboamcpp.cpp b/oam/oamcpp/liboamcpp.cpp index 7ce05e2d9..0ec1e7b1c 100644 --- a/oam/oamcpp/liboamcpp.cpp +++ b/oam/oamcpp/liboamcpp.cpp @@ -1272,10 +1272,12 @@ void Oam::setSystemConfig(const std::string module, ModuleConfig moduleconfig) * ********************************************************************/ -void Oam::addModule(DeviceNetworkList devicenetworklist, const std::string password, const std::string mysqlpw) +void Oam::addModule(DeviceNetworkList devicenetworklist, const std::string password, const std::string mysqlpw, + bool storeHostnames) { // build and send msg - int returnStatus = sendMsgToProcMgr2(ADDMODULE, devicenetworklist, FORCEFUL, ACK_YES, password, mysqlpw); + int returnStatus = sendAddModuleToProcMgr(ADDMODULE, devicenetworklist, FORCEFUL, ACK_YES, storeHostnames, + password, mysqlpw); if (returnStatus != API_SUCCESS) exceptionControl("addModule", returnStatus); @@ -9929,7 +9931,6 @@ int Oam::sendMsgToProcMgr2(messageqcpp::ByteStream::byte requestType, DeviceNetw msg << (ByteStream::byte) gracefulflag; msg << (ByteStream::byte) ackflag; msg << (ByteStream::byte) requestManual; - msg << (uint16_t) devicenetworklist.size(); DeviceNetworkList::iterator pt = devicenetworklist.begin(); @@ -9947,7 +9948,6 @@ int Oam::sendMsgToProcMgr2(messageqcpp::ByteStream::byte requestType, DeviceNetw msg << " "; else msg << (*pt).DisableState; - msg << (uint16_t) (*pt).hostConfigList.size(); HostConfigList::iterator pt1 = (*pt).hostConfigList.begin(); @@ -10007,6 +10007,123 @@ int Oam::sendMsgToProcMgr2(messageqcpp::ByteStream::byte requestType, DeviceNetw return returnStatus; } +/* A slightly different version of sendMsgToProcMgr2. Add-module needs to send one add'l + parameter, and this was the best of a couple bad options. */ +int Oam::sendAddModuleToProcMgr(messageqcpp::ByteStream::byte requestType, DeviceNetworkList devicenetworklist, + GRACEFUL_FLAG gracefulflag, ACK_FLAG ackflag, bool storeHostnames, const std::string password, + const std::string mysqlpw) +{ + if (!checkSystemRunning()) + return API_CONN_REFUSED; + + int returnStatus = API_TIMEOUT; //default + ByteStream msg; + ByteStream receivedMSG; + ByteStream::byte msgType; + ByteStream::byte actionType; + ByteStream::byte status; + + // get current requesting process, an error will occur if process is a UI tool (not kept in Status Table) + // this will be used to determine if this is a manually or auto request down within Process-Monitor + bool requestManual; + myProcessStatus_t t; + + try + { + t = getMyProcessStatus(); + requestManual = false; // set to auto + } + catch (...) + { + requestManual = true; // set to manual + } + + // setup message + msg << (ByteStream::byte) REQUEST; + msg << requestType; + msg << (std::string) " "; + msg << (ByteStream::byte) gracefulflag; + msg << (ByteStream::byte) ackflag; + msg << (ByteStream::byte) requestManual; + msg << (uint8_t) storeHostnames; + msg << (uint16_t) devicenetworklist.size(); + + DeviceNetworkList::iterator pt = devicenetworklist.begin(); + + for ( ; pt != devicenetworklist.end() ; pt++) + { + msg << (*pt).DeviceName; + + if ( (*pt).UserTempDeviceName.empty() ) + msg << " "; + else + msg << (*pt).UserTempDeviceName; + + if ( (*pt).DisableState.empty() ) + msg << " "; + else + msg << (*pt).DisableState; + msg << (uint16_t) (*pt).hostConfigList.size(); + + HostConfigList::iterator pt1 = (*pt).hostConfigList.begin(); + + for ( ; pt1 != (*pt).hostConfigList.end() ; pt1++) + { + msg << (*pt1).IPAddr; + msg << (*pt1).HostName; + msg << (*pt1).NicID; + } + } + + msg << password; + msg << mysqlpw; + + try + { + //send the msg to Process Manager + MessageQueueClient procmgr("ProcMgr"); + procmgr.write(msg); + + // check for Ack msg if needed + if ( ackflag == ACK_YES ) + { + // wait 15 minutes for ACK from Process Manager + struct timespec ts = { 900, 0 }; + + receivedMSG = procmgr.read(&ts); + + if (receivedMSG.length() > 0) + { + receivedMSG >> msgType; + receivedMSG >> actionType; + receivedMSG >> status; + + if ( msgType == oam::ACK && actionType == requestType) + { + // ACK for this request + returnStatus = status; + } + } + else // timeout + returnStatus = API_TIMEOUT; + } + else + // No ACK, assume write success + returnStatus = API_SUCCESS; + + // shutdown connection + procmgr.shutdown(); + } + catch (...) + { + returnStatus = API_FAILURE; + } + + return returnStatus; +} + + + /*************************************************************************** * * Function: sendMsgToProcMgr3 diff --git a/oam/oamcpp/liboamcpp.h b/oam/oamcpp/liboamcpp.h index 39e2a51de..5d5a09c6c 100644 --- a/oam/oamcpp/liboamcpp.h +++ b/oam/oamcpp/liboamcpp.h @@ -1488,7 +1488,8 @@ public: * @param DeviceNetworkConfig the Modules added * @param password Host Root Password */ - EXPORT void addModule(DeviceNetworkList devicenetworklist, const std::string password, const std::string mysqlpw); + EXPORT void addModule(DeviceNetworkList devicenetworklist, const std::string password, const std::string mysqlpw, + bool storeHostnames); /** @brief remove Module * @@ -2503,6 +2504,12 @@ private: int sendMsgToProcMgr2(messageqcpp::ByteStream::byte requestType, DeviceNetworkList devicenetworklist, GRACEFUL_FLAG gracefulflag, ACK_FLAG ackflag, const std::string password = oam::UnassignedName, const std::string mysqlpw = oam::UnassignedName); + /** @brief a slightly different version of sendMsgToProcMgr2, which is for addmodule only. + */ + int sendAddModuleToProcMgr(messageqcpp::ByteStream::byte requestType, DeviceNetworkList devicenetworklist, + GRACEFUL_FLAG gracefulflag, ACK_FLAG ackflag, bool storeHostnames, const std::string password = oam::UnassignedName, + const std::string mysqlpw = oam::UnassignedName); + /** @brief build and send request message to Process Manager * Check for status messages */ diff --git a/oamapps/mcsadmin/mcsadmin.cpp b/oamapps/mcsadmin/mcsadmin.cpp index 454c7a3ea..180e3e18e 100644 --- a/oamapps/mcsadmin/mcsadmin.cpp +++ b/oamapps/mcsadmin/mcsadmin.cpp @@ -5677,6 +5677,7 @@ int processCommand(string* arguments) DeviceNetworkList enabledevicenetworklist; HostConfig hostconfig; + bool storeHostnames = false; string moduleType; string moduleName; int moduleCount; @@ -5687,12 +5688,12 @@ int processCommand(string* arguments) umStorageNames umstoragenames; int hostArg; int dbrootPerPM = 0; - + //check if module type or module name was entered if ( arguments[1].size() == 2 ) { //Module Type was entered - if (arguments[3] == "" && cloud == oam::UnassignedName) + if (arguments[4] == "" && cloud == oam::UnassignedName) { // need at least arguments cout << endl << "**** addModule Failed : Missing a required Parameter, enter 'help' for additional information" << endl; @@ -5702,8 +5703,47 @@ int processCommand(string* arguments) //Module Type was entered moduleType = arguments[1]; moduleCount = atoi(arguments[2].c_str()); + hostArg = 4; + + // MCOL-1607. Check whether we should store host names or IP addresses. + if (arguments[3] != "" && (arguments[3][0] == 'y' || arguments[3][0] == 'Y')) + storeHostnames = true; + + //check for a non-distrubuted install setup, dont need password + if ( DistributedInstall != "y" ) + { + if (arguments[5] != "") + password = arguments[5]; + else + { + cout << endl; + string prompt = "Enter the 'User' Password or 'ssh' if configured with ssh-keys"; + password = dataPrompt(prompt); + } + } + + if (arguments[6] != "") + dbrootPerPM = atoi(arguments[6].c_str()); + } + else + { + //Module Name was entered + if (arguments[2] == "" && cloud == oam::UnassignedName) + { + // need at least arguments + cout << endl << "**** addModule Failed : Missing a required Parameter, enter 'help' for additional information" << endl; + break; + } + + moduleName = arguments[1]; + moduleType = arguments[1].substr(0, MAX_MODULE_TYPE_SIZE); + moduleCount = 1; hostArg = 3; + // MCOL-1607. Check whether we should store host names or IP addresses. + if (arguments[2] != "" && (arguments[2][0] == 'y' || arguments[2][0] == 'Y')) + storeHostnames = true; + //check for a non-distrubuted install setup, dont need password if ( DistributedInstall != "y" ) { @@ -5720,37 +5760,6 @@ int processCommand(string* arguments) if (arguments[5] != "") dbrootPerPM = atoi(arguments[5].c_str()); } - else - { - //Module Name was entered - if (arguments[2] == "" && cloud == oam::UnassignedName) - { - // need at least arguments - cout << endl << "**** addModule Failed : Missing a required Parameter, enter 'help' for additional information" << endl; - break; - } - - moduleName = arguments[1]; - moduleType = arguments[1].substr(0, MAX_MODULE_TYPE_SIZE); - moduleCount = 1; - hostArg = 2; - - //check for a non-distrubuted install setup, dont need password - if ( DistributedInstall != "y" ) - { - if (arguments[3] != "") - password = arguments[3]; - else - { - cout << endl; - string prompt = "Enter the 'User' Password or 'ssh' if configured with ssh-keys"; - password = dataPrompt(prompt); - } - } - - if (arguments[4] != "") - dbrootPerPM = atoi(arguments[4].c_str()); - } //do we needed this check???? if ( moduleCount < 1 || moduleCount > 10 ) @@ -5942,7 +5951,24 @@ int processCommand(string* arguments) string hostName; string IPAddress; - if ( cloud == "amazon-ec2") + // MCOL-1607. Store hostnames in the config file if they entered one */ + if (storeHostnames) + { + // special case + if (cloud == "amazon-vpc" && *listPT1 == "autoassign") + { + hostName = oam::UnassignedName; + IPAddress = *listPT1; + } + else if (oam.isValidIP(*listPT1)) // they entered an IP addr + { + hostName = oam::UnassignedName; + IPAddress = *listPT1; + } + else // they entered a hostname + IPAddress = hostName = *listPT1; + } + else if ( cloud == "amazon-ec2") { hostName = *listPT1; @@ -5995,14 +6021,13 @@ int processCommand(string* arguments) // non-amazon hostName = *listPT1; IPAddress = oam.getIPAddress(hostName); - if ( IPAddress.empty() ) { // prompt for IP Address string prompt = "IP Address of " + hostName + " not found, enter IP Address or enter 'abort'"; IPAddress = dataPrompt(prompt); - - if ( IPAddress == "abort" || !oam.isValidIP(IPAddress)) + + if ( IPAddress == "abort" || !oam.isValidIP(IPAddress) ) return 1; } } @@ -6040,18 +6065,22 @@ int processCommand(string* arguments) { string prompt = "DataRedundancy is configured for dedicated network, enter a hostname"; DataRedundancyHostname = dataPrompt(prompt); - DataRedundancyIPAddress = oam.getIPAddress(DataRedundancyHostname); - - if ( DataRedundancyIPAddress.empty() ) + if (storeHostnames) + DataRedundancyIPAddress = DataRedundancyHostname; + else { - // prompt for IP Address - string prompt = "IP Address of " + DataRedundancyHostname + " not found, enter IP Address"; - DataRedundancyIPAddress = dataPrompt(prompt); + DataRedundancyIPAddress = oam.getIPAddress(DataRedundancyHostname); - if (!oam.isValidIP(DataRedundancyIPAddress)) - return 1; + if ( DataRedundancyIPAddress.empty() ) + { + // prompt for IP Address + string prompt = "IP Address of " + DataRedundancyHostname + " not found, enter IP Address"; + DataRedundancyIPAddress = dataPrompt(prompt); + + if (!oam.isValidIP(DataRedundancyIPAddress)) + return 1; + } } - sysConfig->setConfig("DataRedundancyConfig", dataDupHostName, DataRedundancyHostname); sysConfig->setConfig("DataRedundancyConfig", dataDupIPaddr, DataRedundancyIPAddress); } @@ -6122,7 +6151,7 @@ int processCommand(string* arguments) cout << "please wait..." << endl; - oam.addModule(devicenetworklist, password, mysqlpassword); + oam.addModule(devicenetworklist, password, mysqlpassword, storeHostnames); cout << "Add Module(s) successfully completed" << endl; diff --git a/procmgr/processmanager.cpp b/procmgr/processmanager.cpp index f21459f22..686243f87 100644 --- a/procmgr/processmanager.cpp +++ b/procmgr/processmanager.cpp @@ -2205,10 +2205,15 @@ void processMSG(messageqcpp::IOSocket* cfIos) string value; uint16_t count, ivalue, nicCount; + uint8_t tmp8; oam::DeviceNetworkConfig devicenetworkconfig; oam::DeviceNetworkList devicenetworklist; oam::HostConfig hostconfig; + bool storeHostnames; + msg >> tmp8; + storeHostnames = (tmp8 != 0); + //get module count to add msg >> count; @@ -2223,7 +2228,7 @@ void processMSG(messageqcpp::IOSocket* cfIos) devicenetworkconfig.UserTempDeviceName = value; msg >> value; devicenetworkconfig.DisableState = value; - + msg >> nicCount; for (int j = 0 ; j < nicCount ; j ++ ) @@ -2244,7 +2249,7 @@ void processMSG(messageqcpp::IOSocket* cfIos) string password; msg >> password; - status = processManager.addModule(devicenetworklist, password); + status = processManager.addModule(devicenetworklist, password, storeHostnames); log.writeLog(__LINE__, "ADDMODULE: ACK received from Process-Monitor, return status = " + oam.itoa(status)); } @@ -4835,7 +4840,8 @@ int ProcessManager::reinitProcessType( std::string processName ) * purpose: Add Module to system configuration * ******************************************************************************************/ -int ProcessManager::addModule(oam::DeviceNetworkList devicenetworklist, std::string password, bool manualFlag) +int ProcessManager::addModule(oam::DeviceNetworkList devicenetworklist, std::string password, bool storeHostnames, + bool manualFlag) { ProcessLog log; Configuration config; @@ -5262,7 +5268,10 @@ int ProcessManager::addModule(oam::DeviceNetworkList devicenetworklist, std::str } hostconfig.HostName = hostName; - hostconfig.IPAddr = IPAddr; + if (storeHostnames) + hostconfig.IPAddr = hostName; + else + hostconfig.IPAddr = IPAddr; hostconfig.NicID = (*pt1).NicID; devicenetworkconfig.hostConfigList.push_back(hostconfig); } diff --git a/procmgr/processmanager.h b/procmgr/processmanager.h index 9a57b38b5..8dc39a540 100644 --- a/procmgr/processmanager.h +++ b/procmgr/processmanager.h @@ -416,7 +416,8 @@ public: /** *@brief Add Module */ - int addModule(oam::DeviceNetworkList devicenetworklist, std::string password, bool manualFlag = true); + int addModule(oam::DeviceNetworkList devicenetworklist, std::string password, bool storeHostnames, + bool manualFlag = true); /** *@brief Configure Module From 103df7bea9e6c90fa798876fb167a57460696a27 Mon Sep 17 00:00:00 2001 From: Patrick LeBlanc Date: Thu, 29 Nov 2018 15:30:16 -0600 Subject: [PATCH 15/33] Fixed a copy-paste error. --- oam/oamcpp/liboamcpp.cpp | 3 +++ 1 file changed, 3 insertions(+) diff --git a/oam/oamcpp/liboamcpp.cpp b/oam/oamcpp/liboamcpp.cpp index 0ec1e7b1c..58df33782 100644 --- a/oam/oamcpp/liboamcpp.cpp +++ b/oam/oamcpp/liboamcpp.cpp @@ -9931,6 +9931,7 @@ int Oam::sendMsgToProcMgr2(messageqcpp::ByteStream::byte requestType, DeviceNetw msg << (ByteStream::byte) gracefulflag; msg << (ByteStream::byte) ackflag; msg << (ByteStream::byte) requestManual; + msg << (uint16_t) devicenetworklist.size(); DeviceNetworkList::iterator pt = devicenetworklist.begin(); @@ -9948,6 +9949,7 @@ int Oam::sendMsgToProcMgr2(messageqcpp::ByteStream::byte requestType, DeviceNetw msg << " "; else msg << (*pt).DisableState; + msg << (uint16_t) (*pt).hostConfigList.size(); HostConfigList::iterator pt1 = (*pt).hostConfigList.begin(); @@ -10063,6 +10065,7 @@ int Oam::sendAddModuleToProcMgr(messageqcpp::ByteStream::byte requestType, Devic msg << " "; else msg << (*pt).DisableState; + msg << (uint16_t) (*pt).hostConfigList.size(); HostConfigList::iterator pt1 = (*pt).hostConfigList.begin(); From 451073158390d3d074ec31b7269e69bee5eebe64 Mon Sep 17 00:00:00 2001 From: Patrick LeBlanc Date: Fri, 30 Nov 2018 09:44:54 -0600 Subject: [PATCH 16/33] MCOL-1607. It works here but I will likely change the param ordering. --- oamapps/mcsadmin/mcsadmin.cpp | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/oamapps/mcsadmin/mcsadmin.cpp b/oamapps/mcsadmin/mcsadmin.cpp index 180e3e18e..637c93161 100644 --- a/oamapps/mcsadmin/mcsadmin.cpp +++ b/oamapps/mcsadmin/mcsadmin.cpp @@ -5693,7 +5693,7 @@ int processCommand(string* arguments) if ( arguments[1].size() == 2 ) { //Module Type was entered - if (arguments[4] == "" && cloud == oam::UnassignedName) + if (arguments[3] == "" && cloud == oam::UnassignedName) { // need at least arguments cout << endl << "**** addModule Failed : Missing a required Parameter, enter 'help' for additional information" << endl; From 710ee4d66bf4b2b4c23901cc0bcb04323a09fb28 Mon Sep 17 00:00:00 2001 From: Patrick LeBlanc Date: Fri, 30 Nov 2018 10:32:19 -0600 Subject: [PATCH 17/33] MCOL-1607. Changed the 'help' documentation for addModule. --- oam/etc/ConsoleCmds.xml | 7 ++++--- 1 file changed, 4 insertions(+), 3 deletions(-) diff --git a/oam/etc/ConsoleCmds.xml b/oam/etc/ConsoleCmds.xml index 9c58d78f3..17cf0581d 100644 --- a/oam/etc/ConsoleCmds.xml +++ b/oam/etc/ConsoleCmds.xml @@ -309,9 +309,10 @@ MariaDB Columnstore Packages and setup the module to make it ready to be restored Required: Module-type or Module-name being added Required: Number-of-Modules being added when Module-type is specified - Optional: Store given hostnames in the configuration instead of IP addresses (y/N) - Optional: Server-Hostnames/Amazon-Instance-Names seperated by commas - Optional: Server-root-password + Required: Store hostnames in the configuration instead of IP addresses (y/n) + If not Amazon, then Required: hostnames separated by commas + If Amazon, then Optional: Amazon-Instance-Names separated by commas + Optional: Server-user-password removeModule From 9d438f0a24f6e89201ced7dfaba3a0252e5231b8 Mon Sep 17 00:00:00 2001 From: David Hall Date: Tue, 11 Dec 2018 09:37:12 -0600 Subject: [PATCH 18/33] MCOL-1676 Window Functions with FLOAT type gives incorrect answer --- utils/windowfunction/windowfunctiontype.cpp | 72 ++++++++++++++++++++- 1 file changed, 69 insertions(+), 3 deletions(-) diff --git a/utils/windowfunction/windowfunctiontype.cpp b/utils/windowfunction/windowfunctiontype.cpp index 9afb4105d..7f3de615d 100755 --- a/utils/windowfunction/windowfunctiontype.cpp +++ b/utils/windowfunction/windowfunctiontype.cpp @@ -304,6 +304,28 @@ template<> void WindowFunctionType::setValue(uint64_t i, string& t) fRow.setStringField(t, i); } +// MCOL-1676 Need a separate specialization for string now. +template<> +void WindowFunctionType::setValue(int ct, int64_t b, int64_t e, int64_t c, string* v) +{ + if (c != WF__BOUND_ALL) + b = e = c; + + uint64_t i = fFieldIndex[0]; + + if (v == NULL) + v = (string*) getNullValueByType(ct, i); + + for (int64_t j = b; j <= e; j++) + { + if (j % 1000 == 0 && fStep->cancelled()) + break; + + fRow.setData(getPointer((*fRowData)[j])); + setValue(i, *v); + } +} + template void WindowFunctionType::setValue(int ct, int64_t b, int64_t e, int64_t c, T* v) { @@ -320,7 +342,53 @@ void WindowFunctionType::setValue(int ct, int64_t b, int64_t e, int64_t c, T* v) break; fRow.setData(getPointer((*fRowData)[j])); - setValue(i, *v); + // MCOL-1676 Set the data based on out column type (ct) + switch (ct) + { + case CalpontSystemCatalog::TINYINT: + case CalpontSystemCatalog::SMALLINT: + case CalpontSystemCatalog::MEDINT: + case CalpontSystemCatalog::INT: + case CalpontSystemCatalog::BIGINT: + case CalpontSystemCatalog::DECIMAL: + { + int64_t iv = *v; + setValue(i, iv); + break; + } + + case CalpontSystemCatalog::UTINYINT: + case CalpontSystemCatalog::USMALLINT: + case CalpontSystemCatalog::UMEDINT: + case CalpontSystemCatalog::UINT: + case CalpontSystemCatalog::UBIGINT: + case CalpontSystemCatalog::UDECIMAL: + { + uint64_t uv = *v; + setValue(i, uv); + break; + } + + case CalpontSystemCatalog::DOUBLE: + case CalpontSystemCatalog::UDOUBLE: + { + double dv = *v; + setValue(i, dv); + break; + } + + case CalpontSystemCatalog::FLOAT: + case CalpontSystemCatalog::UFLOAT: + { + float fv = *v; + setValue(i, fv); + break; + } + default: + { + setValue(i, *v); + } + } } } @@ -443,8 +511,6 @@ template void WindowFunctionType::setValue(int, int64_t, int64_t, int64 template void WindowFunctionType::setValue(int, int64_t, int64_t, int64_t, uint64_t*); template void WindowFunctionType::setValue(int, int64_t, int64_t, int64_t, float*); template void WindowFunctionType::setValue(int, int64_t, int64_t, int64_t, double*); -template void WindowFunctionType::setValue(int, int64_t, int64_t, int64_t, string*); - void* WindowFunctionType::getNullValueByType(int ct, int pos) { From 9888f88ac5d96f7732784e36d5d8e85ce140f9cd Mon Sep 17 00:00:00 2001 From: Andrew Hutchings Date: Mon, 17 Dec 2018 09:08:30 +0000 Subject: [PATCH 19/33] MCOL-1662 Use version buffer for UPDATEs For some reason version buffer is not turned up for update queries which means that the version number for dictionary blocks is not changed. This can lead to dirty cache reads resulting in _CpNoTf_ in the results. This patch turns on version buffer for updates. --- writeengine/server/we_dmlcommandproc.cpp | 1 + 1 file changed, 1 insertion(+) diff --git a/writeengine/server/we_dmlcommandproc.cpp b/writeengine/server/we_dmlcommandproc.cpp index 90831afd6..006c06b48 100644 --- a/writeengine/server/we_dmlcommandproc.cpp +++ b/writeengine/server/we_dmlcommandproc.cpp @@ -2515,6 +2515,7 @@ uint8_t WE_DMLCommandProc::processUpdate(messageqcpp::ByteStream& bs, boost::scoped_array preBlkNums(new int[columnsUpdated.size()]); boost::scoped_array oids(new OID[columnsUpdated.size()]); + BRMWrapper::setUseVb(true); for (unsigned int j = 0; j < columnsUpdated.size(); j++) { //timer.start("lookupsyscat"); From 7c47ea181f8fc3349adbae0547d5ddb65e9e01ba Mon Sep 17 00:00:00 2001 From: Andrew Hutchings Date: Thu, 20 Dec 2018 15:32:32 +0000 Subject: [PATCH 20/33] MCOL-1726 Make cleartablelock remove txn from BRM With an API txn (and likely other scenarios) cleartablelock will roll back the txn and lock but leave a stale txn pointer in the BRM. This causes restart/shutdown to warn that transactions are still open. This patch makes sure BRM removes the txn from the list. --- tools/cleartablelock/cleartablelock.cpp | 6 ++++++ 1 file changed, 6 insertions(+) diff --git a/tools/cleartablelock/cleartablelock.cpp b/tools/cleartablelock/cleartablelock.cpp index 1a907510e..8c08ea224 100644 --- a/tools/cleartablelock/cleartablelock.cpp +++ b/tools/cleartablelock/cleartablelock.cpp @@ -620,6 +620,12 @@ int main(int argc, char** argv) rc = execBulkRollbackReq( msgQueClts, pmList, &brm, tInfo, tblName.toString(), rollbackOnly, errMsg ); + + BRM::TxnID txn; + txn.id = tInfo.ownerTxnID; + txn.valid = true; + brm.rolledback(txn); + if (rc != 0) { logFinalStatus( tblName.toString(), lockID, errMsg ); From 93fb0902950342fb9ed01c6a0c766bafc0355b97 Mon Sep 17 00:00:00 2001 From: Ben Thompson Date: Fri, 21 Dec 2018 13:54:47 -0600 Subject: [PATCH 21/33] Update VERSION 1.2.3 --- VERSION | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/VERSION b/VERSION index dee66414e..ebe283cd3 100644 --- a/VERSION +++ b/VERSION @@ -1,4 +1,4 @@ COLUMNSTORE_VERSION_MAJOR=1 COLUMNSTORE_VERSION_MINOR=2 -COLUMNSTORE_VERSION_PATCH=2 +COLUMNSTORE_VERSION_PATCH=3 COLUMNSTORE_VERSION_RELEASE=1 From 35a17a87c40b348625785add23786d1095fa40fa Mon Sep 17 00:00:00 2001 From: Roman Nozdrin Date: Tue, 25 Dec 2018 12:50:18 +0300 Subject: [PATCH 22/33] MCOL-1829 Subquery with limited order by could potentially return onordered set. There were two code mistakes: Eq::operator() always returned true for any pair and Hasher::operator() always returned 0 as a key. --- dbcon/joblist/limitedorderby.cpp | 21 ++++++--------------- dbcon/joblist/tupleaggregatestep.cpp | 4 ++++ utils/rowgroup/rowgroup.h | 1 + utils/windowfunction/idborderby.cpp | 7 +++++-- 4 files changed, 16 insertions(+), 17 deletions(-) diff --git a/dbcon/joblist/limitedorderby.cpp b/dbcon/joblist/limitedorderby.cpp index 7da1accfe..d5db83552 100644 --- a/dbcon/joblist/limitedorderby.cpp +++ b/dbcon/joblist/limitedorderby.cpp @@ -113,7 +113,6 @@ void LimitedOrderBy::processRow(const rowgroup::Row& row) if (fOrderByQueue.size() < fStart + fCount) { copyRow(row, &fRow0); - //memcpy(fRow0.getData(), row.getData(), row.getSize()); OrderByRow newRow(fRow0, fRule); fOrderByQueue.push(newRow); @@ -121,8 +120,6 @@ void LimitedOrderBy::processRow(const rowgroup::Row& row) if (fDistinct) fDistinctMap->insert(fRow0.getPointer()); - //fDistinctMap->insert(make_pair((fRow0.getData()+2), fRow0.getData())); - fRowGroup.incRowCount(); fRow0.nextRow(); @@ -150,23 +147,16 @@ void LimitedOrderBy::processRow(const rowgroup::Row& row) { OrderByRow swapRow = fOrderByQueue.top(); row1.setData(swapRow.fData); - fOrderByQueue.pop(); - if (!fDistinct) + copyRow(row, &row1); + + if (fDistinct) { - copyRow(row, &row1); - //memcpy(swapRow.fData, row.getData(), row.getSize()); - } - else - { - fDistinctMap->erase(row.getPointer()); - copyRow(row, &row1); + fDistinctMap->erase(fOrderByQueue.top().fData); fDistinctMap->insert(row1.getPointer()); - //fDistinctMap->erase(fDistinctMap->find(row.getData() + 2)); - //memcpy(swapRow.fData, row.getData(), row.getSize()); - //fDistinctMap->insert(make_pair((swapRow.fData+2), swapRow.fData)); } + fOrderByQueue.pop(); fOrderByQueue.push(swapRow); } } @@ -228,6 +218,7 @@ void LimitedOrderBy::finalize() const OrderByRow& topRow = fOrderByQueue.top(); row1.setData(topRow.fData); copyRow(row1, &fRow0); + //cerr << "LimitedOrderBy::finalize fRow0 " << fRow0.toString() << endl; fRowGroup.incRowCount(); offset--; fRow0.prevRow(rSize); diff --git a/dbcon/joblist/tupleaggregatestep.cpp b/dbcon/joblist/tupleaggregatestep.cpp index 9ddf42cc9..4c1893115 100644 --- a/dbcon/joblist/tupleaggregatestep.cpp +++ b/dbcon/joblist/tupleaggregatestep.cpp @@ -5554,6 +5554,8 @@ void TupleAggregateStep::threadedAggregateRowGroups(uint32_t threadID) rowIn.copyField(distRow[j], k, multiDist->subAggregators()[j]->getGroupByCols()[k].get()->fInputColumnIndex); } + // TBD This approach could potentiall + // put all values in on bucket. bucketID = distRow[j].hash(hashLens[j] - 1) % fNumOfBuckets; rowBucketVecs[bucketID][j].push_back(rowIn.getPointer()); rowIn.nextRow(); @@ -5572,6 +5574,8 @@ void TupleAggregateStep::threadedAggregateRowGroups(uint32_t threadID) for (uint64_t i = 0; i < fRowGroupIns[threadID].getRowCount(); ++i) { // The key is the groupby columns, which are the leading columns. + // TBD This approach could potentiall + // put all values in on bucket. int bucketID = rowIn.hash(hashLens[0] - 1) % fNumOfBuckets; rowBucketVecs[bucketID][0].push_back(rowIn.getPointer()); rowIn.nextRow(); diff --git a/utils/rowgroup/rowgroup.h b/utils/rowgroup/rowgroup.h index e1c3073c4..ca05360c2 100644 --- a/utils/rowgroup/rowgroup.h +++ b/utils/rowgroup/rowgroup.h @@ -1183,6 +1183,7 @@ inline bool Row::equals(const Row& r2, const std::vector& keyCols) con inline bool Row::equals(const Row& r2, uint32_t lastCol) const { + // This check fires with empty r2 only. if (lastCol >= columnCount) return true; diff --git a/utils/windowfunction/idborderby.cpp b/utils/windowfunction/idborderby.cpp index 8a021f8d8..bac14bec8 100644 --- a/utils/windowfunction/idborderby.cpp +++ b/utils/windowfunction/idborderby.cpp @@ -461,7 +461,8 @@ uint64_t IdbOrderBy::Hasher::operator()(const Row::Pointer& p) const { Row& row = ts->row1; row.setPointer(p); - uint64_t ret = row.hash(colCount); + // MCOL-1829 Row::h uses colcount as an array idx down a callstack. + uint64_t ret = row.hash();//(colCount - 1); //cout << "hash(): returning " << ret << " for row: " << row.toString() << endl; return ret; } @@ -471,7 +472,9 @@ bool IdbOrderBy::Eq::operator()(const Row::Pointer& d1, const Row::Pointer& d2) Row& r1 = ts->row1, &r2 = ts->row2; r1.setPointer(d1); r2.setPointer(d2); - bool ret = r1.equals(r2, colCount); + // MCOL-1829 Row::equals uses 2nd argument as container size boundary + // so it must be column count - 1. + bool ret = r1.equals(r2, colCount - 1); //cout << "equals(): returning " << (int) ret << " for r1: " << r1.toString() << " r2: " << r2.toString() // << endl; From 6deb5e1bfde201c6e54ce1959014459d43e7e3b8 Mon Sep 17 00:00:00 2001 From: Gagan Goel Date: Tue, 8 Jan 2019 20:15:04 -0500 Subject: [PATCH 23/33] MCOL-2057 Fix truncate(x,d) when d < 0 for unsigned int data types For unsigned data types in the int family (tinyint, smallint, int, bigint), truncate(x, d) now properly applies zeros when d < 0. --- utils/funcexp/func_truncate.cpp | 29 ++++++++++++++++++++++++++++- 1 file changed, 28 insertions(+), 1 deletion(-) diff --git a/utils/funcexp/func_truncate.cpp b/utils/funcexp/func_truncate.cpp index b822caacd..2da482ab8 100644 --- a/utils/funcexp/func_truncate.cpp +++ b/utils/funcexp/func_truncate.cpp @@ -136,7 +136,34 @@ uint64_t Func_truncate::getUintVal(Row& row, bool& isNull, CalpontSystemCatalog::ColType& op_ct) { - return parm[0]->data()->getUintVal(row, isNull); + uint64_t val = parm[0]->data()->getUintVal(row, isNull); + + if (isNull) + return val; + + int64_t d = parm[1]->data()->getIntVal(row, isNull); + + if (isNull || d >= 0) + return val; + + uint64_t p = 1; + int64_t i = (-d); + + // Handle overflow since p can't have more than 19 0's + if (i >= 20) + { + val = 0; + } + else + { + while (i--) + p *= 10; + + val /= p; + val *= p; + } + + return val; } From fe94776ac6248da418de13fa1d732b1d7559f80c Mon Sep 17 00:00:00 2001 From: Patrick LeBlanc Date: Wed, 9 Jan 2019 16:31:38 -0600 Subject: [PATCH 24/33] MCOL-2062: bug parsing scientific notation in cpimport Made it use strtold instead of the 'manual' string parsing stuff it was doing. --- writeengine/shared/we_convertor.cpp | 107 +++++++--------------------- 1 file changed, 24 insertions(+), 83 deletions(-) diff --git a/writeengine/shared/we_convertor.cpp b/writeengine/shared/we_convertor.cpp index f4ad696d2..8ad344ce3 100644 --- a/writeengine/shared/we_convertor.cpp +++ b/writeengine/shared/we_convertor.cpp @@ -174,93 +174,34 @@ const std::string Convertor::int2Str(int val) * converted long long for specified "field" ******************************************************************************/ /* static */ + long long Convertor::convertDecimalString( const char* field, - int fieldLength, - int scale ) + int fieldLength, + int scale) { - long long llVal = 0; - - int nDigitsBeforeDecPt = 0; - int nDigitsAfterDecPt = 0; - long long roundUp = 0; //@bug 3405 round off decimal column values - - // Determine the number of digits before and after the decimal point - char* posDecPt = (char*)memchr(field, '.', fieldLength); - if (posDecPt) - { - nDigitsBeforeDecPt = posDecPt - field; - nDigitsAfterDecPt = fieldLength - nDigitsBeforeDecPt - 1; - - //@bug 3405 round off decimal column values - // We look at the scale+1 digit to see if we need to round up. - if (nDigitsAfterDecPt > scale) - { - char roundOffDigit = *(posDecPt + 1 + scale); - if ( (roundOffDigit > '4') && - (roundOffDigit <='9') ) // round up - { - roundUp = 1; - - // We can't just use the sign of llVal to determine whether to - // add +1 or -1, because if we read in -0.005 with scale 2, we - // end up parsing "-0.00", which yields 0; meaning we lose the - // sign. So better (though maybe slower) to look for any lead- - // ing negative sign in the input string. - for (int k=0; k LLONG_MAX || dval < LLONG_MIN) + errno = ERANGE; else - { - nDigitsBeforeDecPt = fieldLength; - nDigitsAfterDecPt = 0; - } - - // Strip out the decimal point by stringing together - // the digits before and after the decimal point. - char* data = (char*)alloca(nDigitsBeforeDecPt + scale + 1); - memcpy(data, field, nDigitsBeforeDecPt); - if (nDigitsAfterDecPt) - { - if (scale > nDigitsAfterDecPt) - memcpy(data + nDigitsBeforeDecPt, - field + nDigitsBeforeDecPt + 1, - nDigitsAfterDecPt); - else // (scale <= nDigitsAfterDecPt) - memcpy(data + nDigitsBeforeDecPt, - field + nDigitsBeforeDecPt + 1, - scale); - } - - // Add on any necessary zero padding at the end - if (scale > nDigitsAfterDecPt) - { - memset(data + nDigitsBeforeDecPt + nDigitsAfterDecPt, - '0', - scale - nDigitsAfterDecPt); - } - - data[nDigitsBeforeDecPt + scale] = '\0'; - - // Convert our constructed decimal string back to a long long - //@bug 1814 Force strtoll to use base 10 - errno = 0; - llVal = strtoll(data, 0, 10); - - //@bug 3405 round off decimal values - if ((roundUp) && (errno == 0)) - llVal += roundUp; - - return llVal; + errno = 0; + + ret = dval; + + // get the fractional part of what's left & round ret up or down. + dval -= ret; + if (dval >= 0.5 && ret < LLONG_MAX) + ++ret; + else if (dval <= -0.5 && ret > LLONG_MIN) + --ret; + return ret; } /******************************************************************************* From d844382a0999a918f1e626f2ae32091809e53d76 Mon Sep 17 00:00:00 2001 From: sneJ- Date: Tue, 15 Jan 2019 20:11:26 +0100 Subject: [PATCH 25/33] MCOL-2068 - first try to add memory mgmt parameter to postConfigure - untested --- oamapps/postConfigure/postConfigure.cpp | 191 +++++++++++++++++------- 1 file changed, 135 insertions(+), 56 deletions(-) diff --git a/oamapps/postConfigure/postConfigure.cpp b/oamapps/postConfigure/postConfigure.cpp index e5680485b..e3b0d525b 100644 --- a/oamapps/postConfigure/postConfigure.cpp +++ b/oamapps/postConfigure/postConfigure.cpp @@ -231,6 +231,8 @@ int main(int argc, char* argv[]) string cmd; string pmIpAddrs = ""; string umIpAddrs = ""; + string numBlocksPctParam = ""; + string totalUmMemoryParam = ""; // struct sysinfo myinfo; @@ -314,7 +316,7 @@ int main(int argc, char* argv[]) cout << " Enter one of the options within [], if available, or" << endl; cout << " Enter a new value" << endl << endl; cout << endl; - cout << "Usage: postConfigure [-h][-c][-u][-p][-qs][-qm][-qa][-port][-i][-n][-d][-sn][-pm-ip-addrs][-um-ip-addrs][-pm-count][-um-count]" << endl; + cout << "Usage: postConfigure [-h][-c][-u][-p][-qs][-qm][-qa][-port][-i][-n][-d][-sn][-pm-ip-addrs][-um-ip-addrs][-pm-count][-um-count][-numBlocksPct][-totalUmMemory]" << endl; cout << " -h Help" << endl; cout << " -c Config File to use to extract configuration data, default is Columnstore.xml.rpmsave" << endl; cout << " -u Upgrade, Install using the Config File from -c, default to Columnstore.xml.rpmsave" << endl; @@ -330,6 +332,10 @@ int main(int argc, char* argv[]) cout << " -pm-ip-addrs Performance Module IP Addresses xxx.xxx.xxx.xxx,xxx.xxx.xxx.xxx" << endl; cout << " -um-ip-addrs User Module IP Addresses xxx.xxx.xxx.xxx,xxx.xxx.xxx.xxx" << endl; cout << " -x Do not resolve IP Addresses from host names" << endl; + cout << " -numBlocksPct amount of physical memory to utilize for disk block caching" << endl; + cout << " (percentages of the total memory can be stated without suffix, explcit values with suffixes M or G)" + cout << " -totalUmMemory amount of physical memory to utilize for joins, intermediate results and set operations on the UM" << endl; + cout << " (percentages of the total memory can be stated without suffix, explcit values with suffixes M or G)" exit (0); } else if (string("-x") == argv[i]) @@ -478,10 +484,32 @@ int main(int argc, char* argv[]) } umNumber = atoi(argv[i]); } + else if ( string("-numBlocksPct") == argv[i] ) + { + i++; + if (i >= argc) + { + cout << " ERROR: Memory settings for numBlocksPct not provided" << endl; + //TODO check suffix. if % is given remove it + exit(1); + } + numBlocksPctParam = argv[i]; + } + else if (string("-totalUmMemory") == argv[i]) + { + i++; + if (i >= argc) + { + cout << " ERROR: Memory settings for totalUmMemory not provided" << endl; + //TODO check suffix. if no %, G, or M is given add % + exit(1); + } + totalUmMemoryParam = argv[i]; + } else { cout << " ERROR: Invalid Argument = " << argv[i] << endl; - cout << " Usage: postConfigure [-h][-c][-u][-p][-qs][-qm][-qa][-port][-i][-n][-d][-sn][-pm-ip-addrs][-um-ip-addrs][-pm-count][-um-count]" << endl; + cout << " Usage: postConfigure [-h][-c][-u][-p][-qs][-qm][-qa][-port][-i][-n][-d][-sn][-pm-ip-addrs][-um-ip-addrs][-pm-count][-um-count][-numBlocksPct][-totalUmMemory]" << endl; exit (1); } } @@ -1629,31 +1657,38 @@ int main(int argc, char* argv[]) case (oam::INSTALL_COMBINE_DM_UM_PM): // combined #1 - dm/um/pm on a single server { // are we using settings from previous config file? - if ( reuseConfig == "n" ) + if (reuseConfig == "n") { - if ( !uncommentCalpontXml("NumBlocksPct") ) - { - cout << "Update Columnstore.xml NumBlocksPct Section" << endl; - exit(1); - } string numBlocksPct; - try - { - numBlocksPct = sysConfig->getConfig("DBBC", "NumBlocksPct"); + // if numBlocksPct was set as command line parameter use the command line parameter value + if (!numBlocksPctParam.empty()) { + numBlocksPct = numBlocksPctParam; } - catch (...) - {} + else { + if (!uncommentCalpontXml("NumBlocksPct")) + { + cout << "Update Columnstore.xml NumBlocksPct Section" << endl; + exit(1); + } - if ( numBlocksPct == "70" || numBlocksPct.empty() ) - { - numBlocksPct = "50"; + try + { + numBlocksPct = sysConfig->getConfig("DBBC", "NumBlocksPct"); + } + catch (...) + { + } - if (hdfs) - numBlocksPct = "25"; + if (numBlocksPct == "70" || numBlocksPct.empty()) + { + numBlocksPct = "50"; + + if (hdfs) + numBlocksPct = "25"; + } } - try { sysConfig->setConfig("DBBC", "NumBlocksPct", numBlocksPct); @@ -1666,11 +1701,20 @@ int main(int argc, char* argv[]) exit(1); } - string percent = "25%"; + + string percent; - if (hdfs) - { - percent = "12%"; + if (!totalUmMemoryParam.empty()) { // if totalUmMemory was set as command line parameter use the command line parameter value + percent = totalUmMemoryParam; + } + else { //otherwise use reasonable defaults + + percent = "25%"; + + if (hdfs) + { + percent = "12%"; + } } cout << " Setting 'TotalUmMemory' to " << percent << endl; @@ -1695,22 +1739,33 @@ int main(int argc, char* argv[]) { try { - string numBlocksPct = sysConfig->getConfig("DBBC", "NumBlocksPct"); - cout << endl; - if ( numBlocksPct.empty() ) - cout << "NOTE: Using the default setting for 'NumBlocksPct' at 70%" << endl; - else - cout << "NOTE: Using previous configuration setting for 'NumBlocksPct' = " << numBlocksPct << "%" << endl; + if (!numBlocksPctParam.empty()) { // if numBlocksPct was set as command line parameter use the command line parameter value + sysConfig->setConfig("DBBC", "NumBlocksPct", numBlocksPctParam); + cout << "NOTE: Setting 'NumBlocksPct' to " << numBlocksPctParam << "%" << endl; + } + else { //otherwise use the settings from the previous config file + string numBlocksPct = sysConfig->getConfig("DBBC", "NumBlocksPct"); - string totalUmMemory = sysConfig->getConfig("HashJoin", "TotalUmMemory"); + if (numBlocksPct.empty()) + cout << "NOTE: Using the default setting for 'NumBlocksPct' at 70%" << endl; + else + cout << "NOTE: Using previous configuration setting for 'NumBlocksPct' = " << numBlocksPct << "%" << endl; + } - cout << " Using previous configuration setting for 'TotalUmMemory' = " << totalUmMemory << endl; + if (!totalUmMemoryParam.empty()) { // if totalUmMemory was set as command line parameter use the command line parameter value + sysConfig->setConfig("HashJoin", "TotalUmMemory", totalUmMemoryParam); + cout << " Setting 'TotalUmMemory' to " << totalUmMemoryParam << endl; + } + else { //otherwise use the settings from the previous config file + string totalUmMemory = sysConfig->getConfig("HashJoin", "TotalUmMemory"); + cout << " Using previous configuration setting for 'TotalUmMemory' = " << totalUmMemory << endl; + } } catch (...) { - cout << "ERROR: Problem reading NumBlocksPct/TotalUmMemory in the MariaDB ColumnStore System Configuration file" << endl; + cout << "ERROR: Problem reading/writing NumBlocksPct/TotalUmMemory in/to the MariaDB ColumnStore System Configuration file" << endl; exit(1); } } @@ -1725,25 +1780,31 @@ int main(int argc, char* argv[]) { string numBlocksPct; - try - { - numBlocksPct = sysConfig->getConfig("DBBC", "NumBlocksPct"); + // if numBlocksPct was set as command line parameter use the command line parameter value + if (!numBlocksPctParam.empty()) { + numBlocksPct = numBlocksPctParam; } - catch (...) - {} + else { + try + { + numBlocksPct = sysConfig->getConfig("DBBC", "NumBlocksPct"); + } + catch (...) + { + } - if ( numBlocksPct.empty() ) - { - numBlocksPct = "70"; + if (numBlocksPct.empty()) + { + numBlocksPct = "70"; - if (hdfs) - numBlocksPct = "35"; + if (hdfs) + numBlocksPct = "35"; + } } try { sysConfig->setConfig("DBBC", "NumBlocksPct", numBlocksPct); - cout << "NOTE: Setting 'NumBlocksPct' to " << numBlocksPct << "%" << endl; } catch (...) @@ -1752,11 +1813,18 @@ int main(int argc, char* argv[]) exit(1); } - string percent = "50%"; + string percent; - if (hdfs) - { - percent = "25%"; + if (!totalUmMemoryParam.empty()) { // if totalUmMemory was set as command line parameter use the command line parameter value + percent = totalUmMemoryParam; + } + else { //otherwise use reasonable defaults + percent = "50%"; + + if (hdfs) + { + percent = "25%"; + } } cout << " Setting 'TotalUmMemory' to " << percent << endl; @@ -1781,20 +1849,31 @@ int main(int argc, char* argv[]) { try { - string numBlocksPct = sysConfig->getConfig("DBBC", "NumBlocksPct"); + if (!numBlocksPctParam.empty()) { // if numBlocksPct was set as command line parameter use the command line parameter value + sysConfig->setConfig("DBBC", "NumBlocksPct", numBlocksPctParam); + cout << "NOTE: Setting 'NumBlocksPct' to " << numBlocksPctParam << "%" << endl; + } + else { //otherwise use the settings from the previous config file + string numBlocksPct = sysConfig->getConfig("DBBC", "NumBlocksPct"); - if ( numBlocksPct.empty() ) - cout << "NOTE: Using the default setting for 'NumBlocksPct' at 70%" << endl; - else - cout << "NOTE: Using previous configuration setting for 'NumBlocksPct' = " << numBlocksPct << "%" << endl; + if (numBlocksPct.empty()) + cout << "NOTE: Using the default setting for 'NumBlocksPct' at 70%" << endl; + else + cout << "NOTE: Using previous configuration setting for 'NumBlocksPct' = " << numBlocksPct << "%" << endl; + } - string totalUmMemory = sysConfig->getConfig("HashJoin", "TotalUmMemory"); - - cout << " Using previous configuration setting for 'TotalUmMemory' = " << totalUmMemory << endl; + if (!totalUmMemoryParam.empty()) { // if totalUmMemory was set as command line parameter use the command line parameter value + sysConfig->setConfig("HashJoin", "TotalUmMemory", totalUmMemoryParam); + cout << " Setting 'TotalUmMemory' to " << totalUmMemoryParam << endl; + } + else { //otherwise use reasonable defaults + string totalUmMemory = sysConfig->getConfig("HashJoin", "TotalUmMemory"); + cout << " Using previous configuration setting for 'TotalUmMemory' = " << totalUmMemory << endl; + } } catch (...) { - cout << "ERROR: Problem reading NumBlocksPct/TotalUmMemory in the MariaDB ColumnStore System Configuration file" << endl; + cout << "ERROR: Problem reading/writing NumBlocksPct/TotalUmMemory in/to the MariaDB ColumnStore System Configuration file" << endl; exit(1); } } From e60ac3001773c6a9db5013de1fa49eb45d87ffdc Mon Sep 17 00:00:00 2001 From: sneJ- Date: Wed, 16 Jan 2019 14:00:09 +0100 Subject: [PATCH 26/33] MCOL-2068 - introduced basic parameter checks --- oamapps/postConfigure/postConfigure.cpp | 52 ++++++++++++++++++++----- 1 file changed, 42 insertions(+), 10 deletions(-) diff --git a/oamapps/postConfigure/postConfigure.cpp b/oamapps/postConfigure/postConfigure.cpp index e3b0d525b..59b1cfc73 100644 --- a/oamapps/postConfigure/postConfigure.cpp +++ b/oamapps/postConfigure/postConfigure.cpp @@ -333,9 +333,9 @@ int main(int argc, char* argv[]) cout << " -um-ip-addrs User Module IP Addresses xxx.xxx.xxx.xxx,xxx.xxx.xxx.xxx" << endl; cout << " -x Do not resolve IP Addresses from host names" << endl; cout << " -numBlocksPct amount of physical memory to utilize for disk block caching" << endl; - cout << " (percentages of the total memory can be stated without suffix, explcit values with suffixes M or G)" + cout << " (percentages of the total memory need to be stated without suffix, explcit values with suffixes M or G)"; cout << " -totalUmMemory amount of physical memory to utilize for joins, intermediate results and set operations on the UM" << endl; - cout << " (percentages of the total memory can be stated without suffix, explcit values with suffixes M or G)" + cout << " (percentages of the total memory need to be stated with suffix %, explcit values with suffixes M or G)"; exit (0); } else if (string("-x") == argv[i]) @@ -490,10 +490,16 @@ int main(int argc, char* argv[]) if (i >= argc) { cout << " ERROR: Memory settings for numBlocksPct not provided" << endl; - //TODO check suffix. if % is given remove it exit(1); } - numBlocksPctParam = argv[i]; + // check that the parameter ends with a number M or G + if (isdigit(argv[i].back()) || argv[i].back() == 'M' || argv[i].bach() == 'G') { + numBlocksPctParam = argv[i]; + } + else { + cout << " ERROR: Memory settings for numBlocksPct need to end on a digit, M or G" << endl; + exit(1); + } } else if (string("-totalUmMemory") == argv[i]) { @@ -501,10 +507,16 @@ int main(int argc, char* argv[]) if (i >= argc) { cout << " ERROR: Memory settings for totalUmMemory not provided" << endl; - //TODO check suffix. if no %, G, or M is given add % exit(1); } - totalUmMemoryParam = argv[i]; + // check that the parameter ends with a %, M, or G + if (argv[i].back() == '%' || argv[i].back() == 'M' || argv[i].bach() == 'G') { + totalUmMemoryParam = argv[i]; + } + else { + cout << " ERROR: Memory settings for totalUmMemory need to end on %, M or G" << endl; + exit(1); + } } else { @@ -1693,7 +1705,12 @@ int main(int argc, char* argv[]) { sysConfig->setConfig("DBBC", "NumBlocksPct", numBlocksPct); - cout << endl << "NOTE: Setting 'NumBlocksPct' to " << numBlocksPct << "%" << endl; + if (numBlocksPct.back() == 'M' || numBlocksPct.back() == 'G') { + cout << endl << "NOTE: Setting 'NumBlocksPct' to " << numBlocksPct << endl; + } + else { + cout << endl << "NOTE: Setting 'NumBlocksPct' to " << numBlocksPct << "%" << endl; + } } catch (...) { @@ -1743,7 +1760,12 @@ int main(int argc, char* argv[]) if (!numBlocksPctParam.empty()) { // if numBlocksPct was set as command line parameter use the command line parameter value sysConfig->setConfig("DBBC", "NumBlocksPct", numBlocksPctParam); - cout << "NOTE: Setting 'NumBlocksPct' to " << numBlocksPctParam << "%" << endl; + if (numBlocksPctParam.back() == 'M' || numBlocksPctParam.back() == 'G') { + cout << endl << "NOTE: Setting 'NumBlocksPct' to " << numBlocksPctParam << endl; + } + else { + cout << endl << "NOTE: Setting 'NumBlocksPct' to " << numBlocksPctParam << "%" << endl; + } } else { //otherwise use the settings from the previous config file string numBlocksPct = sysConfig->getConfig("DBBC", "NumBlocksPct"); @@ -1805,7 +1827,12 @@ int main(int argc, char* argv[]) try { sysConfig->setConfig("DBBC", "NumBlocksPct", numBlocksPct); - cout << "NOTE: Setting 'NumBlocksPct' to " << numBlocksPct << "%" << endl; + if (numBlocksPct.back() == 'M' || numBlocksPct.back() == 'G') { + cout << endl << "NOTE: Setting 'NumBlocksPct' to " << numBlocksPct << endl; + } + else { + cout << endl << "NOTE: Setting 'NumBlocksPct' to " << numBlocksPct << "%" << endl; + } } catch (...) { @@ -1851,7 +1878,12 @@ int main(int argc, char* argv[]) { if (!numBlocksPctParam.empty()) { // if numBlocksPct was set as command line parameter use the command line parameter value sysConfig->setConfig("DBBC", "NumBlocksPct", numBlocksPctParam); - cout << "NOTE: Setting 'NumBlocksPct' to " << numBlocksPctParam << "%" << endl; + if (numBlocksPctParam.back() == 'M' || numBlocksPctParam.back() == 'G') { + cout << endl << "NOTE: Setting 'NumBlocksPct' to " << numBlocksPctParam << endl; + } + else { + cout << endl << "NOTE: Setting 'NumBlocksPct' to " << numBlocksPctParam << "%" << endl; + } } else { //otherwise use the settings from the previous config file string numBlocksPct = sysConfig->getConfig("DBBC", "NumBlocksPct"); From d04ebca2c4f7e3d71fa77f85eb665ef706cf9aa6 Mon Sep 17 00:00:00 2001 From: sneJ- Date: Wed, 16 Jan 2019 15:06:08 +0100 Subject: [PATCH 27/33] MCOL-2068 - changed .back() into .rbegin() to be compileable in non C++11 environments --- oamapps/postConfigure/postConfigure.cpp | 12 ++++++------ 1 file changed, 6 insertions(+), 6 deletions(-) diff --git a/oamapps/postConfigure/postConfigure.cpp b/oamapps/postConfigure/postConfigure.cpp index 59b1cfc73..9453560c6 100644 --- a/oamapps/postConfigure/postConfigure.cpp +++ b/oamapps/postConfigure/postConfigure.cpp @@ -493,7 +493,7 @@ int main(int argc, char* argv[]) exit(1); } // check that the parameter ends with a number M or G - if (isdigit(argv[i].back()) || argv[i].back() == 'M' || argv[i].bach() == 'G') { + if (isdigit(argv[i].rbegin()) || argv[i].rbegin() == 'M' || argv[i].rbegin() == 'G') { numBlocksPctParam = argv[i]; } else { @@ -510,7 +510,7 @@ int main(int argc, char* argv[]) exit(1); } // check that the parameter ends with a %, M, or G - if (argv[i].back() == '%' || argv[i].back() == 'M' || argv[i].bach() == 'G') { + if (argv[i].rbegin() == '%' || argv[i].rbegin() == 'M' || argv[i].rbegin() == 'G') { totalUmMemoryParam = argv[i]; } else { @@ -1705,7 +1705,7 @@ int main(int argc, char* argv[]) { sysConfig->setConfig("DBBC", "NumBlocksPct", numBlocksPct); - if (numBlocksPct.back() == 'M' || numBlocksPct.back() == 'G') { + if (numBlocksPct.rbegin() == 'M' || numBlocksPct.rbegin() == 'G') { cout << endl << "NOTE: Setting 'NumBlocksPct' to " << numBlocksPct << endl; } else { @@ -1760,7 +1760,7 @@ int main(int argc, char* argv[]) if (!numBlocksPctParam.empty()) { // if numBlocksPct was set as command line parameter use the command line parameter value sysConfig->setConfig("DBBC", "NumBlocksPct", numBlocksPctParam); - if (numBlocksPctParam.back() == 'M' || numBlocksPctParam.back() == 'G') { + if (numBlocksPctParam.rbegin() == 'M' || numBlocksPctParam.rbegin() == 'G') { cout << endl << "NOTE: Setting 'NumBlocksPct' to " << numBlocksPctParam << endl; } else { @@ -1827,7 +1827,7 @@ int main(int argc, char* argv[]) try { sysConfig->setConfig("DBBC", "NumBlocksPct", numBlocksPct); - if (numBlocksPct.back() == 'M' || numBlocksPct.back() == 'G') { + if (numBlocksPct.rbegin() == 'M' || numBlocksPct.rbegin() == 'G') { cout << endl << "NOTE: Setting 'NumBlocksPct' to " << numBlocksPct << endl; } else { @@ -1878,7 +1878,7 @@ int main(int argc, char* argv[]) { if (!numBlocksPctParam.empty()) { // if numBlocksPct was set as command line parameter use the command line parameter value sysConfig->setConfig("DBBC", "NumBlocksPct", numBlocksPctParam); - if (numBlocksPctParam.back() == 'M' || numBlocksPctParam.back() == 'G') { + if (numBlocksPctParam.rbegin() == 'M' || numBlocksPctParam.rbegin() == 'G') { cout << endl << "NOTE: Setting 'NumBlocksPct' to " << numBlocksPctParam << endl; } else { From 0689ec1b5bff76b3b768ab3a936d6129ad97932f Mon Sep 17 00:00:00 2001 From: sneJ- Date: Wed, 16 Jan 2019 15:43:57 +0100 Subject: [PATCH 28/33] MCOL-2068 maybe the pointer helps --- oamapps/postConfigure/postConfigure.cpp | 12 ++++++------ 1 file changed, 6 insertions(+), 6 deletions(-) diff --git a/oamapps/postConfigure/postConfigure.cpp b/oamapps/postConfigure/postConfigure.cpp index 9453560c6..9f9391802 100644 --- a/oamapps/postConfigure/postConfigure.cpp +++ b/oamapps/postConfigure/postConfigure.cpp @@ -493,7 +493,7 @@ int main(int argc, char* argv[]) exit(1); } // check that the parameter ends with a number M or G - if (isdigit(argv[i].rbegin()) || argv[i].rbegin() == 'M' || argv[i].rbegin() == 'G') { + if (isdigit(*argv[i].rbegin()) || *argv[i].rbegin() == 'M' || *argv[i].rbegin() == 'G') { numBlocksPctParam = argv[i]; } else { @@ -510,7 +510,7 @@ int main(int argc, char* argv[]) exit(1); } // check that the parameter ends with a %, M, or G - if (argv[i].rbegin() == '%' || argv[i].rbegin() == 'M' || argv[i].rbegin() == 'G') { + if (*argv[i].rbegin() == '%' || *argv[i].rbegin() == 'M' || *argv[i].rbegin() == 'G') { totalUmMemoryParam = argv[i]; } else { @@ -1705,7 +1705,7 @@ int main(int argc, char* argv[]) { sysConfig->setConfig("DBBC", "NumBlocksPct", numBlocksPct); - if (numBlocksPct.rbegin() == 'M' || numBlocksPct.rbegin() == 'G') { + if (*numBlocksPct.rbegin() == 'M' || *numBlocksPct.rbegin() == 'G') { cout << endl << "NOTE: Setting 'NumBlocksPct' to " << numBlocksPct << endl; } else { @@ -1760,7 +1760,7 @@ int main(int argc, char* argv[]) if (!numBlocksPctParam.empty()) { // if numBlocksPct was set as command line parameter use the command line parameter value sysConfig->setConfig("DBBC", "NumBlocksPct", numBlocksPctParam); - if (numBlocksPctParam.rbegin() == 'M' || numBlocksPctParam.rbegin() == 'G') { + if (*numBlocksPctParam.rbegin() == 'M' || *numBlocksPctParam.rbegin() == 'G') { cout << endl << "NOTE: Setting 'NumBlocksPct' to " << numBlocksPctParam << endl; } else { @@ -1827,7 +1827,7 @@ int main(int argc, char* argv[]) try { sysConfig->setConfig("DBBC", "NumBlocksPct", numBlocksPct); - if (numBlocksPct.rbegin() == 'M' || numBlocksPct.rbegin() == 'G') { + if (*numBlocksPct.rbegin() == 'M' || *numBlocksPct.rbegin() == 'G') { cout << endl << "NOTE: Setting 'NumBlocksPct' to " << numBlocksPct << endl; } else { @@ -1878,7 +1878,7 @@ int main(int argc, char* argv[]) { if (!numBlocksPctParam.empty()) { // if numBlocksPct was set as command line parameter use the command line parameter value sysConfig->setConfig("DBBC", "NumBlocksPct", numBlocksPctParam); - if (numBlocksPctParam.rbegin() == 'M' || numBlocksPctParam.rbegin() == 'G') { + if (*numBlocksPctParam.rbegin() == 'M' || *numBlocksPctParam.rbegin() == 'G') { cout << endl << "NOTE: Setting 'NumBlocksPct' to " << numBlocksPctParam << endl; } else { From fdc9f558406143ddca0806e8486991692a9d6671 Mon Sep 17 00:00:00 2001 From: sneJ- Date: Wed, 16 Jan 2019 16:15:32 +0100 Subject: [PATCH 29/33] MCOL-2068 - next try --- oamapps/postConfigure/postConfigure.cpp | 12 ++++-------- 1 file changed, 4 insertions(+), 8 deletions(-) diff --git a/oamapps/postConfigure/postConfigure.cpp b/oamapps/postConfigure/postConfigure.cpp index 9f9391802..e51f3c474 100644 --- a/oamapps/postConfigure/postConfigure.cpp +++ b/oamapps/postConfigure/postConfigure.cpp @@ -492,11 +492,9 @@ int main(int argc, char* argv[]) cout << " ERROR: Memory settings for numBlocksPct not provided" << endl; exit(1); } + numBlocksPctParam = argv[i]; // check that the parameter ends with a number M or G - if (isdigit(*argv[i].rbegin()) || *argv[i].rbegin() == 'M' || *argv[i].rbegin() == 'G') { - numBlocksPctParam = argv[i]; - } - else { + if (!(isdigit(*numBlocksPctParam.rbegin()) || *numBlocksPctParam.rbegin() == 'M' || *numBlocksPctParam.rbegin() == 'G')) { cout << " ERROR: Memory settings for numBlocksPct need to end on a digit, M or G" << endl; exit(1); } @@ -509,11 +507,9 @@ int main(int argc, char* argv[]) cout << " ERROR: Memory settings for totalUmMemory not provided" << endl; exit(1); } + totalUmMemoryParam = argv[i]; // check that the parameter ends with a %, M, or G - if (*argv[i].rbegin() == '%' || *argv[i].rbegin() == 'M' || *argv[i].rbegin() == 'G') { - totalUmMemoryParam = argv[i]; - } - else { + if (!(*totalUmMemoryParam.rbegin() == '%' || *totalUmMemoryParam.rbegin() == 'M' || *totalUmMemoryParam.rbegin() == 'G')) { cout << " ERROR: Memory settings for totalUmMemory need to end on %, M or G" << endl; exit(1); } From 83f493eef391acabe2b5aadf79941e96c9fc7ecc Mon Sep 17 00:00:00 2001 From: sneJ- Date: Wed, 16 Jan 2019 20:14:56 +0100 Subject: [PATCH 30/33] MCOL-2068 - added the memory command line parameters to the installer as well to support single type installations --- oamapps/postConfigure/installer.cpp | 189 ++++++++++++++++++------ oamapps/postConfigure/postConfigure.cpp | 14 +- 2 files changed, 153 insertions(+), 50 deletions(-) diff --git a/oamapps/postConfigure/installer.cpp b/oamapps/postConfigure/installer.cpp index 94850deae..1bc52f173 100644 --- a/oamapps/postConfigure/installer.cpp +++ b/oamapps/postConfigure/installer.cpp @@ -126,10 +126,22 @@ int main(int argc, char* argv[]) mysqlpw = argv[10]; installer_debug = argv[11]; - if (argc >= 13) + string totalBlocksPctParam = ""; + string totalUmMemoryParam = ""; + if (argc >= 13) { installDir = argv[12]; - else + if (argc >= 15) { + if (!string(argv[13]).compare("-")) { + totalBlocksPctParam = argv[13]; + } + if (!string(argv[14]).compare("-")) { + totalUmMemoryParam = argv[14]; + } + } + } + else { installDir = "/usr/local/mariadb/columnstore"; + } ofstream file("/dev/null"); @@ -151,6 +163,12 @@ int main(int argc, char* argv[]) cout << mysqlpw << endl; cout << installer_debug << endl; cout << installDir << endl; + if (!totalBlocksPctParam.empty()) { + cout << totalBlocksPctParam << endl; + } + if (!totalUmMemoryParam.empty()) { + cout << totalUmMemoryParam << endl; + } // restore cout stream buffer cout.rdbuf (strm_buffer); @@ -255,16 +273,26 @@ int main(int argc, char* argv[]) return false; } - string numBlocksPct = "50"; - - if ( DBRootStorageType == "hdfs") - numBlocksPct = "25"; + string numBlocksPct; + if (numBlocksPctParam.empty()) { + numBlocksPct = "50"; + if (DBRootStorageType == "hdfs") + numBlocksPct = "25"; + } + else { + numBlocksPct = numBlocksPctParam; + } try { sysConfig->setConfig("DBBC", "NumBlocksPct", numBlocksPct); - cout << endl << "NOTE: Setting 'NumBlocksPct' to " << numBlocksPct << "%" << endl; + if (*numBlocksPct.rbegin() == 'M' || *numBlocksPct.rbegin() == 'G') { + cout << endl << "NOTE: Setting 'NumBlocksPct' to " << numBlocksPct << endl; + } + else { + cout << endl << "NOTE: Setting 'NumBlocksPct' to " << numBlocksPct << "%" << endl; + } } catch (...) { @@ -282,15 +310,21 @@ int main(int argc, char* argv[]) // long long total = myinfo.totalram / 1024 / 1000; // adjust max memory, 25% of total memory - string percent = "25%"; + string percent; + if (totalUmMemoryParam.empty()) { + percent = "25%"; - if ( DBRootStorageType == "hdfs") - { - percent = "12%"; + if (DBRootStorageType == "hdfs") + { + percent = "12%"; + } + + cout << " Setting 'TotalUmMemory' to " << percent << " of total memory. " << endl; + } + else { + percent = totalUmMemoryParam; + cout << " Setting 'TotalUmMemory' to " << percent << endl; } - - cout << " Setting 'TotalUmMemory' to " << percent << " of total memory. " << endl; - try { sysConfig->setConfig("HashJoin", "TotalUmMemory", percent); @@ -305,29 +339,47 @@ int main(int argc, char* argv[]) { try { - string numBlocksPct = sysConfig->getConfig("DBBC", "NumBlocksPct"); - string totalUmMemory = sysConfig->getConfig("HashJoin", "TotalUmMemory"); + string numBlocksPct; + string totalUmMemory; - if (numBlocksPct.empty() || numBlocksPct == "" ) - { - numBlocksPct = "50"; - - try - { - sysConfig->setConfig("DBBC", "NumBlocksPct", "50"); - - cout << endl << "NOTE: Setting 'NumBlocksPct' to 50%" << endl; - } - catch (...) - { - cout << "ERROR: Problem setting NumBlocksPct in the MariaDB ColumnStore System Configuration file" << endl; - exit(1); - } + if (!numBlocksPctParam.empty()) { + numBlocksPct = numBlocksPctParam; + } + else { + numBlocksPct = sysConfig->getConfig("DBBC", "NumBlocksPct"); + } + if (!totalUmMemoryParam.empty()) { + totalUmMemory = totalUmMemoryParam; + } + else { + totalUmMemory = sysConfig->getConfig("HashJoin", "TotalUmMemory"); } - cout << endl << "NOTE: Using previous configuration setting for 'NumBlocksPct' = " << numBlocksPct << "%" << endl; - cout << " Using previous configuration setting for 'TotalUmMemory' = " << totalUmMemory << endl; + if (numBlocksPct.empty() || numBlocksPct == "") + { + numBlocksPct = "50"; + } + if (totalUmMemory.empty() || totalUmMemory == "") { + totalUmMemory = "25%"; + } + try + { + sysConfig->setConfig("DBBC", "NumBlocksPct", numBlocksPct); + sysConfig->setConfig("HashJoin", "TotalUmMemory", totalUmMemory); + if (*numBlocksPct.rbegin() == 'M' || *numBlocksPct.rbegin() == 'G') { + cout << endl << "NOTE: Setting 'NumBlocksPct' to " << numBlocksPct << endl; + } + else { + cout << endl << "NOTE: Setting 'NumBlocksPct' to " << numBlocksPct << "%" << endl; + } + cout << " Setting 'TotalUmMemory' to " << totalUmMemory << endl; + } + catch (...) + { + cout << "ERROR: Problem setting NumBlocksPct/TotalUmMemory in the MariaDB ColumnStore System Configuration file" << endl; + exit(1); + } } catch (...) { @@ -350,7 +402,23 @@ int main(int argc, char* argv[]) // are we using settings from previous config file? if ( reuseConfig == "n" ) { - cout << endl << "NOTE: Using the default setting for 'NumBlocksPct' at 70%" << endl; + string numBlocksPct = "70" + string totalUmMemory = "50%"; + + if (!numBlocksPctParam.empty()) { + numBlocksPct = numBlocksPctParam; + cout << endl << "NOTE: Setting 'NumBlocksPct' to " << numBlocksPct << endl; + } + else { + cout << endl << "NOTE: Using the default setting for 'NumBlocksPct' at " << numBlocksPct << "%" << endl; + } + if (!totalUmMemoryParam.empty()) { + totalUmMemory = totalUmMemoryParam; + cout << endl << "Setting 'TotalUmMemory' to " << totalUmMemory << endl; + } + else { + cout << endl << "Setting 'TotalUmMemory' to " << totalUmMemory << " of total memory." << endl; + } try { @@ -358,17 +426,14 @@ int main(int argc, char* argv[]) } catch (...) {} - // adjust max memory, 50% of total memory - - cout << endl << "Setting 'TotalUmMemory' to 50% of total memory." << endl; - try { - sysConfig->setConfig("HashJoin", "TotalUmMemory", "50%"); + sysConfig->setConfig("DBBC", "NumBlocksPct", numBlocksPct); + sysConfig->setConfig("HashJoin", "TotalUmMemory", totalUmMemory); } catch (...) { - cout << "ERROR: Problem setting TotalUmMemory in the MariaDB ColumnStore System Configuration file" << endl; + cout << "ERROR: Problem setting TotalUmMemory/NumBlocksPct in the MariaDB ColumnStore System Configuration file" << endl; exit(1); } } @@ -376,15 +441,47 @@ int main(int argc, char* argv[]) { try { - string numBlocksPct = sysConfig->getConfig("DBBC", "NumBlocksPct"); - string totalUmMemory = sysConfig->getConfig("HashJoin", "TotalUmMemory"); + string numBlocksPct; + string totalUmMemory; - if (numBlocksPct.empty() || numBlocksPct == "" ) + if (!numBlocksPctParam.empty()) { + numBlocksPct = numBlocksPctParam; + } + else { + numBlocksPct = sysConfig->getConfig("DBBC", "NumBlocksPct"); + } + if (!totalUmMemoryParam.empty()) { + totalUmMemory = totalUmMemoryParam; + } + else { + totalUmMemory = sysConfig->getConfig("HashJoin", "TotalUmMemory"); + } + + if (numBlocksPct.empty() || numBlocksPct == "") + { numBlocksPct = "70"; + } + if (totalUmMemory.empty() || totalUmMemory == "") { + totalUmMemory = "50%"; + } + try + { + sysConfig->setConfig("DBBC", "NumBlocksPct", numBlocksPct); + sysConfig->setConfig("HashJoin", "TotalUmMemory", totalUmMemory); - cout << endl << "NOTE: Using previous configuration setting for 'NumBlocksPct' = " << numBlocksPct << "%" << endl; - cout << " Using previous configuration setting for 'TotalUmMemory' = " << totalUmMemory << endl; - + if (*numBlocksPct.rbegin() == 'M' || *numBlocksPct.rbegin() == 'G') { + cout << endl << "NOTE: Setting 'NumBlocksPct' to " << numBlocksPct << endl; + } + else { + cout << endl << "NOTE: Setting 'NumBlocksPct' to " << numBlocksPct << "%" << endl; + } + cout << " Setting 'TotalUmMemory' to " << totalUmMemory << endl; + } + catch (...) + { + cout << "ERROR: Problem setting NumBlocksPct/TotalUmMemory in the MariaDB ColumnStore System Configuration file" << endl; + exit(1); + } } catch (...) { diff --git a/oamapps/postConfigure/postConfigure.cpp b/oamapps/postConfigure/postConfigure.cpp index e51f3c474..8796e9b5f 100644 --- a/oamapps/postConfigure/postConfigure.cpp +++ b/oamapps/postConfigure/postConfigure.cpp @@ -333,9 +333,9 @@ int main(int argc, char* argv[]) cout << " -um-ip-addrs User Module IP Addresses xxx.xxx.xxx.xxx,xxx.xxx.xxx.xxx" << endl; cout << " -x Do not resolve IP Addresses from host names" << endl; cout << " -numBlocksPct amount of physical memory to utilize for disk block caching" << endl; - cout << " (percentages of the total memory need to be stated without suffix, explcit values with suffixes M or G)"; + cout << " (percentages of the total memory need to be stated without suffix, explcit values with suffixes M or G)" << endl; cout << " -totalUmMemory amount of physical memory to utilize for joins, intermediate results and set operations on the UM" << endl; - cout << " (percentages of the total memory need to be stated with suffix %, explcit values with suffixes M or G)"; + cout << " (percentages of the total memory need to be stated with suffix %, explcit values with suffixes M or G)" << endl; exit (0); } else if (string("-x") == argv[i]) @@ -994,7 +994,6 @@ int main(int argc, char* argv[]) offLineAppCheck(); checkMysqlPort(mysqlPort, sysConfig); - if ( !writeConfig(sysConfig) ) { cout << "ERROR: Failed trying to update MariaDB ColumnStore System Configuration file" << endl; @@ -1003,7 +1002,14 @@ int main(int argc, char* argv[]) cout << endl << "===== Performing Configuration Setup and MariaDB ColumnStore Startup =====" << endl; - cmd = installDir + "/bin/installer dummy.rpm dummy.rpm dummy.rpm dummy.rpm dummy.rpm initial dummy " + reuseConfig + " --nodeps ' ' 1 " + installDir; + if (numBlocksPctParam.empty()) { + numBlocksPctParam = "-"; + } + if (totalUmMemoryParam.empty()) { + totalUmMemoryParam = "-"; + } + + cmd = installDir + "/bin/installer dummy.rpm dummy.rpm dummy.rpm dummy.rpm dummy.rpm initial dummy " + reuseConfig + " --nodeps ' ' 1 " + installDir + " " + totalBlocksPctParam + " " + totalUmMemoryParam; system(cmd.c_str()); exit(0); } From 2c5b65294c5846c6125fbcca8f3f699c2c38bf98 Mon Sep 17 00:00:00 2001 From: sneJ- Date: Wed, 16 Jan 2019 20:45:14 +0100 Subject: [PATCH 31/33] MCOL-2068 typo fixed --- oamapps/postConfigure/installer.cpp | 10 +++++----- 1 file changed, 5 insertions(+), 5 deletions(-) diff --git a/oamapps/postConfigure/installer.cpp b/oamapps/postConfigure/installer.cpp index 1bc52f173..428911ebe 100644 --- a/oamapps/postConfigure/installer.cpp +++ b/oamapps/postConfigure/installer.cpp @@ -126,13 +126,13 @@ int main(int argc, char* argv[]) mysqlpw = argv[10]; installer_debug = argv[11]; - string totalBlocksPctParam = ""; + string numBlocksPctParam = ""; string totalUmMemoryParam = ""; if (argc >= 13) { installDir = argv[12]; if (argc >= 15) { if (!string(argv[13]).compare("-")) { - totalBlocksPctParam = argv[13]; + numBlocksPctParam = argv[13]; } if (!string(argv[14]).compare("-")) { totalUmMemoryParam = argv[14]; @@ -163,8 +163,8 @@ int main(int argc, char* argv[]) cout << mysqlpw << endl; cout << installer_debug << endl; cout << installDir << endl; - if (!totalBlocksPctParam.empty()) { - cout << totalBlocksPctParam << endl; + if (!numBlocksPctParam.empty()) { + cout << numBlocksPctParam << endl; } if (!totalUmMemoryParam.empty()) { cout << totalUmMemoryParam << endl; @@ -402,7 +402,7 @@ int main(int argc, char* argv[]) // are we using settings from previous config file? if ( reuseConfig == "n" ) { - string numBlocksPct = "70" + string numBlocksPct = "70"; string totalUmMemory = "50%"; if (!numBlocksPctParam.empty()) { From 573a740e8aa5cf352b5353f79363f7072d0fc938 Mon Sep 17 00:00:00 2001 From: sneJ- Date: Wed, 16 Jan 2019 21:08:12 +0100 Subject: [PATCH 32/33] MCOL-2068 typo --- oamapps/postConfigure/postConfigure.cpp | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/oamapps/postConfigure/postConfigure.cpp b/oamapps/postConfigure/postConfigure.cpp index 8796e9b5f..c92697127 100644 --- a/oamapps/postConfigure/postConfigure.cpp +++ b/oamapps/postConfigure/postConfigure.cpp @@ -1009,7 +1009,7 @@ int main(int argc, char* argv[]) totalUmMemoryParam = "-"; } - cmd = installDir + "/bin/installer dummy.rpm dummy.rpm dummy.rpm dummy.rpm dummy.rpm initial dummy " + reuseConfig + " --nodeps ' ' 1 " + installDir + " " + totalBlocksPctParam + " " + totalUmMemoryParam; + cmd = installDir + "/bin/installer dummy.rpm dummy.rpm dummy.rpm dummy.rpm dummy.rpm initial dummy " + reuseConfig + " --nodeps ' ' 1 " + installDir + " " + numBlocksPctParam + " " + totalUmMemoryParam; system(cmd.c_str()); exit(0); } From 17a446c37cca96c71c6abc924a8b00de73554e88 Mon Sep 17 00:00:00 2001 From: sneJ- Date: Thu, 17 Jan 2019 12:23:59 +0100 Subject: [PATCH 33/33] MCOL-2068 - string comparison fixed --- oamapps/postConfigure/installer.cpp | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/oamapps/postConfigure/installer.cpp b/oamapps/postConfigure/installer.cpp index 428911ebe..81b69e548 100644 --- a/oamapps/postConfigure/installer.cpp +++ b/oamapps/postConfigure/installer.cpp @@ -131,10 +131,10 @@ int main(int argc, char* argv[]) if (argc >= 13) { installDir = argv[12]; if (argc >= 15) { - if (!string(argv[13]).compare("-")) { + if (string(argv[13]) != "-") { numBlocksPctParam = argv[13]; } - if (!string(argv[14]).compare("-")) { + if (string(argv[14]) != "-") { totalUmMemoryParam = argv[14]; } }