From 7fa7dddc9bc920a3a5078ed8ddfcb5838ff4babd Mon Sep 17 00:00:00 2001 From: david hill Date: Wed, 23 May 2018 14:45:47 -0500 Subject: [PATCH 01/55] bump release --- README | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) diff --git a/README b/README index 8186d1ed7..ca0d9cd19 100644 --- a/README +++ b/README @@ -1,9 +1,9 @@ -This is MariaDB ColumnStore 1.0.12 -MariaDB ColumnStore 1.0.12 is the development version of MariaDB ColumnStore. +This is MariaDB ColumnStore 1.0.14 +MariaDB ColumnStore 1.0.14 is the development version of MariaDB ColumnStore. It is built by porting InfiniDB 4.6.7 on MariaDB 10.1.26 and adding entirely new features not found anywhere else. -MariaDB ColumnStore 1.0.12 is an GA release. This is the first MariaDB +MariaDB ColumnStore 1.0.14 is an GA release. This is the first MariaDB ColumnStore release, not all features planned for the MariaDB ColumnStore 1.0 series are included in this release. From 60eb6516a25bc49a45adb2f4613a885458d9d058 Mon Sep 17 00:00:00 2001 From: david hill Date: Wed, 23 May 2018 14:46:31 -0500 Subject: [PATCH 02/55] bump release --- README.md | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) diff --git a/README.md b/README.md index 40c1e99ec..5cd3824eb 100644 --- a/README.md +++ b/README.md @@ -1,9 +1,9 @@ -#MariaDB ColumnStore Storage/Execution engine 1.0.12 -MariaDB ColumnStore 1.0.12 is the development version of MariaDB ColumnStore. +#MariaDB ColumnStore Storage/Execution engine 1.0.14 +MariaDB ColumnStore 1.0.14 is the development version of MariaDB ColumnStore. It is built by porting InfiniDB 4.6.7 on MariaDB 10.1.26 and adding entirely new features not found anywhere else. -#MariaDB ColumnStore 1.0.12 is an GA release. +#MariaDB ColumnStore 1.0.14 is an GA release. #Building This repository is not meant to be built independently outside of the server. This repository is integrated into http://mariadb-corporation/mariadb-columnstore-server (ie, the *server*) as a git submodule. As such, you can find complete build instructions on *the server* page. From 4ad2c0a8f2cb77554db2f9a7311522e5bfa6c046 Mon Sep 17 00:00:00 2001 From: david hill Date: Wed, 23 May 2018 14:47:05 -0500 Subject: [PATCH 03/55] bump release --- VERSION | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/VERSION b/VERSION index 7053d9518..1e84dad87 100644 --- a/VERSION +++ b/VERSION @@ -1,4 +1,4 @@ COLUMNSTORE_VERSION_MAJOR=1 COLUMNSTORE_VERSION_MINOR=0 -COLUMNSTORE_VERSION_PATCH=13 +COLUMNSTORE_VERSION_PATCH=14 COLUMNSTORE_VERSION_RELEASE=1 From 6664dc832c99ea3c181288d712f5f9cc0896d547 Mon Sep 17 00:00:00 2001 From: David Hill Date: Thu, 16 Aug 2018 08:27:47 -0500 Subject: [PATCH 04/55] update version to 1.0.15 --- README | 8 ++++---- README.md | 8 ++++---- VERSION | 2 +- 3 files changed, 9 insertions(+), 9 deletions(-) diff --git a/README b/README index ca0d9cd19..3eac5c82e 100644 --- a/README +++ b/README @@ -1,9 +1,9 @@ -This is MariaDB ColumnStore 1.0.14 -MariaDB ColumnStore 1.0.14 is the development version of MariaDB ColumnStore. -It is built by porting InfiniDB 4.6.7 on MariaDB 10.1.26 and adding entirely +This is MariaDB ColumnStore 1.0.15 +MariaDB ColumnStore 1.0.15 is the development version of MariaDB ColumnStore. +It is built by porting InfiniDB 4.6.7 on MariaDB 10.1.28 and adding entirely new features not found anywhere else. -MariaDB ColumnStore 1.0.14 is an GA release. This is the first MariaDB +MariaDB ColumnStore 1.0.15 is an GA release. This is the first MariaDB ColumnStore release, not all features planned for the MariaDB ColumnStore 1.0 series are included in this release. diff --git a/README.md b/README.md index 5cd3824eb..aa84a12ea 100644 --- a/README.md +++ b/README.md @@ -1,9 +1,9 @@ -#MariaDB ColumnStore Storage/Execution engine 1.0.14 -MariaDB ColumnStore 1.0.14 is the development version of MariaDB ColumnStore. -It is built by porting InfiniDB 4.6.7 on MariaDB 10.1.26 and adding entirely +#MariaDB ColumnStore Storage/Execution engine 1.0.15 +MariaDB ColumnStore 1.0.15 is the development version of MariaDB ColumnStore. +It is built by porting InfiniDB 4.6.7 on MariaDB 10.1.28 and adding entirely new features not found anywhere else. -#MariaDB ColumnStore 1.0.14 is an GA release. +#MariaDB ColumnStore 1.0.15 is an GA release. #Building This repository is not meant to be built independently outside of the server. This repository is integrated into http://mariadb-corporation/mariadb-columnstore-server (ie, the *server*) as a git submodule. As such, you can find complete build instructions on *the server* page. diff --git a/VERSION b/VERSION index 1e84dad87..50ab911ca 100644 --- a/VERSION +++ b/VERSION @@ -1,4 +1,4 @@ COLUMNSTORE_VERSION_MAJOR=1 COLUMNSTORE_VERSION_MINOR=0 -COLUMNSTORE_VERSION_PATCH=14 +COLUMNSTORE_VERSION_PATCH=15 COLUMNSTORE_VERSION_RELEASE=1 From bccd2acd4001579853a0f68b90c6262b65354012 Mon Sep 17 00:00:00 2001 From: Andrew Hutchings Date: Thu, 16 Aug 2018 15:57:42 +0100 Subject: [PATCH 05/55] Update README.md Fix server version --- README.md | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/README.md b/README.md index aa84a12ea..6a43e516e 100644 --- a/README.md +++ b/README.md @@ -1,6 +1,6 @@ #MariaDB ColumnStore Storage/Execution engine 1.0.15 MariaDB ColumnStore 1.0.15 is the development version of MariaDB ColumnStore. -It is built by porting InfiniDB 4.6.7 on MariaDB 10.1.28 and adding entirely +It is built by porting InfiniDB 4.6.7 on MariaDB 10.1.35 and adding entirely new features not found anywhere else. #MariaDB ColumnStore 1.0.15 is an GA release. From 9cb83957701516eb735bfee89a4f0ecb5e9a2f4a Mon Sep 17 00:00:00 2001 From: Andrew Hutchings Date: Thu, 16 Aug 2018 15:58:01 +0100 Subject: [PATCH 06/55] Update README Fix server version --- README | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/README b/README index 3eac5c82e..6e3bf8d59 100644 --- a/README +++ b/README @@ -1,6 +1,6 @@ This is MariaDB ColumnStore 1.0.15 MariaDB ColumnStore 1.0.15 is the development version of MariaDB ColumnStore. -It is built by porting InfiniDB 4.6.7 on MariaDB 10.1.28 and adding entirely +It is built by porting InfiniDB 4.6.7 on MariaDB 10.1.35 and adding entirely new features not found anywhere else. MariaDB ColumnStore 1.0.15 is an GA release. This is the first MariaDB From 36a2a7e2774280b816ee3ae7d403e22e6fc64cbe Mon Sep 17 00:00:00 2001 From: david hill Date: Mon, 20 Aug 2018 10:50:42 -0500 Subject: [PATCH 07/55] bump version to 1.0.16 --- README | 6 +++--- README.md | 6 +++--- VERSION | 2 +- 3 files changed, 7 insertions(+), 7 deletions(-) diff --git a/README b/README index 6e3bf8d59..ccc95c61b 100644 --- a/README +++ b/README @@ -1,9 +1,9 @@ -This is MariaDB ColumnStore 1.0.15 -MariaDB ColumnStore 1.0.15 is the development version of MariaDB ColumnStore. +This is MariaDB ColumnStore 1.0.16 +MariaDB ColumnStore 1.0.16 is the development version of MariaDB ColumnStore. It is built by porting InfiniDB 4.6.7 on MariaDB 10.1.35 and adding entirely new features not found anywhere else. -MariaDB ColumnStore 1.0.15 is an GA release. This is the first MariaDB +MariaDB ColumnStore 1.0.16 is an GA release. This is the first MariaDB ColumnStore release, not all features planned for the MariaDB ColumnStore 1.0 series are included in this release. diff --git a/README.md b/README.md index 6a43e516e..3ad645174 100644 --- a/README.md +++ b/README.md @@ -1,9 +1,9 @@ -#MariaDB ColumnStore Storage/Execution engine 1.0.15 -MariaDB ColumnStore 1.0.15 is the development version of MariaDB ColumnStore. +#MariaDB ColumnStore Storage/Execution engine 1.0.16 +MariaDB ColumnStore 1.0.16 is the development version of MariaDB ColumnStore. It is built by porting InfiniDB 4.6.7 on MariaDB 10.1.35 and adding entirely new features not found anywhere else. -#MariaDB ColumnStore 1.0.15 is an GA release. +#MariaDB ColumnStore 1.0.16 is an GA release. #Building This repository is not meant to be built independently outside of the server. This repository is integrated into http://mariadb-corporation/mariadb-columnstore-server (ie, the *server*) as a git submodule. As such, you can find complete build instructions on *the server* page. diff --git a/VERSION b/VERSION index 50ab911ca..b13a94d3f 100644 --- a/VERSION +++ b/VERSION @@ -1,4 +1,4 @@ COLUMNSTORE_VERSION_MAJOR=1 COLUMNSTORE_VERSION_MINOR=0 -COLUMNSTORE_VERSION_PATCH=15 +COLUMNSTORE_VERSION_PATCH=16 COLUMNSTORE_VERSION_RELEASE=1 From 60fb620dcb4b213a3a5c81292c0acae925a714cd Mon Sep 17 00:00:00 2001 From: Patrick LeBlanc Date: Mon, 5 Nov 2018 13:33:56 -0600 Subject: [PATCH 08/55] MCOL-1654. Querystats table creation broken. Escaped the ` chars to make the shell happy. Conflicts: dbcon/mysql/install_calpont_mysql.sh --- dbcon/mysql/install_calpont_mysql.sh | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/dbcon/mysql/install_calpont_mysql.sh b/dbcon/mysql/install_calpont_mysql.sh index 9b55f9448..ecea9d503 100755 --- a/dbcon/mysql/install_calpont_mysql.sh +++ b/dbcon/mysql/install_calpont_mysql.sh @@ -99,7 +99,7 @@ CREATE TABLE IF NOT EXISTS infinidb_querystats.querystats query varchar(8000), startTime timestamp NOT NULL, endTime timestamp NOT NULL, - rows bigint, + \`rows\` bigint, errno int, phyIO bigint, cacheIO bigint, From 35662d3258b9329a944477fcddf1e02f212ed84e Mon Sep 17 00:00:00 2001 From: Patrick LeBlanc Date: Wed, 14 Nov 2018 17:15:14 -0600 Subject: [PATCH 09/55] MCOL-1844. Preserve user-added args in 'myCnf-include-args.text' across upgrades. --- oam/install_scripts/pre-uninstall | 1 + oamapps/postConfigure/mycnfUpgrade.cpp | 59 +++++++++++++++++++++++++- 2 files changed, 59 insertions(+), 1 deletion(-) diff --git a/oam/install_scripts/pre-uninstall b/oam/install_scripts/pre-uninstall index 8709d369c..8621cd9ac 100755 --- a/oam/install_scripts/pre-uninstall +++ b/oam/install_scripts/pre-uninstall @@ -153,6 +153,7 @@ if [ $quiet != 1 ]; then #make copy of Columnstore.xml /bin/cp -f $installdir/etc/Columnstore.xml $installdir/etc/Columnstore.xml.rpmsave > /dev/null 2>&1 /bin/cp -f $installdir/mysql/my.cnf $installdir/mysql/my.cnf.rpmsave > /dev/null 2>&1 + cp $installdir/bin/myCnf-include-args.text $installdir/bin/myCnf-include-args.text.rpmsave >& /dev/null rm -f $installdir/etc/AlarmConfig.xml.installSave fi diff --git a/oamapps/postConfigure/mycnfUpgrade.cpp b/oamapps/postConfigure/mycnfUpgrade.cpp index a8808e740..18b606a2b 100644 --- a/oamapps/postConfigure/mycnfUpgrade.cpp +++ b/oamapps/postConfigure/mycnfUpgrade.cpp @@ -54,6 +54,59 @@ using namespace std; using namespace oam; + +/* MCOL-1844. On an upgrade, the user may have customized options in their old + * myCnf-include-args.text file. Merge it with the packaged version, and then process as we + * have before. + */ +string rtrim(const string &in) { + string::const_reverse_iterator rbegin = in.rbegin(); + while (rbegin != in.rend() && isspace(*rbegin)) + ++rbegin; + return string(in.begin(), rbegin.base()); +} + +void mergeMycnfIncludeArgs() +{ + string userArgsFilename = startup::StartUp::installDir() + "/bin/myCnf-include-args.text.rpmsave"; + string packagedArgsFilename = startup::StartUp::installDir() + "/bin/myCnf-include-args.text"; + ifstream userArgs(userArgsFilename.c_str()); + fstream packagedArgs(packagedArgsFilename.c_str(), ios::in); + + if (!userArgs || !packagedArgs) + return; + + // de-dup the args and comments in both files + set argMerger; + set comments; + string line; + while (getline(packagedArgs, line)) { + line = rtrim(line); + if (line[0] == '#') + comments.insert(line); + else if (line.size() > 0) + argMerger.insert(line); + } + while (getline(userArgs, line)) { + line = rtrim(line); + if (line[0] == '#') + comments.insert(line); + else if (line.size() > 0) + argMerger.insert(line); + } + userArgs.close(); + packagedArgs.close(); + + // write the merged version, comments first. They'll get ordered + // alphabetically but, meh. + packagedArgs.open(packagedArgsFilename.c_str(), ios::out | ios::trunc); + for (set::iterator it = comments.begin(); it != comments.end(); it++) + packagedArgs << *it << endl; + for (set::iterator it = argMerger.begin(); it != argMerger.end(); it++) + packagedArgs << *it << endl; + packagedArgs.close(); +} + int main(int argc, char *argv[]) { Oam oam; @@ -84,6 +137,10 @@ int main(int argc, char *argv[]) exit (1); } + // MCOL-1844. The user may have added options to their myCnf-include-args file. Merge + // myCnf-include-args.text with myCnf-include-args.text.rpmsave, save in myCnf-include-args.text + mergeMycnfIncludeArgs(); + //include arguments file string includeFile = startup::StartUp::installDir() + "/bin/myCnf-include-args.text"; ifstream includefile (includeFile.c_str()); @@ -144,7 +201,7 @@ int main(int argc, char *argv[]) ofstream newFile (mycnfFile.c_str()); //create new file - int fd = open(mycnfFile.c_str(), O_RDWR|O_CREAT, 0666); + int fd = open(mycnfFile.c_str(), O_RDWR|O_CREAT, 0644); copy(lines.begin(), lines.end(), ostream_iterator(newFile, "\n")); newFile.close(); From 165a5c76bf75548109602f3415541d5a4157e54f Mon Sep 17 00:00:00 2001 From: Patrick LeBlanc Date: Thu, 15 Nov 2018 09:14:20 -0600 Subject: [PATCH 10/55] MCOL-1844. Removed an unassociated change I made. --- oamapps/postConfigure/mycnfUpgrade.cpp | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/oamapps/postConfigure/mycnfUpgrade.cpp b/oamapps/postConfigure/mycnfUpgrade.cpp index 18b606a2b..75b9a3d3a 100644 --- a/oamapps/postConfigure/mycnfUpgrade.cpp +++ b/oamapps/postConfigure/mycnfUpgrade.cpp @@ -201,7 +201,7 @@ int main(int argc, char *argv[]) ofstream newFile (mycnfFile.c_str()); //create new file - int fd = open(mycnfFile.c_str(), O_RDWR|O_CREAT, 0644); + int fd = open(mycnfFile.c_str(), O_RDWR|O_CREAT, 0666); copy(lines.begin(), lines.end(), ostream_iterator(newFile, "\n")); newFile.close(); From 0f764ed74157f85167b6148d2722570d5f77ccc2 Mon Sep 17 00:00:00 2001 From: Patrick LeBlanc Date: Tue, 20 Nov 2018 10:49:05 -0600 Subject: [PATCH 11/55] MCOL-1607. WIP. Added the new option, no other change yet. --- oamapps/postConfigure/postConfigure.cpp | 171 ++++++++++++------------ 1 file changed, 85 insertions(+), 86 deletions(-) diff --git a/oamapps/postConfigure/postConfigure.cpp b/oamapps/postConfigure/postConfigure.cpp index bb828b5be..bfc630cd6 100644 --- a/oamapps/postConfigure/postConfigure.cpp +++ b/oamapps/postConfigure/postConfigure.cpp @@ -14,7 +14,7 @@ You should have received a copy of the GNU General Public License along with this program; if not, write to the Free Software Foundation, Inc., 51 Franklin Street, Fifth Floor, Boston, - MA 02110-1301, USA. */ + MA 02110-1301, USA. */ /****************************************************************************************** * $Id: postConfigure.cpp 64 2006-10-12 22:21:51Z dhill $ @@ -192,6 +192,7 @@ bool nonDistributeFlag = false; bool single_server_quick_install = false; bool multi_server_quick_install = false; bool amazon_quick_install = false; +bool doNotResolveHostNames = false; string DataFileEnvFile; @@ -275,12 +276,12 @@ int main(int argc, char* argv[]) usergroup = getgid(); string SUDO = ""; - if (user != 0) + if (user != 0) { rootUser = false; SUDO = "sudo "; } - + char* p = getenv("USER"); if (p && *p) @@ -298,7 +299,7 @@ int main(int argc, char* argv[]) for ( int i = 1; i < argc; i++ ) { - if( string("-h") == argv[i] ) + if( string("-h") == argv[i] ) { cout << endl; cout << "This is the MariaDB ColumnStore System Configuration and Installation tool." << endl; @@ -322,13 +323,18 @@ int main(int argc, char* argv[]) cout << " -qs Quick Install - Single Server" << endl; cout << " -qm Quick Install - Multi Server" << endl; cout << " -port MariaDB ColumnStore Port Address" << endl; - cout << " -i Non-root Install directory, Only use for non-root installs" << endl; - cout << " -n Non-distributed install, meaning postConfigure will not install packages on remote nodes" << endl; - cout << " -d Distributed install, meaning postConfigure will install packages on remote nodes" << endl; + cout << " -i Non-root Install directory, Only use for non-root installs" << endl; + cout << " -n Non-distributed install, meaning postConfigure will not install packages on remote nodes" << endl; + cout << " -d Distributed install, meaning postConfigure will install packages on remote nodes" << endl; cout << " -sn System Name" << endl; cout << " -pm-ip-addrs Performance Module IP Addresses xxx.xxx.xxx.xxx,xxx.xxx.xxx.xxx" << endl; cout << " -um-ip-addrs User Module IP Addresses xxx.xxx.xxx.xxx,xxx.xxx.xxx.xxx" << endl; + cout << " -x Do not resolve IP Addresses from host names" << endl; exit (0); + } + else if (string("-x") == argv[i]) + { + doNotResolveHostNames = true; } else if( string("-qs") == argv[i] ) { @@ -349,33 +355,33 @@ int main(int argc, char* argv[]) nodeps = "--nodeps"; else if ( string("-o") == argv[i] ) startOfflinePrompt = true; - else if( string("-c") == argv[i] ) + else if( string("-c") == argv[i] ) { i++; - if (i >= argc ) + if (i >= argc ) { cout << " ERROR: Config File not provided" << endl; exit (1); } oldFileName = argv[i]; - if ( oldFileName.find("Columnstore.xml") == string::npos ) + if ( oldFileName.find("Columnstore.xml") == string::npos ) { cout << " ERROR: Config File is not a Columnstore.xml file name" << endl; exit (1); } } - else if( string("-p") == argv[i] ) + else if( string("-p") == argv[i] ) { i++; - if (i >= argc ) + if (i >= argc ) { cout << " ERROR: Password not provided" << endl; exit (1); } password = argv[i]; - if ( password.find("-") != string::npos ) + if ( password.find("-") != string::npos ) { cout << " ERROR: Valid Password not provided" << endl; exit (1); @@ -394,10 +400,10 @@ int main(int argc, char* argv[]) nonDistribute = false; nonDistributeFlag = true; } - else if( string("-port") == argv[i] ) + else if( string("-port") == argv[i] ) { i++; - if (i >= argc ) + if (i >= argc ) { cout << " ERROR: MariaDB ColumnStore Port ID not supplied" << endl; exit (1); @@ -411,10 +417,10 @@ int main(int argc, char* argv[]) exit (1); } } - else if( string("-i") == argv[i] ) + else if( string("-i") == argv[i] ) { i++; - if (i >= argc ) + if (i >= argc ) { cout << " ERROR: Path not provided" << endl; exit (1); @@ -422,50 +428,50 @@ int main(int argc, char* argv[]) installDir = argv[i]; } - else if( string("-sn") == argv[i] ) + else if( string("-sn") == argv[i] ) { i++; - if (i >= argc ) + if (i >= argc ) { cout << " ERROR: System-name not provided" << endl; exit (1); } systemName = argv[i]; } - else if( string("-pm-ip-addrs") == argv[i] ) + else if( string("-pm-ip-addrs") == argv[i] ) { i++; - if (i >= argc ) + if (i >= argc ) { cout << " ERROR: PM-IP-ADRESSES not provided" << endl; exit (1); } pmIpAddrs = argv[i]; } - else if( string("-um-ip-addrs") == argv[i] ) + else if( string("-um-ip-addrs") == argv[i] ) { i++; - if (i >= argc ) + if (i >= argc ) { cout << " ERROR: UM-IP-ADRESSES not provided" << endl; exit (1); } umIpAddrs = argv[i]; } - else if( string("-pm-count") == argv[i] ) + else if( string("-pm-count") == argv[i] ) { i++; - if (i >= argc ) + if (i >= argc ) { cout << " ERROR: PM-COUNT not provided" << endl; exit (1); } pmNumber = atoi(argv[i]); } - else if( string("-um-count") == argv[i] ) + else if( string("-um-count") == argv[i] ) { i++; - if (i >= argc ) + if (i >= argc ) { cout << " ERROR: UM-COUNT not provided" << endl; exit (1); @@ -479,7 +485,7 @@ int main(int argc, char* argv[]) exit (1); } } - + //check if quick install multi-server has been given ip address if (multi_server_quick_install) { @@ -553,7 +559,7 @@ int main(int argc, char* argv[]) // redirectStandardOutputToFile(postConfigureLog, false ); } } - + //check if MariaDB ColumnStore is up and running if (oam.checkSystemRunning()) { @@ -567,7 +573,7 @@ int main(int argc, char* argv[]) cout << "ERROR: Configuration File not setup" << endl; exit(1); } - + //determine package type string EEPackageType; @@ -605,7 +611,7 @@ int main(int argc, char* argv[]) cout << "ERROR: Failed trying to update MariaDB ColumnStore System Configuration file" << endl; exit(1); } - + //check for local ip address as pm1 ModuleConfig moduleconfig; @@ -789,7 +795,7 @@ int main(int argc, char* argv[]) } cout << endl; - + if (single_server_quick_install) { cout << "===== Quick Install Single-Server Configuration =====" << endl << endl; @@ -797,13 +803,13 @@ int main(int argc, char* argv[]) cout << "Single-Server install is used when there will only be 1 server configured" << endl; cout << "on the system. It can also be used for production systems, if the plan is" << endl; cout << "to stay single-server." << endl; - + singleServerInstall = "1"; } else if (multi_server_quick_install) { cout << "===== Quick Install Multi-Server Configuration =====" << endl << endl; - + cout << "Multi-Server install defaulting to using local storage" << endl; singleServerInstall = "2"; @@ -811,7 +817,7 @@ int main(int argc, char* argv[]) else if (amazon_quick_install) { cout << "===== Quick Install Amazon Configuration =====" << endl << endl; - + cout << "Amazon AMI EC2 install defaulting to using local storage" << endl; singleServerInstall = "2"; @@ -842,7 +848,7 @@ int main(int argc, char* argv[]) else singleServerInstall = "2"; - while(true) + while(true) { string temp = singleServerInstall; prompt = "Select the type of System Server install [1=single, 2=multi] (" + singleServerInstall + ") > "; @@ -889,7 +895,7 @@ int main(int argc, char* argv[]) //setup to Columnstore.xml file for single server singleServerConfigSetup(sysConfig); } - + //module ProcessConfig.xml to setup all apps on the pm if ( !updateProcessConfig() ) cout << "Update ProcessConfig.xml error" << endl; @@ -967,7 +973,7 @@ int main(int argc, char* argv[]) } // perform multi-node install - + try { sysConfig->setConfig(InstallSection, "SingleServerInstall", "n"); @@ -987,7 +993,7 @@ int main(int argc, char* argv[]) // // Multi-server install // - + ModuleIP InputModuleIP; ModuleIpList InputModuleIPList; @@ -1001,7 +1007,7 @@ int main(int argc, char* argv[]) } catch(...) {} - + if (umIpAddrs == "" ) { // set Server Type Installation to combined @@ -1026,10 +1032,10 @@ int main(int argc, char* argv[]) InputModuleIP.moduleName = module; InputModuleIPList.push_back(InputModuleIP); } - + umNumber = id-1; } - + if (pmIpAddrs != "" ) { int id = 1; @@ -1045,16 +1051,16 @@ int main(int argc, char* argv[]) InputModuleIP.moduleName = module; InputModuleIPList.push_back(InputModuleIP); } - + pmNumber = id-1; } - if ( !writeConfig(sysConfig) ) + if ( !writeConfig(sysConfig) ) { cout << "ERROR: Failed trying to update MariaDB ColumnStore System Configuration file" << endl; exit(1); } - + MaxNicID = 1; } else @@ -1067,13 +1073,13 @@ int main(int argc, char* argv[]) } catch(...) {} - + try { sysConfig->setConfig(InstallSection, "Cloud", "amazon-vpc"); } catch(...) {} - + if (umNumber == 0 ) { // set Server Type Installation to combined @@ -1084,12 +1090,12 @@ int main(int argc, char* argv[]) {} } - if ( !writeConfig(sysConfig) ) + if ( !writeConfig(sysConfig) ) { cout << "ERROR: Failed trying to update MariaDB ColumnStore System Configuration file" << endl; exit(1); } - + MaxNicID = 1; } } @@ -1350,7 +1356,7 @@ int main(int argc, char* argv[]) //amazon install setup check bool amazonInstall = false; string cloud = oam::UnassignedName; - + if (!multi_server_quick_install) { string amazonLog = tmpDir + "/amazon.log"; @@ -1367,7 +1373,7 @@ int main(int argc, char* argv[]) // not running on amazon with ec2-api-tools if (amazon_quick_install) { - cout << "ERROR: Amazon Quick Installer was specified, bu the Amazon CLI API packages isnt installed, exiting" << endl; + cout << "ERROR: Amazon Quick Installer was specified, but the Amazon CLI API packages is not installed, exiting" << endl; exit(1); } @@ -1380,7 +1386,7 @@ int main(int argc, char* argv[]) // not running on amazon with ec2-api-tools if (amazon_quick_install) { - cout << "ERROR: Amazon Quick Installer was specified, bu the AMazon CLI API packages isnt installed, exiting" << endl; + cout << "ERROR: Amazon Quick Installer was specified, but the Amazon CLI API packages is not installed, exiting" << endl; exit(1); } @@ -1390,7 +1396,7 @@ int main(int argc, char* argv[]) amazonInstall = true; } } - + try { cloud = sysConfig->getConfig(InstallSection, "Cloud"); @@ -1717,13 +1723,6 @@ int main(int argc, char* argv[]) // are we using settings from previous config file? if ( reuseConfig == "n" ) { - - - - - - - string numBlocksPct; try @@ -1877,14 +1876,14 @@ int main(int argc, char* argv[]) catch (...) {} } - + if ( moduleType == "um") if ( umNumber != 0 ) moduleCount = umNumber; - + if ( moduleType == "pm") if ( pmNumber != 0 ) - moduleCount = pmNumber; + moduleCount = pmNumber; //verify and setup of modules count switch ( IserverTypeInstall ) @@ -2044,7 +2043,7 @@ int main(int argc, char* argv[]) moduleDisableState = oam::ENABLEDSTATE; //setup HostName/IPAddress for each NIC - + string moduleHostName = oam::UnassignedName; string moduleIPAddr = oam::UnassignedIpAddr; @@ -2063,7 +2062,7 @@ int main(int argc, char* argv[]) } } } - + unsigned int nicID=1; for( ; nicID < MaxNicID +1 ; nicID++ ) { @@ -2076,9 +2075,9 @@ int main(int argc, char* argv[]) for( ; listPT != sysModuleTypeConfig.moduletypeconfig[i].ModuleNetworkList.end() ; listPT++) { - if (newModuleName == (*listPT).DeviceName) + if (newModuleName == (*listPT).DeviceName) { - if ( nicID == 1 ) + if ( nicID == 1 ) { moduleDisableState = (*listPT).DisableState; @@ -2091,7 +2090,7 @@ int main(int argc, char* argv[]) for( ; pt1 != (*listPT).hostConfigList.end() ; pt1++) { - if ((*pt1).NicID == nicID) + if ((*pt1).NicID == nicID) { moduleHostName = (*pt1).HostName; moduleIPAddr = (*pt1).IPAddr; @@ -2103,8 +2102,8 @@ int main(int argc, char* argv[]) } } } - - if ( nicID == 1 ) + + if ( nicID == 1 ) { if ( moduleDisableState != oam::ENABLEDSTATE ) { @@ -2948,7 +2947,7 @@ int main(int argc, char* argv[]) } } } - + vector dbroots; string tempdbrootList; @@ -3497,7 +3496,7 @@ int main(int argc, char* argv[]) } if ( IserverTypeInstall != oam::INSTALL_COMBINE_DM_UM_PM || - pmNumber > 1 ) + pmNumber > 1 ) { if ( password.empty() ) { @@ -3531,10 +3530,10 @@ int main(int argc, char* argv[]) if ( p1 == p2 ) { password = p2; break; - } + } else cout << "Password mismatch, please re-enter" << endl; - } + } //add single quote for special characters if ( password != "ssh" ) @@ -3543,7 +3542,7 @@ int main(int argc, char* argv[]) } } - } + } } int thread_id = 0; @@ -3824,7 +3823,7 @@ int main(int argc, char* argv[]) cout << " DONE" << endl; } } - + //configure data redundancy if (DataRedundancy) { @@ -4039,7 +4038,7 @@ int main(int argc, char* argv[]) } } } - + string dbbuilderLog = tmpDir + "/dbbuilder.log"; if (hdfs) @@ -4060,7 +4059,7 @@ int main(int argc, char* argv[]) //send message to procmon's to run upgrade script int status = sendUpgradeRequest(IserverTypeInstall, pmwithum); - + if ( status != 0 ) { cout << endl << "MariaDB Columnstore Install Failed" << endl << endl; exit(1); @@ -4423,7 +4422,7 @@ bool updateProcessConfig() vector oldModule; string newModule = ">pm"; oldModule.push_back(">um"); - + string fileName = installDir + "/etc/ProcessConfig.xml"; //Save a copy of the original version @@ -4564,9 +4563,9 @@ bool uncommentCalpontXml( string entry) */ bool makeRClocal(string moduleType, string moduleName, int IserverTypeInstall) { - + return true; - + vector lines; string mount1; @@ -5147,7 +5146,7 @@ bool storageSetup(bool amazonInstall) //check if hadoop is installed string hadoopLog = tmpDir + "/hadoop.log"; - + string cmd = "which hadoop > " + hadoopLog + " 2>&1"; system(cmd.c_str()); @@ -5732,7 +5731,7 @@ void setSystemName() Oam oam; //setup System Name - + if ( systemName.empty() ) { try { @@ -5743,7 +5742,7 @@ void setSystemName() systemName = oam::UnassignedName; } } - + if ( systemName.empty() ) systemName = "columnstore-1"; @@ -5759,7 +5758,7 @@ void setSystemName() callFree(pcommand); } } - + try { sysConfig->setConfig(SystemSection, "SystemName", systemName); @@ -6926,7 +6925,7 @@ bool glusterSetup(string password) void singleServerConfigSetup(Config* sysConfig) { - + try { sysConfig->setConfig("ExeMgr1", "IPAddr", "127.0.0.1"); From cadfb0be928bf5a9308902a4a2b3dcb074acb686 Mon Sep 17 00:00:00 2001 From: Patrick LeBlanc Date: Tue, 20 Nov 2018 14:44:30 -0600 Subject: [PATCH 12/55] Very speculative change to postConfigure. 99% sure it won't work. --- oamapps/postConfigure/postConfigure.cpp | 12 ++++++------ 1 file changed, 6 insertions(+), 6 deletions(-) diff --git a/oamapps/postConfigure/postConfigure.cpp b/oamapps/postConfigure/postConfigure.cpp index bfc630cd6..0c8d2e0ac 100644 --- a/oamapps/postConfigure/postConfigure.cpp +++ b/oamapps/postConfigure/postConfigure.cpp @@ -129,7 +129,7 @@ void singleServerConfigSetup(Config* sysConfig); void remoteInstallThread(void*); -bool glusterSetup(string password); +bool glusterSetup(string password, bool doNotResolveHostNames); std::string launchInstance(ModuleIP moduleip); @@ -2446,7 +2446,7 @@ int main(int argc, char* argv[]) newModuleIPAddr = oam::UnassignedIpAddr; else { - if (!amazonInstall) + if (!amazonInstall && !doNotResolveHostNames) { if ( moduleIPAddr == oam::UnassignedIpAddr ) { @@ -3896,7 +3896,7 @@ int main(int argc, char* argv[]) { cout << endl << "===== Configuring MariaDB ColumnStore Data Redundancy Functionality =====" << endl << endl; - if (!glusterSetup(password)) + if (!glusterSetup(password, doNotResolveHostNames)) { cout << "ERROR: Problem setting up ColumnStore Data Redundancy" << endl; exit(1); @@ -6274,7 +6274,7 @@ std::string launchInstance(ModuleIP moduleip) return instanceName; } -bool glusterSetup(string password) +bool glusterSetup(string password, bool doNotResolveHostNames) { Oam oam; int dataRedundancyCopies = 0; @@ -6428,7 +6428,7 @@ bool glusterSetup(string password) callFree(pcommand); } - if ( moduleIPAddr == oam::UnassignedIpAddr ) + if ( moduleIPAddr == oam::UnassignedIpAddr) { //get IP Address string IPAddress = oam.getIPAddress( moduleHostName); @@ -6443,7 +6443,7 @@ bool glusterSetup(string password) moduleIPAddr = "unassigned"; //prompt for IP address - while (true) + while (!doNotResolveHostNames) { prompt = "Enter PM #" + oam.itoa(DataRedundancyConfigs[pm].pmID) + " IP Address of " + moduleHostName + " (" + moduleIPAddr + ") > "; pcommand = callReadline(prompt.c_str()); From d75659aa90fc5d310e1fe6c5c9e67e725a58f84e Mon Sep 17 00:00:00 2001 From: Patrick LeBlanc Date: Tue, 20 Nov 2018 15:53:52 -0600 Subject: [PATCH 13/55] Some changes to postConfigure to get hostname put in IPAddr field. --- oamapps/postConfigure/postConfigure.cpp | 22 +++++++++++++++------- 1 file changed, 15 insertions(+), 7 deletions(-) diff --git a/oamapps/postConfigure/postConfigure.cpp b/oamapps/postConfigure/postConfigure.cpp index 0c8d2e0ac..e5680485b 100644 --- a/oamapps/postConfigure/postConfigure.cpp +++ b/oamapps/postConfigure/postConfigure.cpp @@ -2446,12 +2446,16 @@ int main(int argc, char* argv[]) newModuleIPAddr = oam::UnassignedIpAddr; else { - if (!amazonInstall && !doNotResolveHostNames) + if (!amazonInstall) { if ( moduleIPAddr == oam::UnassignedIpAddr ) { //get IP Address - string IPAddress = oam.getIPAddress( newModuleHostName); + string IPAddress; + if (doNotResolveHostNames) + IPAddress = newModuleHostName; + else + IPAddress = oam.getIPAddress( newModuleHostName); if ( !IPAddress.empty() ) newModuleIPAddr = IPAddress; @@ -2467,7 +2471,7 @@ int main(int argc, char* argv[]) //prompt for IP address while (true) { - prompt = "Enter Nic Interface #" + oam.itoa(nicID) + " IP Address of " + newModuleHostName + " (" + newModuleIPAddr + ") > "; + prompt = "Enter Nic Interface #" + oam.itoa(nicID) + " IP Address or hostname of " + newModuleHostName + " (" + newModuleIPAddr + ") > "; pcommand = callReadline(prompt.c_str()); if (pcommand) @@ -2488,7 +2492,7 @@ int main(int argc, char* argv[]) continue; } - if (oam.isValidIP(newModuleIPAddr)) + if (oam.isValidIP(newModuleIPAddr) || doNotResolveHostNames) { //check and see if hostname already used bool matchFound = false; @@ -6431,7 +6435,11 @@ bool glusterSetup(string password, bool doNotResolveHostNames) if ( moduleIPAddr == oam::UnassignedIpAddr) { //get IP Address - string IPAddress = oam.getIPAddress( moduleHostName); + string IPAddress; + if (doNotResolveHostNames) + IPAddress = moduleHostName; + else + IPAddress = oam.getIPAddress( moduleHostName); if ( !IPAddress.empty() ) moduleIPAddr = IPAddress; @@ -6443,7 +6451,7 @@ bool glusterSetup(string password, bool doNotResolveHostNames) moduleIPAddr = "unassigned"; //prompt for IP address - while (!doNotResolveHostNames) + while (true) { prompt = "Enter PM #" + oam.itoa(DataRedundancyConfigs[pm].pmID) + " IP Address of " + moduleHostName + " (" + moduleIPAddr + ") > "; pcommand = callReadline(prompt.c_str()); @@ -6466,7 +6474,7 @@ bool glusterSetup(string password, bool doNotResolveHostNames) continue; } - if (oam.isValidIP(moduleIPAddr)) + if (oam.isValidIP(moduleIPAddr) || doNotResolveHostNames) { // run ping test to validate From 0cef3bf31ca8995b675ee860eb08da9e5ad17f3e Mon Sep 17 00:00:00 2001 From: David Hall Date: Thu, 29 Nov 2018 13:31:12 -0600 Subject: [PATCH 14/55] MCOL-1983 Have regr_intercept, regr_slope and regr_r2 return NULL for 1 or 0 rows in set --- utils/regr/regr_intercept.cpp | 2 +- utils/regr/regr_r2.cpp | 2 +- utils/regr/regr_slope.cpp | 2 +- utils/windowfunction/wf_udaf.cpp | 2 +- 4 files changed, 4 insertions(+), 4 deletions(-) diff --git a/utils/regr/regr_intercept.cpp b/utils/regr/regr_intercept.cpp index 6d4c35a47..011488d9d 100644 --- a/utils/regr/regr_intercept.cpp +++ b/utils/regr/regr_intercept.cpp @@ -139,7 +139,7 @@ mcsv1_UDAF::ReturnCode regr_intercept::evaluate(mcsv1Context* context, static_an { struct regr_intercept_data* data = (struct regr_intercept_data*)context->getUserData()->data; double N = data->cnt; - if (N > 0) + if (N > 1) { double sumx = data->sumx; double sumy = data->sumy; diff --git a/utils/regr/regr_r2.cpp b/utils/regr/regr_r2.cpp index 34e8888e8..96b0ada18 100644 --- a/utils/regr/regr_r2.cpp +++ b/utils/regr/regr_r2.cpp @@ -144,7 +144,7 @@ mcsv1_UDAF::ReturnCode regr_r2::evaluate(mcsv1Context* context, static_any::any& { struct regr_r2_data* data = (struct regr_r2_data*)context->getUserData()->data; double N = data->cnt; - if (N > 0) + if (N > 1) { double sumx = data->sumx; double sumy = data->sumy; diff --git a/utils/regr/regr_slope.cpp b/utils/regr/regr_slope.cpp index da178673a..fdb94795a 100644 --- a/utils/regr/regr_slope.cpp +++ b/utils/regr/regr_slope.cpp @@ -139,7 +139,7 @@ mcsv1_UDAF::ReturnCode regr_slope::evaluate(mcsv1Context* context, static_any::a { struct regr_slope_data* data = (struct regr_slope_data*)context->getUserData()->data; double N = data->cnt; - if (N > 0) + if (N > 1) { // COVAR_POP(y, x) / VAR_POP(x) double sumx = data->sumx; diff --git a/utils/windowfunction/wf_udaf.cpp b/utils/windowfunction/wf_udaf.cpp index 2b7e28abc..5986c2872 100644 --- a/utils/windowfunction/wf_udaf.cpp +++ b/utils/windowfunction/wf_udaf.cpp @@ -84,7 +84,6 @@ void WF_udaf::resetData() getContext().getFunction()->reset(&getContext()); fDistinctMap.clear(); WindowFunctionType::resetData(); - fValOut.reset(); } void WF_udaf::parseParms(const std::vector& parms) @@ -709,6 +708,7 @@ void WF_udaf::operator()(int64_t b, int64_t e, int64_t c) mcsv1sdk::mcsv1_UDAF::ReturnCode rc; uint64_t colOut = fFieldIndex[0]; bool isNull = false; + fValOut.reset(); if ((fFrameUnit == WF__FRAME_ROWS) || (fPrev == -1) || From 7ad0aa442060fa888322c220aad182b6f329b431 Mon Sep 17 00:00:00 2001 From: Patrick LeBlanc Date: Thu, 29 Nov 2018 15:20:05 -0600 Subject: [PATCH 15/55] MCOL-1607. Tentative change to include addModule. This might or might not let addModule store host names instead of IP addresses. --- oam/etc/ConsoleCmds.xml | 5 +- oam/oamcpp/liboamcpp.cpp | 125 ++++++++++++++++++++++++++++++++-- oam/oamcpp/liboamcpp.h | 9 ++- oamapps/mcsadmin/mcsadmin.cpp | 123 ++++++++++++++++++++------------- procmgr/processmanager.cpp | 17 +++-- procmgr/processmanager.h | 3 +- 6 files changed, 223 insertions(+), 59 deletions(-) diff --git a/oam/etc/ConsoleCmds.xml b/oam/etc/ConsoleCmds.xml index 56a6e678f..9c58d78f3 100644 --- a/oam/etc/ConsoleCmds.xml +++ b/oam/etc/ConsoleCmds.xml @@ -309,8 +309,9 @@ MariaDB Columnstore Packages and setup the module to make it ready to be restored Required: Module-type or Module-name being added Required: Number-of-Modules being added when Module-type is specified - Optional: Server-Hostnames/Amazon-Instance-Names seperated by commas - Optional: Server-root-password + Optional: Store given hostnames in the configuration instead of IP addresses (y/N) + Optional: Server-Hostnames/Amazon-Instance-Names seperated by commas + Optional: Server-root-password removeModule diff --git a/oam/oamcpp/liboamcpp.cpp b/oam/oamcpp/liboamcpp.cpp index 7ce05e2d9..0ec1e7b1c 100644 --- a/oam/oamcpp/liboamcpp.cpp +++ b/oam/oamcpp/liboamcpp.cpp @@ -1272,10 +1272,12 @@ void Oam::setSystemConfig(const std::string module, ModuleConfig moduleconfig) * ********************************************************************/ -void Oam::addModule(DeviceNetworkList devicenetworklist, const std::string password, const std::string mysqlpw) +void Oam::addModule(DeviceNetworkList devicenetworklist, const std::string password, const std::string mysqlpw, + bool storeHostnames) { // build and send msg - int returnStatus = sendMsgToProcMgr2(ADDMODULE, devicenetworklist, FORCEFUL, ACK_YES, password, mysqlpw); + int returnStatus = sendAddModuleToProcMgr(ADDMODULE, devicenetworklist, FORCEFUL, ACK_YES, storeHostnames, + password, mysqlpw); if (returnStatus != API_SUCCESS) exceptionControl("addModule", returnStatus); @@ -9929,7 +9931,6 @@ int Oam::sendMsgToProcMgr2(messageqcpp::ByteStream::byte requestType, DeviceNetw msg << (ByteStream::byte) gracefulflag; msg << (ByteStream::byte) ackflag; msg << (ByteStream::byte) requestManual; - msg << (uint16_t) devicenetworklist.size(); DeviceNetworkList::iterator pt = devicenetworklist.begin(); @@ -9947,7 +9948,6 @@ int Oam::sendMsgToProcMgr2(messageqcpp::ByteStream::byte requestType, DeviceNetw msg << " "; else msg << (*pt).DisableState; - msg << (uint16_t) (*pt).hostConfigList.size(); HostConfigList::iterator pt1 = (*pt).hostConfigList.begin(); @@ -10007,6 +10007,123 @@ int Oam::sendMsgToProcMgr2(messageqcpp::ByteStream::byte requestType, DeviceNetw return returnStatus; } +/* A slightly different version of sendMsgToProcMgr2. Add-module needs to send one add'l + parameter, and this was the best of a couple bad options. */ +int Oam::sendAddModuleToProcMgr(messageqcpp::ByteStream::byte requestType, DeviceNetworkList devicenetworklist, + GRACEFUL_FLAG gracefulflag, ACK_FLAG ackflag, bool storeHostnames, const std::string password, + const std::string mysqlpw) +{ + if (!checkSystemRunning()) + return API_CONN_REFUSED; + + int returnStatus = API_TIMEOUT; //default + ByteStream msg; + ByteStream receivedMSG; + ByteStream::byte msgType; + ByteStream::byte actionType; + ByteStream::byte status; + + // get current requesting process, an error will occur if process is a UI tool (not kept in Status Table) + // this will be used to determine if this is a manually or auto request down within Process-Monitor + bool requestManual; + myProcessStatus_t t; + + try + { + t = getMyProcessStatus(); + requestManual = false; // set to auto + } + catch (...) + { + requestManual = true; // set to manual + } + + // setup message + msg << (ByteStream::byte) REQUEST; + msg << requestType; + msg << (std::string) " "; + msg << (ByteStream::byte) gracefulflag; + msg << (ByteStream::byte) ackflag; + msg << (ByteStream::byte) requestManual; + msg << (uint8_t) storeHostnames; + msg << (uint16_t) devicenetworklist.size(); + + DeviceNetworkList::iterator pt = devicenetworklist.begin(); + + for ( ; pt != devicenetworklist.end() ; pt++) + { + msg << (*pt).DeviceName; + + if ( (*pt).UserTempDeviceName.empty() ) + msg << " "; + else + msg << (*pt).UserTempDeviceName; + + if ( (*pt).DisableState.empty() ) + msg << " "; + else + msg << (*pt).DisableState; + msg << (uint16_t) (*pt).hostConfigList.size(); + + HostConfigList::iterator pt1 = (*pt).hostConfigList.begin(); + + for ( ; pt1 != (*pt).hostConfigList.end() ; pt1++) + { + msg << (*pt1).IPAddr; + msg << (*pt1).HostName; + msg << (*pt1).NicID; + } + } + + msg << password; + msg << mysqlpw; + + try + { + //send the msg to Process Manager + MessageQueueClient procmgr("ProcMgr"); + procmgr.write(msg); + + // check for Ack msg if needed + if ( ackflag == ACK_YES ) + { + // wait 15 minutes for ACK from Process Manager + struct timespec ts = { 900, 0 }; + + receivedMSG = procmgr.read(&ts); + + if (receivedMSG.length() > 0) + { + receivedMSG >> msgType; + receivedMSG >> actionType; + receivedMSG >> status; + + if ( msgType == oam::ACK && actionType == requestType) + { + // ACK for this request + returnStatus = status; + } + } + else // timeout + returnStatus = API_TIMEOUT; + } + else + // No ACK, assume write success + returnStatus = API_SUCCESS; + + // shutdown connection + procmgr.shutdown(); + } + catch (...) + { + returnStatus = API_FAILURE; + } + + return returnStatus; +} + + + /*************************************************************************** * * Function: sendMsgToProcMgr3 diff --git a/oam/oamcpp/liboamcpp.h b/oam/oamcpp/liboamcpp.h index 39e2a51de..5d5a09c6c 100644 --- a/oam/oamcpp/liboamcpp.h +++ b/oam/oamcpp/liboamcpp.h @@ -1488,7 +1488,8 @@ public: * @param DeviceNetworkConfig the Modules added * @param password Host Root Password */ - EXPORT void addModule(DeviceNetworkList devicenetworklist, const std::string password, const std::string mysqlpw); + EXPORT void addModule(DeviceNetworkList devicenetworklist, const std::string password, const std::string mysqlpw, + bool storeHostnames); /** @brief remove Module * @@ -2503,6 +2504,12 @@ private: int sendMsgToProcMgr2(messageqcpp::ByteStream::byte requestType, DeviceNetworkList devicenetworklist, GRACEFUL_FLAG gracefulflag, ACK_FLAG ackflag, const std::string password = oam::UnassignedName, const std::string mysqlpw = oam::UnassignedName); + /** @brief a slightly different version of sendMsgToProcMgr2, which is for addmodule only. + */ + int sendAddModuleToProcMgr(messageqcpp::ByteStream::byte requestType, DeviceNetworkList devicenetworklist, + GRACEFUL_FLAG gracefulflag, ACK_FLAG ackflag, bool storeHostnames, const std::string password = oam::UnassignedName, + const std::string mysqlpw = oam::UnassignedName); + /** @brief build and send request message to Process Manager * Check for status messages */ diff --git a/oamapps/mcsadmin/mcsadmin.cpp b/oamapps/mcsadmin/mcsadmin.cpp index 454c7a3ea..180e3e18e 100644 --- a/oamapps/mcsadmin/mcsadmin.cpp +++ b/oamapps/mcsadmin/mcsadmin.cpp @@ -5677,6 +5677,7 @@ int processCommand(string* arguments) DeviceNetworkList enabledevicenetworklist; HostConfig hostconfig; + bool storeHostnames = false; string moduleType; string moduleName; int moduleCount; @@ -5687,12 +5688,12 @@ int processCommand(string* arguments) umStorageNames umstoragenames; int hostArg; int dbrootPerPM = 0; - + //check if module type or module name was entered if ( arguments[1].size() == 2 ) { //Module Type was entered - if (arguments[3] == "" && cloud == oam::UnassignedName) + if (arguments[4] == "" && cloud == oam::UnassignedName) { // need at least arguments cout << endl << "**** addModule Failed : Missing a required Parameter, enter 'help' for additional information" << endl; @@ -5702,8 +5703,47 @@ int processCommand(string* arguments) //Module Type was entered moduleType = arguments[1]; moduleCount = atoi(arguments[2].c_str()); + hostArg = 4; + + // MCOL-1607. Check whether we should store host names or IP addresses. + if (arguments[3] != "" && (arguments[3][0] == 'y' || arguments[3][0] == 'Y')) + storeHostnames = true; + + //check for a non-distrubuted install setup, dont need password + if ( DistributedInstall != "y" ) + { + if (arguments[5] != "") + password = arguments[5]; + else + { + cout << endl; + string prompt = "Enter the 'User' Password or 'ssh' if configured with ssh-keys"; + password = dataPrompt(prompt); + } + } + + if (arguments[6] != "") + dbrootPerPM = atoi(arguments[6].c_str()); + } + else + { + //Module Name was entered + if (arguments[2] == "" && cloud == oam::UnassignedName) + { + // need at least arguments + cout << endl << "**** addModule Failed : Missing a required Parameter, enter 'help' for additional information" << endl; + break; + } + + moduleName = arguments[1]; + moduleType = arguments[1].substr(0, MAX_MODULE_TYPE_SIZE); + moduleCount = 1; hostArg = 3; + // MCOL-1607. Check whether we should store host names or IP addresses. + if (arguments[2] != "" && (arguments[2][0] == 'y' || arguments[2][0] == 'Y')) + storeHostnames = true; + //check for a non-distrubuted install setup, dont need password if ( DistributedInstall != "y" ) { @@ -5720,37 +5760,6 @@ int processCommand(string* arguments) if (arguments[5] != "") dbrootPerPM = atoi(arguments[5].c_str()); } - else - { - //Module Name was entered - if (arguments[2] == "" && cloud == oam::UnassignedName) - { - // need at least arguments - cout << endl << "**** addModule Failed : Missing a required Parameter, enter 'help' for additional information" << endl; - break; - } - - moduleName = arguments[1]; - moduleType = arguments[1].substr(0, MAX_MODULE_TYPE_SIZE); - moduleCount = 1; - hostArg = 2; - - //check for a non-distrubuted install setup, dont need password - if ( DistributedInstall != "y" ) - { - if (arguments[3] != "") - password = arguments[3]; - else - { - cout << endl; - string prompt = "Enter the 'User' Password or 'ssh' if configured with ssh-keys"; - password = dataPrompt(prompt); - } - } - - if (arguments[4] != "") - dbrootPerPM = atoi(arguments[4].c_str()); - } //do we needed this check???? if ( moduleCount < 1 || moduleCount > 10 ) @@ -5942,7 +5951,24 @@ int processCommand(string* arguments) string hostName; string IPAddress; - if ( cloud == "amazon-ec2") + // MCOL-1607. Store hostnames in the config file if they entered one */ + if (storeHostnames) + { + // special case + if (cloud == "amazon-vpc" && *listPT1 == "autoassign") + { + hostName = oam::UnassignedName; + IPAddress = *listPT1; + } + else if (oam.isValidIP(*listPT1)) // they entered an IP addr + { + hostName = oam::UnassignedName; + IPAddress = *listPT1; + } + else // they entered a hostname + IPAddress = hostName = *listPT1; + } + else if ( cloud == "amazon-ec2") { hostName = *listPT1; @@ -5995,14 +6021,13 @@ int processCommand(string* arguments) // non-amazon hostName = *listPT1; IPAddress = oam.getIPAddress(hostName); - if ( IPAddress.empty() ) { // prompt for IP Address string prompt = "IP Address of " + hostName + " not found, enter IP Address or enter 'abort'"; IPAddress = dataPrompt(prompt); - - if ( IPAddress == "abort" || !oam.isValidIP(IPAddress)) + + if ( IPAddress == "abort" || !oam.isValidIP(IPAddress) ) return 1; } } @@ -6040,18 +6065,22 @@ int processCommand(string* arguments) { string prompt = "DataRedundancy is configured for dedicated network, enter a hostname"; DataRedundancyHostname = dataPrompt(prompt); - DataRedundancyIPAddress = oam.getIPAddress(DataRedundancyHostname); - - if ( DataRedundancyIPAddress.empty() ) + if (storeHostnames) + DataRedundancyIPAddress = DataRedundancyHostname; + else { - // prompt for IP Address - string prompt = "IP Address of " + DataRedundancyHostname + " not found, enter IP Address"; - DataRedundancyIPAddress = dataPrompt(prompt); + DataRedundancyIPAddress = oam.getIPAddress(DataRedundancyHostname); - if (!oam.isValidIP(DataRedundancyIPAddress)) - return 1; + if ( DataRedundancyIPAddress.empty() ) + { + // prompt for IP Address + string prompt = "IP Address of " + DataRedundancyHostname + " not found, enter IP Address"; + DataRedundancyIPAddress = dataPrompt(prompt); + + if (!oam.isValidIP(DataRedundancyIPAddress)) + return 1; + } } - sysConfig->setConfig("DataRedundancyConfig", dataDupHostName, DataRedundancyHostname); sysConfig->setConfig("DataRedundancyConfig", dataDupIPaddr, DataRedundancyIPAddress); } @@ -6122,7 +6151,7 @@ int processCommand(string* arguments) cout << "please wait..." << endl; - oam.addModule(devicenetworklist, password, mysqlpassword); + oam.addModule(devicenetworklist, password, mysqlpassword, storeHostnames); cout << "Add Module(s) successfully completed" << endl; diff --git a/procmgr/processmanager.cpp b/procmgr/processmanager.cpp index f21459f22..686243f87 100644 --- a/procmgr/processmanager.cpp +++ b/procmgr/processmanager.cpp @@ -2205,10 +2205,15 @@ void processMSG(messageqcpp::IOSocket* cfIos) string value; uint16_t count, ivalue, nicCount; + uint8_t tmp8; oam::DeviceNetworkConfig devicenetworkconfig; oam::DeviceNetworkList devicenetworklist; oam::HostConfig hostconfig; + bool storeHostnames; + msg >> tmp8; + storeHostnames = (tmp8 != 0); + //get module count to add msg >> count; @@ -2223,7 +2228,7 @@ void processMSG(messageqcpp::IOSocket* cfIos) devicenetworkconfig.UserTempDeviceName = value; msg >> value; devicenetworkconfig.DisableState = value; - + msg >> nicCount; for (int j = 0 ; j < nicCount ; j ++ ) @@ -2244,7 +2249,7 @@ void processMSG(messageqcpp::IOSocket* cfIos) string password; msg >> password; - status = processManager.addModule(devicenetworklist, password); + status = processManager.addModule(devicenetworklist, password, storeHostnames); log.writeLog(__LINE__, "ADDMODULE: ACK received from Process-Monitor, return status = " + oam.itoa(status)); } @@ -4835,7 +4840,8 @@ int ProcessManager::reinitProcessType( std::string processName ) * purpose: Add Module to system configuration * ******************************************************************************************/ -int ProcessManager::addModule(oam::DeviceNetworkList devicenetworklist, std::string password, bool manualFlag) +int ProcessManager::addModule(oam::DeviceNetworkList devicenetworklist, std::string password, bool storeHostnames, + bool manualFlag) { ProcessLog log; Configuration config; @@ -5262,7 +5268,10 @@ int ProcessManager::addModule(oam::DeviceNetworkList devicenetworklist, std::str } hostconfig.HostName = hostName; - hostconfig.IPAddr = IPAddr; + if (storeHostnames) + hostconfig.IPAddr = hostName; + else + hostconfig.IPAddr = IPAddr; hostconfig.NicID = (*pt1).NicID; devicenetworkconfig.hostConfigList.push_back(hostconfig); } diff --git a/procmgr/processmanager.h b/procmgr/processmanager.h index 9a57b38b5..8dc39a540 100644 --- a/procmgr/processmanager.h +++ b/procmgr/processmanager.h @@ -416,7 +416,8 @@ public: /** *@brief Add Module */ - int addModule(oam::DeviceNetworkList devicenetworklist, std::string password, bool manualFlag = true); + int addModule(oam::DeviceNetworkList devicenetworklist, std::string password, bool storeHostnames, + bool manualFlag = true); /** *@brief Configure Module From 4e74bbc7d7af6c59f11e0e6c5550ab48e121c06b Mon Sep 17 00:00:00 2001 From: David Hall Date: Thu, 29 Nov 2018 15:29:57 -0600 Subject: [PATCH 16/55] MCOL-1676 -- setvalue based on output column type rather than input column type. --- utils/windowfunction/windowfunctiontype.cpp | 73 ++++++++++++++++++++- 1 file changed, 71 insertions(+), 2 deletions(-) diff --git a/utils/windowfunction/windowfunctiontype.cpp b/utils/windowfunction/windowfunctiontype.cpp index efede3ef5..bdfe8c73c 100644 --- a/utils/windowfunction/windowfunctiontype.cpp +++ b/utils/windowfunction/windowfunctiontype.cpp @@ -325,6 +325,29 @@ template<> void WindowFunctionType::setValue(uint64_t i, string& t) fRow.setStringField(t, i); } +//template void WindowFunctionType::setValue(int, int64_t, int64_t, int64_t, string*); +template<> +void WindowFunctionType::setValue(int ct, int64_t b, int64_t e, int64_t c, string* v) +{ + if (c != WF__BOUND_ALL) + b = e = c; + + uint64_t i = fFieldIndex[0]; + + if (v == NULL) + v = (string*) getNullValueByType(ct, i); + + for (int64_t j = b; j <= e; j++) + { + if (j % 1000 == 0 && fStep->cancelled()) + break; + + fRow.setData(getPointer((*fRowData)[j])); + // MCOL-1985 Set the data based on out column type (ct) + setValue(i, *v); + } +} + template void WindowFunctionType::setValue(int ct, int64_t b, int64_t e, int64_t c, T* v) { @@ -342,7 +365,53 @@ void WindowFunctionType::setValue(int ct, int64_t b, int64_t e, int64_t c, T* v) break; fRow.setData(getPointer((*fRowData)[j])); - setValue(i, *v); + // MCOL-1985 Set the data based on out column type (ct) + switch (ct) + { + case CalpontSystemCatalog::TINYINT: + case CalpontSystemCatalog::SMALLINT: + case CalpontSystemCatalog::MEDINT: + case CalpontSystemCatalog::INT: + case CalpontSystemCatalog::BIGINT: + case CalpontSystemCatalog::DECIMAL: + { + int64_t iv = *v; + setValue(i, iv); + break; + } + + case CalpontSystemCatalog::UTINYINT: + case CalpontSystemCatalog::USMALLINT: + case CalpontSystemCatalog::UMEDINT: + case CalpontSystemCatalog::UINT: + case CalpontSystemCatalog::UBIGINT: + case CalpontSystemCatalog::UDECIMAL: + { + uint64_t uv = *v; + setValue(i, uv); + break; + } + + case CalpontSystemCatalog::DOUBLE: + case CalpontSystemCatalog::UDOUBLE: + { + double dv = *v; + setValue(i, dv); + break; + } + + case CalpontSystemCatalog::FLOAT: + case CalpontSystemCatalog::UFLOAT: + { + float fv = *v; + setValue(i, fv); + break; + } + default: + { + setValue(i, *v); + } + } } } @@ -476,7 +545,7 @@ template void WindowFunctionType::setValue(int, int64_t, int64_t, int64 template void WindowFunctionType::setValue(int, int64_t, int64_t, int64_t, uint64_t*); template void WindowFunctionType::setValue(int, int64_t, int64_t, int64_t, float*); template void WindowFunctionType::setValue(int, int64_t, int64_t, int64_t, double*); -template void WindowFunctionType::setValue(int, int64_t, int64_t, int64_t, string*); +//template void WindowFunctionType::setValue(int, int64_t, int64_t, int64_t, string*); void* WindowFunctionType::getNullValueByType(int ct, int pos) From 103df7bea9e6c90fa798876fb167a57460696a27 Mon Sep 17 00:00:00 2001 From: Patrick LeBlanc Date: Thu, 29 Nov 2018 15:30:16 -0600 Subject: [PATCH 17/55] Fixed a copy-paste error. --- oam/oamcpp/liboamcpp.cpp | 3 +++ 1 file changed, 3 insertions(+) diff --git a/oam/oamcpp/liboamcpp.cpp b/oam/oamcpp/liboamcpp.cpp index 0ec1e7b1c..58df33782 100644 --- a/oam/oamcpp/liboamcpp.cpp +++ b/oam/oamcpp/liboamcpp.cpp @@ -9931,6 +9931,7 @@ int Oam::sendMsgToProcMgr2(messageqcpp::ByteStream::byte requestType, DeviceNetw msg << (ByteStream::byte) gracefulflag; msg << (ByteStream::byte) ackflag; msg << (ByteStream::byte) requestManual; + msg << (uint16_t) devicenetworklist.size(); DeviceNetworkList::iterator pt = devicenetworklist.begin(); @@ -9948,6 +9949,7 @@ int Oam::sendMsgToProcMgr2(messageqcpp::ByteStream::byte requestType, DeviceNetw msg << " "; else msg << (*pt).DisableState; + msg << (uint16_t) (*pt).hostConfigList.size(); HostConfigList::iterator pt1 = (*pt).hostConfigList.begin(); @@ -10063,6 +10065,7 @@ int Oam::sendAddModuleToProcMgr(messageqcpp::ByteStream::byte requestType, Devic msg << " "; else msg << (*pt).DisableState; + msg << (uint16_t) (*pt).hostConfigList.size(); HostConfigList::iterator pt1 = (*pt).hostConfigList.begin(); From 451073158390d3d074ec31b7269e69bee5eebe64 Mon Sep 17 00:00:00 2001 From: Patrick LeBlanc Date: Fri, 30 Nov 2018 09:44:54 -0600 Subject: [PATCH 18/55] MCOL-1607. It works here but I will likely change the param ordering. --- oamapps/mcsadmin/mcsadmin.cpp | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/oamapps/mcsadmin/mcsadmin.cpp b/oamapps/mcsadmin/mcsadmin.cpp index 180e3e18e..637c93161 100644 --- a/oamapps/mcsadmin/mcsadmin.cpp +++ b/oamapps/mcsadmin/mcsadmin.cpp @@ -5693,7 +5693,7 @@ int processCommand(string* arguments) if ( arguments[1].size() == 2 ) { //Module Type was entered - if (arguments[4] == "" && cloud == oam::UnassignedName) + if (arguments[3] == "" && cloud == oam::UnassignedName) { // need at least arguments cout << endl << "**** addModule Failed : Missing a required Parameter, enter 'help' for additional information" << endl; From 710ee4d66bf4b2b4c23901cc0bcb04323a09fb28 Mon Sep 17 00:00:00 2001 From: Patrick LeBlanc Date: Fri, 30 Nov 2018 10:32:19 -0600 Subject: [PATCH 19/55] MCOL-1607. Changed the 'help' documentation for addModule. --- oam/etc/ConsoleCmds.xml | 7 ++++--- 1 file changed, 4 insertions(+), 3 deletions(-) diff --git a/oam/etc/ConsoleCmds.xml b/oam/etc/ConsoleCmds.xml index 9c58d78f3..17cf0581d 100644 --- a/oam/etc/ConsoleCmds.xml +++ b/oam/etc/ConsoleCmds.xml @@ -309,9 +309,10 @@ MariaDB Columnstore Packages and setup the module to make it ready to be restored Required: Module-type or Module-name being added Required: Number-of-Modules being added when Module-type is specified - Optional: Store given hostnames in the configuration instead of IP addresses (y/N) - Optional: Server-Hostnames/Amazon-Instance-Names seperated by commas - Optional: Server-root-password + Required: Store hostnames in the configuration instead of IP addresses (y/n) + If not Amazon, then Required: hostnames separated by commas + If Amazon, then Optional: Amazon-Instance-Names separated by commas + Optional: Server-user-password removeModule From 86ebce7b3998cfe438cd71d04a1f1c5ae56127e7 Mon Sep 17 00:00:00 2001 From: David Hall Date: Mon, 10 Dec 2018 11:42:01 -0600 Subject: [PATCH 20/55] MCOL-1981 regr_avgx and regr_avgy should return NULL if count is zero --- utils/regr/regr_avgx.cpp | 6 +----- utils/regr/regr_avgy.cpp | 6 +----- 2 files changed, 2 insertions(+), 10 deletions(-) diff --git a/utils/regr/regr_avgx.cpp b/utils/regr/regr_avgx.cpp index 2041647a5..c366f4e5a 100644 --- a/utils/regr/regr_avgx.cpp +++ b/utils/regr/regr_avgx.cpp @@ -123,11 +123,7 @@ mcsv1_UDAF::ReturnCode regr_avgx::evaluate(mcsv1Context* context, static_any::an { struct regr_avgx_data* data = (struct regr_avgx_data*)context->getUserData()->data; - if (data->cnt == 0) - { - valOut = 0; - } - else + if (data->cnt > 0) { valOut = data->sum / (double)data->cnt; } diff --git a/utils/regr/regr_avgy.cpp b/utils/regr/regr_avgy.cpp index 69c654acf..87a729263 100644 --- a/utils/regr/regr_avgy.cpp +++ b/utils/regr/regr_avgy.cpp @@ -121,11 +121,7 @@ mcsv1_UDAF::ReturnCode regr_avgy::evaluate(mcsv1Context* context, static_any::an { struct regr_avgy_data* data = (struct regr_avgy_data*)context->getUserData()->data; - if (data->cnt == 0) - { - valOut = 0; - } - else + if (data->cnt > 0) { valOut = data->sum / (double)data->cnt; } From 9d438f0a24f6e89201ced7dfaba3a0252e5231b8 Mon Sep 17 00:00:00 2001 From: David Hall Date: Tue, 11 Dec 2018 09:37:12 -0600 Subject: [PATCH 21/55] MCOL-1676 Window Functions with FLOAT type gives incorrect answer --- utils/windowfunction/windowfunctiontype.cpp | 72 ++++++++++++++++++++- 1 file changed, 69 insertions(+), 3 deletions(-) diff --git a/utils/windowfunction/windowfunctiontype.cpp b/utils/windowfunction/windowfunctiontype.cpp index 9afb4105d..7f3de615d 100755 --- a/utils/windowfunction/windowfunctiontype.cpp +++ b/utils/windowfunction/windowfunctiontype.cpp @@ -304,6 +304,28 @@ template<> void WindowFunctionType::setValue(uint64_t i, string& t) fRow.setStringField(t, i); } +// MCOL-1676 Need a separate specialization for string now. +template<> +void WindowFunctionType::setValue(int ct, int64_t b, int64_t e, int64_t c, string* v) +{ + if (c != WF__BOUND_ALL) + b = e = c; + + uint64_t i = fFieldIndex[0]; + + if (v == NULL) + v = (string*) getNullValueByType(ct, i); + + for (int64_t j = b; j <= e; j++) + { + if (j % 1000 == 0 && fStep->cancelled()) + break; + + fRow.setData(getPointer((*fRowData)[j])); + setValue(i, *v); + } +} + template void WindowFunctionType::setValue(int ct, int64_t b, int64_t e, int64_t c, T* v) { @@ -320,7 +342,53 @@ void WindowFunctionType::setValue(int ct, int64_t b, int64_t e, int64_t c, T* v) break; fRow.setData(getPointer((*fRowData)[j])); - setValue(i, *v); + // MCOL-1676 Set the data based on out column type (ct) + switch (ct) + { + case CalpontSystemCatalog::TINYINT: + case CalpontSystemCatalog::SMALLINT: + case CalpontSystemCatalog::MEDINT: + case CalpontSystemCatalog::INT: + case CalpontSystemCatalog::BIGINT: + case CalpontSystemCatalog::DECIMAL: + { + int64_t iv = *v; + setValue(i, iv); + break; + } + + case CalpontSystemCatalog::UTINYINT: + case CalpontSystemCatalog::USMALLINT: + case CalpontSystemCatalog::UMEDINT: + case CalpontSystemCatalog::UINT: + case CalpontSystemCatalog::UBIGINT: + case CalpontSystemCatalog::UDECIMAL: + { + uint64_t uv = *v; + setValue(i, uv); + break; + } + + case CalpontSystemCatalog::DOUBLE: + case CalpontSystemCatalog::UDOUBLE: + { + double dv = *v; + setValue(i, dv); + break; + } + + case CalpontSystemCatalog::FLOAT: + case CalpontSystemCatalog::UFLOAT: + { + float fv = *v; + setValue(i, fv); + break; + } + default: + { + setValue(i, *v); + } + } } } @@ -443,8 +511,6 @@ template void WindowFunctionType::setValue(int, int64_t, int64_t, int64 template void WindowFunctionType::setValue(int, int64_t, int64_t, int64_t, uint64_t*); template void WindowFunctionType::setValue(int, int64_t, int64_t, int64_t, float*); template void WindowFunctionType::setValue(int, int64_t, int64_t, int64_t, double*); -template void WindowFunctionType::setValue(int, int64_t, int64_t, int64_t, string*); - void* WindowFunctionType::getNullValueByType(int ct, int pos) { From cc70726a33976cb8069e7bd9f5fdf0a37573bf12 Mon Sep 17 00:00:00 2001 From: David Hall Date: Thu, 13 Dec 2018 15:49:11 -0600 Subject: [PATCH 22/55] MCOL-1983 For MCOL-1676 the reset of fValOut to NULL happens too soon. --- utils/windowfunction/wf_udaf.cpp | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/utils/windowfunction/wf_udaf.cpp b/utils/windowfunction/wf_udaf.cpp index 5986c2872..91ae5f9b2 100644 --- a/utils/windowfunction/wf_udaf.cpp +++ b/utils/windowfunction/wf_udaf.cpp @@ -708,12 +708,12 @@ void WF_udaf::operator()(int64_t b, int64_t e, int64_t c) mcsv1sdk::mcsv1_UDAF::ReturnCode rc; uint64_t colOut = fFieldIndex[0]; bool isNull = false; - fValOut.reset(); if ((fFrameUnit == WF__FRAME_ROWS) || (fPrev == -1) || (!fPeer->operator()(getPointer(fRowData->at(c)), getPointer(fRowData->at(fPrev))))) { + fValOut.reset(); // for unbounded - current row special handling if (fPrev >= b && fPrev < c) b = c; From 9888f88ac5d96f7732784e36d5d8e85ce140f9cd Mon Sep 17 00:00:00 2001 From: Andrew Hutchings Date: Mon, 17 Dec 2018 09:08:30 +0000 Subject: [PATCH 23/55] MCOL-1662 Use version buffer for UPDATEs For some reason version buffer is not turned up for update queries which means that the version number for dictionary blocks is not changed. This can lead to dirty cache reads resulting in _CpNoTf_ in the results. This patch turns on version buffer for updates. --- writeengine/server/we_dmlcommandproc.cpp | 1 + 1 file changed, 1 insertion(+) diff --git a/writeengine/server/we_dmlcommandproc.cpp b/writeengine/server/we_dmlcommandproc.cpp index 90831afd6..006c06b48 100644 --- a/writeengine/server/we_dmlcommandproc.cpp +++ b/writeengine/server/we_dmlcommandproc.cpp @@ -2515,6 +2515,7 @@ uint8_t WE_DMLCommandProc::processUpdate(messageqcpp::ByteStream& bs, boost::scoped_array preBlkNums(new int[columnsUpdated.size()]); boost::scoped_array oids(new OID[columnsUpdated.size()]); + BRMWrapper::setUseVb(true); for (unsigned int j = 0; j < columnsUpdated.size(); j++) { //timer.start("lookupsyscat"); From 7c47ea181f8fc3349adbae0547d5ddb65e9e01ba Mon Sep 17 00:00:00 2001 From: Andrew Hutchings Date: Thu, 20 Dec 2018 15:32:32 +0000 Subject: [PATCH 24/55] MCOL-1726 Make cleartablelock remove txn from BRM With an API txn (and likely other scenarios) cleartablelock will roll back the txn and lock but leave a stale txn pointer in the BRM. This causes restart/shutdown to warn that transactions are still open. This patch makes sure BRM removes the txn from the list. --- tools/cleartablelock/cleartablelock.cpp | 6 ++++++ 1 file changed, 6 insertions(+) diff --git a/tools/cleartablelock/cleartablelock.cpp b/tools/cleartablelock/cleartablelock.cpp index 1a907510e..8c08ea224 100644 --- a/tools/cleartablelock/cleartablelock.cpp +++ b/tools/cleartablelock/cleartablelock.cpp @@ -620,6 +620,12 @@ int main(int argc, char** argv) rc = execBulkRollbackReq( msgQueClts, pmList, &brm, tInfo, tblName.toString(), rollbackOnly, errMsg ); + + BRM::TxnID txn; + txn.id = tInfo.ownerTxnID; + txn.valid = true; + brm.rolledback(txn); + if (rc != 0) { logFinalStatus( tblName.toString(), lockID, errMsg ); From 93fb0902950342fb9ed01c6a0c766bafc0355b97 Mon Sep 17 00:00:00 2001 From: Ben Thompson Date: Fri, 21 Dec 2018 13:54:47 -0600 Subject: [PATCH 25/55] Update VERSION 1.2.3 --- VERSION | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/VERSION b/VERSION index dee66414e..ebe283cd3 100644 --- a/VERSION +++ b/VERSION @@ -1,4 +1,4 @@ COLUMNSTORE_VERSION_MAJOR=1 COLUMNSTORE_VERSION_MINOR=2 -COLUMNSTORE_VERSION_PATCH=2 +COLUMNSTORE_VERSION_PATCH=3 COLUMNSTORE_VERSION_RELEASE=1 From 35a17a87c40b348625785add23786d1095fa40fa Mon Sep 17 00:00:00 2001 From: Roman Nozdrin Date: Tue, 25 Dec 2018 12:50:18 +0300 Subject: [PATCH 26/55] MCOL-1829 Subquery with limited order by could potentially return onordered set. There were two code mistakes: Eq::operator() always returned true for any pair and Hasher::operator() always returned 0 as a key. --- dbcon/joblist/limitedorderby.cpp | 21 ++++++--------------- dbcon/joblist/tupleaggregatestep.cpp | 4 ++++ utils/rowgroup/rowgroup.h | 1 + utils/windowfunction/idborderby.cpp | 7 +++++-- 4 files changed, 16 insertions(+), 17 deletions(-) diff --git a/dbcon/joblist/limitedorderby.cpp b/dbcon/joblist/limitedorderby.cpp index 7da1accfe..d5db83552 100644 --- a/dbcon/joblist/limitedorderby.cpp +++ b/dbcon/joblist/limitedorderby.cpp @@ -113,7 +113,6 @@ void LimitedOrderBy::processRow(const rowgroup::Row& row) if (fOrderByQueue.size() < fStart + fCount) { copyRow(row, &fRow0); - //memcpy(fRow0.getData(), row.getData(), row.getSize()); OrderByRow newRow(fRow0, fRule); fOrderByQueue.push(newRow); @@ -121,8 +120,6 @@ void LimitedOrderBy::processRow(const rowgroup::Row& row) if (fDistinct) fDistinctMap->insert(fRow0.getPointer()); - //fDistinctMap->insert(make_pair((fRow0.getData()+2), fRow0.getData())); - fRowGroup.incRowCount(); fRow0.nextRow(); @@ -150,23 +147,16 @@ void LimitedOrderBy::processRow(const rowgroup::Row& row) { OrderByRow swapRow = fOrderByQueue.top(); row1.setData(swapRow.fData); - fOrderByQueue.pop(); - if (!fDistinct) + copyRow(row, &row1); + + if (fDistinct) { - copyRow(row, &row1); - //memcpy(swapRow.fData, row.getData(), row.getSize()); - } - else - { - fDistinctMap->erase(row.getPointer()); - copyRow(row, &row1); + fDistinctMap->erase(fOrderByQueue.top().fData); fDistinctMap->insert(row1.getPointer()); - //fDistinctMap->erase(fDistinctMap->find(row.getData() + 2)); - //memcpy(swapRow.fData, row.getData(), row.getSize()); - //fDistinctMap->insert(make_pair((swapRow.fData+2), swapRow.fData)); } + fOrderByQueue.pop(); fOrderByQueue.push(swapRow); } } @@ -228,6 +218,7 @@ void LimitedOrderBy::finalize() const OrderByRow& topRow = fOrderByQueue.top(); row1.setData(topRow.fData); copyRow(row1, &fRow0); + //cerr << "LimitedOrderBy::finalize fRow0 " << fRow0.toString() << endl; fRowGroup.incRowCount(); offset--; fRow0.prevRow(rSize); diff --git a/dbcon/joblist/tupleaggregatestep.cpp b/dbcon/joblist/tupleaggregatestep.cpp index 9ddf42cc9..4c1893115 100644 --- a/dbcon/joblist/tupleaggregatestep.cpp +++ b/dbcon/joblist/tupleaggregatestep.cpp @@ -5554,6 +5554,8 @@ void TupleAggregateStep::threadedAggregateRowGroups(uint32_t threadID) rowIn.copyField(distRow[j], k, multiDist->subAggregators()[j]->getGroupByCols()[k].get()->fInputColumnIndex); } + // TBD This approach could potentiall + // put all values in on bucket. bucketID = distRow[j].hash(hashLens[j] - 1) % fNumOfBuckets; rowBucketVecs[bucketID][j].push_back(rowIn.getPointer()); rowIn.nextRow(); @@ -5572,6 +5574,8 @@ void TupleAggregateStep::threadedAggregateRowGroups(uint32_t threadID) for (uint64_t i = 0; i < fRowGroupIns[threadID].getRowCount(); ++i) { // The key is the groupby columns, which are the leading columns. + // TBD This approach could potentiall + // put all values in on bucket. int bucketID = rowIn.hash(hashLens[0] - 1) % fNumOfBuckets; rowBucketVecs[bucketID][0].push_back(rowIn.getPointer()); rowIn.nextRow(); diff --git a/utils/rowgroup/rowgroup.h b/utils/rowgroup/rowgroup.h index e1c3073c4..ca05360c2 100644 --- a/utils/rowgroup/rowgroup.h +++ b/utils/rowgroup/rowgroup.h @@ -1183,6 +1183,7 @@ inline bool Row::equals(const Row& r2, const std::vector& keyCols) con inline bool Row::equals(const Row& r2, uint32_t lastCol) const { + // This check fires with empty r2 only. if (lastCol >= columnCount) return true; diff --git a/utils/windowfunction/idborderby.cpp b/utils/windowfunction/idborderby.cpp index 8a021f8d8..bac14bec8 100644 --- a/utils/windowfunction/idborderby.cpp +++ b/utils/windowfunction/idborderby.cpp @@ -461,7 +461,8 @@ uint64_t IdbOrderBy::Hasher::operator()(const Row::Pointer& p) const { Row& row = ts->row1; row.setPointer(p); - uint64_t ret = row.hash(colCount); + // MCOL-1829 Row::h uses colcount as an array idx down a callstack. + uint64_t ret = row.hash();//(colCount - 1); //cout << "hash(): returning " << ret << " for row: " << row.toString() << endl; return ret; } @@ -471,7 +472,9 @@ bool IdbOrderBy::Eq::operator()(const Row::Pointer& d1, const Row::Pointer& d2) Row& r1 = ts->row1, &r2 = ts->row2; r1.setPointer(d1); r2.setPointer(d2); - bool ret = r1.equals(r2, colCount); + // MCOL-1829 Row::equals uses 2nd argument as container size boundary + // so it must be column count - 1. + bool ret = r1.equals(r2, colCount - 1); //cout << "equals(): returning " << (int) ret << " for r1: " << r1.toString() << " r2: " << r2.toString() // << endl; From eb75d3cd7ea285ad845041527cae32aaafc29a27 Mon Sep 17 00:00:00 2001 From: David Hall Date: Wed, 2 Jan 2019 10:31:53 -0600 Subject: [PATCH 27/55] MCOL-1983 Reject non-numeric arguments for regr_*** functions that require numeric --- utils/regr/corr.cpp | 7 ++++ utils/regr/covar_pop.cpp | 7 ++++ utils/regr/covar_samp.cpp | 7 ++++ utils/regr/regr_avgx.cpp | 7 ++++ utils/regr/regr_avgy.cpp | 7 ++++ utils/regr/regr_intercept.cpp | 7 ++++ utils/regr/regr_r2.cpp | 7 ++++ utils/regr/regr_slope.cpp | 8 +++- utils/regr/regr_sxx.cpp | 7 ++++ utils/regr/regr_sxy.cpp | 7 ++++ utils/regr/regr_syy.cpp | 7 ++++ utils/regr/regrmysql.cpp | 75 ++++++++++++++++++++++++++++++++++- utils/udfsdk/mcsv1_udaf.h | 2 +- 13 files changed, 152 insertions(+), 3 deletions(-) diff --git a/utils/regr/corr.cpp b/utils/regr/corr.cpp index 9d12680ea..13efbe897 100644 --- a/utils/regr/corr.cpp +++ b/utils/regr/corr.cpp @@ -57,6 +57,13 @@ mcsv1_UDAF::ReturnCode corr::init(mcsv1Context* context, context->setErrorMessage("corr() with other than 2 arguments"); return mcsv1_UDAF::ERROR; } + if (!(isNumeric(colTypes[0].dataType) && isNumeric(colTypes[1].dataType))) + { + // The error message will be prepended with + // "The storage engine for the table doesn't support " + context->setErrorMessage("corr() with non-numeric arguments"); + return mcsv1_UDAF::ERROR; + } context->setUserDataSize(sizeof(corr_data)); context->setResultType(CalpontSystemCatalog::DOUBLE); diff --git a/utils/regr/covar_pop.cpp b/utils/regr/covar_pop.cpp index 51d9a036f..6313e49d2 100644 --- a/utils/regr/covar_pop.cpp +++ b/utils/regr/covar_pop.cpp @@ -55,6 +55,13 @@ mcsv1_UDAF::ReturnCode covar_pop::init(mcsv1Context* context, context->setErrorMessage("covar_pop() with other than 2 arguments"); return mcsv1_UDAF::ERROR; } + if (!(isNumeric(colTypes[0].dataType) && isNumeric(colTypes[1].dataType))) + { + // The error message will be prepended with + // "The storage engine for the table doesn't support " + context->setErrorMessage("covar_pop() with non-numeric arguments"); + return mcsv1_UDAF::ERROR; + } context->setUserDataSize(sizeof(covar_pop_data)); context->setResultType(CalpontSystemCatalog::DOUBLE); diff --git a/utils/regr/covar_samp.cpp b/utils/regr/covar_samp.cpp index b0ebb168b..0a79ff31a 100644 --- a/utils/regr/covar_samp.cpp +++ b/utils/regr/covar_samp.cpp @@ -55,6 +55,13 @@ mcsv1_UDAF::ReturnCode covar_samp::init(mcsv1Context* context, context->setErrorMessage("covar_samp() with other than 2 arguments"); return mcsv1_UDAF::ERROR; } + if (!(isNumeric(colTypes[0].dataType) && isNumeric(colTypes[1].dataType))) + { + // The error message will be prepended with + // "The storage engine for the table doesn't support " + context->setErrorMessage("covar_samp() with non-numeric arguments"); + return mcsv1_UDAF::ERROR; + } context->setUserDataSize(sizeof(covar_samp_data)); context->setResultType(CalpontSystemCatalog::DOUBLE); diff --git a/utils/regr/regr_avgx.cpp b/utils/regr/regr_avgx.cpp index c366f4e5a..e601b4172 100644 --- a/utils/regr/regr_avgx.cpp +++ b/utils/regr/regr_avgx.cpp @@ -63,6 +63,13 @@ mcsv1_UDAF::ReturnCode regr_avgx::init(mcsv1Context* context, context->setErrorMessage("regr_avgx() with a non-numeric x argument"); return mcsv1_UDAF::ERROR; } + if (!(isNumeric(colTypes[1].dataType))) + { + // The error message will be prepended with + // "The storage engine for the table doesn't support " + context->setErrorMessage("regr_avgx() with a non-numeric independant (second) argument"); + return mcsv1_UDAF::ERROR; + } context->setUserDataSize(sizeof(regr_avgx_data)); context->setResultType(CalpontSystemCatalog::DOUBLE); diff --git a/utils/regr/regr_avgy.cpp b/utils/regr/regr_avgy.cpp index 87a729263..0d0cc58dd 100644 --- a/utils/regr/regr_avgy.cpp +++ b/utils/regr/regr_avgy.cpp @@ -63,6 +63,13 @@ mcsv1_UDAF::ReturnCode regr_avgy::init(mcsv1Context* context, context->setErrorMessage("regr_avgy() with a non-numeric x argument"); return mcsv1_UDAF::ERROR; } + if (!(isNumeric(colTypes[0].dataType))) + { + // The error message will be prepended with + // "The storage engine for the table doesn't support " + context->setErrorMessage("regr_avgy() with a non-numeric dependant (first) argument"); + return mcsv1_UDAF::ERROR; + } context->setUserDataSize(sizeof(regr_avgy_data)); context->setResultType(CalpontSystemCatalog::DOUBLE); diff --git a/utils/regr/regr_intercept.cpp b/utils/regr/regr_intercept.cpp index 011488d9d..d46d3117c 100644 --- a/utils/regr/regr_intercept.cpp +++ b/utils/regr/regr_intercept.cpp @@ -56,6 +56,13 @@ mcsv1_UDAF::ReturnCode regr_intercept::init(mcsv1Context* context, context->setErrorMessage("regr_intercept() with other than 2 arguments"); return mcsv1_UDAF::ERROR; } + if (!(isNumeric(colTypes[0].dataType) && isNumeric(colTypes[1].dataType))) + { + // The error message will be prepended with + // "The storage engine for the table doesn't support " + context->setErrorMessage("regr_intercept() with non-numeric arguments"); + return mcsv1_UDAF::ERROR; + } context->setUserDataSize(sizeof(regr_intercept_data)); context->setResultType(CalpontSystemCatalog::DOUBLE); diff --git a/utils/regr/regr_r2.cpp b/utils/regr/regr_r2.cpp index 96b0ada18..f159d834a 100644 --- a/utils/regr/regr_r2.cpp +++ b/utils/regr/regr_r2.cpp @@ -57,6 +57,13 @@ mcsv1_UDAF::ReturnCode regr_r2::init(mcsv1Context* context, context->setErrorMessage("regr_r2() with other than 2 arguments"); return mcsv1_UDAF::ERROR; } + if (!(isNumeric(colTypes[0].dataType) && isNumeric(colTypes[1].dataType))) + { + // The error message will be prepended with + // "The storage engine for the table doesn't support " + context->setErrorMessage("regr_r2() with non-numeric arguments"); + return mcsv1_UDAF::ERROR; + } context->setUserDataSize(sizeof(regr_r2_data)); context->setResultType(CalpontSystemCatalog::DOUBLE); diff --git a/utils/regr/regr_slope.cpp b/utils/regr/regr_slope.cpp index fdb94795a..6790e223e 100644 --- a/utils/regr/regr_slope.cpp +++ b/utils/regr/regr_slope.cpp @@ -56,7 +56,13 @@ mcsv1_UDAF::ReturnCode regr_slope::init(mcsv1Context* context, context->setErrorMessage("regr_slope() with other than 2 arguments"); return mcsv1_UDAF::ERROR; } - + if (!(isNumeric(colTypes[0].dataType) && isNumeric(colTypes[1].dataType))) + { + // The error message will be prepended with + // "The storage engine for the table doesn't support " + context->setErrorMessage("regr_slope() with non-numeric arguments"); + return mcsv1_UDAF::ERROR; + } context->setUserDataSize(sizeof(regr_slope_data)); context->setResultType(CalpontSystemCatalog::DOUBLE); context->setColWidth(8); diff --git a/utils/regr/regr_sxx.cpp b/utils/regr/regr_sxx.cpp index 3f06af61b..b4c467fc3 100644 --- a/utils/regr/regr_sxx.cpp +++ b/utils/regr/regr_sxx.cpp @@ -54,6 +54,13 @@ mcsv1_UDAF::ReturnCode regr_sxx::init(mcsv1Context* context, context->setErrorMessage("regr_sxx() with other than 2 arguments"); return mcsv1_UDAF::ERROR; } + if (!(isNumeric(colTypes[1].dataType))) + { + // The error message will be prepended with + // "The storage engine for the table doesn't support " + context->setErrorMessage("regr_sxx() with a non-numeric independant (second) argument"); + return mcsv1_UDAF::ERROR; + } context->setUserDataSize(sizeof(regr_sxx_data)); context->setResultType(CalpontSystemCatalog::DOUBLE); diff --git a/utils/regr/regr_sxy.cpp b/utils/regr/regr_sxy.cpp index e6d005597..9cdbffb6a 100644 --- a/utils/regr/regr_sxy.cpp +++ b/utils/regr/regr_sxy.cpp @@ -55,6 +55,13 @@ mcsv1_UDAF::ReturnCode regr_sxy::init(mcsv1Context* context, context->setErrorMessage("regr_sxy() with other than 2 arguments"); return mcsv1_UDAF::ERROR; } + if (!(isNumeric(colTypes[0].dataType) && isNumeric(colTypes[1].dataType))) + { + // The error message will be prepended with + // "The storage engine for the table doesn't support " + context->setErrorMessage("regr_sxy() with non-numeric arguments"); + return mcsv1_UDAF::ERROR; + } context->setUserDataSize(sizeof(regr_sxy_data)); context->setResultType(CalpontSystemCatalog::DOUBLE); diff --git a/utils/regr/regr_syy.cpp b/utils/regr/regr_syy.cpp index d0841f723..98e614a8e 100644 --- a/utils/regr/regr_syy.cpp +++ b/utils/regr/regr_syy.cpp @@ -54,6 +54,13 @@ mcsv1_UDAF::ReturnCode regr_syy::init(mcsv1Context* context, context->setErrorMessage("regr_syy() with other than 2 arguments"); return mcsv1_UDAF::ERROR; } + if (!(isNumeric(colTypes[0].dataType))) + { + // The error message will be prepended with + // "The storage engine for the table doesn't support " + context->setErrorMessage("regr_syy() with a non-numeric dependant (first) argument"); + return mcsv1_UDAF::ERROR; + } context->setUserDataSize(sizeof(regr_syy_data)); context->setResultType(CalpontSystemCatalog::DOUBLE); diff --git a/utils/regr/regrmysql.cpp b/utils/regr/regrmysql.cpp index 08f46bb11..4980108e3 100644 --- a/utils/regr/regrmysql.cpp +++ b/utils/regr/regrmysql.cpp @@ -2,12 +2,30 @@ #include #include #include +#include using namespace std; #include "idb_mysql.h" namespace { +inline bool isNumeric(int type, const char* attr) +{ + if (type == INT_RESULT || type == REAL_RESULT || type == DECIMAL_RESULT) + { + return true; + } +#if _MSC_VER + if (_strnicmp("NULL", attr, 4) == 0)) +#else + if (strncasecmp("NULL", attr, 4) == 0) +#endif + { + return true; + } + return false; +} + inline double cvtArgToDouble(int t, const char* v) { double d = 0.0; @@ -144,7 +162,12 @@ extern "C" strcpy(message,"regr_avgx() requires two arguments"); return 1; } - + if (!(isNumeric(args->arg_type[1], args->attributes[1]))) + { + strcpy(message,"regr_avgx() with a non-numeric independant (second) argument"); + return 1; + } + if (!(data = (struct regr_avgx_data*) malloc(sizeof(struct regr_avgx_data)))) { strmov(message,"Couldn't allocate memory"); @@ -228,6 +251,11 @@ extern "C" strcpy(message,"regr_avgy() requires two arguments"); return 1; } + if (!(isNumeric(args->arg_type[0], args->attributes[0]))) + { + strcpy(message,"regr_avgy() with a non-numeric dependant (first) argument"); + return 1; + } if (!(data = (struct regr_avgy_data*) malloc(sizeof(struct regr_avgy_data)))) { @@ -394,6 +422,11 @@ extern "C" strcpy(message,"regr_slope() requires two arguments"); return 1; } + if (!(isNumeric(args->arg_type[0], args->attributes[0]) && isNumeric(args->arg_type[1], args->attributes[1]))) + { + strcpy(message,"regr_slope() with non-numeric arguments"); + return 1; + } if (!(data = (struct regr_slope_data*) malloc(sizeof(struct regr_slope_data)))) { @@ -505,6 +538,11 @@ extern "C" strcpy(message,"regr_intercept() requires two arguments"); return 1; } + if (!(isNumeric(args->arg_type[0], args->attributes[0]) && isNumeric(args->arg_type[1], args->attributes[1]))) + { + strcpy(message,"regr_intercept() with non-numeric arguments"); + return 1; + } if (!(data = (struct regr_intercept_data*) malloc(sizeof(struct regr_intercept_data)))) { @@ -619,6 +657,11 @@ extern "C" strcpy(message,"regr_r2() requires two arguments"); return 1; } + if (!(isNumeric(args->arg_type[0], args->attributes[0]) && isNumeric(args->arg_type[1], args->attributes[1]))) + { + strcpy(message,"regr_r2() with non-numeric arguments"); + return 1; + } if (!(data = (struct regr_r2_data*) malloc(sizeof(struct regr_r2_data)))) { @@ -748,6 +791,11 @@ extern "C" strcpy(message,"corr() requires two arguments"); return 1; } + if (!(isNumeric(args->arg_type[0], args->attributes[0]) && isNumeric(args->arg_type[1], args->attributes[1]))) + { + strcpy(message,"corr() with non-numeric arguments"); + return 1; + } if (!(data = (struct corr_data*) malloc(sizeof(struct corr_data)))) { @@ -874,6 +922,11 @@ extern "C" strcpy(message,"regr_sxx() requires two arguments"); return 1; } + if (!(isNumeric(args->arg_type[1], args->attributes[1]))) + { + strcpy(message,"regr_avgx() with a non-numeric independant (second) argument"); + return 1; + } if (!(data = (struct regr_sxx_data*) malloc(sizeof(struct regr_sxx_data)))) { @@ -970,6 +1023,11 @@ extern "C" strcpy(message,"regr_syy() requires two arguments"); return 1; } + if (!(isNumeric(args->arg_type[0], args->attributes[0]))) + { + strcpy(message,"regr_syy() with a non-numeric dependant (first) argument"); + return 1; + } if (!(data = (struct regr_syy_data*) malloc(sizeof(struct regr_syy_data)))) { @@ -1068,6 +1126,11 @@ extern "C" strcpy(message,"regr_sxy() requires two arguments"); return 1; } + if (!(isNumeric(args->arg_type[0], args->attributes[0]) && isNumeric(args->arg_type[1], args->attributes[1]))) + { + strcpy(message,"regr_sxy() with non-numeric arguments"); + return 1; + } if (!(data = (struct regr_sxy_data*) malloc(sizeof(struct regr_sxy_data)))) { @@ -1171,6 +1234,11 @@ extern "C" strcpy(message,"covar_pop() requires two arguments"); return 1; } + if (!(isNumeric(args->arg_type[0], args->attributes[0]) && isNumeric(args->arg_type[1], args->attributes[1]))) + { + strcpy(message,"covar_pop() with non-numeric arguments"); + return 1; + } if (!(data = (struct covar_pop_data*) malloc(sizeof(struct covar_pop_data)))) { @@ -1273,6 +1341,11 @@ extern "C" strcpy(message,"covar_samp() requires two arguments"); return 1; } + if (!(isNumeric(args->arg_type[0], args->attributes[0]) && isNumeric(args->arg_type[1], args->attributes[1]))) + { + strcpy(message,"covar_samp() with non-numeric arguments"); + return 1; + } if (!(data = (struct covar_samp_data*) malloc(sizeof(struct covar_samp_data)))) { diff --git a/utils/udfsdk/mcsv1_udaf.h b/utils/udfsdk/mcsv1_udaf.h index ec0d0cb79..d6ba04483 100644 --- a/utils/udfsdk/mcsv1_udaf.h +++ b/utils/udfsdk/mcsv1_udaf.h @@ -976,7 +976,7 @@ inline mcsv1_UDAF::ReturnCode mcsv1_UDAF::createUserData(UserData*& userData, in template inline T mcsv1_UDAF::convertAnyTo(static_any::any& valIn) { - T val; + T val = 0; if (valIn.compatible(longTypeId)) { val = valIn.cast(); From 6deb5e1bfde201c6e54ce1959014459d43e7e3b8 Mon Sep 17 00:00:00 2001 From: Gagan Goel Date: Tue, 8 Jan 2019 20:15:04 -0500 Subject: [PATCH 28/55] MCOL-2057 Fix truncate(x,d) when d < 0 for unsigned int data types For unsigned data types in the int family (tinyint, smallint, int, bigint), truncate(x, d) now properly applies zeros when d < 0. --- utils/funcexp/func_truncate.cpp | 29 ++++++++++++++++++++++++++++- 1 file changed, 28 insertions(+), 1 deletion(-) diff --git a/utils/funcexp/func_truncate.cpp b/utils/funcexp/func_truncate.cpp index b822caacd..2da482ab8 100644 --- a/utils/funcexp/func_truncate.cpp +++ b/utils/funcexp/func_truncate.cpp @@ -136,7 +136,34 @@ uint64_t Func_truncate::getUintVal(Row& row, bool& isNull, CalpontSystemCatalog::ColType& op_ct) { - return parm[0]->data()->getUintVal(row, isNull); + uint64_t val = parm[0]->data()->getUintVal(row, isNull); + + if (isNull) + return val; + + int64_t d = parm[1]->data()->getIntVal(row, isNull); + + if (isNull || d >= 0) + return val; + + uint64_t p = 1; + int64_t i = (-d); + + // Handle overflow since p can't have more than 19 0's + if (i >= 20) + { + val = 0; + } + else + { + while (i--) + p *= 10; + + val /= p; + val *= p; + } + + return val; } From fe94776ac6248da418de13fa1d732b1d7559f80c Mon Sep 17 00:00:00 2001 From: Patrick LeBlanc Date: Wed, 9 Jan 2019 16:31:38 -0600 Subject: [PATCH 29/55] MCOL-2062: bug parsing scientific notation in cpimport Made it use strtold instead of the 'manual' string parsing stuff it was doing. --- writeengine/shared/we_convertor.cpp | 107 +++++++--------------------- 1 file changed, 24 insertions(+), 83 deletions(-) diff --git a/writeengine/shared/we_convertor.cpp b/writeengine/shared/we_convertor.cpp index f4ad696d2..8ad344ce3 100644 --- a/writeengine/shared/we_convertor.cpp +++ b/writeengine/shared/we_convertor.cpp @@ -174,93 +174,34 @@ const std::string Convertor::int2Str(int val) * converted long long for specified "field" ******************************************************************************/ /* static */ + long long Convertor::convertDecimalString( const char* field, - int fieldLength, - int scale ) + int fieldLength, + int scale) { - long long llVal = 0; - - int nDigitsBeforeDecPt = 0; - int nDigitsAfterDecPt = 0; - long long roundUp = 0; //@bug 3405 round off decimal column values - - // Determine the number of digits before and after the decimal point - char* posDecPt = (char*)memchr(field, '.', fieldLength); - if (posDecPt) - { - nDigitsBeforeDecPt = posDecPt - field; - nDigitsAfterDecPt = fieldLength - nDigitsBeforeDecPt - 1; - - //@bug 3405 round off decimal column values - // We look at the scale+1 digit to see if we need to round up. - if (nDigitsAfterDecPt > scale) - { - char roundOffDigit = *(posDecPt + 1 + scale); - if ( (roundOffDigit > '4') && - (roundOffDigit <='9') ) // round up - { - roundUp = 1; - - // We can't just use the sign of llVal to determine whether to - // add +1 or -1, because if we read in -0.005 with scale 2, we - // end up parsing "-0.00", which yields 0; meaning we lose the - // sign. So better (though maybe slower) to look for any lead- - // ing negative sign in the input string. - for (int k=0; k LLONG_MAX || dval < LLONG_MIN) + errno = ERANGE; else - { - nDigitsBeforeDecPt = fieldLength; - nDigitsAfterDecPt = 0; - } - - // Strip out the decimal point by stringing together - // the digits before and after the decimal point. - char* data = (char*)alloca(nDigitsBeforeDecPt + scale + 1); - memcpy(data, field, nDigitsBeforeDecPt); - if (nDigitsAfterDecPt) - { - if (scale > nDigitsAfterDecPt) - memcpy(data + nDigitsBeforeDecPt, - field + nDigitsBeforeDecPt + 1, - nDigitsAfterDecPt); - else // (scale <= nDigitsAfterDecPt) - memcpy(data + nDigitsBeforeDecPt, - field + nDigitsBeforeDecPt + 1, - scale); - } - - // Add on any necessary zero padding at the end - if (scale > nDigitsAfterDecPt) - { - memset(data + nDigitsBeforeDecPt + nDigitsAfterDecPt, - '0', - scale - nDigitsAfterDecPt); - } - - data[nDigitsBeforeDecPt + scale] = '\0'; - - // Convert our constructed decimal string back to a long long - //@bug 1814 Force strtoll to use base 10 - errno = 0; - llVal = strtoll(data, 0, 10); - - //@bug 3405 round off decimal values - if ((roundUp) && (errno == 0)) - llVal += roundUp; - - return llVal; + errno = 0; + + ret = dval; + + // get the fractional part of what's left & round ret up or down. + dval -= ret; + if (dval >= 0.5 && ret < LLONG_MAX) + ++ret; + else if (dval <= -0.5 && ret > LLONG_MIN) + --ret; + return ret; } /******************************************************************************* From 7b4cec575747f22fabba57886f7f968952c93e08 Mon Sep 17 00:00:00 2001 From: Roman Nozdrin Date: Fri, 11 Jan 2019 17:07:54 +0300 Subject: [PATCH 30/55] MCOL-1829 Subquery with limited order by could potentially return onordered set. There were two code mistakes: Eq::operator() always returned true for any pair and Hasher::operator() always returned 0 as a key. --- dbcon/joblist/limitedorderby.cpp | 20 ++++++-------------- utils/windowfunction/idborderby.cpp | 7 +++++-- 2 files changed, 11 insertions(+), 16 deletions(-) diff --git a/dbcon/joblist/limitedorderby.cpp b/dbcon/joblist/limitedorderby.cpp index 4c19440ee..c78ed43fa 100644 --- a/dbcon/joblist/limitedorderby.cpp +++ b/dbcon/joblist/limitedorderby.cpp @@ -108,14 +108,12 @@ void LimitedOrderBy::processRow(const rowgroup::Row& row) if (fOrderByQueue.size() < fStart+fCount) { copyRow(row, &fRow0); - //memcpy(fRow0.getData(), row.getData(), row.getSize()); OrderByRow newRow(fRow0, fRule); fOrderByQueue.push(newRow); // add to the distinct map if (fDistinct) fDistinctMap->insert(fRow0.getPointer()); - //fDistinctMap->insert(make_pair((fRow0.getData()+2), fRow0.getData())); fRowGroup.incRowCount(); fRow0.nextRow(); @@ -142,21 +140,15 @@ void LimitedOrderBy::processRow(const rowgroup::Row& row) { OrderByRow swapRow = fOrderByQueue.top(); row1.setData(swapRow.fData); - fOrderByQueue.pop(); - if (!fDistinct) + copyRow(row, &row1); + + if (fDistinct) { - copyRow(row, &row1); - //memcpy(swapRow.fData, row.getData(), row.getSize()); - } - else - { - fDistinctMap->erase(row.getPointer()); - copyRow(row, &row1); + fDistinctMap->erase(fOrderByQueue.top().fData); fDistinctMap->insert(row1.getPointer()); - //fDistinctMap->erase(fDistinctMap->find(row.getData() + 2)); - //memcpy(swapRow.fData, row.getData(), row.getSize()); - //fDistinctMap->insert(make_pair((swapRow.fData+2), swapRow.fData)); } + + fOrderByQueue.pop(); fOrderByQueue.push(swapRow); } } diff --git a/utils/windowfunction/idborderby.cpp b/utils/windowfunction/idborderby.cpp index 1375af542..7e6d6acd5 100644 --- a/utils/windowfunction/idborderby.cpp +++ b/utils/windowfunction/idborderby.cpp @@ -443,7 +443,8 @@ uint64_t IdbOrderBy::Hasher::operator()(const Row::Pointer &p) const { Row &row = ts->row1; row.setPointer(p); - uint64_t ret = row.hash(colCount); + // MCOL-1829 Row::h uses colcount as an array idx down a callstack. + uint64_t ret = row.hash(); //cout << "hash(): returning " << ret << " for row: " << row.toString() << endl; return ret; } @@ -453,7 +454,9 @@ bool IdbOrderBy::Eq::operator()(const Row::Pointer &d1, const Row::Pointer &d2) Row &r1 = ts->row1, &r2 = ts->row2; r1.setPointer(d1); r2.setPointer(d2); - bool ret = r1.equals(r2, colCount); + // MCOL-1829 Row::equals uses 2nd argument as container size boundary + // so it must be column count - 1. + bool ret = r1.equals(r2, colCount - 1); //cout << "equals(): returning " << (int) ret << " for r1: " << r1.toString() << " r2: " << r2.toString() // << endl; From 771988dd1f95c4ebd2562d5b5847e9aefb49d309 Mon Sep 17 00:00:00 2001 From: Roman Nozdrin Date: Tue, 15 Jan 2019 10:13:04 +0300 Subject: [PATCH 31/55] MCOL-2052 IS.columnstore_files now prints correct number of records for any relation out. Return statement called in generate_result() returns too early. --- dbcon/mysql/is_columnstore_files.cpp | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/dbcon/mysql/is_columnstore_files.cpp b/dbcon/mysql/is_columnstore_files.cpp index 1a5fdad1e..b64c64151 100644 --- a/dbcon/mysql/is_columnstore_files.cpp +++ b/dbcon/mysql/is_columnstore_files.cpp @@ -110,7 +110,7 @@ static int generate_result(BRM::OID_t oid, BRM::DBRM *emp, TABLE *table, THD *th if (iter->blockOffset > 0) { iter++; - return 0; + continue; } try From d844382a0999a918f1e626f2ae32091809e53d76 Mon Sep 17 00:00:00 2001 From: sneJ- Date: Tue, 15 Jan 2019 20:11:26 +0100 Subject: [PATCH 32/55] MCOL-2068 - first try to add memory mgmt parameter to postConfigure - untested --- oamapps/postConfigure/postConfigure.cpp | 191 +++++++++++++++++------- 1 file changed, 135 insertions(+), 56 deletions(-) diff --git a/oamapps/postConfigure/postConfigure.cpp b/oamapps/postConfigure/postConfigure.cpp index e5680485b..e3b0d525b 100644 --- a/oamapps/postConfigure/postConfigure.cpp +++ b/oamapps/postConfigure/postConfigure.cpp @@ -231,6 +231,8 @@ int main(int argc, char* argv[]) string cmd; string pmIpAddrs = ""; string umIpAddrs = ""; + string numBlocksPctParam = ""; + string totalUmMemoryParam = ""; // struct sysinfo myinfo; @@ -314,7 +316,7 @@ int main(int argc, char* argv[]) cout << " Enter one of the options within [], if available, or" << endl; cout << " Enter a new value" << endl << endl; cout << endl; - cout << "Usage: postConfigure [-h][-c][-u][-p][-qs][-qm][-qa][-port][-i][-n][-d][-sn][-pm-ip-addrs][-um-ip-addrs][-pm-count][-um-count]" << endl; + cout << "Usage: postConfigure [-h][-c][-u][-p][-qs][-qm][-qa][-port][-i][-n][-d][-sn][-pm-ip-addrs][-um-ip-addrs][-pm-count][-um-count][-numBlocksPct][-totalUmMemory]" << endl; cout << " -h Help" << endl; cout << " -c Config File to use to extract configuration data, default is Columnstore.xml.rpmsave" << endl; cout << " -u Upgrade, Install using the Config File from -c, default to Columnstore.xml.rpmsave" << endl; @@ -330,6 +332,10 @@ int main(int argc, char* argv[]) cout << " -pm-ip-addrs Performance Module IP Addresses xxx.xxx.xxx.xxx,xxx.xxx.xxx.xxx" << endl; cout << " -um-ip-addrs User Module IP Addresses xxx.xxx.xxx.xxx,xxx.xxx.xxx.xxx" << endl; cout << " -x Do not resolve IP Addresses from host names" << endl; + cout << " -numBlocksPct amount of physical memory to utilize for disk block caching" << endl; + cout << " (percentages of the total memory can be stated without suffix, explcit values with suffixes M or G)" + cout << " -totalUmMemory amount of physical memory to utilize for joins, intermediate results and set operations on the UM" << endl; + cout << " (percentages of the total memory can be stated without suffix, explcit values with suffixes M or G)" exit (0); } else if (string("-x") == argv[i]) @@ -478,10 +484,32 @@ int main(int argc, char* argv[]) } umNumber = atoi(argv[i]); } + else if ( string("-numBlocksPct") == argv[i] ) + { + i++; + if (i >= argc) + { + cout << " ERROR: Memory settings for numBlocksPct not provided" << endl; + //TODO check suffix. if % is given remove it + exit(1); + } + numBlocksPctParam = argv[i]; + } + else if (string("-totalUmMemory") == argv[i]) + { + i++; + if (i >= argc) + { + cout << " ERROR: Memory settings for totalUmMemory not provided" << endl; + //TODO check suffix. if no %, G, or M is given add % + exit(1); + } + totalUmMemoryParam = argv[i]; + } else { cout << " ERROR: Invalid Argument = " << argv[i] << endl; - cout << " Usage: postConfigure [-h][-c][-u][-p][-qs][-qm][-qa][-port][-i][-n][-d][-sn][-pm-ip-addrs][-um-ip-addrs][-pm-count][-um-count]" << endl; + cout << " Usage: postConfigure [-h][-c][-u][-p][-qs][-qm][-qa][-port][-i][-n][-d][-sn][-pm-ip-addrs][-um-ip-addrs][-pm-count][-um-count][-numBlocksPct][-totalUmMemory]" << endl; exit (1); } } @@ -1629,31 +1657,38 @@ int main(int argc, char* argv[]) case (oam::INSTALL_COMBINE_DM_UM_PM): // combined #1 - dm/um/pm on a single server { // are we using settings from previous config file? - if ( reuseConfig == "n" ) + if (reuseConfig == "n") { - if ( !uncommentCalpontXml("NumBlocksPct") ) - { - cout << "Update Columnstore.xml NumBlocksPct Section" << endl; - exit(1); - } string numBlocksPct; - try - { - numBlocksPct = sysConfig->getConfig("DBBC", "NumBlocksPct"); + // if numBlocksPct was set as command line parameter use the command line parameter value + if (!numBlocksPctParam.empty()) { + numBlocksPct = numBlocksPctParam; } - catch (...) - {} + else { + if (!uncommentCalpontXml("NumBlocksPct")) + { + cout << "Update Columnstore.xml NumBlocksPct Section" << endl; + exit(1); + } - if ( numBlocksPct == "70" || numBlocksPct.empty() ) - { - numBlocksPct = "50"; + try + { + numBlocksPct = sysConfig->getConfig("DBBC", "NumBlocksPct"); + } + catch (...) + { + } - if (hdfs) - numBlocksPct = "25"; + if (numBlocksPct == "70" || numBlocksPct.empty()) + { + numBlocksPct = "50"; + + if (hdfs) + numBlocksPct = "25"; + } } - try { sysConfig->setConfig("DBBC", "NumBlocksPct", numBlocksPct); @@ -1666,11 +1701,20 @@ int main(int argc, char* argv[]) exit(1); } - string percent = "25%"; + + string percent; - if (hdfs) - { - percent = "12%"; + if (!totalUmMemoryParam.empty()) { // if totalUmMemory was set as command line parameter use the command line parameter value + percent = totalUmMemoryParam; + } + else { //otherwise use reasonable defaults + + percent = "25%"; + + if (hdfs) + { + percent = "12%"; + } } cout << " Setting 'TotalUmMemory' to " << percent << endl; @@ -1695,22 +1739,33 @@ int main(int argc, char* argv[]) { try { - string numBlocksPct = sysConfig->getConfig("DBBC", "NumBlocksPct"); - cout << endl; - if ( numBlocksPct.empty() ) - cout << "NOTE: Using the default setting for 'NumBlocksPct' at 70%" << endl; - else - cout << "NOTE: Using previous configuration setting for 'NumBlocksPct' = " << numBlocksPct << "%" << endl; + if (!numBlocksPctParam.empty()) { // if numBlocksPct was set as command line parameter use the command line parameter value + sysConfig->setConfig("DBBC", "NumBlocksPct", numBlocksPctParam); + cout << "NOTE: Setting 'NumBlocksPct' to " << numBlocksPctParam << "%" << endl; + } + else { //otherwise use the settings from the previous config file + string numBlocksPct = sysConfig->getConfig("DBBC", "NumBlocksPct"); - string totalUmMemory = sysConfig->getConfig("HashJoin", "TotalUmMemory"); + if (numBlocksPct.empty()) + cout << "NOTE: Using the default setting for 'NumBlocksPct' at 70%" << endl; + else + cout << "NOTE: Using previous configuration setting for 'NumBlocksPct' = " << numBlocksPct << "%" << endl; + } - cout << " Using previous configuration setting for 'TotalUmMemory' = " << totalUmMemory << endl; + if (!totalUmMemoryParam.empty()) { // if totalUmMemory was set as command line parameter use the command line parameter value + sysConfig->setConfig("HashJoin", "TotalUmMemory", totalUmMemoryParam); + cout << " Setting 'TotalUmMemory' to " << totalUmMemoryParam << endl; + } + else { //otherwise use the settings from the previous config file + string totalUmMemory = sysConfig->getConfig("HashJoin", "TotalUmMemory"); + cout << " Using previous configuration setting for 'TotalUmMemory' = " << totalUmMemory << endl; + } } catch (...) { - cout << "ERROR: Problem reading NumBlocksPct/TotalUmMemory in the MariaDB ColumnStore System Configuration file" << endl; + cout << "ERROR: Problem reading/writing NumBlocksPct/TotalUmMemory in/to the MariaDB ColumnStore System Configuration file" << endl; exit(1); } } @@ -1725,25 +1780,31 @@ int main(int argc, char* argv[]) { string numBlocksPct; - try - { - numBlocksPct = sysConfig->getConfig("DBBC", "NumBlocksPct"); + // if numBlocksPct was set as command line parameter use the command line parameter value + if (!numBlocksPctParam.empty()) { + numBlocksPct = numBlocksPctParam; } - catch (...) - {} + else { + try + { + numBlocksPct = sysConfig->getConfig("DBBC", "NumBlocksPct"); + } + catch (...) + { + } - if ( numBlocksPct.empty() ) - { - numBlocksPct = "70"; + if (numBlocksPct.empty()) + { + numBlocksPct = "70"; - if (hdfs) - numBlocksPct = "35"; + if (hdfs) + numBlocksPct = "35"; + } } try { sysConfig->setConfig("DBBC", "NumBlocksPct", numBlocksPct); - cout << "NOTE: Setting 'NumBlocksPct' to " << numBlocksPct << "%" << endl; } catch (...) @@ -1752,11 +1813,18 @@ int main(int argc, char* argv[]) exit(1); } - string percent = "50%"; + string percent; - if (hdfs) - { - percent = "25%"; + if (!totalUmMemoryParam.empty()) { // if totalUmMemory was set as command line parameter use the command line parameter value + percent = totalUmMemoryParam; + } + else { //otherwise use reasonable defaults + percent = "50%"; + + if (hdfs) + { + percent = "25%"; + } } cout << " Setting 'TotalUmMemory' to " << percent << endl; @@ -1781,20 +1849,31 @@ int main(int argc, char* argv[]) { try { - string numBlocksPct = sysConfig->getConfig("DBBC", "NumBlocksPct"); + if (!numBlocksPctParam.empty()) { // if numBlocksPct was set as command line parameter use the command line parameter value + sysConfig->setConfig("DBBC", "NumBlocksPct", numBlocksPctParam); + cout << "NOTE: Setting 'NumBlocksPct' to " << numBlocksPctParam << "%" << endl; + } + else { //otherwise use the settings from the previous config file + string numBlocksPct = sysConfig->getConfig("DBBC", "NumBlocksPct"); - if ( numBlocksPct.empty() ) - cout << "NOTE: Using the default setting for 'NumBlocksPct' at 70%" << endl; - else - cout << "NOTE: Using previous configuration setting for 'NumBlocksPct' = " << numBlocksPct << "%" << endl; + if (numBlocksPct.empty()) + cout << "NOTE: Using the default setting for 'NumBlocksPct' at 70%" << endl; + else + cout << "NOTE: Using previous configuration setting for 'NumBlocksPct' = " << numBlocksPct << "%" << endl; + } - string totalUmMemory = sysConfig->getConfig("HashJoin", "TotalUmMemory"); - - cout << " Using previous configuration setting for 'TotalUmMemory' = " << totalUmMemory << endl; + if (!totalUmMemoryParam.empty()) { // if totalUmMemory was set as command line parameter use the command line parameter value + sysConfig->setConfig("HashJoin", "TotalUmMemory", totalUmMemoryParam); + cout << " Setting 'TotalUmMemory' to " << totalUmMemoryParam << endl; + } + else { //otherwise use reasonable defaults + string totalUmMemory = sysConfig->getConfig("HashJoin", "TotalUmMemory"); + cout << " Using previous configuration setting for 'TotalUmMemory' = " << totalUmMemory << endl; + } } catch (...) { - cout << "ERROR: Problem reading NumBlocksPct/TotalUmMemory in the MariaDB ColumnStore System Configuration file" << endl; + cout << "ERROR: Problem reading/writing NumBlocksPct/TotalUmMemory in/to the MariaDB ColumnStore System Configuration file" << endl; exit(1); } } From e60ac3001773c6a9db5013de1fa49eb45d87ffdc Mon Sep 17 00:00:00 2001 From: sneJ- Date: Wed, 16 Jan 2019 14:00:09 +0100 Subject: [PATCH 33/55] MCOL-2068 - introduced basic parameter checks --- oamapps/postConfigure/postConfigure.cpp | 52 ++++++++++++++++++++----- 1 file changed, 42 insertions(+), 10 deletions(-) diff --git a/oamapps/postConfigure/postConfigure.cpp b/oamapps/postConfigure/postConfigure.cpp index e3b0d525b..59b1cfc73 100644 --- a/oamapps/postConfigure/postConfigure.cpp +++ b/oamapps/postConfigure/postConfigure.cpp @@ -333,9 +333,9 @@ int main(int argc, char* argv[]) cout << " -um-ip-addrs User Module IP Addresses xxx.xxx.xxx.xxx,xxx.xxx.xxx.xxx" << endl; cout << " -x Do not resolve IP Addresses from host names" << endl; cout << " -numBlocksPct amount of physical memory to utilize for disk block caching" << endl; - cout << " (percentages of the total memory can be stated without suffix, explcit values with suffixes M or G)" + cout << " (percentages of the total memory need to be stated without suffix, explcit values with suffixes M or G)"; cout << " -totalUmMemory amount of physical memory to utilize for joins, intermediate results and set operations on the UM" << endl; - cout << " (percentages of the total memory can be stated without suffix, explcit values with suffixes M or G)" + cout << " (percentages of the total memory need to be stated with suffix %, explcit values with suffixes M or G)"; exit (0); } else if (string("-x") == argv[i]) @@ -490,10 +490,16 @@ int main(int argc, char* argv[]) if (i >= argc) { cout << " ERROR: Memory settings for numBlocksPct not provided" << endl; - //TODO check suffix. if % is given remove it exit(1); } - numBlocksPctParam = argv[i]; + // check that the parameter ends with a number M or G + if (isdigit(argv[i].back()) || argv[i].back() == 'M' || argv[i].bach() == 'G') { + numBlocksPctParam = argv[i]; + } + else { + cout << " ERROR: Memory settings for numBlocksPct need to end on a digit, M or G" << endl; + exit(1); + } } else if (string("-totalUmMemory") == argv[i]) { @@ -501,10 +507,16 @@ int main(int argc, char* argv[]) if (i >= argc) { cout << " ERROR: Memory settings for totalUmMemory not provided" << endl; - //TODO check suffix. if no %, G, or M is given add % exit(1); } - totalUmMemoryParam = argv[i]; + // check that the parameter ends with a %, M, or G + if (argv[i].back() == '%' || argv[i].back() == 'M' || argv[i].bach() == 'G') { + totalUmMemoryParam = argv[i]; + } + else { + cout << " ERROR: Memory settings for totalUmMemory need to end on %, M or G" << endl; + exit(1); + } } else { @@ -1693,7 +1705,12 @@ int main(int argc, char* argv[]) { sysConfig->setConfig("DBBC", "NumBlocksPct", numBlocksPct); - cout << endl << "NOTE: Setting 'NumBlocksPct' to " << numBlocksPct << "%" << endl; + if (numBlocksPct.back() == 'M' || numBlocksPct.back() == 'G') { + cout << endl << "NOTE: Setting 'NumBlocksPct' to " << numBlocksPct << endl; + } + else { + cout << endl << "NOTE: Setting 'NumBlocksPct' to " << numBlocksPct << "%" << endl; + } } catch (...) { @@ -1743,7 +1760,12 @@ int main(int argc, char* argv[]) if (!numBlocksPctParam.empty()) { // if numBlocksPct was set as command line parameter use the command line parameter value sysConfig->setConfig("DBBC", "NumBlocksPct", numBlocksPctParam); - cout << "NOTE: Setting 'NumBlocksPct' to " << numBlocksPctParam << "%" << endl; + if (numBlocksPctParam.back() == 'M' || numBlocksPctParam.back() == 'G') { + cout << endl << "NOTE: Setting 'NumBlocksPct' to " << numBlocksPctParam << endl; + } + else { + cout << endl << "NOTE: Setting 'NumBlocksPct' to " << numBlocksPctParam << "%" << endl; + } } else { //otherwise use the settings from the previous config file string numBlocksPct = sysConfig->getConfig("DBBC", "NumBlocksPct"); @@ -1805,7 +1827,12 @@ int main(int argc, char* argv[]) try { sysConfig->setConfig("DBBC", "NumBlocksPct", numBlocksPct); - cout << "NOTE: Setting 'NumBlocksPct' to " << numBlocksPct << "%" << endl; + if (numBlocksPct.back() == 'M' || numBlocksPct.back() == 'G') { + cout << endl << "NOTE: Setting 'NumBlocksPct' to " << numBlocksPct << endl; + } + else { + cout << endl << "NOTE: Setting 'NumBlocksPct' to " << numBlocksPct << "%" << endl; + } } catch (...) { @@ -1851,7 +1878,12 @@ int main(int argc, char* argv[]) { if (!numBlocksPctParam.empty()) { // if numBlocksPct was set as command line parameter use the command line parameter value sysConfig->setConfig("DBBC", "NumBlocksPct", numBlocksPctParam); - cout << "NOTE: Setting 'NumBlocksPct' to " << numBlocksPctParam << "%" << endl; + if (numBlocksPctParam.back() == 'M' || numBlocksPctParam.back() == 'G') { + cout << endl << "NOTE: Setting 'NumBlocksPct' to " << numBlocksPctParam << endl; + } + else { + cout << endl << "NOTE: Setting 'NumBlocksPct' to " << numBlocksPctParam << "%" << endl; + } } else { //otherwise use the settings from the previous config file string numBlocksPct = sysConfig->getConfig("DBBC", "NumBlocksPct"); From d04ebca2c4f7e3d71fa77f85eb665ef706cf9aa6 Mon Sep 17 00:00:00 2001 From: sneJ- Date: Wed, 16 Jan 2019 15:06:08 +0100 Subject: [PATCH 34/55] MCOL-2068 - changed .back() into .rbegin() to be compileable in non C++11 environments --- oamapps/postConfigure/postConfigure.cpp | 12 ++++++------ 1 file changed, 6 insertions(+), 6 deletions(-) diff --git a/oamapps/postConfigure/postConfigure.cpp b/oamapps/postConfigure/postConfigure.cpp index 59b1cfc73..9453560c6 100644 --- a/oamapps/postConfigure/postConfigure.cpp +++ b/oamapps/postConfigure/postConfigure.cpp @@ -493,7 +493,7 @@ int main(int argc, char* argv[]) exit(1); } // check that the parameter ends with a number M or G - if (isdigit(argv[i].back()) || argv[i].back() == 'M' || argv[i].bach() == 'G') { + if (isdigit(argv[i].rbegin()) || argv[i].rbegin() == 'M' || argv[i].rbegin() == 'G') { numBlocksPctParam = argv[i]; } else { @@ -510,7 +510,7 @@ int main(int argc, char* argv[]) exit(1); } // check that the parameter ends with a %, M, or G - if (argv[i].back() == '%' || argv[i].back() == 'M' || argv[i].bach() == 'G') { + if (argv[i].rbegin() == '%' || argv[i].rbegin() == 'M' || argv[i].rbegin() == 'G') { totalUmMemoryParam = argv[i]; } else { @@ -1705,7 +1705,7 @@ int main(int argc, char* argv[]) { sysConfig->setConfig("DBBC", "NumBlocksPct", numBlocksPct); - if (numBlocksPct.back() == 'M' || numBlocksPct.back() == 'G') { + if (numBlocksPct.rbegin() == 'M' || numBlocksPct.rbegin() == 'G') { cout << endl << "NOTE: Setting 'NumBlocksPct' to " << numBlocksPct << endl; } else { @@ -1760,7 +1760,7 @@ int main(int argc, char* argv[]) if (!numBlocksPctParam.empty()) { // if numBlocksPct was set as command line parameter use the command line parameter value sysConfig->setConfig("DBBC", "NumBlocksPct", numBlocksPctParam); - if (numBlocksPctParam.back() == 'M' || numBlocksPctParam.back() == 'G') { + if (numBlocksPctParam.rbegin() == 'M' || numBlocksPctParam.rbegin() == 'G') { cout << endl << "NOTE: Setting 'NumBlocksPct' to " << numBlocksPctParam << endl; } else { @@ -1827,7 +1827,7 @@ int main(int argc, char* argv[]) try { sysConfig->setConfig("DBBC", "NumBlocksPct", numBlocksPct); - if (numBlocksPct.back() == 'M' || numBlocksPct.back() == 'G') { + if (numBlocksPct.rbegin() == 'M' || numBlocksPct.rbegin() == 'G') { cout << endl << "NOTE: Setting 'NumBlocksPct' to " << numBlocksPct << endl; } else { @@ -1878,7 +1878,7 @@ int main(int argc, char* argv[]) { if (!numBlocksPctParam.empty()) { // if numBlocksPct was set as command line parameter use the command line parameter value sysConfig->setConfig("DBBC", "NumBlocksPct", numBlocksPctParam); - if (numBlocksPctParam.back() == 'M' || numBlocksPctParam.back() == 'G') { + if (numBlocksPctParam.rbegin() == 'M' || numBlocksPctParam.rbegin() == 'G') { cout << endl << "NOTE: Setting 'NumBlocksPct' to " << numBlocksPctParam << endl; } else { From 0689ec1b5bff76b3b768ab3a936d6129ad97932f Mon Sep 17 00:00:00 2001 From: sneJ- Date: Wed, 16 Jan 2019 15:43:57 +0100 Subject: [PATCH 35/55] MCOL-2068 maybe the pointer helps --- oamapps/postConfigure/postConfigure.cpp | 12 ++++++------ 1 file changed, 6 insertions(+), 6 deletions(-) diff --git a/oamapps/postConfigure/postConfigure.cpp b/oamapps/postConfigure/postConfigure.cpp index 9453560c6..9f9391802 100644 --- a/oamapps/postConfigure/postConfigure.cpp +++ b/oamapps/postConfigure/postConfigure.cpp @@ -493,7 +493,7 @@ int main(int argc, char* argv[]) exit(1); } // check that the parameter ends with a number M or G - if (isdigit(argv[i].rbegin()) || argv[i].rbegin() == 'M' || argv[i].rbegin() == 'G') { + if (isdigit(*argv[i].rbegin()) || *argv[i].rbegin() == 'M' || *argv[i].rbegin() == 'G') { numBlocksPctParam = argv[i]; } else { @@ -510,7 +510,7 @@ int main(int argc, char* argv[]) exit(1); } // check that the parameter ends with a %, M, or G - if (argv[i].rbegin() == '%' || argv[i].rbegin() == 'M' || argv[i].rbegin() == 'G') { + if (*argv[i].rbegin() == '%' || *argv[i].rbegin() == 'M' || *argv[i].rbegin() == 'G') { totalUmMemoryParam = argv[i]; } else { @@ -1705,7 +1705,7 @@ int main(int argc, char* argv[]) { sysConfig->setConfig("DBBC", "NumBlocksPct", numBlocksPct); - if (numBlocksPct.rbegin() == 'M' || numBlocksPct.rbegin() == 'G') { + if (*numBlocksPct.rbegin() == 'M' || *numBlocksPct.rbegin() == 'G') { cout << endl << "NOTE: Setting 'NumBlocksPct' to " << numBlocksPct << endl; } else { @@ -1760,7 +1760,7 @@ int main(int argc, char* argv[]) if (!numBlocksPctParam.empty()) { // if numBlocksPct was set as command line parameter use the command line parameter value sysConfig->setConfig("DBBC", "NumBlocksPct", numBlocksPctParam); - if (numBlocksPctParam.rbegin() == 'M' || numBlocksPctParam.rbegin() == 'G') { + if (*numBlocksPctParam.rbegin() == 'M' || *numBlocksPctParam.rbegin() == 'G') { cout << endl << "NOTE: Setting 'NumBlocksPct' to " << numBlocksPctParam << endl; } else { @@ -1827,7 +1827,7 @@ int main(int argc, char* argv[]) try { sysConfig->setConfig("DBBC", "NumBlocksPct", numBlocksPct); - if (numBlocksPct.rbegin() == 'M' || numBlocksPct.rbegin() == 'G') { + if (*numBlocksPct.rbegin() == 'M' || *numBlocksPct.rbegin() == 'G') { cout << endl << "NOTE: Setting 'NumBlocksPct' to " << numBlocksPct << endl; } else { @@ -1878,7 +1878,7 @@ int main(int argc, char* argv[]) { if (!numBlocksPctParam.empty()) { // if numBlocksPct was set as command line parameter use the command line parameter value sysConfig->setConfig("DBBC", "NumBlocksPct", numBlocksPctParam); - if (numBlocksPctParam.rbegin() == 'M' || numBlocksPctParam.rbegin() == 'G') { + if (*numBlocksPctParam.rbegin() == 'M' || *numBlocksPctParam.rbegin() == 'G') { cout << endl << "NOTE: Setting 'NumBlocksPct' to " << numBlocksPctParam << endl; } else { From fdc9f558406143ddca0806e8486991692a9d6671 Mon Sep 17 00:00:00 2001 From: sneJ- Date: Wed, 16 Jan 2019 16:15:32 +0100 Subject: [PATCH 36/55] MCOL-2068 - next try --- oamapps/postConfigure/postConfigure.cpp | 12 ++++-------- 1 file changed, 4 insertions(+), 8 deletions(-) diff --git a/oamapps/postConfigure/postConfigure.cpp b/oamapps/postConfigure/postConfigure.cpp index 9f9391802..e51f3c474 100644 --- a/oamapps/postConfigure/postConfigure.cpp +++ b/oamapps/postConfigure/postConfigure.cpp @@ -492,11 +492,9 @@ int main(int argc, char* argv[]) cout << " ERROR: Memory settings for numBlocksPct not provided" << endl; exit(1); } + numBlocksPctParam = argv[i]; // check that the parameter ends with a number M or G - if (isdigit(*argv[i].rbegin()) || *argv[i].rbegin() == 'M' || *argv[i].rbegin() == 'G') { - numBlocksPctParam = argv[i]; - } - else { + if (!(isdigit(*numBlocksPctParam.rbegin()) || *numBlocksPctParam.rbegin() == 'M' || *numBlocksPctParam.rbegin() == 'G')) { cout << " ERROR: Memory settings for numBlocksPct need to end on a digit, M or G" << endl; exit(1); } @@ -509,11 +507,9 @@ int main(int argc, char* argv[]) cout << " ERROR: Memory settings for totalUmMemory not provided" << endl; exit(1); } + totalUmMemoryParam = argv[i]; // check that the parameter ends with a %, M, or G - if (*argv[i].rbegin() == '%' || *argv[i].rbegin() == 'M' || *argv[i].rbegin() == 'G') { - totalUmMemoryParam = argv[i]; - } - else { + if (!(*totalUmMemoryParam.rbegin() == '%' || *totalUmMemoryParam.rbegin() == 'M' || *totalUmMemoryParam.rbegin() == 'G')) { cout << " ERROR: Memory settings for totalUmMemory need to end on %, M or G" << endl; exit(1); } From 009f70b4d4466b6a5e192747dd217fd06fd78c05 Mon Sep 17 00:00:00 2001 From: Ben Thompson Date: Wed, 16 Jan 2019 12:58:18 -0600 Subject: [PATCH 37/55] MCOL-2009 MCOL-1702 : join() inside of joblist::abort caused threads to be blocked on an abort --- dbcon/joblist/joblist.cpp | 4 ---- 1 file changed, 4 deletions(-) diff --git a/dbcon/joblist/joblist.cpp b/dbcon/joblist/joblist.cpp index 934dae71d..e5eb86b26 100644 --- a/dbcon/joblist/joblist.cpp +++ b/dbcon/joblist/joblist.cpp @@ -1067,10 +1067,6 @@ void JobList::abort() fQuery[i]->abort(); for (i = 0; i < fProject.size(); i++) fProject[i]->abort(); - for (i = 0; i < fQuery.size(); i++) - fQuery[i]->join(); - for (i = 0; i < fProject.size(); i++) - fProject[i]->join(); } } From 83f493eef391acabe2b5aadf79941e96c9fc7ecc Mon Sep 17 00:00:00 2001 From: sneJ- Date: Wed, 16 Jan 2019 20:14:56 +0100 Subject: [PATCH 38/55] MCOL-2068 - added the memory command line parameters to the installer as well to support single type installations --- oamapps/postConfigure/installer.cpp | 189 ++++++++++++++++++------ oamapps/postConfigure/postConfigure.cpp | 14 +- 2 files changed, 153 insertions(+), 50 deletions(-) diff --git a/oamapps/postConfigure/installer.cpp b/oamapps/postConfigure/installer.cpp index 94850deae..1bc52f173 100644 --- a/oamapps/postConfigure/installer.cpp +++ b/oamapps/postConfigure/installer.cpp @@ -126,10 +126,22 @@ int main(int argc, char* argv[]) mysqlpw = argv[10]; installer_debug = argv[11]; - if (argc >= 13) + string totalBlocksPctParam = ""; + string totalUmMemoryParam = ""; + if (argc >= 13) { installDir = argv[12]; - else + if (argc >= 15) { + if (!string(argv[13]).compare("-")) { + totalBlocksPctParam = argv[13]; + } + if (!string(argv[14]).compare("-")) { + totalUmMemoryParam = argv[14]; + } + } + } + else { installDir = "/usr/local/mariadb/columnstore"; + } ofstream file("/dev/null"); @@ -151,6 +163,12 @@ int main(int argc, char* argv[]) cout << mysqlpw << endl; cout << installer_debug << endl; cout << installDir << endl; + if (!totalBlocksPctParam.empty()) { + cout << totalBlocksPctParam << endl; + } + if (!totalUmMemoryParam.empty()) { + cout << totalUmMemoryParam << endl; + } // restore cout stream buffer cout.rdbuf (strm_buffer); @@ -255,16 +273,26 @@ int main(int argc, char* argv[]) return false; } - string numBlocksPct = "50"; - - if ( DBRootStorageType == "hdfs") - numBlocksPct = "25"; + string numBlocksPct; + if (numBlocksPctParam.empty()) { + numBlocksPct = "50"; + if (DBRootStorageType == "hdfs") + numBlocksPct = "25"; + } + else { + numBlocksPct = numBlocksPctParam; + } try { sysConfig->setConfig("DBBC", "NumBlocksPct", numBlocksPct); - cout << endl << "NOTE: Setting 'NumBlocksPct' to " << numBlocksPct << "%" << endl; + if (*numBlocksPct.rbegin() == 'M' || *numBlocksPct.rbegin() == 'G') { + cout << endl << "NOTE: Setting 'NumBlocksPct' to " << numBlocksPct << endl; + } + else { + cout << endl << "NOTE: Setting 'NumBlocksPct' to " << numBlocksPct << "%" << endl; + } } catch (...) { @@ -282,15 +310,21 @@ int main(int argc, char* argv[]) // long long total = myinfo.totalram / 1024 / 1000; // adjust max memory, 25% of total memory - string percent = "25%"; + string percent; + if (totalUmMemoryParam.empty()) { + percent = "25%"; - if ( DBRootStorageType == "hdfs") - { - percent = "12%"; + if (DBRootStorageType == "hdfs") + { + percent = "12%"; + } + + cout << " Setting 'TotalUmMemory' to " << percent << " of total memory. " << endl; + } + else { + percent = totalUmMemoryParam; + cout << " Setting 'TotalUmMemory' to " << percent << endl; } - - cout << " Setting 'TotalUmMemory' to " << percent << " of total memory. " << endl; - try { sysConfig->setConfig("HashJoin", "TotalUmMemory", percent); @@ -305,29 +339,47 @@ int main(int argc, char* argv[]) { try { - string numBlocksPct = sysConfig->getConfig("DBBC", "NumBlocksPct"); - string totalUmMemory = sysConfig->getConfig("HashJoin", "TotalUmMemory"); + string numBlocksPct; + string totalUmMemory; - if (numBlocksPct.empty() || numBlocksPct == "" ) - { - numBlocksPct = "50"; - - try - { - sysConfig->setConfig("DBBC", "NumBlocksPct", "50"); - - cout << endl << "NOTE: Setting 'NumBlocksPct' to 50%" << endl; - } - catch (...) - { - cout << "ERROR: Problem setting NumBlocksPct in the MariaDB ColumnStore System Configuration file" << endl; - exit(1); - } + if (!numBlocksPctParam.empty()) { + numBlocksPct = numBlocksPctParam; + } + else { + numBlocksPct = sysConfig->getConfig("DBBC", "NumBlocksPct"); + } + if (!totalUmMemoryParam.empty()) { + totalUmMemory = totalUmMemoryParam; + } + else { + totalUmMemory = sysConfig->getConfig("HashJoin", "TotalUmMemory"); } - cout << endl << "NOTE: Using previous configuration setting for 'NumBlocksPct' = " << numBlocksPct << "%" << endl; - cout << " Using previous configuration setting for 'TotalUmMemory' = " << totalUmMemory << endl; + if (numBlocksPct.empty() || numBlocksPct == "") + { + numBlocksPct = "50"; + } + if (totalUmMemory.empty() || totalUmMemory == "") { + totalUmMemory = "25%"; + } + try + { + sysConfig->setConfig("DBBC", "NumBlocksPct", numBlocksPct); + sysConfig->setConfig("HashJoin", "TotalUmMemory", totalUmMemory); + if (*numBlocksPct.rbegin() == 'M' || *numBlocksPct.rbegin() == 'G') { + cout << endl << "NOTE: Setting 'NumBlocksPct' to " << numBlocksPct << endl; + } + else { + cout << endl << "NOTE: Setting 'NumBlocksPct' to " << numBlocksPct << "%" << endl; + } + cout << " Setting 'TotalUmMemory' to " << totalUmMemory << endl; + } + catch (...) + { + cout << "ERROR: Problem setting NumBlocksPct/TotalUmMemory in the MariaDB ColumnStore System Configuration file" << endl; + exit(1); + } } catch (...) { @@ -350,7 +402,23 @@ int main(int argc, char* argv[]) // are we using settings from previous config file? if ( reuseConfig == "n" ) { - cout << endl << "NOTE: Using the default setting for 'NumBlocksPct' at 70%" << endl; + string numBlocksPct = "70" + string totalUmMemory = "50%"; + + if (!numBlocksPctParam.empty()) { + numBlocksPct = numBlocksPctParam; + cout << endl << "NOTE: Setting 'NumBlocksPct' to " << numBlocksPct << endl; + } + else { + cout << endl << "NOTE: Using the default setting for 'NumBlocksPct' at " << numBlocksPct << "%" << endl; + } + if (!totalUmMemoryParam.empty()) { + totalUmMemory = totalUmMemoryParam; + cout << endl << "Setting 'TotalUmMemory' to " << totalUmMemory << endl; + } + else { + cout << endl << "Setting 'TotalUmMemory' to " << totalUmMemory << " of total memory." << endl; + } try { @@ -358,17 +426,14 @@ int main(int argc, char* argv[]) } catch (...) {} - // adjust max memory, 50% of total memory - - cout << endl << "Setting 'TotalUmMemory' to 50% of total memory." << endl; - try { - sysConfig->setConfig("HashJoin", "TotalUmMemory", "50%"); + sysConfig->setConfig("DBBC", "NumBlocksPct", numBlocksPct); + sysConfig->setConfig("HashJoin", "TotalUmMemory", totalUmMemory); } catch (...) { - cout << "ERROR: Problem setting TotalUmMemory in the MariaDB ColumnStore System Configuration file" << endl; + cout << "ERROR: Problem setting TotalUmMemory/NumBlocksPct in the MariaDB ColumnStore System Configuration file" << endl; exit(1); } } @@ -376,15 +441,47 @@ int main(int argc, char* argv[]) { try { - string numBlocksPct = sysConfig->getConfig("DBBC", "NumBlocksPct"); - string totalUmMemory = sysConfig->getConfig("HashJoin", "TotalUmMemory"); + string numBlocksPct; + string totalUmMemory; - if (numBlocksPct.empty() || numBlocksPct == "" ) + if (!numBlocksPctParam.empty()) { + numBlocksPct = numBlocksPctParam; + } + else { + numBlocksPct = sysConfig->getConfig("DBBC", "NumBlocksPct"); + } + if (!totalUmMemoryParam.empty()) { + totalUmMemory = totalUmMemoryParam; + } + else { + totalUmMemory = sysConfig->getConfig("HashJoin", "TotalUmMemory"); + } + + if (numBlocksPct.empty() || numBlocksPct == "") + { numBlocksPct = "70"; + } + if (totalUmMemory.empty() || totalUmMemory == "") { + totalUmMemory = "50%"; + } + try + { + sysConfig->setConfig("DBBC", "NumBlocksPct", numBlocksPct); + sysConfig->setConfig("HashJoin", "TotalUmMemory", totalUmMemory); - cout << endl << "NOTE: Using previous configuration setting for 'NumBlocksPct' = " << numBlocksPct << "%" << endl; - cout << " Using previous configuration setting for 'TotalUmMemory' = " << totalUmMemory << endl; - + if (*numBlocksPct.rbegin() == 'M' || *numBlocksPct.rbegin() == 'G') { + cout << endl << "NOTE: Setting 'NumBlocksPct' to " << numBlocksPct << endl; + } + else { + cout << endl << "NOTE: Setting 'NumBlocksPct' to " << numBlocksPct << "%" << endl; + } + cout << " Setting 'TotalUmMemory' to " << totalUmMemory << endl; + } + catch (...) + { + cout << "ERROR: Problem setting NumBlocksPct/TotalUmMemory in the MariaDB ColumnStore System Configuration file" << endl; + exit(1); + } } catch (...) { diff --git a/oamapps/postConfigure/postConfigure.cpp b/oamapps/postConfigure/postConfigure.cpp index e51f3c474..8796e9b5f 100644 --- a/oamapps/postConfigure/postConfigure.cpp +++ b/oamapps/postConfigure/postConfigure.cpp @@ -333,9 +333,9 @@ int main(int argc, char* argv[]) cout << " -um-ip-addrs User Module IP Addresses xxx.xxx.xxx.xxx,xxx.xxx.xxx.xxx" << endl; cout << " -x Do not resolve IP Addresses from host names" << endl; cout << " -numBlocksPct amount of physical memory to utilize for disk block caching" << endl; - cout << " (percentages of the total memory need to be stated without suffix, explcit values with suffixes M or G)"; + cout << " (percentages of the total memory need to be stated without suffix, explcit values with suffixes M or G)" << endl; cout << " -totalUmMemory amount of physical memory to utilize for joins, intermediate results and set operations on the UM" << endl; - cout << " (percentages of the total memory need to be stated with suffix %, explcit values with suffixes M or G)"; + cout << " (percentages of the total memory need to be stated with suffix %, explcit values with suffixes M or G)" << endl; exit (0); } else if (string("-x") == argv[i]) @@ -994,7 +994,6 @@ int main(int argc, char* argv[]) offLineAppCheck(); checkMysqlPort(mysqlPort, sysConfig); - if ( !writeConfig(sysConfig) ) { cout << "ERROR: Failed trying to update MariaDB ColumnStore System Configuration file" << endl; @@ -1003,7 +1002,14 @@ int main(int argc, char* argv[]) cout << endl << "===== Performing Configuration Setup and MariaDB ColumnStore Startup =====" << endl; - cmd = installDir + "/bin/installer dummy.rpm dummy.rpm dummy.rpm dummy.rpm dummy.rpm initial dummy " + reuseConfig + " --nodeps ' ' 1 " + installDir; + if (numBlocksPctParam.empty()) { + numBlocksPctParam = "-"; + } + if (totalUmMemoryParam.empty()) { + totalUmMemoryParam = "-"; + } + + cmd = installDir + "/bin/installer dummy.rpm dummy.rpm dummy.rpm dummy.rpm dummy.rpm initial dummy " + reuseConfig + " --nodeps ' ' 1 " + installDir + " " + totalBlocksPctParam + " " + totalUmMemoryParam; system(cmd.c_str()); exit(0); } From 2c5b65294c5846c6125fbcca8f3f699c2c38bf98 Mon Sep 17 00:00:00 2001 From: sneJ- Date: Wed, 16 Jan 2019 20:45:14 +0100 Subject: [PATCH 39/55] MCOL-2068 typo fixed --- oamapps/postConfigure/installer.cpp | 10 +++++----- 1 file changed, 5 insertions(+), 5 deletions(-) diff --git a/oamapps/postConfigure/installer.cpp b/oamapps/postConfigure/installer.cpp index 1bc52f173..428911ebe 100644 --- a/oamapps/postConfigure/installer.cpp +++ b/oamapps/postConfigure/installer.cpp @@ -126,13 +126,13 @@ int main(int argc, char* argv[]) mysqlpw = argv[10]; installer_debug = argv[11]; - string totalBlocksPctParam = ""; + string numBlocksPctParam = ""; string totalUmMemoryParam = ""; if (argc >= 13) { installDir = argv[12]; if (argc >= 15) { if (!string(argv[13]).compare("-")) { - totalBlocksPctParam = argv[13]; + numBlocksPctParam = argv[13]; } if (!string(argv[14]).compare("-")) { totalUmMemoryParam = argv[14]; @@ -163,8 +163,8 @@ int main(int argc, char* argv[]) cout << mysqlpw << endl; cout << installer_debug << endl; cout << installDir << endl; - if (!totalBlocksPctParam.empty()) { - cout << totalBlocksPctParam << endl; + if (!numBlocksPctParam.empty()) { + cout << numBlocksPctParam << endl; } if (!totalUmMemoryParam.empty()) { cout << totalUmMemoryParam << endl; @@ -402,7 +402,7 @@ int main(int argc, char* argv[]) // are we using settings from previous config file? if ( reuseConfig == "n" ) { - string numBlocksPct = "70" + string numBlocksPct = "70"; string totalUmMemory = "50%"; if (!numBlocksPctParam.empty()) { From 573a740e8aa5cf352b5353f79363f7072d0fc938 Mon Sep 17 00:00:00 2001 From: sneJ- Date: Wed, 16 Jan 2019 21:08:12 +0100 Subject: [PATCH 40/55] MCOL-2068 typo --- oamapps/postConfigure/postConfigure.cpp | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/oamapps/postConfigure/postConfigure.cpp b/oamapps/postConfigure/postConfigure.cpp index 8796e9b5f..c92697127 100644 --- a/oamapps/postConfigure/postConfigure.cpp +++ b/oamapps/postConfigure/postConfigure.cpp @@ -1009,7 +1009,7 @@ int main(int argc, char* argv[]) totalUmMemoryParam = "-"; } - cmd = installDir + "/bin/installer dummy.rpm dummy.rpm dummy.rpm dummy.rpm dummy.rpm initial dummy " + reuseConfig + " --nodeps ' ' 1 " + installDir + " " + totalBlocksPctParam + " " + totalUmMemoryParam; + cmd = installDir + "/bin/installer dummy.rpm dummy.rpm dummy.rpm dummy.rpm dummy.rpm initial dummy " + reuseConfig + " --nodeps ' ' 1 " + installDir + " " + numBlocksPctParam + " " + totalUmMemoryParam; system(cmd.c_str()); exit(0); } From 17a446c37cca96c71c6abc924a8b00de73554e88 Mon Sep 17 00:00:00 2001 From: sneJ- Date: Thu, 17 Jan 2019 12:23:59 +0100 Subject: [PATCH 41/55] MCOL-2068 - string comparison fixed --- oamapps/postConfigure/installer.cpp | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/oamapps/postConfigure/installer.cpp b/oamapps/postConfigure/installer.cpp index 428911ebe..81b69e548 100644 --- a/oamapps/postConfigure/installer.cpp +++ b/oamapps/postConfigure/installer.cpp @@ -131,10 +131,10 @@ int main(int argc, char* argv[]) if (argc >= 13) { installDir = argv[12]; if (argc >= 15) { - if (!string(argv[13]).compare("-")) { + if (string(argv[13]) != "-") { numBlocksPctParam = argv[13]; } - if (!string(argv[14]).compare("-")) { + if (string(argv[14]) != "-") { totalUmMemoryParam = argv[14]; } } From 16a6a0a27c87fbe3cde9fb25c09c3b1761db3bf3 Mon Sep 17 00:00:00 2001 From: David Hall Date: Mon, 21 Jan 2019 14:39:51 -0600 Subject: [PATCH 42/55] MCOL-1983 Use long double for internal calculations of regr_*** functions. --- utils/regr/corr.cpp | 34 +++++++++++++++++----------------- utils/regr/covar_pop.cpp | 16 ++++++++-------- utils/regr/covar_samp.cpp | 16 ++++++++-------- utils/regr/regr_avgx.cpp | 4 ++-- utils/regr/regr_avgy.cpp | 4 ++-- utils/regr/regr_intercept.cpp | 22 +++++++++++----------- utils/regr/regr_r2.cpp | 32 ++++++++++++++++---------------- utils/regr/regr_slope.cpp | 24 ++++++++++++------------ utils/regr/regr_sxx.cpp | 12 ++++++------ utils/regr/regr_sxy.cpp | 22 +++++++++++----------- utils/regr/regr_syy.cpp | 14 +++++++------- 11 files changed, 100 insertions(+), 100 deletions(-) diff --git a/utils/regr/corr.cpp b/utils/regr/corr.cpp index 13efbe897..c1c388da9 100644 --- a/utils/regr/corr.cpp +++ b/utils/regr/corr.cpp @@ -39,11 +39,11 @@ static Add_corr_ToUDAFMap addToMap; struct corr_data { uint64_t cnt; - double sumx; - double sumx2; // sum of (x squared) - double sumy; - double sumy2; // sum of (y squared) - double sumxy; // sum of x * y + long double sumx; + long double sumx2; // sum of (x squared) + long double sumy; + long double sumy2; // sum of (y squared) + long double sumxy; // sum of x * y }; @@ -153,29 +153,29 @@ mcsv1_UDAF::ReturnCode corr::evaluate(mcsv1Context* context, static_any::any& va double N = data->cnt; if (N > 1) { - double sumx = data->sumx; - double sumy = data->sumy; - double sumx2 = data->sumx2; - double sumy2 = data->sumy2; - double sumxy = data->sumxy; + long double sumx = data->sumx; + long double sumy = data->sumy; + long double sumx2 = data->sumx2; + long double sumy2 = data->sumy2; + long double sumxy = data->sumxy; - double var_popx = (sumx2 - (sumx * sumx / N)) / N; + long double var_popx = (sumx2 - (sumx * sumx / N)) / N; if (var_popx == 0) { // When var_popx is 0, NULL is the result. return mcsv1_UDAF::SUCCESS; } - double var_popy = (sumy2 - (sumy * sumy / N)) / N; + long double var_popy = (sumy2 - (sumy * sumy / N)) / N; if (var_popy == 0) { // When var_popy is 0, NULL is the result return mcsv1_UDAF::SUCCESS; } - double std_popx = sqrt(var_popx); - double std_popy = sqrt(var_popy); - double covar_pop = (sumxy - ((sumx * sumy) / N)) / N; - double corr = covar_pop / (std_popy * std_popx); - valOut = corr; + long double std_popx = sqrt(var_popx); + long double std_popy = sqrt(var_popy); + long double covar_pop = (sumxy - ((sumx * sumy) / N)) / N; + long double corr = covar_pop / (std_popy * std_popx); + valOut = static_cast(corr); } return mcsv1_UDAF::SUCCESS; } diff --git a/utils/regr/covar_pop.cpp b/utils/regr/covar_pop.cpp index 6313e49d2..876be1f30 100644 --- a/utils/regr/covar_pop.cpp +++ b/utils/regr/covar_pop.cpp @@ -39,9 +39,9 @@ static Add_covar_pop_ToUDAFMap addToMap; struct covar_pop_data { uint64_t cnt; - double sumx; - double sumy; - double sumxy; // sum of x * y + long double sumx; + long double sumy; + long double sumxy; // sum of x * y }; @@ -145,12 +145,12 @@ mcsv1_UDAF::ReturnCode covar_pop::evaluate(mcsv1Context* context, static_any::an double N = data->cnt; if (N > 0) { - double sumx = data->sumx; - double sumy = data->sumy; - double sumxy = data->sumxy; + long double sumx = data->sumx; + long double sumy = data->sumy; + long double sumxy = data->sumxy; - double covar_pop = (sumxy - ((sumx * sumy) / N)) / N ; - valOut = covar_pop; + long double covar_pop = (sumxy - ((sumx * sumy) / N)) / N ; + valOut = static_cast(covar_pop); } return mcsv1_UDAF::SUCCESS; } diff --git a/utils/regr/covar_samp.cpp b/utils/regr/covar_samp.cpp index 0a79ff31a..ccc302046 100644 --- a/utils/regr/covar_samp.cpp +++ b/utils/regr/covar_samp.cpp @@ -39,9 +39,9 @@ static Add_covar_samp_ToUDAFMap addToMap; struct covar_samp_data { uint64_t cnt; - double sumx; - double sumy; - double sumxy; // sum of x * y + long double sumx; + long double sumy; + long double sumxy; // sum of x * y }; @@ -145,12 +145,12 @@ mcsv1_UDAF::ReturnCode covar_samp::evaluate(mcsv1Context* context, static_any::a double N = data->cnt; if (N > 1) { - double sumx = data->sumx; - double sumy = data->sumy; - double sumxy = data->sumxy; + long double sumx = data->sumx; + long double sumy = data->sumy; + long double sumxy = data->sumxy; - double covar_samp = (sumxy - ((sumx * sumy) / N)) / (N - 1); - valOut = covar_samp; + long double covar_samp = (sumxy - ((sumx * sumy) / N)) / (N - 1); + valOut = static_cast(covar_samp); } else if (N == 1) diff --git a/utils/regr/regr_avgx.cpp b/utils/regr/regr_avgx.cpp index e601b4172..bf010e648 100644 --- a/utils/regr/regr_avgx.cpp +++ b/utils/regr/regr_avgx.cpp @@ -40,7 +40,7 @@ static Add_regr_avgx_ToUDAFMap addToMap; // Use the simple data model struct regr_avgx_data { - double sum; + long double sum; uint64_t cnt; }; @@ -132,7 +132,7 @@ mcsv1_UDAF::ReturnCode regr_avgx::evaluate(mcsv1Context* context, static_any::an if (data->cnt > 0) { - valOut = data->sum / (double)data->cnt; + valOut = static_cast(data->sum / (long double)data->cnt); } return mcsv1_UDAF::SUCCESS; diff --git a/utils/regr/regr_avgy.cpp b/utils/regr/regr_avgy.cpp index 0d0cc58dd..7325d991f 100644 --- a/utils/regr/regr_avgy.cpp +++ b/utils/regr/regr_avgy.cpp @@ -40,7 +40,7 @@ static Add_regr_avgy_ToUDAFMap addToMap; // Use the simple data model struct regr_avgy_data { - double sum; + long double sum; uint64_t cnt; }; @@ -130,7 +130,7 @@ mcsv1_UDAF::ReturnCode regr_avgy::evaluate(mcsv1Context* context, static_any::an if (data->cnt > 0) { - valOut = data->sum / (double)data->cnt; + valOut = static_cast(data->sum / (long double)data->cnt); } return mcsv1_UDAF::SUCCESS; } diff --git a/utils/regr/regr_intercept.cpp b/utils/regr/regr_intercept.cpp index d46d3117c..df9310f03 100644 --- a/utils/regr/regr_intercept.cpp +++ b/utils/regr/regr_intercept.cpp @@ -39,10 +39,10 @@ static Add_regr_intercept_ToUDAFMap addToMap; struct regr_intercept_data { uint64_t cnt; - double sumx; - double sumx2; // sum of (x squared) - double sumy; - double sumxy; // sum of (x*y) + long double sumx; + long double sumx2; // sum of (x squared) + long double sumy; + long double sumxy; // sum of x * y }; @@ -148,15 +148,15 @@ mcsv1_UDAF::ReturnCode regr_intercept::evaluate(mcsv1Context* context, static_an double N = data->cnt; if (N > 1) { - double sumx = data->sumx; - double sumy = data->sumy; - double sumx2 = data->sumx2; - double sumxy = data->sumxy; - double numerator = sumy * sumx2 - sumx * sumxy; - double var_pop = (N * sumx2) - (sumx * sumx); + long double sumx = data->sumx; + long double sumy = data->sumy; + long double sumx2 = data->sumx2; + long double sumxy = data->sumxy; + long double numerator = sumy * sumx2 - sumx * sumxy; + long double var_pop = (N * sumx2) - (sumx * sumx); if (var_pop != 0) { - valOut = numerator / var_pop; + valOut = static_cast(numerator / var_pop); } } return mcsv1_UDAF::SUCCESS; diff --git a/utils/regr/regr_r2.cpp b/utils/regr/regr_r2.cpp index f159d834a..1abd3ea2e 100644 --- a/utils/regr/regr_r2.cpp +++ b/utils/regr/regr_r2.cpp @@ -39,11 +39,11 @@ static Add_regr_r2_ToUDAFMap addToMap; struct regr_r2_data { uint64_t cnt; - double sumx; - double sumx2; // sum of (x squared) - double sumy; - double sumy2; // sum of (y squared) - double sumxy; // sum of x * y + long double sumx; + long double sumx2; // sum of (x squared) + long double sumy; + long double sumy2; // sum of (y squared) + long double sumxy; // sum of x * y }; @@ -153,13 +153,13 @@ mcsv1_UDAF::ReturnCode regr_r2::evaluate(mcsv1Context* context, static_any::any& double N = data->cnt; if (N > 1) { - double sumx = data->sumx; - double sumy = data->sumy; - double sumx2 = data->sumx2; - double sumy2 = data->sumy2; - double sumxy = data->sumxy; + long double sumx = data->sumx; + long double sumy = data->sumy; + long double sumx2 = data->sumx2; + long double sumy2 = data->sumy2; + long double sumxy = data->sumxy; - double var_popx = (sumx2 - (sumx * sumx / N)) / N; + long double var_popx = (sumx2 - (sumx * sumx / N)) / N; if (var_popx == 0) { // When var_popx is 0, NULL is the result. @@ -172,11 +172,11 @@ mcsv1_UDAF::ReturnCode regr_r2::evaluate(mcsv1Context* context, static_any::any& valOut = 1.0; return mcsv1_UDAF::SUCCESS; } - double std_popx = sqrt(var_popx); - double std_popy = sqrt(var_popy); - double covar_pop = (sumxy - ((sumx * sumy) / N)) / N; - double corr = covar_pop / (std_popy * std_popx); - valOut = corr * corr; + long double std_popx = sqrt(var_popx); + long double std_popy = sqrt(var_popy); + long double covar_pop = (sumxy - ((sumx * sumy) / N)) / N; + long double corr = covar_pop / (std_popy * std_popx); + valOut = static_cast(corr * corr); } return mcsv1_UDAF::SUCCESS; } diff --git a/utils/regr/regr_slope.cpp b/utils/regr/regr_slope.cpp index 6790e223e..de9eab5c7 100644 --- a/utils/regr/regr_slope.cpp +++ b/utils/regr/regr_slope.cpp @@ -39,10 +39,10 @@ static Add_regr_slope_ToUDAFMap addToMap; struct regr_slope_data { uint64_t cnt; - double sumx; - double sumx2; // sum of (x squared) - double sumy; - double sumxy; // sum of (x*y) + long double sumx; + long double sumx2; // sum of (x squared) + long double sumy; + long double sumxy; // sum of x * y }; @@ -148,16 +148,16 @@ mcsv1_UDAF::ReturnCode regr_slope::evaluate(mcsv1Context* context, static_any::a if (N > 1) { // COVAR_POP(y, x) / VAR_POP(x) - double sumx = data->sumx; - double sumy = data->sumy; - double sumx2 = data->sumx2; - double sumxy = data->sumxy; - double covar_pop = N * sumxy - sumx * sumy; - double var_pop = N * sumx2 - sumx * sumx; + long double sumx = data->sumx; + long double sumy = data->sumy; + long double sumx2 = data->sumx2; + long double sumxy = data->sumxy; + long double covar_pop = N * sumxy - sumx * sumy; + long double var_pop = N * sumx2 - sumx * sumx; if (var_pop != 0) { - double slope = covar_pop / var_pop; - valOut = slope; + long double slope = covar_pop / var_pop; + valOut = static_cast(slope); } } return mcsv1_UDAF::SUCCESS; diff --git a/utils/regr/regr_sxx.cpp b/utils/regr/regr_sxx.cpp index b4c467fc3..5769a227b 100644 --- a/utils/regr/regr_sxx.cpp +++ b/utils/regr/regr_sxx.cpp @@ -39,8 +39,8 @@ static Add_regr_sxx_ToUDAFMap addToMap; struct regr_sxx_data { uint64_t cnt; - double sumx; - double sumx2; // sum of (x squared) + long double sumx; + long double sumx2; // sum of (x squared) }; @@ -128,11 +128,11 @@ mcsv1_UDAF::ReturnCode regr_sxx::evaluate(mcsv1Context* context, static_any::any double N = data->cnt; if (N > 0) { - double sumx = data->sumx; - double sumx2 = data->sumx2; + long double sumx = data->sumx; + long double sumx2 = data->sumx2; - double var_popx = (sumx2 - (sumx * sumx / N)) / N; - valOut = data->cnt * var_popx; + long double var_popx = (sumx2 - (sumx * sumx / N)) / N; + valOut = static_cast(data->cnt * var_popx); } return mcsv1_UDAF::SUCCESS; } diff --git a/utils/regr/regr_sxy.cpp b/utils/regr/regr_sxy.cpp index 9cdbffb6a..76e1373c4 100644 --- a/utils/regr/regr_sxy.cpp +++ b/utils/regr/regr_sxy.cpp @@ -39,9 +39,9 @@ static Add_regr_sxy_ToUDAFMap addToMap; struct regr_sxy_data { uint64_t cnt; - double sumx; - double sumy; - double sumxy; // sum of x * y + long double sumx; + long double sumy; + long double sumxy; // sum of x * y }; @@ -88,8 +88,8 @@ mcsv1_UDAF::ReturnCode regr_sxy::nextValue(mcsv1Context* context, ColumnDatum* v static_any::any& valIn_y = valsIn[0].columnData; static_any::any& valIn_x = valsIn[1].columnData; struct regr_sxy_data* data = (struct regr_sxy_data*)context->getUserData()->data; - double valx = 0.0; - double valy = 0.0; + long double valx = 0.0; + long double valy = 0.0; valx = convertAnyTo(valIn_x); valy = convertAnyTo(valIn_y); @@ -145,13 +145,13 @@ mcsv1_UDAF::ReturnCode regr_sxy::evaluate(mcsv1Context* context, static_any::any double N = data->cnt; if (N > 0) { - double sumx = data->sumx; - double sumy = data->sumy; - double sumxy = data->sumxy; + long double sumx = data->sumx; + long double sumy = data->sumy; + long double sumxy = data->sumxy; - double covar_pop = (sumxy - ((sumx * sumy) / N)) / N; - double regr_sxy = data->cnt * covar_pop; - valOut = regr_sxy; + long double covar_pop = (sumxy - ((sumx * sumy) / N)) / N; + long double regr_sxy = data->cnt * covar_pop; + valOut = static_cast(regr_sxy); } return mcsv1_UDAF::SUCCESS; } diff --git a/utils/regr/regr_syy.cpp b/utils/regr/regr_syy.cpp index 98e614a8e..014a28389 100644 --- a/utils/regr/regr_syy.cpp +++ b/utils/regr/regr_syy.cpp @@ -39,8 +39,8 @@ static Add_regr_syy_ToUDAFMap addToMap; struct regr_syy_data { uint64_t cnt; - double sumy; - double sumy2; // sum of (y squared) + long double sumy; + long double sumy2; // sum of (y squared) }; @@ -125,14 +125,14 @@ mcsv1_UDAF::ReturnCode regr_syy::subEvaluate(mcsv1Context* context, const UserDa mcsv1_UDAF::ReturnCode regr_syy::evaluate(mcsv1Context* context, static_any::any& valOut) { struct regr_syy_data* data = (struct regr_syy_data*)context->getUserData()->data; - double N = data->cnt; + long double N = data->cnt; if (N > 0) { - double sumy = data->sumy; - double sumy2 = data->sumy2; + long double sumy = data->sumy; + long double sumy2 = data->sumy2; - double var_popy = (sumy2 - (sumy * sumy / N)) / N; - valOut = data->cnt * var_popy; + long double var_popy = (sumy2 - (sumy * sumy / N)) / N; + valOut = static_cast(data->cnt * var_popy); } return mcsv1_UDAF::SUCCESS; } From 098e49bd01bfc0ae07da480bb092c716f337bd1d Mon Sep 17 00:00:00 2001 From: David Hall Date: Wed, 23 Jan 2019 10:29:12 -0600 Subject: [PATCH 43/55] MCOL-2001 Make the parameters for redistribute case insensitive --- oamapps/mcsadmin/mcsadmin.cpp | 9 +++++---- 1 file changed, 5 insertions(+), 4 deletions(-) diff --git a/oamapps/mcsadmin/mcsadmin.cpp b/oamapps/mcsadmin/mcsadmin.cpp index f2ea97596..8b31353ac 100644 --- a/oamapps/mcsadmin/mcsadmin.cpp +++ b/oamapps/mcsadmin/mcsadmin.cpp @@ -32,6 +32,7 @@ extern int h_errno; #include "boost/filesystem/path.hpp" #include "boost/scoped_ptr.hpp" #include "boost/tokenizer.hpp" +#include "boost/algorithm/string/predicate.hpp" #include "sessionmanager.h" #include "dbrm.h" #include "messagequeue.h" @@ -677,7 +678,7 @@ int processCommand(string* arguments) vector srcDbroots; // all of the currently configured dbroots vector destDbroots; // srcDbroots - removeDbroots set::iterator dbiter; - if (arguments[1] == "start") + if (boost::iequals(arguments[1], "start")) { // Get a list of all the configured dbroots in the xml file. DBRootConfigList dbRootConfigList; @@ -688,7 +689,7 @@ int processCommand(string* arguments) // The user may choose to redistribute in such a way as to // leave certain dbroots empty, presumably for later removal. - if (arguments[2] == "remove") + if (boost::iequals(arguments[2], "remove")) { int dbroot; bool error = false; @@ -792,7 +793,7 @@ int processCommand(string* arguments) SendToWES(oam, bs); } - else if (arguments[1] == "stop") + else if (boost::iequals(arguments[1], "stop")) { ByteStream bs; // message WES ID, sequence #, action id @@ -802,7 +803,7 @@ int processCommand(string* arguments) bs.append((const ByteStream::byte*) &header, sizeof(header)); SendToWES(oam, bs); } - else if (arguments[1] == "status") + else if (boost::iequals(arguments[1], "status")) { ByteStream bs; // message WES ID, sequence #, action id From e12f0fc984b226c775fbe6814dc3f500af1826ca Mon Sep 17 00:00:00 2001 From: David Mott Date: Fri, 25 Jan 2019 06:54:16 -0600 Subject: [PATCH 44/55] fix to MCOL-2110 Cant build engine out-of-source Prefixing SERVER_BUILD_INCLUDE_DIR and SERVER_SOURCE_ROOT_DIR with CMAKE_BINARY_DIR works in the case of the engine and server build folders being siblings to each other and relative paths supplied to cmake. If the build folders are not siblings and absolute paths are supplied to cmake it fails. Using GET_FILENAME_COMPONENT will resolve the correct path in both cases. --- CMakeLists.txt | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) mode change 100755 => 100644 CMakeLists.txt diff --git a/CMakeLists.txt b/CMakeLists.txt old mode 100755 new mode 100644 index c0d708d6e..b8545c295 --- a/CMakeLists.txt +++ b/CMakeLists.txt @@ -172,13 +172,13 @@ SET (ENGINE_WRITE_LIBS ddlpackageproc ddlpackage dmlpackageproc dmlpackage SET (ENGINE_COMMON_LDFLAGS "") IF (SERVER_BUILD_INCLUDE_DIR) - SET (SERVER_BUILD_INCLUDE_DIR ${CMAKE_BINARY_DIR}/${SERVER_BUILD_INCLUDE_DIR}) + GET_FILENAME_COMPONENT (SERVER_BUILD_INCLUDE_DIR ${SERVER_BUILD_INCLUDE_DIR} ABSOLUTE) ELSE() SET (SERVER_BUILD_INCLUDE_DIR ${CMAKE_BINARY_DIR}/../include) ENDIF() IF (SERVER_SOURCE_ROOT_DIR) - SET (SERVER_SOURCE_ROOT_DIR ${CMAKE_BINARY_DIR}/${SERVER_SOURCE_ROOT_DIR}) + GET_FILENAME_COMPONENT (SERVER_SOURCE_ROOT_DIR ${SERVER_SOURCE_ROOT_DIR} ABSOLUTE) ELSE() SET (SERVER_SOURCE_ROOT_DIR ${CMAKE_BINARY_DIR}/..) ENDIF() From 38bd6e1e3c13432dc730565c89ddd9dbf73d2fe3 Mon Sep 17 00:00:00 2001 From: David Mott Date: Fri, 25 Jan 2019 10:33:59 -0600 Subject: [PATCH 45/55] respect relative paths for SERVER_BUILD_INCLUDE_DIR and SERVER_SOURCE_ROOT_DIR --- CMakeLists.txt | 8 ++++++-- 1 file changed, 6 insertions(+), 2 deletions(-) diff --git a/CMakeLists.txt b/CMakeLists.txt index b8545c295..7379cd23c 100644 --- a/CMakeLists.txt +++ b/CMakeLists.txt @@ -172,13 +172,17 @@ SET (ENGINE_WRITE_LIBS ddlpackageproc ddlpackage dmlpackageproc dmlpackage SET (ENGINE_COMMON_LDFLAGS "") IF (SERVER_BUILD_INCLUDE_DIR) - GET_FILENAME_COMPONENT (SERVER_BUILD_INCLUDE_DIR ${SERVER_BUILD_INCLUDE_DIR} ABSOLUTE) + IF (NOT IS_ABSOLUTE ${SERVER_BUILD_INCLUDE_DIR}) + SET (SERVER_BUILD_INCLUDE_DIR ${CMAKE_BINARY_DIR}/${SERVER_BUILD_INCLUDE_DIR}) + ENDIF() ELSE() SET (SERVER_BUILD_INCLUDE_DIR ${CMAKE_BINARY_DIR}/../include) ENDIF() IF (SERVER_SOURCE_ROOT_DIR) - GET_FILENAME_COMPONENT (SERVER_SOURCE_ROOT_DIR ${SERVER_SOURCE_ROOT_DIR} ABSOLUTE) + IF (NOT IS_ABSOLUTE ${SERVER_SOURCE_ROOT_DIR}) + SET (SERVER_SOURCE_ROOT_DIR ${CMAKE_BINARY_DIR}/${SERVER_SOURCE_ROOT_DIR}) + ENDIF() ELSE() SET (SERVER_SOURCE_ROOT_DIR ${CMAKE_BINARY_DIR}/..) ENDIF() From 6d65b138523e04c67ac16b1619086a87852f177b Mon Sep 17 00:00:00 2001 From: Roman Nozdrin Date: Sat, 26 Jan 2019 16:30:02 +0300 Subject: [PATCH 46/55] MCOL-901 Significanlty reduced memory consumption for group_concat(). RowGroup default constructor allocates memory too generously. Removed commented code from groupconcat.cpp. --- dbcon/joblist/groupconcat.cpp | 56 ++++++++--------------------------- utils/rowgroup/rowgroup.cpp | 13 ++++---- 2 files changed, 20 insertions(+), 49 deletions(-) diff --git a/dbcon/joblist/groupconcat.cpp b/dbcon/joblist/groupconcat.cpp index afc91a2ec..11429d053 100644 --- a/dbcon/joblist/groupconcat.cpp +++ b/dbcon/joblist/groupconcat.cpp @@ -344,8 +344,6 @@ void GroupConcatAgUM::merge(const rowgroup::Row& inRow, int64_t i) joblist::GroupConcatAgUM* gccAg = *((joblist::GroupConcatAgUM**)(data + inRow.getOffset(i))); fConcator->merge(gccAg->concator().get()); -// don't reset -// gccAg->orderBy().reset(); } @@ -393,12 +391,11 @@ GroupConcator::~GroupConcator() void GroupConcator::initialize(const rowgroup::SP_GroupConcat& gcc) { + // MCOL-901 This value comes from the Server and it is + // too high(3MB) to allocate it for every instance. fGroupConcatLen = gcc->fSize; fCurrentLength -= strlen(gcc->fSeparator.c_str()); - fOutputString.reset(new uint8_t[fGroupConcatLen + 2]); - memset(fOutputString.get(), 0, fGroupConcatLen + 2); - fConstCols = gcc->fConstCols; fConstantLen = strlen(gcc->fSeparator.c_str()); @@ -477,7 +474,6 @@ void GroupConcator::outputRow(std::ostringstream& oss, const rowgroup::Row& row) case CalpontSystemCatalog::TEXT: { oss << row.getStringField(*i).c_str(); - //oss << row.getStringField(*i); break; } @@ -620,11 +616,6 @@ int64_t GroupConcator::lengthEstimate(const rowgroup::Row& row) { int64_t colWidth = row.getStringLength(*i); fieldLen += colWidth; // getStringLength() does the same thing as below - //assert(!row.usesStringTable()); - //int64_t colWidth = row.getColumnWidth(*i); - //uint8_t* pStr = row.getData() + row.getOffset(*i); - //while ((*pStr++ > 0) && (fieldLen < colWidth)) - // fieldLen++; break; } @@ -752,7 +743,6 @@ void GroupConcatOrderBy::initialize(const rowgroup::SP_GroupConcat& gcc) uint64_t GroupConcatOrderBy::getKeyLength() const { // only distinct the concatenated columns - //return (fRow0.getOffset(fConcatColumns.size()) - 2); return fConcatColumns.size() - 1; // cols 0 to fConcatColumns will be conpared } @@ -771,8 +761,6 @@ void GroupConcatOrderBy::processRow(const rowgroup::Row& row) if (fCurrentLength < fGroupConcatLen) { copyRow(row, &fRow0); - //cout << "length < GB limit: " << fRow0.toString() << endl; - //memcpy(fRow0.getData(), row.getData(), row.getSize()); // the RID is no meaning here, use it to store the estimated length. int16_t estLen = lengthEstimate(fRow0); fRow0.setRid(estLen); @@ -862,7 +850,6 @@ void GroupConcatOrderBy::merge(GroupConcator* gc) if (fDistinct) fDistinctMap->insert(row.fData); - //fDistinctMap->insert(make_pair((row.fData+2), row.fData)); } else if (fOrderByCond.size() > 0 && fRule.less(row.fData, fOrderByQueue.top().fData)) @@ -876,8 +863,6 @@ void GroupConcatOrderBy::merge(GroupConcator* gc) { fDistinctMap->erase(swapRow.fData); fDistinctMap->insert(row.fData); - //fDistinctMap->erase(fDistinctMap->find(swapRow.fData + 2)); - //fDistinctMap->insert(make_pair((row.fData+2), row.fData)); } row1.setData(row.fData); @@ -893,7 +878,6 @@ void GroupConcatOrderBy::merge(GroupConcator* gc) void GroupConcatOrderBy::getResult(uint8_t* buff, const string& sep) { -#if 1 ostringstream oss; bool addSep = false; @@ -919,28 +903,12 @@ void GroupConcatOrderBy::getResult(uint8_t* buff, const string& sep) rowStack.pop(); } - strncpy((char*) buff, oss.str().c_str(), fGroupConcatLen); -#else - ostringstream oss; - bool addSep = false; - priority_queue::reverse_iterator rit; + size_t resultSize = oss.str().size(); + fOutputString.reset(new uint8_t[resultSize + 2]); + memset(fOutputString.get(), 0, resultSize + 2); - for (rit = fOrderByQueue.rbegin(); rit != fOrderByQueue.rend(); ++rit) - { - if (addSep) - oss << sep; - else - addSep = true; - - const OrderByRow& topRow = *rit; - fRow0.setData(topRow.fData); - outputRow(oss, fRow0); - } - - fOrderByQueue.clear(); - - strncpy((char*) buff, oss.str().c_str(), fGroupConcatLen); -#endif + strncpy((char*)fOutputString.get(), + oss.str().c_str(), resultSize); } uint8_t* GroupConcator::getResult(const string& sep) @@ -1010,7 +978,6 @@ void GroupConcatNoOrder::initialize(const rowgroup::SP_GroupConcat& gcc) throw IDBExcept(fErrorCode); } - //fData.reset(new uint8_t[fRowGroup.getDataSize(fRowsPerRG)]); fData.reinit(fRowGroup, fRowsPerRG); fRowGroup.setData(&fData); fRowGroup.resetRowGroup(0); @@ -1025,7 +992,6 @@ void GroupConcatNoOrder::processRow(const rowgroup::Row& row) if (fCurrentLength < fGroupConcatLen && concatColIsNull(row) == false) { copyRow(row, &fRow); - //memcpy(fRow.getData(), row.getData(), row.getSize()); // the RID is no meaning here, use it to store the estimated length. int16_t estLen = lengthEstimate(fRow); @@ -1049,7 +1015,6 @@ void GroupConcatNoOrder::processRow(const rowgroup::Row& row) fDataQueue.push(fData); fData.reinit(fRowGroup, fRowsPerRG); - //fData.reset(new uint8_t[fRowGroup.getDataSize(fRowsPerRG)]); fRowGroup.setData(&fData); fRowGroup.resetRowGroup(0); fRowGroup.getRow(0, &fRow); @@ -1100,7 +1065,12 @@ void GroupConcatNoOrder::getResult(uint8_t* buff, const string& sep) fDataQueue.pop(); } - strncpy((char*) buff, oss.str().c_str(), fGroupConcatLen); + size_t resultSize = oss.str().size(); + fOutputString.reset(new uint8_t[resultSize + 2]); + memset(fOutputString.get(), 0, resultSize + 2); + + strncpy((char*)fOutputString.get(), + oss.str().c_str(), resultSize); } diff --git a/utils/rowgroup/rowgroup.cpp b/utils/rowgroup/rowgroup.cpp index 3411f6fa6..3ecb4998f 100644 --- a/utils/rowgroup/rowgroup.cpp +++ b/utils/rowgroup/rowgroup.cpp @@ -1211,12 +1211,13 @@ int64_t Row::getSignedNullValue(uint32_t colIndex) const RowGroup::RowGroup() : columnCount(0), data(NULL), rgData(NULL), strings(NULL), useStringTable(true), hasLongStringField(false), sTableThreshold(20) { - oldOffsets.reserve(1024); - oids.reserve(1024); - keys.reserve(1024); - types.reserve(1024); - scale.reserve(1024); - precision.reserve(1024); + // 1024 is too generous to waste. + oldOffsets.reserve(10); + oids.reserve(10); + keys.reserve(10); + types.reserve(10); + scale.reserve(10); + precision.reserve(10); } RowGroup::RowGroup(uint32_t colCount, From 773a665dabd8d5de7a00c4563f17713e06a79363 Mon Sep 17 00:00:00 2001 From: Roman Nozdrin Date: Wed, 2 Jan 2019 21:09:55 +0300 Subject: [PATCH 47/55] MCOL-2050 ORDER BY with OFFSET in subquery now produces correct result. --- dbcon/joblist/limitedorderby.cpp | 25 +++++++++++++------------ 1 file changed, 13 insertions(+), 12 deletions(-) diff --git a/dbcon/joblist/limitedorderby.cpp b/dbcon/joblist/limitedorderby.cpp index 64caf078c..de7f9bcb6 100644 --- a/dbcon/joblist/limitedorderby.cpp +++ b/dbcon/joblist/limitedorderby.cpp @@ -173,10 +173,10 @@ void LimitedOrderBy::finalize() if (fOrderByQueue.size() > 0) { - uint64_t newSize = fRowsPerRG * fRowGroup.getRowSize(); - fMemSize += newSize; + uint64_t memSizeInc = fRowsPerRG * fRowGroup.getRowSize(); + fMemSize += memSizeInc; - if (!fRm->getMemory(newSize, fSessionMemLimit)) + if (!fRm->getMemory(memSizeInc, fSessionMemLimit)) { cerr << IDBErrorInfo::instance()->errorMsg(fErrorCode) << " @" << __FILE__ << ":" << __LINE__; @@ -185,10 +185,12 @@ void LimitedOrderBy::finalize() uint64_t offset = 0; uint64_t i = 0; + // Reduce queue size by an offset value if it applicable. + uint64_t queueSizeWoOffset = fOrderByQueue.size() > fStart ? + fOrderByQueue.size() - fStart : 0; list tempRGDataList; - - // Skip first LIMIT rows in the the RowGroup - if ( fCount <= fOrderByQueue.size() ) + + if ( fCount <= queueSizeWoOffset ) { offset = fCount % fRowsPerRG; if(!offset && fCount > 0) @@ -196,11 +198,11 @@ void LimitedOrderBy::finalize() } else { - offset = fOrderByQueue.size() % fRowsPerRG; - if(!offset && fOrderByQueue.size() > 0) + offset = queueSizeWoOffset % fRowsPerRG; + if(!offset && queueSizeWoOffset > 0) offset = fRowsPerRG; } - + list::iterator tempListIter = tempRGDataList.begin(); i = 0; @@ -217,7 +219,6 @@ void LimitedOrderBy::finalize() const OrderByRow& topRow = fOrderByQueue.top(); row1.setData(topRow.fData); copyRow(row1, &fRow0); - //cerr << "LimitedOrderBy::finalize fRow0 " << fRow0.toString() << endl; fRowGroup.incRowCount(); offset--; fRow0.prevRow(rSize); @@ -226,9 +227,9 @@ void LimitedOrderBy::finalize() if(offset == (uint64_t)-1) { tempRGDataList.push_front(fData); - fMemSize += newSize; + fMemSize += memSizeInc; - if (!fRm->getMemory(newSize, fSessionMemLimit)) + if (!fRm->getMemory(memSizeInc, fSessionMemLimit)) { cerr << IDBErrorInfo::instance()->errorMsg(fErrorCode) << " @" << __FILE__ << ":" << __LINE__; From dd05b3af13f834edeac57b22dfdd4d93cfac81b5 Mon Sep 17 00:00:00 2001 From: sneJ- Date: Fri, 1 Feb 2019 15:00:26 +0100 Subject: [PATCH 48/55] MCOL-2129 - added a new postConfigure flag -xr to reverse dns lookup given hostnames and use that value instead of the original hostname value --- oamapps/postConfigure/postConfigure.cpp | 209 ++++++++++++++---------- 1 file changed, 121 insertions(+), 88 deletions(-) diff --git a/oamapps/postConfigure/postConfigure.cpp b/oamapps/postConfigure/postConfigure.cpp index c92697127..c4e9b2126 100644 --- a/oamapps/postConfigure/postConfigure.cpp +++ b/oamapps/postConfigure/postConfigure.cpp @@ -55,7 +55,6 @@ #include #include #include -#include #include /* for strncpy */ #include @@ -111,8 +110,6 @@ typedef std::vector ModuleIpList; void offLineAppCheck(); bool setOSFiles(string parentOAMModuleName, int serverTypeInstall); bool checkSaveConfigFile(); -string getModuleName(); -bool setModuleName(string moduleName); bool updateBash(); bool makeModuleFile(string moduleName, string parentOAMModuleName); bool updateProcessConfig(); @@ -126,6 +123,7 @@ bool singleServerDBrootSetup(); bool copyFstab(string moduleName); bool attachVolume(string instanceName, string volumeName, string deviceName, string dbrootPath); void singleServerConfigSetup(Config* sysConfig); +std::string resolveHostNameToReverseDNSName(std::string hostname); void remoteInstallThread(void*); @@ -193,6 +191,7 @@ bool single_server_quick_install = false; bool multi_server_quick_install = false; bool amazon_quick_install = false; bool doNotResolveHostNames = false; +bool resolveHostNamesToReverseDNSNames = false; string DataFileEnvFile; @@ -316,7 +315,7 @@ int main(int argc, char* argv[]) cout << " Enter one of the options within [], if available, or" << endl; cout << " Enter a new value" << endl << endl; cout << endl; - cout << "Usage: postConfigure [-h][-c][-u][-p][-qs][-qm][-qa][-port][-i][-n][-d][-sn][-pm-ip-addrs][-um-ip-addrs][-pm-count][-um-count][-numBlocksPct][-totalUmMemory]" << endl; + cout << "Usage: postConfigure [-h][-c][-u][-p][-qs][-qm][-qa][-port][-i][-n][-d][-sn][-pm-ip-addrs][-um-ip-addrs][-pm-count][-um-count][-x][-xr][-numBlocksPct][-totalUmMemory]" << endl; cout << " -h Help" << endl; cout << " -c Config File to use to extract configuration data, default is Columnstore.xml.rpmsave" << endl; cout << " -u Upgrade, Install using the Config File from -c, default to Columnstore.xml.rpmsave" << endl; @@ -332,6 +331,7 @@ int main(int argc, char* argv[]) cout << " -pm-ip-addrs Performance Module IP Addresses xxx.xxx.xxx.xxx,xxx.xxx.xxx.xxx" << endl; cout << " -um-ip-addrs User Module IP Addresses xxx.xxx.xxx.xxx,xxx.xxx.xxx.xxx" << endl; cout << " -x Do not resolve IP Addresses from host names" << endl; + cout << " -xr Resolve host names into their reverse DNS host names. Only applied in combination with -x" << endl; cout << " -numBlocksPct amount of physical memory to utilize for disk block caching" << endl; cout << " (percentages of the total memory need to be stated without suffix, explcit values with suffixes M or G)" << endl; cout << " -totalUmMemory amount of physical memory to utilize for joins, intermediate results and set operations on the UM" << endl; @@ -341,6 +341,10 @@ int main(int argc, char* argv[]) else if (string("-x") == argv[i]) { doNotResolveHostNames = true; + } + else if (string("-xr") == argv[i]) + { + resolveHostNamesToReverseDNSNames = true; } else if( string("-qs") == argv[i] ) { @@ -517,7 +521,7 @@ int main(int argc, char* argv[]) else { cout << " ERROR: Invalid Argument = " << argv[i] << endl; - cout << " Usage: postConfigure [-h][-c][-u][-p][-qs][-qm][-qa][-port][-i][-n][-d][-sn][-pm-ip-addrs][-um-ip-addrs][-pm-count][-um-count][-numBlocksPct][-totalUmMemory]" << endl; + cout << " Usage: postConfigure [-h][-c][-u][-p][-qs][-qm][-qa][-port][-i][-n][-d][-sn][-pm-ip-addrs][-um-ip-addrs][-pm-count][-um-count][-x][-xr][-numBlocksPct][-totalUmMemory]" << endl; exit (1); } } @@ -2566,7 +2570,12 @@ int main(int argc, char* argv[]) //get IP Address string IPAddress; if (doNotResolveHostNames) - IPAddress = newModuleHostName; + if (resolveHostNamesToReverseDNSNames) { + IPAddress = resolveHostNameToReverseDNSName(newModuleHostName); + } + else { + IPAddress = newModuleHostName; + } else IPAddress = oam.getIPAddress( newModuleHostName); @@ -6550,7 +6559,12 @@ bool glusterSetup(string password, bool doNotResolveHostNames) //get IP Address string IPAddress; if (doNotResolveHostNames) - IPAddress = moduleHostName; + if (resolveHostNamesToReverseDNSNames) { + IPAddress = resolveHostNameToReverseDNSName(moduleHostName); + } + else { + IPAddress = moduleHostName; + } else IPAddress = oam.getIPAddress( moduleHostName); @@ -7011,29 +7025,29 @@ bool glusterSetup(string password, bool doNotResolveHostNames) command = "sudo gluster volume set dbroot" + oam.itoa(dbrootID) + " storage.owner-uid " + oam.itoa(user) + " >> /tmp/glusterCommands.txt 2>&1";; status = system(command.c_str()); - if (WEXITSTATUS(status) != 0 ) - { - cout << "ERROR: Failed to start dbroot" << oam.itoa(dbrootID) << endl; - exit(1); - } +if (WEXITSTATUS(status) != 0) +{ + cout << "ERROR: Failed to start dbroot" << oam.itoa(dbrootID) << endl; + exit(1); +} - command = "sudo gluster volume set dbroot" + oam.itoa(dbrootID) + " storage.owner-gid " + oam.itoa(group) + " >> /tmp/glusterCommands.txt 2>&1";; - status = system(command.c_str()); +command = "sudo gluster volume set dbroot" + oam.itoa(dbrootID) + " storage.owner-gid " + oam.itoa(group) + " >> /tmp/glusterCommands.txt 2>&1";; +status = system(command.c_str()); - if (WEXITSTATUS(status) != 0 ) - { - cout << "ERROR: Failed to start dbroot" << oam.itoa(dbrootID) << endl; - exit(1); - } +if (WEXITSTATUS(status) != 0) +{ + cout << "ERROR: Failed to start dbroot" << oam.itoa(dbrootID) << endl; + exit(1); +} - command = "sudo gluster volume start dbroot" + oam.itoa(dbrootID) + " >> /tmp/glusterCommands.txt 2>&1"; - status = system(command.c_str()); +command = "sudo gluster volume start dbroot" + oam.itoa(dbrootID) + " >> /tmp/glusterCommands.txt 2>&1"; +status = system(command.c_str()); - if (WEXITSTATUS(status) != 0 ) - { - cout << "ERROR: Failed to start dbroot" << oam.itoa(dbrootID) << endl; - exit(1); - } +if (WEXITSTATUS(status) != 0) +{ + cout << "ERROR: Failed to start dbroot" << oam.itoa(dbrootID) << endl; + exit(1); +} } cout << "DONE" << endl; @@ -7047,70 +7061,89 @@ bool glusterSetup(string password, bool doNotResolveHostNames) void singleServerConfigSetup(Config* sysConfig) { - try - { - sysConfig->setConfig("ExeMgr1", "IPAddr", "127.0.0.1"); - sysConfig->setConfig("ExeMgr1", "Module", "pm1"); - sysConfig->setConfig("ProcMgr", "IPAddr", "127.0.0.1"); - sysConfig->setConfig("ProcMgr_Alarm", "IPAddr", "127.0.0.1"); - sysConfig->setConfig("ProcStatusControl", "IPAddr", "127.0.0.1"); - sysConfig->setConfig("pm1_ProcessMonitor", "IPAddr", "127.0.0.1"); - sysConfig->setConfig("pm1_ServerMonitor", "IPAddr", "127.0.0.1"); - sysConfig->setConfig("pm1_WriteEngineServer", "IPAddr", "127.0.0.1"); - sysConfig->setConfig("DDLProc", "IPAddr", "127.0.0.1"); - sysConfig->setConfig("DMLProc", "IPAddr", "127.0.0.1"); - sysConfig->setConfig("PMS1", "IPAddr", "127.0.0.1"); - sysConfig->setConfig("PMS2", "IPAddr", "127.0.0.1"); - sysConfig->setConfig("PMS3", "IPAddr", "127.0.0.1"); - sysConfig->setConfig("PMS4", "IPAddr", "127.0.0.1"); - sysConfig->setConfig("PMS5", "IPAddr", "127.0.0.1"); - sysConfig->setConfig("PMS6", "IPAddr", "127.0.0.1"); - sysConfig->setConfig("PMS7", "IPAddr", "127.0.0.1"); - sysConfig->setConfig("PMS8", "IPAddr", "127.0.0.1"); - sysConfig->setConfig("PMS9", "IPAddr", "127.0.0.1"); - sysConfig->setConfig("PMS10", "IPAddr", "127.0.0.1"); - sysConfig->setConfig("PMS11", "IPAddr", "127.0.0.1"); - sysConfig->setConfig("PMS12", "IPAddr", "127.0.0.1"); - sysConfig->setConfig("PMS13", "IPAddr", "127.0.0.1"); - sysConfig->setConfig("PMS14", "IPAddr", "127.0.0.1"); - sysConfig->setConfig("PMS15", "IPAddr", "127.0.0.1"); - sysConfig->setConfig("PMS16", "IPAddr", "127.0.0.1"); - sysConfig->setConfig("PMS17", "IPAddr", "127.0.0.1"); - sysConfig->setConfig("PMS18", "IPAddr", "127.0.0.1"); - sysConfig->setConfig("PMS19", "IPAddr", "127.0.0.1"); - sysConfig->setConfig("PMS20", "IPAddr", "127.0.0.1"); - sysConfig->setConfig("PMS21", "IPAddr", "127.0.0.1"); - sysConfig->setConfig("PMS22", "IPAddr", "127.0.0.1"); - sysConfig->setConfig("PMS23", "IPAddr", "127.0.0.1"); - sysConfig->setConfig("PMS24", "IPAddr", "127.0.0.1"); - sysConfig->setConfig("PMS25", "IPAddr", "127.0.0.1"); - sysConfig->setConfig("PMS26", "IPAddr", "127.0.0.1"); - sysConfig->setConfig("PMS27", "IPAddr", "127.0.0.1"); - sysConfig->setConfig("PMS28", "IPAddr", "127.0.0.1"); - sysConfig->setConfig("PMS29", "IPAddr", "127.0.0.1"); - sysConfig->setConfig("PMS30", "IPAddr", "127.0.0.1"); - sysConfig->setConfig("PMS31", "IPAddr", "127.0.0.1"); - sysConfig->setConfig("PMS32", "IPAddr", "127.0.0.1"); - sysConfig->setConfig("SystemModuleConfig", "ModuleCount2", "0"); - sysConfig->setConfig("SystemModuleConfig", "ModuleIPAddr1-1-3", "127.0.0.1"); - sysConfig->setConfig("SystemModuleConfig", "ModuleHostName1-1-3", "localhost"); - sysConfig->setConfig("DBRM_Controller", "IPAddr", "127.0.0.1"); - sysConfig->setConfig("DBRM_Worker1", "IPAddr", "127.0.0.1"); - sysConfig->setConfig("DBRM_Worker1", "Module", "pm1"); - sysConfig->setConfig("DBBC", "NumBlocksPct", "50"); - sysConfig->setConfig("Installation", "InitialInstallFlag", "y"); - sysConfig->setConfig("Installation", "SingleServerInstall", "y"); - sysConfig->setConfig("HashJoin", "TotalUmMemory", "25%"); - } - catch (...) - { - cout << "ERROR: Problem setting for Single Server in the MariaDB ColumnStore System Configuration file" << endl; - exit(1); - } + try + { + sysConfig->setConfig("ExeMgr1", "IPAddr", "127.0.0.1"); + sysConfig->setConfig("ExeMgr1", "Module", "pm1"); + sysConfig->setConfig("ProcMgr", "IPAddr", "127.0.0.1"); + sysConfig->setConfig("ProcMgr_Alarm", "IPAddr", "127.0.0.1"); + sysConfig->setConfig("ProcStatusControl", "IPAddr", "127.0.0.1"); + sysConfig->setConfig("pm1_ProcessMonitor", "IPAddr", "127.0.0.1"); + sysConfig->setConfig("pm1_ServerMonitor", "IPAddr", "127.0.0.1"); + sysConfig->setConfig("pm1_WriteEngineServer", "IPAddr", "127.0.0.1"); + sysConfig->setConfig("DDLProc", "IPAddr", "127.0.0.1"); + sysConfig->setConfig("DMLProc", "IPAddr", "127.0.0.1"); + sysConfig->setConfig("PMS1", "IPAddr", "127.0.0.1"); + sysConfig->setConfig("PMS2", "IPAddr", "127.0.0.1"); + sysConfig->setConfig("PMS3", "IPAddr", "127.0.0.1"); + sysConfig->setConfig("PMS4", "IPAddr", "127.0.0.1"); + sysConfig->setConfig("PMS5", "IPAddr", "127.0.0.1"); + sysConfig->setConfig("PMS6", "IPAddr", "127.0.0.1"); + sysConfig->setConfig("PMS7", "IPAddr", "127.0.0.1"); + sysConfig->setConfig("PMS8", "IPAddr", "127.0.0.1"); + sysConfig->setConfig("PMS9", "IPAddr", "127.0.0.1"); + sysConfig->setConfig("PMS10", "IPAddr", "127.0.0.1"); + sysConfig->setConfig("PMS11", "IPAddr", "127.0.0.1"); + sysConfig->setConfig("PMS12", "IPAddr", "127.0.0.1"); + sysConfig->setConfig("PMS13", "IPAddr", "127.0.0.1"); + sysConfig->setConfig("PMS14", "IPAddr", "127.0.0.1"); + sysConfig->setConfig("PMS15", "IPAddr", "127.0.0.1"); + sysConfig->setConfig("PMS16", "IPAddr", "127.0.0.1"); + sysConfig->setConfig("PMS17", "IPAddr", "127.0.0.1"); + sysConfig->setConfig("PMS18", "IPAddr", "127.0.0.1"); + sysConfig->setConfig("PMS19", "IPAddr", "127.0.0.1"); + sysConfig->setConfig("PMS20", "IPAddr", "127.0.0.1"); + sysConfig->setConfig("PMS21", "IPAddr", "127.0.0.1"); + sysConfig->setConfig("PMS22", "IPAddr", "127.0.0.1"); + sysConfig->setConfig("PMS23", "IPAddr", "127.0.0.1"); + sysConfig->setConfig("PMS24", "IPAddr", "127.0.0.1"); + sysConfig->setConfig("PMS25", "IPAddr", "127.0.0.1"); + sysConfig->setConfig("PMS26", "IPAddr", "127.0.0.1"); + sysConfig->setConfig("PMS27", "IPAddr", "127.0.0.1"); + sysConfig->setConfig("PMS28", "IPAddr", "127.0.0.1"); + sysConfig->setConfig("PMS29", "IPAddr", "127.0.0.1"); + sysConfig->setConfig("PMS30", "IPAddr", "127.0.0.1"); + sysConfig->setConfig("PMS31", "IPAddr", "127.0.0.1"); + sysConfig->setConfig("PMS32", "IPAddr", "127.0.0.1"); + sysConfig->setConfig("SystemModuleConfig", "ModuleCount2", "0"); + sysConfig->setConfig("SystemModuleConfig", "ModuleIPAddr1-1-3", "127.0.0.1"); + sysConfig->setConfig("SystemModuleConfig", "ModuleHostName1-1-3", "localhost"); + sysConfig->setConfig("DBRM_Controller", "IPAddr", "127.0.0.1"); + sysConfig->setConfig("DBRM_Worker1", "IPAddr", "127.0.0.1"); + sysConfig->setConfig("DBRM_Worker1", "Module", "pm1"); + sysConfig->setConfig("DBBC", "NumBlocksPct", "50"); + sysConfig->setConfig("Installation", "InitialInstallFlag", "y"); + sysConfig->setConfig("Installation", "SingleServerInstall", "y"); + sysConfig->setConfig("HashJoin", "TotalUmMemory", "25%"); + } + catch (...) + { + cout << "ERROR: Problem setting for Single Server in the MariaDB ColumnStore System Configuration file" << endl; + exit(1); + } - return; + return; } - +/** + Resolves the given hostname into its reverse DNS name. + + @param hostname the hostname to resolve. + @return the reverse dns name of given hostname or an empty string in case the hostname could not be resolved. +*/ +std::string resolveHostNameToReverseDNSName(std::string hostname) { + struct hostent *hp = gethostbyname(hostname.c_str()); + if (hp == NULL) { + std::cout << "Error: Couldn't resolve hostname " << hostname << " to ip address" << std::endl; + return ""; + } + struct hostent *rl = gethostbyaddr(hp->h_addr_list[0], sizeof hp->h_addr_list[0], AF_INET); + if (rl == NULL) { + std::cout << "Error: Couldn't resolve ip address of hostname " << hostname << " back to a hostname" << std::endl; + return ""; + } + hostname = rl->h_name; + return hostname; +} // vim:ts=4 sw=4: From bf4a215cc8d721be09cd2c311ab95c951ae8cef9 Mon Sep 17 00:00:00 2001 From: David Hall Date: Mon, 4 Feb 2019 12:32:04 -0600 Subject: [PATCH 49/55] MCOL-2001 make redistribute parameters case insensitive. --- oamapps/mcsadmin/mcsadmin.cpp | 25 ++++++++++++++++++++----- 1 file changed, 20 insertions(+), 5 deletions(-) diff --git a/oamapps/mcsadmin/mcsadmin.cpp b/oamapps/mcsadmin/mcsadmin.cpp index 8b31353ac..dd7262d1e 100644 --- a/oamapps/mcsadmin/mcsadmin.cpp +++ b/oamapps/mcsadmin/mcsadmin.cpp @@ -32,7 +32,6 @@ extern int h_errno; #include "boost/filesystem/path.hpp" #include "boost/scoped_ptr.hpp" #include "boost/tokenizer.hpp" -#include "boost/algorithm/string/predicate.hpp" #include "sessionmanager.h" #include "dbrm.h" #include "messagequeue.h" @@ -678,7 +677,11 @@ int processCommand(string* arguments) vector srcDbroots; // all of the currently configured dbroots vector destDbroots; // srcDbroots - removeDbroots set::iterator dbiter; - if (boost::iequals(arguments[1], "start")) +#if _MSC_VER + if (_strnicmp(arguments[1].c_str(), "start", 5) == 0)) +#else + if (strncasecmp(arguments[1].c_str(), "start", 5) == 0) +#endif { // Get a list of all the configured dbroots in the xml file. DBRootConfigList dbRootConfigList; @@ -689,7 +692,11 @@ int processCommand(string* arguments) // The user may choose to redistribute in such a way as to // leave certain dbroots empty, presumably for later removal. - if (boost::iequals(arguments[2], "remove")) +#if _MSC_VER + if (_strnicmp(arguments[1].c_str(), "remove", 6) == 0)) +#else + if (strncasecmp(arguments[1].c_str(), "remove", 6) == 0) +#endif { int dbroot; bool error = false; @@ -793,7 +800,11 @@ int processCommand(string* arguments) SendToWES(oam, bs); } - else if (boost::iequals(arguments[1], "stop")) +#if _MSC_VER + if (_strnicmp(arguments[1].c_str(), "stop", 4) == 0)) +#else + if (strncasecmp(arguments[1].c_str(), "stop", 4) == 0) +#endif { ByteStream bs; // message WES ID, sequence #, action id @@ -803,7 +814,11 @@ int processCommand(string* arguments) bs.append((const ByteStream::byte*) &header, sizeof(header)); SendToWES(oam, bs); } - else if (boost::iequals(arguments[1], "status")) +#if _MSC_VER + if (_strnicmp(arguments[1].c_str(), "status", 6) == 0)) +#else + if (strncasecmp(arguments[1].c_str(), "status", 6) == 0) +#endif { ByteStream bs; // message WES ID, sequence #, action id From b81455744d938c442d248b09b10ebbee65e4e2ee Mon Sep 17 00:00:00 2001 From: Ben Thompson Date: Tue, 5 Feb 2019 13:43:21 -0600 Subject: [PATCH 50/55] Revert "Kill Jemalloc" This reverts commit e03f7dc88503c7044ebc5befdd432238e31d3f16. --- CMakeLists.txt | 8 ++++++- FindJeMalloc.cmake | 44 ++++++++++++++++++++++++++++++++++++++ build/infinidb.spec.in | 1 + build/infinidb.spec.in.4.5 | 1 + 4 files changed, 53 insertions(+), 1 deletion(-) create mode 100644 FindJeMalloc.cmake diff --git a/CMakeLists.txt b/CMakeLists.txt index 0932307ed..2645adc5c 100644 --- a/CMakeLists.txt +++ b/CMakeLists.txt @@ -92,6 +92,12 @@ if (NOT LIBXML2_FOUND) MESSAGE(FATAL_ERROR "Could not find a usable libxml2 development environment!") endif() +INCLUDE (FindJeMalloc.cmake) +if (NOT JEMALLOC_FOUND) + message(NOTICE "jemalloc not found! Compiling with standard malloc") + SET(JEMALLOC_LIBRARIES "") +endif() + FIND_PROGRAM(AWK_EXECUTABLE awk DOC "path to the awk executable") if(NOT AWK_EXECUTABLE) message(FATAL_ERROR "awk not found!") @@ -123,7 +129,7 @@ SET (ENGINE_LOCALDIR "${INSTALL_ENGINE}/local") SET (ENGINE_MYSQLDIR "${INSTALL_ENGINE}/mysql") SET (ENGINE_TOOLSDIR "${INSTALL_ENGINE}/tools") -SET (ENGINE_COMMON_LIBS messageqcpp loggingcpp configcpp idbboot ${Boost_LIBRARIES} xml2 pthread rt) +SET (ENGINE_COMMON_LIBS messageqcpp loggingcpp configcpp idbboot ${Boost_LIBRARIES} xml2 pthread rt ${JEMALLOC_LIBRARIES}) SET (ENGINE_OAM_LIBS oamcpp alarmmanager) SET (ENGINE_BRM_LIBS brm idbdatafile cacheutils rwlock ${ENGINE_OAM_LIBS} ${ENGINE_COMMON_LIBS}) SET (ENGINE_EXEC_LIBS joblist execplan windowfunction joiner rowgroup funcexp udfsdk dataconvert common compress mysqlcl_idb querystats querytele thrift threadpool ${ENGINE_BRM_LIBS}) diff --git a/FindJeMalloc.cmake b/FindJeMalloc.cmake new file mode 100644 index 000000000..c97bb1b9f --- /dev/null +++ b/FindJeMalloc.cmake @@ -0,0 +1,44 @@ +# - Try to find jemalloc headers and libraries. +# +# Usage of this module as follows: +# +# find_package(JeMalloc) +# +# Variables used by this module, they can change the default behaviour and need +# to be set before calling find_package: +# +# JEMALLOC_ROOT_DIR Set this variable to the root installation of +# jemalloc if the module has problems finding +# the proper installation path. +# +# Variables defined by this module: +# +# JEMALLOC_FOUND System has jemalloc libs/headers +# JEMALLOC_LIBRARIES The jemalloc library/libraries +# JEMALLOC_INCLUDE_DIR The location of jemalloc headers + +find_path(JEMALLOC_ROOT_DIR + NAMES include/jemalloc/jemalloc.h +) + +find_library(JEMALLOC_LIBRARIES + NAMES jemalloc + HINTS ${JEMALLOC_ROOT_DIR}/lib +) + +find_path(JEMALLOC_INCLUDE_DIR + NAMES jemalloc/jemalloc.h + HINTS ${JEMALLOC_ROOT_DIR}/include +) + +include(FindPackageHandleStandardArgs) +find_package_handle_standard_args(JeMalloc DEFAULT_MSG + JEMALLOC_LIBRARIES + JEMALLOC_INCLUDE_DIR +) + +mark_as_advanced( + JEMALLOC_ROOT_DIR + JEMALLOC_LIBRARIES + JEMALLOC_INCLUDE_DIR +) diff --git a/build/infinidb.spec.in b/build/infinidb.spec.in index 20a44e834..2531044a0 100644 --- a/build/infinidb.spec.in +++ b/build/infinidb.spec.in @@ -336,6 +336,7 @@ rm -rf $RPM_BUILD_ROOT $RPM_BUILD_DIR/%{name}-%{version}-%{release} /usr/local/mariadb/columnstore/lib/libmysqlcl_idb.so.1.0.0 /usr/local/mariadb/columnstore/lib/libquerystats.so.1.0.0 /usr/local/mariadb/columnstore/lib/libwriteengineredistribute.so.1.0.0 +/usr/local/mariadb/columnstore/lib/libjemalloc.so.3.3.0 /usr/local/mariadb/columnstore/lib/libidbdatafile.so.1.0.0 /usr/local/mariadb/columnstore/lib/hdfs-20.so /usr/local/mariadb/columnstore/lib/hdfs-12.so diff --git a/build/infinidb.spec.in.4.5 b/build/infinidb.spec.in.4.5 index 966f2a73a..b8145d580 100755 --- a/build/infinidb.spec.in.4.5 +++ b/build/infinidb.spec.in.4.5 @@ -338,6 +338,7 @@ rm -rf $RPM_BUILD_ROOT $RPM_BUILD_DIR/%{name}-%{version}-%{release} /usr/local/Calpont/lib/libmysqlcl_idb.so.1.0.0 /usr/local/Calpont/lib/libquerystats.so.1.0.0 /usr/local/Calpont/lib/libwriteengineredistribute.so.1.0.0 +/usr/local/Calpont/lib/libjemalloc.so.3.3.0 /usr/local/Calpont/lib/libidbdatafile.so.1.0.0 /usr/local/Calpont/lib/hdfs-20.so /usr/local/Calpont/lib/hdfs-12.so From 52a741e6efdd7d22b29701ab1cce0ae86278886e Mon Sep 17 00:00:00 2001 From: Ben Thompson Date: Tue, 5 Feb 2019 17:18:02 -0600 Subject: [PATCH 51/55] MCOL-2136: Force use of jemalloc for build and install. --- CMakeLists.txt | 2 +- build/infinidb.spec.in | 1 - build/infinidb.spec.in.4.5 | 1 - cpackEngineDEB.cmake | 2 +- cpackEngineRPM.cmake | 4 ++-- 5 files changed, 4 insertions(+), 6 deletions(-) diff --git a/CMakeLists.txt b/CMakeLists.txt index 2645adc5c..c72a74d61 100644 --- a/CMakeLists.txt +++ b/CMakeLists.txt @@ -94,7 +94,7 @@ endif() INCLUDE (FindJeMalloc.cmake) if (NOT JEMALLOC_FOUND) - message(NOTICE "jemalloc not found! Compiling with standard malloc") + message(FATAL_ERROR "jemalloc not found!") SET(JEMALLOC_LIBRARIES "") endif() diff --git a/build/infinidb.spec.in b/build/infinidb.spec.in index 2531044a0..20a44e834 100644 --- a/build/infinidb.spec.in +++ b/build/infinidb.spec.in @@ -336,7 +336,6 @@ rm -rf $RPM_BUILD_ROOT $RPM_BUILD_DIR/%{name}-%{version}-%{release} /usr/local/mariadb/columnstore/lib/libmysqlcl_idb.so.1.0.0 /usr/local/mariadb/columnstore/lib/libquerystats.so.1.0.0 /usr/local/mariadb/columnstore/lib/libwriteengineredistribute.so.1.0.0 -/usr/local/mariadb/columnstore/lib/libjemalloc.so.3.3.0 /usr/local/mariadb/columnstore/lib/libidbdatafile.so.1.0.0 /usr/local/mariadb/columnstore/lib/hdfs-20.so /usr/local/mariadb/columnstore/lib/hdfs-12.so diff --git a/build/infinidb.spec.in.4.5 b/build/infinidb.spec.in.4.5 index b8145d580..966f2a73a 100755 --- a/build/infinidb.spec.in.4.5 +++ b/build/infinidb.spec.in.4.5 @@ -338,7 +338,6 @@ rm -rf $RPM_BUILD_ROOT $RPM_BUILD_DIR/%{name}-%{version}-%{release} /usr/local/Calpont/lib/libmysqlcl_idb.so.1.0.0 /usr/local/Calpont/lib/libquerystats.so.1.0.0 /usr/local/Calpont/lib/libwriteengineredistribute.so.1.0.0 -/usr/local/Calpont/lib/libjemalloc.so.3.3.0 /usr/local/Calpont/lib/libidbdatafile.so.1.0.0 /usr/local/Calpont/lib/hdfs-20.so /usr/local/Calpont/lib/hdfs-12.so diff --git a/cpackEngineDEB.cmake b/cpackEngineDEB.cmake index 30d3d0ab9..13865c05b 100644 --- a/cpackEngineDEB.cmake +++ b/cpackEngineDEB.cmake @@ -60,7 +60,7 @@ SET(CPACK_DEBIAN_LIBS_PACKAGE_PROVIDES "mariadb-columnstore-libs") SET(CPACK_DEBIAN_PLATFORM_PACKAGE_PROVIDES "mariadb-columnstore-platform") SET(CPACK_DEBIAN_STORAGE-ENGINE_PACKAGE_PROVIDES "mariadb-columnstore-storage-engine") -SET(CPACK_DEBIAN_PLATFORM_PACKAGE_DEPENDS "expect, libboost-all-dev, mariadb-columnstore-libs") +SET(CPACK_DEBIAN_PLATFORM_PACKAGE_DEPENDS "expect, libboost-all-dev, mariadb-columnstore-libs, libjemalloc1") SET(CPACK_DEBIAN_STORAGE-ENGINE_PACKAGE_DEPENDS "mariadb-columnstore-libs") diff --git a/cpackEngineRPM.cmake b/cpackEngineRPM.cmake index 98974325d..12da14498 100644 --- a/cpackEngineRPM.cmake +++ b/cpackEngineRPM.cmake @@ -90,9 +90,9 @@ if (${REDHAT_VERSION_NUMBER} EQUAL 6) # Disable auto require as this will also try to pull Boost via RPM SET(CPACK_RPM_PACKAGE_AUTOREQPROV " no") elseif (${SUSE_VERSION_NUMBER} EQUAL 12) - SETA(CPACK_RPM_platform_PACKAGE_REQUIRES "expect" "boost-devel >= 1.54.0" "mariadb-columnstore-libs") + SETA(CPACK_RPM_platform_PACKAGE_REQUIRES "expect" "boost-devel >= 1.54.0" "mariadb-columnstore-libs" "jemalloc") else () - SETA(CPACK_RPM_platform_PACKAGE_REQUIRES "expect" "boost >= 1.53.0" "mariadb-columnstore-libs") + SETA(CPACK_RPM_platform_PACKAGE_REQUIRES "expect" "boost >= 1.53.0" "mariadb-columnstore-libs" "jemalloc") endif() SETA(CPACK_RPM_storage-engine_PACKAGE_REQUIRES "mariadb-columnstore-libs") From f805478eb91173676012c481e8dc81511707963e Mon Sep 17 00:00:00 2001 From: Andrew Hutchings Date: Fri, 8 Feb 2019 17:55:50 +0000 Subject: [PATCH 52/55] MCOL-2149 Fix cpimport decimal saturation If we saturate int64_t during string -> decimal conversion then end processing there instead of continuing. This preserves a good saturation value. --- writeengine/shared/we_convertor.cpp | 21 ++++++++++++++------- 1 file changed, 14 insertions(+), 7 deletions(-) diff --git a/writeengine/shared/we_convertor.cpp b/writeengine/shared/we_convertor.cpp index 8ad344ce3..7dcad3986 100644 --- a/writeengine/shared/we_convertor.cpp +++ b/writeengine/shared/we_convertor.cpp @@ -182,19 +182,26 @@ long long Convertor::convertDecimalString( { long double dval = strtold(field, NULL); long long ret = 0; - + // move scale digits to the left of the decimal point for (int i = 0; i < scale; i++) dval *= 10; - + // range check against int64 - if (dval > LLONG_MAX || dval < LLONG_MIN) + if (dval > LLONG_MAX) + { errno = ERANGE; - else - errno = 0; - + return LLONG_MAX; + } + if (dval < LLONG_MIN) + { + errno = ERANGE; + return LLONG_MIN; + } + errno = 0; + ret = dval; - + // get the fractional part of what's left & round ret up or down. dval -= ret; if (dval >= 0.5 && ret < LLONG_MAX) From 9c71f3d48086b5963a020cc9bd9d2316ca22a0cd Mon Sep 17 00:00:00 2001 From: Andrew Hutchings Date: Wed, 13 Feb 2019 21:23:08 +0000 Subject: [PATCH 53/55] Bump version to 1.0.17 --- README.md | 8 ++++---- VERSION | 2 +- 2 files changed, 5 insertions(+), 5 deletions(-) diff --git a/README.md b/README.md index 3ad645174..bac34bdd2 100644 --- a/README.md +++ b/README.md @@ -1,9 +1,9 @@ -#MariaDB ColumnStore Storage/Execution engine 1.0.16 -MariaDB ColumnStore 1.0.16 is the development version of MariaDB ColumnStore. -It is built by porting InfiniDB 4.6.7 on MariaDB 10.1.35 and adding entirely +#MariaDB ColumnStore Storage/Execution engine 1.0.17 +MariaDB ColumnStore 1.0.17 is the development version of MariaDB ColumnStore. +It is built by porting InfiniDB 4.6.7 on MariaDB 10.1 and adding entirely new features not found anywhere else. -#MariaDB ColumnStore 1.0.16 is an GA release. +#MariaDB ColumnStore 1.0.17 is an GA release. #Building This repository is not meant to be built independently outside of the server. This repository is integrated into http://mariadb-corporation/mariadb-columnstore-server (ie, the *server*) as a git submodule. As such, you can find complete build instructions on *the server* page. diff --git a/VERSION b/VERSION index b13a94d3f..7e6391a76 100644 --- a/VERSION +++ b/VERSION @@ -1,4 +1,4 @@ COLUMNSTORE_VERSION_MAJOR=1 COLUMNSTORE_VERSION_MINOR=0 -COLUMNSTORE_VERSION_PATCH=16 +COLUMNSTORE_VERSION_PATCH=17 COLUMNSTORE_VERSION_RELEASE=1 From 620519384593c0b2ada9dcb24ce52b9a6e1463c8 Mon Sep 17 00:00:00 2001 From: Andrew Hutchings Date: Wed, 13 Feb 2019 21:50:44 +0000 Subject: [PATCH 54/55] Disable jemalloc on SLES --- CMakeLists.txt | 13 +++++++++---- 1 file changed, 9 insertions(+), 4 deletions(-) diff --git a/CMakeLists.txt b/CMakeLists.txt index 387d34f3a..2053bbad7 100755 --- a/CMakeLists.txt +++ b/CMakeLists.txt @@ -112,11 +112,16 @@ if (NOT SNAPPY_FOUND) MESSAGE(FATAL_ERROR "Snappy not found please install snappy-devel for CentOS/RedHat or libsnappy-dev for Ubuntu/Debian") endif() -INCLUDE (FindJeMalloc.cmake) -if (NOT JEMALLOC_FOUND) - message(FATAL_ERROR "jemalloc not found!") +# Jemalloc has issues with SLES 12, so disable for now +IF (EXISTS "/etc/SuSE-release") SET(JEMALLOC_LIBRARIES "") -endif() +ELSE () + INCLUDE (FindJeMalloc.cmake) + if (NOT JEMALLOC_FOUND) + message(FATAL_ERROR "jemalloc not found!") + SET(JEMALLOC_LIBRARIES "") + endif() +ENDIF () FIND_PROGRAM(AWK_EXECUTABLE awk DOC "path to the awk executable") if(NOT AWK_EXECUTABLE) From 7dfcc3c69ca8dd641c35864275f1cfb0cd0dbf23 Mon Sep 17 00:00:00 2001 From: Patrick LeBlanc Date: Thu, 14 Feb 2019 10:01:27 -0600 Subject: [PATCH 55/55] MCOL-1607. Let postconfigure store hostnames in the configuration. Made postconfig resolve hostname -> IP for its 'am I PM1' check. Made the text of one of the prompts more consistent. Made one of the paths it goes down to get ip addresses do hostname -> IP. Unrelated to ticket, just something Daniel noticed. --- oamapps/postConfigure/postConfigure.cpp | 25 ++++++++++++++++++++++--- 1 file changed, 22 insertions(+), 3 deletions(-) diff --git a/oamapps/postConfigure/postConfigure.cpp b/oamapps/postConfigure/postConfigure.cpp index c4e9b2126..fdfc1482c 100644 --- a/oamapps/postConfigure/postConfigure.cpp +++ b/oamapps/postConfigure/postConfigure.cpp @@ -662,9 +662,14 @@ int main(int argc, char* argv[]) if (moduleconfig.hostConfigList.size() > 0 ) { HostConfigList::iterator pt1 = moduleconfig.hostConfigList.begin(); - string PM1ipAdd = (*pt1).IPAddr; + + // MCOL-1607. The 'am I pm1?' check below requires an ipaddr. + string PM1ipAdd = oam.getIPAddress((*pt1).IPAddr.c_str()); + if (PM1ipAdd.empty()) + PM1ipAdd = (*pt1).IPAddr; // this is what it was doing before + //cout << PM1ipAdd << endl; - + if ( PM1ipAdd != "127.0.0.1" ) { if ( PM1ipAdd != "0.0.0.0") @@ -2602,6 +2607,13 @@ int main(int argc, char* argv[]) callFree(pcommand); } + + if (!doNotResolveHostNames) + { + string ugh = oam.getIPAddress(newModuleIPAddr); + if (ugh.length() > 0) + newModuleIPAddr = ugh; + } if (newModuleIPAddr == "127.0.0.1" || newModuleIPAddr == "0.0.0.0" || newModuleIPAddr == "128.0.0.1") { @@ -6580,7 +6592,7 @@ bool glusterSetup(string password, bool doNotResolveHostNames) //prompt for IP address while (true) { - prompt = "Enter PM #" + oam.itoa(DataRedundancyConfigs[pm].pmID) + " IP Address of " + moduleHostName + " (" + moduleIPAddr + ") > "; + prompt = "Enter PM #" + oam.itoa(DataRedundancyConfigs[pm].pmID) + " IP Address or hostname of " + moduleHostName + " (" + moduleIPAddr + ") > "; pcommand = callReadline(prompt.c_str()); if (pcommand) @@ -6589,6 +6601,13 @@ bool glusterSetup(string password, bool doNotResolveHostNames) callFree(pcommand); } + + if (!doNotResolveHostNames) + { + string ugh = oam.getIPAddress(moduleIPAddr); + if (ugh.length() > 0) + moduleIPAddr = ugh; + } if (moduleIPAddr == "127.0.0.1" || moduleIPAddr == "0.0.0.0" || moduleIPAddr == "128.0.0.1") {