From b1d5f54abea9e3507662b040f7fd22da2a0e57ec Mon Sep 17 00:00:00 2001 From: Andrew Hutchings Date: Wed, 14 Nov 2018 14:29:58 +0000 Subject: [PATCH 01/15] MCOL-1868 Fix error in unused code There was a a bad line in some code that we don't currently compile. This patch fixes that code in case we use it in the future. --- utils/funcexp/func_concat_ws.cpp | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/utils/funcexp/func_concat_ws.cpp b/utils/funcexp/func_concat_ws.cpp index 2718f20b3..d6253cf10 100644 --- a/utils/funcexp/func_concat_ws.cpp +++ b/utils/funcexp/func_concat_ws.cpp @@ -93,7 +93,7 @@ string Func_concat_ws::getStrVal(Row& row, string tmp; for ( uint32_t i = 1 ; i < parm.size() ; i++) { - string(stringValue(parm[i], row, isNull).c_str(), tmp); + stringValue(parm[i], row, isNull, tmp); str += tmp; if (isNull) From 5fd94e1438571347f0877a63e7f6e75177e58091 Mon Sep 17 00:00:00 2001 From: Patrick LeBlanc Date: Wed, 14 Nov 2018 17:15:14 -0600 Subject: [PATCH 02/15] MCOL-1844. Preserve user-added args in 'myCnf-include-args.text' across upgrades. --- oam/install_scripts/pre-uninstall | 1 + oamapps/postConfigure/mycnfUpgrade.cpp | 59 +++++++++++++++++++++++++- 2 files changed, 59 insertions(+), 1 deletion(-) diff --git a/oam/install_scripts/pre-uninstall b/oam/install_scripts/pre-uninstall index 2f58b5421..33a6328f2 100755 --- a/oam/install_scripts/pre-uninstall +++ b/oam/install_scripts/pre-uninstall @@ -138,6 +138,7 @@ if [ $quiet != 1 ]; then #make copy of Columnstore.xml /bin/cp -f $installdir/etc/Columnstore.xml $installdir/etc/Columnstore.xml.rpmsave > /dev/null 2>&1 /bin/cp -f $installdir/mysql/my.cnf $installdir/mysql/my.cnf.rpmsave > /dev/null 2>&1 + cp $installdir/bin/myCnf-include-args.text $installdir/bin/myCnf-include-args.text.rpmsave >& /dev/null rm -f $installdir/etc/AlarmConfig.xml.installSave fi diff --git a/oamapps/postConfigure/mycnfUpgrade.cpp b/oamapps/postConfigure/mycnfUpgrade.cpp index a8808e740..18b606a2b 100644 --- a/oamapps/postConfigure/mycnfUpgrade.cpp +++ b/oamapps/postConfigure/mycnfUpgrade.cpp @@ -54,6 +54,59 @@ using namespace std; using namespace oam; + +/* MCOL-1844. On an upgrade, the user may have customized options in their old + * myCnf-include-args.text file. Merge it with the packaged version, and then process as we + * have before. + */ +string rtrim(const string &in) { + string::const_reverse_iterator rbegin = in.rbegin(); + while (rbegin != in.rend() && isspace(*rbegin)) + ++rbegin; + return string(in.begin(), rbegin.base()); +} + +void mergeMycnfIncludeArgs() +{ + string userArgsFilename = startup::StartUp::installDir() + "/bin/myCnf-include-args.text.rpmsave"; + string packagedArgsFilename = startup::StartUp::installDir() + "/bin/myCnf-include-args.text"; + ifstream userArgs(userArgsFilename.c_str()); + fstream packagedArgs(packagedArgsFilename.c_str(), ios::in); + + if (!userArgs || !packagedArgs) + return; + + // de-dup the args and comments in both files + set argMerger; + set comments; + string line; + while (getline(packagedArgs, line)) { + line = rtrim(line); + if (line[0] == '#') + comments.insert(line); + else if (line.size() > 0) + argMerger.insert(line); + } + while (getline(userArgs, line)) { + line = rtrim(line); + if (line[0] == '#') + comments.insert(line); + else if (line.size() > 0) + argMerger.insert(line); + } + userArgs.close(); + packagedArgs.close(); + + // write the merged version, comments first. They'll get ordered + // alphabetically but, meh. + packagedArgs.open(packagedArgsFilename.c_str(), ios::out | ios::trunc); + for (set::iterator it = comments.begin(); it != comments.end(); it++) + packagedArgs << *it << endl; + for (set::iterator it = argMerger.begin(); it != argMerger.end(); it++) + packagedArgs << *it << endl; + packagedArgs.close(); +} + int main(int argc, char *argv[]) { Oam oam; @@ -84,6 +137,10 @@ int main(int argc, char *argv[]) exit (1); } + // MCOL-1844. The user may have added options to their myCnf-include-args file. Merge + // myCnf-include-args.text with myCnf-include-args.text.rpmsave, save in myCnf-include-args.text + mergeMycnfIncludeArgs(); + //include arguments file string includeFile = startup::StartUp::installDir() + "/bin/myCnf-include-args.text"; ifstream includefile (includeFile.c_str()); @@ -144,7 +201,7 @@ int main(int argc, char *argv[]) ofstream newFile (mycnfFile.c_str()); //create new file - int fd = open(mycnfFile.c_str(), O_RDWR|O_CREAT, 0666); + int fd = open(mycnfFile.c_str(), O_RDWR|O_CREAT, 0644); copy(lines.begin(), lines.end(), ostream_iterator(newFile, "\n")); newFile.close(); From f8267a54b694594c78085b3ee61f7c8ddaa9f5c0 Mon Sep 17 00:00:00 2001 From: David Hill Date: Fri, 16 Nov 2018 14:13:31 -0600 Subject: [PATCH 03/15] MCOL-1944 - correct the non-root user:group setup --- oam/install_scripts/syslogSetup.sh | 38 +++++++++++++----------------- 1 file changed, 17 insertions(+), 21 deletions(-) diff --git a/oam/install_scripts/syslogSetup.sh b/oam/install_scripts/syslogSetup.sh index e2d690fb4..703f62ed0 100755 --- a/oam/install_scripts/syslogSetup.sh +++ b/oam/install_scripts/syslogSetup.sh @@ -14,9 +14,22 @@ rsyslog7=0 user=`whoami 2>/dev/null` +#set default names groupname=adm username=syslog +# determine username/groupname + +if [ -f /var/log/messages ]; then + username=`stat -c "%U %G" /var/log/messages | awk '{print $1}'` + groupname=`stat -c "%U %G" /var/log/messages | awk '{print $2}'` +fi + +if [ -f /var/log/syslog ]; then + username=`stat -c "%U %G" /var/log/syslog | awk '{print $1}'` + groupname=`stat -c "%U %G" /var/log/syslog | awk '{print $2}'` +fi + for arg in "$@"; do if [ `expr -- "$arg" : '--prefix='` -eq 9 ]; then prefix="`echo $arg | awk -F= '{print $2}'`" @@ -161,8 +174,8 @@ makeDir() { test -d /var/log/mariadb/columnstore/archive || mkdir /var/log/mariadb/columnstore/archive >/dev/null 2>&1 test -d /var/log/mariadb/columnstore/corefiles || mkdir /var/log/mariadb/columnstore/corefiles >/dev/null 2>&1 test -d /var/log/mariadb/columnstore/trace || mkdir /var/log/mariadb/columnstore/trace >/dev/null 2>&1 - chmod 777 -R /var/log/mariadb/columnstore - chown $user:$user -R /var/log/mariadb + chmod 750 -R /var/log/mariadb/columnstore + chown $username:$groupname -R /var/log/mariadb } install() { @@ -170,9 +183,6 @@ makeDir checkSyslog if [ ! -z "$syslog_conf" ] ; then $installdir/bin/setConfig -d Installation SystemLogConfigFile ${syslog_conf} >/dev/null 2>&1 - if [ $user != "root" ]; then - chown $user:$user /home/$user/mariadb/columnstore/etc/* - fi if [ "$syslog_conf" == /etc/rsyslog.d/columnstore.conf ] || [ "$syslog_conf" == /etc/rsyslog.d/49-columnstore.conf ]; then @@ -188,21 +198,6 @@ if [ ! -z "$syslog_conf" ] ; then #set the syslog for ColumnStore logging # remove older version incase it was installed by previous build rm -rf /etc/rsyslog.d/columnstore.conf - - # determine username/groupname - - if [ -f /var/log/messages ]; then - user=`stat -c "%U %G" /var/log/messages | awk '{print $1}'` - group=`stat -c "%U %G" /var/log/messages | awk '{print $2}'` - fi - - if [ -f /var/log/syslog ]; then - user=`stat -c "%U %G" /var/log/syslog | awk '{print $1}'` - group=`stat -c "%U %G" /var/log/syslog | awk '{print $2}'` - fi - - # set permissions - chown $user:$group -R /var/log/mariadb > /dev/null 2>&1 if [ $rsyslog7 == 1 ]; then rm -f /etc/rsyslog.d/49-columnstore.conf @@ -210,6 +205,7 @@ if [ ! -z "$syslog_conf" ] ; then sed -i -e s/groupname/$groupname/g ${syslog_conf} sed -i -e s/username/$username/g ${syslog_conf} + chmod 644 ${syslog_conf} else cp ${columnstoreSyslogFile} ${syslog_conf} fi @@ -233,7 +229,7 @@ if [ ! -z "$syslog_conf" ] ; then if [ $? -eq 0 ]; then if [ -f ${syslog_conf}.columnstoreSave ] ; then #uninstall the syslog for ColumnStore logging - v -f ${syslog_conf} ${syslog_conf}.ColumnStoreBackup + mv -f ${syslog_conf} ${syslog_conf}.ColumnStoreBackup mv -f ${syslog_conf}.columnstoreSave ${syslog_conf} >/dev/null 2>&1 if [ ! -f ${syslog_conf} ] ; then cp ${syslog_conf}.ColumnStoreBackup ${syslog_conf} From 9bf8df6a6f120008281ff14ed937f08356fad390 Mon Sep 17 00:00:00 2001 From: David Hill Date: Mon, 19 Nov 2018 09:07:01 -0600 Subject: [PATCH 04/15] MCOL-1947 - change module and home alias --- oam/install_scripts/columnstoreAlias | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/oam/install_scripts/columnstoreAlias b/oam/install_scripts/columnstoreAlias index 255eb7e7e..623cf26b8 100644 --- a/oam/install_scripts/columnstoreAlias +++ b/oam/install_scripts/columnstoreAlias @@ -4,7 +4,7 @@ alias mcsmysql='/usr/local/mariadb/columnstore/mysql/bin/mysql --defaults-extra- alias ma=/usr/local/mariadb/columnstore/bin/mcsadmin alias mcsadmin=/usr/local/mariadb/columnstore/bin/mcsadmin alias cpimport=/usr/local/mariadb/columnstore/bin/cpimport -alias home='cd /usr/local/mariadb/columnstore' +alias mcshome='cd /usr/local/mariadb/columnstore' alias log='cd /var/log/mariadb/columnstore/' alias core='cd /var/log/mariadb/columnstore/corefiles' alias tmsg='tail -f /var/log/messages' @@ -14,4 +14,4 @@ alias terror='tail -f /var/log/mariadb/columnstore/err.log' alias twarning='tail -f /var/log/mariadb/columnstore/warning.log' alias tcrit='tail -f /var/log/mariadb/columnstore/crit.log' alias dbrm='cd /usr/local/mariadb/columnstore/data1/systemFiles/dbrm' -alias module='cat /usr/local/mariadb/columnstore/local/module' +alias mcsmodule='cat /usr/local/mariadb/columnstore/local/module' From 3bf269b3f7cf131c5195e353deda04f9edbf217a Mon Sep 17 00:00:00 2001 From: Roman Nozdrin Date: Tue, 20 Nov 2018 13:44:47 +0300 Subject: [PATCH 05/15] MCOL-1519 GROUP BY handler now uses an appropriate SELECT_LEX structure. Before that handler used an outter query SELECT_LEX that could give incorrect information for subquery. --- dbcon/mysql/ha_calpont.cpp | 14 +++++--------- dbcon/mysql/ha_calpont_execplan.cpp | 6 ++---- 2 files changed, 7 insertions(+), 13 deletions(-) diff --git a/dbcon/mysql/ha_calpont.cpp b/dbcon/mysql/ha_calpont.cpp index 03289ba60..e16a8ba83 100644 --- a/dbcon/mysql/ha_calpont.cpp +++ b/dbcon/mysql/ha_calpont.cpp @@ -1198,8 +1198,6 @@ void check_walk(const Item* item, void* arg) } } -#include - /*@brief create_calpont_group_by_handler- Creates handler*/ /*********************************************************** * DESCRIPTION: @@ -1221,19 +1219,19 @@ static group_by_handler* create_calpont_group_by_handler(THD* thd, Query* query) { ha_calpont_group_by_handler* handler = NULL; - LEX* lex = thd->lex; - SELECT_LEX *select_lex = &lex->select_lex; + SELECT_LEX *select_lex = query->from->select_lex; // Create a handler if query is valid. See comments for details. if ( thd->infinidb_vtable.vtable_state == THD::INFINIDB_DISABLE_VTABLE && ( thd->variables.infinidb_vtable_mode == 0 || thd->variables.infinidb_vtable_mode == 2 ) - && ( query->group_by || thd->lex->select_lex.with_sum_func ) ) + && ( query->group_by || select_lex->with_sum_func ) ) { bool unsupported_feature = false; // Impossible HAVING or WHERE - if ( ( select_lex->having && select_lex->having_value == Item::COND_FALSE ) - || ( select_lex->cond_value && select_lex->cond_value == Item::COND_FALSE ) ) + if ( ( query->having && select_lex->having_value == Item::COND_FALSE ) + || ( select_lex->cond_count > 0 + && select_lex->cond_value == Item::COND_FALSE ) ) { unsupported_feature = true; } @@ -1265,8 +1263,6 @@ create_calpont_group_by_handler(THD* thd, Query* query) } } - std::cerr << "create_calpont_group_by_handler handler " << handler << std::endl; - return handler; } diff --git a/dbcon/mysql/ha_calpont_execplan.cpp b/dbcon/mysql/ha_calpont_execplan.cpp index a872ace4f..6e1f223d5 100644 --- a/dbcon/mysql/ha_calpont_execplan.cpp +++ b/dbcon/mysql/ha_calpont_execplan.cpp @@ -8248,13 +8248,11 @@ int cp_get_table_plan(THD* thd, SCSEP& csep, cal_table_info& ti) int cp_get_group_plan(THD* thd, SCSEP& csep, cal_impl_if::cal_group_info& gi) { - LEX* lex = thd->lex; - idbassert(lex != 0); - SELECT_LEX select_lex = lex->select_lex; + SELECT_LEX *select_lex = gi.groupByTables->select_lex; gp_walk_info gwi; gwi.thd = thd; - int status = getGroupPlan(gwi, select_lex, csep, gi); + int status = getGroupPlan(gwi, *select_lex, csep, gi); #ifdef DEBUG_WALK_COND cerr << "---------------- cp_get_group_plan EXECUTION PLAN ----------------" << endl; From 575fdf22847cec41276fff8bf4530186d0eca803 Mon Sep 17 00:00:00 2001 From: David Hill Date: Tue, 20 Nov 2018 09:11:11 -0600 Subject: [PATCH 06/15] MCOL-1944 --- oam/install_scripts/syslogSetup.sh | 7 +++++-- 1 file changed, 5 insertions(+), 2 deletions(-) diff --git a/oam/install_scripts/syslogSetup.sh b/oam/install_scripts/syslogSetup.sh index 703f62ed0..c1d758396 100755 --- a/oam/install_scripts/syslogSetup.sh +++ b/oam/install_scripts/syslogSetup.sh @@ -11,6 +11,7 @@ prefix=/usr/local installdir=$prefix/mariadb/columnstore syslog_conf=nofile rsyslog7=0 +non-root-user="no" user=`whoami 2>/dev/null` @@ -41,6 +42,7 @@ for arg in "$@"; do user="`echo $arg | awk -F= '{print $2}'`" groupname=$user username=$user + non-root-user="yes" elif [ `expr -- "$arg" : '--..*'` -ge 3 ]; then echo "ignoring unknown argument: $arg" 1>&2 elif [ `expr -- "$arg" : '--'` -eq 2 ]; then @@ -174,7 +176,9 @@ makeDir() { test -d /var/log/mariadb/columnstore/archive || mkdir /var/log/mariadb/columnstore/archive >/dev/null 2>&1 test -d /var/log/mariadb/columnstore/corefiles || mkdir /var/log/mariadb/columnstore/corefiles >/dev/null 2>&1 test -d /var/log/mariadb/columnstore/trace || mkdir /var/log/mariadb/columnstore/trace >/dev/null 2>&1 - chmod 750 -R /var/log/mariadb/columnstore + if [ $non-root-user == "yes" ]; then + chmod 777 /var/log/mariadb/columnstore + fi chown $username:$groupname -R /var/log/mariadb } @@ -248,7 +252,6 @@ if [ ! -z "$syslog_conf" ] ; then rm -f /etc/logrotate.d/columnstore restartSyslog - fi } From 332d07328e34498c3c01c9e8a85d49ab04f05249 Mon Sep 17 00:00:00 2001 From: David Hill Date: Tue, 20 Nov 2018 09:35:02 -0600 Subject: [PATCH 07/15] MCOL-1944 --- oam/install_scripts/syslogSetup.sh | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) diff --git a/oam/install_scripts/syslogSetup.sh b/oam/install_scripts/syslogSetup.sh index c1d758396..84980eef8 100755 --- a/oam/install_scripts/syslogSetup.sh +++ b/oam/install_scripts/syslogSetup.sh @@ -11,7 +11,7 @@ prefix=/usr/local installdir=$prefix/mariadb/columnstore syslog_conf=nofile rsyslog7=0 -non-root-user="no" +non_root_user=no user=`whoami 2>/dev/null` @@ -42,7 +42,7 @@ for arg in "$@"; do user="`echo $arg | awk -F= '{print $2}'`" groupname=$user username=$user - non-root-user="yes" + non_root_user=yes elif [ `expr -- "$arg" : '--..*'` -ge 3 ]; then echo "ignoring unknown argument: $arg" 1>&2 elif [ `expr -- "$arg" : '--'` -eq 2 ]; then @@ -176,7 +176,7 @@ makeDir() { test -d /var/log/mariadb/columnstore/archive || mkdir /var/log/mariadb/columnstore/archive >/dev/null 2>&1 test -d /var/log/mariadb/columnstore/corefiles || mkdir /var/log/mariadb/columnstore/corefiles >/dev/null 2>&1 test -d /var/log/mariadb/columnstore/trace || mkdir /var/log/mariadb/columnstore/trace >/dev/null 2>&1 - if [ $non-root-user == "yes" ]; then + if [ $non_root_user == "yes" ]; then chmod 777 /var/log/mariadb/columnstore fi chown $username:$groupname -R /var/log/mariadb From 4d70ccd679db034124345e2c12edc7bbd37f5912 Mon Sep 17 00:00:00 2001 From: David Hill Date: Tue, 20 Nov 2018 13:21:34 -0600 Subject: [PATCH 08/15] MCOL-1944 - fix issues with logging after upgrade --- oam/install_scripts/syslogSetup.sh | 20 ++++++++++++++------ 1 file changed, 14 insertions(+), 6 deletions(-) diff --git a/oam/install_scripts/syslogSetup.sh b/oam/install_scripts/syslogSetup.sh index 84980eef8..a900c2e84 100755 --- a/oam/install_scripts/syslogSetup.sh +++ b/oam/install_scripts/syslogSetup.sh @@ -172,14 +172,19 @@ fi } makeDir() { - test -d /var/log/mariadb/columnstore || mkdir -p /var/log/mariadb/columnstore >/dev/null 2>&1 + if [ ! -d /var/log/mariadb/columnstore ];then + mkdir -p /var/log/mariadb/columnstore >/dev/null 2>&1 + chown $username:$groupname -R /var/log/mariadb + fi + + if [ $non_root_user == "yes" ]; then + chmod 777 /var/log/mariadb + chmod 777 /var/log/mariadb/columnstore + fi + test -d /var/log/mariadb/columnstore/archive || mkdir /var/log/mariadb/columnstore/archive >/dev/null 2>&1 test -d /var/log/mariadb/columnstore/corefiles || mkdir /var/log/mariadb/columnstore/corefiles >/dev/null 2>&1 test -d /var/log/mariadb/columnstore/trace || mkdir /var/log/mariadb/columnstore/trace >/dev/null 2>&1 - if [ $non_root_user == "yes" ]; then - chmod 777 /var/log/mariadb/columnstore - fi - chown $username:$groupname -R /var/log/mariadb } install() { @@ -187,12 +192,15 @@ makeDir checkSyslog if [ ! -z "$syslog_conf" ] ; then $installdir/bin/setConfig -d Installation SystemLogConfigFile ${syslog_conf} >/dev/null 2>&1 + if [ $non_root_user == "yes" ]; then + chown $user:$user $installdir/etc/Columnstore.xml* + fi + rm -f ${syslog_conf}.columnstoreSave if [ "$syslog_conf" == /etc/rsyslog.d/columnstore.conf ] || [ "$syslog_conf" == /etc/rsyslog.d/49-columnstore.conf ]; then i=1 else - rm -f ${syslog_conf}.columnstoreSave cp ${syslog_conf} ${syslog_conf}.columnstoreSave >/dev/null 2>&1 sed -i '/# MariaDB/,$d' ${syslog_conf}.columnstoreSave > /dev/null 2>&1 fi From 05ce8e24ef88cfcacdeea58e56da243a52434620 Mon Sep 17 00:00:00 2001 From: David Hill Date: Tue, 20 Nov 2018 15:13:20 -0600 Subject: [PATCH 09/15] MCOL-1944 - do chmod on all directories --- oam/install_scripts/syslogSetup.sh | 12 ++++++++---- 1 file changed, 8 insertions(+), 4 deletions(-) diff --git a/oam/install_scripts/syslogSetup.sh b/oam/install_scripts/syslogSetup.sh index a900c2e84..31f457505 100755 --- a/oam/install_scripts/syslogSetup.sh +++ b/oam/install_scripts/syslogSetup.sh @@ -174,17 +174,21 @@ fi makeDir() { if [ ! -d /var/log/mariadb/columnstore ];then mkdir -p /var/log/mariadb/columnstore >/dev/null 2>&1 + test -d /var/log/mariadb/columnstore/archive || mkdir /var/log/mariadb/columnstore/archive >/dev/null 2>&1 + test -d /var/log/mariadb/columnstore/corefiles || mkdir /var/log/mariadb/columnstore/corefiles >/dev/null 2>&1 + test -d /var/log/mariadb/columnstore/trace || mkdir /var/log/mariadb/columnstore/trace >/dev/null 2>&1 chown $username:$groupname -R /var/log/mariadb + else + test -d /var/log/mariadb/columnstore/archive || mkdir /var/log/mariadb/columnstore/archive >/dev/null 2>&1 + test -d /var/log/mariadb/columnstore/corefiles || mkdir /var/log/mariadb/columnstore/corefiles >/dev/null 2>&1 + test -d /var/log/mariadb/columnstore/trace || mkdir /var/log/mariadb/columnstore/trace >/dev/null 2>&1 fi - + if [ $non_root_user == "yes" ]; then chmod 777 /var/log/mariadb chmod 777 /var/log/mariadb/columnstore fi - test -d /var/log/mariadb/columnstore/archive || mkdir /var/log/mariadb/columnstore/archive >/dev/null 2>&1 - test -d /var/log/mariadb/columnstore/corefiles || mkdir /var/log/mariadb/columnstore/corefiles >/dev/null 2>&1 - test -d /var/log/mariadb/columnstore/trace || mkdir /var/log/mariadb/columnstore/trace >/dev/null 2>&1 } install() { From bb096737c3d216b2f4a39b425745d29a8465e1f2 Mon Sep 17 00:00:00 2001 From: David Hill Date: Tue, 20 Nov 2018 16:47:22 -0600 Subject: [PATCH 10/15] MCOL-1944 - move cplogger from post-install to syslogSetup --- oam/install_scripts/syslogSetup.sh | 5 +++++ 1 file changed, 5 insertions(+) diff --git a/oam/install_scripts/syslogSetup.sh b/oam/install_scripts/syslogSetup.sh index 31f457505..da8bd7684 100755 --- a/oam/install_scripts/syslogSetup.sh +++ b/oam/install_scripts/syslogSetup.sh @@ -232,6 +232,11 @@ if [ ! -z "$syslog_conf" ] ; then chmod 644 /etc/logrotate.d/columnstore restartSyslog + + #log install message + test -f $installdir/post/functions && . $installdir/post/functions + LD_LIBRARY_PATH=$LD_LIBRARY_PATH:$installdir/lib $installdir/bin/cplogger -i 19 "***** MariaDB Columnstore Installed *****" + fi } From 430f7c494cd69482b5ca8069486e39e8b87caddc Mon Sep 17 00:00:00 2001 From: David Hill Date: Wed, 21 Nov 2018 10:25:19 -0600 Subject: [PATCH 11/15] MCOL-1944 - move cplogger to syslogSetup.sh --- oam/install_scripts/post-install | 8 -------- oam/install_scripts/syslogSetup.sh | 32 ++++++++++++++++++++++-------- 2 files changed, 24 insertions(+), 16 deletions(-) diff --git a/oam/install_scripts/post-install b/oam/install_scripts/post-install index 6fac1eaae..d82bbb953 100755 --- a/oam/install_scripts/post-install +++ b/oam/install_scripts/post-install @@ -315,14 +315,6 @@ if [ -z "aws" ]; then $installdir/bin/MCSgetCredentials.sh >/dev/null 2>&1 fi -#log install message -test -f $installdir/post/functions && . $installdir/post/functions -if [ $user = "root" ]; then - $installdir/bin/cplogger -i 19 "***** MariaDB Columnstore Installed *****" -else - LD_LIBRARY_PATH=$LD_LIBRARY_PATH:$installdir/lib $installdir/bin/cplogger -i 19 "***** MariaDB Columnstore Installed *****" -fi - #setup hadoop hadoop=`which hadoop 2>/dev/null` if [ -z "$hadoop" ]; then diff --git a/oam/install_scripts/syslogSetup.sh b/oam/install_scripts/syslogSetup.sh index da8bd7684..22cf99054 100755 --- a/oam/install_scripts/syslogSetup.sh +++ b/oam/install_scripts/syslogSetup.sh @@ -227,16 +227,32 @@ if [ ! -z "$syslog_conf" ] ; then fi fi - # install Columnstore Log Rotate File - cp $installdir/bin/columnstoreLogRotate /etc/logrotate.d/columnstore > /dev/null 2>&1 - chmod 644 /etc/logrotate.d/columnstore - restartSyslog - - #log install message - test -f $installdir/post/functions && . $installdir/post/functions - LD_LIBRARY_PATH=$LD_LIBRARY_PATH:$installdir/lib $installdir/bin/cplogger -i 19 "***** MariaDB Columnstore Installed *****" + # install Columnstore Log Rotate File + if [ -d /etc/logrotate.d ]; then + cp $installdir/bin/columnstoreLogRotate /etc/logrotate.d/columnstore > /dev/null 2>&1 + chmod 644 /etc/logrotate.d/columnstore + + #do the logrotate to start with a fresh log file during install + logrotate -f /etc/logrotate.d/columnstore > /dev/null 2>&1 + fi + + #log install message and find the least permission that allows logging to work + CHMOD_LIST=("750" "770" "775" "777") + for CHMOD in "${CHMOD_LIST[@]}"; do + chmod $CHMOD /var/log/mariadb + chmod $CHMOD /var/log/mariadb/columnstore + + test -f $installdir/post/functions && . $installdir/post/functions + LD_LIBRARY_PATH=$LD_LIBRARY_PATH:$installdir/lib $installdir/bin/cplogger -i 19 "***** MariaDB Columnstore Installed *****" + + if [ -f /var/log/mariadb/columnstore/info.log ]; then + if [ ! -s /var/log/mariadb/columnstore/info.log ]; then + break + fi + fi + done fi } From 6563f48e32d26df2e5df6bd152cb300eb32a44cd Mon Sep 17 00:00:00 2001 From: Roman Nozdrin Date: Tue, 20 Nov 2018 20:47:11 +0300 Subject: [PATCH 12/15] MCOL-1786 Reduce the performance degradation caused by iequals. --- writeengine/bulk/we_bulkloadbuffer.cpp | 114 +++++++++++-------------- writeengine/bulk/we_bulkloadbuffer.h | 9 ++ 2 files changed, 59 insertions(+), 64 deletions(-) diff --git a/writeengine/bulk/we_bulkloadbuffer.cpp b/writeengine/bulk/we_bulkloadbuffer.cpp index 877f0b033..da568f9a6 100644 --- a/writeengine/bulk/we_bulkloadbuffer.cpp +++ b/writeengine/bulk/we_bulkloadbuffer.cpp @@ -31,8 +31,6 @@ #include #include -#include - #include "we_bulkload.h" #include "we_bulkloadbuffer.h" #include "we_brm.h" @@ -370,18 +368,11 @@ void BulkLoadBuffer::convert(char* field, int fieldLength, { errno = 0; - if (iequals(field, "true")) - { - fVal = 1; - } - else - { #ifdef _MSC_VER - fVal = (float)strtod( field, 0 ); + fVal = (float)strtod( field, 0 ); #else - fVal = strtof( field, 0 ); + fVal = strtof( field, 0 ); #endif - } if (errno == ERANGE) { @@ -414,6 +405,11 @@ void BulkLoadBuffer::convert(char* field, int fieldLength, fVal = minFltSat; bufStats.satCount++; } + if ( fVal == 0 + && isTrueWord(const_cast(field), fieldLength) ) + { + fVal = 1; + } } } @@ -474,14 +470,7 @@ void BulkLoadBuffer::convert(char* field, int fieldLength, { errno = 0; - if (iequals(field, "true")) - { - dVal = 1; - } - else - { - dVal = strtod(field, 0); - } + dVal = strtod(field, 0); if (errno == ERANGE) { @@ -514,6 +503,11 @@ void BulkLoadBuffer::convert(char* field, int fieldLength, dVal = column.fMinDblSat; bufStats.satCount++; } + else if (dVal == 0 + && isTrueWord(const_cast(field), fieldLength)) + { + dVal = 1; + } } } @@ -629,12 +623,6 @@ void BulkLoadBuffer::convert(char* field, int fieldLength, } else { - if (iequals(field, "true")) - { - strcpy(field, "1"); - fieldLength = 1; - } - if ( (column.dataType == CalpontSystemCatalog::DECIMAL ) || (column.dataType == CalpontSystemCatalog::UDECIMAL) ) { @@ -664,6 +652,11 @@ void BulkLoadBuffer::convert(char* field, int fieldLength, origVal = static_cast(column.fMaxIntSat); bSatVal = true; } + else if ( origVal == 0 + && isTrueWord(const_cast(field), fieldLength) ) + { + origVal = 1; + } if (bSatVal) bufStats.satCount++; @@ -722,15 +715,8 @@ void BulkLoadBuffer::convert(char* field, int fieldLength, { errno = 0; - if (iequals(field, "true")) - { - origVal = 1; - } - else - { - origVal = strtoll(field, 0, 10); - } - + origVal = strtoll(field, 0, 10); + if (errno == ERANGE) bSatVal = true; } @@ -747,6 +733,11 @@ void BulkLoadBuffer::convert(char* field, int fieldLength, origVal = static_cast(column.fMaxIntSat); bSatVal = true; } + else if ( origVal == 0 + && isTrueWord(const_cast(field), fieldLength) ) + { + origVal = 1; + } if (bSatVal) bufStats.satCount++; @@ -805,12 +796,12 @@ void BulkLoadBuffer::convert(char* field, int fieldLength, } else { - if (iequals(field, "true")) + if (isTrueWord(const_cast(field), fieldLength)) { strcpy(field, "1"); fieldLength = 1; } - + if ( (column.dataType == CalpontSystemCatalog::DECIMAL ) || (column.dataType == CalpontSystemCatalog::UDECIMAL)) { @@ -898,14 +889,7 @@ void BulkLoadBuffer::convert(char* field, int fieldLength, { errno = 0; - if (iequals(field, "true")) - { - origVal = 1; - } - else - { - origVal = strtoll(field, 0, 10); - } + origVal = strtoll(field, 0, 10); if (errno == ERANGE) bSatVal = true; @@ -923,6 +907,11 @@ void BulkLoadBuffer::convert(char* field, int fieldLength, origVal = static_cast(column.fMaxIntSat); bSatVal = true; } + else if ( origVal == 0 + && isTrueWord(const_cast(field), fieldLength) ) + { + origVal = 1; + } if (bSatVal) bufStats.satCount++; @@ -981,12 +970,12 @@ void BulkLoadBuffer::convert(char* field, int fieldLength, } else { - if (iequals(field, "true")) + if (isTrueWord(const_cast(field), fieldLength)) { strcpy(field, "1"); fieldLength = 1; } - + if ( (column.dataType == CalpontSystemCatalog::DECIMAL) || (column.dataType == CalpontSystemCatalog::UDECIMAL)) { @@ -1018,6 +1007,7 @@ void BulkLoadBuffer::convert(char* field, int fieldLength, bSatVal = true; } + if (bSatVal) bufStats.satCount++; @@ -1199,14 +1189,7 @@ void BulkLoadBuffer::convert(char* field, int fieldLength, { errno = 0; - if (iequals(field, "true")) - { - ullVal = 1; - } - else - { - ullVal = strtoull(field, 0, 10); - } + ullVal = strtoull(field, 0, 10); if (errno == ERANGE) bSatVal = true; @@ -1220,6 +1203,11 @@ void BulkLoadBuffer::convert(char* field, int fieldLength, ullVal = column.fMaxIntSat; bSatVal = true; } + else if ( ullVal == 0 + && isTrueWord(const_cast(field), fieldLength) ) + { + ullVal = 1; + } if (bSatVal) bufStats.satCount++; @@ -1276,14 +1264,7 @@ void BulkLoadBuffer::convert(char* field, int fieldLength, { errno = 0; - if (iequals(field, "true")) - { - origVal = 1; - } - else - { - origVal = strtoll(field, 0, 10); - } + origVal = strtoll(field, 0, 10); if (errno == ERANGE) bSatVal = true; @@ -1301,6 +1282,11 @@ void BulkLoadBuffer::convert(char* field, int fieldLength, origVal = static_cast(column.fMaxIntSat); bSatVal = true; } + else if ( origVal == 0 + && isTrueWord(const_cast(field), fieldLength) ) + { + origVal = 1; + } if (bSatVal) bufStats.satCount++; @@ -1361,12 +1347,12 @@ void BulkLoadBuffer::convert(char* field, int fieldLength, } else { - if (iequals(field, "true")) + if (isTrueWord(const_cast(field), fieldLength)) { strcpy(field, "1"); fieldLength = 1; } - + if ( (column.dataType == CalpontSystemCatalog::DECIMAL) || (column.dataType == CalpontSystemCatalog::UDECIMAL)) { diff --git a/writeengine/bulk/we_bulkloadbuffer.h b/writeengine/bulk/we_bulkloadbuffer.h index 85d60980c..649e9847e 100644 --- a/writeengine/bulk/we_bulkloadbuffer.h +++ b/writeengine/bulk/we_bulkloadbuffer.h @@ -383,5 +383,14 @@ public: } }; +inline bool isTrueWord(const char *field, int fieldLength) +{ + //return false; + return fieldLength == 4 && ( field[0] == 'T' || field[0] == 't' ) + && ( field[1] == 'R' || field[1] == 'r' ) + && ( field[2] == 'U' || field[2] == 'u' ) + && ( field[3] == 'E' || field[3] == 'e' ); +} + } #endif From 7c1853feab6f55874e8e2306f9870b2c44b10e5a Mon Sep 17 00:00:00 2001 From: David Hill Date: Wed, 21 Nov 2018 10:52:48 -0600 Subject: [PATCH 13/15] MCOL-1944 - move cplogger to syslogSetup.sh --- oam/install_scripts/syslogSetup.sh | 9 +++------ 1 file changed, 3 insertions(+), 6 deletions(-) diff --git a/oam/install_scripts/syslogSetup.sh b/oam/install_scripts/syslogSetup.sh index 22cf99054..7b088f522 100755 --- a/oam/install_scripts/syslogSetup.sh +++ b/oam/install_scripts/syslogSetup.sh @@ -183,12 +183,6 @@ makeDir() { test -d /var/log/mariadb/columnstore/corefiles || mkdir /var/log/mariadb/columnstore/corefiles >/dev/null 2>&1 test -d /var/log/mariadb/columnstore/trace || mkdir /var/log/mariadb/columnstore/trace >/dev/null 2>&1 fi - - if [ $non_root_user == "yes" ]; then - chmod 777 /var/log/mariadb - chmod 777 /var/log/mariadb/columnstore - fi - } install() { @@ -243,6 +237,9 @@ if [ ! -z "$syslog_conf" ] ; then for CHMOD in "${CHMOD_LIST[@]}"; do chmod $CHMOD /var/log/mariadb chmod $CHMOD /var/log/mariadb/columnstore + chmod $CHMOD /var/log/mariadb/columnstore/archive + chmod $CHMOD /var/log/mariadb/columnstore/corefiles + chmod $CHMOD /var/log/mariadb/columnstore/trace test -f $installdir/post/functions && . $installdir/post/functions LD_LIBRARY_PATH=$LD_LIBRARY_PATH:$installdir/lib $installdir/bin/cplogger -i 19 "***** MariaDB Columnstore Installed *****" From bb23798e7413b026828065b8c5d0dba9a35496a3 Mon Sep 17 00:00:00 2001 From: David Hill Date: Wed, 21 Nov 2018 11:10:27 -0600 Subject: [PATCH 14/15] MCOL-1944 - move cplogger to syslogSetup.sh --- oam/install_scripts/syslogSetup.sh | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/oam/install_scripts/syslogSetup.sh b/oam/install_scripts/syslogSetup.sh index 7b088f522..b937b7ab2 100755 --- a/oam/install_scripts/syslogSetup.sh +++ b/oam/install_scripts/syslogSetup.sh @@ -172,7 +172,7 @@ fi } makeDir() { - if [ ! -d /var/log/mariadb/columnstore ];then + if [ ! -d /var/log/mariadb/columnstore ]; then mkdir -p /var/log/mariadb/columnstore >/dev/null 2>&1 test -d /var/log/mariadb/columnstore/archive || mkdir /var/log/mariadb/columnstore/archive >/dev/null 2>&1 test -d /var/log/mariadb/columnstore/corefiles || mkdir /var/log/mariadb/columnstore/corefiles >/dev/null 2>&1 @@ -246,7 +246,7 @@ if [ ! -z "$syslog_conf" ] ; then if [ -f /var/log/mariadb/columnstore/info.log ]; then if [ ! -s /var/log/mariadb/columnstore/info.log ]; then - break + exit 0 fi fi done From b422396371f2ef431b077b9f305c23a40482cbac Mon Sep 17 00:00:00 2001 From: David Hill Date: Wed, 21 Nov 2018 11:43:05 -0600 Subject: [PATCH 15/15] MCOL-1944 - move cplogger to syslogSetup.sh --- oam/install_scripts/syslogSetup.sh | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/oam/install_scripts/syslogSetup.sh b/oam/install_scripts/syslogSetup.sh index b937b7ab2..babbd1c88 100755 --- a/oam/install_scripts/syslogSetup.sh +++ b/oam/install_scripts/syslogSetup.sh @@ -245,7 +245,7 @@ if [ ! -z "$syslog_conf" ] ; then LD_LIBRARY_PATH=$LD_LIBRARY_PATH:$installdir/lib $installdir/bin/cplogger -i 19 "***** MariaDB Columnstore Installed *****" if [ -f /var/log/mariadb/columnstore/info.log ]; then - if [ ! -s /var/log/mariadb/columnstore/info.log ]; then + if [ -s /var/log/mariadb/columnstore/info.log ]; then exit 0 fi fi