From 436a99a8fd4a59bd7288da59172dc711aaea8d54 Mon Sep 17 00:00:00 2001 From: david hill Date: Wed, 29 Nov 2017 08:54:44 -0600 Subject: [PATCH 1/6] MCOL-1055- backport clustertester fixes from 1.1 --- oam/install_scripts/remote_scp_get.sh | 4 +- oam/install_scripts/remote_scp_put.sh | 4 +- .../clusterTester/columnstoreClusterTester.sh | 63 ++++++++++--------- 3 files changed, 41 insertions(+), 30 deletions(-) diff --git a/oam/install_scripts/remote_scp_get.sh b/oam/install_scripts/remote_scp_get.sh index 703cc4b90..61a1de219 100755 --- a/oam/install_scripts/remote_scp_get.sh +++ b/oam/install_scripts/remote_scp_get.sh @@ -7,7 +7,9 @@ # Argument 2 - Remote Server root password # Argument 3 - Command set timeout 10 -set USERNAME $env(USER)"@" +exec whoami >/tmp/whoami.tmp +set USERNAME [exec cat /tmp/whoami.tmp] +exec rm -f /tmp/whoami.tmp set SERVER [lindex $argv 0] set PASSWORD [lindex $argv 1] set FILE [lindex $argv 2] diff --git a/oam/install_scripts/remote_scp_put.sh b/oam/install_scripts/remote_scp_put.sh index a97d907ad..c94ea666e 100755 --- a/oam/install_scripts/remote_scp_put.sh +++ b/oam/install_scripts/remote_scp_put.sh @@ -7,7 +7,9 @@ # Argument 2 - Remote Server root password # Argument 3 - Command set timeout 30 -set USERNAME $env(USER)"@" +exec whoami >/tmp/whoami.tmp +set USERNAME [exec cat /tmp/whoami.tmp] +exec rm -f /tmp/whoami.tmp set SERVER [lindex $argv 0] set PASSWORD [lindex $argv 1] set FILE [lindex $argv 2] diff --git a/utils/clusterTester/columnstoreClusterTester.sh b/utils/clusterTester/columnstoreClusterTester.sh index 89f0e34c8..acc39d23a 100755 --- a/utils/clusterTester/columnstoreClusterTester.sh +++ b/utils/clusterTester/columnstoreClusterTester.sh @@ -14,6 +14,10 @@ OS_LIST=("centos6" "centos7" "debian8" "debian9" "suse12" "ubuntu16") NODE_IPADDRESS="" +if [ -z "$COLUMNSTORE_INSTALL_DIR" ]; then + COLUMNSTORE_INSTALL_DIR=/usr/local/mariadb/columnstore +fi + checkContinue() { if [ "$CHECK" = false ]; then @@ -198,7 +202,7 @@ checkLocalOS() echo "" #get local OS - `./os_detect.sh > /tmp/os_detect 2>&1` + `$COLUMNSTORE_INSTALL_DIR/bin/os_detect.sh > /tmp/os_detect 2>&1` if [ "$?" -eq 0 ]; then localOS=`cat /tmp/os_detect | grep "Operating System name" | cut -f2 -d '"'` echo "Local Node OS System Name : $localOS" @@ -318,7 +322,7 @@ checkSSH() echo "" for ipadd in "${NODE_IPADDRESS[@]}"; do - `./remote_command.sh $ipadd $PASSWORD ls 1 > /dev/null 2>&1`; + `$COLUMNSTORE_INSTALL_DIR/bin/remote_command.sh $ipadd $PASSWORD ls 1 > /dev/null 2>&1`; rc="$?" if [ $rc -eq 0 ] || ( [ $rc -eq 2 ] && [ $OS == "suse12" ] ) ; then if [ $PASSWORD == "ssh" ] ; then @@ -345,7 +349,7 @@ checkRemoteDir() `sudo rm -f /tmp/*_check > /dev/null 2>&1` for ipadd in "${NODE_IPADDRESS[@]}"; do - `./remote_command.sh $ipadd $PASSWORD 'sudo rm -f /tmp/*_check > /dev/null 2>&1' 1 > /tmp/remote_command_check 2>&1` + `$COLUMNSTORE_INSTALL_DIR/bin/remote_command.sh $ipadd $PASSWORD 'sudo rm -f /tmp/*_check > /dev/null 2>&1' 1 > /tmp/remote_command_check 2>&1` done if [ "$USER" != "root" ]; then @@ -356,7 +360,7 @@ checkRemoteDir() echo "" for ipadd in "${NODE_IPADDRESS[@]}"; do - `./remote_command.sh $ipadd $PASSWORD 'touch /tmp/cs_check' 1 > /tmp/remote_command_check 2>&1` + `$COLUMNSTORE_INSTALL_DIR/bin/remote_command.sh $ipadd $PASSWORD 'touch /tmp/cs_check' 1 > /tmp/remote_command_check 2>&1` rc="$?" if [ $rc -eq 0 ] || ( [ $rc -eq 2 ] && [ $OS == "suse12" ] ) ; then `grep "Permission denied" /tmp/remote_command_check > /dev/null 2>&1` @@ -372,7 +376,7 @@ checkRemoteDir() REPORTPASS=false fi - `./remote_command.sh $ipadd $PASSWORD 'touch /dev/shm/cs_check' 1 > /tmp/remote_command_check 2>&1` + `$COLUMNSTORE_INSTALL_DIR/bin/remote_command.sh $ipadd $PASSWORD 'touch /dev/shm/cs_check' 1 > /tmp/remote_command_check 2>&1` rc="$?" if [ $rc -eq 0 ] || ( [ $rc -eq 2 ] && [ $OS == "suse12" ] ) ; then `grep "Permission denied" /tmp/remote_command_check > /dev/null 2>&1` @@ -408,25 +412,26 @@ checkOS() echo "" pass=true + `/bin/cp -f $COLUMNSTORE_INSTALL_DIR/bin/os_detect.sh /tmp/.` for ipadd in "${NODE_IPADDRESS[@]}"; do - `./remote_scp_put.sh $ipadd $PASSWORD os_detect.sh 1 > /tmp/remote_scp_put_check 2>&1` + `$COLUMNSTORE_INSTALL_DIR/bin/remote_scp_put.sh $ipadd $PASSWORD /tmp/os_detect.sh 1 > /tmp/remote_scp_put_check 2>&1` if [ "$?" -ne 0 ]; then echo "Error running remote_scp_put.sh to $ipadd Node, check /tmp/remote_scp_put_check" + exit 1 else - `./remote_command.sh $ipadd $PASSWORD './os_detect.sh > /tmp/os_detect 2>&1' 1 > /tmp/remote_command_check` + `$COLUMNSTORE_INSTALL_DIR/bin/remote_command.sh $ipadd $PASSWORD /tmp/os_detect.sh 1 > /tmp/remote_command_check` rc="$?" - `./remote_scp_get.sh $ipadd $PASSWORD /tmp/os_detect > /tmp/remote_scp_get_check 2>&1` if [ "$?" -ne 0 ]; then - echo "Error running remote_scp_get.sh to $ipadd Node, check /tmp/remote_scp_get_check" + echo "Error running remote_command.sh /tmp/os_detect.sh on $ipadd Node, check /tmp/remote_command_check" + exit 1 else - remoteOS=`cat os_detect | grep "Operating System name" | cut -f2 -d '"'` + remoteOS=`cat /tmp/remote_command_check | grep "Operating System name" | cut -f2 -d '"'` echo "$ipadd Node OS Version : $remoteOS" if [ $localOS != $remoteOS ]; then echo "${bold}Failed${normal}, $ipadd has a different OS than local node" pass=false REPORTPASS=false fi - rm -f os_detect fi fi done @@ -454,12 +459,13 @@ checkLocale() pass=true for ipadd in "${NODE_IPADDRESS[@]}"; do - `./remote_command.sh $ipadd $PASSWORD 'locale | grep LANG= > /tmp/locale_check 2>&1' 1 > /tmp/remote_command_check` + `$COLUMNSTORE_INSTALL_DIR/bin/remote_command.sh $ipadd $PASSWORD 'locale | grep LANG= > /tmp/locale_check 2>&1' 1 > /tmp/remote_command_check` rc="$?" if [ $rc -eq 0 ] || ( [ $rc -eq 2 ] && [ $OS == "suse12" ] ) ; then - `./remote_scp_get.sh $ipadd $PASSWORD /tmp/locale_check > /tmp/remote_scp_get_check 2>&1` + `$COLUMNSTORE_INSTALL_DIR/bin/remote_scp_get.sh $ipadd $PASSWORD /tmp/locale_check > /tmp/remote_scp_get_check 2>&1` if [ "$?" -ne 0 ]; then echo "Error running remote_scp_get.sh to $ipadd Node, check /tmp/remote_scp_get_check" + exit 1 else echo "$ipadd Node Locale : `cat locale_check`" `diff /tmp/locale_check locale_check > /dev/null 2>&1` @@ -472,6 +478,7 @@ checkLocale() fi else echo "Error running remote_command.sh to $ipadd Node, check /tmp/remote_command_check" + exit 1 pass=false REPORTPASS=false fi @@ -506,7 +513,7 @@ checkSELINUX() fi for ipadd in "${NODE_IPADDRESS[@]}"; do - `./remote_scp_get.sh $ipadd $PASSWORD /etc/selinux/config > /tmp/remote_scp_get_check 2>&1` + `$COLUMNSTORE_INSTALL_DIR/bin/remote_scp_get.sh $ipadd $PASSWORD /etc/selinux/config > /tmp/remote_scp_get_check 2>&1` if [ "$?" -ne 0 ]; then echo "$ipadd Node SELINUX setting is Not Enabled" else @@ -572,14 +579,14 @@ checkFirewalls() # 'sysconfig not on remote node for firewall in "${FIREWALL_LIST[@]}"; do pass=true - `./remote_command.sh $ipadd $PASSWORD "service '$firewall' status > /tmp/firewall_check 2>&1" 1 > /tmp/remote_command_check` + `$COLUMNSTORE_INSTALL_DIR/bin/remote_command.sh $ipadd $PASSWORD "service '$firewall' status > /tmp/firewall_check 2>&1" 1 > /tmp/remote_command_check` if [ "$?" -eq 0 ]; then echo "${bold}Failed${normal}, $ipadd Node $firewall service is Active, please disable" pass=false fpass=false REPORTPASS=false else - `./remote_command.sh $ipadd $PASSWORD "systemctl status '$firewall' > /tmp/firewall_check 2>&1" 1 > /tmp/remote_command_check` + `$COLUMNSTORE_INSTALL_DIR/bin/remote_command.sh $ipadd $PASSWORD "systemctl status '$firewall' > /tmp/firewall_check 2>&1" 1 > /tmp/remote_command_check` if [ "$?" -eq 0 ]; then echo "${bold}Failed${normal}, $ipadd Node $firewall service is Active, please disable" pass=false @@ -619,7 +626,7 @@ checkFirewalls() fi for ipadd in "${NODE_IPADDRESS[@]}"; do - `./remote_command.sh $ipadd $PASSWORD '/sbin/rcSuSEfirewall2 status > /tmp/rcSuSEfirewall2_check 2>&1' 1 > /tmp/remote_command_check` + `$COLUMNSTORE_INSTALL_DIR/bin/remote_command.sh $ipadd $PASSWORD '/sbin/rcSuSEfirewall2 status > /tmp/rcSuSEfirewall2_check 2>&1' 1 > /tmp/remote_command_check` rc="$?" if [ $rc -eq 0 ] ; then echo "${bold}Failed${normal}, $ipadd Node rcSuSEfirewall2 service is Enabled, please disable" @@ -674,14 +681,14 @@ checkTime() #get local epoch time localTime=`date +%s` for ipadd in "${NODE_IPADDRESS[@]}"; do - `./remote_command.sh $ipadd $PASSWORD 'date +%s > /tmp/time_check' > /tmp/time_check` + `$COLUMNSTORE_INSTALL_DIR/bin/remote_command.sh $ipadd $PASSWORD 'date +%s > /tmp/time_check' > /tmp/time_check` rc="$?" if [ $rc -ne 0 ] ; then echo $ipadd " Node ${bold}Failed${normal} date/time check failed, check /tmp/time_check" pass=false REPORTPASS=false else - `./remote_scp_get.sh $ipadd $PASSWORD /tmp/time_check > /tmp/remote_scp_get_check 2>&1` + `$COLUMNSTORE_INSTALL_DIR/bin/remote_scp_get.sh $ipadd $PASSWORD /tmp/time_check > /tmp/remote_scp_get_check 2>&1` if [ "$?" -ne 0 ]; then echo "Error running remote_scp_get.sh to $ipadd Node, check /tmp/remote_scp_get_check" else @@ -756,14 +763,14 @@ checkPackages() for ipadd in "${NODE_IPADDRESS[@]}"; do for PKG in "${CENTOS_PKG[@]}"; do if [ $OS == "centos6" ] && [ $PKG == "boost" ]; then - `./remote_command.sh $ipadd $PASSWORD 'ls /usr/lib/libboost_regex.so > /dev/null 2>&1' 1 > /tmp/remote_command_check 2>&1` + `$COLUMNSTORE_INSTALL_DIR/bin/remote_command.sh $ipadd $PASSWORD 'ls /usr/lib/libboost_regex.so > /dev/null 2>&1' 1 > /tmp/remote_command_check 2>&1` if [ $? -ne 0 ] ; then echo "${bold}Failed${normal}, $ipadd Node ${bold}boost libraries${normal} not installed" pass=false REPORTPASS=false fi else - `./remote_command.sh $ipadd $PASSWORD "yum list installed '$PKG' > /tmp/pkg_check 2>&1" 1 > /tmp/remote_command_check 2>&1` + `$COLUMNSTORE_INSTALL_DIR/bin/remote_command.sh $ipadd $PASSWORD "yum list installed '$PKG' > /tmp/pkg_check 2>&1" 1 > /tmp/remote_command_check 2>&1` rc="$?" if [ $rc -eq 2 ] ; then echo "${bold}Failed${normal}, $ipadd Node, 'yum' not installed" @@ -821,7 +828,7 @@ checkPackages() if [ "$IPADDRESSES" != "" ]; then for ipadd in "${NODE_IPADDRESS[@]}"; do for PKG in "${SUSE_PKG[@]}"; do - `./remote_command.sh $ipadd $PASSWORD "rpm -qi '$PKG' > /tmp/pkg_check 2>&1" 1 > /tmp/remote_command_check 2>&1` + `$COLUMNSTORE_INSTALL_DIR/bin/remote_command.sh $ipadd $PASSWORD "rpm -qi '$PKG' > /tmp/pkg_check 2>&1" 1 > /tmp/remote_command_check 2>&1` rc="$?" if [ $rc -ne 0 ] ; then echo "${bold}Failed${normal}, $ipadd Node package ${bold}${PKG}${normal} is not installed, please install" @@ -873,8 +880,8 @@ checkPackages() if [ "$IPADDRESSES" != "" ]; then for ipadd in "${NODE_IPADDRESS[@]}"; do for PKG in "${UBUNTU_PKG[@]}"; do - `./remote_command.sh $ipadd $PASSWORD "dpkg -s '$PKG' > /tmp/pkg_check 2>&1" 1 > /tmp/remote_command_check 2>&1` - `./remote_scp_get.sh $ipadd $PASSWORD /tmp/pkg_check > /tmp/remote_scp_get_check 2>&1` + `$COLUMNSTORE_INSTALL_DIR/bin/remote_command.sh $ipadd $PASSWORD "dpkg -s '$PKG' > /tmp/pkg_check 2>&1" 1 > /tmp/remote_command_check 2>&1` + `$COLUMNSTORE_INSTALL_DIR/bin/remote_scp_get.sh $ipadd $PASSWORD /tmp/pkg_check > /tmp/remote_scp_get_check 2>&1` if [ "$?" -ne 0 ]; then echo "Error running remote_scp_get.sh to $ipadd Node, check /tmp/remote_scp_get_check" else @@ -938,8 +945,8 @@ checkPackages() if [ "$IPADDRESSES" != "" ]; then for ipadd in "${NODE_IPADDRESS[@]}"; do for PKG in "${DEBIAN_PKG[@]}"; do - `./remote_command.sh $ipadd $PASSWORD "dpkg -s '$PKG' > /tmp/pkg_check 2>&1" 1 > /tmp/remote_command_check 2>&1` - `./remote_scp_get.sh $ipadd $PASSWORD /tmp/pkg_check > /tmp/remote_scp_get_check 2>&1` + `$COLUMNSTORE_INSTALL_DIR/bin/remote_command.sh $ipadd $PASSWORD "dpkg -s '$PKG' > /tmp/pkg_check 2>&1" 1 > /tmp/remote_command_check 2>&1` + `$COLUMNSTORE_INSTALL_DIR/bin/remote_scp_get.sh $ipadd $PASSWORD /tmp/pkg_check > /tmp/remote_scp_get_check 2>&1` if [ "$?" -ne 0 ]; then echo "Error running remote_scp_get.sh to $ipadd Node, check /tmp/remote_scp_get_check" else @@ -1003,8 +1010,8 @@ checkPackages() if [ "$IPADDRESSES" != "" ]; then for ipadd in "${NODE_IPADDRESS[@]}"; do for PKG in "${DEBIAN9_PKG[@]}"; do - `./remote_command.sh $ipadd $PASSWORD "dpkg -s '$PKG' > /tmp/pkg_check 2>&1" 1 > /tmp/remote_command_check 2>&1` - `./remote_scp_get.sh $ipadd $PASSWORD /tmp/pkg_check > /tmp/remote_scp_get_check 2>&1` + `$COLUMNSTORE_INSTALL_DIR/bin/remote_command.sh $ipadd $PASSWORD "dpkg -s '$PKG' > /tmp/pkg_check 2>&1" 1 > /tmp/remote_command_check 2>&1` + `$COLUMNSTORE_INSTALL_DIR/bin/remote_scp_get.sh $ipadd $PASSWORD /tmp/pkg_check > /tmp/remote_scp_get_check 2>&1` if [ "$?" -ne 0 ]; then echo "Error running remote_scp_get.sh to $ipadd Node, check /tmp/remote_scp_get_check" else From b657938709862e3339651a6b926b52123264f6db Mon Sep 17 00:00:00 2001 From: Andrew Hutchings Date: Fri, 1 Dec 2017 19:32:35 +0000 Subject: [PATCH 2/6] MCOL-1068 Improve compression_ratio() procedure * Uncompressed columns caused a miscalculation for compression ratio * We now show a ratio such as 2:1 rather than a percentage * compressed_data_size instead of file_size is used to show the compression of actual data, ignoring the pre-allocated segment --- dbcon/mysql/columnstore_info.sql | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/dbcon/mysql/columnstore_info.sql b/dbcon/mysql/columnstore_info.sql index 23a6a0ed6..691ea754d 100644 --- a/dbcon/mysql/columnstore_info.sql +++ b/dbcon/mysql/columnstore_info.sql @@ -82,7 +82,7 @@ DROP PROCEDURE IF EXISTS `compression_ratio` // CREATE PROCEDURE compression_ratio() BEGIN -SELECT CONCAT((SELECT SUM(file_size) FROM information_schema.columnstore_files WHERE compressed_data_size IS NOT NULL) / (SELECT SUM(data_size) FROM information_schema.columnstore_extents) * 100, '%') COMPRESSION_RATIO; +SELECT CONCAT((SELECT SUM(data_size) FROM information_schema.columnstore_extents ce left join information_schema.columnstore_columns cc on ce.object_id = cc.object_id where compression_type='Snappy') / (SELECT SUM(compressed_data_size) FROM information_schema.columnstore_files WHERE compressed_data_size IS NOT NULL), ':1') COMPRESSION_RATIO; END // DELIMITER ; From 252880ebdf54096e7f9a3f811a241986fee96d51 Mon Sep 17 00:00:00 2001 From: Ben Thompson Date: Tue, 5 Dec 2017 16:58:06 -0600 Subject: [PATCH 3/6] MCOL-446: update myCnf-include-args.text to fix variable name. Modify mycnfUpgrade to use regex search instead of find for more accurate variable searches, this would cause problem with generic variable 'port'. --- oam/install_scripts/myCnf-include-args.text | 2 +- oamapps/postConfigure/mycnfUpgrade.cpp | 16 +++++++--------- 2 files changed, 8 insertions(+), 10 deletions(-) diff --git a/oam/install_scripts/myCnf-include-args.text b/oam/install_scripts/myCnf-include-args.text index bc2c4c74f..40545c7ed 100644 --- a/oam/install_scripts/myCnf-include-args.text +++ b/oam/install_scripts/myCnf-include-args.text @@ -1,7 +1,7 @@ # List of my.cnf arguments that should be checked and saved during upgrade install # infinidb_local_query -log-bin=mysql-bin +log-bin server-id max_length_for_sort_data tmpdir diff --git a/oamapps/postConfigure/mycnfUpgrade.cpp b/oamapps/postConfigure/mycnfUpgrade.cpp index aa61c2cf6..a8808e740 100644 --- a/oamapps/postConfigure/mycnfUpgrade.cpp +++ b/oamapps/postConfigure/mycnfUpgrade.cpp @@ -47,7 +47,7 @@ #include #include #include - +#include #include "liboamcpp.h" #include "installdir.h" @@ -107,6 +107,7 @@ int main(int argc, char *argv[]) { includeArg = line; + boost::regex icludeArgRegEx("^#*\\s*" + includeArg + "\\s*="); //see if in my.cnf.rpmsave ifstream mycnfsavefile (mycnfsaveFile.c_str()); char line[200]; @@ -114,8 +115,7 @@ int main(int argc, char *argv[]) while (mycnfsavefile.getline(line, 200)) { oldbuf = line; - string::size_type pos = oldbuf.find(includeArg,0); - if ( pos != string::npos ) { + if ( boost::regex_search(oldbuf.begin(),oldbuf.end(),icludeArgRegEx) ) { //found in my.cnf.rpmsave, check if this is commented out if ( line[0] != '#' ) { @@ -129,8 +129,7 @@ int main(int argc, char *argv[]) while (mycnffile.getline(line1, 200)) { newbuf = line1; - string::size_type pos = newbuf.find(includeArg,0); - if ( pos != string::npos ) { + if ( boost::regex_search(newbuf.begin(),newbuf.end(),icludeArgRegEx) ) { newbuf = oldbuf; cout << "Updated argument: " << includeArg << endl; updated = true; @@ -161,8 +160,8 @@ int main(int argc, char *argv[]) while (mycnffile.getline(line1, 200)) { newbuf = line1; - string::size_type pos = newbuf.find("[mysqld]",0); - if ( pos != string::npos ) { + boost::regex mysqldSectionRegEx("\\[mysqld\\]"); + if ( boost::regex_search(newbuf.begin(),newbuf.end(),mysqldSectionRegEx) ) { lines.push_back(newbuf); newbuf = oldbuf; cout << "Added argument: " << includeArg << endl; @@ -183,10 +182,9 @@ int main(int argc, char *argv[]) newFile.close(); close(fd); + break; } } - - break; } } } From 26f7344dc05bc7fcf9111d75420e91e20536dfbc Mon Sep 17 00:00:00 2001 From: Ben Thompson Date: Tue, 5 Dec 2017 16:59:45 -0600 Subject: [PATCH 4/6] MCOL-445: Modify getConfig and setConfig to be case insensitive on variable names. --- utils/configcpp/xmlparser.cpp | 25 +++++++++++++++++-------- 1 file changed, 17 insertions(+), 8 deletions(-) diff --git a/utils/configcpp/xmlparser.cpp b/utils/configcpp/xmlparser.cpp index 96b12734b..8ebe967ca 100644 --- a/utils/configcpp/xmlparser.cpp +++ b/utils/configcpp/xmlparser.cpp @@ -22,6 +22,7 @@ #include "config.h" #include +#include #include #include #include @@ -43,12 +44,14 @@ const string XMLParser::getConfig(const xmlDocPtr doc, const string& section, co cur1 = cur1->xmlChildrenNode; while (cur1 != NULL) { - if ((!xmlStrcmp(cur1->name, (const xmlChar *)section.c_str()))) + string cur1name = (const char*)cur1->name; + if ((boost::iequals(cur1name, section))) { xmlNodePtr cur2 = cur1->xmlChildrenNode; while (cur2 != NULL) { - if ((!xmlStrcmp(cur2->name, (const xmlChar*)name.c_str()))) + string cur2name = (const char*)cur2->name; + if ((boost::iequals(cur2name, name))) { xmlNodePtr cur3 = cur2->xmlChildrenNode; if (cur3) @@ -75,12 +78,14 @@ void XMLParser::getConfig(const xmlDocPtr doc, const string& section, const stri cur1 = cur1->xmlChildrenNode; while (cur1 != NULL) { - if ((!xmlStrcmp(cur1->name, (const xmlChar *)section.c_str()))) + string cur1name = (const char*)cur1->name; + if ((boost::iequals(cur1name, section))) { xmlNodePtr cur2 = cur1->xmlChildrenNode; while (cur2 != NULL) { - if ((!xmlStrcmp(cur2->name, (const xmlChar*)name.c_str()))) + string cur2name = (const char*)cur2->name; + if ((boost::iequals(cur2name, name))) { res.clear(); xmlNodePtr cur3 = cur2->xmlChildrenNode; @@ -106,12 +111,14 @@ void XMLParser::setConfig(xmlDocPtr doc, const string& section, const string& na cur1 = cur1->xmlChildrenNode; while (cur1 != NULL) { - if (xmlStrcmp(cur1->name, (const xmlChar *)section.c_str()) == 0) + string cur1name = (const char*)cur1->name; + if (boost::iequals(cur1name, section)) { cur2 = cur1->xmlChildrenNode; while (cur2 != NULL) { - if (xmlStrcmp(cur2->name, (const xmlChar*)name.c_str()) == 0) + string cur2name = (const char*)cur2->name; + if (boost::iequals(cur2name, name)) { xmlNodePtr cur3 = cur2->xmlChildrenNode; if (cur3 == NULL) @@ -160,14 +167,16 @@ void XMLParser::delConfig(xmlDocPtr doc, const string& section, const string& na cur1 = cur1->xmlChildrenNode; while (cur1 != NULL) { - if ((!xmlStrcmp(cur1->name, (const xmlChar *)section.c_str()))) + string cur1name = (const char*)cur1->name; + if ((boost::iequals(cur1name, section))) { xmlNodePtr cur2 = cur1->xmlChildrenNode; while (cur2 != NULL) { xmlNodePtr tmp = cur2; cur2 = cur2->next; - if ((!xmlStrcmp(tmp->name, (const xmlChar*)name.c_str()))) + string tmpname = (const char*)tmp->name; + if ((boost::iequals(tmpname, name))) { xmlUnlinkNode(tmp); xmlFreeNode(tmp); From 34799d8d308a8ac6dba920e858c77cfc327b9ff8 Mon Sep 17 00:00:00 2001 From: David Hall Date: Thu, 7 Dec 2017 10:48:06 -0600 Subject: [PATCH 5/6] MCOL-994 handle a second abbreviated extent in case it moved dbroots because of redistribute remove. --- versioning/BRM/extentmap.h | 4 +- writeengine/bulk/we_columninfo.cpp | 58 ++++++++++---------- writeengine/shared/we_fileop.cpp | 70 +++++++++++++++++++++--- writeengine/shared/we_log.h | 2 +- writeengine/splitter/we_sdhandler.cpp | 2 +- writeengine/wrapper/we_colopcompress.cpp | 8 ++- 6 files changed, 102 insertions(+), 42 deletions(-) diff --git a/versioning/BRM/extentmap.h b/versioning/BRM/extentmap.h index b067ed024..e7a796711 100644 --- a/versioning/BRM/extentmap.h +++ b/versioning/BRM/extentmap.h @@ -684,8 +684,8 @@ public: * * Function should only be called for System Catalog OIDs, as it assumes * the OID is fully contained on a single DBRoot, returning the first - * DBRoot found. This only makes since for a System Catalog OID, because - * all other column OIDs can span multiple DBRoots. + * DBRoot found. This only makes sence for a System Catalog + * OID, because all other column OIDs can span multiple DBRoots. * * @param oid The system catalog OID * @param dbRoot (out) the DBRoot holding the system catalog OID diff --git a/writeengine/bulk/we_columninfo.cpp b/writeengine/bulk/we_columninfo.cpp index dda374750..832789b8a 100644 --- a/writeengine/bulk/we_columninfo.cpp +++ b/writeengine/bulk/we_columninfo.cpp @@ -111,7 +111,7 @@ unsigned int compactVarBinary(char* charTmpBuf, int fieldLength) } } - + namespace WriteEngine { @@ -209,7 +209,7 @@ ColumnInfo::ColumnInfo(Log* logger, // fAutoIncMgr = new ColumnAutoIncJob(logger); } } - + //------------------------------------------------------------------------------ // ColumnInfo destructor //------------------------------------------------------------------------------ @@ -250,7 +250,7 @@ void ColumnInfo::clearMemory( ) fDictBlocks.clear(); } - + //------------------------------------------------------------------------------ // If at the start of the job, We have encountered a PM that has no DB file for // this column, or whose HWM extent is disabled; then this function is called @@ -281,7 +281,7 @@ void ColumnInfo::setupDelayedFileCreation( column.compressionType, dbRoot, partition, segment ); } - + //------------------------------------------------------------------------------ // Create a DB file as part of delayed file creation. See setupDelayedFile- // Creation for an explanation. @@ -506,7 +506,7 @@ int ColumnInfo::createDelayedFileIfNeeded( const std::string& tableName ) return rc; } - + //------------------------------------------------------------------------------ // Add an extent for this column. The next segment file in the DBRoot, // partition, segment number rotation will be selected for the extent. @@ -613,7 +613,7 @@ int ColumnInfo::extendColumn( bool saveLBIDForCP ) return rc; } - + //------------------------------------------------------------------------------ // Add a new extent to this column, at the specified DBRoot. Partition may be // used if DBRoot is empty. @@ -773,7 +773,7 @@ int ColumnInfo::extendColumnNewExtent( return NO_ERROR; } - + //------------------------------------------------------------------------------ // Fill out existing partial extent to extent boundary, so that we can resume // inserting rows on an extent boundary basis. This use case should only take @@ -881,7 +881,7 @@ int ColumnInfo::extendColumnOldExtent( return NO_ERROR; } - + //------------------------------------------------------------------------------ // Either add or update the File object, so that it has the updated HWM. // We will access this info to update the HWM in the ExtentMap at the end @@ -933,7 +933,7 @@ void ColumnInfo::addToSegFileList( File& dataFile, HWM hwm ) fSegFileUpdateList.push_back( dataFile ); } } - + //------------------------------------------------------------------------------ // Reset file offset data member attributes when we start working on the next // extent. @@ -961,7 +961,7 @@ int ColumnInfo::resetFileOffsetsNewExtent(const char* /*hdr*/) return NO_ERROR; } - + //------------------------------------------------------------------------------ // Set current size of file in raw (uncompressed) bytes, given the specified // hwm. abbrevFlag indicates whether this is a fixed size abbreviated extent. @@ -994,7 +994,7 @@ void ColumnInfo::setFileSize( HWM hwm, int abbrevFlag ) } } } - + //------------------------------------------------------------------------------ // If we are dealing with the first extent in the first segment file for this // column, and the segment file is still equal to 256K rows, then we set the @@ -1026,7 +1026,7 @@ void ColumnInfo::setAbbrevExtentCheck( ) } } } - + //------------------------------------------------------------------------------ // If this is an abbreviated extent, we expand the extent to a full extent on // disk, by initializing the necessary number of remaining blocks. @@ -1102,7 +1102,7 @@ int ColumnInfo::expandAbbrevExtent( bool bRetainFilePos ) return NO_ERROR; } - + //------------------------------------------------------------------------------ // Close the current Column file. //------------------------------------------------------------------------------ @@ -1116,7 +1116,7 @@ int ColumnInfo::closeColumnFile(bool /*bCompletingExtent*/, bool /*bAbort*/) return NO_ERROR; } - + //------------------------------------------------------------------------------ // Initialize fLastInputRowInCurrentExtent used in detecting when a Read Buffer // is crossing an extent boundary, so that we can accurately track the min/max @@ -1163,7 +1163,7 @@ void ColumnInfo::lastInputRowInExtentInit( bool bIsNewExtent ) bIsNewExtent ); } } - + //------------------------------------------------------------------------------ // Increment fLastRIDInExtent to the end of the next extent. //------------------------------------------------------------------------------ @@ -1171,7 +1171,7 @@ void ColumnInfo::lastInputRowInExtentInc( ) { fLastInputRowInCurrentExtent += fRowsPerExtent; } - + //------------------------------------------------------------------------------ // Parsing is complete for this column. Flush pending data. Close the current // segment file, and corresponding dictionary store file (if applicable). Also @@ -1234,7 +1234,7 @@ int ColumnInfo::finishParsing( ) return NO_ERROR; } - + //------------------------------------------------------------------------------ // Store updated column information in BRMReporter for this column at EOJ; // so that Extent Map CP information and HWM's can be updated. @@ -1259,7 +1259,7 @@ void ColumnInfo::getBRMUpdateInfo( BRMReporter& brmReporter ) if (entriesAdded > 0) getCPInfoForBRM( brmReporter ); } - + //------------------------------------------------------------------------------ // Get updated Casual Partition (CP) information for BRM for this column at EOJ. //------------------------------------------------------------------------------ @@ -1267,7 +1267,7 @@ void ColumnInfo::getCPInfoForBRM( BRMReporter& brmReporter ) { fColExtInf->getCPInfoForBRM(column, brmReporter); } - + //------------------------------------------------------------------------------ // Get updated HWM information for BRM for this column at EOJ. // Returns count of the number of HWM entries added to the BRMReporter. @@ -1340,7 +1340,7 @@ int ColumnInfo::getHWMInfoForBRM( BRMReporter& brmReporter ) return entriesAdded; } - + //------------------------------------------------------------------------------ // Setup initial extent we will begin loading at start of import. // DBRoot, partition, segment, etc for the starting extent are specified. @@ -1460,7 +1460,7 @@ int ColumnInfo::setupInitialColumnExtent( return NO_ERROR; } - + //------------------------------------------------------------------------------ // Prepare the initial column segment file for import. //------------------------------------------------------------------------------ @@ -1521,7 +1521,7 @@ int ColumnInfo::setupInitialColumnFile( HWM oldHwm, HWM hwm ) return NO_ERROR; } - + //------------------------------------------------------------------------------ // Update the number of bytes in the file, and the free space still remaining. //------------------------------------------------------------------------------ @@ -1542,7 +1542,7 @@ bool ColumnInfo::isFileComplete() const return false; } - + //------------------------------------------------------------------------------ // Initialize last used auto-increment value from the current "next" // auto-increment value taken from the system catalog (or BRM). @@ -1575,7 +1575,7 @@ int ColumnInfo::finishAutoInc( ) return rc; } - + //------------------------------------------------------------------------------ // Get current dbroot, partition, segment, and HWM for this column. // @@ -1597,7 +1597,7 @@ void ColumnInfo::getSegFileInfo( DBRootExtentInfo& fileInfo ) else fileInfo.fLocalHwm = 0; } - + //------------------------------------------------------------------------------ // Open a new or existing Dictionary store file based on the DBRoot, // partition, and segment settings in curCol.dataFile. @@ -1741,7 +1741,7 @@ int ColumnInfo::openDctnryStore( bool bMustExist ) return rc; } - + //------------------------------------------------------------------------------ // Close the current Dictionary store file. //------------------------------------------------------------------------------ @@ -1773,7 +1773,7 @@ int ColumnInfo::closeDctnryStore(bool bAbort) return rc; } - + //------------------------------------------------------------------------------ // Update dictionary store file with specified strings, and return the assigned // tokens (tokenbuf) to be stored in the corresponding column token file. @@ -1835,7 +1835,7 @@ int ColumnInfo::updateDctnryStore(char* buf, return NO_ERROR; } - + //------------------------------------------------------------------------------ // No action necessary for uncompressed dictionary files //------------------------------------------------------------------------------ @@ -1856,7 +1856,7 @@ int ColumnInfo::truncateDctnryStore( { return NO_ERROR; } - + //------------------------------------------------------------------------------ // utility to convert a Status enumeration to a string //------------------------------------------------------------------------------ diff --git a/writeengine/shared/we_fileop.cpp b/writeengine/shared/we_fileop.cpp index c70d1df09..d213305f6 100644 --- a/writeengine/shared/we_fileop.cpp +++ b/writeengine/shared/we_fileop.cpp @@ -593,8 +593,7 @@ int FileOp::extendFile( return ERR_FILE_NOT_EXIST; } - pFile = openFile( oid, dbRoot, partition, segment, - segFile, "r+b" );//old file + pFile = openFile( oid, dbRoot, partition, segment, segFile, "r+b" );//old file if (pFile == 0) { ostringstream oss; @@ -613,7 +612,7 @@ int FileOp::extendFile( if ( isDebug(DEBUG_1) && getLogger() ) { std::ostringstream oss; - oss << "Opening existing column file" << + oss << "Opening existing column file (extendFile)" << ": OID-" << oid << "; DBRoot-" << dbRoot << "; part-" << partition << @@ -651,19 +650,55 @@ int FileOp::extendFile( ostringstream oss; oss << "oid: " << oid << " with path " << segFile << "; new extent fbo " << hwm << "; number of " - "compressed chunks " << ptrCount; + "compressed chunks " << ptrCount << + "; chunkIndex " << chunkIndex; logging::Message::Args args; args.add("compressed"); args.add(oss.str()); SimpleSysLog::instance()->logMsg(args, logging::LOG_TYPE_ERROR, logging::M0103); - return ERR_FILE_NEW_EXTENT_FBO; + // Expand the partial extent to full with emptyVal + // Since fillCompColumnExtentEmptyChunks() messes with the + // file on disk, we need to close it and reopen after or + // the cache isn't updated. + if ((pFile)) + closeFile( pFile ); + pFile = NULL; + string failedTask; // For return error message, if any. + rc = FileOp::fillCompColumnExtentEmptyChunks(oid, + width, + emptyVal, + dbRoot, + partition, + segment, + hwm, + segFile, + failedTask); + if (rc != NO_ERROR) + { + if (getLogger()) + { + std::ostringstream oss; + oss << "FileOp::extendFile: error padding partial compressed extent for " << + "column OID-" << oid << + "; DBRoot-" << dbRoot << + "; part-" << partition << + "; seg-" << segment << + "; hwm-" << hwm << + " " << failedTask.c_str(); + getLogger()->logMsg( oss.str(), rc, MSGLVL_CRITICAL ); + } + return rc; + } + pFile = openFile( oid, dbRoot, partition, segment, segFile, "r+b" ); // modified file } + // Get the latest file header for the caller. If a partial extent was filled out, + // this will be different than when we first read the headers. if (hdrs) { - memcpy(hdrs, hdrsIn, sizeof(hdrsIn) ); + RETURN_ON_ERROR( readHeaders(pFile, hdrs) ); } } else @@ -693,7 +728,26 @@ int FileOp::extendFile( SimpleSysLog::instance()->logMsg(args, logging::LOG_TYPE_ERROR, logging::M0103); - return ERR_FILE_NEW_EXTENT_FBO; + // Expand the partial extent to full with emptyVal + // This generally won't ever happen, as uncompressed files + // are created with full extents. + rc = FileOp::expandAbbrevColumnExtent( pFile, dbRoot, + emptyVal, width); + if (rc != NO_ERROR) + { + if (getLogger()) + { + std::ostringstream oss; + oss << "FileOp::extendFile: error padding partial uncompressed extent for " << + "column OID-" << oid << + "; DBRoot-" << dbRoot << + "; part-" << partition << + "; seg-" << segment << + "; hwm-" << hwm; + getLogger()->logMsg( oss.str(), rc, MSGLVL_CRITICAL ); + } + return rc; + } } } } @@ -2598,7 +2652,7 @@ int FileOp::expandAbbrevColumnExtent( } // Add blocks to turn the abbreviated extent into a full extent. - int rc = initColumnExtent(pFile, dbRoot, blksToAdd, emptyVal, width, + int rc = FileOp::initColumnExtent(pFile, dbRoot, blksToAdd, emptyVal, width, false, // existing file true, // expand existing extent false); // n/a since not adding new extent diff --git a/writeengine/shared/we_log.h b/writeengine/shared/we_log.h index 871ddfc02..42efb7c94 100644 --- a/writeengine/shared/we_log.h +++ b/writeengine/shared/we_log.h @@ -47,7 +47,7 @@ namespace WriteEngine { const std::string MSG_LEVEL_STR[] = { "INFO", - "INFO", + "INFO2", "WARN", "ERR ", "CRIT" }; diff --git a/writeengine/splitter/we_sdhandler.cpp b/writeengine/splitter/we_sdhandler.cpp index 6b9c148e3..985c90d74 100644 --- a/writeengine/splitter/we_sdhandler.cpp +++ b/writeengine/splitter/we_sdhandler.cpp @@ -992,7 +992,7 @@ void WESDHandler::onNakResponse(int PmId) { // Increase the read timeout for this PM thread so that it won't // consume too much CPU void WESDHandler::onEodResponse(int PmId) { - if (getDebugLvl()) + if (getDebugLvl()>2) cout << "Received a EOD from " << PmId << endl; if (fRef.fCmdArgs.getMode() == 0) diff --git a/writeengine/wrapper/we_colopcompress.cpp b/writeengine/wrapper/we_colopcompress.cpp index d38258b50..1c8938f7e 100644 --- a/writeengine/wrapper/we_colopcompress.cpp +++ b/writeengine/wrapper/we_colopcompress.cpp @@ -188,7 +188,13 @@ int ColumnOpCompress1::expandAbbrevColumnExtent( IDBDataFile* pFile, uint16_t dbRoot, uint64_t emptyVal, int width) { // update the uncompressed initial chunk to full chunk - RETURN_ON_ERROR(m_chunkManager->expandAbbrevColumnExtent(pFile, emptyVal, width)); + int rc = m_chunkManager->expandAbbrevColumnExtent(pFile, emptyVal, width); + // ERR_COMP_FILE_NOT_FOUND is acceptable here. It just means that the + // file hasn't been loaded into the chunk manager yet. No big deal. + if (rc != NO_ERROR && rc != ERR_COMP_FILE_NOT_FOUND) + { + return rc; + } // let the base to physically expand extent. return FileOp::expandAbbrevColumnExtent(pFile, dbRoot, emptyVal, width); From 1699c20c488a8454bc652bad00e11b03290a4621 Mon Sep 17 00:00:00 2001 From: david hill Date: Mon, 11 Dec 2017 10:23:02 -0600 Subject: [PATCH 6/6] MCOL-1095 - fixed debian 9 lib dep --- utils/clusterTester/columnstoreClusterTester.sh | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/utils/clusterTester/columnstoreClusterTester.sh b/utils/clusterTester/columnstoreClusterTester.sh index acc39d23a..78fa3f889 100755 --- a/utils/clusterTester/columnstoreClusterTester.sh +++ b/utils/clusterTester/columnstoreClusterTester.sh @@ -978,7 +978,7 @@ checkPackages() fi fi - declare -a DEBIAN9_PKG=("libboost-all-dev" "expect" "libdbi-perl" "perl" "openssl" "file" "sudo" "libreadline5" "rsync" "libsnappy1V5" "net-tools" "libioa1") + declare -a DEBIAN9_PKG=("libboost-all-dev" "expect" "libdbi-perl" "perl" "openssl" "file" "sudo" "libreadline5" "rsync" "libsnappy1V5" "net-tools" "libaio1") if [ "$OS" == "debian9" ]; then if [ ! `which dpkg 2>/dev/null` ] ; then