From 369b5e1fb93eba84068f6e91dd37c9d552c27476 Mon Sep 17 00:00:00 2001 From: david hill Date: Tue, 23 Aug 2016 15:59:13 -0500 Subject: [PATCH 01/61] change readme --- README.md | 6 ++++-- 1 file changed, 4 insertions(+), 2 deletions(-) diff --git a/README.md b/README.md index 693f09193..5df193e89 100644 --- a/README.md +++ b/README.md @@ -1,9 +1,9 @@ #MariaDB ColumnStore Storage/Execution engine 1.0.1 -MariaDB ColumnStore 1.0.1 is the development version of MariaDB ColumnStore. +MariaDB ColumnStore 1.0.2 is the development version of MariaDB ColumnStore. It is built by porting InfiniDB 4.6.2 on MariaDB 10.1.14 and adding entirely new features not found anywhere else. -#MariaDB ColumnStore 1.0.1 is an Alpha release. +#MariaDB ColumnStore 1.0.2 is an Alpha release. This is the first MariaDB ColumnStore release, not all features planned for the MariaDB ColumnStore 1.0 series are included in this release. @@ -14,3 +14,5 @@ A few things to notice: #Building This repository is not meant to be built independently outside of the server. This repository is integrated into http://mariadb-corporation/mariadb-columnstore-server (ie, the *server*) as a git submodule. As such, you can find complete build instructions on *the server* page. + https://github.com/mariadb-corporation/mariadb-columnstore-server + From d7cbaad3e88a3a229df388cfe04078c1ce51cf9e Mon Sep 17 00:00:00 2001 From: david hill Date: Wed, 24 Aug 2016 08:42:56 -0500 Subject: [PATCH 02/61] update readme --- README.md | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/README.md b/README.md index 5df193e89..86a9168be 100644 --- a/README.md +++ b/README.md @@ -1,4 +1,4 @@ -#MariaDB ColumnStore Storage/Execution engine 1.0.1 +#MariaDB ColumnStore Storage/Execution engine 1.0.2 MariaDB ColumnStore 1.0.2 is the development version of MariaDB ColumnStore. It is built by porting InfiniDB 4.6.2 on MariaDB 10.1.14 and adding entirely new features not found anywhere else. From 2ca70ef647716e2d70a8f2ca0fcde1e3bcbbad19 Mon Sep 17 00:00:00 2001 From: david hill Date: Wed, 24 Aug 2016 08:44:57 -0500 Subject: [PATCH 03/61] update readme --- README | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) diff --git a/README b/README index 0b05354b1..70b9f790f 100644 --- a/README +++ b/README @@ -1,9 +1,9 @@ -This is MariaDB ColumnStore 1.0.1 -MariaDB ColumnStore 1.0.1 is the development version of MariaDB ColumnStore. +This is MariaDB ColumnStore 1.0.2 +MariaDB ColumnStore 1.0.2 is the development version of MariaDB ColumnStore. It is built by porting InfiniDB 4.6.2 on MariaDB 10.1.14 and adding entirely new features not found anywhere else. -MariaDB ColumnStore 1.0.1 is an Alpha release. This is the first MariaDB +MariaDB ColumnStore 1.0.2 is an Alpha release. This is the first MariaDB ColumnStore release, not all features planned for the MariaDB ColumnStore 1.0 series are included in this release. From e6dba743b005643b92fe8c667036b3ef455e4740 Mon Sep 17 00:00:00 2001 From: Andrew Ernst Date: Fri, 23 Sep 2016 09:13:10 -0700 Subject: [PATCH 04/61] Fixes what is likely a misspelling on the mcsadmin> prompt Instead of mscadmin> this changes the prompt to mcsadmin> --- oamapps/mcsadmin/mcsadmin.cpp | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/oamapps/mcsadmin/mcsadmin.cpp b/oamapps/mcsadmin/mcsadmin.cpp index 2786ad472..910d0367e 100644 --- a/oamapps/mcsadmin/mcsadmin.cpp +++ b/oamapps/mcsadmin/mcsadmin.cpp @@ -349,7 +349,7 @@ int main(int argc, char *argv[]) } // read input - pcommand = readline("mscadmin> "); + pcommand = readline("mcsadmin> "); if (!pcommand) // user hit -D pcommand = strdup("exit"); From 40db11e2185ab73d352d539ecdb03cc638579d54 Mon Sep 17 00:00:00 2001 From: david hill Date: Thu, 24 Nov 2016 09:14:05 -0600 Subject: [PATCH 05/61] Update README.md --- README.md | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/README.md b/README.md index b561c70a3..efeca3c89 100644 --- a/README.md +++ b/README.md @@ -3,7 +3,7 @@ MariaDB ColumnStore 1.0.5 is the development version of MariaDB ColumnStore. It is built by porting InfiniDB 4.6.2 on MariaDB 10.1.19 and adding entirely new features not found anywhere else. -#MariaDB ColumnStore 1.0.5 is an Beta release. +#MariaDB ColumnStore 1.0.5 is an RC release. This is the first MariaDB ColumnStore release, not all features planned for the MariaDB ColumnStore 1.0 series are included in this release. From 02a2fbeae6bc9de7f009044815a22b2cd8fe120e Mon Sep 17 00:00:00 2001 From: David Hill Date: Thu, 12 Jan 2017 11:33:27 -0600 Subject: [PATCH 06/61] MCOL-488 - added back of Alarm Config File --- oam/install_scripts/post-install | 3 +++ oam/install_scripts/pre-uninstall | 1 + 2 files changed, 4 insertions(+) diff --git a/oam/install_scripts/post-install b/oam/install_scripts/post-install index 160cacc97..5de1b981b 100755 --- a/oam/install_scripts/post-install +++ b/oam/install_scripts/post-install @@ -231,6 +231,9 @@ if [ $? -eq 0 ]; then cat /tmp/syslog_install.log fi +#backup copy of Alarm Config File +/bin/cp -f $installdir/etc/AlarmConfig.xml $installdir/etc/AlarmConfig.xml.installSave > /dev/null 2>&1 + #setup alias commands eval userhome=~$user bashFile=$userhome/.bashrc diff --git a/oam/install_scripts/pre-uninstall b/oam/install_scripts/pre-uninstall index 1e50b1a35..8709d369c 100755 --- a/oam/install_scripts/pre-uninstall +++ b/oam/install_scripts/pre-uninstall @@ -153,6 +153,7 @@ if [ $quiet != 1 ]; then #make copy of Columnstore.xml /bin/cp -f $installdir/etc/Columnstore.xml $installdir/etc/Columnstore.xml.rpmsave > /dev/null 2>&1 /bin/cp -f $installdir/mysql/my.cnf $installdir/mysql/my.cnf.rpmsave > /dev/null 2>&1 + rm -f $installdir/etc/AlarmConfig.xml.installSave fi #umount file systems From 4874134a2144d89dfefc73ff159f250500b90a35 Mon Sep 17 00:00:00 2001 From: David Hill Date: Fri, 13 Jan 2017 09:58:23 -0600 Subject: [PATCH 07/61] fix storage type prompt --- oamapps/postConfigure/postConfigure.cpp | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/oamapps/postConfigure/postConfigure.cpp b/oamapps/postConfigure/postConfigure.cpp index 3a9a7a2d8..cb02fc091 100644 --- a/oamapps/postConfigure/postConfigure.cpp +++ b/oamapps/postConfigure/postConfigure.cpp @@ -4551,7 +4551,7 @@ bool storageSetup(bool amazonInstall) callFree(pcommand); } - if ( PMVolumeType == "standard" || PMVolumeType == "gp2" || PMVolumeType == "io1" || PMVolumeType == "gp2" || PMVolumeType == "io1") + if ( PMVolumeType == "standard" || PMVolumeType == "gp2" || PMVolumeType == "io1" || PMVolumeType == "sc1" || PMVolumeType == "st1") break; else { From db0c47f763060489fa4ce500bfa9e92f89bd5106 Mon Sep 17 00:00:00 2001 From: david hill Date: Fri, 13 Jan 2017 10:02:59 -0600 Subject: [PATCH 08/61] change release number --- README.md | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) diff --git a/README.md b/README.md index efeca3c89..62a8a218d 100644 --- a/README.md +++ b/README.md @@ -1,9 +1,9 @@ -#MariaDB ColumnStore Storage/Execution engine 1.0.5 -MariaDB ColumnStore 1.0.5 is the development version of MariaDB ColumnStore. +#MariaDB ColumnStore Storage/Execution engine 1.0.6 +MariaDB ColumnStore 1.0.6 is the development version of MariaDB ColumnStore. It is built by porting InfiniDB 4.6.2 on MariaDB 10.1.19 and adding entirely new features not found anywhere else. -#MariaDB ColumnStore 1.0.5 is an RC release. +#MariaDB ColumnStore 1.0.6 is an GA release. This is the first MariaDB ColumnStore release, not all features planned for the MariaDB ColumnStore 1.0 series are included in this release. From e53c20a0ec8c47f584780cf97ce6ec77618a7d12 Mon Sep 17 00:00:00 2001 From: david hill Date: Fri, 13 Jan 2017 10:04:42 -0600 Subject: [PATCH 09/61] change release number --- README | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) diff --git a/README b/README index 85722e3c3..ac59b3816 100644 --- a/README +++ b/README @@ -1,9 +1,9 @@ -This is MariaDB ColumnStore 1.0.4 -MariaDB ColumnStore 1.0.4 is the development version of MariaDB ColumnStore. +This is MariaDB ColumnStore 1.0.6 +MariaDB ColumnStore 1.0.6 is the development version of MariaDB ColumnStore. It is built by porting InfiniDB 4.6.2 on MariaDB 10.1.19 and adding entirely new features not found anywhere else. -MariaDB ColumnStore 1.0.4 is an Beta release. This is the first MariaDB +MariaDB ColumnStore 1.0.6 is an GA release. This is the first MariaDB ColumnStore release, not all features planned for the MariaDB ColumnStore 1.0 series are included in this release. From fa0fde9a73583c52a0890e52662cb611141b7257 Mon Sep 17 00:00:00 2001 From: David Hill Date: Fri, 13 Jan 2017 14:24:14 -0600 Subject: [PATCH 10/61] mcol-462, fixed um and pmvolumnesize in prompt to be minsize --- oamapps/postConfigure/postConfigure.cpp | 19 ++++++++++--------- 1 file changed, 10 insertions(+), 9 deletions(-) diff --git a/oamapps/postConfigure/postConfigure.cpp b/oamapps/postConfigure/postConfigure.cpp index cb02fc091..ba698361f 100644 --- a/oamapps/postConfigure/postConfigure.cpp +++ b/oamapps/postConfigure/postConfigure.cpp @@ -4248,7 +4248,7 @@ bool storageSetup(bool amazonInstall) {} if ( UMVolumeSize.empty() || UMVolumeSize == "" || UMVolumeSize == oam::UnassignedName) - UMVolumeSize = "10"; + UMVolumeSize = minSize; while(true) { @@ -4572,14 +4572,6 @@ bool storageSetup(bool amazonInstall) } cout << endl; - try { - oam.getSystemConfig("PMVolumeSize", PMVolumeSize); - } - catch(...) - {} - - if ( PMVolumeSize.empty() || PMVolumeSize == "" || PMVolumeSize == oam::UnassignedName) - PMVolumeSize = "100"; string minSize = "1"; string maxSize = "16384"; @@ -4593,6 +4585,15 @@ bool storageSetup(bool amazonInstall) if (PMVolumeType == "standard") maxSize = "1024"; + try { + oam.getSystemConfig("PMVolumeSize", PMVolumeSize); + } + catch(...) + {} + + if ( PMVolumeSize.empty() || PMVolumeSize == "" || PMVolumeSize == oam::UnassignedName) + PMVolumeSize = minSize; + while(true) { string prompt = "Enter EBS Volume storage size in GB: [" + minSize + "," + maxSize + "] (" + PMVolumeSize + ") > "; From dec640a6ac791093b978b4e914d5d66e4ce35f33 Mon Sep 17 00:00:00 2001 From: Andrew Hutchings Date: Wed, 18 Jan 2017 15:11:01 +0000 Subject: [PATCH 11/61] Update README.md for 1.0.7 --- README.md | 15 +++++---------- 1 file changed, 5 insertions(+), 10 deletions(-) diff --git a/README.md b/README.md index 7bda7e247..e9a325907 100644 --- a/README.md +++ b/README.md @@ -1,15 +1,10 @@ -#MariaDB ColumnStore Storage/Execution engine 1.0.6 -MariaDB ColumnStore 1.0.6 is the development version of MariaDB ColumnStore. -It is built by porting InfiniDB 4.6.7 on MariaDB 10.1.19 and adding entirely +#MariaDB ColumnStore Storage/Execution engine 1.0.7 +MariaDB ColumnStore 1.0.7 is the development version of MariaDB ColumnStore. +It is built by porting InfiniDB 4.6.7 on MariaDB 10.1.21 and adding entirely new features not found anywhere else. -#MariaDB ColumnStore 1.0.6 is an GA release. -This is the first MariaDB ColumnStore release, not all features planned for the MariaDB ColumnStore 1.0 -series are included in this release. - -Additional features will be pushed in future releases. -A few things to notice: -- Do not use alpha releases on production systems. +#MariaDB ColumnStore 1.0.7 is an GA release. +This is the first MariaDB ColumnStore release. #Building This repository is not meant to be built independently outside of the server. This repository is integrated into http://mariadb-corporation/mariadb-columnstore-server (ie, the *server*) as a git submodule. As such, you can find complete build instructions on *the server* page. From 2dd2c14312a734bfc70fdad1870d0ab67d015ae8 Mon Sep 17 00:00:00 2001 From: david hill Date: Wed, 18 Jan 2017 11:22:15 -0600 Subject: [PATCH 12/61] MCOL-475 - removed extra } --- oam/install_scripts/rsync.sh | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/oam/install_scripts/rsync.sh b/oam/install_scripts/rsync.sh index d11720344..49f93af69 100755 --- a/oam/install_scripts/rsync.sh +++ b/oam/install_scripts/rsync.sh @@ -52,7 +52,7 @@ expect { -re "failed" { send_user " FAILED: Failure, check tmp log\n" ; exit 1 } -re "Permission denied" { send_user " FAILED: Invalid password\n" ; exit 1 } -re "total size" { exit 0 } - } + } exit 0 From a64851f56cf84db1fa519037b477421dccea89a8 Mon Sep 17 00:00:00 2001 From: Andrew Hutchings Date: Wed, 18 Jan 2017 17:28:19 +0000 Subject: [PATCH 13/61] MCOL-477 Add validation to alter autoincrement If the data isn't validated and someone gives bad input it could cause corruption. This validates the input. --- dbcon/ddlpackageproc/altertableprocessor.cpp | 62 ++++++++++++++++++++ 1 file changed, 62 insertions(+) diff --git a/dbcon/ddlpackageproc/altertableprocessor.cpp b/dbcon/ddlpackageproc/altertableprocessor.cpp index b4cdd1a4c..d73c3a624 100644 --- a/dbcon/ddlpackageproc/altertableprocessor.cpp +++ b/dbcon/ddlpackageproc/altertableprocessor.cpp @@ -1854,6 +1854,68 @@ void AlterTableProcessor::tableComment(uint32_t sessionID, execplan::CalpontSyst throw std::runtime_error("Table does not have an autoincrement column"); } int32_t oid = systemCatalogPtr->autoColumOid(tableName); + + CalpontSystemCatalog::ColType type = systemCatalogPtr->colType(oid); + + bool validated = true; + bool negative = false; + + switch (type.colDataType) + { + case CalpontSystemCatalog::BIGINT: + if (static_cast(nextVal) > MAX_BIGINT) + validated = false; + if (static_cast(nextVal) < 1) + negative = true; + break; + case CalpontSystemCatalog::UBIGINT: + if (nextVal > MAX_UBIGINT) + validated = false; + break; + case CalpontSystemCatalog::INT: + case CalpontSystemCatalog::MEDINT: + if (static_cast(nextVal) > MAX_INT) + validated = false; + if (static_cast(nextVal) < 1) + negative = true; + break; + case CalpontSystemCatalog::UINT: + case CalpontSystemCatalog::UMEDINT: + if (nextVal > MAX_UINT) + validated = false; + break; + case CalpontSystemCatalog::SMALLINT: + if (static_cast(nextVal) > MAX_SMALLINT) + validated = false; + if (static_cast(nextVal) < 1) + negative = true; + break; + case CalpontSystemCatalog::USMALLINT: + if (nextVal > MAX_USMALLINT) + validated = false; + break; + case CalpontSystemCatalog::TINYINT: + if (static_cast(nextVal) > MAX_TINYINT) + validated = false; + if (static_cast(nextVal) < 1) + negative = true; + break; + case CalpontSystemCatalog::UTINYINT: + if (nextVal > MAX_UTINYINT) + validated = false; + break; + default: + break; + } + if (!validated) + { + throw std::runtime_error(IDBErrorInfo::instance()->errorMsg(ERR_INVALID_START_VALUE)); + } + if (negative) + { + throw std::runtime_error(IDBErrorInfo::instance()->errorMsg(ERR_NEGATIVE_STARTVALUE)); + } + fDbrm->resetAISequence(oid, nextVal); ByteStream bs; bs.restart(); From 271bfb525ee0518d9d770904b093fab70c52c872 Mon Sep 17 00:00:00 2001 From: Andrew Hutchings Date: Wed, 18 Jan 2017 17:35:59 +0000 Subject: [PATCH 14/61] Also check for zero Needed for unsigned int columns --- dbcon/ddlpackageproc/altertableprocessor.cpp | 6 ++++++ 1 file changed, 6 insertions(+) diff --git a/dbcon/ddlpackageproc/altertableprocessor.cpp b/dbcon/ddlpackageproc/altertableprocessor.cpp index d73c3a624..df2d77eb8 100644 --- a/dbcon/ddlpackageproc/altertableprocessor.cpp +++ b/dbcon/ddlpackageproc/altertableprocessor.cpp @@ -1838,6 +1838,12 @@ void AlterTableProcessor::tableComment(uint32_t sessionID, execplan::CalpontSyst { throw std::runtime_error(IDBErrorInfo::instance()->errorMsg(ERR_INVALID_START_VALUE)); } + // Checks if zero and throws appropriate error (despite message name) + // negative checks are below + if (nextVal == 0) + { + throw std::runtime_error(IDBErrorInfo::instance()->errorMsg(ERR_NEGATIVE_STARTVALUE)); + } } else { From 06a997628940e62bd61d318a2b42fb56a68904ba Mon Sep 17 00:00:00 2001 From: david hill Date: Wed, 18 Jan 2017 14:32:54 -0600 Subject: [PATCH 15/61] mcol-481 - just changed the note --- oamapps/postConfigure/postConfigure.cpp | 11 +++++------ 1 file changed, 5 insertions(+), 6 deletions(-) diff --git a/oamapps/postConfigure/postConfigure.cpp b/oamapps/postConfigure/postConfigure.cpp index ba698361f..f8e66311a 100644 --- a/oamapps/postConfigure/postConfigure.cpp +++ b/oamapps/postConfigure/postConfigure.cpp @@ -666,10 +666,9 @@ int main(int argc, char *argv[]) exit(1); } - cout << "NOTE: MySQL Replication feature can be enabled to have MariaDB ColumnStore" << endl; - cout << " setup the functionality of MySQL Replication on the system." << endl; - cout << " Or the feature can be disabled, if for example, if you have replication" << endl; - cout << " handled by another application" << endl << endl; + cout << "NOTE: The MariaDB ColumnStore Schema Sync feature will replicate all of the" << endl; + cout << " schemas and InnoDB tables across the User Module nodes. This feature can be enabled" << endl; + cout << " or disabled, for example, if you wish to configure your own replication post installation." << endl << endl; try { MySQLRep = sysConfig->getConfig(InstallSection, "MySQLRep"); @@ -684,9 +683,9 @@ int main(int argc, char *argv[]) while(true) { if ( mysqlRep ) - prompt = "MariaDB ColumnStore MySQL Replication feature is Enabled, do you want to leave enabled? [y,n] (y) > "; + prompt = "MariaDB ColumnStore Schema Sync feature is Enabled, do you want to leave enabled? [y,n] (y) > "; else - prompt = "MariaDB ColumnStore MySQL Replication feature, do you want to enable? [y,n] (y) > "; + prompt = "MariaDB ColumnStore Schema Sync feature, do you want to enable? [y,n] (y) > "; pcommand = callReadline(prompt.c_str()); if (pcommand) { From a89ab2f2bf315fd9d72b0ca33beeb25108db6d55 Mon Sep 17 00:00:00 2001 From: Andrew Hutchings Date: Thu, 19 Jan 2017 16:43:27 +0000 Subject: [PATCH 16/61] MCOL-454 Fix HWM edge case Sometimes block offset can be > 0 but HWM can be 0 this causes a calculation error in the DATA_SIZE field of I_S.columnstore_extents --- dbcon/mysql/is_columnstore_extents.cpp | 10 +++++++++- 1 file changed, 9 insertions(+), 1 deletion(-) diff --git a/dbcon/mysql/is_columnstore_extents.cpp b/dbcon/mysql/is_columnstore_extents.cpp index 1d1098eda..9f7e11710 100644 --- a/dbcon/mysql/is_columnstore_extents.cpp +++ b/dbcon/mysql/is_columnstore_extents.cpp @@ -152,7 +152,15 @@ static int is_columnstore_extents_fill(THD *thd, TABLE_LIST *tables, COND *cond) default: table->field[14]->store("Unknown", strlen("Unknown"), cs); } - table->field[15]->store((iter->HWM - iter->blockOffset + 1) * 8192); + // MCOL-454: special case, sometimes blockOffset can be > 0 and HWM can be 0 + if (iter->HWM == 0) + { + table->field[15]->store(8192); + } + else + { + table->field[15]->store((iter->HWM - iter->blockOffset + 1) * 8192); + } if (schema_table_store_record(thd, table)) { From ddf6d15bec6ab64a50c681f5293dae2019a4c755 Mon Sep 17 00:00:00 2001 From: david hill Date: Thu, 19 Jan 2017 15:04:00 -0600 Subject: [PATCH 17/61] MCOL-481 fix --- oamapps/postConfigure/postConfigure.cpp | 47 ++++++++++--------------- procmgr/processmanager.cpp | 22 ++---------- 2 files changed, 21 insertions(+), 48 deletions(-) diff --git a/oamapps/postConfigure/postConfigure.cpp b/oamapps/postConfigure/postConfigure.cpp index f8e66311a..1679a07d1 100644 --- a/oamapps/postConfigure/postConfigure.cpp +++ b/oamapps/postConfigure/postConfigure.cpp @@ -703,37 +703,26 @@ int main(int argc, char *argv[]) exit(1); } - if ( mysqlRep ) - { // current enabled - if ( answer == "y" ) { - mysqlRep = true; - MySQLRep = "y"; - } - else - { - mysqlRep = false; - MySQLRep = "n"; - } - } - else - { // currently disabled - if ( answer == "y" ) { - mysqlRep = false; - MySQLRep = "n"; - } - else - { - mysqlRep = true; - MySQLRep = "y"; - } - } + if ( answer == "y" ) { + mysqlRep = true; + MySQLRep = "y"; + } + else + { + mysqlRep = false; + MySQLRep = "n"; + } - try { - sysConfig->setConfig(InstallSection, "MySQLRep", MySQLRep); - } - catch(...) - {} + try { + sysConfig->setConfig(InstallSection, "MySQLRep", MySQLRep); + } + catch(...) + {} + if ( !writeConfig(sysConfig) ) { + cout << "ERROR: Failed trying to update MariaDB ColumnStore System Configuration file" << endl; + exit(1); + } switch ( IserverTypeInstall ) { case (oam::INSTALL_COMBINE_DM_UM_PM): // combined #1 - dm/um/pm on a single server diff --git a/procmgr/processmanager.cpp b/procmgr/processmanager.cpp index c5efc3c31..8dedf915d 100644 --- a/procmgr/processmanager.cpp +++ b/procmgr/processmanager.cpp @@ -10054,15 +10054,7 @@ int ProcessManager::setMySQLReplication(oam::DeviceNetworkList devicenetworklist try { Config* sysConfig = Config::makeConfig(); if ( sysConfig->getConfig("DBRM_Controller", "NumWorkers") == "1" ) { - //disable mysqlrep - log.writeLog(__LINE__, "Disable MySQL Replication", LOG_TYPE_DEBUG); - try { - oam.setSystemConfig("MySQLRep", "n"); - } - catch(...) {} - - enable = false; - distributeDB = true; + return oam::API_SUCCESS; } } catch(...) @@ -10080,17 +10072,9 @@ int ProcessManager::setMySQLReplication(oam::DeviceNetworkList devicenetworklist catch(...) {} - if ( moduletypeconfig.ModuleCount < 1 ) + if ( moduletypeconfig.ModuleCount < 2 ) { - //disable mysqlrep - log.writeLog(__LINE__, "Disable MySQL Replication", LOG_TYPE_DEBUG); - try { - oam.setSystemConfig("MySQLRep", "n"); - } - catch(...) {} - - enable = false; - distributeDB = true; + return oam::API_SUCCESS; } } From faea63b9cbc050e35490c6cc90ee0900558d1aca Mon Sep 17 00:00:00 2001 From: David Hall Date: Fri, 20 Jan 2017 10:10:24 -0600 Subject: [PATCH 18/61] MCOL-455: fix crash caused by rewind of unopened redistribute plan file during display when no partitions have been planned to be moved. --- .../we_redistributecontrolthread.cpp | 46 ++++++++++++------- 1 file changed, 29 insertions(+), 17 deletions(-) diff --git a/writeengine/redistribute/we_redistributecontrolthread.cpp b/writeengine/redistribute/we_redistributecontrolthread.cpp index dd498dd2a..c43f83c3b 100644 --- a/writeengine/redistribute/we_redistributecontrolthread.cpp +++ b/writeengine/redistribute/we_redistributecontrolthread.cpp @@ -316,7 +316,7 @@ int RedistributeControlThread::makeRedistributePlan() targetDbroots.insert(*j); } } - // At this point, there are two concepts of target. (1)Those fTargetSet, which is the + // At this point, there are two concepts of target. (1)Those in fTargetSet, which is the // set of dbroots the user wants partitions on and (2) those in targetDbroots, a subset of // fTargetSet, which is those that actually have room, based on average, for more data. @@ -423,6 +423,7 @@ int RedistributeControlThread::makeRedistributePlan() if (dbPartVec[*targetDbroot].size() < partCount) { tdbroot = *targetDbroot; + partCount = dbPartVec[*targetDbroot].size(); } } if (tdbroot == 0) @@ -495,15 +496,22 @@ void RedistributeControlThread::dumpPlanToFile(uint64_t oid, vectorfPlanFilePtr); - - ByteStream bs; - uint32_t entryId = 0; - long entrySize = sizeof(RedistributePlanEntry); - fControl->logMessage(string("Redistribution Plan:")); - while (entryId++ < fEntryCount) + try { - try + if (!fControl->fPlanFilePtr) + { + ostringstream oss; + oss << "No data is schefuled to be moved" << endl; + fControl->logMessage(oss.str()); + return; + } + rewind(fControl->fPlanFilePtr); + + ByteStream bs; + uint32_t entryId = 0; + long entrySize = sizeof(RedistributePlanEntry); + fControl->logMessage(string("Redistribution Plan:")); + while (entryId++ < fEntryCount) { RedistributePlanEntry entry; errno = 0; @@ -522,14 +530,18 @@ void RedistributeControlThread::displayPlan() << " moves from dbroot " << entry.source << " to " << entry.destination << endl; fControl->logMessage(oss.str()); } - catch (std::exception &e) - { - cout << "exception during display of plan: " << e.what() << endl; - } - catch (...) - { - cout << "exception during display of plan" << endl; - } + } + catch (std::exception &e) + { + ostringstream oss; + oss << "exception during display of plan: " << e.what() << endl; + fControl->logMessage(oss.str()); + } + catch (...) + { + ostringstream oss; + oss << "exception during display of plan" << endl; + fControl->logMessage(oss.str()); } } From d054d0edc72e007370d62d42f6447497967df7bb Mon Sep 17 00:00:00 2001 From: David Hall Date: Fri, 20 Jan 2017 10:37:43 -0600 Subject: [PATCH 19/61] MCOL-455 spell check --- writeengine/redistribute/we_redistributecontrolthread.cpp | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/writeengine/redistribute/we_redistributecontrolthread.cpp b/writeengine/redistribute/we_redistributecontrolthread.cpp index c43f83c3b..652b48565 100644 --- a/writeengine/redistribute/we_redistributecontrolthread.cpp +++ b/writeengine/redistribute/we_redistributecontrolthread.cpp @@ -501,7 +501,7 @@ void RedistributeControlThread::displayPlan() if (!fControl->fPlanFilePtr) { ostringstream oss; - oss << "No data is schefuled to be moved" << endl; + oss << "No data is scheduled to be moved" << endl; fControl->logMessage(oss.str()); return; } From 75b4d0648190401de4b92106261c0e69dbae585c Mon Sep 17 00:00:00 2001 From: David Hill Date: Fri, 20 Jan 2017 19:16:54 -0600 Subject: [PATCH 20/61] MCOL-455 - add start remove to help --- oam/etc/ConsoleCmds.xml | 1 + 1 file changed, 1 insertion(+) diff --git a/oam/etc/ConsoleCmds.xml b/oam/etc/ConsoleCmds.xml index 1373a211c..4bf12d36b 100644 --- a/oam/etc/ConsoleCmds.xml +++ b/oam/etc/ConsoleCmds.xml @@ -31,6 +31,7 @@ START to begin a redistribution STOP to stop redistribution before completion STATUS to to view statistics and progress + START REMOVE to redistribute and move data off the given dbroot to other dbroots findObjectFile From 588dc7a1220578e470eb7d4e5cfee34a5bfe7335 Mon Sep 17 00:00:00 2001 From: David Hill Date: Fri, 20 Jan 2017 19:25:48 -0600 Subject: [PATCH 21/61] MCOL-455 - add start remove to help --- oam/etc/ConsoleCmds.xml | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) diff --git a/oam/etc/ConsoleCmds.xml b/oam/etc/ConsoleCmds.xml index 4bf12d36b..01f6001cc 100644 --- a/oam/etc/ConsoleCmds.xml +++ b/oam/etc/ConsoleCmds.xml @@ -29,9 +29,9 @@ redistributeData Redistribute table data accross all dbroots to balance disk usage START to begin a redistribution - STOP to stop redistribution before completion - STATUS to to view statistics and progress - START REMOVE to redistribute and move data off the given dbroot to other dbroots + START REMOVE n to being a redistribution where data is removed from dbroot 'n' + STOP to stop redistribution before completion + STATUS to to view statistics and progress findObjectFile From a75c597d3b4ca15fe052aa4bcfc2e903dfdc388c Mon Sep 17 00:00:00 2001 From: david hill Date: Fri, 20 Jan 2017 19:49:26 -0600 Subject: [PATCH 22/61] MCOL-528 - removed schema sync prompt and default to on when local query is selected --- oamapps/postConfigure/postConfigure.cpp | 127 +++++++++++++----------- 1 file changed, 69 insertions(+), 58 deletions(-) diff --git a/oamapps/postConfigure/postConfigure.cpp b/oamapps/postConfigure/postConfigure.cpp index 1679a07d1..1facf1a7c 100644 --- a/oamapps/postConfigure/postConfigure.cpp +++ b/oamapps/postConfigure/postConfigure.cpp @@ -666,64 +666,6 @@ int main(int argc, char *argv[]) exit(1); } - cout << "NOTE: The MariaDB ColumnStore Schema Sync feature will replicate all of the" << endl; - cout << " schemas and InnoDB tables across the User Module nodes. This feature can be enabled" << endl; - cout << " or disabled, for example, if you wish to configure your own replication post installation." << endl << endl; - - try { - MySQLRep = sysConfig->getConfig(InstallSection, "MySQLRep"); - } - catch(...) - {} - - if ( MySQLRep == "y" ) - mysqlRep = true; - - string answer = "y"; - - while(true) { - if ( mysqlRep ) - prompt = "MariaDB ColumnStore Schema Sync feature is Enabled, do you want to leave enabled? [y,n] (y) > "; - else - prompt = "MariaDB ColumnStore Schema Sync feature, do you want to enable? [y,n] (y) > "; - - pcommand = callReadline(prompt.c_str()); - if (pcommand) { - if (strlen(pcommand) > 0) answer = pcommand; - callFree(pcommand); - } - - if ( answer == "y" || answer == "n" ) { - cout << endl; - break; - } - else - cout << "Invalid Entry, please enter 'y' for yes or 'n' for no" << endl; - if ( noPrompting ) - exit(1); - } - - if ( answer == "y" ) { - mysqlRep = true; - MySQLRep = "y"; - } - else - { - mysqlRep = false; - MySQLRep = "n"; - } - - try { - sysConfig->setConfig(InstallSection, "MySQLRep", MySQLRep); - } - catch(...) - {} - - if ( !writeConfig(sysConfig) ) { - cout << "ERROR: Failed trying to update MariaDB ColumnStore System Configuration file" << endl; - exit(1); - } - switch ( IserverTypeInstall ) { case (oam::INSTALL_COMBINE_DM_UM_PM): // combined #1 - dm/um/pm on a single server { @@ -813,6 +755,75 @@ int main(int argc, char *argv[]) break; } + // check for Schema Schema is Local Query wasnt selected + if (!pmwithum) + { + cout << "NOTE: The MariaDB ColumnStore Schema Sync feature will replicate all of the" << endl; + cout << " schemas and InnoDB tables across the User Module nodes. This feature can be enabled" << endl; + cout << " or disabled, for example, if you wish to configure your own replication post installation." << endl << endl; + + try { + MySQLRep = sysConfig->getConfig(InstallSection, "MySQLRep"); + } + catch(...) + {} + + if ( MySQLRep == "y" ) + mysqlRep = true; + + string answer = "y"; + + while(true) { + if ( mysqlRep ) + prompt = "MariaDB ColumnStore Schema Sync feature is Enabled, do you want to leave enabled? [y,n] (y) > "; + else + prompt = "MariaDB ColumnStore Schema Sync feature, do you want to enable? [y,n] (y) > "; + + pcommand = callReadline(prompt.c_str()); + if (pcommand) { + if (strlen(pcommand) > 0) answer = pcommand; + callFree(pcommand); + } + + if ( answer == "y" || answer == "n" ) { + cout << endl; + break; + } + else + cout << "Invalid Entry, please enter 'y' for yes or 'n' for no" << endl; + + if ( noPrompting ) + exit(1); + } + + if ( answer == "y" ) { + mysqlRep = true; + MySQLRep = "y"; + } + else + { + mysqlRep = false; + MySQLRep = "n"; + } + + try { + sysConfig->setConfig(InstallSection, "MySQLRep", MySQLRep); + } + catch(...) + {} + } + else + { //Schema Sync is default as on when Local Query is Selected + mysqlRep = true; + MySQLRep = "y"; + + try { + sysConfig->setConfig(InstallSection, "MySQLRep", MySQLRep); + } + catch(...) + {} + } + if ( !writeConfig(sysConfig) ) { cout << "ERROR: Failed trying to update MariaDB ColumnStore System Configuration file" << endl; exit(1); From e84573af8cfece8a701c82e4efb04dddc6d5058d Mon Sep 17 00:00:00 2001 From: david hill Date: Fri, 20 Jan 2017 19:57:22 -0600 Subject: [PATCH 23/61] MCOL-527 - change UM device name from sdf to xvdf --- oam/oamcpp/liboamcpp.cpp | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/oam/oamcpp/liboamcpp.cpp b/oam/oamcpp/liboamcpp.cpp index 3d35f35b7..810aac264 100644 --- a/oam/oamcpp/liboamcpp.cpp +++ b/oam/oamcpp/liboamcpp.cpp @@ -5987,7 +5987,7 @@ namespace oam } //attach and format volumes - device = "/dev/sdf"; + device = "/dev/xvdf"; string localInstance = getEC2LocalInstance(); From a6193ec7cde6c26900eed23d0358cdab3857f4a8 Mon Sep 17 00:00:00 2001 From: david hill Date: Fri, 20 Jan 2017 23:15:00 -0600 Subject: [PATCH 24/61] MCOL-527 - add code for non-root --- procmon/processmonitor.cpp | 7 ++++++- 1 file changed, 6 insertions(+), 1 deletion(-) diff --git a/procmon/processmonitor.cpp b/procmon/processmonitor.cpp index f11598a0d..b45e5be84 100644 --- a/procmon/processmonitor.cpp +++ b/procmon/processmonitor.cpp @@ -5501,7 +5501,12 @@ bool ProcessMonitor::amazonVolumeCheck(int dbrootID) string status = oam.getEC2VolumeStatus(volumeName); if ( status == "attached" ) { - string cmd = "mount " + deviceName + " " + startup::StartUp::installDir() + "/mysql/db -t ext2 -o defaults > /dev/null"; + string cmd; + if ( rootUser) + cmd = "mount " + deviceName + " " + startup::StartUp::installDir() + "/mysql/db -t ext2 -o defaults > /tmp/um_mount.log"; + else + cmd = "sudo mount " + deviceName + " " + startup::StartUp::installDir() + "/mysql/db -t ext2 -o defaults > /tmp/um_mount.log"; + system(cmd.c_str()); log.writeLog(__LINE__, "mount cmd: " + cmd, LOG_TYPE_DEBUG); From df084281b89910fda4fd675bf5b7ee7ecb8ee947 Mon Sep 17 00:00:00 2001 From: David Hill Date: Sat, 21 Jan 2017 15:51:25 -0600 Subject: [PATCH 25/61] MCOL-527 - change to run mysql scripts when um disk is mounted --- procmon/processmonitor.cpp | 81 +++++++++++++++++++++++++++++++++++--- 1 file changed, 76 insertions(+), 5 deletions(-) diff --git a/procmon/processmonitor.cpp b/procmon/processmonitor.cpp index b45e5be84..e474c69d5 100644 --- a/procmon/processmonitor.cpp +++ b/procmon/processmonitor.cpp @@ -5271,6 +5271,8 @@ int ProcessMonitor::runMasterDist(std::string& password, std::string& slaveModul string cmd = startup::StartUp::installDir() + "/bin/rsync.sh " + ipAddr + " " + password + " " + startup::StartUp::installDir() + " 1 > /tmp/master-dist_" + slaveModule + ".log"; system(cmd.c_str()); + + log.writeLog(__LINE__, "cmd = " + cmd, LOG_TYPE_DEBUG); string logFile = "/tmp/master-dist_" + slaveModule + ".log"; if (!oam.checkLogStatus(logFile, "FAILED")) @@ -5503,15 +5505,47 @@ bool ProcessMonitor::amazonVolumeCheck(int dbrootID) if ( status == "attached" ) { string cmd; if ( rootUser) - cmd = "mount " + deviceName + " " + startup::StartUp::installDir() + "/mysql/db -t ext2 -o defaults > /tmp/um_mount.log"; + cmd = "mount " + deviceName + " " + startup::StartUp::installDir() + "/mysql/db -t ext2 -o noatime,nodiratime,noauto > /tmp/um_mount.log"; else - cmd = "sudo mount " + deviceName + " " + startup::StartUp::installDir() + "/mysql/db -t ext2 -o defaults > /tmp/um_mount.log"; + cmd = "sudo mount " + deviceName + " " + startup::StartUp::installDir() + "/mysql/db -t ext2 -o noatime,nodiratime,noauto,user > /tmp/um_mount.log"; system(cmd.c_str()); log.writeLog(__LINE__, "mount cmd: " + cmd, LOG_TYPE_DEBUG); - cmd = "chown mysql:mysql -R " + startup::StartUp::installDir() + "/mysql/db"; + if ( rootUser) + cmd = "chown -R mysql:mysql " + startup::StartUp::installDir() + "/mysql/db"; + else + cmd = "sudo chown -R " + USER + ":" + USER + " " + startup::StartUp::installDir() + "/mysql/db"; + system(cmd.c_str()); + log.writeLog(__LINE__, "chown cmd: " + cmd, LOG_TYPE_DEBUG); + + //check for setup files in mysq/db, if not, create them for a new install + string file = startup::StartUp::installDir() + "/mysql/db/mysql"; + ifstream new_file (file.c_str()); + if (!new_file) { + string cmd; + cmd = startup::StartUp::installDir() + "/bin/post-mysqld-install --installdir=" + startup::StartUp::installDir() + " > /tmp/post-mysqld-install.log 2>&1"; + log.writeLog(__LINE__, "cmd: " + cmd, LOG_TYPE_DEBUG); + + int rtnCode = system(cmd.c_str()); + if (WEXITSTATUS(rtnCode) != 0) { + log.writeLog(__LINE__, "amazonVolumeCheck function failed, post-mysqld-install error" , LOG_TYPE_ERROR); + return false; + } + else + log.writeLog(__LINE__, "amazonVolumeCheck function, post-mysqld-install passed" , LOG_TYPE_DEBUG); + + cmd = startup::StartUp::installDir() + "/bin/post-mysql-install --installdir=" + startup::StartUp::installDir() + " > /tmp/post-mysql-install.log";; + log.writeLog(__LINE__, "cmd: " + cmd, LOG_TYPE_DEBUG); + rtnCode = system(cmd.c_str()); + if (WEXITSTATUS(rtnCode) != 0) { + log.writeLog(__LINE__, "amazonVolumeCheck function failed, post-mysql-install error" , LOG_TYPE_ERROR); + return false; + } + else + log.writeLog(__LINE__, "amazonVolumeCheck function, post-mysql-install passed" , LOG_TYPE_DEBUG); + } log.writeLog(__LINE__, "amazonVolumeCheck function successfully completed, volume attached: " + volumeName, LOG_TYPE_DEBUG); return true; @@ -5536,12 +5570,49 @@ bool ProcessMonitor::amazonVolumeCheck(int dbrootID) {} if (oam.attachEC2Volume(volumeName, deviceName, instanceName)) { - string cmd = "mount " + deviceName + " " + startup::StartUp::installDir() + "/mysql/db -t ext2 -o defaults > /dev/null"; + string cmd; + if ( rootUser) + cmd = "mount " + deviceName + " " + startup::StartUp::installDir() + "/mysql/db -t ext2 -o noatime,nodiratime,noauto > /tmp/um_mount.log"; + else + cmd = "sudo mount " + deviceName + " " + startup::StartUp::installDir() + "/mysql/db -t ext2 -o noatime,nodiratime,noauto,user > /tmp/um_mount.log"; + system(cmd.c_str()); log.writeLog(__LINE__, "mount cmd: " + cmd, LOG_TYPE_DEBUG); - cmd = "chown mysql:mysql -R " + startup::StartUp::installDir() + "/mysql/db"; + if ( rootUser) + cmd = "chown -R mysql:mysql " + startup::StartUp::installDir() + "/mysql/db"; + else + cmd = "sudo chown -R " + USER + ":" + USER + " " + startup::StartUp::installDir() + "/mysql/db"; + system(cmd.c_str()); + log.writeLog(__LINE__, "chown cmd: " + cmd, LOG_TYPE_DEBUG); + + //check for setup files in mysq/db, if not, create them for a new install + string file = startup::StartUp::installDir() + "/mysql/db/mysql"; + ifstream new_file (file.c_str()); + if (!new_file) { + string cmd; + cmd = startup::StartUp::installDir() + "/bin/post-mysqld-install --installdir=" + startup::StartUp::installDir() + " > /tmp/post-mysqld-install.log 2>&1"; + log.writeLog(__LINE__, "cmd: " + cmd, LOG_TYPE_DEBUG); + int rtnCode = system(cmd.c_str()); + if (WEXITSTATUS(rtnCode) != 0) { + log.writeLog(__LINE__, "amazonVolumeCheck function failed, post-mysqld-install error" , LOG_TYPE_ERROR); + return false; + } + else + log.writeLog(__LINE__, "amazonVolumeCheck function, post-mysqld-install passed" , LOG_TYPE_DEBUG); + + cmd = startup::StartUp::installDir() + "/bin/post-mysql-install --installdir=" + startup::StartUp::installDir() + " > /tmp/post-mysql-install.log";; + log.writeLog(__LINE__, "cmd: " + cmd, LOG_TYPE_DEBUG); + + rtnCode = system(cmd.c_str()); + if (WEXITSTATUS(rtnCode) != 0) { + log.writeLog(__LINE__, "amazonVolumeCheck function failed, post-mysql-install error" , LOG_TYPE_ERROR); + return false; + } + else + log.writeLog(__LINE__, "amazonVolumeCheck function, post-mysql-install passed" , LOG_TYPE_DEBUG); + } return true; } From eabacb07f661ab45db72fe05d66faf6000da8c35 Mon Sep 17 00:00:00 2001 From: David Hall Date: Fri, 3 Mar 2017 15:57:41 -0600 Subject: [PATCH 26/61] MCOL-597 Take Window Functions just to get a compile. No other changes were needed. --- dbcon/mysql/ha_calpont.cpp | 8 ---- dbcon/mysql/ha_calpont_execplan.cpp | 61 +++++++++++++++++------------ dbcon/mysql/ha_calpont_impl.cpp | 12 +----- dbcon/mysql/ha_calpont_impl.h | 1 - dbcon/mysql/ha_window_function.cpp | 13 +++--- 5 files changed, 46 insertions(+), 49 deletions(-) diff --git a/dbcon/mysql/ha_calpont.cpp b/dbcon/mysql/ha_calpont.cpp index 31b08451c..16e1e0def 100755 --- a/dbcon/mysql/ha_calpont.cpp +++ b/dbcon/mysql/ha_calpont.cpp @@ -121,7 +121,6 @@ static int calpont_commit(handlerton *hton, THD* thd, bool all); static int calpont_rollback(handlerton *hton, THD* thd, bool all); static int calpont_close_connection ( handlerton *hton, THD* thd ); -static void calpont_set_error( THD*, uint64_t, LEX_STRING*, uint32_t); handlerton *calpont_hton; /* Variables for example share methods */ @@ -219,7 +218,6 @@ static int columnstore_init_func(void *p) calpont_hton->commit= calpont_commit; calpont_hton->rollback= calpont_rollback; calpont_hton->close_connection = calpont_close_connection; - calpont_hton->set_error= calpont_set_error; DBUG_RETURN(0); } @@ -248,7 +246,6 @@ static int infinidb_init_func(void *p) calpont_hton->commit= calpont_commit; calpont_hton->rollback= calpont_rollback; calpont_hton->close_connection = calpont_close_connection; - calpont_hton->set_error= calpont_set_error; DBUG_RETURN(0); } @@ -372,11 +369,6 @@ static int calpont_close_connection ( handlerton *hton, THD* thd ) return rc; } -static void calpont_set_error(THD* thd, uint64_t errCode, LEX_STRING* args, uint32_t argCount) -{ - return ha_calpont_impl_set_error(thd, errCode, args, argCount); -} - ha_calpont::ha_calpont(handlerton *hton, TABLE_SHARE *table_arg) : handler(hton, table_arg), int_table_flags(HA_BINLOG_STMT_CAPABLE | HA_BINLOG_ROW_CAPABLE | diff --git a/dbcon/mysql/ha_calpont_execplan.cpp b/dbcon/mysql/ha_calpont_execplan.cpp index 22aa4742b..233a3fcfc 100755 --- a/dbcon/mysql/ha_calpont_execplan.cpp +++ b/dbcon/mysql/ha_calpont_execplan.cpp @@ -699,11 +699,13 @@ void debug_walk(const Item *item, void *arg) cout << ": " << endl; break; } +#if 0 case Item::WINDOW_FUNC_ITEM: { cout << "Window Function Item" << endl; break; } +#endif default: { cout << "UNKNOWN_ITEM type " << item->type() << endl; @@ -2453,6 +2455,7 @@ ReturnedColumn* buildReturnedColumn(Item* item, gp_walk_info& gwi, bool& nonSupp rc = new ConstantColumn(valStr); break; } +#if 0 case Item::WINDOW_FUNC_ITEM: { return buildWindowFunctionColumn(item, gwi, nonSupport); @@ -2469,6 +2472,7 @@ ReturnedColumn* buildReturnedColumn(Item* item, gp_walk_info& gwi, bool& nonSupp rc->resultType(srcp->resultType()); break; } +#endif case Item::SUBSELECT_ITEM: { gwi.hasSubSelect = true; @@ -3500,23 +3504,24 @@ ReturnedColumn* buildAggregateColumn(Item* item, gp_walk_info& gwi) end=order_item + gc->order_field();order_item < end; order_item++) { - Item *ord_col= *(*order_item)->item; - if (ord_col->type() == Item::INT_ITEM) - { - Item_int* id = (Item_int*)ord_col; - if (id->val_int() > (int)selCols.size()) - { - gwi.fatalParseError = true; - return NULL; - } - rc = selCols[id->val_int()-1]->clone(); - rc->orderPos(id->val_int()-1); - } - else - { - rc = buildReturnedColumn(ord_col, gwi, gwi.fatalParseError); - } - rc->asc((*order_item)->asc); + Item *ord_col= *(*order_item)->item; + if (ord_col->type() == Item::INT_ITEM) + { + Item_int* id = (Item_int*)ord_col; + if (id->val_int() > (int)selCols.size()) + { + gwi.fatalParseError = true; + return NULL; + } + rc = selCols[id->val_int()-1]->clone(); + rc->orderPos(id->val_int()-1); + } + else + { + rc = buildReturnedColumn(ord_col, gwi, gwi.fatalParseError); + } + // 10.2 TODO: direction is now a tri-state flag + rc->asc((*order_item)->direction == ORDER::ORDER_ASC ? true : false); orderCols.push_back(SRCP(rc)); } @@ -4431,6 +4436,7 @@ void gp_walk(const Item *item, void *arg) gwip->rcWorkStack.push(buildReturnedColumn(itp, *gwip, gwip->fatalParseError)); break; } +#if 0 case Item::WINDOW_FUNC_ITEM: { gwip->hasWindowFunc = true; @@ -4440,6 +4446,7 @@ void gp_walk(const Item *item, void *arg) gwip->rcWorkStack.push(af); break; } +#endif case Item::COPY_STR_ITEM: printf("********** received COPY_STR_ITEM *********\n"); break; @@ -4623,9 +4630,11 @@ void parse_item (Item *item, vector& field_vec, bool& hasNonSupport setError(item->thd(), ER_CHECK_NOT_IMPLEMENTED, parseErrorText); break; } +#if 0 case Item::WINDOW_FUNC_ITEM: parseInfo |= AF_BIT; break; +#endif default: break; } @@ -5515,6 +5524,7 @@ int getSelectPlan(gp_walk_info& gwi, SELECT_LEX& select_lex, SCSEP& csep, bool i setError(gwi.thd, ER_CHECK_NOT_IMPLEMENTED, gwi.parseErrorText, gwi); return ER_CHECK_NOT_IMPLEMENTED; } +#if 0 case Item::WINDOW_FUNC_ITEM: { SRCP srcp(buildWindowFunctionColumn(item, gwi, gwi.fatalParseError)); @@ -5528,6 +5538,7 @@ int getSelectPlan(gp_walk_info& gwi, SELECT_LEX& select_lex, SCSEP& csep, bool i gwi.returnedCols.push_back(srcp); break; } +#endif default: { break; @@ -5680,6 +5691,7 @@ int getSelectPlan(gp_walk_info& gwi, SELECT_LEX& select_lex, SCSEP& csep, bool i // check if window functions are in order by. InfiniDB process order by list if // window functions are involved, either in order by or projection. +#if 0 bool hasWindowFunc = gwi.hasWindowFunc; gwi.hasWindowFunc = false; for (; groupcol; groupcol= groupcol->next) @@ -5695,7 +5707,7 @@ int getSelectPlan(gp_walk_info& gwi, SELECT_LEX& select_lex, SCSEP& csep, bool i return ER_CHECK_NOT_IMPLEMENTED; } gwi.hasWindowFunc = hasWindowFunc; - +#endif groupcol = reinterpret_cast(select_lex.group_list.first); for (; groupcol; groupcol= groupcol->next) @@ -5941,12 +5953,13 @@ int getSelectPlan(gp_walk_info& gwi, SELECT_LEX& select_lex, SCSEP& csep, bool i // check if window functions are in order by. InfiniDB process order by list if // window functions are involved, either in order by or projection. +#if 0 for (; ordercol; ordercol= ordercol->next) { if ((*(ordercol->item))->type() == Item::WINDOW_FUNC_ITEM) gwi.hasWindowFunc = true; } - +#endif // re-visit the first of ordercol list ordercol = reinterpret_cast(order_list.first); @@ -5997,7 +6010,7 @@ int getSelectPlan(gp_walk_info& gwi, SELECT_LEX& select_lex, SCSEP& csep, bool i return ER_CHECK_NOT_IMPLEMENTED; } } - if (ordercol->asc) + if (ordercol->direction == ORDER::ORDER_ASC) rc->asc(true); else rc->asc(false); @@ -6030,7 +6043,7 @@ int getSelectPlan(gp_walk_info& gwi, SELECT_LEX& select_lex, SCSEP& csep, bool i ostringstream oss; oss << ordercol->counter; ord_cols += oss.str(); - if (!ordercol->asc) + if (!ordercol->direction == ORDER::ORDER_ASC) ord_cols += " desc"; continue; } @@ -6155,7 +6168,7 @@ int getSelectPlan(gp_walk_info& gwi, SELECT_LEX& select_lex, SCSEP& csep, bool i gwi.returnedCols.push_back(srcp); ord_cols += " `" + escapeBackTick(str.c_ptr()) + "`"; } - if (!ordercol->asc) + if (!ordercol->direction == ORDER::ORDER_ASC) ord_cols += " desc"; continue; } @@ -6219,7 +6232,7 @@ int getSelectPlan(gp_walk_info& gwi, SELECT_LEX& select_lex, SCSEP& csep, bool i ord_item->print(&str, QT_INFINIDB); ord_cols += str.c_ptr(); } - if (!ordercol->asc) + if (!ordercol->direction == ORDER::ORDER_ASC) ord_cols += " desc"; } } @@ -6559,7 +6572,7 @@ int getSelectPlan(gp_walk_info& gwi, SELECT_LEX& select_lex, SCSEP& csep, bool i ord_item->print(&str, QT_INFINIDB_NO_QUOTE); ord_cols += string(str.c_ptr()); } - if (!ordercol->asc) + if (!ordercol->direction == ORDER::ORDER_ASC) ord_cols += " desc"; } } diff --git a/dbcon/mysql/ha_calpont_impl.cpp b/dbcon/mysql/ha_calpont_impl.cpp index f86a8a73c..931048cec 100755 --- a/dbcon/mysql/ha_calpont_impl.cpp +++ b/dbcon/mysql/ha_calpont_impl.cpp @@ -1160,7 +1160,7 @@ uint32_t doUpdateDelete(THD *thd) { multi_delete* deleteTable = (multi_delete*)((thd->lex->select_lex.join)->result); first_table= (TABLE_LIST*) deleteTable->get_tables(); - if (deleteTable->num_of_tables == 1) + if (deleteTable->get_num_of_tables() == 1) { schemaName = first_table->db; tableName = first_table->table_name; @@ -4434,15 +4434,5 @@ int ha_calpont_impl_rnd_pos(uchar *buf, uchar *pos) return ER_INTERNAL_ERROR; } -// Called from mysql parser to set IDB error for window functions -void ha_calpont_impl_set_error(THD* thd, uint64_t errCode, LEX_STRING* args, uint32_t argCount) -{ - IDEBUG( cout << "ha_calpont_impl_ser_error" << endl); - Message::Args arguments; - for (uint32_t i = 0; i < argCount; i++) - arguments.add(args[i].str); - string emsg = logging::IDBErrorInfo::instance()->errorMsg(errCode, arguments); - setError(thd, ER_INTERNAL_ERROR, emsg); -} // vim:sw=4 ts=4: diff --git a/dbcon/mysql/ha_calpont_impl.h b/dbcon/mysql/ha_calpont_impl.h index 58f680a41..79fc6933b 100644 --- a/dbcon/mysql/ha_calpont_impl.h +++ b/dbcon/mysql/ha_calpont_impl.h @@ -48,7 +48,6 @@ extern int ha_calpont_impl_external_lock(THD *thd, TABLE* table, int lock_type); extern int ha_calpont_impl_update_row(); extern int ha_calpont_impl_delete_row(); extern int ha_calpont_impl_rnd_pos(uchar *buf, uchar *pos); -extern void ha_calpont_impl_set_error(THD* thd, uint64_t errCode, LEX_STRING* args, uint32_t argCount); #endif #ifdef NEED_CALPONT_INTERFACE diff --git a/dbcon/mysql/ha_window_function.cpp b/dbcon/mysql/ha_window_function.cpp index 4081baffd..d15e2c2bf 100755 --- a/dbcon/mysql/ha_window_function.cpp +++ b/dbcon/mysql/ha_window_function.cpp @@ -63,6 +63,7 @@ ReturnedColumn* nullOnError(gp_walk_info& gwi) return NULL; } +#if 0 WF_FRAME frame(BOUND& bound) { switch (bound) @@ -81,7 +82,7 @@ WF_FRAME frame(BOUND& bound) return WF_UNKNOWN; } } - +#endif ReturnedColumn* buildBoundExp(WF_Boundary& bound, SRCP& order, gp_walk_info& gwi) { if (!(gwi.thd->infinidb_vtable.cal_conn_info)) @@ -188,12 +189,13 @@ ReturnedColumn* buildWindowFunctionColumn(Item* item, gp_walk_info& gwi, bool& n cal_connection_info* ci = reinterpret_cast(gwi.thd->infinidb_vtable.cal_conn_info); gwi.hasWindowFunc = true; - Item_func_window* wf = (Item_func_window*)item; - string funcName = wf->func_name(); +// Item_func_window* wf = (Item_func_window*)item; + string funcName /*= wf->func_name()*/; WindowFunctionColumn* ac = new WindowFunctionColumn(funcName); - ac->distinct(wf->isDistinct()); - Window_context *wf_ctx = wf->window_ctx(); +// ac->distinct(wf->isDistinct()); +// Window_context *wf_ctx = wf->window_ctx(); SRCP srcp; +#if 0 // arguments vector funcParms; @@ -433,6 +435,7 @@ ReturnedColumn* buildWindowFunctionColumn(Item* item, gp_walk_info& gwi, bool& n // put ac on windowFuncList gwi.windowFuncList.push_back(ac); +#endif return ac; } From 4c4d516bc8066653bf8a95b5dad2d2824eb5f5ea Mon Sep 17 00:00:00 2001 From: David Hall Date: Mon, 13 Mar 2017 13:20:57 -0500 Subject: [PATCH 27/61] MCOL-597 stabilization --- dbcon/mysql/CMakeLists.txt | 10 +- dbcon/mysql/ha_calpont_execplan.cpp | 19 +- dbcon/mysql/ha_window_function.cpp | 2 +- dbcon/mysql/install_calpont_mysql.sh | 2 +- dbcon/mysql/my.cnf | 1 + dbcon/mysql/mysql.vpj | 582 ++++++++++++++++++--------- utils/funcexp/func_div.cpp | 4 +- utils/loggingcpp/ErrorMessage.txt | 2 +- 8 files changed, 415 insertions(+), 207 deletions(-) diff --git a/dbcon/mysql/CMakeLists.txt b/dbcon/mysql/CMakeLists.txt index d478843b0..4c423ed16 100644 --- a/dbcon/mysql/CMakeLists.txt +++ b/dbcon/mysql/CMakeLists.txt @@ -25,7 +25,7 @@ set_source_files_properties(ha_calpont.cpp PROPERTIES COMPILE_FLAGS "-fno-rtti - add_library(calmysql SHARED ${libcalmysql_SRCS}) -target_link_libraries(calmysql ${ENGINE_LDFLAGS} ${ENGINE_WRITE_LIBS} ${NETSNMP_LIBRARIES} threadpool) +target_link_libraries(calmysql ${ENGINE_LDFLAGS} ${ENGINE_WRITE_LIBS} ${NETSNMP_LIBRARIES} mysqlservices threadpool) set_target_properties(calmysql PROPERTIES VERSION 1.0.0 SOVERSION 1) @@ -34,7 +34,7 @@ SET ( is_columnstore_tables_SRCS ) add_library(is_columnstore_tables SHARED ${is_columnstore_tables_SRCS}) -target_link_libraries(is_columnstore_tables ${ENGINE_LDFLAGS} ${ENGINE_WRITE_LIBS} threadpool) +target_link_libraries(is_columnstore_tables ${ENGINE_LDFLAGS} ${ENGINE_WRITE_LIBS} mysqlservices threadpool) # Don't prepend .so file with 'lib' set_target_properties(is_columnstore_tables PROPERTIES PREFIX "") @@ -45,7 +45,7 @@ SET ( is_columnstore_columns_SRCS ) add_library(is_columnstore_columns SHARED ${is_columnstore_columns_SRCS}) -target_link_libraries(is_columnstore_columns ${ENGINE_LDFLAGS} ${ENGINE_WRITE_LIBS} threadpool) +target_link_libraries(is_columnstore_columns ${ENGINE_LDFLAGS} ${ENGINE_WRITE_LIBS} mysqlservices threadpool) # Don't prepend .so file with 'lib' set_target_properties(is_columnstore_columns PROPERTIES PREFIX "") @@ -56,7 +56,7 @@ SET ( is_columnstore_extents_SRCS ) add_library(is_columnstore_extents SHARED ${is_columnstore_extents_SRCS}) -target_link_libraries(is_columnstore_extents ${ENGINE_LDFLAGS} ${ENGINE_WRITE_LIBS} threadpool) +target_link_libraries(is_columnstore_extents ${ENGINE_LDFLAGS} ${ENGINE_WRITE_LIBS} mysqlservices threadpool) # Don't prepend .so file with 'lib' set_target_properties(is_columnstore_extents PROPERTIES PREFIX "") @@ -67,7 +67,7 @@ SET ( is_columnstore_files_SRCS ) add_library(is_columnstore_files SHARED ${is_columnstore_files_SRCS}) -target_link_libraries(is_columnstore_files ${ENGINE_LDFLAGS} ${ENGINE_WRITE_LIBS} threadpool) +target_link_libraries(is_columnstore_files ${ENGINE_LDFLAGS} ${ENGINE_WRITE_LIBS} mysqlservices threadpool) # Don't prepend .so file with 'lib' set_target_properties(is_columnstore_files PROPERTIES PREFIX "") diff --git a/dbcon/mysql/ha_calpont_execplan.cpp b/dbcon/mysql/ha_calpont_execplan.cpp index 233a3fcfc..443ce5bcd 100755 --- a/dbcon/mysql/ha_calpont_execplan.cpp +++ b/dbcon/mysql/ha_calpont_execplan.cpp @@ -423,7 +423,7 @@ void debug_walk(const Item *item, void *arg) char* item_name = item->name; if (!item_name) { - item_name = ""; + item_name = (char*)""; } switch (isp->sum_func()) { @@ -1111,6 +1111,15 @@ bool buildRowColumnFilter(gp_walk_info* gwip, RowColumn* rhs, RowColumn* lhs, It bool buildPredicateItem(Item_func* ifp, gp_walk_info* gwip) { boost::shared_ptr sop(new PredicateOperator(ifp->func_name())); + if (ifp->functype() == Item_func::LIKE_FUNC) + { + // Starting with MariaDB 10.2, LIKE uses a negated flag instead of FUNC_NOT + // Further processing is done below as before for LIKE + if (((Item_func_like*)ifp)->negated) + { + sop->reverseOp(); + } + } if (!(gwip->thd->infinidb_vtable.cal_conn_info)) gwip->thd->infinidb_vtable.cal_conn_info = (void*)(new cal_connection_info()); cal_connection_info* ci = reinterpret_cast(gwip->thd->infinidb_vtable.cal_conn_info); @@ -6043,7 +6052,7 @@ int getSelectPlan(gp_walk_info& gwi, SELECT_LEX& select_lex, SCSEP& csep, bool i ostringstream oss; oss << ordercol->counter; ord_cols += oss.str(); - if (!ordercol->direction == ORDER::ORDER_ASC) + if (ordercol->direction != ORDER::ORDER_ASC) ord_cols += " desc"; continue; } @@ -6168,7 +6177,7 @@ int getSelectPlan(gp_walk_info& gwi, SELECT_LEX& select_lex, SCSEP& csep, bool i gwi.returnedCols.push_back(srcp); ord_cols += " `" + escapeBackTick(str.c_ptr()) + "`"; } - if (!ordercol->direction == ORDER::ORDER_ASC) + if (ordercol->direction != ORDER::ORDER_ASC) ord_cols += " desc"; continue; } @@ -6232,7 +6241,7 @@ int getSelectPlan(gp_walk_info& gwi, SELECT_LEX& select_lex, SCSEP& csep, bool i ord_item->print(&str, QT_INFINIDB); ord_cols += str.c_ptr(); } - if (!ordercol->direction == ORDER::ORDER_ASC) + if (ordercol->direction != ORDER::ORDER_ASC) ord_cols += " desc"; } } @@ -6572,7 +6581,7 @@ int getSelectPlan(gp_walk_info& gwi, SELECT_LEX& select_lex, SCSEP& csep, bool i ord_item->print(&str, QT_INFINIDB_NO_QUOTE); ord_cols += string(str.c_ptr()); } - if (!ordercol->direction == ORDER::ORDER_ASC) + if (ordercol->direction != ORDER::ORDER_ASC) ord_cols += " desc"; } } diff --git a/dbcon/mysql/ha_window_function.cpp b/dbcon/mysql/ha_window_function.cpp index d15e2c2bf..8af20511e 100755 --- a/dbcon/mysql/ha_window_function.cpp +++ b/dbcon/mysql/ha_window_function.cpp @@ -186,7 +186,7 @@ ReturnedColumn* buildWindowFunctionColumn(Item* item, gp_walk_info& gwi, bool& n //cout << str.c_ptr() << endl; if (!(gwi.thd->infinidb_vtable.cal_conn_info)) gwi.thd->infinidb_vtable.cal_conn_info = (void*)(new cal_connection_info()); - cal_connection_info* ci = reinterpret_cast(gwi.thd->infinidb_vtable.cal_conn_info); +// cal_connection_info* ci = reinterpret_cast(gwi.thd->infinidb_vtable.cal_conn_info); gwi.hasWindowFunc = true; // Item_func_window* wf = (Item_func_window*)item; diff --git a/dbcon/mysql/install_calpont_mysql.sh b/dbcon/mysql/install_calpont_mysql.sh index 9b55f9448..611b50ef3 100755 --- a/dbcon/mysql/install_calpont_mysql.sh +++ b/dbcon/mysql/install_calpont_mysql.sh @@ -99,7 +99,7 @@ CREATE TABLE IF NOT EXISTS infinidb_querystats.querystats query varchar(8000), startTime timestamp NOT NULL, endTime timestamp NOT NULL, - rows bigint, + `rows` bigint, errno int, phyIO bigint, cacheIO bigint, diff --git a/dbcon/mysql/my.cnf b/dbcon/mysql/my.cnf index 3793e8baa..0fb2acd0e 100644 --- a/dbcon/mysql/my.cnf +++ b/dbcon/mysql/my.cnf @@ -44,6 +44,7 @@ query_cache_size = 0 thread_stack = 512K lower_case_table_names=1 group_concat_max_len=512 +sql_mode="ERROR_FOR_DIVISION_BY_ZERO,NO_AUTO_CREATE_USER,NO_ENGINE_SUBSTITUTION" # Enable compression by default on create, set to 0 to turn off infinidb_compression_type=2 diff --git a/dbcon/mysql/mysql.vpj b/dbcon/mysql/mysql.vpj index cdb8f4f12..fd92a4562 100644 --- a/dbcon/mysql/mysql.vpj +++ b/dbcon/mysql/mysql.vpj @@ -197,6 +197,198 @@ + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + @@ -218,204 +410,210 @@ - - - - - - - - - - - - - - - - - - - - - - - - - - + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - 0 ? val1 + 0.5 : val1 - 0.5); // MCOL-176 If abs(int_val2) is small enough (like -1), then, this may cause overflow. @@ -78,7 +78,7 @@ uint64_t Func_div::getUintVal(rowgroup::Row& row, if (val2 == 0) { isNull = true; - return NULL; + return 0; } return val1 / val2; } diff --git a/utils/loggingcpp/ErrorMessage.txt b/utils/loggingcpp/ErrorMessage.txt index 1266a21e4..c4d134899 100644 --- a/utils/loggingcpp/ErrorMessage.txt +++ b/utils/loggingcpp/ErrorMessage.txt @@ -27,7 +27,7 @@ 1008 ERR_CREATE_DATATYPE_NOT_SUPPORT The syntax or the data type(s) is not supported by Columnstore. Please check the Columnstore syntax guide for supported syntax or data types. 1010 ERR_AGGREGATE_TYPE_NOT_SUPPORT '%1%' for column type '%2%' isn't supported. 1011 ERR_DML_VIEW %1% on VIEW is currently not supported. -#1012 ERR_UPDATE_NOT_SUPPORT_FEATURE This version of Columstore supports update of only one table at a time. +1012 ERR_UPDATE_NOT_SUPPORT_FEATURE This version of Columnstore supports update of only one table at a time. 1014 ERR_ROLLUP_NOT_SUPPORT Rollup is currently not supported. 1015 ERR_OUTER_JOIN_SUBSELECT Subquery on OUTER JOIN ON clause is currently not supported. 1016 ERR_PARTITION_BY_RANGE The column type %1% is currently not supported in %2% function. From 9afd92f74f12cbac17a83a82af76bf6b327fc410 Mon Sep 17 00:00:00 2001 From: David Hall Date: Tue, 21 Mar 2017 16:27:09 -0500 Subject: [PATCH 28/61] MCOL-597 Fix up Windows Functions --- dbcon/execplan/execplan.vpj | 622 +++++++++++++------------- dbcon/execplan/windowfunctioncolumn.h | 2 +- dbcon/joblist/joblist.vpj | 2 + dbcon/mysql/ha_calpont_execplan.cpp | 21 +- dbcon/mysql/ha_calpont_impl.cpp | 2 +- dbcon/mysql/ha_window_function.cpp | 293 +++++++++--- dbcon/mysql/idb_mysql.h | 1 + utils/windowfunction/wf_lead_lag.cpp | 31 ++ 8 files changed, 578 insertions(+), 396 deletions(-) diff --git a/dbcon/execplan/execplan.vpj b/dbcon/execplan/execplan.vpj index 956fe3dac..6f964bc80 100644 --- a/dbcon/execplan/execplan.vpj +++ b/dbcon/execplan/execplan.vpj @@ -1,326 +1,302 @@ - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - + Version="10.0" + VendorName="SlickEdit" + TemplateName="GNU C/C++" + WorkingDir="."> + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + diff --git a/dbcon/execplan/windowfunctioncolumn.h b/dbcon/execplan/windowfunctioncolumn.h index d8b9fec75..477660b2f 100644 --- a/dbcon/execplan/windowfunctioncolumn.h +++ b/dbcon/execplan/windowfunctioncolumn.h @@ -41,7 +41,7 @@ class ByteStream; */ namespace execplan { -// This enum is made consistant with mysql item_func_window +// This enum is made consistant with mysql Item_window_func enum WF_FRAME { WF_PRECEDING = 0, diff --git a/dbcon/joblist/joblist.vpj b/dbcon/joblist/joblist.vpj index 20b1d6278..cbf9aa13a 100644 --- a/dbcon/joblist/joblist.vpj +++ b/dbcon/joblist/joblist.vpj @@ -248,6 +248,7 @@ + + #include #include @@ -38,7 +38,6 @@ #include #include #include -//#define NDEBUG #include #include #include @@ -699,13 +698,11 @@ void debug_walk(const Item *item, void *arg) cout << ": " << endl; break; } -#if 0 case Item::WINDOW_FUNC_ITEM: { cout << "Window Function Item" << endl; break; } -#endif default: { cout << "UNKNOWN_ITEM type " << item->type() << endl; @@ -2464,11 +2461,11 @@ ReturnedColumn* buildReturnedColumn(Item* item, gp_walk_info& gwi, bool& nonSupp rc = new ConstantColumn(valStr); break; } -#if 0 case Item::WINDOW_FUNC_ITEM: { return buildWindowFunctionColumn(item, gwi, nonSupport); } +#if INTERVAL_ITEM case Item::INTERVAL_ITEM: { Item_interval* interval = (Item_interval*)item; @@ -2481,7 +2478,7 @@ ReturnedColumn* buildReturnedColumn(Item* item, gp_walk_info& gwi, bool& nonSupp rc->resultType(srcp->resultType()); break; } -#endif +#endif case Item::SUBSELECT_ITEM: { gwi.hasSubSelect = true; @@ -4445,17 +4442,15 @@ void gp_walk(const Item *item, void *arg) gwip->rcWorkStack.push(buildReturnedColumn(itp, *gwip, gwip->fatalParseError)); break; } -#if 0 case Item::WINDOW_FUNC_ITEM: { gwip->hasWindowFunc = true; - Item_func_window* ifa = (Item_func_window*)item; + Item_window_func* ifa = (Item_window_func*)item; ReturnedColumn* af = buildWindowFunctionColumn(ifa, *gwip, gwip->fatalParseError); if (af) gwip->rcWorkStack.push(af); break; } -#endif case Item::COPY_STR_ITEM: printf("********** received COPY_STR_ITEM *********\n"); break; @@ -4639,11 +4634,9 @@ void parse_item (Item *item, vector& field_vec, bool& hasNonSupport setError(item->thd(), ER_CHECK_NOT_IMPLEMENTED, parseErrorText); break; } -#if 0 case Item::WINDOW_FUNC_ITEM: parseInfo |= AF_BIT; break; -#endif default: break; } @@ -5533,7 +5526,6 @@ int getSelectPlan(gp_walk_info& gwi, SELECT_LEX& select_lex, SCSEP& csep, bool i setError(gwi.thd, ER_CHECK_NOT_IMPLEMENTED, gwi.parseErrorText, gwi); return ER_CHECK_NOT_IMPLEMENTED; } -#if 0 case Item::WINDOW_FUNC_ITEM: { SRCP srcp(buildWindowFunctionColumn(item, gwi, gwi.fatalParseError)); @@ -5547,7 +5539,6 @@ int getSelectPlan(gp_walk_info& gwi, SELECT_LEX& select_lex, SCSEP& csep, bool i gwi.returnedCols.push_back(srcp); break; } -#endif default: { break; @@ -5700,7 +5691,6 @@ int getSelectPlan(gp_walk_info& gwi, SELECT_LEX& select_lex, SCSEP& csep, bool i // check if window functions are in order by. InfiniDB process order by list if // window functions are involved, either in order by or projection. -#if 0 bool hasWindowFunc = gwi.hasWindowFunc; gwi.hasWindowFunc = false; for (; groupcol; groupcol= groupcol->next) @@ -5716,7 +5706,6 @@ int getSelectPlan(gp_walk_info& gwi, SELECT_LEX& select_lex, SCSEP& csep, bool i return ER_CHECK_NOT_IMPLEMENTED; } gwi.hasWindowFunc = hasWindowFunc; -#endif groupcol = reinterpret_cast(select_lex.group_list.first); for (; groupcol; groupcol= groupcol->next) @@ -5962,13 +5951,11 @@ int getSelectPlan(gp_walk_info& gwi, SELECT_LEX& select_lex, SCSEP& csep, bool i // check if window functions are in order by. InfiniDB process order by list if // window functions are involved, either in order by or projection. -#if 0 for (; ordercol; ordercol= ordercol->next) { if ((*(ordercol->item))->type() == Item::WINDOW_FUNC_ITEM) gwi.hasWindowFunc = true; } -#endif // re-visit the first of ordercol list ordercol = reinterpret_cast(order_list.first); diff --git a/dbcon/mysql/ha_calpont_impl.cpp b/dbcon/mysql/ha_calpont_impl.cpp index 931048cec..cf8e86c57 100755 --- a/dbcon/mysql/ha_calpont_impl.cpp +++ b/dbcon/mysql/ha_calpont_impl.cpp @@ -3007,7 +3007,7 @@ int ha_calpont_impl_rnd_next(uchar *buf, TABLE* table) rc = fetchNextRow(buf, ti, ci); } catch (std::exception& e) { - string emsg = string("Lost connection to ExeMgr while fetching: ") + e.what(); + string emsg = string("Error while fetching from ExeMgr: ") + e.what(); setError(thd, ER_INTERNAL_ERROR, emsg); CalpontSystemCatalog::removeCalpontSystemCatalog(tid2sid(thd->thread_id)); return ER_INTERNAL_ERROR; diff --git a/dbcon/mysql/ha_window_function.cpp b/dbcon/mysql/ha_window_function.cpp index 8af20511e..965d14824 100755 --- a/dbcon/mysql/ha_window_function.cpp +++ b/dbcon/mysql/ha_window_function.cpp @@ -63,26 +63,26 @@ ReturnedColumn* nullOnError(gp_walk_info& gwi) return NULL; } -#if 0 -WF_FRAME frame(BOUND& bound) +WF_FRAME frame(Window_frame_bound::Bound_precedence_type bound, Item* offset) { switch (bound) { - case PRECEDING: - return WF_PRECEDING; - case FOLLOWING: - return WF_FOLLOWING; - case UNBOUNDED_PRECEDING: - return WF_UNBOUNDED_PRECEDING; - case UNBOUNDED_FOLLOWING: - return WF_UNBOUNDED_FOLLOWING; - case CURRENT_ROW: + case Window_frame_bound::PRECEDING: + if (offset) + return WF_PRECEDING; + else + return WF_UNBOUNDED_PRECEDING; + case Window_frame_bound::FOLLOWING: + if (offset) + return WF_FOLLOWING; + else + return WF_UNBOUNDED_FOLLOWING; + case Window_frame_bound::CURRENT: // Offset is meaningless return WF_CURRENT_ROW; default: return WF_UNKNOWN; } } -#endif ReturnedColumn* buildBoundExp(WF_Boundary& bound, SRCP& order, gp_walk_info& gwi) { if (!(gwi.thd->infinidb_vtable.cal_conn_info)) @@ -178,6 +178,96 @@ ReturnedColumn* buildBoundExp(WF_Boundary& bound, SRCP& order, gp_walk_info& gwi return rc; } +// Since columnstore implemented Windows Functions before MariaDB, we need +// map from the enum MariaDB uses to the string that columnstore uses to +// identify the function type. +string ConvertFuncName(Item_sum* item) +{ + switch (item->sum_func()) + { + case Item_sum::COUNT_FUNC: + if (!item->arguments()[0]->name) + return "COUNT(*)"; + return "COUNT"; + break; + case Item_sum::COUNT_DISTINCT_FUNC: + return "COUNT_DISTINCT"; + break; + case Item_sum::SUM_FUNC: + return "SUM"; + break; + case Item_sum::SUM_DISTINCT_FUNC: + return "SUM_DISTINCT"; + break; + case Item_sum::AVG_FUNC: + return "AVG"; + break; + case Item_sum::AVG_DISTINCT_FUNC: + return "AVG_DISTINCT"; + break; + case Item_sum::MIN_FUNC: + return "MIN"; + break; + case Item_sum::MAX_FUNC: + return "MAX"; + break; + case Item_sum::STD_FUNC: + return "STDDEV_POP"; + break; + case Item_sum::VARIANCE_FUNC: + return "VAR_POP"; + break; + case Item_sum::SUM_BIT_FUNC: + if (strcmp(item->func_name(), "bit_or(") == 0) + return "BIT_OR"; + if (strcmp(item->func_name(), "bit_and(") == 0) + return "BIT_AND"; + if (strcmp(item->func_name(), "bit_xor(") == 0) + return "BIT_XOR"; + break; + case Item_sum::UDF_SUM_FUNC: + return "UDF_SUM_FUNC"; // Not supported + break; + case Item_sum::GROUP_CONCAT_FUNC: + return "GROUP_CONCAT"; // Not supported + break; + case Item_sum::ROW_NUMBER_FUNC: + return "ROW_NUMBER"; + break; + case Item_sum::RANK_FUNC: + return "RANK"; + break; + case Item_sum::DENSE_RANK_FUNC: + return "DENSE_RANK"; + break; + case Item_sum::PERCENT_RANK_FUNC: + return "PERCENT_RANK"; + break; + case Item_sum::CUME_DIST_FUNC: + return "CUME_DIST"; + break; + case Item_sum::NTILE_FUNC: + return "NTILE"; + break; + case Item_sum::FIRST_VALUE_FUNC: + return "FIRST_VALUE"; + break; + case Item_sum::LAST_VALUE_FUNC: + return "LAST_VALUE"; + break; + case Item_sum::NTH_VALUE_FUNC: + return "NTH_VALUE"; + break; + case Item_sum::LEAD_FUNC: + return "LEAD"; + break; + case Item_sum::LAG_FUNC: + return "LAG"; + break; + }; + return ""; +} + ReturnedColumn* buildWindowFunctionColumn(Item* item, gp_walk_info& gwi, bool& nonSupport) { //@todo fix print for create view @@ -186,37 +276,82 @@ ReturnedColumn* buildWindowFunctionColumn(Item* item, gp_walk_info& gwi, bool& n //cout << str.c_ptr() << endl; if (!(gwi.thd->infinidb_vtable.cal_conn_info)) gwi.thd->infinidb_vtable.cal_conn_info = (void*)(new cal_connection_info()); -// cal_connection_info* ci = reinterpret_cast(gwi.thd->infinidb_vtable.cal_conn_info); + cal_connection_info* ci = reinterpret_cast(gwi.thd->infinidb_vtable.cal_conn_info); gwi.hasWindowFunc = true; -// Item_func_window* wf = (Item_func_window*)item; - string funcName /*= wf->func_name()*/; + Item_window_func* wf = (Item_window_func*)item; + string funcName = ConvertFuncName(wf->window_func()); WindowFunctionColumn* ac = new WindowFunctionColumn(funcName); -// ac->distinct(wf->isDistinct()); -// Window_context *wf_ctx = wf->window_ctx(); + ac->distinct(wf->window_func()->has_with_distinct()); + Window_spec *win_spec = wf->window_spec; SRCP srcp; -#if 0 + // arguments + vector funcParms; + Item_sum* item_sum = (Item_sum*)wf->arguments()[0]; + for (uint32_t i = 0; i < item_sum->argument_count(); i++) + { + srcp.reset(buildReturnedColumn((item_sum->arguments()[i]), gwi, nonSupport)); + if (!srcp) + return nullOnError(gwi); + funcParms.push_back(srcp); + if (gwi.clauseType == WHERE && !gwi.rcWorkStack.empty()) + gwi.rcWorkStack.pop(); + } + // Some functions, such as LEAD/LAG don't have all parameters implemented in the + // front end. Add dummies here to make the backend use defaults. + // Some of these will be temporary until they are implemented in the front end. + // Others need to stay because the back end expects them, but the front end + // no longer sends them. + // This case is kept in enum order in hopes the compiler can optimize + switch (wf->window_func()->sum_func()) + { + case Item_sum::COUNT_FUNC: + case Item_sum::COUNT_DISTINCT_FUNC: + break; + case Item_sum::FIRST_VALUE_FUNC: + srcp.reset(new ConstantColumn("1", (uint64_t)1, ConstantColumn::NUM)); // OFFSET (always one) + funcParms.push_back(srcp); + srcp.reset(new ConstantColumn("1", (uint64_t)1, ConstantColumn::NUM)); // FROM_FIRST + funcParms.push_back(srcp); + srcp.reset(new ConstantColumn("1", (uint64_t)1, ConstantColumn::NUM)); // IGNORE/RESPECT NULLS. 1 => RESPECT + funcParms.push_back(srcp); + break; + case Item_sum::LAST_VALUE_FUNC: + srcp.reset(new ConstantColumn("1", (uint64_t)1, ConstantColumn::NUM)); // OFFSET (always one) + funcParms.push_back(srcp); + srcp.reset(new ConstantColumn("0", (uint64_t)0, ConstantColumn::NUM)); // FROM_LAST + funcParms.push_back(srcp); + srcp.reset(new ConstantColumn("1", (uint64_t)1, ConstantColumn::NUM)); // IGNORE/RESPECT NULLS. 1 => RESPECT + funcParms.push_back(srcp); + break; + case Item_sum::NTH_VALUE_FUNC: + // When the front end supports these paramters, this needs modification + srcp.reset(new ConstantColumn("1", (uint64_t)1, ConstantColumn::NUM)); // FROM FIRST/LAST 1 => FIRST + funcParms.push_back(srcp); + srcp.reset(new ConstantColumn("1", (uint64_t)1, ConstantColumn::NUM)); // IGNORE/RESPECT NULLS. 1 => RESPECT + funcParms.push_back(srcp); + break; + case Item_sum::LEAD_FUNC: + case Item_sum::LAG_FUNC: + // When the front end supports these paramters, this needs modification + srcp.reset(new ConstantColumn("", ConstantColumn::NULLDATA)); // Default to fill in for NULL values + funcParms.push_back(srcp); + srcp.reset(new ConstantColumn("1", (uint64_t)1, ConstantColumn::NUM)); // IGNORE/RESPECT NULLS. 1 => RESPECT + funcParms.push_back(srcp); + break; + default: + break; + }; - // arguments - vector funcParms; - for (uint32_t i = 0; i < wf->argument_count(); i++) - { - srcp.reset(buildReturnedColumn(wf->arguments()[i], gwi, nonSupport)); - if (!srcp) - return nullOnError(gwi); - funcParms.push_back(srcp); - if (gwi.clauseType == WHERE && !gwi.rcWorkStack.empty()) - gwi.rcWorkStack.pop(); - } - ac->functionParms(funcParms); + ac->functionParms(funcParms); // Partition by - if (wf_ctx) + if (win_spec) { vector partitions; - for (uint32_t i = 0; i < wf_ctx->partition_count; i++) + for (ORDER *ord= win_spec->partition_list->first; ord; ord=ord->next) { - srcp.reset(buildReturnedColumn(wf_ctx->partitions[i], gwi, nonSupport)); + srcp.reset(buildReturnedColumn(*ord->item, gwi, nonSupport)); if (!srcp) return nullOnError(gwi); partitions.push_back(srcp); @@ -224,40 +359,42 @@ ReturnedColumn* buildWindowFunctionColumn(Item* item, gp_walk_info& gwi, bool& n ac->partitions(partitions); // Order by - if (wf_ctx->ordering) + if (win_spec->order_list) { WF_OrderBy orderBy; // order columns - if (wf_ctx->ordering->orders) + if (win_spec->order_list) { vector orders; - ORDER* orderCol = reinterpret_cast(wf_ctx->ordering->orders->first); + ORDER* orderCol = reinterpret_cast(win_spec->order_list->first); for (; orderCol; orderCol= orderCol->next) { Item* orderItem = *(orderCol->item); srcp.reset(buildReturnedColumn(orderItem, gwi, nonSupport)); if (!srcp) return nullOnError(gwi); - srcp->asc(orderCol->asc); - srcp->nullsFirst(orderCol->nulls); // nulls 1-nulls first 0-nulls last - orders.push_back(srcp); + srcp->asc(orderCol->direction == ORDER::ORDER_ASC ? true : false); +// srcp->nullsFirst(orderCol->nulls); // nulls 2-default, 1-nulls first, 0-nulls last + srcp->nullsFirst(1); // WINDOWS TODO: implement NULLS FIRST/LAST in 10.2 front end + orders.push_back(srcp); } orderBy.fOrders = orders; } // window frame WF_Frame frm; - if (wf_ctx->ordering->frame) + if (win_spec->window_frame) { - frm.fIsRange = wf_ctx->ordering->frame->isRange; + frm.fIsRange = win_spec->window_frame->units == Window_frame::UNITS_RANGE; // start - if (wf_ctx->ordering->frame->start) + if (win_spec->window_frame->top_bound) { - frm.fStart.fFrame = frame(wf_ctx->ordering->frame->start->bound); + frm.fStart.fFrame = frame(win_spec->window_frame->top_bound->precedence_type, + win_spec->window_frame->top_bound->offset); // offset NULL means UNBOUNDED - if (wf_ctx->ordering->frame->start->item) + if (win_spec->window_frame->top_bound->offset) { - frm.fStart.fVal.reset(buildReturnedColumn(wf_ctx->ordering->frame->start->item, gwi, nonSupport)); + frm.fStart.fVal.reset(buildReturnedColumn(win_spec->window_frame->top_bound->offset, gwi, nonSupport)); if (!frm.fStart.fVal) return nullOnError(gwi); @@ -294,12 +431,13 @@ ReturnedColumn* buildWindowFunctionColumn(Item* item, gp_walk_info& gwi, bool& n } // end - if (wf_ctx->ordering->frame->end) + if (win_spec->window_frame->bottom_bound) { - frm.fEnd.fFrame = frame(wf_ctx->ordering->frame->end->bound); - if (wf_ctx->ordering->frame->end->item) + frm.fEnd.fFrame = frame(win_spec->window_frame->bottom_bound->precedence_type, + win_spec->window_frame->bottom_bound->offset); + if (win_spec->window_frame->bottom_bound->offset) { - frm.fEnd.fVal.reset(buildReturnedColumn(wf_ctx->ordering->frame->end->item, gwi, nonSupport)); + frm.fEnd.fVal.reset(buildReturnedColumn(win_spec->window_frame->bottom_bound->offset, gwi, nonSupport)); if (!frm.fEnd.fVal) return nullOnError(gwi); @@ -406,8 +544,57 @@ ReturnedColumn* buildWindowFunctionColumn(Item* item, gp_walk_info& gwi, bool& n } else { - frm.fStart.fFrame = WF_UNBOUNDED_PRECEDING; - frm.fEnd.fFrame = WF_CURRENT_ROW; + // Certain function types have different default boundaries + // This case is kept in enum order in hopes the compiler can optimize + switch (wf->window_func()->sum_func()) + { + case Item_sum::COUNT_FUNC: + case Item_sum::COUNT_DISTINCT_FUNC: + case Item_sum::SUM_FUNC: + case Item_sum::SUM_DISTINCT_FUNC: + case Item_sum::AVG_FUNC: + case Item_sum::AVG_DISTINCT_FUNC: + case Item_sum::MIN_FUNC: + case Item_sum::MAX_FUNC: + case Item_sum::STD_FUNC: + case Item_sum::VARIANCE_FUNC: + case Item_sum::SUM_BIT_FUNC: + case Item_sum::UDF_SUM_FUNC: + case Item_sum::GROUP_CONCAT_FUNC: + frm.fStart.fFrame = WF_UNBOUNDED_PRECEDING; + frm.fEnd.fFrame = WF_CURRENT_ROW; + break; + case Item_sum::ROW_NUMBER_FUNC: + case Item_sum::RANK_FUNC: + frm.fStart.fFrame = WF_UNBOUNDED_PRECEDING; + frm.fEnd.fFrame = WF_UNBOUNDED_FOLLOWING; + break; + case Item_sum::DENSE_RANK_FUNC: + case Item_sum::PERCENT_RANK_FUNC: + case Item_sum::CUME_DIST_FUNC: + frm.fStart.fFrame = WF_UNBOUNDED_PRECEDING; + frm.fEnd.fFrame = WF_CURRENT_ROW; + break; + case Item_sum::NTILE_FUNC: + frm.fStart.fFrame = WF_UNBOUNDED_PRECEDING; + frm.fEnd.fFrame = WF_UNBOUNDED_FOLLOWING; + break; + case Item_sum::FIRST_VALUE_FUNC: + case Item_sum::LAST_VALUE_FUNC: + case Item_sum::NTH_VALUE_FUNC: + frm.fStart.fFrame = WF_UNBOUNDED_PRECEDING; + frm.fEnd.fFrame = WF_CURRENT_ROW; + break; + case Item_sum::LEAD_FUNC: + case Item_sum::LAG_FUNC: + frm.fStart.fFrame = WF_UNBOUNDED_PRECEDING; + frm.fEnd.fFrame = WF_UNBOUNDED_FOLLOWING; + break; + default: + frm.fStart.fFrame = WF_UNBOUNDED_PRECEDING; + frm.fEnd.fFrame = WF_CURRENT_ROW; + break; + }; } orderBy.fFrame = frm; @@ -423,8 +610,7 @@ ReturnedColumn* buildWindowFunctionColumn(Item* item, gp_walk_info& gwi, bool& n return NULL; } - ac->resultType(colType_MysqlToIDB(wf)); - + ac->resultType(colType_MysqlToIDB(wf->arguments()[0])); // bug5736. Make the result type double for some window functions when // infinidb_double_for_decimal_math is set. ac->adjustResultType(); @@ -435,7 +621,6 @@ ReturnedColumn* buildWindowFunctionColumn(Item* item, gp_walk_info& gwi, bool& n // put ac on windowFuncList gwi.windowFuncList.push_back(ac); -#endif return ac; } diff --git a/dbcon/mysql/idb_mysql.h b/dbcon/mysql/idb_mysql.h index f8eb5d5c0..670f0e0e0 100644 --- a/dbcon/mysql/idb_mysql.h +++ b/dbcon/mysql/idb_mysql.h @@ -65,6 +65,7 @@ template bool isnan(T); #include "sql_table.h" #include "sql_select.h" +#include "item_windowfunc.h" // Now clean up the pollution as best we can... #undef min diff --git a/utils/windowfunction/wf_lead_lag.cpp b/utils/windowfunction/wf_lead_lag.cpp index 1ab9818f3..8d3b6169b 100644 --- a/utils/windowfunction/wf_lead_lag.cpp +++ b/utils/windowfunction/wf_lead_lag.cpp @@ -121,6 +121,11 @@ void WF_lead_lag::parseParms(const std::vector& parms) { // lead | lag fLead = 1; + fRespectNulls = true; + fDefNull = false; + fDefault = (T)0; + fOffsetNull = false; + fOffset = 0; if (fFunctionId == WF__LAG) fLead = -1; @@ -148,6 +153,32 @@ void WF_lead_lag::parseParms(const std::vector& parms) idbassert(cc != NULL); bool isNull = false; // dummy, harded coded fRespectNulls = (cc->getIntVal(fRow, isNull) > 0); +#if 0 + // parms[1]: offset + for (std::vector::size_type i = 1/*ignore 0th element*/; i < parms.size(); ++i) + { + ConstantColumn* cc = dynamic_cast(parms[i].get()); + if (cc != NULL) + { + fOffset = cc->getIntVal(fRow, fOffsetNull) * fLead; // row not used, no need to setData. + continue; + } + + cc = dynamic_cast(parms[i].get()); + if (cc != NULL) + { + getConstValue(cc, fDefault, fDefNull); + continue; + } + // IGNORE/RESPECT nulls is currently broken in the front end + cc = dynamic_cast(parms[i].get()); + if (cc != NULL) + { + bool isNull = false; // dummy, harded coded + fRespectNulls = (cc->getIntVal(fRow, isNull) > 0); + } + } +#endif } From 5b334140e73ad190827eef568c454c1a19cae165 Mon Sep 17 00:00:00 2001 From: David Hall Date: Tue, 28 Mar 2017 13:03:28 -0500 Subject: [PATCH 29/61] MCOL-597 Fix up Windows Functions --- dbcon/joblist/CMakeLists.txt | 2 +- dbcon/mysql/ha_calpont_execplan.cpp | 48 ++++++++++++++++++------- dbcon/mysql/ha_window_function.cpp | 27 ++++++++------ oam/install_scripts/post-mysqld-install | 10 +++--- utils/querystats/CMakeLists.txt | 2 -- versioning/BRM/CMakeLists.txt | 8 ++--- 6 files changed, 61 insertions(+), 36 deletions(-) diff --git a/dbcon/joblist/CMakeLists.txt b/dbcon/joblist/CMakeLists.txt index 7fac3a859..0a7880413 100644 --- a/dbcon/joblist/CMakeLists.txt +++ b/dbcon/joblist/CMakeLists.txt @@ -61,7 +61,7 @@ set(joblist_LIB_SRCS add_library(joblist SHARED ${joblist_LIB_SRCS}) -target_link_libraries(joblist ${NETSNMP_LIBRARIES} -L${SERVER_BUILD_INCLUDE_DIR}/../libmysql/ libmysqlclient_r.so) +target_link_libraries(joblist -L${SERVER_BUILD_INCLUDE_DIR}/../libmariadb/libmariadb/ libmariadb.so) set_target_properties(joblist PROPERTIES VERSION 1.0.0 SOVERSION 1) diff --git a/dbcon/mysql/ha_calpont_execplan.cpp b/dbcon/mysql/ha_calpont_execplan.cpp index e4cfe46ce..f0aa37176 100755 --- a/dbcon/mysql/ha_calpont_execplan.cpp +++ b/dbcon/mysql/ha_calpont_execplan.cpp @@ -496,6 +496,11 @@ void debug_walk(const Item *item, void *arg) '.' << ifp->field_name << endl; break; } + else if (field->type() == Item::FUNC_ITEM) + { + Item_func* ifp = (Item_func*)field; + cout << "CACHED REF FUNC_ITEM " << ifp->func_name() << endl; + } else if (field->type() == Item::REF_ITEM) { Item_ref* ifr = (Item_ref*)field; @@ -563,7 +568,20 @@ void debug_walk(const Item *item, void *arg) ifp->field_name << endl; break; } - cout << "UNKNOWN REF ITEM type " << ref->real_item()->type() << endl; + else if (ref->real_item()->type() == Item::FUNC_ITEM) + { + Item_func* ifp = (Item_func*)ref->real_item(); + cout << "REF FUNC_ITEM " << ifp->func_name() << endl; + } + else if (ref->real_item()->type() == Item::WINDOW_FUNC_ITEM) + { + Item_window_func* ifp = (Item_window_func*)ref->real_item(); + cout << "REF WINDOW_FUNC_ITEM " << ifp->window_func()->func_name() << endl; + } + else + { + cout << "UNKNOWN REF ITEM type " << ref->real_item()->type() << endl; + } break; } case Item::ROW_ITEM: @@ -700,7 +718,8 @@ void debug_walk(const Item *item, void *arg) } case Item::WINDOW_FUNC_ITEM: { - cout << "Window Function Item" << endl; + Item_window_func* ifp = (Item_window_func*)item; + cout << "Window Function Item " << ifp->window_func()->func_name() << endl; break; } default: @@ -2407,18 +2426,19 @@ ReturnedColumn* buildReturnedColumn(Item* item, gp_walk_info& gwi, bool& nonSupp case Item::REF_ITEM: { Item_ref* ref = (Item_ref*)item; - if ((*(ref->ref))->type() == Item::SUM_FUNC_ITEM) - { + switch ((*(ref->ref))->type()) + { + case Item::SUM_FUNC_ITEM: return buildAggregateColumn(*(ref->ref), gwi); - } - else if ((*(ref->ref))->type() == Item::FIELD_ITEM) + case Item::FIELD_ITEM: return buildReturnedColumn(*(ref->ref), gwi, nonSupport); - else if ((*(ref->ref))->type() == Item::REF_ITEM) + case Item::REF_ITEM: return buildReturnedColumn(*(((Item_ref*)(*(ref->ref)))->ref), gwi, nonSupport); - else if ((*(ref->ref))->type() == Item::FUNC_ITEM) + case Item::FUNC_ITEM: return buildFunctionColumn((Item_func*)(*(ref->ref)), gwi, nonSupport); - else - { + case Item::WINDOW_FUNC_ITEM: + return buildWindowFunctionColumn(*(ref->ref), gwi, nonSupport); + default: gwi.fatalParseError = true; gwi.parseErrorText = "Unknown REF item"; break; @@ -4398,8 +4418,6 @@ void gp_walk(const Item *item, void *arg) gwip->subQuery = orig; gwip->lastSub = existsSub; } - -#if MYSQL_VERSION_ID >= 50172 else if (sub->substype() == Item_subselect::IN_SUBS) { if (!((Item_in_subselect*)sub)->getOptimizer() && gwip->thd->derived_tables_processing) @@ -4411,7 +4429,6 @@ void gp_walk(const Item *item, void *arg) break; } } -#endif // store a dummy subselect object. the transform is handled in item_func. SubSelect *subselect = new SubSelect(); gwip->rcWorkStack.push(subselect); @@ -4603,6 +4620,11 @@ void parse_item (Item *item, vector& field_vec, bool& hasNonSupport item = (*(ref->ref)); continue; } + else if ((*(ref->ref))->type() == Item::WINDOW_FUNC_ITEM) + { + parseInfo |= AF_BIT; + break; + } else { cout << "UNKNOWN REF Item" << endl; diff --git a/dbcon/mysql/ha_window_function.cpp b/dbcon/mysql/ha_window_function.cpp index 965d14824..9867278cb 100755 --- a/dbcon/mysql/ha_window_function.cpp +++ b/dbcon/mysql/ha_window_function.cpp @@ -212,10 +212,16 @@ string ConvertFuncName(Item_sum* item) return "MAX"; break; case Item_sum::STD_FUNC: - return "STDDEV_POP"; + if (((Item_sum_variance*)item)->sample) + return "STDDEV_SAMP"; + else + return "STDDEV_POP"; break; - case Item_sum::VARIANCE_FUNC: - return "VAR_POP"; + case Item_sum::VARIANCE_FUNC: + if (((Item_sum_variance*)item)->sample) + return "VAR_SAMP"; + else + return "VAR_POP"; break; case Item_sum::SUM_BIT_FUNC: if (strcmp(item->func_name(), "bit_or(") == 0) @@ -375,7 +381,7 @@ ReturnedColumn* buildWindowFunctionColumn(Item* item, gp_walk_info& gwi, bool& n return nullOnError(gwi); srcp->asc(orderCol->direction == ORDER::ORDER_ASC ? true : false); // srcp->nullsFirst(orderCol->nulls); // nulls 2-default, 1-nulls first, 0-nulls last - srcp->nullsFirst(1); // WINDOWS TODO: implement NULLS FIRST/LAST in 10.2 front end + srcp->nullsFirst(orderCol->direction == ORDER::ORDER_ASC ? 1 : 0); // WINDOWS TODO: implement NULLS FIRST/LAST in 10.2 front end orders.push_back(srcp); } orderBy.fOrders = orders; @@ -554,8 +560,15 @@ ReturnedColumn* buildWindowFunctionColumn(Item* item, gp_walk_info& gwi, bool& n case Item_sum::SUM_DISTINCT_FUNC: case Item_sum::AVG_FUNC: case Item_sum::AVG_DISTINCT_FUNC: + frm.fStart.fFrame = WF_UNBOUNDED_PRECEDING; + frm.fEnd.fFrame = WF_CURRENT_ROW; + break; case Item_sum::MIN_FUNC: case Item_sum::MAX_FUNC: + frm.fStart.fFrame = WF_UNBOUNDED_PRECEDING; +// frm.fEnd.fFrame = WF_UNBOUNDED_FOLLOWING; + frm.fEnd.fFrame = WF_CURRENT_ROW; + break; case Item_sum::STD_FUNC: case Item_sum::VARIANCE_FUNC: case Item_sum::SUM_BIT_FUNC: @@ -566,15 +579,9 @@ ReturnedColumn* buildWindowFunctionColumn(Item* item, gp_walk_info& gwi, bool& n break; case Item_sum::ROW_NUMBER_FUNC: case Item_sum::RANK_FUNC: - frm.fStart.fFrame = WF_UNBOUNDED_PRECEDING; - frm.fEnd.fFrame = WF_UNBOUNDED_FOLLOWING; - break; case Item_sum::DENSE_RANK_FUNC: case Item_sum::PERCENT_RANK_FUNC: case Item_sum::CUME_DIST_FUNC: - frm.fStart.fFrame = WF_UNBOUNDED_PRECEDING; - frm.fEnd.fFrame = WF_CURRENT_ROW; - break; case Item_sum::NTILE_FUNC: frm.fStart.fFrame = WF_UNBOUNDED_PRECEDING; frm.fEnd.fFrame = WF_UNBOUNDED_FOLLOWING; diff --git a/oam/install_scripts/post-mysqld-install b/oam/install_scripts/post-mysqld-install index 546945b22..7fa634c54 100755 --- a/oam/install_scripts/post-mysqld-install +++ b/oam/install_scripts/post-mysqld-install @@ -28,11 +28,11 @@ for arg in "$@"; do fi done -cd $installdir/mysql/lib -for file in libmysqlclient libmysqlclient_r; do - ln -sf $file.so.18.0.0 $file.so - ln -sf $file.so.18.0.0 $file.so.18 -done +#cd $installdir/mysql/lib +#for file in libmysqlclient libmysqlclient_r; do +# ln -sf $file.so.18.0.0 $file.so +# ln -sf $file.so.18.0.0 $file.so.18 +#done cd $installdir/mysql/lib/plugin for file in ha_archive ha_blackhole ha_federated ha_innodb; do ln -sf $file.so $file.so.0 diff --git a/utils/querystats/CMakeLists.txt b/utils/querystats/CMakeLists.txt index 935898818..940749a6b 100644 --- a/utils/querystats/CMakeLists.txt +++ b/utils/querystats/CMakeLists.txt @@ -8,8 +8,6 @@ set(querystats_LIB_SRCS querystats.cpp) add_library(querystats SHARED ${querystats_LIB_SRCS}) -target_link_libraries(querystats -L${SERVER_BUILD_INCLUDE_DIR}/../libmysql/ libmysqlclient_r.so) - set_target_properties(querystats PROPERTIES VERSION 1.0.0 SOVERSION 1) install(TARGETS querystats DESTINATION ${ENGINE_LIBDIR} COMPONENT libs) diff --git a/versioning/BRM/CMakeLists.txt b/versioning/BRM/CMakeLists.txt index 2ce60254d..de392d6b1 100644 --- a/versioning/BRM/CMakeLists.txt +++ b/versioning/BRM/CMakeLists.txt @@ -32,8 +32,6 @@ set(brm_LIB_SRCS add_library(brm SHARED ${brm_LIB_SRCS}) -target_link_libraries(brm ${NETSNMP_LIBRARIES}) - set_target_properties(brm PROPERTIES VERSION 1.0.0 SOVERSION 1) install(TARGETS brm DESTINATION ${ENGINE_LIBDIR} COMPONENT libs) @@ -45,7 +43,7 @@ set(controllernode_SRCS masternode.cpp masterdbrmnode.cpp) add_executable(controllernode ${controllernode_SRCS}) -target_link_libraries(controllernode ${ENGINE_LDFLAGS} ${ENGINE_OAM_LIBS} ${ENGINE_EXEC_LIBS} ${NETSNMP_LIBRARIES}) +target_link_libraries(controllernode ${ENGINE_LDFLAGS} ${ENGINE_OAM_LIBS} ${ENGINE_EXEC_LIBS} ) install(TARGETS controllernode DESTINATION ${ENGINE_BINDIR} COMPONENT platform) @@ -56,7 +54,7 @@ set(workernode_SRCS slavenode.cpp) add_executable(workernode ${workernode_SRCS}) -target_link_libraries(workernode ${ENGINE_LDFLAGS} ${ENGINE_OAM_LIBS} ${ENGINE_EXEC_LIBS} ${NETSNMP_LIBRARIES}) +target_link_libraries(workernode ${ENGINE_LDFLAGS} ${ENGINE_OAM_LIBS} ${ENGINE_EXEC_LIBS}) install(TARGETS workernode DESTINATION ${ENGINE_BINDIR} COMPONENT platform) @@ -67,7 +65,7 @@ set(dbrmctl_SRCS dbrmctl.cpp) add_executable(dbrmctl ${dbrmctl_SRCS}) -target_link_libraries(dbrmctl ${ENGINE_LDFLAGS} ${ENGINE_OAM_LIBS} ${ENGINE_EXEC_LIBS} ${NETSNMP_LIBRARIES}) +target_link_libraries(dbrmctl ${ENGINE_LDFLAGS} ${ENGINE_OAM_LIBS} ${ENGINE_EXEC_LIBS}) install(TARGETS dbrmctl DESTINATION ${ENGINE_BINDIR} COMPONENT platform) From cbc90827af86283dcbac5f9ca74dc802858cdf58 Mon Sep 17 00:00:00 2001 From: david hill Date: Tue, 23 Aug 2016 15:59:13 -0500 Subject: [PATCH 30/61] change readme --- README.md | 16 +++++++++------- 1 file changed, 9 insertions(+), 7 deletions(-) diff --git a/README.md b/README.md index 577ac33c1..5df193e89 100644 --- a/README.md +++ b/README.md @@ -1,16 +1,18 @@ -#MariaDB ColumnStore Storage/Execution engine 1.1.0 -MariaDB ColumnStore 1.1.0 is the development version of MariaDB ColumnStore. -It is built by porting InfiniDB 4.6.7 on MariaDB 10.1.21 and adding entirely +#MariaDB ColumnStore Storage/Execution engine 1.0.1 +MariaDB ColumnStore 1.0.2 is the development version of MariaDB ColumnStore. +It is built by porting InfiniDB 4.6.2 on MariaDB 10.1.14 and adding entirely new features not found anywhere else. -#MariaDB ColumnStore 1.1.0 is an development release. +#MariaDB ColumnStore 1.0.2 is an Alpha release. +This is the first MariaDB ColumnStore release, not all features planned for the MariaDB ColumnStore 1.0 +series are included in this release. -- Do not use development releases on production systems. +Additional features will be pushed in future releases. +A few things to notice: +- Do not use alpha releases on production systems. #Building This repository is not meant to be built independently outside of the server. This repository is integrated into http://mariadb-corporation/mariadb-columnstore-server (ie, the *server*) as a git submodule. As such, you can find complete build instructions on *the server* page. https://github.com/mariadb-corporation/mariadb-columnstore-server -#Issue tracking -Issue tracking of MariaDB ColumnStore happens in JIRA, https://jira.mariadb.org/browse/MCOL From c5a46404f4af26d2091204c9960ad66f3f307e0c Mon Sep 17 00:00:00 2001 From: david hill Date: Wed, 18 Jan 2017 14:32:54 -0600 Subject: [PATCH 31/61] mcol-481 - just changed the note --- oamapps/postConfigure/postConfigure.cpp | 203 ++++++++---------------- 1 file changed, 69 insertions(+), 134 deletions(-) diff --git a/oamapps/postConfigure/postConfigure.cpp b/oamapps/postConfigure/postConfigure.cpp index 92b848fcd..f8e66311a 100644 --- a/oamapps/postConfigure/postConfigure.cpp +++ b/oamapps/postConfigure/postConfigure.cpp @@ -52,16 +52,6 @@ #include #include -#include -#include -#include -#include -#include /* for strncpy */ -#include -#include -#include -#include - #include #include #include "boost/filesystem/operations.hpp" @@ -433,61 +423,6 @@ int main(int argc, char *argv[]) exit(1); } - //check for local ip address as pm1 - ModuleConfig moduleconfig; - - try - { - oam.getSystemConfig("pm1", moduleconfig); - if (moduleconfig.hostConfigList.size() > 0 ) - { - HostConfigList::iterator pt1 = moduleconfig.hostConfigList.begin(); - string PM1ipAdd = (*pt1).IPAddr; - cout << PM1ipAdd << endl; - - if ( PM1ipAdd != "127.0.0.1" && PM1ipAdd != "0.0.0.0") - { - struct ifaddrs *ifap, *ifa; - struct sockaddr_in *sa; - char *addr; - bool found = false; - - getifaddrs (&ifap); - for (ifa = ifap; ifa; ifa = ifa->ifa_next) { - if (ifa->ifa_addr->sa_family==AF_INET) { - sa = (struct sockaddr_in *) ifa->ifa_addr; - addr = inet_ntoa(sa->sin_addr); - printf("Interface: %s\tAddress: %s\n", ifa->ifa_name, addr); - - if ( PM1ipAdd == addr ) - { - //match - found = true; - } - } - - if (found) - break; - } - - freeifaddrs(ifap); - - if (!found) - { - cout << endl; - cout << "ERROR: postConfigure install can only be done on the PM1" << endl; - cout << "designated node. The configured PM1 IP address doesn't match the local" << endl; - cout << "IP Address. exiting..." << endl; - exit(1); - } - } - } - } - catch(...) - {} - - - // run my.cnf upgrade script if ( reuseConfig == "y" ) { @@ -731,6 +666,75 @@ int main(int argc, char *argv[]) exit(1); } + cout << "NOTE: The MariaDB ColumnStore Schema Sync feature will replicate all of the" << endl; + cout << " schemas and InnoDB tables across the User Module nodes. This feature can be enabled" << endl; + cout << " or disabled, for example, if you wish to configure your own replication post installation." << endl << endl; + + try { + MySQLRep = sysConfig->getConfig(InstallSection, "MySQLRep"); + } + catch(...) + {} + + if ( MySQLRep == "y" ) + mysqlRep = true; + + string answer = "y"; + + while(true) { + if ( mysqlRep ) + prompt = "MariaDB ColumnStore Schema Sync feature is Enabled, do you want to leave enabled? [y,n] (y) > "; + else + prompt = "MariaDB ColumnStore Schema Sync feature, do you want to enable? [y,n] (y) > "; + + pcommand = callReadline(prompt.c_str()); + if (pcommand) { + if (strlen(pcommand) > 0) answer = pcommand; + callFree(pcommand); + } + + if ( answer == "y" || answer == "n" ) { + cout << endl; + break; + } + else + cout << "Invalid Entry, please enter 'y' for yes or 'n' for no" << endl; + if ( noPrompting ) + exit(1); + } + + if ( mysqlRep ) + { // current enabled + if ( answer == "y" ) { + mysqlRep = true; + MySQLRep = "y"; + } + else + { + mysqlRep = false; + MySQLRep = "n"; + } + } + else + { // currently disabled + if ( answer == "y" ) { + mysqlRep = false; + MySQLRep = "n"; + } + else + { + mysqlRep = true; + MySQLRep = "y"; + } + } + + try { + sysConfig->setConfig(InstallSection, "MySQLRep", MySQLRep); + } + catch(...) + {} + + switch ( IserverTypeInstall ) { case (oam::INSTALL_COMBINE_DM_UM_PM): // combined #1 - dm/um/pm on a single server { @@ -820,75 +824,6 @@ int main(int argc, char *argv[]) break; } - // check for Schema Schema is Local Query wasnt selected - if (!pmwithum) - { - cout << "NOTE: The MariaDB ColumnStore Schema Sync feature will replicate all of the" << endl; - cout << " schemas and InnoDB tables across the User Module nodes. This feature can be enabled" << endl; - cout << " or disabled, for example, if you wish to configure your own replication post installation." << endl << endl; - - try { - MySQLRep = sysConfig->getConfig(InstallSection, "MySQLRep"); - } - catch(...) - {} - - if ( MySQLRep == "y" ) - mysqlRep = true; - - string answer = "y"; - - while(true) { - if ( mysqlRep ) - prompt = "MariaDB ColumnStore Schema Sync feature is Enabled, do you want to leave enabled? [y,n] (y) > "; - else - prompt = "MariaDB ColumnStore Schema Sync feature, do you want to enable? [y,n] (y) > "; - - pcommand = callReadline(prompt.c_str()); - if (pcommand) { - if (strlen(pcommand) > 0) answer = pcommand; - callFree(pcommand); - } - - if ( answer == "y" || answer == "n" ) { - cout << endl; - break; - } - else - cout << "Invalid Entry, please enter 'y' for yes or 'n' for no" << endl; - - if ( noPrompting ) - exit(1); - } - - if ( answer == "y" ) { - mysqlRep = true; - MySQLRep = "y"; - } - else - { - mysqlRep = false; - MySQLRep = "n"; - } - - try { - sysConfig->setConfig(InstallSection, "MySQLRep", MySQLRep); - } - catch(...) - {} - } - else - { //Schema Sync is default as on when Local Query is Selected - mysqlRep = true; - MySQLRep = "y"; - - try { - sysConfig->setConfig(InstallSection, "MySQLRep", MySQLRep); - } - catch(...) - {} - } - if ( !writeConfig(sysConfig) ) { cout << "ERROR: Failed trying to update MariaDB ColumnStore System Configuration file" << endl; exit(1); From e4106c78d6df48ff491537ebab519b7336dd961e Mon Sep 17 00:00:00 2001 From: david hill Date: Thu, 19 Jan 2017 15:04:00 -0600 Subject: [PATCH 32/61] MCOL-481 fix --- oamapps/postConfigure/postConfigure.cpp | 47 ++++++++++--------------- 1 file changed, 18 insertions(+), 29 deletions(-) diff --git a/oamapps/postConfigure/postConfigure.cpp b/oamapps/postConfigure/postConfigure.cpp index f8e66311a..1679a07d1 100644 --- a/oamapps/postConfigure/postConfigure.cpp +++ b/oamapps/postConfigure/postConfigure.cpp @@ -703,37 +703,26 @@ int main(int argc, char *argv[]) exit(1); } - if ( mysqlRep ) - { // current enabled - if ( answer == "y" ) { - mysqlRep = true; - MySQLRep = "y"; - } - else - { - mysqlRep = false; - MySQLRep = "n"; - } - } - else - { // currently disabled - if ( answer == "y" ) { - mysqlRep = false; - MySQLRep = "n"; - } - else - { - mysqlRep = true; - MySQLRep = "y"; - } - } + if ( answer == "y" ) { + mysqlRep = true; + MySQLRep = "y"; + } + else + { + mysqlRep = false; + MySQLRep = "n"; + } - try { - sysConfig->setConfig(InstallSection, "MySQLRep", MySQLRep); - } - catch(...) - {} + try { + sysConfig->setConfig(InstallSection, "MySQLRep", MySQLRep); + } + catch(...) + {} + if ( !writeConfig(sysConfig) ) { + cout << "ERROR: Failed trying to update MariaDB ColumnStore System Configuration file" << endl; + exit(1); + } switch ( IserverTypeInstall ) { case (oam::INSTALL_COMBINE_DM_UM_PM): // combined #1 - dm/um/pm on a single server From 8276d2e905a1bbd36e54ae5e0b204ad51e7c2fc8 Mon Sep 17 00:00:00 2001 From: David Hall Date: Fri, 20 Jan 2017 10:10:24 -0600 Subject: [PATCH 33/61] MCOL-455: fix crash caused by rewind of unopened redistribute plan file during display when no partitions have been planned to be moved. --- writeengine/redistribute/we_redistributecontrolthread.cpp | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/writeengine/redistribute/we_redistributecontrolthread.cpp b/writeengine/redistribute/we_redistributecontrolthread.cpp index 652b48565..c43f83c3b 100644 --- a/writeengine/redistribute/we_redistributecontrolthread.cpp +++ b/writeengine/redistribute/we_redistributecontrolthread.cpp @@ -501,7 +501,7 @@ void RedistributeControlThread::displayPlan() if (!fControl->fPlanFilePtr) { ostringstream oss; - oss << "No data is scheduled to be moved" << endl; + oss << "No data is schefuled to be moved" << endl; fControl->logMessage(oss.str()); return; } From b6d700e47924fde2a77ad68a040cfa3a719fe149 Mon Sep 17 00:00:00 2001 From: David Hill Date: Fri, 20 Jan 2017 19:16:54 -0600 Subject: [PATCH 34/61] MCOL-455 - add start remove to help --- oam/etc/ConsoleCmds.xml | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) diff --git a/oam/etc/ConsoleCmds.xml b/oam/etc/ConsoleCmds.xml index b5ee9b85d..4bf12d36b 100644 --- a/oam/etc/ConsoleCmds.xml +++ b/oam/etc/ConsoleCmds.xml @@ -29,9 +29,9 @@ redistributeData Redistribute table data accross all dbroots to balance disk usage START to begin a redistribution - START REMOVE [dbroots] to redistribute to all but the enumerated dbroots, leaving those empty - STOP to stop redistribution before completion - STATUS to to view statistics and progress + STOP to stop redistribution before completion + STATUS to to view statistics and progress + START REMOVE to redistribute and move data off the given dbroot to other dbroots findObjectFile From a4851e842e9aa38021eecb1f387216ec92d87824 Mon Sep 17 00:00:00 2001 From: david hill Date: Fri, 20 Jan 2017 23:15:00 -0600 Subject: [PATCH 35/61] MCOL-527 - add code for non-root --- procmon/processmonitor.cpp | 86 +++++--------------------------------- 1 file changed, 10 insertions(+), 76 deletions(-) diff --git a/procmon/processmonitor.cpp b/procmon/processmonitor.cpp index af7f4dd2d..b45e5be84 100644 --- a/procmon/processmonitor.cpp +++ b/procmon/processmonitor.cpp @@ -4835,6 +4835,11 @@ int ProcessMonitor::changeMyCnf(std::string type) } } + pos = buf.find("slave-skip-errors=all",0); + if ( pos != string::npos ) { + buf = "# slave-skip-errors=all"; + } + //output to temp file lines.push_back(buf); } @@ -5266,8 +5271,6 @@ int ProcessMonitor::runMasterDist(std::string& password, std::string& slaveModul string cmd = startup::StartUp::installDir() + "/bin/rsync.sh " + ipAddr + " " + password + " " + startup::StartUp::installDir() + " 1 > /tmp/master-dist_" + slaveModule + ".log"; system(cmd.c_str()); - - log.writeLog(__LINE__, "cmd = " + cmd, LOG_TYPE_DEBUG); string logFile = "/tmp/master-dist_" + slaveModule + ".log"; if (!oam.checkLogStatus(logFile, "FAILED")) @@ -5500,47 +5503,15 @@ bool ProcessMonitor::amazonVolumeCheck(int dbrootID) if ( status == "attached" ) { string cmd; if ( rootUser) - cmd = "mount " + deviceName + " " + startup::StartUp::installDir() + "/mysql/db -t ext2 -o noatime,nodiratime,noauto > /tmp/um_mount.log"; + cmd = "mount " + deviceName + " " + startup::StartUp::installDir() + "/mysql/db -t ext2 -o defaults > /tmp/um_mount.log"; else - cmd = "sudo mount " + deviceName + " " + startup::StartUp::installDir() + "/mysql/db -t ext2 -o noatime,nodiratime,noauto,user > /tmp/um_mount.log"; + cmd = "sudo mount " + deviceName + " " + startup::StartUp::installDir() + "/mysql/db -t ext2 -o defaults > /tmp/um_mount.log"; system(cmd.c_str()); log.writeLog(__LINE__, "mount cmd: " + cmd, LOG_TYPE_DEBUG); - if ( rootUser) - cmd = "chown -R mysql:mysql " + startup::StartUp::installDir() + "/mysql/db"; - else - cmd = "sudo chown -R " + USER + ":" + USER + " " + startup::StartUp::installDir() + "/mysql/db"; - + cmd = "chown mysql:mysql -R " + startup::StartUp::installDir() + "/mysql/db"; system(cmd.c_str()); - log.writeLog(__LINE__, "chown cmd: " + cmd, LOG_TYPE_DEBUG); - - //check for setup files in mysq/db, if not, create them for a new install - string file = startup::StartUp::installDir() + "/mysql/db/mysql"; - ifstream new_file (file.c_str()); - if (!new_file) { - string cmd; - cmd = startup::StartUp::installDir() + "/bin/post-mysqld-install --installdir=" + startup::StartUp::installDir() + " > /tmp/post-mysqld-install.log 2>&1"; - log.writeLog(__LINE__, "cmd: " + cmd, LOG_TYPE_DEBUG); - - int rtnCode = system(cmd.c_str()); - if (WEXITSTATUS(rtnCode) != 0) { - log.writeLog(__LINE__, "amazonVolumeCheck function failed, post-mysqld-install error" , LOG_TYPE_ERROR); - return false; - } - else - log.writeLog(__LINE__, "amazonVolumeCheck function, post-mysqld-install passed" , LOG_TYPE_DEBUG); - - cmd = startup::StartUp::installDir() + "/bin/post-mysql-install --installdir=" + startup::StartUp::installDir() + " > /tmp/post-mysql-install.log";; - log.writeLog(__LINE__, "cmd: " + cmd, LOG_TYPE_DEBUG); - rtnCode = system(cmd.c_str()); - if (WEXITSTATUS(rtnCode) != 0) { - log.writeLog(__LINE__, "amazonVolumeCheck function failed, post-mysql-install error" , LOG_TYPE_ERROR); - return false; - } - else - log.writeLog(__LINE__, "amazonVolumeCheck function, post-mysql-install passed" , LOG_TYPE_DEBUG); - } log.writeLog(__LINE__, "amazonVolumeCheck function successfully completed, volume attached: " + volumeName, LOG_TYPE_DEBUG); return true; @@ -5565,49 +5536,12 @@ bool ProcessMonitor::amazonVolumeCheck(int dbrootID) {} if (oam.attachEC2Volume(volumeName, deviceName, instanceName)) { - string cmd; - if ( rootUser) - cmd = "mount " + deviceName + " " + startup::StartUp::installDir() + "/mysql/db -t ext2 -o noatime,nodiratime,noauto > /tmp/um_mount.log"; - else - cmd = "sudo mount " + deviceName + " " + startup::StartUp::installDir() + "/mysql/db -t ext2 -o noatime,nodiratime,noauto,user > /tmp/um_mount.log"; - + string cmd = "mount " + deviceName + " " + startup::StartUp::installDir() + "/mysql/db -t ext2 -o defaults > /dev/null"; system(cmd.c_str()); log.writeLog(__LINE__, "mount cmd: " + cmd, LOG_TYPE_DEBUG); - if ( rootUser) - cmd = "chown -R mysql:mysql " + startup::StartUp::installDir() + "/mysql/db"; - else - cmd = "sudo chown -R " + USER + ":" + USER + " " + startup::StartUp::installDir() + "/mysql/db"; - + cmd = "chown mysql:mysql -R " + startup::StartUp::installDir() + "/mysql/db"; system(cmd.c_str()); - log.writeLog(__LINE__, "chown cmd: " + cmd, LOG_TYPE_DEBUG); - - //check for setup files in mysq/db, if not, create them for a new install - string file = startup::StartUp::installDir() + "/mysql/db/mysql"; - ifstream new_file (file.c_str()); - if (!new_file) { - string cmd; - cmd = startup::StartUp::installDir() + "/bin/post-mysqld-install --installdir=" + startup::StartUp::installDir() + " > /tmp/post-mysqld-install.log 2>&1"; - log.writeLog(__LINE__, "cmd: " + cmd, LOG_TYPE_DEBUG); - int rtnCode = system(cmd.c_str()); - if (WEXITSTATUS(rtnCode) != 0) { - log.writeLog(__LINE__, "amazonVolumeCheck function failed, post-mysqld-install error" , LOG_TYPE_ERROR); - return false; - } - else - log.writeLog(__LINE__, "amazonVolumeCheck function, post-mysqld-install passed" , LOG_TYPE_DEBUG); - - cmd = startup::StartUp::installDir() + "/bin/post-mysql-install --installdir=" + startup::StartUp::installDir() + " > /tmp/post-mysql-install.log";; - log.writeLog(__LINE__, "cmd: " + cmd, LOG_TYPE_DEBUG); - - rtnCode = system(cmd.c_str()); - if (WEXITSTATUS(rtnCode) != 0) { - log.writeLog(__LINE__, "amazonVolumeCheck function failed, post-mysql-install error" , LOG_TYPE_ERROR); - return false; - } - else - log.writeLog(__LINE__, "amazonVolumeCheck function, post-mysql-install passed" , LOG_TYPE_DEBUG); - } return true; } From 92df60f6d56de0cfcdd104922e0e0c4da1e756a7 Mon Sep 17 00:00:00 2001 From: David Hall Date: Fri, 21 Apr 2017 09:37:07 -0500 Subject: [PATCH 36/61] MCOL-597 some clean up and make Release builds work --- dbcon/mysql/CMakeLists.txt | 10 +++++----- dbcon/mysql/ha_calpont_execplan.cpp | 2 +- dbcon/mysql/ha_calpont_impl.cpp | 10 ++++++---- dbcon/mysql/idb_mysql.h | 1 + utils/windowfunction/wf_lead_lag.cpp | 26 -------------------------- 5 files changed, 13 insertions(+), 36 deletions(-) diff --git a/dbcon/mysql/CMakeLists.txt b/dbcon/mysql/CMakeLists.txt index 4c423ed16..015ad58ea 100644 --- a/dbcon/mysql/CMakeLists.txt +++ b/dbcon/mysql/CMakeLists.txt @@ -25,7 +25,7 @@ set_source_files_properties(ha_calpont.cpp PROPERTIES COMPILE_FLAGS "-fno-rtti - add_library(calmysql SHARED ${libcalmysql_SRCS}) -target_link_libraries(calmysql ${ENGINE_LDFLAGS} ${ENGINE_WRITE_LIBS} ${NETSNMP_LIBRARIES} mysqlservices threadpool) +target_link_libraries(calmysql ${ENGINE_LDFLAGS} ${ENGINE_WRITE_LIBS} ${NETSNMP_LIBRARIES} ${SERVER_BUILD_INCLUDE_DIR}/../libservices/libmysqlservices.a threadpool) set_target_properties(calmysql PROPERTIES VERSION 1.0.0 SOVERSION 1) @@ -34,7 +34,7 @@ SET ( is_columnstore_tables_SRCS ) add_library(is_columnstore_tables SHARED ${is_columnstore_tables_SRCS}) -target_link_libraries(is_columnstore_tables ${ENGINE_LDFLAGS} ${ENGINE_WRITE_LIBS} mysqlservices threadpool) +target_link_libraries(is_columnstore_tables ${ENGINE_LDFLAGS} ${ENGINE_WRITE_LIBS} ${SERVER_BUILD_INCLUDE_DIR}/../libservices/libmysqlservices.a threadpool) # Don't prepend .so file with 'lib' set_target_properties(is_columnstore_tables PROPERTIES PREFIX "") @@ -45,7 +45,7 @@ SET ( is_columnstore_columns_SRCS ) add_library(is_columnstore_columns SHARED ${is_columnstore_columns_SRCS}) -target_link_libraries(is_columnstore_columns ${ENGINE_LDFLAGS} ${ENGINE_WRITE_LIBS} mysqlservices threadpool) +target_link_libraries(is_columnstore_columns ${ENGINE_LDFLAGS} ${ENGINE_WRITE_LIBS} ${SERVER_BUILD_INCLUDE_DIR}/../libservices/libmysqlservices.a threadpool) # Don't prepend .so file with 'lib' set_target_properties(is_columnstore_columns PROPERTIES PREFIX "") @@ -56,7 +56,7 @@ SET ( is_columnstore_extents_SRCS ) add_library(is_columnstore_extents SHARED ${is_columnstore_extents_SRCS}) -target_link_libraries(is_columnstore_extents ${ENGINE_LDFLAGS} ${ENGINE_WRITE_LIBS} mysqlservices threadpool) +target_link_libraries(is_columnstore_extents ${ENGINE_LDFLAGS} ${ENGINE_WRITE_LIBS} ${SERVER_BUILD_INCLUDE_DIR}/../libservices/libmysqlservices.a threadpool) # Don't prepend .so file with 'lib' set_target_properties(is_columnstore_extents PROPERTIES PREFIX "") @@ -67,7 +67,7 @@ SET ( is_columnstore_files_SRCS ) add_library(is_columnstore_files SHARED ${is_columnstore_files_SRCS}) -target_link_libraries(is_columnstore_files ${ENGINE_LDFLAGS} ${ENGINE_WRITE_LIBS} mysqlservices threadpool) +target_link_libraries(is_columnstore_files ${ENGINE_LDFLAGS} ${ENGINE_WRITE_LIBS} ${SERVER_BUILD_INCLUDE_DIR}/../libservices/libmysqlservices.a threadpool) # Don't prepend .so file with 'lib' set_target_properties(is_columnstore_files PROPERTIES PREFIX "") diff --git a/dbcon/mysql/ha_calpont_execplan.cpp b/dbcon/mysql/ha_calpont_execplan.cpp index f0aa37176..924314aad 100755 --- a/dbcon/mysql/ha_calpont_execplan.cpp +++ b/dbcon/mysql/ha_calpont_execplan.cpp @@ -21,7 +21,7 @@ */ /** @file */ -#define DEBUG_WALK_COND +//#define DEBUG_WALK_COND #include #include #include diff --git a/dbcon/mysql/ha_calpont_impl.cpp b/dbcon/mysql/ha_calpont_impl.cpp index c5ce0c7b1..6f7f45744 100755 --- a/dbcon/mysql/ha_calpont_impl.cpp +++ b/dbcon/mysql/ha_calpont_impl.cpp @@ -4380,10 +4380,11 @@ int ha_calpont_impl_external_lock(THD *thd, TABLE* table, int lock_type) // table mode if (thd->infinidb_vtable.vtable_state == THD::INFINIDB_DISABLE_VTABLE) { - if (ci->traceFlags & 1) - push_warning(thd, Sql_condition::WARN_LEVEL_NOTE, 9999, mapiter->second.conn_hndl->queryStats.c_str()); if (mapiter->second.conn_hndl) { + if (ci->traceFlags & 1) + push_warning(thd, Sql_condition::WARN_LEVEL_NOTE, 9999, mapiter->second.conn_hndl->queryStats.c_str()); + ci->queryStats = mapiter->second.conn_hndl->queryStats; ci->extendedStats = mapiter->second.conn_hndl->extendedStats; ci->miniStats = mapiter->second.conn_hndl->miniStats; @@ -4407,11 +4408,12 @@ int ha_calpont_impl_external_lock(THD *thd, TABLE* table, int lock_type) { if (thd->infinidb_vtable.vtable_state == THD::INFINIDB_SELECT_VTABLE) { + if (!ci->cal_conn_hndl) + return 0; + if (ci->traceFlags & 1) push_warning(thd, Sql_condition::WARN_LEVEL_NOTE, 9999, ci->cal_conn_hndl->queryStats.c_str()); - if (!ci->cal_conn_hndl) - return 0; ci->queryStats = ci->cal_conn_hndl->queryStats; ci->extendedStats = ci->cal_conn_hndl->extendedStats; ci->miniStats = ci->cal_conn_hndl->miniStats; diff --git a/dbcon/mysql/idb_mysql.h b/dbcon/mysql/idb_mysql.h index 670f0e0e0..637f517d7 100644 --- a/dbcon/mysql/idb_mysql.h +++ b/dbcon/mysql/idb_mysql.h @@ -65,6 +65,7 @@ template bool isnan(T); #include "sql_table.h" #include "sql_select.h" +#include "mysqld_error.h" #include "item_windowfunc.h" // Now clean up the pollution as best we can... diff --git a/utils/windowfunction/wf_lead_lag.cpp b/utils/windowfunction/wf_lead_lag.cpp index 8d3b6169b..85ad54f70 100644 --- a/utils/windowfunction/wf_lead_lag.cpp +++ b/utils/windowfunction/wf_lead_lag.cpp @@ -153,32 +153,6 @@ void WF_lead_lag::parseParms(const std::vector& parms) idbassert(cc != NULL); bool isNull = false; // dummy, harded coded fRespectNulls = (cc->getIntVal(fRow, isNull) > 0); -#if 0 - // parms[1]: offset - for (std::vector::size_type i = 1/*ignore 0th element*/; i < parms.size(); ++i) - { - ConstantColumn* cc = dynamic_cast(parms[i].get()); - if (cc != NULL) - { - fOffset = cc->getIntVal(fRow, fOffsetNull) * fLead; // row not used, no need to setData. - continue; - } - - cc = dynamic_cast(parms[i].get()); - if (cc != NULL) - { - getConstValue(cc, fDefault, fDefNull); - continue; - } - // IGNORE/RESPECT nulls is currently broken in the front end - cc = dynamic_cast(parms[i].get()); - if (cc != NULL) - { - bool isNull = false; // dummy, harded coded - fRespectNulls = (cc->getIntVal(fRow, isNull) > 0); - } - } -#endif } From 2aec526b55b5ed0ea83c639ed2aa84b61b554ff4 Mon Sep 17 00:00:00 2001 From: david hill Date: Tue, 23 Aug 2016 15:59:13 -0500 Subject: [PATCH 37/61] change readme --- README.md | 16 +++++++++------- 1 file changed, 9 insertions(+), 7 deletions(-) diff --git a/README.md b/README.md index 577ac33c1..5df193e89 100644 --- a/README.md +++ b/README.md @@ -1,16 +1,18 @@ -#MariaDB ColumnStore Storage/Execution engine 1.1.0 -MariaDB ColumnStore 1.1.0 is the development version of MariaDB ColumnStore. -It is built by porting InfiniDB 4.6.7 on MariaDB 10.1.21 and adding entirely +#MariaDB ColumnStore Storage/Execution engine 1.0.1 +MariaDB ColumnStore 1.0.2 is the development version of MariaDB ColumnStore. +It is built by porting InfiniDB 4.6.2 on MariaDB 10.1.14 and adding entirely new features not found anywhere else. -#MariaDB ColumnStore 1.1.0 is an development release. +#MariaDB ColumnStore 1.0.2 is an Alpha release. +This is the first MariaDB ColumnStore release, not all features planned for the MariaDB ColumnStore 1.0 +series are included in this release. -- Do not use development releases on production systems. +Additional features will be pushed in future releases. +A few things to notice: +- Do not use alpha releases on production systems. #Building This repository is not meant to be built independently outside of the server. This repository is integrated into http://mariadb-corporation/mariadb-columnstore-server (ie, the *server*) as a git submodule. As such, you can find complete build instructions on *the server* page. https://github.com/mariadb-corporation/mariadb-columnstore-server -#Issue tracking -Issue tracking of MariaDB ColumnStore happens in JIRA, https://jira.mariadb.org/browse/MCOL From 0b3fc9e570dbd0dc8cd91c88ce525c78d3d17f64 Mon Sep 17 00:00:00 2001 From: david hill Date: Wed, 24 Aug 2016 08:42:56 -0500 Subject: [PATCH 38/61] update readme --- README.md | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/README.md b/README.md index 5df193e89..86a9168be 100644 --- a/README.md +++ b/README.md @@ -1,4 +1,4 @@ -#MariaDB ColumnStore Storage/Execution engine 1.0.1 +#MariaDB ColumnStore Storage/Execution engine 1.0.2 MariaDB ColumnStore 1.0.2 is the development version of MariaDB ColumnStore. It is built by porting InfiniDB 4.6.2 on MariaDB 10.1.14 and adding entirely new features not found anywhere else. From db1fe58dc82e7d906b1ee40646d3a9484506a2d9 Mon Sep 17 00:00:00 2001 From: david hill Date: Wed, 24 Aug 2016 08:44:57 -0500 Subject: [PATCH 39/61] update readme --- README | 8 ++++---- 1 file changed, 4 insertions(+), 4 deletions(-) diff --git a/README b/README index 9ebf34303..70b9f790f 100644 --- a/README +++ b/README @@ -1,9 +1,9 @@ -This is MariaDB ColumnStore 1.0.6 -MariaDB ColumnStore 1.0.6 is the development version of MariaDB ColumnStore. -It is built by porting InfiniDB 4.6.7 on MariaDB 10.1.19 and adding entirely +This is MariaDB ColumnStore 1.0.2 +MariaDB ColumnStore 1.0.2 is the development version of MariaDB ColumnStore. +It is built by porting InfiniDB 4.6.2 on MariaDB 10.1.14 and adding entirely new features not found anywhere else. -MariaDB ColumnStore 1.0.6 is an GA release. This is the first MariaDB +MariaDB ColumnStore 1.0.2 is an Alpha release. This is the first MariaDB ColumnStore release, not all features planned for the MariaDB ColumnStore 1.0 series are included in this release. From 834504ed94be803650b2541af2d575a70a34069e Mon Sep 17 00:00:00 2001 From: david hill Date: Thu, 24 Nov 2016 09:14:05 -0600 Subject: [PATCH 40/61] Update README.md --- README.md | 10 ++++++---- 1 file changed, 6 insertions(+), 4 deletions(-) diff --git a/README.md b/README.md index 86a9168be..efeca3c89 100644 --- a/README.md +++ b/README.md @@ -1,9 +1,9 @@ -#MariaDB ColumnStore Storage/Execution engine 1.0.2 -MariaDB ColumnStore 1.0.2 is the development version of MariaDB ColumnStore. -It is built by porting InfiniDB 4.6.2 on MariaDB 10.1.14 and adding entirely +#MariaDB ColumnStore Storage/Execution engine 1.0.5 +MariaDB ColumnStore 1.0.5 is the development version of MariaDB ColumnStore. +It is built by porting InfiniDB 4.6.2 on MariaDB 10.1.19 and adding entirely new features not found anywhere else. -#MariaDB ColumnStore 1.0.2 is an Alpha release. +#MariaDB ColumnStore 1.0.5 is an RC release. This is the first MariaDB ColumnStore release, not all features planned for the MariaDB ColumnStore 1.0 series are included in this release. @@ -16,3 +16,5 @@ This repository is not meant to be built independently outside of the server. T https://github.com/mariadb-corporation/mariadb-columnstore-server +#Issue tracking +Issue tracking of MariaDB ColumnStore happens in JIRA, https://jira.mariadb.org/browse/MCOL From e7b90286a8d1638cf171784654e2eef32064bba6 Mon Sep 17 00:00:00 2001 From: david hill Date: Fri, 13 Jan 2017 10:02:59 -0600 Subject: [PATCH 41/61] change release number --- README.md | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) diff --git a/README.md b/README.md index efeca3c89..62a8a218d 100644 --- a/README.md +++ b/README.md @@ -1,9 +1,9 @@ -#MariaDB ColumnStore Storage/Execution engine 1.0.5 -MariaDB ColumnStore 1.0.5 is the development version of MariaDB ColumnStore. +#MariaDB ColumnStore Storage/Execution engine 1.0.6 +MariaDB ColumnStore 1.0.6 is the development version of MariaDB ColumnStore. It is built by porting InfiniDB 4.6.2 on MariaDB 10.1.19 and adding entirely new features not found anywhere else. -#MariaDB ColumnStore 1.0.5 is an RC release. +#MariaDB ColumnStore 1.0.6 is an GA release. This is the first MariaDB ColumnStore release, not all features planned for the MariaDB ColumnStore 1.0 series are included in this release. From f8329d281f6e1a8b23f4b8897b6090fe904fd9ce Mon Sep 17 00:00:00 2001 From: david hill Date: Fri, 13 Jan 2017 10:04:42 -0600 Subject: [PATCH 42/61] change release number --- README | 8 ++++---- 1 file changed, 4 insertions(+), 4 deletions(-) diff --git a/README b/README index 70b9f790f..ac59b3816 100644 --- a/README +++ b/README @@ -1,9 +1,9 @@ -This is MariaDB ColumnStore 1.0.2 -MariaDB ColumnStore 1.0.2 is the development version of MariaDB ColumnStore. -It is built by porting InfiniDB 4.6.2 on MariaDB 10.1.14 and adding entirely +This is MariaDB ColumnStore 1.0.6 +MariaDB ColumnStore 1.0.6 is the development version of MariaDB ColumnStore. +It is built by porting InfiniDB 4.6.2 on MariaDB 10.1.19 and adding entirely new features not found anywhere else. -MariaDB ColumnStore 1.0.2 is an Alpha release. This is the first MariaDB +MariaDB ColumnStore 1.0.6 is an GA release. This is the first MariaDB ColumnStore release, not all features planned for the MariaDB ColumnStore 1.0 series are included in this release. From 0a4a46b52a2ba8132760139231e7612388a48920 Mon Sep 17 00:00:00 2001 From: david hill Date: Wed, 18 Jan 2017 14:32:54 -0600 Subject: [PATCH 43/61] mcol-481 - just changed the note --- oamapps/postConfigure/postConfigure.cpp | 371 +++++++++++------------- 1 file changed, 175 insertions(+), 196 deletions(-) diff --git a/oamapps/postConfigure/postConfigure.cpp b/oamapps/postConfigure/postConfigure.cpp index 01115e8b6..f8e66311a 100644 --- a/oamapps/postConfigure/postConfigure.cpp +++ b/oamapps/postConfigure/postConfigure.cpp @@ -52,17 +52,6 @@ #include #include -#include -#include -#include -#include - -#include /* for strncpy */ -#include -#include -#include -#include - #include #include #include "boost/filesystem/operations.hpp" @@ -109,7 +98,7 @@ bool updateProcessConfig(int serverTypeInstall); bool uncommentCalpontXml( string entry); bool makeRClocal(string moduleType, string moduleName, int IserverTypeInstall); bool createDbrootDirs(string DBRootStorageType); -bool pkgCheck(std::string columnstorePackage); +bool pkgCheck(); bool storageSetup(bool amazonInstall); void setSystemName(); bool singleServerDBrootSetup(); @@ -126,7 +115,7 @@ typedef struct ModuleIP_struct std::string launchInstance(ModuleIP moduleip); -string columnstorePackage; +string calpontPackage1; //string calpontPackage2; //string calpontPackage3; //string mysqlPackage; @@ -408,7 +397,7 @@ int main(int argc, char *argv[]) //check if MariaDB ColumnStore is up and running if (oam.checkSystemRunning()) { cout << "MariaDB ColumnStore is running, can't run postConfigure while MariaDB ColumnStore is running. Exiting.." << endl; - exit (1); + exit (0); } //if InitialInstallFlag is set, then an install has been done @@ -434,60 +423,6 @@ int main(int argc, char *argv[]) exit(1); } - //check for local ip address as pm1 - ModuleConfig moduleconfig; - - try - { - oam.getSystemConfig("pm1", moduleconfig); - if (moduleconfig.hostConfigList.size() > 0 ) - { - HostConfigList::iterator pt1 = moduleconfig.hostConfigList.begin(); - string PM1ipAdd = (*pt1).IPAddr; - //cout << PM1ipAdd << endl; - - if ( PM1ipAdd != "127.0.0.1" && PM1ipAdd != "0.0.0.0") - { - struct ifaddrs *ifap, *ifa; - struct sockaddr_in *sa; - char *addr; - bool found = false; - - getifaddrs (&ifap); - for (ifa = ifap; ifa; ifa = ifa->ifa_next) { - if (ifa->ifa_addr->sa_family==AF_INET) { - sa = (struct sockaddr_in *) ifa->ifa_addr; - addr = inet_ntoa(sa->sin_addr); - //printf("Interface: %s\tAddress: %s\n", ifa->ifa_name, addr); - - if ( PM1ipAdd == addr ) - { - //match - found = true; - } - } - - if (found) - break; - } - - freeifaddrs(ifap); - - if (!found) - { - cout << endl; - cout << "ERROR: postConfigure install can only be done on the PM1" << endl; - cout << "designated node. The configured PM1 IP address doesn't match the local" << endl; - cout << "IP Address. exiting..." << endl; - exit(1); - } - } - } - } - catch(...) - {} - - // run my.cnf upgrade script if ( reuseConfig == "y" ) { @@ -731,6 +666,75 @@ int main(int argc, char *argv[]) exit(1); } + cout << "NOTE: The MariaDB ColumnStore Schema Sync feature will replicate all of the" << endl; + cout << " schemas and InnoDB tables across the User Module nodes. This feature can be enabled" << endl; + cout << " or disabled, for example, if you wish to configure your own replication post installation." << endl << endl; + + try { + MySQLRep = sysConfig->getConfig(InstallSection, "MySQLRep"); + } + catch(...) + {} + + if ( MySQLRep == "y" ) + mysqlRep = true; + + string answer = "y"; + + while(true) { + if ( mysqlRep ) + prompt = "MariaDB ColumnStore Schema Sync feature is Enabled, do you want to leave enabled? [y,n] (y) > "; + else + prompt = "MariaDB ColumnStore Schema Sync feature, do you want to enable? [y,n] (y) > "; + + pcommand = callReadline(prompt.c_str()); + if (pcommand) { + if (strlen(pcommand) > 0) answer = pcommand; + callFree(pcommand); + } + + if ( answer == "y" || answer == "n" ) { + cout << endl; + break; + } + else + cout << "Invalid Entry, please enter 'y' for yes or 'n' for no" << endl; + if ( noPrompting ) + exit(1); + } + + if ( mysqlRep ) + { // current enabled + if ( answer == "y" ) { + mysqlRep = true; + MySQLRep = "y"; + } + else + { + mysqlRep = false; + MySQLRep = "n"; + } + } + else + { // currently disabled + if ( answer == "y" ) { + mysqlRep = false; + MySQLRep = "n"; + } + else + { + mysqlRep = true; + MySQLRep = "y"; + } + } + + try { + sysConfig->setConfig(InstallSection, "MySQLRep", MySQLRep); + } + catch(...) + {} + + switch ( IserverTypeInstall ) { case (oam::INSTALL_COMBINE_DM_UM_PM): // combined #1 - dm/um/pm on a single server { @@ -820,75 +824,6 @@ int main(int argc, char *argv[]) break; } - // check for Schema Schema is Local Query wasnt selected - if (!pmwithum) - { - cout << "NOTE: The MariaDB ColumnStore Schema Sync feature will replicate all of the" << endl; - cout << " schemas and InnoDB tables across the User Module nodes. This feature can be enabled" << endl; - cout << " or disabled, for example, if you wish to configure your own replication post installation." << endl << endl; - - try { - MySQLRep = sysConfig->getConfig(InstallSection, "MySQLRep"); - } - catch(...) - {} - - if ( MySQLRep == "y" ) - mysqlRep = true; - - string answer = "y"; - - while(true) { - if ( mysqlRep ) - prompt = "MariaDB ColumnStore Schema Sync feature is Enabled, do you want to leave enabled? [y,n] (y) > "; - else - prompt = "MariaDB ColumnStore Schema Sync feature, do you want to enable? [y,n] (y) > "; - - pcommand = callReadline(prompt.c_str()); - if (pcommand) { - if (strlen(pcommand) > 0) answer = pcommand; - callFree(pcommand); - } - - if ( answer == "y" || answer == "n" ) { - cout << endl; - break; - } - else - cout << "Invalid Entry, please enter 'y' for yes or 'n' for no" << endl; - - if ( noPrompting ) - exit(1); - } - - if ( answer == "y" ) { - mysqlRep = true; - MySQLRep = "y"; - } - else - { - mysqlRep = false; - MySQLRep = "n"; - } - - try { - sysConfig->setConfig(InstallSection, "MySQLRep", MySQLRep); - } - catch(...) - {} - } - else - { //Schema Sync is default as on when Local Query is Selected - mysqlRep = true; - MySQLRep = "y"; - - try { - sysConfig->setConfig(InstallSection, "MySQLRep", MySQLRep); - } - catch(...) - {} - } - if ( !writeConfig(sysConfig) ) { cout << "ERROR: Failed trying to update MariaDB ColumnStore System Configuration file" << endl; exit(1); @@ -2676,42 +2611,7 @@ int main(int argc, char *argv[]) /* create a thread_data_t argument array */ thread_data_t thr_data[childmodulelist.size()]; - // determine package type - string EEPackageType; - - if (!rootUser) - EEPackageType = "binary"; - else - { - int rtnCode = system("rpm -qi mariadb-columnstore-platform > /tmp/columnstore.txt 2>&1"); - if (WEXITSTATUS(rtnCode) == 0) - EEPackageType = "rpm"; - else { - rtnCode = system("dpkg -s mariadb-columnstore-platform > /tmp/columnstore.txt 2>&1"); - if (WEXITSTATUS(rtnCode) == 0) - EEPackageType = "deb"; - else - EEPackageType = "binary"; - } - } - - try { - sysConfig->setConfig(InstallSection, "EEPackageType", EEPackageType); - } - catch(...) - { - cout << "ERROR: Problem setting EEPackageType from the MariaDB ColumnStore System Configuration file" << endl; - exit(1); - } - - if ( !writeConfig(sysConfig) ) { - cout << "ERROR: Failed trying to update MariaDB ColumnStore System Configuration file" << endl; - exit(1); - } - - string install = "y"; - if ( IserverTypeInstall != oam::INSTALL_COMBINE_DM_UM_PM || pmNumber > 1 ) { // @@ -2754,6 +2654,39 @@ int main(int argc, char *argv[]) cout << endl; + //Write out Updated System Configuration File + string EEPackageType; + if ( rootUser ) + { + try { + EEPackageType = sysConfig->getConfig(InstallSection, "EEPackageType"); + } + catch(...) + { + cout << "ERROR: Problem getting EEPackageType from the MariaDB ColumnStore System Configuration file" << endl; + exit(1); + } + } + else //nonroot, default to binary + EEPackageType = "binary"; + + while(true) { + prompt = "Enter the Package Type being installed to other servers [rpm,deb,binary] (" + EEPackageType + ") > "; + pcommand = callReadline(prompt); + if (pcommand) { + if (strlen(pcommand) > 0) EEPackageType = pcommand; + callFree(pcommand); + } + + if ( EEPackageType == "rpm" || EEPackageType == "deb" || EEPackageType == "binary" ) { + break; + } + cout << "Invalid Package Type, please re-enter" << endl; + EEPackageType = "rpm"; + if ( noPrompting ) + exit(1); + } + if ( EEPackageType == "rpm" ) { cout << "Performing an MariaDB ColumnStore System install using RPM packages" << endl; @@ -2773,20 +2706,76 @@ int main(int argc, char *argv[]) } } + //Write out Updated System Configuration File + try { + sysConfig->setConfig(InstallSection, "EEPackageType", EEPackageType); + } + catch(...) + { + cout << "ERROR: Problem setting EEPackageType from the MariaDB ColumnStore System Configuration file" << endl; + exit(1); + } + + if ( !writeConfig(sysConfig) ) { + cout << "ERROR: Failed trying to update MariaDB ColumnStore System Configuration file" << endl; + exit(1); + } + //check if pkgs are located in $HOME directory string version = systemsoftware.Version + "-" + systemsoftware.Release; - if ( EEPackageType == "rpm") - columnstorePackage = HOME + "/" + "mariadb-columnstore-" + version + "*.rpm.tar.gz"; - else - if ( EEPackageType == "deb") - columnstorePackage = HOME + "/" + "mariadb-columnstore-" + version + "*.deb.tar.gz"; + if ( EEPackageType != "binary") { + string separator = "-"; + calpontPackage1 = "mariadb-columnstore-*" + separator + version; + //calpontPackage2 = "mariadb-columnstore-libs" + separator + version; + //calpontPackage3 = "mariadb-columnstore-enterprise" + separator + version; + //mysqlPackage = "mariadb-columnstore-storage-engine" + separator + version; + //mysqldPackage = "mariadb-columnstore-mysql" + separator + version; + + if( !pkgCheck() ) { + exit(1); + } else - columnstorePackage = HOME + "/" + "mariadb-columnstore-" + version + "*.bin.tar.gz"; + { + //mariadb + calpontPackage1 = "mariadb-columnstore-*" + separator + version; + calpontPackage1 = HOME + "/" + calpontPackage1 + "*." + EEPackageType; + //calpontPackage2 = HOME + "/" + calpontPackage2 + "*." + EEPackageType; + //calpontPackage3 = HOME + "/" + calpontPackage3 + "*." + EEPackageType; + //mysqlPackage = HOME + "/" + mysqlPackage + "*." + EEPackageType; + //mysqldPackage = HOME + "/" + mysqldPackage + "*." + EEPackageType; + } + } + else + { + // binary + //string fileName = installDir + "/bin/healthcheck"; + //ifstream file (fileName.c_str()); + //if (!file) // CE + calpontPackage1 = "mariadb-columnstore-" + version; + //else // EE + //calpontPackage1 = "mariadb-columnstore-ent-" + version; + //calpontPackage2 = "dummy"; + //calpontPackage3 = "dummy"; + //mysqlPackage = calpontPackage1; + //mysqldPackage = calpontPackage1; + + if( !pkgCheck() ) + exit(1); + calpontPackage1 = HOME + "/" + calpontPackage1 + "*.bin.tar.gz"; + //calpontPackage2 = "dummy"; + //calpontPackage3 = "dummy"; + } + + //If ent pkg is not there, mark it as such + //{ + // glob_t gt; + // memset(>, 0, sizeof(gt)); + // if (glob(calpontPackage3.c_str(), 0, 0, >) != 0) + // calpontPackage3 = "dummy.rpm"; + // globfree(>); + //} - if( !pkgCheck(columnstorePackage) ) - exit(1); - if ( password.empty() ) { cout << endl; @@ -2996,7 +2985,7 @@ int main(int argc, char *argv[]) // checkRemoteMysqlPort(remoteModuleIP, remoteModuleName, USER, password, mysqlPort, sysConfig); cmd = installDir + "/bin/binary_installer.sh " + remoteModuleName + " " + - remoteModuleIP + " " + password + " " + columnstorePackage + " " + remoteModuleType + + remoteModuleIP + " " + password + " " + calpontPackage1 + " " + remoteModuleType + " initial " + binservertype + " " + mysqlPort + " " + remote_installer_debug + " " + installDir + " " + debug_logfile; @@ -3066,7 +3055,7 @@ int main(int argc, char *argv[]) if ( pmwithum ) binservertype = "pmwithum"; cmd = installDir + "/bin/binary_installer.sh " + remoteModuleName + " " + remoteModuleIP + - " " + password + " " + columnstorePackage + " " + remoteModuleType + " initial " + + " " + password + " " + calpontPackage1 + " " + remoteModuleType + " initial " + binservertype + " " + mysqlPort + " " + remote_installer_debug + " " + installDir + " " + debug_logfile; @@ -3683,14 +3672,6 @@ bool setOSFiles(string parentOAMModuleName, int serverTypeInstall) system(cmd.c_str()); } - //check and do the amazon credentials file - string fileName = HOME + "/.aws/credentials"; - ifstream oldFile (fileName.c_str()); - if (!oldFile) - return allfound; - - string cmd = "cp " + fileName + " " + installDir + "/local/etc/. > /dev/null 2>&1"; - system(cmd.c_str()); return allfound; } @@ -3987,14 +3968,14 @@ bool createDbrootDirs(string DBRootStorageType) /* * pkgCheck */ -bool pkgCheck(string columnstorePackage) +bool pkgCheck() { while(true) { - string cmd = "ls " + columnstorePackage + " > /tmp/calpontpkgs"; + string cmd = "ls " + HOME + " | grep " + calpontPackage1 + " > /tmp/calpontpkgs"; system(cmd.c_str()); - string pkg = columnstorePackage; + string pkg = calpontPackage1; string fileName = "/tmp/calpontpkgs"; ifstream oldFile (fileName.c_str()); if (oldFile) { @@ -4858,7 +4839,6 @@ bool storageSetup(bool amazonInstall) void setSystemName() { - Oam oam; //setup System Name try { systemName = sysConfig->getConfig(SystemSection, "SystemName"); @@ -4881,7 +4861,6 @@ void setSystemName() try { sysConfig->setConfig(SystemSection, "SystemName", systemName); - oam.changeMyCnf( "server_audit_syslog_info", systemName ); } catch(...) { From 5e2f8cf6c970385bd7512eb4c7cfce58b405aabe Mon Sep 17 00:00:00 2001 From: david hill Date: Thu, 19 Jan 2017 15:04:00 -0600 Subject: [PATCH 44/61] MCOL-481 fix --- oamapps/postConfigure/postConfigure.cpp | 47 ++++++++++--------------- 1 file changed, 18 insertions(+), 29 deletions(-) diff --git a/oamapps/postConfigure/postConfigure.cpp b/oamapps/postConfigure/postConfigure.cpp index f8e66311a..1679a07d1 100644 --- a/oamapps/postConfigure/postConfigure.cpp +++ b/oamapps/postConfigure/postConfigure.cpp @@ -703,37 +703,26 @@ int main(int argc, char *argv[]) exit(1); } - if ( mysqlRep ) - { // current enabled - if ( answer == "y" ) { - mysqlRep = true; - MySQLRep = "y"; - } - else - { - mysqlRep = false; - MySQLRep = "n"; - } - } - else - { // currently disabled - if ( answer == "y" ) { - mysqlRep = false; - MySQLRep = "n"; - } - else - { - mysqlRep = true; - MySQLRep = "y"; - } - } + if ( answer == "y" ) { + mysqlRep = true; + MySQLRep = "y"; + } + else + { + mysqlRep = false; + MySQLRep = "n"; + } - try { - sysConfig->setConfig(InstallSection, "MySQLRep", MySQLRep); - } - catch(...) - {} + try { + sysConfig->setConfig(InstallSection, "MySQLRep", MySQLRep); + } + catch(...) + {} + if ( !writeConfig(sysConfig) ) { + cout << "ERROR: Failed trying to update MariaDB ColumnStore System Configuration file" << endl; + exit(1); + } switch ( IserverTypeInstall ) { case (oam::INSTALL_COMBINE_DM_UM_PM): // combined #1 - dm/um/pm on a single server From bdd1d22d4ea7be6784d2d59c6be84478efcd5d6a Mon Sep 17 00:00:00 2001 From: David Hill Date: Fri, 20 Jan 2017 19:16:54 -0600 Subject: [PATCH 45/61] MCOL-455 - add start remove to help --- oam/etc/ConsoleCmds.xml | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) diff --git a/oam/etc/ConsoleCmds.xml b/oam/etc/ConsoleCmds.xml index 01f6001cc..4bf12d36b 100644 --- a/oam/etc/ConsoleCmds.xml +++ b/oam/etc/ConsoleCmds.xml @@ -29,9 +29,9 @@ redistributeData Redistribute table data accross all dbroots to balance disk usage START to begin a redistribution - START REMOVE n to being a redistribution where data is removed from dbroot 'n' - STOP to stop redistribution before completion - STATUS to to view statistics and progress + STOP to stop redistribution before completion + STATUS to to view statistics and progress + START REMOVE to redistribute and move data off the given dbroot to other dbroots findObjectFile From 277d3b095fe0d22e988e49b31cd9d69bca7d7a19 Mon Sep 17 00:00:00 2001 From: David Hill Date: Fri, 20 Jan 2017 19:25:48 -0600 Subject: [PATCH 46/61] MCOL-455 - add start remove to help --- oam/etc/ConsoleCmds.xml | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) diff --git a/oam/etc/ConsoleCmds.xml b/oam/etc/ConsoleCmds.xml index 4bf12d36b..01f6001cc 100644 --- a/oam/etc/ConsoleCmds.xml +++ b/oam/etc/ConsoleCmds.xml @@ -29,9 +29,9 @@ redistributeData Redistribute table data accross all dbroots to balance disk usage START to begin a redistribution - STOP to stop redistribution before completion - STATUS to to view statistics and progress - START REMOVE to redistribute and move data off the given dbroot to other dbroots + START REMOVE n to being a redistribution where data is removed from dbroot 'n' + STOP to stop redistribution before completion + STATUS to to view statistics and progress findObjectFile From c7bbb78b73d4f94cf657b6b31d1e785d5c479d55 Mon Sep 17 00:00:00 2001 From: david hill Date: Fri, 20 Jan 2017 19:49:26 -0600 Subject: [PATCH 47/61] MCOL-528 - removed schema sync prompt and default to on when local query is selected --- oamapps/postConfigure/postConfigure.cpp | 127 +++++++++++++----------- 1 file changed, 69 insertions(+), 58 deletions(-) diff --git a/oamapps/postConfigure/postConfigure.cpp b/oamapps/postConfigure/postConfigure.cpp index 1679a07d1..1facf1a7c 100644 --- a/oamapps/postConfigure/postConfigure.cpp +++ b/oamapps/postConfigure/postConfigure.cpp @@ -666,64 +666,6 @@ int main(int argc, char *argv[]) exit(1); } - cout << "NOTE: The MariaDB ColumnStore Schema Sync feature will replicate all of the" << endl; - cout << " schemas and InnoDB tables across the User Module nodes. This feature can be enabled" << endl; - cout << " or disabled, for example, if you wish to configure your own replication post installation." << endl << endl; - - try { - MySQLRep = sysConfig->getConfig(InstallSection, "MySQLRep"); - } - catch(...) - {} - - if ( MySQLRep == "y" ) - mysqlRep = true; - - string answer = "y"; - - while(true) { - if ( mysqlRep ) - prompt = "MariaDB ColumnStore Schema Sync feature is Enabled, do you want to leave enabled? [y,n] (y) > "; - else - prompt = "MariaDB ColumnStore Schema Sync feature, do you want to enable? [y,n] (y) > "; - - pcommand = callReadline(prompt.c_str()); - if (pcommand) { - if (strlen(pcommand) > 0) answer = pcommand; - callFree(pcommand); - } - - if ( answer == "y" || answer == "n" ) { - cout << endl; - break; - } - else - cout << "Invalid Entry, please enter 'y' for yes or 'n' for no" << endl; - if ( noPrompting ) - exit(1); - } - - if ( answer == "y" ) { - mysqlRep = true; - MySQLRep = "y"; - } - else - { - mysqlRep = false; - MySQLRep = "n"; - } - - try { - sysConfig->setConfig(InstallSection, "MySQLRep", MySQLRep); - } - catch(...) - {} - - if ( !writeConfig(sysConfig) ) { - cout << "ERROR: Failed trying to update MariaDB ColumnStore System Configuration file" << endl; - exit(1); - } - switch ( IserverTypeInstall ) { case (oam::INSTALL_COMBINE_DM_UM_PM): // combined #1 - dm/um/pm on a single server { @@ -813,6 +755,75 @@ int main(int argc, char *argv[]) break; } + // check for Schema Schema is Local Query wasnt selected + if (!pmwithum) + { + cout << "NOTE: The MariaDB ColumnStore Schema Sync feature will replicate all of the" << endl; + cout << " schemas and InnoDB tables across the User Module nodes. This feature can be enabled" << endl; + cout << " or disabled, for example, if you wish to configure your own replication post installation." << endl << endl; + + try { + MySQLRep = sysConfig->getConfig(InstallSection, "MySQLRep"); + } + catch(...) + {} + + if ( MySQLRep == "y" ) + mysqlRep = true; + + string answer = "y"; + + while(true) { + if ( mysqlRep ) + prompt = "MariaDB ColumnStore Schema Sync feature is Enabled, do you want to leave enabled? [y,n] (y) > "; + else + prompt = "MariaDB ColumnStore Schema Sync feature, do you want to enable? [y,n] (y) > "; + + pcommand = callReadline(prompt.c_str()); + if (pcommand) { + if (strlen(pcommand) > 0) answer = pcommand; + callFree(pcommand); + } + + if ( answer == "y" || answer == "n" ) { + cout << endl; + break; + } + else + cout << "Invalid Entry, please enter 'y' for yes or 'n' for no" << endl; + + if ( noPrompting ) + exit(1); + } + + if ( answer == "y" ) { + mysqlRep = true; + MySQLRep = "y"; + } + else + { + mysqlRep = false; + MySQLRep = "n"; + } + + try { + sysConfig->setConfig(InstallSection, "MySQLRep", MySQLRep); + } + catch(...) + {} + } + else + { //Schema Sync is default as on when Local Query is Selected + mysqlRep = true; + MySQLRep = "y"; + + try { + sysConfig->setConfig(InstallSection, "MySQLRep", MySQLRep); + } + catch(...) + {} + } + if ( !writeConfig(sysConfig) ) { cout << "ERROR: Failed trying to update MariaDB ColumnStore System Configuration file" << endl; exit(1); From 96bf638dafb421b1d2c08810f3bf12d29e217e49 Mon Sep 17 00:00:00 2001 From: david hill Date: Fri, 20 Jan 2017 23:15:00 -0600 Subject: [PATCH 48/61] MCOL-527 - add code for non-root --- procmon/processmonitor.cpp | 86 +++++--------------------------------- 1 file changed, 10 insertions(+), 76 deletions(-) diff --git a/procmon/processmonitor.cpp b/procmon/processmonitor.cpp index af7f4dd2d..b45e5be84 100644 --- a/procmon/processmonitor.cpp +++ b/procmon/processmonitor.cpp @@ -4835,6 +4835,11 @@ int ProcessMonitor::changeMyCnf(std::string type) } } + pos = buf.find("slave-skip-errors=all",0); + if ( pos != string::npos ) { + buf = "# slave-skip-errors=all"; + } + //output to temp file lines.push_back(buf); } @@ -5266,8 +5271,6 @@ int ProcessMonitor::runMasterDist(std::string& password, std::string& slaveModul string cmd = startup::StartUp::installDir() + "/bin/rsync.sh " + ipAddr + " " + password + " " + startup::StartUp::installDir() + " 1 > /tmp/master-dist_" + slaveModule + ".log"; system(cmd.c_str()); - - log.writeLog(__LINE__, "cmd = " + cmd, LOG_TYPE_DEBUG); string logFile = "/tmp/master-dist_" + slaveModule + ".log"; if (!oam.checkLogStatus(logFile, "FAILED")) @@ -5500,47 +5503,15 @@ bool ProcessMonitor::amazonVolumeCheck(int dbrootID) if ( status == "attached" ) { string cmd; if ( rootUser) - cmd = "mount " + deviceName + " " + startup::StartUp::installDir() + "/mysql/db -t ext2 -o noatime,nodiratime,noauto > /tmp/um_mount.log"; + cmd = "mount " + deviceName + " " + startup::StartUp::installDir() + "/mysql/db -t ext2 -o defaults > /tmp/um_mount.log"; else - cmd = "sudo mount " + deviceName + " " + startup::StartUp::installDir() + "/mysql/db -t ext2 -o noatime,nodiratime,noauto,user > /tmp/um_mount.log"; + cmd = "sudo mount " + deviceName + " " + startup::StartUp::installDir() + "/mysql/db -t ext2 -o defaults > /tmp/um_mount.log"; system(cmd.c_str()); log.writeLog(__LINE__, "mount cmd: " + cmd, LOG_TYPE_DEBUG); - if ( rootUser) - cmd = "chown -R mysql:mysql " + startup::StartUp::installDir() + "/mysql/db"; - else - cmd = "sudo chown -R " + USER + ":" + USER + " " + startup::StartUp::installDir() + "/mysql/db"; - + cmd = "chown mysql:mysql -R " + startup::StartUp::installDir() + "/mysql/db"; system(cmd.c_str()); - log.writeLog(__LINE__, "chown cmd: " + cmd, LOG_TYPE_DEBUG); - - //check for setup files in mysq/db, if not, create them for a new install - string file = startup::StartUp::installDir() + "/mysql/db/mysql"; - ifstream new_file (file.c_str()); - if (!new_file) { - string cmd; - cmd = startup::StartUp::installDir() + "/bin/post-mysqld-install --installdir=" + startup::StartUp::installDir() + " > /tmp/post-mysqld-install.log 2>&1"; - log.writeLog(__LINE__, "cmd: " + cmd, LOG_TYPE_DEBUG); - - int rtnCode = system(cmd.c_str()); - if (WEXITSTATUS(rtnCode) != 0) { - log.writeLog(__LINE__, "amazonVolumeCheck function failed, post-mysqld-install error" , LOG_TYPE_ERROR); - return false; - } - else - log.writeLog(__LINE__, "amazonVolumeCheck function, post-mysqld-install passed" , LOG_TYPE_DEBUG); - - cmd = startup::StartUp::installDir() + "/bin/post-mysql-install --installdir=" + startup::StartUp::installDir() + " > /tmp/post-mysql-install.log";; - log.writeLog(__LINE__, "cmd: " + cmd, LOG_TYPE_DEBUG); - rtnCode = system(cmd.c_str()); - if (WEXITSTATUS(rtnCode) != 0) { - log.writeLog(__LINE__, "amazonVolumeCheck function failed, post-mysql-install error" , LOG_TYPE_ERROR); - return false; - } - else - log.writeLog(__LINE__, "amazonVolumeCheck function, post-mysql-install passed" , LOG_TYPE_DEBUG); - } log.writeLog(__LINE__, "amazonVolumeCheck function successfully completed, volume attached: " + volumeName, LOG_TYPE_DEBUG); return true; @@ -5565,49 +5536,12 @@ bool ProcessMonitor::amazonVolumeCheck(int dbrootID) {} if (oam.attachEC2Volume(volumeName, deviceName, instanceName)) { - string cmd; - if ( rootUser) - cmd = "mount " + deviceName + " " + startup::StartUp::installDir() + "/mysql/db -t ext2 -o noatime,nodiratime,noauto > /tmp/um_mount.log"; - else - cmd = "sudo mount " + deviceName + " " + startup::StartUp::installDir() + "/mysql/db -t ext2 -o noatime,nodiratime,noauto,user > /tmp/um_mount.log"; - + string cmd = "mount " + deviceName + " " + startup::StartUp::installDir() + "/mysql/db -t ext2 -o defaults > /dev/null"; system(cmd.c_str()); log.writeLog(__LINE__, "mount cmd: " + cmd, LOG_TYPE_DEBUG); - if ( rootUser) - cmd = "chown -R mysql:mysql " + startup::StartUp::installDir() + "/mysql/db"; - else - cmd = "sudo chown -R " + USER + ":" + USER + " " + startup::StartUp::installDir() + "/mysql/db"; - + cmd = "chown mysql:mysql -R " + startup::StartUp::installDir() + "/mysql/db"; system(cmd.c_str()); - log.writeLog(__LINE__, "chown cmd: " + cmd, LOG_TYPE_DEBUG); - - //check for setup files in mysq/db, if not, create them for a new install - string file = startup::StartUp::installDir() + "/mysql/db/mysql"; - ifstream new_file (file.c_str()); - if (!new_file) { - string cmd; - cmd = startup::StartUp::installDir() + "/bin/post-mysqld-install --installdir=" + startup::StartUp::installDir() + " > /tmp/post-mysqld-install.log 2>&1"; - log.writeLog(__LINE__, "cmd: " + cmd, LOG_TYPE_DEBUG); - int rtnCode = system(cmd.c_str()); - if (WEXITSTATUS(rtnCode) != 0) { - log.writeLog(__LINE__, "amazonVolumeCheck function failed, post-mysqld-install error" , LOG_TYPE_ERROR); - return false; - } - else - log.writeLog(__LINE__, "amazonVolumeCheck function, post-mysqld-install passed" , LOG_TYPE_DEBUG); - - cmd = startup::StartUp::installDir() + "/bin/post-mysql-install --installdir=" + startup::StartUp::installDir() + " > /tmp/post-mysql-install.log";; - log.writeLog(__LINE__, "cmd: " + cmd, LOG_TYPE_DEBUG); - - rtnCode = system(cmd.c_str()); - if (WEXITSTATUS(rtnCode) != 0) { - log.writeLog(__LINE__, "amazonVolumeCheck function failed, post-mysql-install error" , LOG_TYPE_ERROR); - return false; - } - else - log.writeLog(__LINE__, "amazonVolumeCheck function, post-mysql-install passed" , LOG_TYPE_DEBUG); - } return true; } From a0c978f9605a52ece7338cd180634563ea7ef16a Mon Sep 17 00:00:00 2001 From: David Hill Date: Sat, 21 Jan 2017 15:51:25 -0600 Subject: [PATCH 49/61] MCOL-527 - change to run mysql scripts when um disk is mounted --- procmon/processmonitor.cpp | 81 +++++++++++++++++++++++++++++++++++--- 1 file changed, 76 insertions(+), 5 deletions(-) diff --git a/procmon/processmonitor.cpp b/procmon/processmonitor.cpp index b45e5be84..e474c69d5 100644 --- a/procmon/processmonitor.cpp +++ b/procmon/processmonitor.cpp @@ -5271,6 +5271,8 @@ int ProcessMonitor::runMasterDist(std::string& password, std::string& slaveModul string cmd = startup::StartUp::installDir() + "/bin/rsync.sh " + ipAddr + " " + password + " " + startup::StartUp::installDir() + " 1 > /tmp/master-dist_" + slaveModule + ".log"; system(cmd.c_str()); + + log.writeLog(__LINE__, "cmd = " + cmd, LOG_TYPE_DEBUG); string logFile = "/tmp/master-dist_" + slaveModule + ".log"; if (!oam.checkLogStatus(logFile, "FAILED")) @@ -5503,15 +5505,47 @@ bool ProcessMonitor::amazonVolumeCheck(int dbrootID) if ( status == "attached" ) { string cmd; if ( rootUser) - cmd = "mount " + deviceName + " " + startup::StartUp::installDir() + "/mysql/db -t ext2 -o defaults > /tmp/um_mount.log"; + cmd = "mount " + deviceName + " " + startup::StartUp::installDir() + "/mysql/db -t ext2 -o noatime,nodiratime,noauto > /tmp/um_mount.log"; else - cmd = "sudo mount " + deviceName + " " + startup::StartUp::installDir() + "/mysql/db -t ext2 -o defaults > /tmp/um_mount.log"; + cmd = "sudo mount " + deviceName + " " + startup::StartUp::installDir() + "/mysql/db -t ext2 -o noatime,nodiratime,noauto,user > /tmp/um_mount.log"; system(cmd.c_str()); log.writeLog(__LINE__, "mount cmd: " + cmd, LOG_TYPE_DEBUG); - cmd = "chown mysql:mysql -R " + startup::StartUp::installDir() + "/mysql/db"; + if ( rootUser) + cmd = "chown -R mysql:mysql " + startup::StartUp::installDir() + "/mysql/db"; + else + cmd = "sudo chown -R " + USER + ":" + USER + " " + startup::StartUp::installDir() + "/mysql/db"; + system(cmd.c_str()); + log.writeLog(__LINE__, "chown cmd: " + cmd, LOG_TYPE_DEBUG); + + //check for setup files in mysq/db, if not, create them for a new install + string file = startup::StartUp::installDir() + "/mysql/db/mysql"; + ifstream new_file (file.c_str()); + if (!new_file) { + string cmd; + cmd = startup::StartUp::installDir() + "/bin/post-mysqld-install --installdir=" + startup::StartUp::installDir() + " > /tmp/post-mysqld-install.log 2>&1"; + log.writeLog(__LINE__, "cmd: " + cmd, LOG_TYPE_DEBUG); + + int rtnCode = system(cmd.c_str()); + if (WEXITSTATUS(rtnCode) != 0) { + log.writeLog(__LINE__, "amazonVolumeCheck function failed, post-mysqld-install error" , LOG_TYPE_ERROR); + return false; + } + else + log.writeLog(__LINE__, "amazonVolumeCheck function, post-mysqld-install passed" , LOG_TYPE_DEBUG); + + cmd = startup::StartUp::installDir() + "/bin/post-mysql-install --installdir=" + startup::StartUp::installDir() + " > /tmp/post-mysql-install.log";; + log.writeLog(__LINE__, "cmd: " + cmd, LOG_TYPE_DEBUG); + rtnCode = system(cmd.c_str()); + if (WEXITSTATUS(rtnCode) != 0) { + log.writeLog(__LINE__, "amazonVolumeCheck function failed, post-mysql-install error" , LOG_TYPE_ERROR); + return false; + } + else + log.writeLog(__LINE__, "amazonVolumeCheck function, post-mysql-install passed" , LOG_TYPE_DEBUG); + } log.writeLog(__LINE__, "amazonVolumeCheck function successfully completed, volume attached: " + volumeName, LOG_TYPE_DEBUG); return true; @@ -5536,12 +5570,49 @@ bool ProcessMonitor::amazonVolumeCheck(int dbrootID) {} if (oam.attachEC2Volume(volumeName, deviceName, instanceName)) { - string cmd = "mount " + deviceName + " " + startup::StartUp::installDir() + "/mysql/db -t ext2 -o defaults > /dev/null"; + string cmd; + if ( rootUser) + cmd = "mount " + deviceName + " " + startup::StartUp::installDir() + "/mysql/db -t ext2 -o noatime,nodiratime,noauto > /tmp/um_mount.log"; + else + cmd = "sudo mount " + deviceName + " " + startup::StartUp::installDir() + "/mysql/db -t ext2 -o noatime,nodiratime,noauto,user > /tmp/um_mount.log"; + system(cmd.c_str()); log.writeLog(__LINE__, "mount cmd: " + cmd, LOG_TYPE_DEBUG); - cmd = "chown mysql:mysql -R " + startup::StartUp::installDir() + "/mysql/db"; + if ( rootUser) + cmd = "chown -R mysql:mysql " + startup::StartUp::installDir() + "/mysql/db"; + else + cmd = "sudo chown -R " + USER + ":" + USER + " " + startup::StartUp::installDir() + "/mysql/db"; + system(cmd.c_str()); + log.writeLog(__LINE__, "chown cmd: " + cmd, LOG_TYPE_DEBUG); + + //check for setup files in mysq/db, if not, create them for a new install + string file = startup::StartUp::installDir() + "/mysql/db/mysql"; + ifstream new_file (file.c_str()); + if (!new_file) { + string cmd; + cmd = startup::StartUp::installDir() + "/bin/post-mysqld-install --installdir=" + startup::StartUp::installDir() + " > /tmp/post-mysqld-install.log 2>&1"; + log.writeLog(__LINE__, "cmd: " + cmd, LOG_TYPE_DEBUG); + int rtnCode = system(cmd.c_str()); + if (WEXITSTATUS(rtnCode) != 0) { + log.writeLog(__LINE__, "amazonVolumeCheck function failed, post-mysqld-install error" , LOG_TYPE_ERROR); + return false; + } + else + log.writeLog(__LINE__, "amazonVolumeCheck function, post-mysqld-install passed" , LOG_TYPE_DEBUG); + + cmd = startup::StartUp::installDir() + "/bin/post-mysql-install --installdir=" + startup::StartUp::installDir() + " > /tmp/post-mysql-install.log";; + log.writeLog(__LINE__, "cmd: " + cmd, LOG_TYPE_DEBUG); + + rtnCode = system(cmd.c_str()); + if (WEXITSTATUS(rtnCode) != 0) { + log.writeLog(__LINE__, "amazonVolumeCheck function failed, post-mysql-install error" , LOG_TYPE_ERROR); + return false; + } + else + log.writeLog(__LINE__, "amazonVolumeCheck function, post-mysql-install passed" , LOG_TYPE_DEBUG); + } return true; } From f9e5b7b507c97f5157f33b059f93b37285f3181f Mon Sep 17 00:00:00 2001 From: David Hall Date: Fri, 3 Mar 2017 15:57:41 -0600 Subject: [PATCH 50/61] MCOL-597 Take Window Functions just to get a compile. No other changes were needed. --- dbcon/mysql/ha_calpont.cpp | 8 ---- dbcon/mysql/ha_calpont_execplan.cpp | 61 +++++++++++++++++------------ dbcon/mysql/ha_calpont_impl.cpp | 12 +----- dbcon/mysql/ha_calpont_impl.h | 1 - dbcon/mysql/ha_window_function.cpp | 13 +++--- 5 files changed, 46 insertions(+), 49 deletions(-) diff --git a/dbcon/mysql/ha_calpont.cpp b/dbcon/mysql/ha_calpont.cpp index 31b08451c..16e1e0def 100755 --- a/dbcon/mysql/ha_calpont.cpp +++ b/dbcon/mysql/ha_calpont.cpp @@ -121,7 +121,6 @@ static int calpont_commit(handlerton *hton, THD* thd, bool all); static int calpont_rollback(handlerton *hton, THD* thd, bool all); static int calpont_close_connection ( handlerton *hton, THD* thd ); -static void calpont_set_error( THD*, uint64_t, LEX_STRING*, uint32_t); handlerton *calpont_hton; /* Variables for example share methods */ @@ -219,7 +218,6 @@ static int columnstore_init_func(void *p) calpont_hton->commit= calpont_commit; calpont_hton->rollback= calpont_rollback; calpont_hton->close_connection = calpont_close_connection; - calpont_hton->set_error= calpont_set_error; DBUG_RETURN(0); } @@ -248,7 +246,6 @@ static int infinidb_init_func(void *p) calpont_hton->commit= calpont_commit; calpont_hton->rollback= calpont_rollback; calpont_hton->close_connection = calpont_close_connection; - calpont_hton->set_error= calpont_set_error; DBUG_RETURN(0); } @@ -372,11 +369,6 @@ static int calpont_close_connection ( handlerton *hton, THD* thd ) return rc; } -static void calpont_set_error(THD* thd, uint64_t errCode, LEX_STRING* args, uint32_t argCount) -{ - return ha_calpont_impl_set_error(thd, errCode, args, argCount); -} - ha_calpont::ha_calpont(handlerton *hton, TABLE_SHARE *table_arg) : handler(hton, table_arg), int_table_flags(HA_BINLOG_STMT_CAPABLE | HA_BINLOG_ROW_CAPABLE | diff --git a/dbcon/mysql/ha_calpont_execplan.cpp b/dbcon/mysql/ha_calpont_execplan.cpp index 4c2f7959b..56df504dd 100755 --- a/dbcon/mysql/ha_calpont_execplan.cpp +++ b/dbcon/mysql/ha_calpont_execplan.cpp @@ -699,11 +699,13 @@ void debug_walk(const Item *item, void *arg) cout << ": " << endl; break; } +#if 0 case Item::WINDOW_FUNC_ITEM: { cout << "Window Function Item" << endl; break; } +#endif default: { cout << "UNKNOWN_ITEM type " << item->type() << endl; @@ -2453,6 +2455,7 @@ ReturnedColumn* buildReturnedColumn(Item* item, gp_walk_info& gwi, bool& nonSupp rc = new ConstantColumn(valStr); break; } +#if 0 case Item::WINDOW_FUNC_ITEM: { return buildWindowFunctionColumn(item, gwi, nonSupport); @@ -2469,6 +2472,7 @@ ReturnedColumn* buildReturnedColumn(Item* item, gp_walk_info& gwi, bool& nonSupp rc->resultType(srcp->resultType()); break; } +#endif case Item::SUBSELECT_ITEM: { gwi.hasSubSelect = true; @@ -3500,23 +3504,24 @@ ReturnedColumn* buildAggregateColumn(Item* item, gp_walk_info& gwi) end=order_item + gc->order_field();order_item < end; order_item++) { - Item *ord_col= *(*order_item)->item; - if (ord_col->type() == Item::INT_ITEM) - { - Item_int* id = (Item_int*)ord_col; - if (id->val_int() > (int)selCols.size()) - { - gwi.fatalParseError = true; - return NULL; - } - rc = selCols[id->val_int()-1]->clone(); - rc->orderPos(id->val_int()-1); - } - else - { - rc = buildReturnedColumn(ord_col, gwi, gwi.fatalParseError); - } - rc->asc((*order_item)->asc); + Item *ord_col= *(*order_item)->item; + if (ord_col->type() == Item::INT_ITEM) + { + Item_int* id = (Item_int*)ord_col; + if (id->val_int() > (int)selCols.size()) + { + gwi.fatalParseError = true; + return NULL; + } + rc = selCols[id->val_int()-1]->clone(); + rc->orderPos(id->val_int()-1); + } + else + { + rc = buildReturnedColumn(ord_col, gwi, gwi.fatalParseError); + } + // 10.2 TODO: direction is now a tri-state flag + rc->asc((*order_item)->direction == ORDER::ORDER_ASC ? true : false); orderCols.push_back(SRCP(rc)); } @@ -4423,6 +4428,7 @@ void gp_walk(const Item *item, void *arg) gwip->rcWorkStack.push(buildReturnedColumn(itp, *gwip, gwip->fatalParseError)); break; } +#if 0 case Item::WINDOW_FUNC_ITEM: { gwip->hasWindowFunc = true; @@ -4432,6 +4438,7 @@ void gp_walk(const Item *item, void *arg) gwip->rcWorkStack.push(af); break; } +#endif case Item::COPY_STR_ITEM: printf("********** received COPY_STR_ITEM *********\n"); break; @@ -4615,9 +4622,11 @@ void parse_item (Item *item, vector& field_vec, bool& hasNonSupport setError(item->thd(), ER_CHECK_NOT_IMPLEMENTED, parseErrorText); break; } +#if 0 case Item::WINDOW_FUNC_ITEM: parseInfo |= AF_BIT; break; +#endif default: break; } @@ -5507,6 +5516,7 @@ int getSelectPlan(gp_walk_info& gwi, SELECT_LEX& select_lex, SCSEP& csep, bool i setError(gwi.thd, ER_CHECK_NOT_IMPLEMENTED, gwi.parseErrorText, gwi); return ER_CHECK_NOT_IMPLEMENTED; } +#if 0 case Item::WINDOW_FUNC_ITEM: { SRCP srcp(buildWindowFunctionColumn(item, gwi, gwi.fatalParseError)); @@ -5520,6 +5530,7 @@ int getSelectPlan(gp_walk_info& gwi, SELECT_LEX& select_lex, SCSEP& csep, bool i gwi.returnedCols.push_back(srcp); break; } +#endif default: { break; @@ -5672,6 +5683,7 @@ int getSelectPlan(gp_walk_info& gwi, SELECT_LEX& select_lex, SCSEP& csep, bool i // check if window functions are in order by. InfiniDB process order by list if // window functions are involved, either in order by or projection. +#if 0 bool hasWindowFunc = gwi.hasWindowFunc; gwi.hasWindowFunc = false; for (; groupcol; groupcol= groupcol->next) @@ -5687,7 +5699,7 @@ int getSelectPlan(gp_walk_info& gwi, SELECT_LEX& select_lex, SCSEP& csep, bool i return ER_CHECK_NOT_IMPLEMENTED; } gwi.hasWindowFunc = hasWindowFunc; - +#endif groupcol = reinterpret_cast(select_lex.group_list.first); for (; groupcol; groupcol= groupcol->next) @@ -5933,12 +5945,13 @@ int getSelectPlan(gp_walk_info& gwi, SELECT_LEX& select_lex, SCSEP& csep, bool i // check if window functions are in order by. InfiniDB process order by list if // window functions are involved, either in order by or projection. +#if 0 for (; ordercol; ordercol= ordercol->next) { if ((*(ordercol->item))->type() == Item::WINDOW_FUNC_ITEM) gwi.hasWindowFunc = true; } - +#endif // re-visit the first of ordercol list ordercol = reinterpret_cast(order_list.first); @@ -5989,7 +6002,7 @@ int getSelectPlan(gp_walk_info& gwi, SELECT_LEX& select_lex, SCSEP& csep, bool i return ER_CHECK_NOT_IMPLEMENTED; } } - if (ordercol->asc) + if (ordercol->direction == ORDER::ORDER_ASC) rc->asc(true); else rc->asc(false); @@ -6022,7 +6035,7 @@ int getSelectPlan(gp_walk_info& gwi, SELECT_LEX& select_lex, SCSEP& csep, bool i ostringstream oss; oss << ordercol->counter; ord_cols += oss.str(); - if (!ordercol->asc) + if (!ordercol->direction == ORDER::ORDER_ASC) ord_cols += " desc"; continue; } @@ -6147,7 +6160,7 @@ int getSelectPlan(gp_walk_info& gwi, SELECT_LEX& select_lex, SCSEP& csep, bool i gwi.returnedCols.push_back(srcp); ord_cols += " `" + escapeBackTick(str.c_ptr()) + "`"; } - if (!ordercol->asc) + if (!ordercol->direction == ORDER::ORDER_ASC) ord_cols += " desc"; continue; } @@ -6211,7 +6224,7 @@ int getSelectPlan(gp_walk_info& gwi, SELECT_LEX& select_lex, SCSEP& csep, bool i ord_item->print(&str, QT_INFINIDB); ord_cols += str.c_ptr(); } - if (!ordercol->asc) + if (!ordercol->direction == ORDER::ORDER_ASC) ord_cols += " desc"; } } @@ -6551,7 +6564,7 @@ int getSelectPlan(gp_walk_info& gwi, SELECT_LEX& select_lex, SCSEP& csep, bool i ord_item->print(&str, QT_INFINIDB_NO_QUOTE); ord_cols += string(str.c_ptr()); } - if (!ordercol->asc) + if (!ordercol->direction == ORDER::ORDER_ASC) ord_cols += " desc"; } } diff --git a/dbcon/mysql/ha_calpont_impl.cpp b/dbcon/mysql/ha_calpont_impl.cpp index 71300fae0..3d5f1e381 100755 --- a/dbcon/mysql/ha_calpont_impl.cpp +++ b/dbcon/mysql/ha_calpont_impl.cpp @@ -1170,7 +1170,7 @@ uint32_t doUpdateDelete(THD *thd) { multi_delete* deleteTable = (multi_delete*)((thd->lex->select_lex.join)->result); first_table= (TABLE_LIST*) deleteTable->get_tables(); - if (deleteTable->num_of_tables == 1) + if (deleteTable->get_num_of_tables() == 1) { schemaName = first_table->db; tableName = first_table->table_name; @@ -4444,15 +4444,5 @@ int ha_calpont_impl_rnd_pos(uchar *buf, uchar *pos) return ER_INTERNAL_ERROR; } -// Called from mysql parser to set IDB error for window functions -void ha_calpont_impl_set_error(THD* thd, uint64_t errCode, LEX_STRING* args, uint32_t argCount) -{ - IDEBUG( cout << "ha_calpont_impl_ser_error" << endl); - Message::Args arguments; - for (uint32_t i = 0; i < argCount; i++) - arguments.add(args[i].str); - string emsg = logging::IDBErrorInfo::instance()->errorMsg(errCode, arguments); - setError(thd, ER_INTERNAL_ERROR, emsg); -} // vim:sw=4 ts=4: diff --git a/dbcon/mysql/ha_calpont_impl.h b/dbcon/mysql/ha_calpont_impl.h index 58f680a41..79fc6933b 100644 --- a/dbcon/mysql/ha_calpont_impl.h +++ b/dbcon/mysql/ha_calpont_impl.h @@ -48,7 +48,6 @@ extern int ha_calpont_impl_external_lock(THD *thd, TABLE* table, int lock_type); extern int ha_calpont_impl_update_row(); extern int ha_calpont_impl_delete_row(); extern int ha_calpont_impl_rnd_pos(uchar *buf, uchar *pos); -extern void ha_calpont_impl_set_error(THD* thd, uint64_t errCode, LEX_STRING* args, uint32_t argCount); #endif #ifdef NEED_CALPONT_INTERFACE diff --git a/dbcon/mysql/ha_window_function.cpp b/dbcon/mysql/ha_window_function.cpp index ee91eed26..c7bed4f15 100755 --- a/dbcon/mysql/ha_window_function.cpp +++ b/dbcon/mysql/ha_window_function.cpp @@ -63,6 +63,7 @@ ReturnedColumn* nullOnError(gp_walk_info& gwi) return NULL; } +#if 0 WF_FRAME frame(BOUND& bound) { switch (bound) @@ -81,7 +82,7 @@ WF_FRAME frame(BOUND& bound) return WF_UNKNOWN; } } - +#endif ReturnedColumn* buildBoundExp(WF_Boundary& bound, SRCP& order, gp_walk_info& gwi) { if (!(gwi.thd->infinidb_vtable.cal_conn_info)) @@ -188,12 +189,13 @@ ReturnedColumn* buildWindowFunctionColumn(Item* item, gp_walk_info& gwi, bool& n cal_connection_info* ci = reinterpret_cast(gwi.thd->infinidb_vtable.cal_conn_info); gwi.hasWindowFunc = true; - Item_func_window* wf = (Item_func_window*)item; - string funcName = wf->func_name(); +// Item_func_window* wf = (Item_func_window*)item; + string funcName /*= wf->func_name()*/; WindowFunctionColumn* ac = new WindowFunctionColumn(funcName); - ac->distinct(wf->isDistinct()); - Window_context *wf_ctx = wf->window_ctx(); +// ac->distinct(wf->isDistinct()); +// Window_context *wf_ctx = wf->window_ctx(); SRCP srcp; +#if 0 // arguments vector funcParms; @@ -436,6 +438,7 @@ ReturnedColumn* buildWindowFunctionColumn(Item* item, gp_walk_info& gwi, bool& n // put ac on windowFuncList gwi.windowFuncList.push_back(ac); +#endif return ac; } From a3c4503bc9f3cc00187c3b27e7a96abda54a2968 Mon Sep 17 00:00:00 2001 From: David Hall Date: Mon, 13 Mar 2017 13:20:57 -0500 Subject: [PATCH 51/61] MCOL-597 stabilization --- dbcon/mysql/CMakeLists.txt | 10 +- dbcon/mysql/ha_calpont_execplan.cpp | 19 +- dbcon/mysql/ha_window_function.cpp | 2 +- dbcon/mysql/install_calpont_mysql.sh | 2 +- dbcon/mysql/my.cnf | 1 + dbcon/mysql/mysql.vpj | 582 ++++++++++++++++++--------- utils/funcexp/func_div.cpp | 4 +- utils/loggingcpp/ErrorMessage.txt | 2 +- 8 files changed, 415 insertions(+), 207 deletions(-) diff --git a/dbcon/mysql/CMakeLists.txt b/dbcon/mysql/CMakeLists.txt index d478843b0..4c423ed16 100644 --- a/dbcon/mysql/CMakeLists.txt +++ b/dbcon/mysql/CMakeLists.txt @@ -25,7 +25,7 @@ set_source_files_properties(ha_calpont.cpp PROPERTIES COMPILE_FLAGS "-fno-rtti - add_library(calmysql SHARED ${libcalmysql_SRCS}) -target_link_libraries(calmysql ${ENGINE_LDFLAGS} ${ENGINE_WRITE_LIBS} ${NETSNMP_LIBRARIES} threadpool) +target_link_libraries(calmysql ${ENGINE_LDFLAGS} ${ENGINE_WRITE_LIBS} ${NETSNMP_LIBRARIES} mysqlservices threadpool) set_target_properties(calmysql PROPERTIES VERSION 1.0.0 SOVERSION 1) @@ -34,7 +34,7 @@ SET ( is_columnstore_tables_SRCS ) add_library(is_columnstore_tables SHARED ${is_columnstore_tables_SRCS}) -target_link_libraries(is_columnstore_tables ${ENGINE_LDFLAGS} ${ENGINE_WRITE_LIBS} threadpool) +target_link_libraries(is_columnstore_tables ${ENGINE_LDFLAGS} ${ENGINE_WRITE_LIBS} mysqlservices threadpool) # Don't prepend .so file with 'lib' set_target_properties(is_columnstore_tables PROPERTIES PREFIX "") @@ -45,7 +45,7 @@ SET ( is_columnstore_columns_SRCS ) add_library(is_columnstore_columns SHARED ${is_columnstore_columns_SRCS}) -target_link_libraries(is_columnstore_columns ${ENGINE_LDFLAGS} ${ENGINE_WRITE_LIBS} threadpool) +target_link_libraries(is_columnstore_columns ${ENGINE_LDFLAGS} ${ENGINE_WRITE_LIBS} mysqlservices threadpool) # Don't prepend .so file with 'lib' set_target_properties(is_columnstore_columns PROPERTIES PREFIX "") @@ -56,7 +56,7 @@ SET ( is_columnstore_extents_SRCS ) add_library(is_columnstore_extents SHARED ${is_columnstore_extents_SRCS}) -target_link_libraries(is_columnstore_extents ${ENGINE_LDFLAGS} ${ENGINE_WRITE_LIBS} threadpool) +target_link_libraries(is_columnstore_extents ${ENGINE_LDFLAGS} ${ENGINE_WRITE_LIBS} mysqlservices threadpool) # Don't prepend .so file with 'lib' set_target_properties(is_columnstore_extents PROPERTIES PREFIX "") @@ -67,7 +67,7 @@ SET ( is_columnstore_files_SRCS ) add_library(is_columnstore_files SHARED ${is_columnstore_files_SRCS}) -target_link_libraries(is_columnstore_files ${ENGINE_LDFLAGS} ${ENGINE_WRITE_LIBS} threadpool) +target_link_libraries(is_columnstore_files ${ENGINE_LDFLAGS} ${ENGINE_WRITE_LIBS} mysqlservices threadpool) # Don't prepend .so file with 'lib' set_target_properties(is_columnstore_files PROPERTIES PREFIX "") diff --git a/dbcon/mysql/ha_calpont_execplan.cpp b/dbcon/mysql/ha_calpont_execplan.cpp index 56df504dd..85fe0b45f 100755 --- a/dbcon/mysql/ha_calpont_execplan.cpp +++ b/dbcon/mysql/ha_calpont_execplan.cpp @@ -423,7 +423,7 @@ void debug_walk(const Item *item, void *arg) char* item_name = item->name; if (!item_name) { - item_name = ""; + item_name = (char*)""; } switch (isp->sum_func()) { @@ -1111,6 +1111,15 @@ bool buildRowColumnFilter(gp_walk_info* gwip, RowColumn* rhs, RowColumn* lhs, It bool buildPredicateItem(Item_func* ifp, gp_walk_info* gwip) { boost::shared_ptr sop(new PredicateOperator(ifp->func_name())); + if (ifp->functype() == Item_func::LIKE_FUNC) + { + // Starting with MariaDB 10.2, LIKE uses a negated flag instead of FUNC_NOT + // Further processing is done below as before for LIKE + if (((Item_func_like*)ifp)->negated) + { + sop->reverseOp(); + } + } if (!(gwip->thd->infinidb_vtable.cal_conn_info)) gwip->thd->infinidb_vtable.cal_conn_info = (void*)(new cal_connection_info()); cal_connection_info* ci = reinterpret_cast(gwip->thd->infinidb_vtable.cal_conn_info); @@ -6035,7 +6044,7 @@ int getSelectPlan(gp_walk_info& gwi, SELECT_LEX& select_lex, SCSEP& csep, bool i ostringstream oss; oss << ordercol->counter; ord_cols += oss.str(); - if (!ordercol->direction == ORDER::ORDER_ASC) + if (ordercol->direction != ORDER::ORDER_ASC) ord_cols += " desc"; continue; } @@ -6160,7 +6169,7 @@ int getSelectPlan(gp_walk_info& gwi, SELECT_LEX& select_lex, SCSEP& csep, bool i gwi.returnedCols.push_back(srcp); ord_cols += " `" + escapeBackTick(str.c_ptr()) + "`"; } - if (!ordercol->direction == ORDER::ORDER_ASC) + if (ordercol->direction != ORDER::ORDER_ASC) ord_cols += " desc"; continue; } @@ -6224,7 +6233,7 @@ int getSelectPlan(gp_walk_info& gwi, SELECT_LEX& select_lex, SCSEP& csep, bool i ord_item->print(&str, QT_INFINIDB); ord_cols += str.c_ptr(); } - if (!ordercol->direction == ORDER::ORDER_ASC) + if (ordercol->direction != ORDER::ORDER_ASC) ord_cols += " desc"; } } @@ -6564,7 +6573,7 @@ int getSelectPlan(gp_walk_info& gwi, SELECT_LEX& select_lex, SCSEP& csep, bool i ord_item->print(&str, QT_INFINIDB_NO_QUOTE); ord_cols += string(str.c_ptr()); } - if (!ordercol->direction == ORDER::ORDER_ASC) + if (ordercol->direction != ORDER::ORDER_ASC) ord_cols += " desc"; } } diff --git a/dbcon/mysql/ha_window_function.cpp b/dbcon/mysql/ha_window_function.cpp index c7bed4f15..4106329da 100755 --- a/dbcon/mysql/ha_window_function.cpp +++ b/dbcon/mysql/ha_window_function.cpp @@ -186,7 +186,7 @@ ReturnedColumn* buildWindowFunctionColumn(Item* item, gp_walk_info& gwi, bool& n //cout << str.c_ptr() << endl; if (!(gwi.thd->infinidb_vtable.cal_conn_info)) gwi.thd->infinidb_vtable.cal_conn_info = (void*)(new cal_connection_info()); - cal_connection_info* ci = reinterpret_cast(gwi.thd->infinidb_vtable.cal_conn_info); +// cal_connection_info* ci = reinterpret_cast(gwi.thd->infinidb_vtable.cal_conn_info); gwi.hasWindowFunc = true; // Item_func_window* wf = (Item_func_window*)item; diff --git a/dbcon/mysql/install_calpont_mysql.sh b/dbcon/mysql/install_calpont_mysql.sh index 9b55f9448..611b50ef3 100755 --- a/dbcon/mysql/install_calpont_mysql.sh +++ b/dbcon/mysql/install_calpont_mysql.sh @@ -99,7 +99,7 @@ CREATE TABLE IF NOT EXISTS infinidb_querystats.querystats query varchar(8000), startTime timestamp NOT NULL, endTime timestamp NOT NULL, - rows bigint, + `rows` bigint, errno int, phyIO bigint, cacheIO bigint, diff --git a/dbcon/mysql/my.cnf b/dbcon/mysql/my.cnf index 9d9907daa..d4f595062 100644 --- a/dbcon/mysql/my.cnf +++ b/dbcon/mysql/my.cnf @@ -44,6 +44,7 @@ query_cache_size = 0 thread_stack = 512K lower_case_table_names=1 group_concat_max_len=512 +sql_mode="ERROR_FOR_DIVISION_BY_ZERO,NO_AUTO_CREATE_USER,NO_ENGINE_SUBSTITUTION" # Enable compression by default on create, set to 0 to turn off infinidb_compression_type=2 diff --git a/dbcon/mysql/mysql.vpj b/dbcon/mysql/mysql.vpj index cdb8f4f12..fd92a4562 100644 --- a/dbcon/mysql/mysql.vpj +++ b/dbcon/mysql/mysql.vpj @@ -197,6 +197,198 @@ + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + @@ -218,204 +410,210 @@ - - - - - - - - - - - - - - - - - - - - - - - - - - + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - 0 ? val1 + 0.5 : val1 - 0.5); // MCOL-176 If abs(int_val2) is small enough (like -1), then, this may cause overflow. @@ -78,7 +78,7 @@ uint64_t Func_div::getUintVal(rowgroup::Row& row, if (val2 == 0) { isNull = true; - return NULL; + return 0; } return val1 / val2; } diff --git a/utils/loggingcpp/ErrorMessage.txt b/utils/loggingcpp/ErrorMessage.txt index 1266a21e4..c4d134899 100644 --- a/utils/loggingcpp/ErrorMessage.txt +++ b/utils/loggingcpp/ErrorMessage.txt @@ -27,7 +27,7 @@ 1008 ERR_CREATE_DATATYPE_NOT_SUPPORT The syntax or the data type(s) is not supported by Columnstore. Please check the Columnstore syntax guide for supported syntax or data types. 1010 ERR_AGGREGATE_TYPE_NOT_SUPPORT '%1%' for column type '%2%' isn't supported. 1011 ERR_DML_VIEW %1% on VIEW is currently not supported. -#1012 ERR_UPDATE_NOT_SUPPORT_FEATURE This version of Columstore supports update of only one table at a time. +1012 ERR_UPDATE_NOT_SUPPORT_FEATURE This version of Columnstore supports update of only one table at a time. 1014 ERR_ROLLUP_NOT_SUPPORT Rollup is currently not supported. 1015 ERR_OUTER_JOIN_SUBSELECT Subquery on OUTER JOIN ON clause is currently not supported. 1016 ERR_PARTITION_BY_RANGE The column type %1% is currently not supported in %2% function. From d7da34c249613f269ea5b6d87afa56d4fddd2af8 Mon Sep 17 00:00:00 2001 From: David Hall Date: Tue, 21 Mar 2017 16:27:09 -0500 Subject: [PATCH 52/61] MCOL-597 Fix up Windows Functions --- dbcon/execplan/execplan.vpj | 622 +++++++++++++------------- dbcon/execplan/windowfunctioncolumn.h | 2 +- dbcon/mysql/ha_calpont_execplan.cpp | 21 +- dbcon/mysql/ha_calpont_impl.cpp | 2 +- dbcon/mysql/ha_window_function.cpp | 293 +++++++++--- dbcon/mysql/idb_mysql.h | 1 + utils/windowfunction/wf_lead_lag.cpp | 31 ++ 7 files changed, 576 insertions(+), 396 deletions(-) diff --git a/dbcon/execplan/execplan.vpj b/dbcon/execplan/execplan.vpj index 956fe3dac..6f964bc80 100644 --- a/dbcon/execplan/execplan.vpj +++ b/dbcon/execplan/execplan.vpj @@ -1,326 +1,302 @@ - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - + Version="10.0" + VendorName="SlickEdit" + TemplateName="GNU C/C++" + WorkingDir="."> + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + diff --git a/dbcon/execplan/windowfunctioncolumn.h b/dbcon/execplan/windowfunctioncolumn.h index d8b9fec75..477660b2f 100644 --- a/dbcon/execplan/windowfunctioncolumn.h +++ b/dbcon/execplan/windowfunctioncolumn.h @@ -41,7 +41,7 @@ class ByteStream; */ namespace execplan { -// This enum is made consistant with mysql item_func_window +// This enum is made consistant with mysql Item_window_func enum WF_FRAME { WF_PRECEDING = 0, diff --git a/dbcon/mysql/ha_calpont_execplan.cpp b/dbcon/mysql/ha_calpont_execplan.cpp index 85fe0b45f..93499aa4d 100755 --- a/dbcon/mysql/ha_calpont_execplan.cpp +++ b/dbcon/mysql/ha_calpont_execplan.cpp @@ -21,7 +21,7 @@ */ /** @file */ -//#define DEBUG_WALK_COND +#define DEBUG_WALK_COND #include #include #include @@ -38,7 +38,6 @@ #include #include #include -//#define NDEBUG #include #include #include @@ -699,13 +698,11 @@ void debug_walk(const Item *item, void *arg) cout << ": " << endl; break; } -#if 0 case Item::WINDOW_FUNC_ITEM: { cout << "Window Function Item" << endl; break; } -#endif default: { cout << "UNKNOWN_ITEM type " << item->type() << endl; @@ -2464,11 +2461,11 @@ ReturnedColumn* buildReturnedColumn(Item* item, gp_walk_info& gwi, bool& nonSupp rc = new ConstantColumn(valStr); break; } -#if 0 case Item::WINDOW_FUNC_ITEM: { return buildWindowFunctionColumn(item, gwi, nonSupport); } +#if INTERVAL_ITEM case Item::INTERVAL_ITEM: { Item_interval* interval = (Item_interval*)item; @@ -2481,7 +2478,7 @@ ReturnedColumn* buildReturnedColumn(Item* item, gp_walk_info& gwi, bool& nonSupp rc->resultType(srcp->resultType()); break; } -#endif +#endif case Item::SUBSELECT_ITEM: { gwi.hasSubSelect = true; @@ -4437,17 +4434,15 @@ void gp_walk(const Item *item, void *arg) gwip->rcWorkStack.push(buildReturnedColumn(itp, *gwip, gwip->fatalParseError)); break; } -#if 0 case Item::WINDOW_FUNC_ITEM: { gwip->hasWindowFunc = true; - Item_func_window* ifa = (Item_func_window*)item; + Item_window_func* ifa = (Item_window_func*)item; ReturnedColumn* af = buildWindowFunctionColumn(ifa, *gwip, gwip->fatalParseError); if (af) gwip->rcWorkStack.push(af); break; } -#endif case Item::COPY_STR_ITEM: printf("********** received COPY_STR_ITEM *********\n"); break; @@ -4631,11 +4626,9 @@ void parse_item (Item *item, vector& field_vec, bool& hasNonSupport setError(item->thd(), ER_CHECK_NOT_IMPLEMENTED, parseErrorText); break; } -#if 0 case Item::WINDOW_FUNC_ITEM: parseInfo |= AF_BIT; break; -#endif default: break; } @@ -5525,7 +5518,6 @@ int getSelectPlan(gp_walk_info& gwi, SELECT_LEX& select_lex, SCSEP& csep, bool i setError(gwi.thd, ER_CHECK_NOT_IMPLEMENTED, gwi.parseErrorText, gwi); return ER_CHECK_NOT_IMPLEMENTED; } -#if 0 case Item::WINDOW_FUNC_ITEM: { SRCP srcp(buildWindowFunctionColumn(item, gwi, gwi.fatalParseError)); @@ -5539,7 +5531,6 @@ int getSelectPlan(gp_walk_info& gwi, SELECT_LEX& select_lex, SCSEP& csep, bool i gwi.returnedCols.push_back(srcp); break; } -#endif default: { break; @@ -5692,7 +5683,6 @@ int getSelectPlan(gp_walk_info& gwi, SELECT_LEX& select_lex, SCSEP& csep, bool i // check if window functions are in order by. InfiniDB process order by list if // window functions are involved, either in order by or projection. -#if 0 bool hasWindowFunc = gwi.hasWindowFunc; gwi.hasWindowFunc = false; for (; groupcol; groupcol= groupcol->next) @@ -5708,7 +5698,6 @@ int getSelectPlan(gp_walk_info& gwi, SELECT_LEX& select_lex, SCSEP& csep, bool i return ER_CHECK_NOT_IMPLEMENTED; } gwi.hasWindowFunc = hasWindowFunc; -#endif groupcol = reinterpret_cast(select_lex.group_list.first); for (; groupcol; groupcol= groupcol->next) @@ -5954,13 +5943,11 @@ int getSelectPlan(gp_walk_info& gwi, SELECT_LEX& select_lex, SCSEP& csep, bool i // check if window functions are in order by. InfiniDB process order by list if // window functions are involved, either in order by or projection. -#if 0 for (; ordercol; ordercol= ordercol->next) { if ((*(ordercol->item))->type() == Item::WINDOW_FUNC_ITEM) gwi.hasWindowFunc = true; } -#endif // re-visit the first of ordercol list ordercol = reinterpret_cast(order_list.first); diff --git a/dbcon/mysql/ha_calpont_impl.cpp b/dbcon/mysql/ha_calpont_impl.cpp index 3d5f1e381..93ce00949 100755 --- a/dbcon/mysql/ha_calpont_impl.cpp +++ b/dbcon/mysql/ha_calpont_impl.cpp @@ -3017,7 +3017,7 @@ int ha_calpont_impl_rnd_next(uchar *buf, TABLE* table) rc = fetchNextRow(buf, ti, ci); } catch (std::exception& e) { - string emsg = string("Lost connection to ExeMgr while fetching: ") + e.what(); + string emsg = string("Error while fetching from ExeMgr: ") + e.what(); setError(thd, ER_INTERNAL_ERROR, emsg); CalpontSystemCatalog::removeCalpontSystemCatalog(tid2sid(thd->thread_id)); return ER_INTERNAL_ERROR; diff --git a/dbcon/mysql/ha_window_function.cpp b/dbcon/mysql/ha_window_function.cpp index 4106329da..33e58fff2 100755 --- a/dbcon/mysql/ha_window_function.cpp +++ b/dbcon/mysql/ha_window_function.cpp @@ -63,26 +63,26 @@ ReturnedColumn* nullOnError(gp_walk_info& gwi) return NULL; } -#if 0 -WF_FRAME frame(BOUND& bound) +WF_FRAME frame(Window_frame_bound::Bound_precedence_type bound, Item* offset) { switch (bound) { - case PRECEDING: - return WF_PRECEDING; - case FOLLOWING: - return WF_FOLLOWING; - case UNBOUNDED_PRECEDING: - return WF_UNBOUNDED_PRECEDING; - case UNBOUNDED_FOLLOWING: - return WF_UNBOUNDED_FOLLOWING; - case CURRENT_ROW: + case Window_frame_bound::PRECEDING: + if (offset) + return WF_PRECEDING; + else + return WF_UNBOUNDED_PRECEDING; + case Window_frame_bound::FOLLOWING: + if (offset) + return WF_FOLLOWING; + else + return WF_UNBOUNDED_FOLLOWING; + case Window_frame_bound::CURRENT: // Offset is meaningless return WF_CURRENT_ROW; default: return WF_UNKNOWN; } } -#endif ReturnedColumn* buildBoundExp(WF_Boundary& bound, SRCP& order, gp_walk_info& gwi) { if (!(gwi.thd->infinidb_vtable.cal_conn_info)) @@ -178,6 +178,96 @@ ReturnedColumn* buildBoundExp(WF_Boundary& bound, SRCP& order, gp_walk_info& gwi return rc; } +// Since columnstore implemented Windows Functions before MariaDB, we need +// map from the enum MariaDB uses to the string that columnstore uses to +// identify the function type. +string ConvertFuncName(Item_sum* item) +{ + switch (item->sum_func()) + { + case Item_sum::COUNT_FUNC: + if (!item->arguments()[0]->name) + return "COUNT(*)"; + return "COUNT"; + break; + case Item_sum::COUNT_DISTINCT_FUNC: + return "COUNT_DISTINCT"; + break; + case Item_sum::SUM_FUNC: + return "SUM"; + break; + case Item_sum::SUM_DISTINCT_FUNC: + return "SUM_DISTINCT"; + break; + case Item_sum::AVG_FUNC: + return "AVG"; + break; + case Item_sum::AVG_DISTINCT_FUNC: + return "AVG_DISTINCT"; + break; + case Item_sum::MIN_FUNC: + return "MIN"; + break; + case Item_sum::MAX_FUNC: + return "MAX"; + break; + case Item_sum::STD_FUNC: + return "STDDEV_POP"; + break; + case Item_sum::VARIANCE_FUNC: + return "VAR_POP"; + break; + case Item_sum::SUM_BIT_FUNC: + if (strcmp(item->func_name(), "bit_or(") == 0) + return "BIT_OR"; + if (strcmp(item->func_name(), "bit_and(") == 0) + return "BIT_AND"; + if (strcmp(item->func_name(), "bit_xor(") == 0) + return "BIT_XOR"; + break; + case Item_sum::UDF_SUM_FUNC: + return "UDF_SUM_FUNC"; // Not supported + break; + case Item_sum::GROUP_CONCAT_FUNC: + return "GROUP_CONCAT"; // Not supported + break; + case Item_sum::ROW_NUMBER_FUNC: + return "ROW_NUMBER"; + break; + case Item_sum::RANK_FUNC: + return "RANK"; + break; + case Item_sum::DENSE_RANK_FUNC: + return "DENSE_RANK"; + break; + case Item_sum::PERCENT_RANK_FUNC: + return "PERCENT_RANK"; + break; + case Item_sum::CUME_DIST_FUNC: + return "CUME_DIST"; + break; + case Item_sum::NTILE_FUNC: + return "NTILE"; + break; + case Item_sum::FIRST_VALUE_FUNC: + return "FIRST_VALUE"; + break; + case Item_sum::LAST_VALUE_FUNC: + return "LAST_VALUE"; + break; + case Item_sum::NTH_VALUE_FUNC: + return "NTH_VALUE"; + break; + case Item_sum::LEAD_FUNC: + return "LEAD"; + break; + case Item_sum::LAG_FUNC: + return "LAG"; + break; + }; + return ""; +} + ReturnedColumn* buildWindowFunctionColumn(Item* item, gp_walk_info& gwi, bool& nonSupport) { //@todo fix print for create view @@ -186,37 +276,82 @@ ReturnedColumn* buildWindowFunctionColumn(Item* item, gp_walk_info& gwi, bool& n //cout << str.c_ptr() << endl; if (!(gwi.thd->infinidb_vtable.cal_conn_info)) gwi.thd->infinidb_vtable.cal_conn_info = (void*)(new cal_connection_info()); -// cal_connection_info* ci = reinterpret_cast(gwi.thd->infinidb_vtable.cal_conn_info); + cal_connection_info* ci = reinterpret_cast(gwi.thd->infinidb_vtable.cal_conn_info); gwi.hasWindowFunc = true; -// Item_func_window* wf = (Item_func_window*)item; - string funcName /*= wf->func_name()*/; + Item_window_func* wf = (Item_window_func*)item; + string funcName = ConvertFuncName(wf->window_func()); WindowFunctionColumn* ac = new WindowFunctionColumn(funcName); -// ac->distinct(wf->isDistinct()); -// Window_context *wf_ctx = wf->window_ctx(); + ac->distinct(wf->window_func()->has_with_distinct()); + Window_spec *win_spec = wf->window_spec; SRCP srcp; -#if 0 + // arguments + vector funcParms; + Item_sum* item_sum = (Item_sum*)wf->arguments()[0]; + for (uint32_t i = 0; i < item_sum->argument_count(); i++) + { + srcp.reset(buildReturnedColumn((item_sum->arguments()[i]), gwi, nonSupport)); + if (!srcp) + return nullOnError(gwi); + funcParms.push_back(srcp); + if (gwi.clauseType == WHERE && !gwi.rcWorkStack.empty()) + gwi.rcWorkStack.pop(); + } + // Some functions, such as LEAD/LAG don't have all parameters implemented in the + // front end. Add dummies here to make the backend use defaults. + // Some of these will be temporary until they are implemented in the front end. + // Others need to stay because the back end expects them, but the front end + // no longer sends them. + // This case is kept in enum order in hopes the compiler can optimize + switch (wf->window_func()->sum_func()) + { + case Item_sum::COUNT_FUNC: + case Item_sum::COUNT_DISTINCT_FUNC: + break; + case Item_sum::FIRST_VALUE_FUNC: + srcp.reset(new ConstantColumn("1", (uint64_t)1, ConstantColumn::NUM)); // OFFSET (always one) + funcParms.push_back(srcp); + srcp.reset(new ConstantColumn("1", (uint64_t)1, ConstantColumn::NUM)); // FROM_FIRST + funcParms.push_back(srcp); + srcp.reset(new ConstantColumn("1", (uint64_t)1, ConstantColumn::NUM)); // IGNORE/RESPECT NULLS. 1 => RESPECT + funcParms.push_back(srcp); + break; + case Item_sum::LAST_VALUE_FUNC: + srcp.reset(new ConstantColumn("1", (uint64_t)1, ConstantColumn::NUM)); // OFFSET (always one) + funcParms.push_back(srcp); + srcp.reset(new ConstantColumn("0", (uint64_t)0, ConstantColumn::NUM)); // FROM_LAST + funcParms.push_back(srcp); + srcp.reset(new ConstantColumn("1", (uint64_t)1, ConstantColumn::NUM)); // IGNORE/RESPECT NULLS. 1 => RESPECT + funcParms.push_back(srcp); + break; + case Item_sum::NTH_VALUE_FUNC: + // When the front end supports these paramters, this needs modification + srcp.reset(new ConstantColumn("1", (uint64_t)1, ConstantColumn::NUM)); // FROM FIRST/LAST 1 => FIRST + funcParms.push_back(srcp); + srcp.reset(new ConstantColumn("1", (uint64_t)1, ConstantColumn::NUM)); // IGNORE/RESPECT NULLS. 1 => RESPECT + funcParms.push_back(srcp); + break; + case Item_sum::LEAD_FUNC: + case Item_sum::LAG_FUNC: + // When the front end supports these paramters, this needs modification + srcp.reset(new ConstantColumn("", ConstantColumn::NULLDATA)); // Default to fill in for NULL values + funcParms.push_back(srcp); + srcp.reset(new ConstantColumn("1", (uint64_t)1, ConstantColumn::NUM)); // IGNORE/RESPECT NULLS. 1 => RESPECT + funcParms.push_back(srcp); + break; + default: + break; + }; - // arguments - vector funcParms; - for (uint32_t i = 0; i < wf->argument_count(); i++) - { - srcp.reset(buildReturnedColumn(wf->arguments()[i], gwi, nonSupport)); - if (!srcp) - return nullOnError(gwi); - funcParms.push_back(srcp); - if (gwi.clauseType == WHERE && !gwi.rcWorkStack.empty()) - gwi.rcWorkStack.pop(); - } - ac->functionParms(funcParms); + ac->functionParms(funcParms); // Partition by - if (wf_ctx) + if (win_spec) { vector partitions; - for (uint32_t i = 0; i < wf_ctx->partition_count; i++) + for (ORDER *ord= win_spec->partition_list->first; ord; ord=ord->next) { - srcp.reset(buildReturnedColumn(wf_ctx->partitions[i], gwi, nonSupport)); + srcp.reset(buildReturnedColumn(*ord->item, gwi, nonSupport)); if (!srcp) return nullOnError(gwi); partitions.push_back(srcp); @@ -224,40 +359,42 @@ ReturnedColumn* buildWindowFunctionColumn(Item* item, gp_walk_info& gwi, bool& n ac->partitions(partitions); // Order by - if (wf_ctx->ordering) + if (win_spec->order_list) { WF_OrderBy orderBy; // order columns - if (wf_ctx->ordering->orders) + if (win_spec->order_list) { vector orders; - ORDER* orderCol = reinterpret_cast(wf_ctx->ordering->orders->first); + ORDER* orderCol = reinterpret_cast(win_spec->order_list->first); for (; orderCol; orderCol= orderCol->next) { Item* orderItem = *(orderCol->item); srcp.reset(buildReturnedColumn(orderItem, gwi, nonSupport)); if (!srcp) return nullOnError(gwi); - srcp->asc(orderCol->asc); - srcp->nullsFirst(orderCol->nulls); // nulls 1-nulls first 0-nulls last - orders.push_back(srcp); + srcp->asc(orderCol->direction == ORDER::ORDER_ASC ? true : false); +// srcp->nullsFirst(orderCol->nulls); // nulls 2-default, 1-nulls first, 0-nulls last + srcp->nullsFirst(1); // WINDOWS TODO: implement NULLS FIRST/LAST in 10.2 front end + orders.push_back(srcp); } orderBy.fOrders = orders; } // window frame WF_Frame frm; - if (wf_ctx->ordering->frame) + if (win_spec->window_frame) { - frm.fIsRange = wf_ctx->ordering->frame->isRange; + frm.fIsRange = win_spec->window_frame->units == Window_frame::UNITS_RANGE; // start - if (wf_ctx->ordering->frame->start) + if (win_spec->window_frame->top_bound) { - frm.fStart.fFrame = frame(wf_ctx->ordering->frame->start->bound); + frm.fStart.fFrame = frame(win_spec->window_frame->top_bound->precedence_type, + win_spec->window_frame->top_bound->offset); // offset NULL means UNBOUNDED - if (wf_ctx->ordering->frame->start->item) + if (win_spec->window_frame->top_bound->offset) { - frm.fStart.fVal.reset(buildReturnedColumn(wf_ctx->ordering->frame->start->item, gwi, nonSupport)); + frm.fStart.fVal.reset(buildReturnedColumn(win_spec->window_frame->top_bound->offset, gwi, nonSupport)); if (!frm.fStart.fVal) return nullOnError(gwi); @@ -295,12 +432,13 @@ ReturnedColumn* buildWindowFunctionColumn(Item* item, gp_walk_info& gwi, bool& n } // end - if (wf_ctx->ordering->frame->end) + if (win_spec->window_frame->bottom_bound) { - frm.fEnd.fFrame = frame(wf_ctx->ordering->frame->end->bound); - if (wf_ctx->ordering->frame->end->item) + frm.fEnd.fFrame = frame(win_spec->window_frame->bottom_bound->precedence_type, + win_spec->window_frame->bottom_bound->offset); + if (win_spec->window_frame->bottom_bound->offset) { - frm.fEnd.fVal.reset(buildReturnedColumn(wf_ctx->ordering->frame->end->item, gwi, nonSupport)); + frm.fEnd.fVal.reset(buildReturnedColumn(win_spec->window_frame->bottom_bound->offset, gwi, nonSupport)); if (!frm.fEnd.fVal) return nullOnError(gwi); @@ -409,8 +547,57 @@ ReturnedColumn* buildWindowFunctionColumn(Item* item, gp_walk_info& gwi, bool& n } else { - frm.fStart.fFrame = WF_UNBOUNDED_PRECEDING; - frm.fEnd.fFrame = WF_CURRENT_ROW; + // Certain function types have different default boundaries + // This case is kept in enum order in hopes the compiler can optimize + switch (wf->window_func()->sum_func()) + { + case Item_sum::COUNT_FUNC: + case Item_sum::COUNT_DISTINCT_FUNC: + case Item_sum::SUM_FUNC: + case Item_sum::SUM_DISTINCT_FUNC: + case Item_sum::AVG_FUNC: + case Item_sum::AVG_DISTINCT_FUNC: + case Item_sum::MIN_FUNC: + case Item_sum::MAX_FUNC: + case Item_sum::STD_FUNC: + case Item_sum::VARIANCE_FUNC: + case Item_sum::SUM_BIT_FUNC: + case Item_sum::UDF_SUM_FUNC: + case Item_sum::GROUP_CONCAT_FUNC: + frm.fStart.fFrame = WF_UNBOUNDED_PRECEDING; + frm.fEnd.fFrame = WF_CURRENT_ROW; + break; + case Item_sum::ROW_NUMBER_FUNC: + case Item_sum::RANK_FUNC: + frm.fStart.fFrame = WF_UNBOUNDED_PRECEDING; + frm.fEnd.fFrame = WF_UNBOUNDED_FOLLOWING; + break; + case Item_sum::DENSE_RANK_FUNC: + case Item_sum::PERCENT_RANK_FUNC: + case Item_sum::CUME_DIST_FUNC: + frm.fStart.fFrame = WF_UNBOUNDED_PRECEDING; + frm.fEnd.fFrame = WF_CURRENT_ROW; + break; + case Item_sum::NTILE_FUNC: + frm.fStart.fFrame = WF_UNBOUNDED_PRECEDING; + frm.fEnd.fFrame = WF_UNBOUNDED_FOLLOWING; + break; + case Item_sum::FIRST_VALUE_FUNC: + case Item_sum::LAST_VALUE_FUNC: + case Item_sum::NTH_VALUE_FUNC: + frm.fStart.fFrame = WF_UNBOUNDED_PRECEDING; + frm.fEnd.fFrame = WF_CURRENT_ROW; + break; + case Item_sum::LEAD_FUNC: + case Item_sum::LAG_FUNC: + frm.fStart.fFrame = WF_UNBOUNDED_PRECEDING; + frm.fEnd.fFrame = WF_UNBOUNDED_FOLLOWING; + break; + default: + frm.fStart.fFrame = WF_UNBOUNDED_PRECEDING; + frm.fEnd.fFrame = WF_CURRENT_ROW; + break; + }; } orderBy.fFrame = frm; @@ -426,8 +613,7 @@ ReturnedColumn* buildWindowFunctionColumn(Item* item, gp_walk_info& gwi, bool& n return NULL; } - ac->resultType(colType_MysqlToIDB(wf)); - + ac->resultType(colType_MysqlToIDB(wf->arguments()[0])); // bug5736. Make the result type double for some window functions when // infinidb_double_for_decimal_math is set. ac->adjustResultType(); @@ -438,7 +624,6 @@ ReturnedColumn* buildWindowFunctionColumn(Item* item, gp_walk_info& gwi, bool& n // put ac on windowFuncList gwi.windowFuncList.push_back(ac); -#endif return ac; } diff --git a/dbcon/mysql/idb_mysql.h b/dbcon/mysql/idb_mysql.h index f8eb5d5c0..670f0e0e0 100644 --- a/dbcon/mysql/idb_mysql.h +++ b/dbcon/mysql/idb_mysql.h @@ -65,6 +65,7 @@ template bool isnan(T); #include "sql_table.h" #include "sql_select.h" +#include "item_windowfunc.h" // Now clean up the pollution as best we can... #undef min diff --git a/utils/windowfunction/wf_lead_lag.cpp b/utils/windowfunction/wf_lead_lag.cpp index 1ab9818f3..8d3b6169b 100644 --- a/utils/windowfunction/wf_lead_lag.cpp +++ b/utils/windowfunction/wf_lead_lag.cpp @@ -121,6 +121,11 @@ void WF_lead_lag::parseParms(const std::vector& parms) { // lead | lag fLead = 1; + fRespectNulls = true; + fDefNull = false; + fDefault = (T)0; + fOffsetNull = false; + fOffset = 0; if (fFunctionId == WF__LAG) fLead = -1; @@ -148,6 +153,32 @@ void WF_lead_lag::parseParms(const std::vector& parms) idbassert(cc != NULL); bool isNull = false; // dummy, harded coded fRespectNulls = (cc->getIntVal(fRow, isNull) > 0); +#if 0 + // parms[1]: offset + for (std::vector::size_type i = 1/*ignore 0th element*/; i < parms.size(); ++i) + { + ConstantColumn* cc = dynamic_cast(parms[i].get()); + if (cc != NULL) + { + fOffset = cc->getIntVal(fRow, fOffsetNull) * fLead; // row not used, no need to setData. + continue; + } + + cc = dynamic_cast(parms[i].get()); + if (cc != NULL) + { + getConstValue(cc, fDefault, fDefNull); + continue; + } + // IGNORE/RESPECT nulls is currently broken in the front end + cc = dynamic_cast(parms[i].get()); + if (cc != NULL) + { + bool isNull = false; // dummy, harded coded + fRespectNulls = (cc->getIntVal(fRow, isNull) > 0); + } + } +#endif } From d531796bf28b5df55d6989d34c47b3272ea48194 Mon Sep 17 00:00:00 2001 From: David Hall Date: Tue, 28 Mar 2017 13:03:28 -0500 Subject: [PATCH 53/61] MCOL-597 Fix up Windows Functions --- dbcon/joblist/CMakeLists.txt | 2 +- dbcon/mysql/ha_calpont_execplan.cpp | 48 ++++++++++++++++++------- dbcon/mysql/ha_window_function.cpp | 27 ++++++++------ oam/install_scripts/post-mysqld-install | 10 +++--- utils/querystats/CMakeLists.txt | 2 -- versioning/BRM/CMakeLists.txt | 8 ++--- 6 files changed, 61 insertions(+), 36 deletions(-) diff --git a/dbcon/joblist/CMakeLists.txt b/dbcon/joblist/CMakeLists.txt index 7fac3a859..0a7880413 100644 --- a/dbcon/joblist/CMakeLists.txt +++ b/dbcon/joblist/CMakeLists.txt @@ -61,7 +61,7 @@ set(joblist_LIB_SRCS add_library(joblist SHARED ${joblist_LIB_SRCS}) -target_link_libraries(joblist ${NETSNMP_LIBRARIES} -L${SERVER_BUILD_INCLUDE_DIR}/../libmysql/ libmysqlclient_r.so) +target_link_libraries(joblist -L${SERVER_BUILD_INCLUDE_DIR}/../libmariadb/libmariadb/ libmariadb.so) set_target_properties(joblist PROPERTIES VERSION 1.0.0 SOVERSION 1) diff --git a/dbcon/mysql/ha_calpont_execplan.cpp b/dbcon/mysql/ha_calpont_execplan.cpp index 93499aa4d..5180e03ae 100755 --- a/dbcon/mysql/ha_calpont_execplan.cpp +++ b/dbcon/mysql/ha_calpont_execplan.cpp @@ -496,6 +496,11 @@ void debug_walk(const Item *item, void *arg) '.' << ifp->field_name << endl; break; } + else if (field->type() == Item::FUNC_ITEM) + { + Item_func* ifp = (Item_func*)field; + cout << "CACHED REF FUNC_ITEM " << ifp->func_name() << endl; + } else if (field->type() == Item::REF_ITEM) { Item_ref* ifr = (Item_ref*)field; @@ -563,7 +568,20 @@ void debug_walk(const Item *item, void *arg) ifp->field_name << endl; break; } - cout << "UNKNOWN REF ITEM type " << ref->real_item()->type() << endl; + else if (ref->real_item()->type() == Item::FUNC_ITEM) + { + Item_func* ifp = (Item_func*)ref->real_item(); + cout << "REF FUNC_ITEM " << ifp->func_name() << endl; + } + else if (ref->real_item()->type() == Item::WINDOW_FUNC_ITEM) + { + Item_window_func* ifp = (Item_window_func*)ref->real_item(); + cout << "REF WINDOW_FUNC_ITEM " << ifp->window_func()->func_name() << endl; + } + else + { + cout << "UNKNOWN REF ITEM type " << ref->real_item()->type() << endl; + } break; } case Item::ROW_ITEM: @@ -700,7 +718,8 @@ void debug_walk(const Item *item, void *arg) } case Item::WINDOW_FUNC_ITEM: { - cout << "Window Function Item" << endl; + Item_window_func* ifp = (Item_window_func*)item; + cout << "Window Function Item " << ifp->window_func()->func_name() << endl; break; } default: @@ -2407,18 +2426,19 @@ ReturnedColumn* buildReturnedColumn(Item* item, gp_walk_info& gwi, bool& nonSupp case Item::REF_ITEM: { Item_ref* ref = (Item_ref*)item; - if ((*(ref->ref))->type() == Item::SUM_FUNC_ITEM) - { + switch ((*(ref->ref))->type()) + { + case Item::SUM_FUNC_ITEM: return buildAggregateColumn(*(ref->ref), gwi); - } - else if ((*(ref->ref))->type() == Item::FIELD_ITEM) + case Item::FIELD_ITEM: return buildReturnedColumn(*(ref->ref), gwi, nonSupport); - else if ((*(ref->ref))->type() == Item::REF_ITEM) + case Item::REF_ITEM: return buildReturnedColumn(*(((Item_ref*)(*(ref->ref)))->ref), gwi, nonSupport); - else if ((*(ref->ref))->type() == Item::FUNC_ITEM) + case Item::FUNC_ITEM: return buildFunctionColumn((Item_func*)(*(ref->ref)), gwi, nonSupport); - else - { + case Item::WINDOW_FUNC_ITEM: + return buildWindowFunctionColumn(*(ref->ref), gwi, nonSupport); + default: gwi.fatalParseError = true; gwi.parseErrorText = "Unknown REF item"; break; @@ -4390,8 +4410,6 @@ void gp_walk(const Item *item, void *arg) gwip->subQuery = orig; gwip->lastSub = existsSub; } - -#if MYSQL_VERSION_ID >= 50172 else if (sub->substype() == Item_subselect::IN_SUBS) { if (!((Item_in_subselect*)sub)->getOptimizer() && gwip->thd->derived_tables_processing) @@ -4403,7 +4421,6 @@ void gp_walk(const Item *item, void *arg) break; } } -#endif // store a dummy subselect object. the transform is handled in item_func. SubSelect *subselect = new SubSelect(); gwip->rcWorkStack.push(subselect); @@ -4595,6 +4612,11 @@ void parse_item (Item *item, vector& field_vec, bool& hasNonSupport item = (*(ref->ref)); continue; } + else if ((*(ref->ref))->type() == Item::WINDOW_FUNC_ITEM) + { + parseInfo |= AF_BIT; + break; + } else { cout << "UNKNOWN REF Item" << endl; diff --git a/dbcon/mysql/ha_window_function.cpp b/dbcon/mysql/ha_window_function.cpp index 33e58fff2..7361dfd51 100755 --- a/dbcon/mysql/ha_window_function.cpp +++ b/dbcon/mysql/ha_window_function.cpp @@ -212,10 +212,16 @@ string ConvertFuncName(Item_sum* item) return "MAX"; break; case Item_sum::STD_FUNC: - return "STDDEV_POP"; + if (((Item_sum_variance*)item)->sample) + return "STDDEV_SAMP"; + else + return "STDDEV_POP"; break; - case Item_sum::VARIANCE_FUNC: - return "VAR_POP"; + case Item_sum::VARIANCE_FUNC: + if (((Item_sum_variance*)item)->sample) + return "VAR_SAMP"; + else + return "VAR_POP"; break; case Item_sum::SUM_BIT_FUNC: if (strcmp(item->func_name(), "bit_or(") == 0) @@ -375,7 +381,7 @@ ReturnedColumn* buildWindowFunctionColumn(Item* item, gp_walk_info& gwi, bool& n return nullOnError(gwi); srcp->asc(orderCol->direction == ORDER::ORDER_ASC ? true : false); // srcp->nullsFirst(orderCol->nulls); // nulls 2-default, 1-nulls first, 0-nulls last - srcp->nullsFirst(1); // WINDOWS TODO: implement NULLS FIRST/LAST in 10.2 front end + srcp->nullsFirst(orderCol->direction == ORDER::ORDER_ASC ? 1 : 0); // WINDOWS TODO: implement NULLS FIRST/LAST in 10.2 front end orders.push_back(srcp); } orderBy.fOrders = orders; @@ -557,8 +563,15 @@ ReturnedColumn* buildWindowFunctionColumn(Item* item, gp_walk_info& gwi, bool& n case Item_sum::SUM_DISTINCT_FUNC: case Item_sum::AVG_FUNC: case Item_sum::AVG_DISTINCT_FUNC: + frm.fStart.fFrame = WF_UNBOUNDED_PRECEDING; + frm.fEnd.fFrame = WF_CURRENT_ROW; + break; case Item_sum::MIN_FUNC: case Item_sum::MAX_FUNC: + frm.fStart.fFrame = WF_UNBOUNDED_PRECEDING; +// frm.fEnd.fFrame = WF_UNBOUNDED_FOLLOWING; + frm.fEnd.fFrame = WF_CURRENT_ROW; + break; case Item_sum::STD_FUNC: case Item_sum::VARIANCE_FUNC: case Item_sum::SUM_BIT_FUNC: @@ -569,15 +582,9 @@ ReturnedColumn* buildWindowFunctionColumn(Item* item, gp_walk_info& gwi, bool& n break; case Item_sum::ROW_NUMBER_FUNC: case Item_sum::RANK_FUNC: - frm.fStart.fFrame = WF_UNBOUNDED_PRECEDING; - frm.fEnd.fFrame = WF_UNBOUNDED_FOLLOWING; - break; case Item_sum::DENSE_RANK_FUNC: case Item_sum::PERCENT_RANK_FUNC: case Item_sum::CUME_DIST_FUNC: - frm.fStart.fFrame = WF_UNBOUNDED_PRECEDING; - frm.fEnd.fFrame = WF_CURRENT_ROW; - break; case Item_sum::NTILE_FUNC: frm.fStart.fFrame = WF_UNBOUNDED_PRECEDING; frm.fEnd.fFrame = WF_UNBOUNDED_FOLLOWING; diff --git a/oam/install_scripts/post-mysqld-install b/oam/install_scripts/post-mysqld-install index 546945b22..7fa634c54 100755 --- a/oam/install_scripts/post-mysqld-install +++ b/oam/install_scripts/post-mysqld-install @@ -28,11 +28,11 @@ for arg in "$@"; do fi done -cd $installdir/mysql/lib -for file in libmysqlclient libmysqlclient_r; do - ln -sf $file.so.18.0.0 $file.so - ln -sf $file.so.18.0.0 $file.so.18 -done +#cd $installdir/mysql/lib +#for file in libmysqlclient libmysqlclient_r; do +# ln -sf $file.so.18.0.0 $file.so +# ln -sf $file.so.18.0.0 $file.so.18 +#done cd $installdir/mysql/lib/plugin for file in ha_archive ha_blackhole ha_federated ha_innodb; do ln -sf $file.so $file.so.0 diff --git a/utils/querystats/CMakeLists.txt b/utils/querystats/CMakeLists.txt index 935898818..940749a6b 100644 --- a/utils/querystats/CMakeLists.txt +++ b/utils/querystats/CMakeLists.txt @@ -8,8 +8,6 @@ set(querystats_LIB_SRCS querystats.cpp) add_library(querystats SHARED ${querystats_LIB_SRCS}) -target_link_libraries(querystats -L${SERVER_BUILD_INCLUDE_DIR}/../libmysql/ libmysqlclient_r.so) - set_target_properties(querystats PROPERTIES VERSION 1.0.0 SOVERSION 1) install(TARGETS querystats DESTINATION ${ENGINE_LIBDIR} COMPONENT libs) diff --git a/versioning/BRM/CMakeLists.txt b/versioning/BRM/CMakeLists.txt index 2ce60254d..de392d6b1 100644 --- a/versioning/BRM/CMakeLists.txt +++ b/versioning/BRM/CMakeLists.txt @@ -32,8 +32,6 @@ set(brm_LIB_SRCS add_library(brm SHARED ${brm_LIB_SRCS}) -target_link_libraries(brm ${NETSNMP_LIBRARIES}) - set_target_properties(brm PROPERTIES VERSION 1.0.0 SOVERSION 1) install(TARGETS brm DESTINATION ${ENGINE_LIBDIR} COMPONENT libs) @@ -45,7 +43,7 @@ set(controllernode_SRCS masternode.cpp masterdbrmnode.cpp) add_executable(controllernode ${controllernode_SRCS}) -target_link_libraries(controllernode ${ENGINE_LDFLAGS} ${ENGINE_OAM_LIBS} ${ENGINE_EXEC_LIBS} ${NETSNMP_LIBRARIES}) +target_link_libraries(controllernode ${ENGINE_LDFLAGS} ${ENGINE_OAM_LIBS} ${ENGINE_EXEC_LIBS} ) install(TARGETS controllernode DESTINATION ${ENGINE_BINDIR} COMPONENT platform) @@ -56,7 +54,7 @@ set(workernode_SRCS slavenode.cpp) add_executable(workernode ${workernode_SRCS}) -target_link_libraries(workernode ${ENGINE_LDFLAGS} ${ENGINE_OAM_LIBS} ${ENGINE_EXEC_LIBS} ${NETSNMP_LIBRARIES}) +target_link_libraries(workernode ${ENGINE_LDFLAGS} ${ENGINE_OAM_LIBS} ${ENGINE_EXEC_LIBS}) install(TARGETS workernode DESTINATION ${ENGINE_BINDIR} COMPONENT platform) @@ -67,7 +65,7 @@ set(dbrmctl_SRCS dbrmctl.cpp) add_executable(dbrmctl ${dbrmctl_SRCS}) -target_link_libraries(dbrmctl ${ENGINE_LDFLAGS} ${ENGINE_OAM_LIBS} ${ENGINE_EXEC_LIBS} ${NETSNMP_LIBRARIES}) +target_link_libraries(dbrmctl ${ENGINE_LDFLAGS} ${ENGINE_OAM_LIBS} ${ENGINE_EXEC_LIBS}) install(TARGETS dbrmctl DESTINATION ${ENGINE_BINDIR} COMPONENT platform) From 587593575c04b8977cbe2fd1818939836e0e8ccc Mon Sep 17 00:00:00 2001 From: david hill Date: Tue, 23 Aug 2016 15:59:13 -0500 Subject: [PATCH 54/61] change readme --- README.md | 10 ++++------ 1 file changed, 4 insertions(+), 6 deletions(-) diff --git a/README.md b/README.md index 62a8a218d..5df193e89 100644 --- a/README.md +++ b/README.md @@ -1,9 +1,9 @@ -#MariaDB ColumnStore Storage/Execution engine 1.0.6 -MariaDB ColumnStore 1.0.6 is the development version of MariaDB ColumnStore. -It is built by porting InfiniDB 4.6.2 on MariaDB 10.1.19 and adding entirely +#MariaDB ColumnStore Storage/Execution engine 1.0.1 +MariaDB ColumnStore 1.0.2 is the development version of MariaDB ColumnStore. +It is built by porting InfiniDB 4.6.2 on MariaDB 10.1.14 and adding entirely new features not found anywhere else. -#MariaDB ColumnStore 1.0.6 is an GA release. +#MariaDB ColumnStore 1.0.2 is an Alpha release. This is the first MariaDB ColumnStore release, not all features planned for the MariaDB ColumnStore 1.0 series are included in this release. @@ -16,5 +16,3 @@ This repository is not meant to be built independently outside of the server. T https://github.com/mariadb-corporation/mariadb-columnstore-server -#Issue tracking -Issue tracking of MariaDB ColumnStore happens in JIRA, https://jira.mariadb.org/browse/MCOL From 67d3a801bb0ee780401cf070b60df6c2cc11b0e9 Mon Sep 17 00:00:00 2001 From: david hill Date: Wed, 18 Jan 2017 14:32:54 -0600 Subject: [PATCH 55/61] mcol-481 - just changed the note --- oamapps/postConfigure/postConfigure.cpp | 138 ++++++++++++------------ 1 file changed, 69 insertions(+), 69 deletions(-) diff --git a/oamapps/postConfigure/postConfigure.cpp b/oamapps/postConfigure/postConfigure.cpp index 1facf1a7c..f8e66311a 100644 --- a/oamapps/postConfigure/postConfigure.cpp +++ b/oamapps/postConfigure/postConfigure.cpp @@ -666,6 +666,75 @@ int main(int argc, char *argv[]) exit(1); } + cout << "NOTE: The MariaDB ColumnStore Schema Sync feature will replicate all of the" << endl; + cout << " schemas and InnoDB tables across the User Module nodes. This feature can be enabled" << endl; + cout << " or disabled, for example, if you wish to configure your own replication post installation." << endl << endl; + + try { + MySQLRep = sysConfig->getConfig(InstallSection, "MySQLRep"); + } + catch(...) + {} + + if ( MySQLRep == "y" ) + mysqlRep = true; + + string answer = "y"; + + while(true) { + if ( mysqlRep ) + prompt = "MariaDB ColumnStore Schema Sync feature is Enabled, do you want to leave enabled? [y,n] (y) > "; + else + prompt = "MariaDB ColumnStore Schema Sync feature, do you want to enable? [y,n] (y) > "; + + pcommand = callReadline(prompt.c_str()); + if (pcommand) { + if (strlen(pcommand) > 0) answer = pcommand; + callFree(pcommand); + } + + if ( answer == "y" || answer == "n" ) { + cout << endl; + break; + } + else + cout << "Invalid Entry, please enter 'y' for yes or 'n' for no" << endl; + if ( noPrompting ) + exit(1); + } + + if ( mysqlRep ) + { // current enabled + if ( answer == "y" ) { + mysqlRep = true; + MySQLRep = "y"; + } + else + { + mysqlRep = false; + MySQLRep = "n"; + } + } + else + { // currently disabled + if ( answer == "y" ) { + mysqlRep = false; + MySQLRep = "n"; + } + else + { + mysqlRep = true; + MySQLRep = "y"; + } + } + + try { + sysConfig->setConfig(InstallSection, "MySQLRep", MySQLRep); + } + catch(...) + {} + + switch ( IserverTypeInstall ) { case (oam::INSTALL_COMBINE_DM_UM_PM): // combined #1 - dm/um/pm on a single server { @@ -755,75 +824,6 @@ int main(int argc, char *argv[]) break; } - // check for Schema Schema is Local Query wasnt selected - if (!pmwithum) - { - cout << "NOTE: The MariaDB ColumnStore Schema Sync feature will replicate all of the" << endl; - cout << " schemas and InnoDB tables across the User Module nodes. This feature can be enabled" << endl; - cout << " or disabled, for example, if you wish to configure your own replication post installation." << endl << endl; - - try { - MySQLRep = sysConfig->getConfig(InstallSection, "MySQLRep"); - } - catch(...) - {} - - if ( MySQLRep == "y" ) - mysqlRep = true; - - string answer = "y"; - - while(true) { - if ( mysqlRep ) - prompt = "MariaDB ColumnStore Schema Sync feature is Enabled, do you want to leave enabled? [y,n] (y) > "; - else - prompt = "MariaDB ColumnStore Schema Sync feature, do you want to enable? [y,n] (y) > "; - - pcommand = callReadline(prompt.c_str()); - if (pcommand) { - if (strlen(pcommand) > 0) answer = pcommand; - callFree(pcommand); - } - - if ( answer == "y" || answer == "n" ) { - cout << endl; - break; - } - else - cout << "Invalid Entry, please enter 'y' for yes or 'n' for no" << endl; - - if ( noPrompting ) - exit(1); - } - - if ( answer == "y" ) { - mysqlRep = true; - MySQLRep = "y"; - } - else - { - mysqlRep = false; - MySQLRep = "n"; - } - - try { - sysConfig->setConfig(InstallSection, "MySQLRep", MySQLRep); - } - catch(...) - {} - } - else - { //Schema Sync is default as on when Local Query is Selected - mysqlRep = true; - MySQLRep = "y"; - - try { - sysConfig->setConfig(InstallSection, "MySQLRep", MySQLRep); - } - catch(...) - {} - } - if ( !writeConfig(sysConfig) ) { cout << "ERROR: Failed trying to update MariaDB ColumnStore System Configuration file" << endl; exit(1); From 9675af540754b3414b99fc13070776c07d42b3d8 Mon Sep 17 00:00:00 2001 From: david hill Date: Thu, 19 Jan 2017 15:04:00 -0600 Subject: [PATCH 56/61] MCOL-481 fix --- oamapps/postConfigure/postConfigure.cpp | 47 ++++++++++--------------- 1 file changed, 18 insertions(+), 29 deletions(-) diff --git a/oamapps/postConfigure/postConfigure.cpp b/oamapps/postConfigure/postConfigure.cpp index f8e66311a..1679a07d1 100644 --- a/oamapps/postConfigure/postConfigure.cpp +++ b/oamapps/postConfigure/postConfigure.cpp @@ -703,37 +703,26 @@ int main(int argc, char *argv[]) exit(1); } - if ( mysqlRep ) - { // current enabled - if ( answer == "y" ) { - mysqlRep = true; - MySQLRep = "y"; - } - else - { - mysqlRep = false; - MySQLRep = "n"; - } - } - else - { // currently disabled - if ( answer == "y" ) { - mysqlRep = false; - MySQLRep = "n"; - } - else - { - mysqlRep = true; - MySQLRep = "y"; - } - } + if ( answer == "y" ) { + mysqlRep = true; + MySQLRep = "y"; + } + else + { + mysqlRep = false; + MySQLRep = "n"; + } - try { - sysConfig->setConfig(InstallSection, "MySQLRep", MySQLRep); - } - catch(...) - {} + try { + sysConfig->setConfig(InstallSection, "MySQLRep", MySQLRep); + } + catch(...) + {} + if ( !writeConfig(sysConfig) ) { + cout << "ERROR: Failed trying to update MariaDB ColumnStore System Configuration file" << endl; + exit(1); + } switch ( IserverTypeInstall ) { case (oam::INSTALL_COMBINE_DM_UM_PM): // combined #1 - dm/um/pm on a single server From a48174de23ef8f7288c58fe3580bcd499003ae13 Mon Sep 17 00:00:00 2001 From: David Hall Date: Fri, 20 Jan 2017 10:10:24 -0600 Subject: [PATCH 57/61] MCOL-455: fix crash caused by rewind of unopened redistribute plan file during display when no partitions have been planned to be moved. --- writeengine/redistribute/we_redistributecontrolthread.cpp | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/writeengine/redistribute/we_redistributecontrolthread.cpp b/writeengine/redistribute/we_redistributecontrolthread.cpp index 652b48565..c43f83c3b 100644 --- a/writeengine/redistribute/we_redistributecontrolthread.cpp +++ b/writeengine/redistribute/we_redistributecontrolthread.cpp @@ -501,7 +501,7 @@ void RedistributeControlThread::displayPlan() if (!fControl->fPlanFilePtr) { ostringstream oss; - oss << "No data is scheduled to be moved" << endl; + oss << "No data is schefuled to be moved" << endl; fControl->logMessage(oss.str()); return; } From 26181beb8517710a690162ae4e39b6897fa0cd5d Mon Sep 17 00:00:00 2001 From: David Hill Date: Fri, 20 Jan 2017 19:16:54 -0600 Subject: [PATCH 58/61] MCOL-455 - add start remove to help --- oam/etc/ConsoleCmds.xml | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) diff --git a/oam/etc/ConsoleCmds.xml b/oam/etc/ConsoleCmds.xml index 01f6001cc..4bf12d36b 100644 --- a/oam/etc/ConsoleCmds.xml +++ b/oam/etc/ConsoleCmds.xml @@ -29,9 +29,9 @@ redistributeData Redistribute table data accross all dbroots to balance disk usage START to begin a redistribution - START REMOVE n to being a redistribution where data is removed from dbroot 'n' - STOP to stop redistribution before completion - STATUS to to view statistics and progress + STOP to stop redistribution before completion + STATUS to to view statistics and progress + START REMOVE to redistribute and move data off the given dbroot to other dbroots findObjectFile From 505464da612394a7657b6db120395b2cc0fbca97 Mon Sep 17 00:00:00 2001 From: david hill Date: Fri, 20 Jan 2017 23:15:00 -0600 Subject: [PATCH 59/61] MCOL-527 - add code for non-root --- procmon/processmonitor.cpp | 81 +++----------------------------------- 1 file changed, 5 insertions(+), 76 deletions(-) diff --git a/procmon/processmonitor.cpp b/procmon/processmonitor.cpp index e474c69d5..b45e5be84 100644 --- a/procmon/processmonitor.cpp +++ b/procmon/processmonitor.cpp @@ -5271,8 +5271,6 @@ int ProcessMonitor::runMasterDist(std::string& password, std::string& slaveModul string cmd = startup::StartUp::installDir() + "/bin/rsync.sh " + ipAddr + " " + password + " " + startup::StartUp::installDir() + " 1 > /tmp/master-dist_" + slaveModule + ".log"; system(cmd.c_str()); - - log.writeLog(__LINE__, "cmd = " + cmd, LOG_TYPE_DEBUG); string logFile = "/tmp/master-dist_" + slaveModule + ".log"; if (!oam.checkLogStatus(logFile, "FAILED")) @@ -5505,47 +5503,15 @@ bool ProcessMonitor::amazonVolumeCheck(int dbrootID) if ( status == "attached" ) { string cmd; if ( rootUser) - cmd = "mount " + deviceName + " " + startup::StartUp::installDir() + "/mysql/db -t ext2 -o noatime,nodiratime,noauto > /tmp/um_mount.log"; + cmd = "mount " + deviceName + " " + startup::StartUp::installDir() + "/mysql/db -t ext2 -o defaults > /tmp/um_mount.log"; else - cmd = "sudo mount " + deviceName + " " + startup::StartUp::installDir() + "/mysql/db -t ext2 -o noatime,nodiratime,noauto,user > /tmp/um_mount.log"; + cmd = "sudo mount " + deviceName + " " + startup::StartUp::installDir() + "/mysql/db -t ext2 -o defaults > /tmp/um_mount.log"; system(cmd.c_str()); log.writeLog(__LINE__, "mount cmd: " + cmd, LOG_TYPE_DEBUG); - if ( rootUser) - cmd = "chown -R mysql:mysql " + startup::StartUp::installDir() + "/mysql/db"; - else - cmd = "sudo chown -R " + USER + ":" + USER + " " + startup::StartUp::installDir() + "/mysql/db"; - + cmd = "chown mysql:mysql -R " + startup::StartUp::installDir() + "/mysql/db"; system(cmd.c_str()); - log.writeLog(__LINE__, "chown cmd: " + cmd, LOG_TYPE_DEBUG); - - //check for setup files in mysq/db, if not, create them for a new install - string file = startup::StartUp::installDir() + "/mysql/db/mysql"; - ifstream new_file (file.c_str()); - if (!new_file) { - string cmd; - cmd = startup::StartUp::installDir() + "/bin/post-mysqld-install --installdir=" + startup::StartUp::installDir() + " > /tmp/post-mysqld-install.log 2>&1"; - log.writeLog(__LINE__, "cmd: " + cmd, LOG_TYPE_DEBUG); - - int rtnCode = system(cmd.c_str()); - if (WEXITSTATUS(rtnCode) != 0) { - log.writeLog(__LINE__, "amazonVolumeCheck function failed, post-mysqld-install error" , LOG_TYPE_ERROR); - return false; - } - else - log.writeLog(__LINE__, "amazonVolumeCheck function, post-mysqld-install passed" , LOG_TYPE_DEBUG); - - cmd = startup::StartUp::installDir() + "/bin/post-mysql-install --installdir=" + startup::StartUp::installDir() + " > /tmp/post-mysql-install.log";; - log.writeLog(__LINE__, "cmd: " + cmd, LOG_TYPE_DEBUG); - rtnCode = system(cmd.c_str()); - if (WEXITSTATUS(rtnCode) != 0) { - log.writeLog(__LINE__, "amazonVolumeCheck function failed, post-mysql-install error" , LOG_TYPE_ERROR); - return false; - } - else - log.writeLog(__LINE__, "amazonVolumeCheck function, post-mysql-install passed" , LOG_TYPE_DEBUG); - } log.writeLog(__LINE__, "amazonVolumeCheck function successfully completed, volume attached: " + volumeName, LOG_TYPE_DEBUG); return true; @@ -5570,49 +5536,12 @@ bool ProcessMonitor::amazonVolumeCheck(int dbrootID) {} if (oam.attachEC2Volume(volumeName, deviceName, instanceName)) { - string cmd; - if ( rootUser) - cmd = "mount " + deviceName + " " + startup::StartUp::installDir() + "/mysql/db -t ext2 -o noatime,nodiratime,noauto > /tmp/um_mount.log"; - else - cmd = "sudo mount " + deviceName + " " + startup::StartUp::installDir() + "/mysql/db -t ext2 -o noatime,nodiratime,noauto,user > /tmp/um_mount.log"; - + string cmd = "mount " + deviceName + " " + startup::StartUp::installDir() + "/mysql/db -t ext2 -o defaults > /dev/null"; system(cmd.c_str()); log.writeLog(__LINE__, "mount cmd: " + cmd, LOG_TYPE_DEBUG); - if ( rootUser) - cmd = "chown -R mysql:mysql " + startup::StartUp::installDir() + "/mysql/db"; - else - cmd = "sudo chown -R " + USER + ":" + USER + " " + startup::StartUp::installDir() + "/mysql/db"; - + cmd = "chown mysql:mysql -R " + startup::StartUp::installDir() + "/mysql/db"; system(cmd.c_str()); - log.writeLog(__LINE__, "chown cmd: " + cmd, LOG_TYPE_DEBUG); - - //check for setup files in mysq/db, if not, create them for a new install - string file = startup::StartUp::installDir() + "/mysql/db/mysql"; - ifstream new_file (file.c_str()); - if (!new_file) { - string cmd; - cmd = startup::StartUp::installDir() + "/bin/post-mysqld-install --installdir=" + startup::StartUp::installDir() + " > /tmp/post-mysqld-install.log 2>&1"; - log.writeLog(__LINE__, "cmd: " + cmd, LOG_TYPE_DEBUG); - int rtnCode = system(cmd.c_str()); - if (WEXITSTATUS(rtnCode) != 0) { - log.writeLog(__LINE__, "amazonVolumeCheck function failed, post-mysqld-install error" , LOG_TYPE_ERROR); - return false; - } - else - log.writeLog(__LINE__, "amazonVolumeCheck function, post-mysqld-install passed" , LOG_TYPE_DEBUG); - - cmd = startup::StartUp::installDir() + "/bin/post-mysql-install --installdir=" + startup::StartUp::installDir() + " > /tmp/post-mysql-install.log";; - log.writeLog(__LINE__, "cmd: " + cmd, LOG_TYPE_DEBUG); - - rtnCode = system(cmd.c_str()); - if (WEXITSTATUS(rtnCode) != 0) { - log.writeLog(__LINE__, "amazonVolumeCheck function failed, post-mysql-install error" , LOG_TYPE_ERROR); - return false; - } - else - log.writeLog(__LINE__, "amazonVolumeCheck function, post-mysql-install passed" , LOG_TYPE_DEBUG); - } return true; } From b070ef321753f3cbc26ecc1cead0b4bdbadefbcb Mon Sep 17 00:00:00 2001 From: David Hall Date: Fri, 21 Apr 2017 09:37:07 -0500 Subject: [PATCH 60/61] MCOL-597 some clean up and make Release builds work --- dbcon/mysql/CMakeLists.txt | 10 +++++----- dbcon/mysql/ha_calpont_execplan.cpp | 2 +- dbcon/mysql/ha_calpont_impl.cpp | 10 ++++++---- dbcon/mysql/idb_mysql.h | 1 + utils/windowfunction/wf_lead_lag.cpp | 26 -------------------------- 5 files changed, 13 insertions(+), 36 deletions(-) diff --git a/dbcon/mysql/CMakeLists.txt b/dbcon/mysql/CMakeLists.txt index 4c423ed16..015ad58ea 100644 --- a/dbcon/mysql/CMakeLists.txt +++ b/dbcon/mysql/CMakeLists.txt @@ -25,7 +25,7 @@ set_source_files_properties(ha_calpont.cpp PROPERTIES COMPILE_FLAGS "-fno-rtti - add_library(calmysql SHARED ${libcalmysql_SRCS}) -target_link_libraries(calmysql ${ENGINE_LDFLAGS} ${ENGINE_WRITE_LIBS} ${NETSNMP_LIBRARIES} mysqlservices threadpool) +target_link_libraries(calmysql ${ENGINE_LDFLAGS} ${ENGINE_WRITE_LIBS} ${NETSNMP_LIBRARIES} ${SERVER_BUILD_INCLUDE_DIR}/../libservices/libmysqlservices.a threadpool) set_target_properties(calmysql PROPERTIES VERSION 1.0.0 SOVERSION 1) @@ -34,7 +34,7 @@ SET ( is_columnstore_tables_SRCS ) add_library(is_columnstore_tables SHARED ${is_columnstore_tables_SRCS}) -target_link_libraries(is_columnstore_tables ${ENGINE_LDFLAGS} ${ENGINE_WRITE_LIBS} mysqlservices threadpool) +target_link_libraries(is_columnstore_tables ${ENGINE_LDFLAGS} ${ENGINE_WRITE_LIBS} ${SERVER_BUILD_INCLUDE_DIR}/../libservices/libmysqlservices.a threadpool) # Don't prepend .so file with 'lib' set_target_properties(is_columnstore_tables PROPERTIES PREFIX "") @@ -45,7 +45,7 @@ SET ( is_columnstore_columns_SRCS ) add_library(is_columnstore_columns SHARED ${is_columnstore_columns_SRCS}) -target_link_libraries(is_columnstore_columns ${ENGINE_LDFLAGS} ${ENGINE_WRITE_LIBS} mysqlservices threadpool) +target_link_libraries(is_columnstore_columns ${ENGINE_LDFLAGS} ${ENGINE_WRITE_LIBS} ${SERVER_BUILD_INCLUDE_DIR}/../libservices/libmysqlservices.a threadpool) # Don't prepend .so file with 'lib' set_target_properties(is_columnstore_columns PROPERTIES PREFIX "") @@ -56,7 +56,7 @@ SET ( is_columnstore_extents_SRCS ) add_library(is_columnstore_extents SHARED ${is_columnstore_extents_SRCS}) -target_link_libraries(is_columnstore_extents ${ENGINE_LDFLAGS} ${ENGINE_WRITE_LIBS} mysqlservices threadpool) +target_link_libraries(is_columnstore_extents ${ENGINE_LDFLAGS} ${ENGINE_WRITE_LIBS} ${SERVER_BUILD_INCLUDE_DIR}/../libservices/libmysqlservices.a threadpool) # Don't prepend .so file with 'lib' set_target_properties(is_columnstore_extents PROPERTIES PREFIX "") @@ -67,7 +67,7 @@ SET ( is_columnstore_files_SRCS ) add_library(is_columnstore_files SHARED ${is_columnstore_files_SRCS}) -target_link_libraries(is_columnstore_files ${ENGINE_LDFLAGS} ${ENGINE_WRITE_LIBS} mysqlservices threadpool) +target_link_libraries(is_columnstore_files ${ENGINE_LDFLAGS} ${ENGINE_WRITE_LIBS} ${SERVER_BUILD_INCLUDE_DIR}/../libservices/libmysqlservices.a threadpool) # Don't prepend .so file with 'lib' set_target_properties(is_columnstore_files PROPERTIES PREFIX "") diff --git a/dbcon/mysql/ha_calpont_execplan.cpp b/dbcon/mysql/ha_calpont_execplan.cpp index 5180e03ae..d3c0c4e01 100755 --- a/dbcon/mysql/ha_calpont_execplan.cpp +++ b/dbcon/mysql/ha_calpont_execplan.cpp @@ -21,7 +21,7 @@ */ /** @file */ -#define DEBUG_WALK_COND +//#define DEBUG_WALK_COND #include #include #include diff --git a/dbcon/mysql/ha_calpont_impl.cpp b/dbcon/mysql/ha_calpont_impl.cpp index 93ce00949..c0dab9b14 100755 --- a/dbcon/mysql/ha_calpont_impl.cpp +++ b/dbcon/mysql/ha_calpont_impl.cpp @@ -4390,10 +4390,11 @@ int ha_calpont_impl_external_lock(THD *thd, TABLE* table, int lock_type) // table mode if (thd->infinidb_vtable.vtable_state == THD::INFINIDB_DISABLE_VTABLE) { - if (ci->traceFlags & 1) - push_warning(thd, Sql_condition::WARN_LEVEL_NOTE, 9999, mapiter->second.conn_hndl->queryStats.c_str()); if (mapiter->second.conn_hndl) { + if (ci->traceFlags & 1) + push_warning(thd, Sql_condition::WARN_LEVEL_NOTE, 9999, mapiter->second.conn_hndl->queryStats.c_str()); + ci->queryStats = mapiter->second.conn_hndl->queryStats; ci->extendedStats = mapiter->second.conn_hndl->extendedStats; ci->miniStats = mapiter->second.conn_hndl->miniStats; @@ -4417,11 +4418,12 @@ int ha_calpont_impl_external_lock(THD *thd, TABLE* table, int lock_type) { if (thd->infinidb_vtable.vtable_state == THD::INFINIDB_SELECT_VTABLE) { + if (!ci->cal_conn_hndl) + return 0; + if (ci->traceFlags & 1) push_warning(thd, Sql_condition::WARN_LEVEL_NOTE, 9999, ci->cal_conn_hndl->queryStats.c_str()); - if (!ci->cal_conn_hndl) - return 0; ci->queryStats = ci->cal_conn_hndl->queryStats; ci->extendedStats = ci->cal_conn_hndl->extendedStats; ci->miniStats = ci->cal_conn_hndl->miniStats; diff --git a/dbcon/mysql/idb_mysql.h b/dbcon/mysql/idb_mysql.h index 670f0e0e0..637f517d7 100644 --- a/dbcon/mysql/idb_mysql.h +++ b/dbcon/mysql/idb_mysql.h @@ -65,6 +65,7 @@ template bool isnan(T); #include "sql_table.h" #include "sql_select.h" +#include "mysqld_error.h" #include "item_windowfunc.h" // Now clean up the pollution as best we can... diff --git a/utils/windowfunction/wf_lead_lag.cpp b/utils/windowfunction/wf_lead_lag.cpp index 8d3b6169b..85ad54f70 100644 --- a/utils/windowfunction/wf_lead_lag.cpp +++ b/utils/windowfunction/wf_lead_lag.cpp @@ -153,32 +153,6 @@ void WF_lead_lag::parseParms(const std::vector& parms) idbassert(cc != NULL); bool isNull = false; // dummy, harded coded fRespectNulls = (cc->getIntVal(fRow, isNull) > 0); -#if 0 - // parms[1]: offset - for (std::vector::size_type i = 1/*ignore 0th element*/; i < parms.size(); ++i) - { - ConstantColumn* cc = dynamic_cast(parms[i].get()); - if (cc != NULL) - { - fOffset = cc->getIntVal(fRow, fOffsetNull) * fLead; // row not used, no need to setData. - continue; - } - - cc = dynamic_cast(parms[i].get()); - if (cc != NULL) - { - getConstValue(cc, fDefault, fDefNull); - continue; - } - // IGNORE/RESPECT nulls is currently broken in the front end - cc = dynamic_cast(parms[i].get()); - if (cc != NULL) - { - bool isNull = false; // dummy, harded coded - fRespectNulls = (cc->getIntVal(fRow, isNull) > 0); - } - } -#endif } From c7f91d749bd28230ff081e0f25241e2c9d83576c Mon Sep 17 00:00:00 2001 From: David Hall Date: Fri, 21 Apr 2017 10:18:06 -0500 Subject: [PATCH 61/61] MCOL-597 Update README and README.md to avoid merge conflict --- README | 2 +- README.md | 16 +++++++--------- 2 files changed, 8 insertions(+), 10 deletions(-) diff --git a/README b/README index ac59b3816..9ebf34303 100644 --- a/README +++ b/README @@ -1,6 +1,6 @@ This is MariaDB ColumnStore 1.0.6 MariaDB ColumnStore 1.0.6 is the development version of MariaDB ColumnStore. -It is built by porting InfiniDB 4.6.2 on MariaDB 10.1.19 and adding entirely +It is built by porting InfiniDB 4.6.7 on MariaDB 10.1.19 and adding entirely new features not found anywhere else. MariaDB ColumnStore 1.0.6 is an GA release. This is the first MariaDB diff --git a/README.md b/README.md index 5df193e89..577ac33c1 100644 --- a/README.md +++ b/README.md @@ -1,18 +1,16 @@ -#MariaDB ColumnStore Storage/Execution engine 1.0.1 -MariaDB ColumnStore 1.0.2 is the development version of MariaDB ColumnStore. -It is built by porting InfiniDB 4.6.2 on MariaDB 10.1.14 and adding entirely +#MariaDB ColumnStore Storage/Execution engine 1.1.0 +MariaDB ColumnStore 1.1.0 is the development version of MariaDB ColumnStore. +It is built by porting InfiniDB 4.6.7 on MariaDB 10.1.21 and adding entirely new features not found anywhere else. -#MariaDB ColumnStore 1.0.2 is an Alpha release. -This is the first MariaDB ColumnStore release, not all features planned for the MariaDB ColumnStore 1.0 -series are included in this release. +#MariaDB ColumnStore 1.1.0 is an development release. -Additional features will be pushed in future releases. -A few things to notice: -- Do not use alpha releases on production systems. +- Do not use development releases on production systems. #Building This repository is not meant to be built independently outside of the server. This repository is integrated into http://mariadb-corporation/mariadb-columnstore-server (ie, the *server*) as a git submodule. As such, you can find complete build instructions on *the server* page. https://github.com/mariadb-corporation/mariadb-columnstore-server +#Issue tracking +Issue tracking of MariaDB ColumnStore happens in JIRA, https://jira.mariadb.org/browse/MCOL