From 973edf5a87935b548ada654c1c29a170190c9855 Mon Sep 17 00:00:00 2001 From: David Hall Date: Wed, 6 Mar 2019 13:29:50 -0600 Subject: [PATCH 01/41] MCOL-2100 Don't print the line "redistributeData must have one of START, STOP or STATUS" if it has one. --- oamapps/mcsadmin/mcsadmin.cpp | 8 ++++---- 1 file changed, 4 insertions(+), 4 deletions(-) diff --git a/oamapps/mcsadmin/mcsadmin.cpp b/oamapps/mcsadmin/mcsadmin.cpp index dd7262d1e..d88944961 100644 --- a/oamapps/mcsadmin/mcsadmin.cpp +++ b/oamapps/mcsadmin/mcsadmin.cpp @@ -801,9 +801,9 @@ int processCommand(string* arguments) SendToWES(oam, bs); } #if _MSC_VER - if (_strnicmp(arguments[1].c_str(), "stop", 4) == 0)) + else if (_strnicmp(arguments[1].c_str(), "stop", 4) == 0)) #else - if (strncasecmp(arguments[1].c_str(), "stop", 4) == 0) + else if (strncasecmp(arguments[1].c_str(), "stop", 4) == 0) #endif { ByteStream bs; @@ -815,9 +815,9 @@ int processCommand(string* arguments) SendToWES(oam, bs); } #if _MSC_VER - if (_strnicmp(arguments[1].c_str(), "status", 6) == 0)) + else if (_strnicmp(arguments[1].c_str(), "status", 6) == 0)) #else - if (strncasecmp(arguments[1].c_str(), "status", 6) == 0) + else if (strncasecmp(arguments[1].c_str(), "status", 6) == 0) #endif { ByteStream bs; From bed4eae7213515fdba22ff7f2e08be313bb28ba5 Mon Sep 17 00:00:00 2001 From: Andrew Hutchings Date: Fri, 29 Mar 2019 15:02:16 +0000 Subject: [PATCH 02/41] Bump to version 1.1.8 --- VERSION | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/VERSION b/VERSION index 04362fcf8..6d304eb90 100644 --- a/VERSION +++ b/VERSION @@ -1,4 +1,4 @@ COLUMNSTORE_VERSION_MAJOR=1 COLUMNSTORE_VERSION_MINOR=1 -COLUMNSTORE_VERSION_PATCH=7 +COLUMNSTORE_VERSION_PATCH=8 COLUMNSTORE_VERSION_RELEASE=1 From 4e72b1f2d924566daf95175744630926fc21bdc3 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?=E6=95=B0=E7=BB=84=E8=B6=8A=E7=95=8C?= Date: Tue, 16 Apr 2019 09:31:49 +0800 Subject: [PATCH 03/41] fix typo in postConfigure --- oamapps/postConfigure/postConfigure.cpp | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/oamapps/postConfigure/postConfigure.cpp b/oamapps/postConfigure/postConfigure.cpp index fe7a3b337..107674507 100644 --- a/oamapps/postConfigure/postConfigure.cpp +++ b/oamapps/postConfigure/postConfigure.cpp @@ -4666,7 +4666,7 @@ bool storageSetup(bool amazonInstall) if ( (glusterInstalled == "y" && singleServerInstall != "1") && hadoopInstalled == "y" ) { - cout << "There are 5 options when configuring the storage: internal, external, DataRedundancy, or hdfs" << endl << endl; + cout << "There are 4 options when configuring the storage: internal, external, DataRedundancy, or hdfs" << endl << endl; prompt = "Select the type of Data Storage [1=internal, 2=external, 3=DataRedundancy, 4=hdfs] (" + storageType + ") > "; } From 83aad0c7b10bb7e0423ae5c7e845c94bc368d0f9 Mon Sep 17 00:00:00 2001 From: Sergey Sedov Date: Fri, 26 Apr 2019 11:10:29 +0300 Subject: [PATCH 04/41] MCOL-2051: CS uses 'NULL' literal as NULL when UPDATEs varchar field: proxying NULL value. Note: empty strings are still considered NULLs later on. --- dbcon/dmlpackage/dmlcolumn.cpp | 4 ---- dbcon/dmlpackage/dmlpkg.h | 5 +++-- dbcon/dmlpackage/updatedmlpackage.cpp | 3 ++- dbcon/mysql/ha_calpont_impl.cpp | 3 ++- 4 files changed, 7 insertions(+), 8 deletions(-) diff --git a/dbcon/dmlpackage/dmlcolumn.cpp b/dbcon/dmlpackage/dmlcolumn.cpp index 9bd0f67a8..f2a98f7d6 100644 --- a/dbcon/dmlpackage/dmlcolumn.cpp +++ b/dbcon/dmlpackage/dmlcolumn.cpp @@ -40,10 +40,6 @@ DMLColumn::DMLColumn(std::string name, std::string value, bool isFromCol, uint32 { fName = name; fData = value; - if (( strcasecmp(value.c_str(), "NULL") == 0) || (value.length() == 0) ) - { - isNULL = true; - } fisNULL = isNULL; fIsFromCol = isFromCol; fFuncScale = funcScale; diff --git a/dbcon/dmlpackage/dmlpkg.h b/dbcon/dmlpackage/dmlpkg.h index 3bef09343..924d974fd 100644 --- a/dbcon/dmlpackage/dmlpkg.h +++ b/dbcon/dmlpackage/dmlpkg.h @@ -409,8 +409,9 @@ public: std::string fColumn; std::string fOperator; std::string fScalarExpression; - bool fFromCol; - uint32_t fFuncScale; + bool fFromCol; + uint32_t fFuncScale; + bool fIsNull = false; }; /** @brief Stores a value list or a query specification diff --git a/dbcon/dmlpackage/updatedmlpackage.cpp b/dbcon/dmlpackage/updatedmlpackage.cpp index 05ca4f117..aa3504010 100644 --- a/dbcon/dmlpackage/updatedmlpackage.cpp +++ b/dbcon/dmlpackage/updatedmlpackage.cpp @@ -245,7 +245,8 @@ void UpdateDMLPackage::buildUpdateFromMysqlBuffer(UpdateSqlStatement& updateStm while (iter != updateStmt.fColAssignmentListPtr->end()) { ColumnAssignment* colaPtr = *iter; - DMLColumn* colPtr = new DMLColumn(colaPtr->fColumn, colaPtr->fScalarExpression, colaPtr->fFromCol, colaPtr->fFuncScale); + DMLColumn* colPtr = new DMLColumn(colaPtr->fColumn, colaPtr->fScalarExpression, colaPtr->fFromCol, colaPtr->fFuncScale, + colaPtr->fIsNull); rowPtr->get_ColumnList().push_back(colPtr); ++iter; diff --git a/dbcon/mysql/ha_calpont_impl.cpp b/dbcon/mysql/ha_calpont_impl.cpp index c1f37c9ac..7bb6ad212 100755 --- a/dbcon/mysql/ha_calpont_impl.cpp +++ b/dbcon/mysql/ha_calpont_impl.cpp @@ -1044,8 +1044,9 @@ uint32_t doUpdateDelete(THD *thd) else if ( value->type() == Item::NULL_ITEM ) { // dmlStmt += "NULL"; - columnAssignmentPtr->fScalarExpression = "NULL"; + columnAssignmentPtr->fScalarExpression = ""; columnAssignmentPtr->fFromCol = false; + columnAssignmentPtr->fIsNull = true; } else if ( value->type() == Item::SUBSELECT_ITEM ) { From 12d9e2ec954d24ec48ff0d948d8a2c06fb26bf11 Mon Sep 17 00:00:00 2001 From: Sergey Sedov Date: Thu, 2 May 2019 09:58:57 +0300 Subject: [PATCH 05/41] fixup! MCOL-2051: CS uses 'NULL' literal as NULL when UPDATEs varchar field Fixed to retain compatibility with CentOS 6 (no C++11 support). Moved initialization to the ColumnAssignment constructor. --- dbcon/dmlpackage/dml.y | 10 ++-------- dbcon/dmlpackage/dmlpkg.h | 10 +++++++++- dbcon/mysql/ha_calpont_impl.cpp | 5 +---- 3 files changed, 12 insertions(+), 13 deletions(-) diff --git a/dbcon/dmlpackage/dml.y b/dbcon/dmlpackage/dml.y index b26319c0b..73dcf26ea 100644 --- a/dbcon/dmlpackage/dml.y +++ b/dbcon/dmlpackage/dml.y @@ -566,17 +566,11 @@ assignment_commalist: assignment: column COMPARISON scalar_exp { - $$ = new ColumnAssignment(); - $$->fColumn = $1; - $$->fOperator = $2; - $$->fScalarExpression = $3; + $$ = new ColumnAssignment($1, $2, $3); } | column COMPARISON NULLX { - $$ = new ColumnAssignment(); - $$->fColumn = $1; - $$->fOperator = $2; - $$->fScalarExpression = $3; + $$ = new ColumnAssignment($1, $2, $3); } ; diff --git a/dbcon/dmlpackage/dmlpkg.h b/dbcon/dmlpackage/dmlpkg.h index 924d974fd..82c561b8e 100644 --- a/dbcon/dmlpackage/dmlpkg.h +++ b/dbcon/dmlpackage/dmlpkg.h @@ -397,6 +397,14 @@ public: class ColumnAssignment { public: + explicit ColumnAssignment( + std::string const& column, + std::string const& op = "=", + std::string const& expr = "") : + fColumn(column), fOperator(op), fScalarExpression(expr), + fFromCol(false), fFuncScale(0), fIsNull(false) + {}; + /** @brief dump to stdout */ std::ostream& put(std::ostream &os) const; @@ -411,7 +419,7 @@ public: std::string fScalarExpression; bool fFromCol; uint32_t fFuncScale; - bool fIsNull = false; + bool fIsNull; }; /** @brief Stores a value list or a query specification diff --git a/dbcon/mysql/ha_calpont_impl.cpp b/dbcon/mysql/ha_calpont_impl.cpp index 7bb6ad212..a6cbea6a9 100755 --- a/dbcon/mysql/ha_calpont_impl.cpp +++ b/dbcon/mysql/ha_calpont_impl.cpp @@ -928,10 +928,7 @@ uint32_t doUpdateDelete(THD *thd) else schemaName = string(item->db_name); - columnAssignmentPtr = new ColumnAssignment(); - columnAssignmentPtr->fColumn = string(item->name); - columnAssignmentPtr->fOperator = "="; - columnAssignmentPtr->fFuncScale = 0; + columnAssignmentPtr = new ColumnAssignment(item->name, "=", ""); Item *value= value_it++; if (value->type() == Item::STRING_ITEM) { From be83194c316e38a2aff28623960fe0818c403a66 Mon Sep 17 00:00:00 2001 From: Andrew Hutchings Date: Thu, 2 May 2019 10:32:49 +0100 Subject: [PATCH 06/41] MCOL-2273 Improve disk usage reporting Use the amount of available space rather than the amount of free space. The two numbers are usually different because the free blocks may be unusable at the time. --- oamapps/serverMonitor/diskMonitor.cpp | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/oamapps/serverMonitor/diskMonitor.cpp b/oamapps/serverMonitor/diskMonitor.cpp index bef1df560..e6989e4f6 100644 --- a/oamapps/serverMonitor/diskMonitor.cpp +++ b/oamapps/serverMonitor/diskMonitor.cpp @@ -240,7 +240,7 @@ void diskMonitor() blksize = buf.f_bsize; blocks = buf.f_blocks; - freeblks = buf.f_bfree; + freeblks = buf.f_bavail; totalBlocks = blocks * blksize; free = freeblks * blksize; @@ -359,7 +359,7 @@ void diskMonitor() blksize = buf.f_bsize; blocks = buf.f_blocks; - freeblks = buf.f_bfree; + freeblks = buf.f_bavail; totalBlocks = blocks * blksize; free = freeblks * blksize; From 72b6e3ef54723d8f283e03d9cb3a2e5f1d0ab879 Mon Sep 17 00:00:00 2001 From: David Hall Date: Thu, 2 May 2019 13:49:21 -0500 Subject: [PATCH 07/41] MCOL-1984 add waitPeriod to config items saved during update --- tools/configMgt/autoConfigure.cpp | 15 ++++++++++++++- 1 file changed, 14 insertions(+), 1 deletion(-) diff --git a/tools/configMgt/autoConfigure.cpp b/tools/configMgt/autoConfigure.cpp index e87401127..4c21fd6c7 100755 --- a/tools/configMgt/autoConfigure.cpp +++ b/tools/configMgt/autoConfigure.cpp @@ -66,6 +66,7 @@ typedef std::vector PerformanceModuleList; int main(int argc, char *argv[]) { setenv("CALPONT_HOME", "./", 1); + std::cout << "autoConfigure : main() *************" << std::endl; Oam oam; string systemParentOAMModuleName; @@ -326,7 +327,19 @@ int main(int argc, char *argv[]) exit(-1); } - //setup System Language + // WaitPeriod + try + { + string waitPeriod = sysConfigOld->getConfig(SystemSection, "WaitPeriod"); + if (waitPeriod.length() > 0) + { + sysConfigNew->setConfig(SystemSection, "WaitPeriod", waitPeriod); + } + } + catch (...) + { } + + //setup System Language string systemLang = "C"; try { systemLang = sysConfigOld->getConfig(SystemSection, "SystemLang"); From 767dd633c21b72cca4dc86c4b3daffa471d723a1 Mon Sep 17 00:00:00 2001 From: David Hall Date: Thu, 2 May 2019 13:52:01 -0500 Subject: [PATCH 08/41] MCOL-1984 remove debug line --- tools/configMgt/autoConfigure.cpp | 1 - 1 file changed, 1 deletion(-) diff --git a/tools/configMgt/autoConfigure.cpp b/tools/configMgt/autoConfigure.cpp index 4c21fd6c7..6cf665dad 100755 --- a/tools/configMgt/autoConfigure.cpp +++ b/tools/configMgt/autoConfigure.cpp @@ -66,7 +66,6 @@ typedef std::vector PerformanceModuleList; int main(int argc, char *argv[]) { setenv("CALPONT_HOME", "./", 1); - std::cout << "autoConfigure : main() *************" << std::endl; Oam oam; string systemParentOAMModuleName; From c444c36da7ce72f7d948c2cf7fe3827e5cfe956f Mon Sep 17 00:00:00 2001 From: Ben Thompson Date: Wed, 8 May 2019 12:00:12 -0500 Subject: [PATCH 09/41] Fix for newer version of CMake/CPack --- cpackEngineRPM.cmake | 230 +------------------------------------------ 1 file changed, 3 insertions(+), 227 deletions(-) diff --git a/cpackEngineRPM.cmake b/cpackEngineRPM.cmake index 866f39cff..b34b8207d 100644 --- a/cpackEngineRPM.cmake +++ b/cpackEngineRPM.cmake @@ -119,235 +119,11 @@ SET(ignored #%define _prefix ${CMAKE_INSTALL_PREFIX} #") -SET(CPACK_RPM_platform_USER_FILELIST -"/usr/local/mariadb/columnstore/bin/DDLProc" -"/usr/local/mariadb/columnstore/bin/ExeMgr" -"/usr/local/mariadb/columnstore/bin/ProcMgr" -"/usr/local/mariadb/columnstore/bin/ProcMon" -"/usr/local/mariadb/columnstore/bin/DMLProc" -"/usr/local/mariadb/columnstore/bin/WriteEngineServer" -"/usr/local/mariadb/columnstore/bin/cpimport" -"/usr/local/mariadb/columnstore/bin/post-install" -"/usr/local/mariadb/columnstore/bin/post-mysql-install" -"/usr/local/mariadb/columnstore/bin/post-mysqld-install" -"/usr/local/mariadb/columnstore/bin/pre-uninstall" -"/usr/local/mariadb/columnstore/bin/PrimProc" -"/usr/local/mariadb/columnstore/bin/DecomSvr" -"/usr/local/mariadb/columnstore/bin/upgrade-columnstore.sh" -"/usr/local/mariadb/columnstore/bin/run.sh" -"/usr/local/mariadb/columnstore/bin/columnstore" -"/usr/local/mariadb/columnstore/bin/columnstoreSyslog" -"/usr/local/mariadb/columnstore/bin/columnstoreSyslog7" -"/usr/local/mariadb/columnstore/bin/columnstoreSyslog-ng" -"/usr/local/mariadb/columnstore/bin/syslogSetup.sh" -"/usr/local/mariadb/columnstore/bin/cplogger" -"/usr/local/mariadb/columnstore/bin/columnstore.def" -"/usr/local/mariadb/columnstore/bin/dbbuilder" -"/usr/local/mariadb/columnstore/bin/cpimport.bin" -"/usr/local/mariadb/columnstore/bin/load_brm" -"/usr/local/mariadb/columnstore/bin/save_brm" -"/usr/local/mariadb/columnstore/bin/dbrmctl" -"/usr/local/mariadb/columnstore/bin/controllernode" -"/usr/local/mariadb/columnstore/bin/reset_locks" -"/usr/local/mariadb/columnstore/bin/workernode" -"/usr/local/mariadb/columnstore/bin/colxml" -"/usr/local/mariadb/columnstore/bin/clearShm" -"/usr/local/mariadb/columnstore/bin/viewtablelock" -"/usr/local/mariadb/columnstore/bin/cleartablelock" -"/usr/local/mariadb/columnstore/bin/mcsadmin" -"/usr/local/mariadb/columnstore/bin/remote_command.sh" -"/usr/local/mariadb/columnstore/bin/postConfigure" -"/usr/local/mariadb/columnstore/bin/columnstoreLogRotate" -"/usr/local/mariadb/columnstore/bin/transactionLog" -"/usr/local/mariadb/columnstore/bin/columnstoreDBWrite" -"/usr/local/mariadb/columnstore/bin/transactionLogArchiver.sh" -"/usr/local/mariadb/columnstore/bin/installer" -"/usr/local/mariadb/columnstore/bin/module_installer.sh" -"/usr/local/mariadb/columnstore/bin/package_installer.sh" -"/usr/local/mariadb/columnstore/bin/startupTests.sh" -"/usr/local/mariadb/columnstore/bin/os_check.sh" -"/usr/local/mariadb/columnstore/bin/remote_scp_put.sh" -"/usr/local/mariadb/columnstore/bin/remotessh.exp" -"/usr/local/mariadb/columnstore/bin/ServerMonitor" -"/usr/local/mariadb/columnstore/bin/master-rep-columnstore.sh" -"/usr/local/mariadb/columnstore/bin/slave-rep-columnstore.sh" -"/usr/local/mariadb/columnstore/bin/rsync.sh" -"/usr/local/mariadb/columnstore/bin/columnstoreSupport" -"/usr/local/mariadb/columnstore/bin/hardwareReport.sh" -"/usr/local/mariadb/columnstore/bin/softwareReport.sh" -"/usr/local/mariadb/columnstore/bin/configReport.sh" -"/usr/local/mariadb/columnstore/bin/logReport.sh" -"/usr/local/mariadb/columnstore/bin/bulklogReport.sh" -"/usr/local/mariadb/columnstore/bin/resourceReport.sh" -"/usr/local/mariadb/columnstore/bin/hadoopReport.sh" -"/usr/local/mariadb/columnstore/bin/alarmReport.sh" -"/usr/local/mariadb/columnstore/bin/remote_command_verify.sh" -"/usr/local/mariadb/columnstore/bin/disable-rep-columnstore.sh" -"/usr/local/mariadb/columnstore/bin/columnstore.service" -"/usr/local/mariadb/columnstore/etc/MessageFile.txt" -"/usr/local/mariadb/columnstore/etc/ErrorMessage.txt" -"/usr/local/mariadb/columnstore/local/module" -"/usr/local/mariadb/columnstore/releasenum" -"/usr/local/mariadb/columnstore/bin/rollback" -"/usr/local/mariadb/columnstore/bin/editem" -"/usr/local/mariadb/columnstore/bin/getConfig" -"/usr/local/mariadb/columnstore/bin/setConfig" -"/usr/local/mariadb/columnstore/bin/setenv-hdfs-12" -"/usr/local/mariadb/columnstore/bin/setenv-hdfs-20" -"/usr/local/mariadb/columnstore/bin/configxml.sh" -"/usr/local/mariadb/columnstore/bin/remote_scp_get.sh" -"/usr/local/mariadb/columnstore/bin/columnstoreAlias" -"/usr/local/mariadb/columnstore/bin/autoConfigure" -"/usr/local/mariadb/columnstore/bin/ddlcleanup" -"/usr/local/mariadb/columnstore/bin/idbmeminfo" -"/usr/local/mariadb/columnstore/bin/MCSInstanceCmds.sh" -"/usr/local/mariadb/columnstore/bin/MCSVolumeCmds.sh" -"/usr/local/mariadb/columnstore/bin/binary_installer.sh" -"/usr/local/mariadb/columnstore/bin/myCnf-include-args.text" -"/usr/local/mariadb/columnstore/bin/myCnf-exclude-args.text" -"/usr/local/mariadb/columnstore/bin/mycnfUpgrade" -"/usr/local/mariadb/columnstore/bin/getMySQLpw" -"/usr/local/mariadb/columnstore/bin/columnstore.conf" -"/usr/local/mariadb/columnstore/post/functions" -"/usr/local/mariadb/columnstore/post/test-001.sh" -"/usr/local/mariadb/columnstore/post/test-002.sh" -"/usr/local/mariadb/columnstore/post/test-003.sh" -"/usr/local/mariadb/columnstore/post/test-004.sh" -"/usr/local/mariadb/columnstore/bin/os_detect.sh" -"/usr/local/mariadb/columnstore/bin/columnstoreClusterTester.sh" -"/usr/local/mariadb/columnstore/bin/mariadb-command-line.sh" -"/usr/local/mariadb/columnstore/bin/quick_installer_single_server.sh" -"/usr/local/mariadb/columnstore/bin/quick_installer_multi_server.sh" -"/usr/local/mariadb/columnstore/bin/quick_installer_amazon.sh" -${ignored}) +SET(CPACK_RPM_platform_USER_FILELIST ${ignored}) -SET(CPACK_RPM_libs_USER_FILELIST -"/usr/local/mariadb/columnstore/lib/libconfigcpp.so.1.0.0" -"/usr/local/mariadb/columnstore/lib/libconfigcpp.so.1" -"/usr/local/mariadb/columnstore/lib/libconfigcpp.so" -"/usr/local/mariadb/columnstore/lib/libddlpackageproc.so.1.0.0" -"/usr/local/mariadb/columnstore/lib/libddlpackageproc.so.1" -"/usr/local/mariadb/columnstore/lib/libddlpackageproc.so" -"/usr/local/mariadb/columnstore/lib/libddlpackage.so.1.0.0" -"/usr/local/mariadb/columnstore/lib/libddlpackage.so.1" -"/usr/local/mariadb/columnstore/lib/libddlpackage.so" -"/usr/local/mariadb/columnstore/lib/libdmlpackageproc.so.1.0.0" -"/usr/local/mariadb/columnstore/lib/libdmlpackageproc.so.1" -"/usr/local/mariadb/columnstore/lib/libdmlpackageproc.so" -"/usr/local/mariadb/columnstore/lib/libdmlpackage.so.1.0.0" -"/usr/local/mariadb/columnstore/lib/libdmlpackage.so.1" -"/usr/local/mariadb/columnstore/lib/libdmlpackage.so" -"/usr/local/mariadb/columnstore/lib/libexecplan.so.1.0.0" -"/usr/local/mariadb/columnstore/lib/libexecplan.so.1" -"/usr/local/mariadb/columnstore/lib/libexecplan.so" -"/usr/local/mariadb/columnstore/lib/libfuncexp.so.1.0.0" -"/usr/local/mariadb/columnstore/lib/libfuncexp.so.1" -"/usr/local/mariadb/columnstore/lib/libfuncexp.so" -"/usr/local/mariadb/columnstore/lib/libudfsdk.so.1.1.0" -"/usr/local/mariadb/columnstore/lib/libudfsdk.so.1" -"/usr/local/mariadb/columnstore/lib/libudfsdk.so" -"/usr/local/mariadb/columnstore/lib/libjoblist.so.1.0.0" -"/usr/local/mariadb/columnstore/lib/libjoblist.so.1" -"/usr/local/mariadb/columnstore/lib/libjoblist.so" -"/usr/local/mariadb/columnstore/lib/libjoiner.so.1.0.0" -"/usr/local/mariadb/columnstore/lib/libjoiner.so.1" -"/usr/local/mariadb/columnstore/lib/libjoiner.so" -"/usr/local/mariadb/columnstore/lib/libloggingcpp.so.1.0.0" -"/usr/local/mariadb/columnstore/lib/libloggingcpp.so.1" -"/usr/local/mariadb/columnstore/lib/libloggingcpp.so" -"/usr/local/mariadb/columnstore/lib/libmessageqcpp.so.1.0.0" -"/usr/local/mariadb/columnstore/lib/libmessageqcpp.so.1" -"/usr/local/mariadb/columnstore/lib/libmessageqcpp.so" -"/usr/local/mariadb/columnstore/lib/liboamcpp.so.1.0.0" -"/usr/local/mariadb/columnstore/lib/liboamcpp.so.1" -"/usr/local/mariadb/columnstore/lib/liboamcpp.so" -"/usr/local/mariadb/columnstore/lib/libalarmmanager.so.1.0.0" -"/usr/local/mariadb/columnstore/lib/libalarmmanager.so.1" -"/usr/local/mariadb/columnstore/lib/libalarmmanager.so" -"/usr/local/mariadb/columnstore/lib/libthreadpool.so.1.0.0" -"/usr/local/mariadb/columnstore/lib/libthreadpool.so.1" -"/usr/local/mariadb/columnstore/lib/libthreadpool.so" -"/usr/local/mariadb/columnstore/lib/libwindowfunction.so.1.0.0" -"/usr/local/mariadb/columnstore/lib/libwindowfunction.so.1" -"/usr/local/mariadb/columnstore/lib/libwindowfunction.so" -"/usr/local/mariadb/columnstore/lib/libwriteengine.so.1.0.0" -"/usr/local/mariadb/columnstore/lib/libwriteengine.so.1" -"/usr/local/mariadb/columnstore/lib/libwriteengine.so" -"/usr/local/mariadb/columnstore/lib/libwriteengineclient.so.1.0.0" -"/usr/local/mariadb/columnstore/lib/libwriteengineclient.so.1" -"/usr/local/mariadb/columnstore/lib/libwriteengineclient.so" -"/usr/local/mariadb/columnstore/lib/libbrm.so.1.0.0" -"/usr/local/mariadb/columnstore/lib/libbrm.so.1" -"/usr/local/mariadb/columnstore/lib/libbrm.so" -"/usr/local/mariadb/columnstore/lib/librwlock.so.1.0.0" -"/usr/local/mariadb/columnstore/lib/librwlock.so.1" -"/usr/local/mariadb/columnstore/lib/librwlock.so" -"/usr/local/mariadb/columnstore/lib/libdataconvert.so.1.0.0" -"/usr/local/mariadb/columnstore/lib/libdataconvert.so.1" -"/usr/local/mariadb/columnstore/lib/libdataconvert.so" -"/usr/local/mariadb/columnstore/lib/librowgroup.so.1.0.0" -"/usr/local/mariadb/columnstore/lib/librowgroup.so.1" -"/usr/local/mariadb/columnstore/lib/librowgroup.so" -"/usr/local/mariadb/columnstore/lib/libcacheutils.so.1.0.0" -"/usr/local/mariadb/columnstore/lib/libcacheutils.so.1" -"/usr/local/mariadb/columnstore/lib/libcacheutils.so" -"/usr/local/mariadb/columnstore/lib/libcommon.so.1.0.0" -"/usr/local/mariadb/columnstore/lib/libcommon.so.1" -"/usr/local/mariadb/columnstore/lib/libcommon.so" -"/usr/local/mariadb/columnstore/lib/libcompress.so.1.0.0" -"/usr/local/mariadb/columnstore/lib/libcompress.so.1" -"/usr/local/mariadb/columnstore/lib/libcompress.so" -"/usr/local/mariadb/columnstore/lib/libddlcleanuputil.so.1.0.0" -"/usr/local/mariadb/columnstore/lib/libddlcleanuputil.so.1" -"/usr/local/mariadb/columnstore/lib/libddlcleanuputil.so" -"/usr/local/mariadb/columnstore/lib/libbatchloader.so.1.0.0" -"/usr/local/mariadb/columnstore/lib/libbatchloader.so.1" -"/usr/local/mariadb/columnstore/lib/libbatchloader.so" -"/usr/local/mariadb/columnstore/lib/libquerystats.so.1.0.0" -"/usr/local/mariadb/columnstore/lib/libquerystats.so.1" -"/usr/local/mariadb/columnstore/lib/libquerystats.so" -"/usr/local/mariadb/columnstore/lib/libwriteengineredistribute.so.1.0.0" -"/usr/local/mariadb/columnstore/lib/libwriteengineredistribute.so.1" -"/usr/local/mariadb/columnstore/lib/libwriteengineredistribute.so" -"/usr/local/mariadb/columnstore/lib/libidbdatafile.so.1.0.0" -"/usr/local/mariadb/columnstore/lib/libidbdatafile.so.1" -"/usr/local/mariadb/columnstore/lib/libidbdatafile.so" -"/usr/local/mariadb/columnstore/lib/libthrift.so.1.0.0" -"/usr/local/mariadb/columnstore/lib/libthrift.so.1" -"/usr/local/mariadb/columnstore/lib/libthrift.so" -"/usr/local/mariadb/columnstore/lib/libquerytele.so.1.0.0" -"/usr/local/mariadb/columnstore/lib/libquerytele.so.1" -"/usr/local/mariadb/columnstore/lib/libquerytele.so" -${ignored}) +SET(CPACK_RPM_libs_USER_FILELIST ${ignored}) -SET(CPACK_RPM_storage-engine_USER_FILELIST -"/usr/local/mariadb/columnstore/lib/libcalmysql.so.1.0.0" -"/usr/local/mariadb/columnstore/lib/libcalmysql.so.1" -"/usr/local/mariadb/columnstore/lib/libcalmysql.so" -"/usr/local/mariadb/columnstore/lib/libudf_mysql.so.1.0.0" -"/usr/local/mariadb/columnstore/lib/libudf_mysql.so.1" -"/usr/local/mariadb/columnstore/lib/libudf_mysql.so" -"/usr/local/mariadb/columnstore/lib/is_columnstore_columns.so" -"/usr/local/mariadb/columnstore/lib/is_columnstore_columns.so.1" -"/usr/local/mariadb/columnstore/lib/is_columnstore_columns.so.1.0.0" -"/usr/local/mariadb/columnstore/lib/is_columnstore_extents.so" -"/usr/local/mariadb/columnstore/lib/is_columnstore_extents.so.1" -"/usr/local/mariadb/columnstore/lib/is_columnstore_extents.so.1.0.0" -"/usr/local/mariadb/columnstore/lib/is_columnstore_tables.so" -"/usr/local/mariadb/columnstore/lib/is_columnstore_tables.so.1" -"/usr/local/mariadb/columnstore/lib/is_columnstore_tables.so.1.0.0" -"/usr/local/mariadb/columnstore/lib/is_columnstore_files.so" -"/usr/local/mariadb/columnstore/lib/is_columnstore_files.so.1" -"/usr/local/mariadb/columnstore/lib/is_columnstore_files.so.1.0.0" -"/usr/local/mariadb/columnstore/mysql/mysql-Columnstore" -"/usr/local/mariadb/columnstore/mysql/install_calpont_mysql.sh" -"/usr/local/mariadb/columnstore/mysql/syscatalog_mysql.sql" -"/usr/local/mariadb/columnstore/mysql/dumpcat_mysql.sql" -"/usr/local/mariadb/columnstore/mysql/calsetuserpriority.sql" -"/usr/local/mariadb/columnstore/mysql/calremoveuserpriority.sql" -"/usr/local/mariadb/columnstore/mysql/calshowprocesslist.sql" -"/usr/local/mariadb/columnstore/mysql/columnstore_info.sql" -${ignored}) +SET(CPACK_RPM_storage-engine_USER_FILELIST ${ignored}) INCLUDE (CPack) From f2d8a33ee5ee299fddebc78cfd4c16501bee348b Mon Sep 17 00:00:00 2001 From: David Hall Date: Thu, 9 May 2019 12:42:18 -0500 Subject: [PATCH 10/41] MCOL-3296 CTRL+C should not double remove from dequeue. --- dmlproc/dmlproc.cpp | 12 +++++++---- dmlproc/dmlprocessor.cpp | 36 +++++++------------------------ dmlproc/dmlprocessor.h | 38 +++++++++++++++++++++++++++++---- utils/threadpool/threadpool.cpp | 20 +++++++++++++++++ 4 files changed, 70 insertions(+), 36 deletions(-) diff --git a/dmlproc/dmlproc.cpp b/dmlproc/dmlproc.cpp index 106977824..990015ac8 100644 --- a/dmlproc/dmlproc.cpp +++ b/dmlproc/dmlproc.cpp @@ -86,6 +86,8 @@ using namespace joblist; namespace fs = boost::filesystem; +ThreadPool DMLServer::fDmlPackagepool(10, 0); + namespace { DistributedEngineComm *Dec; @@ -551,16 +553,16 @@ int main(int argc, char* argv[]) int temp; int serverThreads = 10; - int serverQueueSize = 50; + int serverQueueSize = 0; const string DMLProc("DMLProc"); temp = toInt(cf->getConfig(DMLProc, "ServerThreads")); if (temp > 0) serverThreads = temp; - temp = toInt(cf->getConfig(DMLProc, "ServerQueueSize")); - if (temp > 0) - serverQueueSize = temp; +// temp = toInt(cf->getConfig(DMLProc, "ServerQueueSize")); +// if (temp > 0) +// serverQueueSize = temp; bool rootUser = true; @@ -599,6 +601,8 @@ int main(int argc, char* argv[]) { JobStep::jobstepThreadPool.setDebug(true); JobStep::jobstepThreadPool.invoke(ThreadPoolMonitor(&JobStep::jobstepThreadPool)); + DMLServer::fDmlPackagepool.setDebug(true); + DMLServer::fDmlPackagepool.invoke(ThreadPoolMonitor(&DMLServer::fDmlPackagepool)); } //set ACTIVE state diff --git a/dmlproc/dmlprocessor.cpp b/dmlproc/dmlprocessor.cpp index 3b3a5cffc..ca0707989 100644 --- a/dmlproc/dmlprocessor.cpp +++ b/dmlproc/dmlprocessor.cpp @@ -413,7 +413,6 @@ int PackageHandler::releaseTableAccess() boost::lock_guard lock(tableOidMutex); if (fTableOid == 0 || (it=tableOidMap.find(fTableOid)) == tableOidMap.end()) { - // This will happen for DML_COMMAND, as we never got the tableoid or called synchTableAccess return 2; // For now, return codes are not used } PackageHandler::tableAccessQueue_t& tableOidQueue = it->second; @@ -433,7 +432,8 @@ int PackageHandler::releaseTableAccess() } else { - tableOidQueue.pop(); // Get off the waiting list. + if (!tableOidQueue.empty()) + tableOidQueue.pop(); // Get off the waiting list. if (tableOidQueue.empty()) { // remove the queue from the map. @@ -447,7 +447,7 @@ int PackageHandler::releaseTableAccess() int PackageHandler::forceReleaseTableAccess() { - // By removing the tcnid from the queue, the logic after the wait in + // By removing the txnid from the queue, the logic after the wait in // synchTableAccess() will release the thread and clean up if needed. std::map::iterator it; boost::lock_guard lock(tableOidMutex); @@ -481,7 +481,8 @@ void PackageHandler::run() std::string stmt; unsigned DMLLoggingId = 21; oam::OamCache* oamCache = oam::OamCache::makeOamCache(); - + SynchTable synchTable; + try { switch( fPackageType ) @@ -508,7 +509,7 @@ void PackageHandler::run() CalpontSystemCatalog::ROPair roPair = fcsc->tableRID(tableName); fTableOid = roPair.objnum; } - synchTableAccess(); // Blocks if another DML thread is using this fTableOid + synchTable.setPackage(this); // Blocks if another DML thread is using this fTableOid } #endif QueryTeleStats qts; @@ -870,7 +871,7 @@ void PackageHandler::run() CalpontSystemCatalog::ROPair roPair = fcsc->tableRID(tableName); fTableOid = roPair.objnum; } - synchTableAccess(); // Blocks if another DML thread is using this fTableOid + synchTable.setPackage(this); // Blocks if another DML thread is using this fTableOid } #endif updatePkg->set_TxnID(fTxnid); @@ -926,7 +927,7 @@ void PackageHandler::run() CalpontSystemCatalog::ROPair roPair = fcsc->tableRID(tableName); fTableOid = roPair.objnum; } - synchTableAccess(); // Blocks if another DML thread is using this fTableOid + synchTable.setPackage(this); // Blocks if another DML thread is using this fTableOid } #endif deletePkg->set_TxnID(fTxnid); @@ -989,13 +990,6 @@ void PackageHandler::run() } break; } -#ifdef MCOL_140 - if (fConcurrentSupport) - { - // MCOL-140 We're done. release the next waiting txn for this fTableOid - releaseTableAccess(); - } -#endif //Log errors if ( (result.result != dmlpackageprocessor::DMLPackageProcessor::NO_ERROR) && (result.result != dmlpackageprocessor::DMLPackageProcessor::IDBRANGE_WARNING) @@ -1017,13 +1011,6 @@ void PackageHandler::run() } catch(std::exception& e) { -#ifdef MCOL_140 - if (fConcurrentSupport) - { - // MCOL-140 We're done. release the next waiting txn for this fTableOid - releaseTableAccess(); - } -#endif cout << "dmlprocessor.cpp PackageHandler::run() package type(" << fPackageType << ") exception: " << e.what() << endl; logging::LoggingID lid(21); @@ -1040,13 +1027,6 @@ void PackageHandler::run() } catch(...) { -#ifdef MCOL_140 - if (fConcurrentSupport) - { - // MCOL-140 We're done. release the next waiting txn for this fTableOid - releaseTableAccess(); - } -#endif logging::LoggingID lid(21); logging::MessageLog ml(lid); logging::Message::Args args; diff --git a/dmlproc/dmlprocessor.h b/dmlproc/dmlprocessor.h index 2faf0cfb2..02f993df6 100644 --- a/dmlproc/dmlprocessor.h +++ b/dmlproc/dmlprocessor.h @@ -120,15 +120,16 @@ private: DMLServer(const DMLServer& rhs); DMLServer& operator=(const DMLServer& rhs); - /** @brief the thread pool for processing dml packages - */ - threadpool::ThreadPool fDmlPackagepool; - int fPackageMaxThreads; /** @brief max number of threads to process dml packages */ int fPackageWorkQueueSize; /** @brief max number of packages waiting in the work queue */ boost::scoped_ptr fMqServer; BRM::DBRM* fDbrm; + +public: + /** @brief the thread pool for processing dml packages + */ + static threadpool::ThreadPool fDmlPackagepool; }; /** @brief Thread to process a single dml package. @@ -184,6 +185,35 @@ private: static boost::mutex tableOidMutex; public: static int clearTableAccess(); + + // MCOL-3296 Add a class to call synchTableAccess on creation and + // releaseTableAccess on destuction for exception safeness. + class SynchTable + { + public: + SynchTable() : fphp(NULL) {}; + SynchTable(PackageHandler* php) + { + setPackage(php); + } + ~SynchTable() + { + if (fphp) + fphp->releaseTableAccess(); + } + bool setPackage(PackageHandler* php) + { + if (fphp) + fphp->releaseTableAccess(); + fphp = php; + if (fphp) + fphp->synchTableAccess(); + return true; + } + private: + PackageHandler* fphp; + }; + }; /** @brief processes dml packages as they arrive diff --git a/utils/threadpool/threadpool.cpp b/utils/threadpool/threadpool.cpp index 602f868c4..892a14ff2 100644 --- a/utils/threadpool/threadpool.cpp +++ b/utils/threadpool/threadpool.cpp @@ -392,6 +392,24 @@ void ThreadPool::beginThread() throw() --fIssued; --waitingFunctorsSize; fWaitingFunctors.erase(todo); + if (fDebug) + { + ostringstream oss; + oss << "Ending thread " << " on " << fName + << " max " << fMaxThreads + << " queue " << fQueueSize + << " threads " << fThreadCount + << " running " << fIssued + << " waiting " << (waitingFunctorsSize - fIssued) + << " total " << waitingFunctorsSize; + logging::Message::Args args; + logging::Message message(0); + args.add(oss.str()); + message.format( args ); + logging::LoggingID lid(22); + logging::MessageLog ml(lid); + ml.logWarningMessage( message ); + } } timeout = boost::get_system_time()+boost::posix_time::minutes(10); @@ -515,6 +533,8 @@ void ThreadPoolMonitor::operator()() << setw(4) << tv.tv_usec/100 << " Name " << fPool->fName << " Active " << fPool->waitingFunctorsSize + << " running " << fPool->fIssued + << " waiting " << (fPool->waitingFunctorsSize - fPool->fIssued) << " ThdCnt " << fPool->fThreadCount << " Max " << fPool->fMaxThreads << " Q " << fPool->fQueueSize From f98097673f72e7fbeae70fc1fd451167828a095a Mon Sep 17 00:00:00 2001 From: David Hall Date: Fri, 10 May 2019 10:04:57 -0500 Subject: [PATCH 11/41] MCOL-3296 Add logging when a dml is blocked because of another dml on the same table --- dmlproc/dmlprocessor.cpp | 53 ++++++++++++++++++++++++++++------------ dmlproc/dmlprocessor.h | 12 ++++----- 2 files changed, 44 insertions(+), 21 deletions(-) diff --git a/dmlproc/dmlprocessor.cpp b/dmlproc/dmlprocessor.cpp index ca0707989..b2ef6e6f6 100644 --- a/dmlproc/dmlprocessor.cpp +++ b/dmlproc/dmlprocessor.cpp @@ -334,7 +334,7 @@ PackageHandler::~PackageHandler() // Rollback will most likely be next. // // A tranasaction for one fTableOid is not blocked by a txn for a different fTableOid. -int PackageHandler::synchTableAccess() +int PackageHandler::synchTableAccess(dmlpackage::CalpontDMLPackage* dmlPackage) { // MCOL-140 Wait for any other DML using this table. std::map::iterator it; @@ -374,9 +374,27 @@ int PackageHandler::synchTableAccess() // tableOidQueue here is the queue holding the waitng transactions for this fTableOid while (true) { - tableOidCond.wait(lock); + // Log something that we're waiting + LoggingID logid(21, fSessionID, fTxnid); + logging::Message::Args args1; + logging::Message msg(1); + ostringstream oss; + oss << "Txn is waiting for" << tableOidQueue.front() << " " << dmlPackage->get_SQLStatement() << "; |" << dmlPackage->get_SchemaName() <<"|"; + args1.add(oss.str()); + args1.add((uint64_t)fTableOid); + msg.format(args1); + logging::Logger logger(logid.fSubsysID); + logger.logMessage(LOG_TYPE_DEBUG, msg, logid); + + tableOidCond.wait(lock); + // In case of CTRL+C, the tableOidQueue could be invalidated + if ((tableOidMap.find(fTableOid))->second != tableOidQueue) + { + break; + } if (tableOidQueue.front() == fTxnid) { + // We're up next. Let's go do stuff. break; } if (tableOidQueue.empty()) @@ -458,6 +476,11 @@ int PackageHandler::forceReleaseTableAccess() } PackageHandler::tableAccessQueue_t& tableOidQueue = it->second; tableOidQueue.erase(fTxnid); + if (tableOidQueue.empty()) + { + // remove the queue from the map. + tableOidMap.erase(fTableOid); + } // release the condition tableOidCond.notify_all(); return 1; @@ -509,7 +532,7 @@ void PackageHandler::run() CalpontSystemCatalog::ROPair roPair = fcsc->tableRID(tableName); fTableOid = roPair.objnum; } - synchTable.setPackage(this); // Blocks if another DML thread is using this fTableOid + synchTable.setPackage(this, &insertPkg); // Blocks if another DML thread is using this fTableOid } #endif QueryTeleStats qts; @@ -871,7 +894,7 @@ void PackageHandler::run() CalpontSystemCatalog::ROPair roPair = fcsc->tableRID(tableName); fTableOid = roPair.objnum; } - synchTable.setPackage(this); // Blocks if another DML thread is using this fTableOid + synchTable.setPackage(this, updatePkg.get()); // Blocks if another DML thread is using this fTableOid } #endif updatePkg->set_TxnID(fTxnid); @@ -927,7 +950,7 @@ void PackageHandler::run() CalpontSystemCatalog::ROPair roPair = fcsc->tableRID(tableName); fTableOid = roPair.objnum; } - synchTable.setPackage(this); // Blocks if another DML thread is using this fTableOid + synchTable.setPackage(this, deletePkg.get()); // Blocks if another DML thread is using this fTableOid } #endif deletePkg->set_TxnID(fTxnid); @@ -1070,16 +1093,6 @@ void PackageHandler::run() void PackageHandler::rollbackPending() { - // Force a release of the processing from MCOL-140 -#ifdef MCOL_140 - if (fConcurrentSupport) - { - // MCOL-140 We're not necessarily the next in line. - // This forces this thread to be released anyway. - forceReleaseTableAccess(); - } -#endif - if (fProcessor.get() == NULL) { // This happens when batch insert @@ -1088,6 +1101,16 @@ void PackageHandler::rollbackPending() fProcessor->setRollbackPending(true); + // Force a release of the processing from MCOL-140 +#ifdef MCOL_140 + if (fConcurrentSupport) + { + // MCOL-140 We're not necessarily the next in line. + // This forces this thread to be released anyway. + forceReleaseTableAccess(); + } +#endif + ostringstream oss; oss << "PackageHandler::rollbackPending: Processing DMLPackage."; DMLProcessor::log(oss.str(), logging::LOG_TYPE_DEBUG); diff --git a/dmlproc/dmlprocessor.h b/dmlproc/dmlprocessor.h index 02f993df6..978fbdda8 100644 --- a/dmlproc/dmlprocessor.h +++ b/dmlproc/dmlprocessor.h @@ -171,12 +171,12 @@ private: // Used to serialize operations because the VSS can't handle inserts // or updates on the same block. // When an Insert, Update or Delete command arrives, we look here - // for the table oid. If found, wait until it is no onger here. + // for the table oid. If found, wait until it is no longer here. // If this transactionID (SCN) is < the transactionID in the table, don't delay // and hope for the best, as we're already out of order. // When the VSS is engineered to handle transactions out of order, all MCOL-140 // code is to be removed. - int synchTableAccess(); + int synchTableAccess(dmlpackage::CalpontDMLPackage* dmlPackage); int releaseTableAccess(); int forceReleaseTableAccess(); typedef iterable_queue tableAccessQueue_t; @@ -192,22 +192,22 @@ public: { public: SynchTable() : fphp(NULL) {}; - SynchTable(PackageHandler* php) + SynchTable(PackageHandler* php, dmlpackage::CalpontDMLPackage* dmlPackage) { - setPackage(php); + setPackage(php, dmlPackage); } ~SynchTable() { if (fphp) fphp->releaseTableAccess(); } - bool setPackage(PackageHandler* php) + bool setPackage(PackageHandler* php, dmlpackage::CalpontDMLPackage* dmlPackage) { if (fphp) fphp->releaseTableAccess(); fphp = php; if (fphp) - fphp->synchTableAccess(); + fphp->synchTableAccess(dmlPackage); return true; } private: From 45df72a416a05a8d07647be9c83629568357f121 Mon Sep 17 00:00:00 2001 From: Andrew Hutchings Date: Tue, 14 May 2019 14:36:46 +0100 Subject: [PATCH 12/41] Fix issue during merge --- dbcon/mysql/ha_calpont_impl.cpp | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/dbcon/mysql/ha_calpont_impl.cpp b/dbcon/mysql/ha_calpont_impl.cpp index e57f332c5..146e026b1 100644 --- a/dbcon/mysql/ha_calpont_impl.cpp +++ b/dbcon/mysql/ha_calpont_impl.cpp @@ -1095,7 +1095,7 @@ uint32_t doUpdateDelete(THD* thd) else schemaName = string(item->db_name); - columnAssignmentPtr = new ColumnAssignment(item->name, "=", ""); + columnAssignmentPtr = new ColumnAssignment(item->name.str, "=", ""); Item* value = value_it++; if (value->type() == Item::STRING_ITEM) From 750b6ea0ffe527640e220d068e0963228323108b Mon Sep 17 00:00:00 2001 From: David Hall Date: Fri, 17 May 2019 12:43:18 -0500 Subject: [PATCH 13/41] MCOL-3314 When setting up statistics functions for the UM, use values that actually exist. --- dbcon/joblist/tupleaggregatestep.cpp | 24 ++++++++++++------------ 1 file changed, 12 insertions(+), 12 deletions(-) diff --git a/dbcon/joblist/tupleaggregatestep.cpp b/dbcon/joblist/tupleaggregatestep.cpp index b4aec4013..143af5de0 100644 --- a/dbcon/joblist/tupleaggregatestep.cpp +++ b/dbcon/joblist/tupleaggregatestep.cpp @@ -2622,8 +2622,8 @@ void TupleAggregateStep::prep1PhaseDistinctAggregate( functionVec2[i]->fAuxColumnIndex = lastCol; // sum(x) - oidsAggDist.push_back(oidsAggDist[j]); - keysAggDist.push_back(keysAggDist[j]); + oidsAggDist.push_back(oidsAgg[j]); + keysAggDist.push_back(keysAgg[j]); scaleAggDist.push_back(0); precisionAggDist.push_back(0); typeAggDist.push_back(CalpontSystemCatalog::LONGDOUBLE); @@ -2631,8 +2631,8 @@ void TupleAggregateStep::prep1PhaseDistinctAggregate( ++lastCol; // sum(x**2) - oidsAggDist.push_back(oidsAggDist[j]); - keysAggDist.push_back(keysAggDist[j]); + oidsAggDist.push_back(oidsAgg[j]); + keysAggDist.push_back(keysAgg[j]); scaleAggDist.push_back(0); precisionAggDist.push_back(-1); typeAggDist.push_back(CalpontSystemCatalog::LONGDOUBLE); @@ -3667,8 +3667,8 @@ void TupleAggregateStep::prep2PhasesAggregate( functionVecUm[i]->fAuxColumnIndex = lastCol; // sum(x) - oidsAggUm.push_back(oidsAggUm[j]); - keysAggUm.push_back(keysAggUm[j]); + oidsAggUm.push_back(oidsAggPm[j]); + keysAggUm.push_back(keysAggPm[j]); scaleAggUm.push_back(0); precisionAggUm.push_back(-1); typeAggUm.push_back(CalpontSystemCatalog::LONGDOUBLE); @@ -3676,8 +3676,8 @@ void TupleAggregateStep::prep2PhasesAggregate( ++lastCol; // sum(x**2) - oidsAggUm.push_back(oidsAggUm[j]); - keysAggUm.push_back(keysAggUm[j]); + oidsAggUm.push_back(oidsAggPm[j]); + keysAggUm.push_back(keysAggPm[j]); scaleAggUm.push_back(0); precisionAggUm.push_back(-1); typeAggUm.push_back(CalpontSystemCatalog::LONGDOUBLE); @@ -4704,8 +4704,8 @@ void TupleAggregateStep::prep2PhasesDistinctAggregate( functionVecUm[i]->fAuxColumnIndex = lastCol; // sum(x) - oidsAggDist.push_back(oidsAggDist[j]); - keysAggDist.push_back(keysAggDist[j]); + oidsAggDist.push_back(oidsAggPm[j]); + keysAggDist.push_back(keysAggPm[j]); scaleAggDist.push_back(0); precisionAggDist.push_back(-1); typeAggDist.push_back(CalpontSystemCatalog::LONGDOUBLE); @@ -4713,8 +4713,8 @@ void TupleAggregateStep::prep2PhasesDistinctAggregate( ++lastCol; // sum(x**2) - oidsAggDist.push_back(oidsAggDist[j]); - keysAggDist.push_back(keysAggDist[j]); + oidsAggDist.push_back(oidsAggPm[j]); + keysAggDist.push_back(keysAggPm[j]); scaleAggDist.push_back(0); precisionAggDist.push_back(-1); typeAggDist.push_back(CalpontSystemCatalog::LONGDOUBLE); From 01ff86096a4092a8a80d8a177620d1f076bb946e Mon Sep 17 00:00:00 2001 From: David Hall Date: Fri, 17 May 2019 12:45:09 -0500 Subject: [PATCH 14/41] MCOL-3314 When setting up statistics functions for the UM, use values that actually exist. --- dbcon/joblist/tupleaggregatestep.cpp | 36 ++++++++++++++-------------- 1 file changed, 18 insertions(+), 18 deletions(-) diff --git a/dbcon/joblist/tupleaggregatestep.cpp b/dbcon/joblist/tupleaggregatestep.cpp index e68bc9f18..07fb2a828 100755 --- a/dbcon/joblist/tupleaggregatestep.cpp +++ b/dbcon/joblist/tupleaggregatestep.cpp @@ -2325,8 +2325,8 @@ void TupleAggregateStep::prep1PhaseDistinctAggregate( throw logic_error("(9)A UDAF function is called but there's no RowUDAFFunctionCol"); } functionVec2[i]->fAuxColumnIndex = lastCol++; - oidsAggDist.push_back(oidsAggDist[j]); // Dummy? - keysAggDist.push_back(keysAggDist[j]); // Dummy? + oidsAggDist.push_back(oidsAgg[j]); // Dummy? + keysAggDist.push_back(keysAgg[j]); // Dummy? scaleAggDist.push_back(0); precisionAggDist.push_back(0); typeAggDist.push_back(CalpontSystemCatalog::UBIGINT); @@ -2340,8 +2340,8 @@ void TupleAggregateStep::prep1PhaseDistinctAggregate( functionVec2[i]->fAuxColumnIndex = lastCol; // sum(x) - oidsAggDist.push_back(oidsAggDist[j]); - keysAggDist.push_back(keysAggDist[j]); + oidsAggDist.push_back(oidsAgg[j]); + keysAggDist.push_back(keysAgg[j]); scaleAggDist.push_back(0); precisionAggDist.push_back(0); typeAggDist.push_back(CalpontSystemCatalog::LONGDOUBLE); @@ -2349,8 +2349,8 @@ void TupleAggregateStep::prep1PhaseDistinctAggregate( ++lastCol; // sum(x**2) - oidsAggDist.push_back(oidsAggDist[j]); - keysAggDist.push_back(keysAggDist[j]); + oidsAggDist.push_back(oidsAgg[j]); + keysAggDist.push_back(keysAgg[j]); scaleAggDist.push_back(0); precisionAggDist.push_back(0); typeAggDist.push_back(CalpontSystemCatalog::LONGDOUBLE); @@ -3258,8 +3258,8 @@ void TupleAggregateStep::prep2PhasesAggregate( throw logic_error("(9)A UDAF function is called but there's no RowUDAFFunctionCol"); } functionVecUm[i]->fAuxColumnIndex = lastCol++; - oidsAggUm.push_back(oidsAggUm[j]); // Dummy? - keysAggUm.push_back(keysAggUm[j]); // Dummy? + oidsAggUm.push_back(oidsAggPm[j]); // Dummy? + keysAggUm.push_back(keysAggPm[j]); // Dummy? scaleAggUm.push_back(0); precisionAggUm.push_back(0); typeAggUm.push_back(CalpontSystemCatalog::UBIGINT); @@ -3273,8 +3273,8 @@ void TupleAggregateStep::prep2PhasesAggregate( functionVecUm[i]->fAuxColumnIndex = lastCol; // sum(x) - oidsAggUm.push_back(oidsAggUm[j]); - keysAggUm.push_back(keysAggUm[j]); + oidsAggUm.push_back(oidsAggPm[j]); + keysAggUm.push_back(keysAggPm[j]); scaleAggUm.push_back(0); precisionAggUm.push_back(0); typeAggUm.push_back(CalpontSystemCatalog::LONGDOUBLE); @@ -3282,8 +3282,8 @@ void TupleAggregateStep::prep2PhasesAggregate( ++lastCol; // sum(x**2) - oidsAggUm.push_back(oidsAggUm[j]); - keysAggUm.push_back(keysAggUm[j]); + oidsAggUm.push_back(oidsAggPm[j]); + keysAggUm.push_back(keysAggPm[j]); scaleAggUm.push_back(0); precisionAggUm.push_back(0); typeAggUm.push_back(CalpontSystemCatalog::LONGDOUBLE); @@ -4189,8 +4189,8 @@ void TupleAggregateStep::prep2PhasesDistinctAggregate( throw logic_error("(9)A UDAF function is called but there's no RowUDAFFunctionCol"); } functionVecUm[i]->fAuxColumnIndex = lastCol++; - oidsAggDist.push_back(oidsAggUm[j]); // Dummy? - keysAggDist.push_back(keysAggUm[j]); // Dummy? + oidsAggDist.push_back(oidsAggPm[j]); // Dummy? + keysAggDist.push_back(keysAggPm[j]); // Dummy? scaleAggDist.push_back(0); precisionAggDist.push_back(0); typeAggDist.push_back(CalpontSystemCatalog::UBIGINT); @@ -4203,8 +4203,8 @@ void TupleAggregateStep::prep2PhasesDistinctAggregate( functionVecUm[i]->fAuxColumnIndex = lastCol; // sum(x) - oidsAggDist.push_back(oidsAggDist[j]); - keysAggDist.push_back(keysAggDist[j]); + oidsAggDist.push_back(oidsAggPm[j]); + keysAggDist.push_back(keysAggPm[j]); scaleAggDist.push_back(0); precisionAggDist.push_back(0); typeAggDist.push_back(CalpontSystemCatalog::LONGDOUBLE); @@ -4212,8 +4212,8 @@ void TupleAggregateStep::prep2PhasesDistinctAggregate( ++lastCol; // sum(x**2) - oidsAggDist.push_back(oidsAggDist[j]); - keysAggDist.push_back(keysAggDist[j]); + oidsAggDist.push_back(oidsAggPm[j]); + keysAggDist.push_back(keysAggPm[j]); scaleAggDist.push_back(0); precisionAggDist.push_back(0); typeAggDist.push_back(CalpontSystemCatalog::LONGDOUBLE); From fa0947caaaa294ffbcd3115b2c9fd7cec47b377b Mon Sep 17 00:00:00 2001 From: Andrew Hutchings Date: Thu, 23 May 2019 13:47:20 +0100 Subject: [PATCH 15/41] Revert "MCOL-1559 remove unused boost/trim header" This reverts commit 28e743bf38e6cc143d21e047cfcc7af105a665c5. --- dbcon/joblist/jlf_execplantojoblist.cpp | 1 + 1 file changed, 1 insertion(+) diff --git a/dbcon/joblist/jlf_execplantojoblist.cpp b/dbcon/joblist/jlf_execplantojoblist.cpp index fff1b12fb..e9068b5e8 100644 --- a/dbcon/joblist/jlf_execplantojoblist.cpp +++ b/dbcon/joblist/jlf_execplantojoblist.cpp @@ -36,6 +36,7 @@ using namespace std; #include #include #include +#include namespace ba = boost::algorithm; #include "calpontexecutionplan.h" From 182e31e2829cc26aa4bf5103a64f1abe55951264 Mon Sep 17 00:00:00 2001 From: Andrew Hutchings Date: Thu, 23 May 2019 13:47:42 +0100 Subject: [PATCH 16/41] Revert "MCOL-1559 backout trim before setting compare. Shouldn't do this in all cases." This reverts commit d78944d9ff894f677b63726d3e4f0a89763c8521. --- dbcon/joblist/jlf_execplantojoblist.cpp | 2 ++ 1 file changed, 2 insertions(+) diff --git a/dbcon/joblist/jlf_execplantojoblist.cpp b/dbcon/joblist/jlf_execplantojoblist.cpp index e9068b5e8..2cad5870e 100644 --- a/dbcon/joblist/jlf_execplantojoblist.cpp +++ b/dbcon/joblist/jlf_execplantojoblist.cpp @@ -1674,6 +1674,8 @@ const JobStepVector doSimpleFilter(SimpleFilter* sf, JobInfo& jobInfo) pds->cardinality(sc->cardinality()); //Add the filter + // MCOL-1559 trim before adding. + boost::trim_right_if(constval, boost::is_any_of(" ")); pds->addFilter(cop, constval); // data list for pcolstep output From 8b1c0f610164cc8ca14e1f701a5f98412d82fc71 Mon Sep 17 00:00:00 2001 From: Andrew Hutchings Date: Thu, 23 May 2019 13:48:00 +0100 Subject: [PATCH 17/41] Revert "MCOL-1559 trim constant varchar string before adding filter" This reverts commit b6484dda4aa5703606cf74fe77e29101625cff33. --- dbcon/joblist/jlf_execplantojoblist.cpp | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/dbcon/joblist/jlf_execplantojoblist.cpp b/dbcon/joblist/jlf_execplantojoblist.cpp index 2cad5870e..4bd41530d 100644 --- a/dbcon/joblist/jlf_execplantojoblist.cpp +++ b/dbcon/joblist/jlf_execplantojoblist.cpp @@ -1636,6 +1636,8 @@ const JobStepVector doSimpleFilter(SimpleFilter* sf, JobInfo& jobInfo) } string constval(cc->constval()); +// boost::trim_right_if(constval, boost::is_any_of(" ")); + CalpontSystemCatalog::OID dictOid = 0; CalpontSystemCatalog::ColType ct = sc->colType(); @@ -1674,8 +1676,6 @@ const JobStepVector doSimpleFilter(SimpleFilter* sf, JobInfo& jobInfo) pds->cardinality(sc->cardinality()); //Add the filter - // MCOL-1559 trim before adding. - boost::trim_right_if(constval, boost::is_any_of(" ")); pds->addFilter(cop, constval); // data list for pcolstep output From 73f7307ca264d775c9b636bbadb813df5da3f998 Mon Sep 17 00:00:00 2001 From: Andrew Hutchings Date: Thu, 23 May 2019 13:48:18 +0100 Subject: [PATCH 18/41] Revert "MCOL-1559 trailing space compare" This reverts commit e2cb6444843494e62ae8eacb0a1c3425400b7f69. --- dbcon/execplan/predicateoperator.h | 12 +++++++++++- 1 file changed, 11 insertions(+), 1 deletion(-) diff --git a/dbcon/execplan/predicateoperator.h b/dbcon/execplan/predicateoperator.h index 08f0c40cf..daa078948 100644 --- a/dbcon/execplan/predicateoperator.h +++ b/dbcon/execplan/predicateoperator.h @@ -456,8 +456,18 @@ inline bool PredicateOperator::getBoolVal(rowgroup::Row& row, bool& isNull, Retu isNull = false; return !ret; } +#if 0 + if (isNull) + return false; - // MCOL-1559 + const std::string& val1 = lop->getStrVal(row, isNull); + + if (isNull) + return false; + + return strCompare(val1, rop->getStrVal(row, isNull)) && !isNull; +#endif + // MCOL-1559 std::string val1 = lop->getStrVal(row, isNull); if (isNull) return false; From f3449fe2f47c3f1f15ba88d4e65ae463b8fab3ae Mon Sep 17 00:00:00 2001 From: Andrew Hutchings Date: Thu, 23 May 2019 13:48:31 +0100 Subject: [PATCH 19/41] Revert "MCOL-1559 trailing space compare" This reverts commit 5b581f53cbe1c0a7b27ac63fdddcffeccd866103. --- dbcon/execplan/predicateoperator.h | 12 ++---------- dbcon/joblist/jlf_execplantojoblist.cpp | 2 +- 2 files changed, 3 insertions(+), 11 deletions(-) diff --git a/dbcon/execplan/predicateoperator.h b/dbcon/execplan/predicateoperator.h index daa078948..70209df5b 100644 --- a/dbcon/execplan/predicateoperator.h +++ b/dbcon/execplan/predicateoperator.h @@ -456,19 +456,11 @@ inline bool PredicateOperator::getBoolVal(rowgroup::Row& row, bool& isNull, Retu isNull = false; return !ret; } -#if 0 - if (isNull) - return false; - - const std::string& val1 = lop->getStrVal(row, isNull); if (isNull) return false; - return strCompare(val1, rop->getStrVal(row, isNull)) && !isNull; -#endif - // MCOL-1559 - std::string val1 = lop->getStrVal(row, isNull); + std::string val1 = lop->getStrVal(row, isNull); if (isNull) return false; @@ -480,7 +472,7 @@ inline bool PredicateOperator::getBoolVal(rowgroup::Row& row, bool& isNull, Retu boost::trim_right_if(val2, boost::is_any_of(" ")); return strCompare(val1, val2); - } + } //FIXME: ??? case execplan::CalpontSystemCatalog::VARBINARY: diff --git a/dbcon/joblist/jlf_execplantojoblist.cpp b/dbcon/joblist/jlf_execplantojoblist.cpp index 4bd41530d..24ec7f50e 100644 --- a/dbcon/joblist/jlf_execplantojoblist.cpp +++ b/dbcon/joblist/jlf_execplantojoblist.cpp @@ -1636,7 +1636,7 @@ const JobStepVector doSimpleFilter(SimpleFilter* sf, JobInfo& jobInfo) } string constval(cc->constval()); -// boost::trim_right_if(constval, boost::is_any_of(" ")); + boost::trim_right_if(constval, boost::is_any_of(" ")); CalpontSystemCatalog::OID dictOid = 0; From e7a331f3f86290972973f717a4dc2e235c1c9ced Mon Sep 17 00:00:00 2001 From: Andrew Hutchings Date: Thu, 23 May 2019 13:48:43 +0100 Subject: [PATCH 20/41] Revert "MCOL-1559 trim spaces before compare" This reverts commit 8d553ae9fbbe8ddcc984966a45f632a1896fdf81. --- dbcon/execplan/predicateoperator.h | 4 ++-- dbcon/mysql/ha_calpont_execplan.cpp | 2 ++ 2 files changed, 4 insertions(+), 2 deletions(-) diff --git a/dbcon/execplan/predicateoperator.h b/dbcon/execplan/predicateoperator.h index 70209df5b..f81f189bb 100644 --- a/dbcon/execplan/predicateoperator.h +++ b/dbcon/execplan/predicateoperator.h @@ -460,11 +460,11 @@ inline bool PredicateOperator::getBoolVal(rowgroup::Row& row, bool& isNull, Retu if (isNull) return false; - std::string val1 = lop->getStrVal(row, isNull); + const std::string& val1 = lop->getStrVal(row, isNull); if (isNull) return false; - std::string val2 = rop->getStrVal(row, isNull); + const std::string& val2 = rop->getStrVal(row, isNull); if (isNull) return false; diff --git a/dbcon/mysql/ha_calpont_execplan.cpp b/dbcon/mysql/ha_calpont_execplan.cpp index 65d0ac4a6..239ae9f0c 100644 --- a/dbcon/mysql/ha_calpont_execplan.cpp +++ b/dbcon/mysql/ha_calpont_execplan.cpp @@ -49,6 +49,7 @@ using namespace std; #include #include +#include #include #include @@ -4868,6 +4869,7 @@ void gp_walk(const Item* item, void* arg) { cval.assign(str->ptr(), str->length()); } +// boost::trim_right_if(cval, boost::is_any_of(" ")); gwip->rcWorkStack.push(new ConstantColumn(cval)); break; From 922e828d6c52a722ba9c4f74c16269de301facba Mon Sep 17 00:00:00 2001 From: Andrew Hutchings Date: Thu, 23 May 2019 13:48:56 +0100 Subject: [PATCH 21/41] Revert "MCOL-1559 trailing white space comparison" This reverts commit aa802f44c534a21ff447b3025ae3e07ac323e337. --- dbcon/execplan/predicateoperator.h | 11 ++--------- 1 file changed, 2 insertions(+), 9 deletions(-) diff --git a/dbcon/execplan/predicateoperator.h b/dbcon/execplan/predicateoperator.h index f81f189bb..b83931394 100644 --- a/dbcon/execplan/predicateoperator.h +++ b/dbcon/execplan/predicateoperator.h @@ -36,7 +36,6 @@ #endif #include #include -#include #include "expressionparser.h" #include "returnedcolumn.h" @@ -461,17 +460,11 @@ inline bool PredicateOperator::getBoolVal(rowgroup::Row& row, bool& isNull, Retu return false; const std::string& val1 = lop->getStrVal(row, isNull); + if (isNull) return false; - const std::string& val2 = rop->getStrVal(row, isNull); - if (isNull) - return false; - - boost::trim_right_if(val1, boost::is_any_of(" ")); - boost::trim_right_if(val2, boost::is_any_of(" ")); - - return strCompare(val1, val2); + return strCompare(val1, rop->getStrVal(row, isNull)) && !isNull; } //FIXME: ??? From 9390ee05fb74c7edb28ff4f3e80bdb2f4b62aeaf Mon Sep 17 00:00:00 2001 From: Andrew Hutchings Date: Thu, 23 May 2019 13:49:08 +0100 Subject: [PATCH 22/41] Revert "MCOL-1559 Some string trailing blank stuff" This reverts commit e5d76e142be7edcbbf0817f5bea594ddffc84973. --- dbcon/joblist/jlf_execplantojoblist.cpp | 2 -- dbcon/joblist/lbidlist.cpp | 1 - dbcon/mysql/ha_calpont_execplan.cpp | 2 -- primitives/primproc/dictstep.cpp | 2 -- primitives/primproc/primitiveserver.cpp | 2 -- 5 files changed, 9 deletions(-) diff --git a/dbcon/joblist/jlf_execplantojoblist.cpp b/dbcon/joblist/jlf_execplantojoblist.cpp index 24ec7f50e..f3782c9d5 100644 --- a/dbcon/joblist/jlf_execplantojoblist.cpp +++ b/dbcon/joblist/jlf_execplantojoblist.cpp @@ -36,7 +36,6 @@ using namespace std; #include #include #include -#include namespace ba = boost::algorithm; #include "calpontexecutionplan.h" @@ -1636,7 +1635,6 @@ const JobStepVector doSimpleFilter(SimpleFilter* sf, JobInfo& jobInfo) } string constval(cc->constval()); - boost::trim_right_if(constval, boost::is_any_of(" ")); CalpontSystemCatalog::OID dictOid = 0; diff --git a/dbcon/joblist/lbidlist.cpp b/dbcon/joblist/lbidlist.cpp index 7852562ef..c317defc9 100644 --- a/dbcon/joblist/lbidlist.cpp +++ b/dbcon/joblist/lbidlist.cpp @@ -749,7 +749,6 @@ bool LBIDList::CasualPartitionPredicate(const int64_t Min, int64_t tMax = Max; dataconvert::DataConvert::trimWhitespace(tMin); dataconvert::DataConvert::trimWhitespace(tMax); - dataconvert::DataConvert::trimWhitespace(value); scan = compareVal(order_swap(tMin), order_swap(tMax), order_swap(value), op, lcf); diff --git a/dbcon/mysql/ha_calpont_execplan.cpp b/dbcon/mysql/ha_calpont_execplan.cpp index 239ae9f0c..65d0ac4a6 100644 --- a/dbcon/mysql/ha_calpont_execplan.cpp +++ b/dbcon/mysql/ha_calpont_execplan.cpp @@ -49,7 +49,6 @@ using namespace std; #include #include -#include #include #include @@ -4869,7 +4868,6 @@ void gp_walk(const Item* item, void* arg) { cval.assign(str->ptr(), str->length()); } -// boost::trim_right_if(cval, boost::is_any_of(" ")); gwip->rcWorkStack.push(new ConstantColumn(cval)); break; diff --git a/primitives/primproc/dictstep.cpp b/primitives/primproc/dictstep.cpp index 47bfeac25..abd99ada3 100644 --- a/primitives/primproc/dictstep.cpp +++ b/primitives/primproc/dictstep.cpp @@ -30,7 +30,6 @@ #include #include -#include #include "bpp.h" #include "primitiveserver.h" @@ -94,7 +93,6 @@ void DictStep::createCommand(ByteStream& bs) for (uint32_t i = 0; i < filterCount; i++) { bs >> strTmp; - boost::trim_right_if(strTmp, boost::is_any_of(" ")); //cout << " " << strTmp << endl; eqFilter->insert(strTmp); } diff --git a/primitives/primproc/primitiveserver.cpp b/primitives/primproc/primitiveserver.cpp index 54f4b7fb6..1e497122e 100644 --- a/primitives/primproc/primitiveserver.cpp +++ b/primitives/primproc/primitiveserver.cpp @@ -28,7 +28,6 @@ #include #include #include -#include //#define NDEBUG #include #include @@ -1778,7 +1777,6 @@ private: for (i = 0; i < count; i++) { *bs >> str; - boost::trim_right_if(str, boost::is_any_of(" ")); filter->insert(str); } From 6b2f67a23747942861d347d15f5cab7c55bc70a5 Mon Sep 17 00:00:00 2001 From: David Hall Date: Thu, 23 May 2019 14:15:58 -0500 Subject: [PATCH 23/41] MCOL-3314 Use something useful for stats functions oid and key during prep1PhaseAggregate --- dbcon/joblist/tupleaggregatestep.cpp | 12 ++++++------ 1 file changed, 6 insertions(+), 6 deletions(-) diff --git a/dbcon/joblist/tupleaggregatestep.cpp b/dbcon/joblist/tupleaggregatestep.cpp index 143af5de0..396e6a6ae 100644 --- a/dbcon/joblist/tupleaggregatestep.cpp +++ b/dbcon/joblist/tupleaggregatestep.cpp @@ -1583,8 +1583,8 @@ void TupleAggregateStep::prep1PhaseAggregate( } functionVec[i]->fAuxColumnIndex = lastCol++; - oidsAgg.push_back(oidsAgg[j]); - keysAgg.push_back(keysAgg[j]); + oidsAgg.push_back(oidsProj[j]); + keysAgg.push_back(keysProj[j]); scaleAgg.push_back(0); precisionAgg.push_back(0); precisionAgg.push_back(0); @@ -1599,8 +1599,8 @@ void TupleAggregateStep::prep1PhaseAggregate( functionVec[i]->fAuxColumnIndex = lastCol; // sum(x) - oidsAgg.push_back(oidsAgg[j]); - keysAgg.push_back(keysAgg[j]); + oidsAgg.push_back(oidsProj[j]); + keysAgg.push_back(keysProj[j]); scaleAgg.push_back(0); precisionAgg.push_back(-1); typeAgg.push_back(CalpontSystemCatalog::LONGDOUBLE); @@ -1608,8 +1608,8 @@ void TupleAggregateStep::prep1PhaseAggregate( ++lastCol; // sum(x**2) - oidsAgg.push_back(oidsAgg[j]); - keysAgg.push_back(keysAgg[j]); + oidsAgg.push_back(oidsProj[j]); + keysAgg.push_back(keysProj[j]); scaleAgg.push_back(0); precisionAgg.push_back(-1); typeAgg.push_back(CalpontSystemCatalog::LONGDOUBLE); From 42f29096f0cc9f639ddc46e0ea0854eebdcf3d6b Mon Sep 17 00:00:00 2001 From: Roman Nozdrin Date: Thu, 23 May 2019 22:22:55 +0300 Subject: [PATCH 24/41] MCOL-3239 CS pushes relevant filter predicates into derived tables. This change disables predicate push for function column that contains both derived table column and non-derived table column. --- dbcon/execplan/functioncolumn.cpp | 10 ++++++++++ 1 file changed, 10 insertions(+) diff --git a/dbcon/execplan/functioncolumn.cpp b/dbcon/execplan/functioncolumn.cpp index 487d42c0b..665e4a4c3 100644 --- a/dbcon/execplan/functioncolumn.cpp +++ b/dbcon/execplan/functioncolumn.cpp @@ -420,6 +420,16 @@ void FunctionColumn::setDerivedTable() break; } } + // MCOL-3239 Block for func column with both + // derived table column and normal table column. + else if (derivedTableAlias == "") + { + if (sc->tableAlias().length()) + { + derivedTableAlias = ""; + break; + } + } } fDerivedTable = derivedTableAlias; From 23b594eb284add47f3b97a6f20dd00eb2cf3a640 Mon Sep 17 00:00:00 2001 From: David Hall Date: Tue, 28 May 2019 15:56:30 -0500 Subject: [PATCH 25/41] MCOL-3304 During prepAggregate, if you set type to LONGDOUBLE, set scale to 0 --- dbcon/execplan/windowfunctioncolumn.cpp | 1 + dbcon/joblist/tupleaggregatestep.cpp | 36 ++++++++++++------------- 2 files changed, 19 insertions(+), 18 deletions(-) diff --git a/dbcon/execplan/windowfunctioncolumn.cpp b/dbcon/execplan/windowfunctioncolumn.cpp index e01c5c010..5c3e57407 100644 --- a/dbcon/execplan/windowfunctioncolumn.cpp +++ b/dbcon/execplan/windowfunctioncolumn.cpp @@ -386,6 +386,7 @@ void WindowFunctionColumn::adjustResultType() fResultType.colDataType = CalpontSystemCatalog::LONGDOUBLE; fResultType.colWidth = sizeof(long double); fResultType.precision = -1; + fResultType.scale = 0; } } diff --git a/dbcon/joblist/tupleaggregatestep.cpp b/dbcon/joblist/tupleaggregatestep.cpp index 143af5de0..23a8e8782 100644 --- a/dbcon/joblist/tupleaggregatestep.cpp +++ b/dbcon/joblist/tupleaggregatestep.cpp @@ -1406,7 +1406,7 @@ void TupleAggregateStep::prep1PhaseAggregate( typeAgg.push_back(CalpontSystemCatalog::LONGDOUBLE); precisionAgg.push_back(-1); widthAgg.push_back(sizeof(long double)); - scaleAgg.push_back(scaleProj[colProj]); + scaleAgg.push_back(0); } break; @@ -1935,7 +1935,7 @@ void TupleAggregateStep::prep1PhaseDistinctAggregate( typeAgg.push_back(CalpontSystemCatalog::LONGDOUBLE); precisionAgg.push_back(-1); widthAgg.push_back(sizeof(long double)); - scaleAgg.push_back(scaleProj[colProj]); + scaleAgg.push_back(0); colAgg++; // has distinct step, put the count column for avg next to the sum @@ -2255,7 +2255,7 @@ void TupleAggregateStep::prep1PhaseDistinctAggregate( typeAggDist.push_back(CalpontSystemCatalog::LONGDOUBLE); precisionAggDist.push_back(-1); widthAggDist.push_back(sizeof(long double)); - scaleAggDist.push_back(scaleProj[colAgg]); + scaleAggDist.push_back(0); } break; @@ -2327,10 +2327,10 @@ void TupleAggregateStep::prep1PhaseDistinctAggregate( { oidsAggDist.push_back(oidsAgg[colAgg]); keysAggDist.push_back(retKey); - scaleAggDist.push_back(scaleAgg[colAgg] >> 8); - precisionAggDist.push_back(precisionAgg[colAgg]); - typeAggDist.push_back(typeAgg[colAgg]); - widthAggDist.push_back(widthAgg[colAgg]); + scaleAggDist.push_back(0); + typeAggDist.push_back(CalpontSystemCatalog::LONGDOUBLE); + precisionAggDist.push_back(-1); + widthAggDist.push_back(sizeof(long double)); } else { @@ -3173,7 +3173,7 @@ void TupleAggregateStep::prep2PhasesAggregate( oidsAggPm.push_back(oidsProj[colProj]); keysAggPm.push_back(aggKey); typeAggPm.push_back(CalpontSystemCatalog::LONGDOUBLE); - scaleAggPm.push_back(scaleProj[colProj]); + scaleAggPm.push_back(0); precisionAggPm.push_back(-1); widthAggPm.push_back(sizeof(long double)); colAggPm++; @@ -3449,10 +3449,10 @@ void TupleAggregateStep::prep2PhasesAggregate( { oidsAggUm.push_back(oidsAggPm[colPm]); keysAggUm.push_back(retKey); - scaleAggUm.push_back(scaleAggPm[colPm] >> 8); - precisionAggUm.push_back(precisionAggPm[colPm]); - typeAggUm.push_back(typeAggPm[colPm]); - widthAggUm.push_back(widthAggPm[colPm]); + scaleAggUm.push_back(0); + typeAggUm.push_back(CalpontSystemCatalog::LONGDOUBLE); + precisionAggUm.push_back(-1); + widthAggUm.push_back(sizeof(long double)); } else { @@ -4027,7 +4027,7 @@ void TupleAggregateStep::prep2PhasesDistinctAggregate( typeAggPm.push_back(CalpontSystemCatalog::LONGDOUBLE); precisionAggPm.push_back(-1); widthAggPm.push_back(sizeof(long double)); - scaleAggPm.push_back(scaleProj[colProj]); + scaleAggPm.push_back(0); colAggPm++; } @@ -4394,7 +4394,7 @@ void TupleAggregateStep::prep2PhasesDistinctAggregate( typeAggDist.push_back(CalpontSystemCatalog::LONGDOUBLE); precisionAggDist.push_back(-1); widthAggDist.push_back(sizeof(long double)); - scaleAggDist.push_back(scaleAggUm[colUm]); + scaleAggDist.push_back(0); } // PM: put the count column for avg next to the sum // let fall through to add a count column for average function @@ -4456,10 +4456,10 @@ void TupleAggregateStep::prep2PhasesDistinctAggregate( { oidsAggDist.push_back(oidsAggUm[colUm]); keysAggDist.push_back(retKey); - scaleAggDist.push_back(scaleAggUm[colUm] >> 8); - precisionAggDist.push_back(precisionAggUm[colUm]); - typeAggDist.push_back(typeAggUm[colUm]); - widthAggDist.push_back(widthAggUm[colUm]); + scaleAggDist.push_back(0); + typeAggDist.push_back(CalpontSystemCatalog::LONGDOUBLE); + precisionAggDist.push_back(-1); + widthAggDist.push_back(sizeof(long double)); } else { From fff18e0dd41f22e97b3ca78497e7cb2f20c924ea Mon Sep 17 00:00:00 2001 From: David Mott Date: Mon, 3 Jun 2019 03:36:48 -0500 Subject: [PATCH 26/41] fix to possible memory corruption --- dbcon/execplan/arithmeticcolumn.cpp | 7 +++++-- 1 file changed, 5 insertions(+), 2 deletions(-) diff --git a/dbcon/execplan/arithmeticcolumn.cpp b/dbcon/execplan/arithmeticcolumn.cpp index aab6c9265..2128a47ed 100644 --- a/dbcon/execplan/arithmeticcolumn.cpp +++ b/dbcon/execplan/arithmeticcolumn.cpp @@ -326,7 +326,8 @@ void ArithmeticColumn::serialize(messageqcpp::ByteStream& b) const ObjectReader::writeParseTree(fExpression, b); b << fTableAlias; b << fData; - b << static_cast(fAsc); + const ByteStream::doublebyte tmp = fAsc; + b << tmp; } void ArithmeticColumn::unserialize(messageqcpp::ByteStream& b) @@ -340,7 +341,9 @@ void ArithmeticColumn::unserialize(messageqcpp::ByteStream& b) fExpression = ObjectReader::createParseTree(b); b >> fTableAlias; b >> fData; - b >> reinterpret_cast< ByteStream::doublebyte&>(fAsc); + ByteStream::doublebyte tmp; + b >> tmp; + fAsc = (tmp); fSimpleColumnList.clear(); fExpression->walk(getSimpleCols, &fSimpleColumnList); From 0983c1becf0e66de8b06024e6504f4ffb7dc245a Mon Sep 17 00:00:00 2001 From: Andrew Hutchings Date: Mon, 10 Jun 2019 16:46:24 +0100 Subject: [PATCH 27/41] MCOL-2230 Fix TIMEDIFF() handling TIMEDIFF would not compare datetime with string correctly, causing a NULL saturation. --- utils/funcexp/func_timediff.cpp | 5 ++++- 1 file changed, 4 insertions(+), 1 deletion(-) diff --git a/utils/funcexp/func_timediff.cpp b/utils/funcexp/func_timediff.cpp index 369bb80a1..c4de35a0d 100644 --- a/utils/funcexp/func_timediff.cpp +++ b/utils/funcexp/func_timediff.cpp @@ -118,7 +118,10 @@ string Func_timediff::getStrVal(rowgroup::Row& row, case execplan::CalpontSystemCatalog::TIME: case execplan::CalpontSystemCatalog::DATETIME: - if (type1 != type2) + // Diff between time and datetime returns NULL in MariaDB + if ((type2 == execplan::CalpontSystemCatalog::TIME || + type2 == execplan::CalpontSystemCatalog::DATETIME) && + type1 != type2) { isNull = true; break; From e3cd20538887c037faea1f93cdda32763f7fc7fb Mon Sep 17 00:00:00 2001 From: Andrew Hutchings Date: Tue, 11 Jun 2019 10:37:04 +0100 Subject: [PATCH 28/41] MCOL-1968 Fix UTF char/varchar min/max handling If the first byte of a char/varchar was > 0x80 then it will break the min/max values for an extent during cpimport. This patch makes the min/max compare unsigned and only switches to signed when storing. In addition send all the LDI / INSERT...SELECT data to cpimport, not truncated. Let cpimport figure out the truncation point. --- dbcon/mysql/ha_calpont_dml.cpp | 18 ++++++++++-------- writeengine/bulk/we_bulkloadbuffer.cpp | 11 +++++++---- 2 files changed, 17 insertions(+), 12 deletions(-) diff --git a/dbcon/mysql/ha_calpont_dml.cpp b/dbcon/mysql/ha_calpont_dml.cpp index c2dc36e47..a9404c223 100755 --- a/dbcon/mysql/ha_calpont_dml.cpp +++ b/dbcon/mysql/ha_calpont_dml.cpp @@ -860,7 +860,13 @@ int ha_calpont_impl_write_batch_row_(uchar *buf, TABLE* table, cal_impl_if::cal_ } case CalpontSystemCatalog::VARCHAR: { - if (nullVal && (ci.columnTypes[colpos].constraintType != CalpontSystemCatalog::NOTNULL_CONSTRAINT)) + size_t length; + if (ci.utf8) + length = (ci.columnTypes[colpos].colWidth * 3); + else + length = ci.columnTypes[colpos].colWidth; + + if (nullVal && (ci.columnTypes[colpos].constraintType != CalpontSystemCatalog::NOTNULL_CONSTRAINT)) { fprintf(ci.filePtr, "%c", ci.delimiter); if (!ci.utf8) @@ -901,6 +907,7 @@ int ha_calpont_impl_write_batch_row_(uchar *buf, TABLE* table, cal_impl_if::cal_ dataLength = *(uint16_t*) buf; buf = buf + 2 ; } + length = dataLength; escape.assign((char*)buf, dataLength); boost::replace_all(escape, "\\", "\\\\"); fprintf(ci.filePtr, "%c%.*s%c%c", ci.enclosed_by, (int)escape.length(), escape.c_str(), ci.enclosed_by, ci.delimiter); @@ -917,8 +924,7 @@ int ha_calpont_impl_write_batch_row_(uchar *buf, TABLE* table, cal_impl_if::cal_ dataLength = *(uint16_t*) buf; buf = buf + 2 ; } - if ( dataLength > ci.columnTypes[colpos].colWidth) - dataLength = ci.columnTypes[colpos].colWidth; + length = dataLength; escape.assign((char*)buf, dataLength); boost::replace_all(escape, "\\", "\\\\"); @@ -926,11 +932,7 @@ int ha_calpont_impl_write_batch_row_(uchar *buf, TABLE* table, cal_impl_if::cal_ fprintf(ci.filePtr, "%c%.*s%c%c", ci.enclosed_by, (int)escape.length(), escape.c_str(), ci.enclosed_by, ci.delimiter); } } - //buf += ci.columnTypes[colpos].colWidth; - if (ci.utf8) - buf += (ci.columnTypes[colpos].colWidth * 3); - else - buf += ci.columnTypes[colpos].colWidth; + buf += length; break; } diff --git a/writeengine/bulk/we_bulkloadbuffer.cpp b/writeengine/bulk/we_bulkloadbuffer.cpp index b1902e98f..7326f987b 100644 --- a/writeengine/bulk/we_bulkloadbuffer.cpp +++ b/writeengine/bulk/we_bulkloadbuffer.cpp @@ -524,13 +524,16 @@ void BulkLoadBuffer::convert(char *field, int fieldLength, } // Swap byte order before comparing character string - int64_t binChar = static_cast( uint64ToStr( - *(reinterpret_cast(charTmpBuf)) ) ); + // Compare must be unsigned + uint64_t compChar = uint64ToStr( *(reinterpret_cast(charTmpBuf)) ); + int64_t binChar = static_cast( compChar ); // Update min/max range - if (binChar < bufStats.minBufferVal) + uint64_t minVal = static_cast( bufStats.minBufferVal ); + uint64_t maxVal = static_cast( bufStats.maxBufferVal ); + if (compChar < minVal) bufStats.minBufferVal = binChar; - if (binChar > bufStats.maxBufferVal) + if (compChar > maxVal) bufStats.maxBufferVal = binChar; pVal = charTmpBuf; From ea6592ea080640252f5eb2ff07c764ef3a350a31 Mon Sep 17 00:00:00 2001 From: David Hall Date: Wed, 12 Jun 2019 14:17:31 -0500 Subject: [PATCH 29/41] MCOL-3343 Try adding aggregate result to jobInfo.nonConstDelCol vector --- dbcon/joblist/joblistfactory.cpp | 11 ++++++++--- 1 file changed, 8 insertions(+), 3 deletions(-) diff --git a/dbcon/joblist/joblistfactory.cpp b/dbcon/joblist/joblistfactory.cpp index 21df156a7..0a141f03a 100644 --- a/dbcon/joblist/joblistfactory.cpp +++ b/dbcon/joblist/joblistfactory.cpp @@ -397,6 +397,7 @@ void checkHavingClause(CalpontSelectExecutionPlan* csep, JobInfo& jobInfo) void preProcessFunctionOnAggregation(const vector& scs, const vector& aggs, + const vector& wcs, JobInfo& jobInfo) { // append the simple columns if not already projected @@ -426,10 +427,14 @@ void preProcessFunctionOnAggregation(const vector& scs, } } - // append the aggregate columns in arithmetic/function cloulmn to the projection list + // append the aggregate columns in arithmetic/function column to the projection list for (vector::const_iterator i = aggs.begin(); i != aggs.end(); i++) { addAggregateColumn(*i, -1, jobInfo.projectionCols, jobInfo); + if (wcs.size() > 0) + { + jobInfo.nonConstDelCols.push_back(SRCP((*i)->clone())); + } } } @@ -481,12 +486,12 @@ void checkReturnedColumns(CalpontSelectExecutionPlan* csep, JobInfo& jobInfo) if (ac != NULL && ac->aggColumnList().size() > 0) { jobInfo.nonConstCols[i]->outputIndex(i); - preProcessFunctionOnAggregation(ac->simpleColumnList(), ac->aggColumnList(), jobInfo); + preProcessFunctionOnAggregation(ac->simpleColumnList(), ac->aggColumnList(), ac->windowfunctionColumnList(), jobInfo); } else if (fc != NULL && fc->aggColumnList().size() > 0) { jobInfo.nonConstCols[i]->outputIndex(i); - preProcessFunctionOnAggregation(fc->simpleColumnList(), fc->aggColumnList(), jobInfo); + preProcessFunctionOnAggregation(fc->simpleColumnList(), fc->aggColumnList(), fc->windowfunctionColumnList(), jobInfo); } } } From 7d22a5945c68456029e10c44ff402b96d8b25836 Mon Sep 17 00:00:00 2001 From: Andrew Hutchings Date: Fri, 14 Jun 2019 15:52:30 +0100 Subject: [PATCH 30/41] MCOL-1989 Fix view in view subquery outer join A view calling a view as part of a subquery outer join was not getting the view name for the derived table columns. Which caused ColumnStore to think it was joining outside of the view and triggered a missing column error. This fix adds the view name from the subquery if one cannot be obtained from the field object. --- dbcon/execplan/calpontselectexecutionplan.h | 5 +++++ dbcon/mysql/ha_calpont_execplan.cpp | 9 ++++++++- dbcon/mysql/ha_from_sub.cpp | 1 + 3 files changed, 14 insertions(+), 1 deletion(-) diff --git a/dbcon/execplan/calpontselectexecutionplan.h b/dbcon/execplan/calpontselectexecutionplan.h index f77d5c0f7..cacb9d3f5 100644 --- a/dbcon/execplan/calpontselectexecutionplan.h +++ b/dbcon/execplan/calpontselectexecutionplan.h @@ -354,6 +354,10 @@ public: void derivedTbAlias(const std::string derivedTbAlias) { fDerivedTbAlias = derivedTbAlias; } const std::string derivedTbAlias() const { return fDerivedTbAlias; } + void derivedTbView(const std::string derivedTbView) { fDerivedTbView = derivedTbView; } + const std::string derivedTbView() const { return fDerivedTbView; } + + void limitStart(const uint64_t limitStart) { fLimitStart = limitStart; } const uint64_t limitStart() const { return fLimitStart; } @@ -565,6 +569,7 @@ private: // for subselect uint64_t fSubType; std::string fDerivedTbAlias; + std::string fDerivedTbView; // for limit uint64_t fLimitStart; diff --git a/dbcon/mysql/ha_calpont_execplan.cpp b/dbcon/mysql/ha_calpont_execplan.cpp index 2383f7ff7..d0e96c412 100755 --- a/dbcon/mysql/ha_calpont_execplan.cpp +++ b/dbcon/mysql/ha_calpont_execplan.cpp @@ -1833,7 +1833,14 @@ SimpleColumn* buildSimpleColFromDerivedTable(gp_walk_info& gwi, Item_field* ifp) sc->colPosition(j); string tableAlias(csep->derivedTbAlias()); sc->tableAlias(lower(tableAlias)); - sc->viewName(lower(viewName)); + if (!viewName.empty()) + { + sc->viewName(viewName); + } + else + { + sc->viewName(csep->derivedTbView()); + } sc->resultType(cols[j]->resultType()); sc->hasAggregate(cols[j]->hasAggregate()); if (col) diff --git a/dbcon/mysql/ha_from_sub.cpp b/dbcon/mysql/ha_from_sub.cpp index 0dd9eeb0e..a12221fc5 100755 --- a/dbcon/mysql/ha_from_sub.cpp +++ b/dbcon/mysql/ha_from_sub.cpp @@ -317,6 +317,7 @@ SCSEP FromSubQuery::transform() gwi.subQuery = this; gwi.viewName = fGwip.viewName; csep->derivedTbAlias(fAlias); // always lower case + csep->derivedTbView(fGwip.viewName.alias); if (getSelectPlan(gwi, *fFromSub, csep) != 0) { From e3e59dd167658810218a4f6c5c4b49b69ff6afba Mon Sep 17 00:00:00 2001 From: Ben Thompson Date: Mon, 17 Jun 2019 12:33:15 -0500 Subject: [PATCH 31/41] Update Version to 1.2.5 --- VERSION | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/VERSION b/VERSION index 91bc7ea08..f67d0ae71 100644 --- a/VERSION +++ b/VERSION @@ -1,4 +1,4 @@ COLUMNSTORE_VERSION_MAJOR=1 COLUMNSTORE_VERSION_MINOR=2 -COLUMNSTORE_VERSION_PATCH=4 +COLUMNSTORE_VERSION_PATCH=5 COLUMNSTORE_VERSION_RELEASE=1 From b05c4e8df4e1f3a2bdbbd14896bce658b074d0fb Mon Sep 17 00:00:00 2001 From: Andrew Hutchings Date: Tue, 18 Jun 2019 16:13:50 +0100 Subject: [PATCH 32/41] MCOL-2243 Only set length and dec when dec is zero Length and dec for columns only needs to be set when it is the result of a function. Otherwise it could lead to odd side-effects in MariaDB. --- dbcon/mysql/ha_calpont_impl.cpp | 14 ++++++++++++-- 1 file changed, 12 insertions(+), 2 deletions(-) diff --git a/dbcon/mysql/ha_calpont_impl.cpp b/dbcon/mysql/ha_calpont_impl.cpp index a6cbea6a9..eb3b85384 100755 --- a/dbcon/mysql/ha_calpont_impl.cpp +++ b/dbcon/mysql/ha_calpont_impl.cpp @@ -579,7 +579,12 @@ int fetchNextRow(uchar *buf, cal_table_info& ti, cal_connection_info* ci) // bug 3485, reserve enough space for the longest float value // -3.402823466E+38 to -1.175494351E-38, 0, and // 1.175494351E-38 to 3.402823466E+38. - (*f)->field_length = 40; + if (!f2->dec) + { + (*f)->field_length = 40; + f2->dec = row.getScale(s); + } + //float float_val = *(float*)(&value); //f2->store(float_val); if (f2->decimals() < (uint32_t)row.getScale(s)) @@ -602,7 +607,12 @@ int fetchNextRow(uchar *buf, cal_table_info& ti, cal_connection_info* ci) // bug 3483, reserve enough space for the longest double value // -1.7976931348623157E+308 to -2.2250738585072014E-308, 0, and // 2.2250738585072014E-308 to 1.7976931348623157E+308. - (*f)->field_length = 310; + if (!f2->dec) + { + (*f)->field_length = 310; + f2->dec = row.getScale(s); + } + //double double_val = *(double*)(&value); //f2->store(double_val); if (f2->decimals() < (uint32_t)row.getScale(s)) From 4cf2c37c1801de6c7cce02d14c439b976e17428e Mon Sep 17 00:00:00 2001 From: David Hall Date: Thu, 20 Jun 2019 09:24:58 -0500 Subject: [PATCH 33/41] MCOL-3343 Handle windowfunction simple column --- dbcon/execplan/windowfunctioncolumn.cpp | 1 - dbcon/joblist/jlf_common.cpp | 58 ++++++++++++++++++++----- dbcon/joblist/jlf_common.h | 7 +-- dbcon/joblist/joblistfactory.cpp | 20 +++++++-- dbcon/joblist/tupleaggregatestep.cpp | 6 ++- dbcon/joblist/windowfunctionstep.cpp | 52 ++++++++++++++++++++++ dbcon/joblist/windowfunctionstep.h | 2 + 7 files changed, 126 insertions(+), 20 deletions(-) diff --git a/dbcon/execplan/windowfunctioncolumn.cpp b/dbcon/execplan/windowfunctioncolumn.cpp index 5c3e57407..e01c5c010 100644 --- a/dbcon/execplan/windowfunctioncolumn.cpp +++ b/dbcon/execplan/windowfunctioncolumn.cpp @@ -386,7 +386,6 @@ void WindowFunctionColumn::adjustResultType() fResultType.colDataType = CalpontSystemCatalog::LONGDOUBLE; fResultType.colWidth = sizeof(long double); fResultType.precision = -1; - fResultType.scale = 0; } } diff --git a/dbcon/joblist/jlf_common.cpp b/dbcon/joblist/jlf_common.cpp index 7ce066427..33e7e9657 100644 --- a/dbcon/joblist/jlf_common.cpp +++ b/dbcon/joblist/jlf_common.cpp @@ -386,19 +386,57 @@ CalpontSystemCatalog::OID tableOid(const SimpleColumn* sc, boost::shared_ptr(sc); uint32_t pseudoType = (pc) ? pc->pseudoType() : execplan::PSEUDO_UNKNOWN; - return getTupleKey_(jobInfo, sc->oid(), sc->columnName(), extractTableAlias(sc), - sc->schemaName(), sc->viewName(), - ((sc->joinInfo() & execplan::JOIN_CORRELATED) != 0), - pseudoType, (sc->isInfiniDB() ? 0 : 1)); + if (sc == NULL) + { + return -1; + } + + if (add) + { + // setTupleInfo first if add is true, ok if already set. + if (sc->schemaName().empty()) + { + SimpleColumn tmp(*sc, jobInfo.sessionId); + tmp.oid(tableOid(sc, jobInfo.csc) + 1 + sc->colPosition()); + key = getTupleKey(jobInfo, &tmp); // sub-query should be there + } + else + { + CalpontSystemCatalog::ColType ct = sc->colType(); + string alias(extractTableAlias(sc)); + CalpontSystemCatalog::OID tblOid = tableOid(sc, jobInfo.csc); + TupleInfo ti(setTupleInfo(ct, sc->oid(), jobInfo, tblOid, sc, alias)); + key = ti.key; + + CalpontSystemCatalog::OID dictOid = isDictCol(ct); + + if (dictOid > 0) + { + ti = setTupleInfo(ct, dictOid, jobInfo, tblOid, sc, alias); + jobInfo.keyInfo->dictKeyMap[key] = ti.key; + key = ti.key; + } + } + } + else + { + // TupleInfo is expected to be set already + return getTupleKey_(jobInfo, sc->oid(), sc->columnName(), extractTableAlias(sc), + sc->schemaName(), sc->viewName(), + ((sc->joinInfo() & execplan::JOIN_CORRELATED) != 0), + pseudoType, (sc->isInfiniDB() ? 0 : 1)); + } + + return key; } - - + uint32_t getTupleKey(JobInfo& jobInfo, const SRCP& srcp, bool add) { int key = -1; @@ -608,7 +646,7 @@ uint32_t getExpTupleKey(const JobInfo& jobInfo, uint64_t eid, bool cr) } -void addAggregateColumn(AggregateColumn* agc, int idx, RetColsVector& vec, JobInfo& jobInfo) +void addAggregateColumn(ReturnedColumn* agc, int idx, RetColsVector& vec, JobInfo& jobInfo) { uint32_t eid = agc->expressionId(); setExpTupleInfo(agc->resultType(), eid, agc->alias(), jobInfo); diff --git a/dbcon/joblist/jlf_common.h b/dbcon/joblist/jlf_common.h index 18c1ea396..6cf28af8c 100644 --- a/dbcon/joblist/jlf_common.h +++ b/dbcon/joblist/jlf_common.h @@ -401,8 +401,9 @@ execplan::CalpontSystemCatalog::OID tableOid(const execplan::SimpleColumn* sc, /** @brief Returns the unique ID to be used in tupleInfo * */ -uint32_t getTupleKey(const JobInfo& jobInfo, - const execplan::SimpleColumn* sc); +uint32_t getTupleKey(JobInfo& jobInfo, + const execplan::SimpleColumn* sc, + bool add = false); uint32_t getTableKey(const JobInfo& jobInfo, execplan::CalpontSystemCatalog::OID tableOid, const std::string& alias, @@ -464,7 +465,7 @@ TupleInfo setExpTupleInfo(const execplan::ReturnedColumn* rc, JobInfo& jobInfo); /** @brief add an aggregate column info * */ -void addAggregateColumn(execplan::AggregateColumn*, int, RetColsVector&, JobInfo&); +void addAggregateColumn(execplan::ReturnedColumn*, int, RetColsVector&, JobInfo&); void makeJobSteps(execplan::CalpontSelectExecutionPlan* csep, JobInfo& jobInfo, JobStepVector& querySteps, JobStepVector& projectSteps, diff --git a/dbcon/joblist/joblistfactory.cpp b/dbcon/joblist/joblistfactory.cpp index 0a141f03a..66245a8e2 100644 --- a/dbcon/joblist/joblistfactory.cpp +++ b/dbcon/joblist/joblistfactory.cpp @@ -274,9 +274,11 @@ const JobStepVector doProject(const RetColsVector& retCols, JobInfo& jobInfo) if (retCols[i]->windowfunctionColumnList().size() > 0) jobInfo.expressionVec.push_back(key); - else if (find(jobInfo.expressionVec.begin(), jobInfo.expressionVec.end(), key) == - jobInfo.expressionVec.end()) + else if (find(jobInfo.expressionVec.begin(), jobInfo.expressionVec.end(), key) + == jobInfo.expressionVec.end()) + { jobInfo.returnedExpressions.push_back(sjstep); + } //put place hold column in projection list jobInfo.pjColList.push_back(ti); @@ -1047,16 +1049,21 @@ const JobStepVector doAggProject(const CalpontSelectExecutionPlan* csep, JobInfo const FunctionColumn* fc = NULL; const WindowFunctionColumn* wc = NULL; bool hasAggCols = false; + bool hasWndCols = false; if ((ac = dynamic_cast(srcp.get())) != NULL) { if (ac->aggColumnList().size() > 0) hasAggCols = true; + if (ac->windowfunctionColumnList().size() > 0) + hasWndCols = true; } else if ((fc = dynamic_cast(srcp.get())) != NULL) { if (fc->aggColumnList().size() > 0) hasAggCols = true; + if (fc->windowfunctionColumnList().size() > 0) + hasWndCols = true; } else if (dynamic_cast(srcp.get()) != NULL) { @@ -1081,7 +1088,7 @@ const JobStepVector doAggProject(const CalpontSelectExecutionPlan* csep, JobInfo TupleInfo ti(setExpTupleInfo(ct, eid, srcp.get()->alias(), jobInfo)); tupleKey = ti.key; - if (hasAggCols) + if (hasAggCols && !hasWndCols) jobInfo.expressionVec.push_back(tupleKey); } @@ -1195,16 +1202,21 @@ const JobStepVector doAggProject(const CalpontSelectExecutionPlan* csep, JobInfo const FunctionColumn* fc = NULL; const WindowFunctionColumn* wc = NULL; bool hasAggCols = false; + bool hasWndCols = false; if ((ac = dynamic_cast(srcp.get())) != NULL) { if (ac->aggColumnList().size() > 0) hasAggCols = true; + if (ac->windowfunctionColumnList().size() > 0) + hasWndCols = true; } else if ((fc = dynamic_cast(srcp.get())) != NULL) { if (fc->aggColumnList().size() > 0) hasAggCols = true; + if (fc->windowfunctionColumnList().size() > 0) + hasWndCols = true; } else if (dynamic_cast(srcp.get()) != NULL) { @@ -1229,7 +1241,7 @@ const JobStepVector doAggProject(const CalpontSelectExecutionPlan* csep, JobInfo TupleInfo ti(setExpTupleInfo(ct, eid, srcp.get()->alias(), jobInfo)); tupleKey = ti.key; - if (hasAggCols) + if (hasAggCols && !hasWndCols) jobInfo.expressionVec.push_back(tupleKey); } diff --git a/dbcon/joblist/tupleaggregatestep.cpp b/dbcon/joblist/tupleaggregatestep.cpp index db5a4e8d8..62f084412 100644 --- a/dbcon/joblist/tupleaggregatestep.cpp +++ b/dbcon/joblist/tupleaggregatestep.cpp @@ -5011,7 +5011,8 @@ void TupleAggregateStep::prepExpressionOnAggregate(SP_ROWAGG_UM_t& aggUM, JobInf uint64_t eid = -1; if (((ac = dynamic_cast(it->get())) != NULL) && - (ac->aggColumnList().size() > 0)) + (ac->aggColumnList().size() > 0) && + (ac->windowfunctionColumnList().size() == 0)) { const vector& scols = ac->simpleColumnList(); simpleColumns.insert(simpleColumns.end(), scols.begin(), scols.end()); @@ -5020,7 +5021,8 @@ void TupleAggregateStep::prepExpressionOnAggregate(SP_ROWAGG_UM_t& aggUM, JobInf expressionVec.push_back(*it); } else if (((fc = dynamic_cast(it->get())) != NULL) && - (fc->aggColumnList().size() > 0)) + (fc->aggColumnList().size() > 0) && + (fc->windowfunctionColumnList().size() == 0)) { const vector& sCols = fc->simpleColumnList(); simpleColumns.insert(simpleColumns.end(), sCols.begin(), sCols.end()); diff --git a/dbcon/joblist/windowfunctionstep.cpp b/dbcon/joblist/windowfunctionstep.cpp index fe43faa77..172a7a8a5 100644 --- a/dbcon/joblist/windowfunctionstep.cpp +++ b/dbcon/joblist/windowfunctionstep.cpp @@ -320,6 +320,38 @@ const string WindowFunctionStep::toString() const return oss.str(); } +void WindowFunctionStep::AddSimplColumn(const vector& scs, + JobInfo& jobInfo) +{ + // append the simple columns if not already projected + set scProjected; + + for (RetColsVector::iterator i = jobInfo.projectionCols.begin(); + i != jobInfo.projectionCols.end(); + i++) + { + SimpleColumn* sc = dynamic_cast(i->get()); + + if (sc != NULL) + { + if (sc->schemaName().empty()) + sc->oid(joblist::tableOid(sc, jobInfo.csc) + 1 + sc->colPosition()); + + scProjected.insert(UniqId(sc)); + } + } + + for (vector::const_iterator i = scs.begin(); i != scs.end(); i++) + { + if (scProjected.find(UniqId(*i)) == scProjected.end()) + { + jobInfo.windowDels.push_back(SRCP((*i)->clone())); + jobInfo.windowSet.insert(getTupleKey(jobInfo, *i, true)); + scProjected.insert(UniqId(*i)); + } + } +} + void WindowFunctionStep::checkWindowFunction(CalpontSelectExecutionPlan* csep, JobInfo& jobInfo) { // window functions in select clause, selected or in expression @@ -404,6 +436,23 @@ void WindowFunctionStep::checkWindowFunction(CalpontSelectExecutionPlan* csep, J if (jobInfo.windowCols.empty()) return; + // Add in the non-window side of arithmetic columns and functions + for (uint64_t i = 0; i < jobInfo.windowExps.size(); i++) + { + const ArithmeticColumn* ac = + dynamic_cast(jobInfo.windowExps[i].get()); + const FunctionColumn* fc = + dynamic_cast(jobInfo.windowExps[i].get()); + + if (ac != NULL && ac->windowfunctionColumnList().size() > 0) + { + AddSimplColumn(ac->simpleColumnList(), jobInfo); + } + else if (fc != NULL && fc->windowfunctionColumnList().size() > 0) + { + AddSimplColumn(fc->simpleColumnList(), jobInfo); + } + } // reconstruct the delivered column list with auxiliary columns set colSet; jobInfo.deliveredCols.resize(0); @@ -445,7 +494,10 @@ void WindowFunctionStep::checkWindowFunction(CalpontSelectExecutionPlan* csep, J key = getTupleKey(jobInfo, *j, true); if (colSet.find(key) == colSet.end()) + { jobInfo.deliveredCols.push_back(*j); + jobInfo.windowSet.insert(getTupleKey(jobInfo, *j, true)); + } colSet.insert(key); } diff --git a/dbcon/joblist/windowfunctionstep.h b/dbcon/joblist/windowfunctionstep.h index f483c6027..f6a695a65 100644 --- a/dbcon/joblist/windowfunctionstep.h +++ b/dbcon/joblist/windowfunctionstep.h @@ -148,6 +148,8 @@ private: void formatMiniStats(); void printCalTrace(); + static void AddSimplColumn(const vector& scs, JobInfo& jobInfo); + class Runner { public: From 2255d41eaf057d5e91be26c3c483aa813f74d35f Mon Sep 17 00:00:00 2001 From: David Hall Date: Thu, 20 Jun 2019 15:54:05 -0500 Subject: [PATCH 34/41] MCOL-3343 Error when no group by for unaggregated fields in window functions. --- dbcon/joblist/windowfunctionstep.cpp | 10 ++++++++-- 1 file changed, 8 insertions(+), 2 deletions(-) diff --git a/dbcon/joblist/windowfunctionstep.cpp b/dbcon/joblist/windowfunctionstep.cpp index 172a7a8a5..1a603a98e 100644 --- a/dbcon/joblist/windowfunctionstep.cpp +++ b/dbcon/joblist/windowfunctionstep.cpp @@ -346,7 +346,10 @@ void WindowFunctionStep::AddSimplColumn(const vector& scs, if (scProjected.find(UniqId(*i)) == scProjected.end()) { jobInfo.windowDels.push_back(SRCP((*i)->clone())); - jobInfo.windowSet.insert(getTupleKey(jobInfo, *i, true)); +// MCOL-3343 Enable this if we decide to allow Window Functions to run with +// aggregates with no group by. MariaDB allows this. Nobody else in the world does. +// There will be more work to get it to function if we try this. +// jobInfo.windowSet.insert(getTupleKey(jobInfo, *i, true)); scProjected.insert(UniqId(*i)); } } @@ -496,7 +499,10 @@ void WindowFunctionStep::checkWindowFunction(CalpontSelectExecutionPlan* csep, J if (colSet.find(key) == colSet.end()) { jobInfo.deliveredCols.push_back(*j); - jobInfo.windowSet.insert(getTupleKey(jobInfo, *j, true)); +// MCOL-3343 Enable this if we decide to allow Window Functions to run with +// aggregates with no group by. MariaDB allows this. Nobody else in the world does. +// There will be more work to get it to function if we try this. +// jobInfo.windowSet.insert(getTupleKey(jobInfo, *j, true)); } colSet.insert(key); From 089ec7d4dcfe08a0014b30862bc9588a2f9439f4 Mon Sep 17 00:00:00 2001 From: Andrew Hutchings Date: Tue, 25 Jun 2019 17:15:30 +0100 Subject: [PATCH 35/41] MCOL-3391 Fix columnstore_upgrade unicode support The columnstore_upgrade() procedure would break on utf8 table names. This patch fixes that. --- dbcon/mysql/columnstore_info.sql | 3 +-- 1 file changed, 1 insertion(+), 2 deletions(-) diff --git a/dbcon/mysql/columnstore_info.sql b/dbcon/mysql/columnstore_info.sql index 7655b5f16..b2f5e50d1 100644 --- a/dbcon/mysql/columnstore_info.sql +++ b/dbcon/mysql/columnstore_info.sql @@ -101,7 +101,7 @@ END // create procedure columnstore_upgrade() `columnstore_upgrade`: BEGIN DECLARE done INTEGER DEFAULT 0; - DECLARE schema_table VARCHAR(100) DEFAULT ""; + DECLARE schema_table VARCHAR(100) CHARACTER SET utf8 DEFAULT ""; DECLARE table_list CURSOR FOR select concat('`', table_schema,'`.`',table_name,'`') from information_schema.tables where engine='columnstore'; DECLARE CONTINUE HANDLER FOR NOT FOUND SET done = 1; OPEN table_list; @@ -115,6 +115,5 @@ create procedure columnstore_upgrade() DEALLOCATE PREPARE stmt; END LOOP; END // -delimiter ; DELIMITER ; From 9c4ac78105d015c0f15605c92fac1331334b0191 Mon Sep 17 00:00:00 2001 From: Andrew Hutchings Date: Wed, 26 Jun 2019 07:50:17 +0100 Subject: [PATCH 36/41] MCOL-2225 Make cross-engine step use backticks If an InnoDB table has spaces in the column names the cross engine step would fail. This patch wraps quotes around the table and column names to support this use case. --- dbcon/execplan/simplecolumn.cpp | 4 ++-- dbcon/joblist/crossenginestep.cpp | 6 +++--- 2 files changed, 5 insertions(+), 5 deletions(-) diff --git a/dbcon/execplan/simplecolumn.cpp b/dbcon/execplan/simplecolumn.cpp index 206194954..75864892e 100644 --- a/dbcon/execplan/simplecolumn.cpp +++ b/dbcon/execplan/simplecolumn.cpp @@ -211,9 +211,9 @@ const string SimpleColumn::data() const if (!fData.empty()) return fData; else if (!fTableAlias.empty()) - return string(fSchemaName + '.' + fTableAlias + '.' + fColumnName); + return string("`" + fSchemaName + "`.`" + fTableAlias + "`.`" + fColumnName + "`"); - return string(fSchemaName + '.' + fTableName + '.' + fColumnName); + return string("`" + fSchemaName + "`.`" + fTableName + "`.`" + fColumnName + "`"); } SimpleColumn& SimpleColumn::operator=(const SimpleColumn& rhs) diff --git a/dbcon/joblist/crossenginestep.cpp b/dbcon/joblist/crossenginestep.cpp index d3cef7928..e46e1fe22 100644 --- a/dbcon/joblist/crossenginestep.cpp +++ b/dbcon/joblist/crossenginestep.cpp @@ -727,17 +727,17 @@ void CrossEngineStep::setProjectBPP(JobStep* jobStep1, JobStep*) else fSelectClause += "SELECT "; - fSelectClause += jobStep1->name(); + fSelectClause += "`" + jobStep1->name() + "`"; } string CrossEngineStep::makeQuery() { ostringstream oss; - oss << fSelectClause << " FROM " << fTable; + oss << fSelectClause << " FROM `" << fTable << "`"; if (fTable.compare(fAlias) != 0) - oss << " " << fAlias; + oss << " `" << fAlias << "`"; if (!fWhereClause.empty()) oss << fWhereClause; From 6bee5f9f61956d5e54a25eecb71f3f5fc8a84dda Mon Sep 17 00:00:00 2001 From: Andrew Hutchings Date: Wed, 3 Jul 2019 08:46:43 +0100 Subject: [PATCH 37/41] MCOL-3395 Always clear dctnry cache on close The dictionary cache needs to be cleared every time it is closed so that on bulk insert we don't mix the cache between columns. --- writeengine/dictionary/we_dctnry.cpp | 3 +-- 1 file changed, 1 insertion(+), 2 deletions(-) diff --git a/writeengine/dictionary/we_dctnry.cpp b/writeengine/dictionary/we_dctnry.cpp index 6200cc248..ea677b341 100644 --- a/writeengine/dictionary/we_dctnry.cpp +++ b/writeengine/dictionary/we_dctnry.cpp @@ -419,8 +419,7 @@ int Dctnry::closeDctnry(bool realClose) return rc; //cout <<"Init called! m_dctnryOID =" << m_dctnryOID << endl; - if (realClose) - freeStringCache( ); + freeStringCache( ); return NO_ERROR; } From 621948660763d05b3fe5aed0ea0695099e248555 Mon Sep 17 00:00:00 2001 From: Andrew Hutchings Date: Wed, 10 Jul 2019 11:07:02 +0100 Subject: [PATCH 38/41] MCOL-3399 Fix VARCHAR LDI length processing Accidentally passed the wrong number of bytes when processing the row provided in LDI. Revert the buf increment code. --- dbcon/mysql/ha_calpont_dml.cpp | 14 ++++---------- 1 file changed, 4 insertions(+), 10 deletions(-) diff --git a/dbcon/mysql/ha_calpont_dml.cpp b/dbcon/mysql/ha_calpont_dml.cpp index 87fbad147..6d1b7626e 100644 --- a/dbcon/mysql/ha_calpont_dml.cpp +++ b/dbcon/mysql/ha_calpont_dml.cpp @@ -862,12 +862,6 @@ int ha_calpont_impl_write_batch_row_(uchar* buf, TABLE* table, cal_impl_if::cal_ case CalpontSystemCatalog::VARCHAR: { - size_t length; - if (ci.utf8) - length = (ci.columnTypes[colpos].colWidth * 3); - else - length = ci.columnTypes[colpos].colWidth; - if (nullVal && (ci.columnTypes[colpos].constraintType != CalpontSystemCatalog::NOTNULL_CONSTRAINT)) { fprintf(ci.filePtr, "%c", ci.delimiter); @@ -911,7 +905,6 @@ int ha_calpont_impl_write_batch_row_(uchar* buf, TABLE* table, cal_impl_if::cal_ dataLength = *(uint16_t*) buf; buf = buf + 2 ; } - length = dataLength; escape.assign((char*)buf, dataLength); boost::replace_all(escape, "\\", "\\\\"); fprintf(ci.filePtr, "%c%.*s%c%c", ci.enclosed_by, (int)escape.length(), escape.c_str(), ci.enclosed_by, ci.delimiter); @@ -928,7 +921,6 @@ int ha_calpont_impl_write_batch_row_(uchar* buf, TABLE* table, cal_impl_if::cal_ dataLength = *(uint16_t*) buf; buf = buf + 2 ; } - length = dataLength; escape.assign((char*)buf, dataLength); boost::replace_all(escape, "\\", "\\\\"); @@ -936,8 +928,10 @@ int ha_calpont_impl_write_batch_row_(uchar* buf, TABLE* table, cal_impl_if::cal_ fprintf(ci.filePtr, "%c%.*s%c%c", ci.enclosed_by, (int)escape.length(), escape.c_str(), ci.enclosed_by, ci.delimiter); } } - buf += length; - + if (ci.utf8) + buf += (ci.columnTypes[colpos].colWidth * 3); + else + buf += ci.columnTypes[colpos].colWidth; break; } From a62761137845e75d25766962548c8efb66848445 Mon Sep 17 00:00:00 2001 From: Andrew Hutchings Date: Thu, 11 Jul 2019 13:32:44 +0100 Subject: [PATCH 39/41] Revert "MCOL-2243 Only set length and dec when dec is zero" This reverts commit b05c4e8df4e1f3a2bdbbd14896bce658b074d0fb. --- dbcon/mysql/ha_calpont_impl.cpp | 14 ++------------ 1 file changed, 2 insertions(+), 12 deletions(-) diff --git a/dbcon/mysql/ha_calpont_impl.cpp b/dbcon/mysql/ha_calpont_impl.cpp index 97d4a80e4..d051abcd7 100644 --- a/dbcon/mysql/ha_calpont_impl.cpp +++ b/dbcon/mysql/ha_calpont_impl.cpp @@ -649,12 +649,7 @@ int fetchNextRow(uchar* buf, cal_table_info& ti, cal_connection_info* ci, bool h // bug 3485, reserve enough space for the longest float value // -3.402823466E+38 to -1.175494351E-38, 0, and // 1.175494351E-38 to 3.402823466E+38. - if (!f2->dec) - { - (*f)->field_length = 40; - f2->dec = row.getScale(s); - } - + (*f)->field_length = 40; //float float_val = *(float*)(&value); //f2->store(float_val); if (f2->decimals() < (uint32_t)row.getScale(s)) @@ -683,12 +678,7 @@ int fetchNextRow(uchar* buf, cal_table_info& ti, cal_connection_info* ci, bool h // bug 3483, reserve enough space for the longest double value // -1.7976931348623157E+308 to -2.2250738585072014E-308, 0, and // 2.2250738585072014E-308 to 1.7976931348623157E+308. - if (!f2->dec) - { - (*f)->field_length = 310; - f2->dec = row.getScale(s); - } - + (*f)->field_length = 310; //double double_val = *(double*)(&value); //f2->store(double_val); if ((f2->decimals() == DECIMAL_NOT_SPECIFIED && row.getScale(s) > 0) From c683b6bfc061193eed8b131dd1a86d932eb6303a Mon Sep 17 00:00:00 2001 From: David Hall Date: Tue, 16 Jul 2019 18:19:03 -0500 Subject: [PATCH 40/41] MCOL-3404 Revert experimental changes from MCOL-3343 --- dbcon/joblist/joblistfactory.cpp | 9 ++------- utils/rowgroup/rowgroup.h | 3 ++- 2 files changed, 4 insertions(+), 8 deletions(-) diff --git a/dbcon/joblist/joblistfactory.cpp b/dbcon/joblist/joblistfactory.cpp index 66245a8e2..152358bf7 100644 --- a/dbcon/joblist/joblistfactory.cpp +++ b/dbcon/joblist/joblistfactory.cpp @@ -399,7 +399,6 @@ void checkHavingClause(CalpontSelectExecutionPlan* csep, JobInfo& jobInfo) void preProcessFunctionOnAggregation(const vector& scs, const vector& aggs, - const vector& wcs, JobInfo& jobInfo) { // append the simple columns if not already projected @@ -433,10 +432,6 @@ void preProcessFunctionOnAggregation(const vector& scs, for (vector::const_iterator i = aggs.begin(); i != aggs.end(); i++) { addAggregateColumn(*i, -1, jobInfo.projectionCols, jobInfo); - if (wcs.size() > 0) - { - jobInfo.nonConstDelCols.push_back(SRCP((*i)->clone())); - } } } @@ -488,12 +483,12 @@ void checkReturnedColumns(CalpontSelectExecutionPlan* csep, JobInfo& jobInfo) if (ac != NULL && ac->aggColumnList().size() > 0) { jobInfo.nonConstCols[i]->outputIndex(i); - preProcessFunctionOnAggregation(ac->simpleColumnList(), ac->aggColumnList(), ac->windowfunctionColumnList(), jobInfo); + preProcessFunctionOnAggregation(ac->simpleColumnList(), ac->aggColumnList(), jobInfo); } else if (fc != NULL && fc->aggColumnList().size() > 0) { jobInfo.nonConstCols[i]->outputIndex(i); - preProcessFunctionOnAggregation(fc->simpleColumnList(), fc->aggColumnList(), fc->windowfunctionColumnList(), jobInfo); + preProcessFunctionOnAggregation(fc->simpleColumnList(), fc->aggColumnList(), jobInfo); } } } diff --git a/utils/rowgroup/rowgroup.h b/utils/rowgroup/rowgroup.h index b91e0f0ef..2334d22bc 100644 --- a/utils/rowgroup/rowgroup.h +++ b/utils/rowgroup/rowgroup.h @@ -1772,7 +1772,8 @@ inline void copyRow(const Row& in, Row* out, uint32_t colCount) { if (UNLIKELY(in.getColTypes()[i] == execplan::CalpontSystemCatalog::VARBINARY || in.getColTypes()[i] == execplan::CalpontSystemCatalog::BLOB || - in.getColTypes()[i] == execplan::CalpontSystemCatalog::TEXT)) + in.getColTypes()[i] == execplan::CalpontSystemCatalog::TEXT || + in.getColTypes()[i] == execplan::CalpontSystemCatalog::CLOB)) out->setVarBinaryField(in.getVarBinaryStringField(i), i); else if (UNLIKELY(in.isLongString(i))) //out->setStringField(in.getStringField(i), i); From 83a892422925bedcbc22f4f873747d28fc47a299 Mon Sep 17 00:00:00 2001 From: Andrew Hutchings Date: Mon, 29 Jul 2019 13:48:28 +0100 Subject: [PATCH 41/41] Fix namespace and class changes --- dbcon/joblist/windowfunctionstep.h | 2 +- dbcon/mysql/ha_calpont_impl.cpp | 6 +----- dmlproc/dmlproc.cpp | 4 ++-- 3 files changed, 4 insertions(+), 8 deletions(-) diff --git a/dbcon/joblist/windowfunctionstep.h b/dbcon/joblist/windowfunctionstep.h index f0154ae0a..eaa020fc3 100644 --- a/dbcon/joblist/windowfunctionstep.h +++ b/dbcon/joblist/windowfunctionstep.h @@ -148,7 +148,7 @@ private: void formatMiniStats(); void printCalTrace(); - static void AddSimplColumn(const vector& scs, JobInfo& jobInfo); + static void AddSimplColumn(const std::vector& scs, JobInfo& jobInfo); class Runner { diff --git a/dbcon/mysql/ha_calpont_impl.cpp b/dbcon/mysql/ha_calpont_impl.cpp index fd0f0fcb8..bb08c6cb9 100644 --- a/dbcon/mysql/ha_calpont_impl.cpp +++ b/dbcon/mysql/ha_calpont_impl.cpp @@ -1594,11 +1594,7 @@ uint32_t doUpdateDelete(THD* thd) { if (timeStampColumnNames.find(onUpdateTimeStampColumns[i]) == timeStampColumnNames.end()) { - columnAssignmentPtr = new ColumnAssignment(); - columnAssignmentPtr->fColumn = string(onUpdateTimeStampColumns[i]); - columnAssignmentPtr->fOperator = "="; - columnAssignmentPtr->fFuncScale = 0; - columnAssignmentPtr->fFromCol = false; + columnAssignmentPtr = new ColumnAssignment(string(onUpdateTimeStampColumns[i]), "=", ""); struct timeval tv; char buf[64]; gettimeofday(&tv, 0); diff --git a/dmlproc/dmlproc.cpp b/dmlproc/dmlproc.cpp index 5adf9404d..09e5b9128 100644 --- a/dmlproc/dmlproc.cpp +++ b/dmlproc/dmlproc.cpp @@ -86,7 +86,7 @@ using namespace joblist; namespace fs = boost::filesystem; -ThreadPool DMLServer::fDmlPackagepool(10, 0); +threadpool::ThreadPool DMLServer::fDmlPackagepool(10, 0); namespace { @@ -658,7 +658,7 @@ int main(int argc, char* argv[]) JobStep::jobstepThreadPool.setDebug(true); JobStep::jobstepThreadPool.invoke(threadpool::ThreadPoolMonitor(&JobStep::jobstepThreadPool)); DMLServer::fDmlPackagepool.setDebug(true); - DMLServer::fDmlPackagepool.invoke(ThreadPoolMonitor(&DMLServer::fDmlPackagepool)); + DMLServer::fDmlPackagepool.invoke(threadpool::ThreadPoolMonitor(&DMLServer::fDmlPackagepool)); } //set ACTIVE state