From 0b885c3830846da36377c089fbe0267a9bc39928 Mon Sep 17 00:00:00 2001 From: jmrojas2332 Date: Tue, 10 Mar 2020 21:12:23 +0000 Subject: [PATCH 01/21] MCOL-3830 post-mysql-install fix on systems not running systemd --- .../columnstore-pre-uninstall.in | 11 ++++++++++- oam/install_scripts/columnstore.in | 10 +++++++++- oam/install_scripts/post-mysql-install | 19 ++++++++++++++----- 3 files changed, 33 insertions(+), 7 deletions(-) diff --git a/oam/install_scripts/columnstore-pre-uninstall.in b/oam/install_scripts/columnstore-pre-uninstall.in index 070633a41..204698d89 100755 --- a/oam/install_scripts/columnstore-pre-uninstall.in +++ b/oam/install_scripts/columnstore-pre-uninstall.in @@ -4,6 +4,15 @@ # # pre-uninstall steps for columnstore install +running_systemd() { + if [ ps --no-headers -o comm 1 == "systemd" ]; then + echo 0 + else + echo 1 + fi + +} + rpmmode=install user=`whoami 2>/dev/null` @@ -28,7 +37,7 @@ columnstore stop > /dev/null 2>&1 # Test we are using systemd systemctl cat mariadb.service > /dev/null 2>&1 -if [ $? -eq 0 ]; then +if [ $? -eq 0 ] && [ $(running_systemd) -eq 0 ]; then systemctl stop mariadb.service > /dev/null 2>&1 else pkill mysqld diff --git a/oam/install_scripts/columnstore.in b/oam/install_scripts/columnstore.in index 9ef7cf7ad..98355d255 100644 --- a/oam/install_scripts/columnstore.in +++ b/oam/install_scripts/columnstore.in @@ -17,6 +17,14 @@ # Short-Description: Start/stop MariaDB Columnstore DW DBMS ### END INIT INFO +running_systemd() { + if [ ps --no-headers -o comm 1 == "systemd" ]; then + echo 0 + else + echo 1 + fi +} + USER=`whoami 2>/dev/null` # Source function library. @@ -95,7 +103,7 @@ stop() { fuser -k 8604/tcp > /dev/null 2>&1 # Test we are using systemd systemctl cat mariadb.service > /dev/null 2>&1 - if [ $? -eq 0 ]; then + if [ $? -eq 0 ] && [ $(running_systemd) -eq 0 ]; then systemctl stop mariadb.service > /dev/null 2>&1 else pkill mysqld diff --git a/oam/install_scripts/post-mysql-install b/oam/install_scripts/post-mysql-install index 81808c5da..de2d920da 100755 --- a/oam/install_scripts/post-mysql-install +++ b/oam/install_scripts/post-mysql-install @@ -4,6 +4,15 @@ # # Post-install steps for calpont-mysql install +# check if running systemd +running_systemd() { + if [ ps --no-headers -o comm 1 == "systemd" ]; then + echo 0 + else + echo 1 + fi +} + # check log for error checkForError() { # check for password error @@ -13,7 +22,7 @@ checkForError() { rm -f ${tmpdir}/error.check # Test we are using systemd systemctl cat mariadb.service > /dev/null 2>&1 - if [ $? -eq 0 ]; then + if [ $? -eq 0 ] && [ $(running_systemd) -eq 0 ]; then systemctl stop mariadb.service > /dev/null 2>&1 else pkill mysqld @@ -62,7 +71,7 @@ done # Restart in the same way that mysqld will be started normally. # Test we are using systemd systemctl cat mariadb.service > /dev/null 2>&1 -if [ $? -eq 0 ]; then +if [ $? -eq 0 ] && [ $(running_systemd) -eq 0 ]; then systemctl stop mariadb.service > /dev/null 2>&1 else pkill mysqld @@ -71,7 +80,7 @@ sleep 2 export MYSQL_OPTS="--skip-grant-tables" # Test we are using systemd systemctl cat mariadb.service > /dev/null 2>&1 -if [ $? -eq 0 ]; then +if [ $? -eq 0 ] && [ $(running_systemd) -eq 0 ]; then systemctl start mariadb.service else /usr/bin/mysqld_safe --skip-grant-tables & @@ -87,7 +96,7 @@ if [ $? -ne 0 ]; then echo "ERROR: Invalid password in .my.cnf, or Columnstore plugin install missing" # Test we are using systemd systemctl cat mariadb.service > /dev/null 2>&1 - if [ $? -eq 0 ]; then + if [ $? -eq 0 ] && [ $(running_systemd) -eq 0 ]; then systemctl stop mariadb.service > /dev/null 2>&1 else pkill mysqld @@ -98,7 +107,7 @@ fi # Test we are using systemd systemctl cat mariadb.service > /dev/null 2>&1 -if [ $? -eq 0 ]; then +if [ $? -eq 0 ] && [ $(running_systemd) -eq 0 ]; then systemctl stop mariadb.service > /dev/null 2>&1 else pkill mysqld From efb97b1415c5d3d7eb3cf19df9f77d30c19b6672 Mon Sep 17 00:00:00 2001 From: Patrick LeBlanc Date: Wed, 11 Mar 2020 11:14:44 -0400 Subject: [PATCH 02/21] MCOL-2022: aws specific options can't use nvme dev names Checkpointing a minimal attempt to disable the aws-specific functionality for new installations. --- oamapps/postConfigure/CMakeLists.txt | 2 +- oamapps/postConfigure/postConfigure.cpp | 18 ++++++++++++------ 2 files changed, 13 insertions(+), 7 deletions(-) diff --git a/oamapps/postConfigure/CMakeLists.txt b/oamapps/postConfigure/CMakeLists.txt index 8f573d0a8..be02574c2 100644 --- a/oamapps/postConfigure/CMakeLists.txt +++ b/oamapps/postConfigure/CMakeLists.txt @@ -65,5 +65,5 @@ install(TARGETS mycnfUpgrade DESTINATION ${ENGINE_BINDIR} COMPONENT columnstore- ########### next target ############### -install(PROGRAMS quick_installer_single_server.sh quick_installer_multi_server.sh quick_installer_amazon.sh +install(PROGRAMS quick_installer_single_server.sh quick_installer_multi_server.sh DESTINATION ${ENGINE_BINDIR} COMPONENT columnstore-platform) diff --git a/oamapps/postConfigure/postConfigure.cpp b/oamapps/postConfigure/postConfigure.cpp index 03cdfb7c0..8de961c07 100644 --- a/oamapps/postConfigure/postConfigure.cpp +++ b/oamapps/postConfigure/postConfigure.cpp @@ -316,11 +316,6 @@ int main(int argc, char* argv[]) multi_server_quick_install = true; noPrompting = true; } - else if( string("-qa") == argv[i] ) - { - amazon_quick_install = true; - noPrompting = true; - } else if ( string("-f") == argv[i] ) nodeps = "--nodeps"; else if ( string("-o") == argv[i] ) @@ -460,7 +455,7 @@ int main(int argc, char* argv[]) else { cout << " ERROR: Invalid Argument = " << argv[i] << endl; - cout << " Usage: postConfigure [-h][-c][-u][-p][-qs][-qm][-qa][-port][-i][-sn][-pm-ip-addrs][-um-ip-addrs][-pm-count][-um-count][-x][-xr][-numBlocksPct][-totalUmMemory]" << endl; + cout << " Usage: postConfigure [-h][-c][-u][-p][-qs][-qm][-port][-i][-sn][-pm-ip-addrs][-um-ip-addrs][-pm-count][-um-count][-x][-xr][-numBlocksPct][-totalUmMemory]" << endl; exit (1); } } @@ -1354,6 +1349,7 @@ int main(int argc, char* argv[]) bool amazonInstall = false; string cloud = oam::UnassignedName; + #if 0 if (!multi_server_quick_install) { string amazonLog = tmpDir + "/amazon.log"; @@ -1393,6 +1389,7 @@ int main(int argc, char* argv[]) amazonInstall = true; } } + #endif try { @@ -1402,6 +1399,15 @@ int main(int argc, char* argv[]) { cloud = oam::UnassignedName; } + + #if XXXPAT + if (cloud.find("amazon") != string::npos) + { + cout << "AWS-specific functionality has been deprecated for this version." << endl + << "Please re-run postConfigure without the upgrade option." << endl; + exit(1); + } + #endif if ( cloud == "disable" ) amazonInstall = false; From 2cd0f8314c9a2928b87efc2c00820529f81d3067 Mon Sep 17 00:00:00 2001 From: Patrick LeBlanc Date: Wed, 11 Mar 2020 17:54:14 -0400 Subject: [PATCH 03/21] MCOL-2022: added 'deprecated' to the assign/unassign elastic ip commands. --- oam/etc/ConsoleCmds.xml | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/oam/etc/ConsoleCmds.xml b/oam/etc/ConsoleCmds.xml index 9a63dc2e1..c36f2bcab 100644 --- a/oam/etc/ConsoleCmds.xml +++ b/oam/etc/ConsoleCmds.xml @@ -274,13 +274,13 @@ assignElasticIPAddress - Assign Amazon Elastic IP Address to a module + Assign Amazon Elastic IP Address to a module (deprecated) Required: Amazon Elastic IP Address Required: Module Name unassignElasticIPAddress - Unassign Amazon Elastic IP Address + Unassign Amazon Elastic IP Address (deprecated) Required: Amazon Elastic IP Address From c073ae0ef9eb1b252929671a3f47f380f0b4f44c Mon Sep 17 00:00:00 2001 From: Patrick LeBlanc Date: Wed, 11 Mar 2020 18:29:38 -0400 Subject: [PATCH 04/21] MCOL-2022. Removed some test code. --- oamapps/postConfigure/postConfigure.cpp | 9 --------- 1 file changed, 9 deletions(-) diff --git a/oamapps/postConfigure/postConfigure.cpp b/oamapps/postConfigure/postConfigure.cpp index 8de961c07..b05eef05e 100644 --- a/oamapps/postConfigure/postConfigure.cpp +++ b/oamapps/postConfigure/postConfigure.cpp @@ -1399,15 +1399,6 @@ int main(int argc, char* argv[]) { cloud = oam::UnassignedName; } - - #if XXXPAT - if (cloud.find("amazon") != string::npos) - { - cout << "AWS-specific functionality has been deprecated for this version." << endl - << "Please re-run postConfigure without the upgrade option." << endl; - exit(1); - } - #endif if ( cloud == "disable" ) amazonInstall = false; From ee682b834f01d6ae01ccba14d777cecbb5cebb5e Mon Sep 17 00:00:00 2001 From: Patrick LeBlanc Date: Fri, 13 Mar 2020 14:50:21 -0400 Subject: [PATCH 05/21] MCOL-3251. Bad file-not-found error msg from PrimProc Wrote a different method for PrimProc to use to get filenames. --- primitives/blockcache/iomanager.cpp | 55 +++++++++++++++++++++++++++-- primitives/blockcache/iomanager.h | 7 +++- 2 files changed, 59 insertions(+), 3 deletions(-) diff --git a/primitives/blockcache/iomanager.cpp b/primitives/blockcache/iomanager.cpp index 097f3cad6..92b3fb28f 100644 --- a/primitives/blockcache/iomanager.cpp +++ b/primitives/blockcache/iomanager.cpp @@ -1356,18 +1356,69 @@ ioManager::ioManager(FileBufferMgr& fbm, } fThreadCount = thrCount; + lastConfigMTime = 0; + loadDBRootCache(); go(); } -void ioManager::buildOidFileName(const BRM::OID_t oid, const uint16_t dbRoot, const uint16_t partNum, const uint32_t segNum, char* file_name) +void ioManager::loadDBRootCache() { + if (fConfig->getLastMTime() == lastConfigMTime) + return; + + char buf[100]; + dbRootCache.clear(); + + // this will grab all of the dbroots on the cluster, not just the ones on this node + // for simplicity. + oam::Oam OAM; + oam::DBRootConfigList dbRoots; + OAM.getSystemDbrootConfig(dbRoots); + for (auto _dbroot : dbRoots) + { + sprintf(buf, "DBRoot%d", _dbroot); + string location = fConfig->getConfig("SystemConfig", buf); + dbRootCache[_dbroot] = location; + } +} + +void ioManager::buildOidFileName(const BRM::OID_t oid, uint16_t dbRoot, const uint16_t partNum, const uint32_t segNum, char* file_name) +{ + // when it's a request for the version buffer, the dbroot comes in as 0 for legacy reasons + if (dbRoot == 0 && oid < 1000) + dbRoot = fdbrm.getDBRootOfVBOID(oid); + + boost::unique_lock lock(dbRootCacheLock); + loadDBRootCache(); + string dbRootPath; + auto it = dbRootCache.find(dbRoot); + if (it == dbRootCache.end()) + { + ostringstream oss; + oss << "(dbroot " << dbRoot << " offline)"; + dbRootPath = oss.str(); + } + else + dbRootPath = it->second; + lock.unlock(); + + // different filenames for the version buffer files + if (oid < 1000) + snprintf(file_name, WriteEngine::FILE_NAME_SIZE, "%s/versionbuffer.cdf", dbRootPath.c_str()); + else + snprintf(file_name, WriteEngine::FILE_NAME_SIZE, "%s/%03u.dir/%03u.dir/%03u.dir/%03u.dir/%03u.dir/FILE%03d.cdf", + dbRootPath.c_str(), oid >> 24, (oid & 0x00ff0000) >> 16, (oid & 0x0000ff00) >> 8, + oid & 0x000000ff, partNum, segNum); + + + /* old version if (fFileOp.getFileName(oid, file_name, dbRoot, partNum, segNum) != WriteEngine::NO_ERROR) { file_name[0] = 0; throw std::runtime_error("fileOp.getFileName failed"); } - //cout << "Oid2Filename o: " << oid << " n: " << file_name << endl; + */ } const int ioManager::localLbidLookup(BRM::LBID_t lbid, diff --git a/primitives/blockcache/iomanager.h b/primitives/blockcache/iomanager.h index 4ac14a9b0..ad09fd0db 100644 --- a/primitives/blockcache/iomanager.h +++ b/primitives/blockcache/iomanager.h @@ -83,7 +83,7 @@ public: uint32_t& fileBlockOffset); void buildOidFileName(const BRM::OID_t oid, - const uint16_t dbRoot, + uint16_t dbRoot, const uint16_t partNum, const uint32_t segNum, char* file_name); @@ -153,6 +153,11 @@ private: uint32_t fDecreaseOpenFilesCount; bool fFDCacheTrace; std::ofstream fFDTraceFile; + + std::map dbRootCache; + boost::mutex dbRootCacheLock; + void loadDBRootCache(); + time_t lastConfigMTime; }; From a65c8e454421f74b023e0540eecb6abae3b2be17 Mon Sep 17 00:00:00 2001 From: Patrick LeBlanc Date: Fri, 13 Mar 2020 15:18:55 -0400 Subject: [PATCH 06/21] MCOL-3251 Moved the new logic into we_fileop to maintain existing encapsulation. --- primitives/blockcache/iomanager.cpp | 55 +---------------------------- primitives/blockcache/iomanager.h | 6 ---- writeengine/shared/we_fileop.cpp | 23 ++++++++++++ writeengine/shared/we_fileop.h | 6 ++++ 4 files changed, 30 insertions(+), 60 deletions(-) diff --git a/primitives/blockcache/iomanager.cpp b/primitives/blockcache/iomanager.cpp index 92b3fb28f..3aa3f4c09 100644 --- a/primitives/blockcache/iomanager.cpp +++ b/primitives/blockcache/iomanager.cpp @@ -1356,69 +1356,16 @@ ioManager::ioManager(FileBufferMgr& fbm, } fThreadCount = thrCount; - lastConfigMTime = 0; - loadDBRootCache(); go(); } -void ioManager::loadDBRootCache() -{ - if (fConfig->getLastMTime() == lastConfigMTime) - return; - - char buf[100]; - dbRootCache.clear(); - - // this will grab all of the dbroots on the cluster, not just the ones on this node - // for simplicity. - oam::Oam OAM; - oam::DBRootConfigList dbRoots; - OAM.getSystemDbrootConfig(dbRoots); - for (auto _dbroot : dbRoots) - { - sprintf(buf, "DBRoot%d", _dbroot); - string location = fConfig->getConfig("SystemConfig", buf); - dbRootCache[_dbroot] = location; - } -} - void ioManager::buildOidFileName(const BRM::OID_t oid, uint16_t dbRoot, const uint16_t partNum, const uint32_t segNum, char* file_name) { // when it's a request for the version buffer, the dbroot comes in as 0 for legacy reasons if (dbRoot == 0 && oid < 1000) dbRoot = fdbrm.getDBRootOfVBOID(oid); - - boost::unique_lock lock(dbRootCacheLock); - loadDBRootCache(); - string dbRootPath; - auto it = dbRootCache.find(dbRoot); - if (it == dbRootCache.end()) - { - ostringstream oss; - oss << "(dbroot " << dbRoot << " offline)"; - dbRootPath = oss.str(); - } - else - dbRootPath = it->second; - lock.unlock(); - // different filenames for the version buffer files - if (oid < 1000) - snprintf(file_name, WriteEngine::FILE_NAME_SIZE, "%s/versionbuffer.cdf", dbRootPath.c_str()); - else - snprintf(file_name, WriteEngine::FILE_NAME_SIZE, "%s/%03u.dir/%03u.dir/%03u.dir/%03u.dir/%03u.dir/FILE%03d.cdf", - dbRootPath.c_str(), oid >> 24, (oid & 0x00ff0000) >> 16, (oid & 0x0000ff00) >> 8, - oid & 0x000000ff, partNum, segNum); - - - /* old version - if (fFileOp.getFileName(oid, file_name, dbRoot, partNum, segNum) != WriteEngine::NO_ERROR) - { - file_name[0] = 0; - throw std::runtime_error("fileOp.getFileName failed"); - } - //cout << "Oid2Filename o: " << oid << " n: " << file_name << endl; - */ + fFileOp.getFileNameForPrimProc(oid, file_name, dbRoot, partNum, segNum); } const int ioManager::localLbidLookup(BRM::LBID_t lbid, diff --git a/primitives/blockcache/iomanager.h b/primitives/blockcache/iomanager.h index ad09fd0db..eb7a9f1af 100644 --- a/primitives/blockcache/iomanager.h +++ b/primitives/blockcache/iomanager.h @@ -153,12 +153,6 @@ private: uint32_t fDecreaseOpenFilesCount; bool fFDCacheTrace; std::ofstream fFDTraceFile; - - std::map dbRootCache; - boost::mutex dbRootCacheLock; - void loadDBRootCache(); - time_t lastConfigMTime; - }; // @bug2631, for remount filesystem by loadBlock() in primitiveserver diff --git a/writeengine/shared/we_fileop.cpp b/writeengine/shared/we_fileop.cpp index f4c65ed52..d81a1927f 100644 --- a/writeengine/shared/we_fileop.cpp +++ b/writeengine/shared/we_fileop.cpp @@ -2412,6 +2412,29 @@ int FileOp::oid2FileName( FID fid, return NO_ERROR; } +void FileOp::getFileNameForPrimProc(FID fid, + char* fullFileName, + uint16_t dbRoot, + uint32_t partition, + uint16_t segment) const +{ + string dbRootPath = Config::getDBRootByNum(dbRoot); + if (dbRootPath.empty()) + { + ostringstream oss; + oss << "(dbroot " << dbRoot << " offline)"; + dbRootPath = oss.str(); + } + + // different filenames for the version buffer files + if (fid < 1000) + snprintf(fullFileName, FILE_NAME_SIZE, "%s/versionbuffer.cdf", dbRootPath.c_str()); + else + snprintf(fullFileName, FILE_NAME_SIZE, "%s/%03u.dir/%03u.dir/%03u.dir/%03u.dir/%03u.dir/FILE%03d.cdf", + dbRootPath.c_str(), fid >> 24, (fid & 0x00ff0000) >> 16, (fid & 0x0000ff00) >> 8, + fid & 0x000000ff, partition, segment); +} + /*********************************************************** * DESCRIPTION: * Search for directory path associated with specified OID. diff --git a/writeengine/shared/we_fileop.h b/writeengine/shared/we_fileop.h index e7f2fa576..4ddc7513f 100644 --- a/writeengine/shared/we_fileop.h +++ b/writeengine/shared/we_fileop.h @@ -310,6 +310,12 @@ public: uint32_t partition, uint16_t segment ) const; + /* Added for MCOL-3251 */ + void getFileNameForPrimProc(FID fid, char* fileName, + uint16_t dbRoot, + uint32_t partition, + uint16_t segment ) const; + /** * @brief Construct directory path for the specified fid (OID), DBRoot, and * partition number. Directory does not have to exist, nor is it created. From 4e7e70b0f1d5506a5d4c9bff6f9cb80c01554cc9 Mon Sep 17 00:00:00 2001 From: Patrick LeBlanc Date: Mon, 16 Mar 2020 13:42:44 -0400 Subject: [PATCH 07/21] MCOL-3887 - postconfig problem processing an upgrade mycnfUpgrade has been looking for our mysql config mods in the /etc/my.cnf/ directory, which is wrong obviously. Looks like an oversight; all of the code around it is looking in the right place. Changed it to look in the right place. --- oamapps/postConfigure/mycnfUpgrade.cpp | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/oamapps/postConfigure/mycnfUpgrade.cpp b/oamapps/postConfigure/mycnfUpgrade.cpp index 33f2f2f91..e35a36d57 100644 --- a/oamapps/postConfigure/mycnfUpgrade.cpp +++ b/oamapps/postConfigure/mycnfUpgrade.cpp @@ -136,7 +136,7 @@ int main(int argc, char* argv[]) } //my.cnf.rpmsave file - string mycnfsaveFile = "/etc/my.cnf/columnstore.cnf.rpmsave"; + string mycnfsaveFile = std::string(MCSMYCNFDIR) + "/columnstore.cnf.rpmsave"; ifstream mycnfsavefile (mycnfsaveFile.c_str()); if (!mycnfsavefile) From f480a28bdf3f182c5da2effd5de1fae7ea0287c3 Mon Sep 17 00:00:00 2001 From: Patrick LeBlanc Date: Tue, 17 Mar 2020 14:49:49 -0400 Subject: [PATCH 08/21] MCOL-3251. Changed the sig of buildOidFileName as per feedback. --- primitives/blockcache/iomanager.cpp | 2 +- primitives/blockcache/iomanager.h | 4 ++-- 2 files changed, 3 insertions(+), 3 deletions(-) diff --git a/primitives/blockcache/iomanager.cpp b/primitives/blockcache/iomanager.cpp index 3aa3f4c09..019368313 100644 --- a/primitives/blockcache/iomanager.cpp +++ b/primitives/blockcache/iomanager.cpp @@ -1359,7 +1359,7 @@ ioManager::ioManager(FileBufferMgr& fbm, go(); } -void ioManager::buildOidFileName(const BRM::OID_t oid, uint16_t dbRoot, const uint16_t partNum, const uint32_t segNum, char* file_name) +void ioManager::buildOidFileName(const BRM::OID_t oid, uint16_t dbRoot, const uint32_t partNum, const uint16_t segNum, char* file_name) { // when it's a request for the version buffer, the dbroot comes in as 0 for legacy reasons if (dbRoot == 0 && oid < 1000) diff --git a/primitives/blockcache/iomanager.h b/primitives/blockcache/iomanager.h index eb7a9f1af..8f0f629ac 100644 --- a/primitives/blockcache/iomanager.h +++ b/primitives/blockcache/iomanager.h @@ -84,8 +84,8 @@ public: void buildOidFileName(const BRM::OID_t oid, uint16_t dbRoot, - const uint16_t partNum, - const uint32_t segNum, + const uint32_t partNum, + const uint16_t segNum, char* file_name); const uint32_t getExtentRows() From 3345f7a43727bdf3c1c302d57408eb5737297fd3 Mon Sep 17 00:00:00 2001 From: Roman Nozdrin Date: Mon, 17 Feb 2020 12:27:23 +0000 Subject: [PATCH 09/21] MCOL-3769 This commit divides create_SH in two parts: before rewrites and after rewrites. We can safely fallback from SH to DH and table mode from 'before rewrites' part. --- dbcon/mysql/ha_mcs_opt_rewrites.cpp | 10 +- dbcon/mysql/ha_mcs_opt_rewrites.h | 2 +- dbcon/mysql/ha_mcs_pushdown.cpp | 152 +++++++++++++++------------- dbcon/mysql/ha_mcs_pushdown.h | 21 ++-- 4 files changed, 98 insertions(+), 87 deletions(-) diff --git a/dbcon/mysql/ha_mcs_opt_rewrites.cpp b/dbcon/mysql/ha_mcs_opt_rewrites.cpp index 00c08d9e0..62c4c057a 100644 --- a/dbcon/mysql/ha_mcs_opt_rewrites.cpp +++ b/dbcon/mysql/ha_mcs_opt_rewrites.cpp @@ -18,7 +18,7 @@ // Search simplify_joins() function in the server's code for detail COND * -simplify_joins_(JOIN *join, List *join_list, COND *conds, bool top, +simplify_joins_mcs(JOIN *join, List *join_list, COND *conds, bool top, bool in_sj) { TABLE_LIST *table; @@ -26,7 +26,7 @@ simplify_joins_(JOIN *join, List *join_list, COND *conds, bool top, TABLE_LIST *prev_table= 0; List_iterator li(*join_list); bool straight_join= MY_TEST(join->select_options & SELECT_STRAIGHT_JOIN); - DBUG_ENTER("simplify_joins"); + DBUG_ENTER("simplify_joins_mcs"); /* Try to simplify join operations from join_list. @@ -54,7 +54,7 @@ simplify_joins_(JOIN *join, List *join_list, COND *conds, bool top, the outer join is converted to an inner join and the corresponding on expression is added to E. */ - expr= simplify_joins_(join, &nested_join->join_list, + expr= simplify_joins_mcs(join, &nested_join->join_list, expr, FALSE, in_sj || table->sj_on_expr); if (!table->prep_on_expr || expr != table->on_expr) @@ -67,7 +67,7 @@ simplify_joins_(JOIN *join, List *join_list, COND *conds, bool top, } nested_join->used_tables= (table_map) 0; nested_join->not_null_tables=(table_map) 0; - conds= simplify_joins_(join, &nested_join->join_list, conds, top, + conds= simplify_joins_mcs(join, &nested_join->join_list, conds, top, in_sj || table->sj_on_expr); used_tables= nested_join->used_tables; not_null_tables= nested_join->not_null_tables; @@ -241,5 +241,5 @@ simplify_joins_(JOIN *join, List *join_list, COND *conds, bool top, li.replace(repl_list); } } - DBUG_RETURN(conds); + DBUG_RETURN(conds); } diff --git a/dbcon/mysql/ha_mcs_opt_rewrites.h b/dbcon/mysql/ha_mcs_opt_rewrites.h index d433c06a5..2b221d1c5 100644 --- a/dbcon/mysql/ha_mcs_opt_rewrites.h +++ b/dbcon/mysql/ha_mcs_opt_rewrites.h @@ -20,7 +20,7 @@ #include "idb_mysql.h" -COND *simplify_joins_(JOIN *join, List *join_list, COND *conds, bool top, bool in_sj); +COND *simplify_joins_mcs(JOIN *join, List *join_list, COND *conds, bool top, bool in_sj); #endif diff --git a/dbcon/mysql/ha_mcs_pushdown.cpp b/dbcon/mysql/ha_mcs_pushdown.cpp index da7c948bd..d73e8a861 100644 --- a/dbcon/mysql/ha_mcs_pushdown.cpp +++ b/dbcon/mysql/ha_mcs_pushdown.cpp @@ -27,7 +27,7 @@ void disable_indices_for_CEJ(THD *thd_) TABLE_LIST* global_list; for (global_list = thd_->lex->query_tables; global_list; global_list = global_list->next_global) { - // MCOL-652 - doing this with derived tables can cause bad things to happen + // MCOL-652 - doing this with derived tables can cause bad things to happen if (!global_list->derived) { global_list->index_hints= new (thd_->mem_root) List(); @@ -42,7 +42,7 @@ void disable_indices_for_CEJ(THD *thd_) } } -bool optimize_unflattened_subqueries_(SELECT_LEX *select_lex) +bool optimize_unflattened_subqueries_mcs(SELECT_LEX *select_lex) { bool result = false; TABLE_LIST *tbl; @@ -52,7 +52,7 @@ bool optimize_unflattened_subqueries_(SELECT_LEX *select_lex) if (tbl->is_view_or_derived()) { SELECT_LEX *dsl = tbl->derived->first_select(); - result = optimize_unflattened_subqueries_(dsl); + result = optimize_unflattened_subqueries_mcs(dsl); } } @@ -516,7 +516,7 @@ create_columnstore_derived_handler(THD* thd, TABLE_LIST *derived) //To search for CROSS JOIN-s we use tree invariant //G(V,E) where [V] = [E]+1 List join_preds_list; - TABLE_LIST *tl; + TABLE_LIST *tl; for (tl = sl->get_table_list(); !unsupported_feature && tl; tl = tl->next_local) { Item_cond* where_icp= 0; @@ -756,68 +756,38 @@ create_columnstore_select_handler(THD* thd, SELECT_LEX* select_lex) { ha_columnstore_select_handler* handler = NULL; - // MCOL-2178 Disable SP support in the select_handler for now. // Check the session variable value to enable/disable use of // select_handler - // Disable processing of select_result_interceptor classes - // which intercept and transform result set rows. E.g.: - // select a,b into @a1, @a2 from t1; - if (!get_select_handler(thd) || (thd->lex)->sphead || - ((thd->lex)->result && - !((select_dumpvar *)(thd->lex)->result)->var_list.is_empty())) + if (!get_select_handler(thd)) { return handler; } - bool unsupported_feature = false; + // Disable SP support in the select_handler for now. + if ((thd->lex)->sphead) + { + return handler; + } + + // Disable processing of select_result_interceptor classes + // which intercept and transform result set rows. E.g.: + // select a,b into @a1, @a2 from t1; + if (((thd->lex)->result && + !((select_dumpvar *)(thd->lex)->result)->var_list.is_empty())) + { + return handler; + } // Select_handler use the short-cut that effectively disables // INSERT..SELECT, LDI, SELECT..INTO OUTFILE if ((thd->lex)->sql_command == SQLCOM_INSERT_SELECT || (thd->lex)->sql_command == SQLCOM_CREATE_TABLE || (thd->lex)->exchange) - { - unsupported_feature = true; - } - - JOIN *join= select_lex->join; - // Next block tries to execute the query using SH very early to fallback - // if execution fails. - if (!unsupported_feature) - { - disable_indices_for_CEJ(thd); - - if (select_lex->handle_derived(thd->lex, DT_MERGE)) - { - // early quit b/c of the error in handle_derived - return handler; - } - - COND *conds = simplify_joins_(join, select_lex->join_list, join->conds, TRUE, FALSE); - // MCOL-3747 IN-TO-EXISTS rewrite inside MDB didn't add - // an equi-JOIN condition. - optimize_unflattened_subqueries_(select_lex); - - if (conds) - { -#ifdef DEBUG_WALK_COND - conds->traverse_cond(cal_impl_if::debug_walk, NULL, Item::POSTFIX); -#endif - join->conds = conds; - } - - // Impossible HAVING or WHERE - // TODO replace with function call - if (unsupported_feature - || select_lex->having_value == Item::COND_FALSE - || select_lex->cond_value == Item::COND_FALSE ) - { - unsupported_feature = true; - restore_optimizer_flags(thd); - } + return handler; } + bool unsupported_feature = false; // Iterate and traverse through the item list and do not create SH // if the unsupported (set/get_user_var) functions are present. TABLE_LIST* table_ptr = select_lex->get_table_list(); @@ -830,31 +800,57 @@ create_columnstore_select_handler(THD* thd, SELECT_LEX* select_lex) item_check(item, &unsupported_feature); if (unsupported_feature) { - break; + return handler; } } } - if (!unsupported_feature) + // We apply dedicated rewrites from MDB here so MDB's data structures + // becomes dirty and CS has to raise an error in case of any problem + // or unsupported feature. + handler= new ha_columnstore_select_handler(thd, select_lex); + JOIN *join= select_lex->join; { - handler= new ha_columnstore_select_handler(thd, select_lex); - mcs_handler_info mhi= mcs_handler_info(reinterpret_cast(handler), SELECT); - // handler::table is the place for the result set - int rc= 0; - // Skip execution for EXPLAIN queries - if (!thd->lex->describe) + disable_indices_for_CEJ(thd); + + if (select_lex->handle_derived(thd->lex, DT_MERGE)) { - rc= ha_cs_impl_pushdown_init(&mhi, handler->table); + unsupported_feature = true; + handler->err_msg.assign("create_columnstore_select_handler(): \ + Internal error occured in SL::handle_derived()"); } - // Return SH even if init fails b/c CS changed SELECT_LEX structures - // with simplify_joins_() - if (rc) + COND *conds = nullptr; + if (!unsupported_feature) + { + conds = simplify_joins_mcs(join, select_lex->join_list, + join->conds, TRUE, FALSE); + } + // MCOL-3747 IN-TO-EXISTS rewrite inside MDB didn't add + // an equi-JOIN condition. + if (!unsupported_feature + && optimize_unflattened_subqueries_mcs(select_lex)) + { unsupported_feature = true; - return handler; + handler->err_msg.assign("create_columnstore_select_handler(): \ + Internal error occured in optimize_unflattened_subqueries_mcs()"); + } + + if (!unsupported_feature && conds) + { +#ifdef DEBUG_WALK_COND + conds->traverse_cond(cal_impl_if::debug_walk, NULL, Item::POSTFIX); +#endif + join->conds = conds; + } } - return NULL; + // We can't raise error now so set an error to raise it later in init_SH. + handler->rewrite_error= unsupported_feature; + + // Return SH even if init fails b/c CS changed SELECT_LEX structures + // with simplify_joins_mcs() + return handler; } /*********************************************************** @@ -868,7 +864,8 @@ ha_columnstore_select_handler::ha_columnstore_select_handler(THD *thd, SELECT_LEX* select_lex) : select_handler(thd, mcs_hton) { - select = select_lex; + select= select_lex; + rewrite_error= false; } /*********************************************************** @@ -892,10 +889,26 @@ int ha_columnstore_select_handler::init_scan() { DBUG_ENTER("ha_columnstore_select_handler::init_scan"); - // Dummy init for SH. Actual init happens in create_SH - // to allow fallback to other handlers if SH fails. int rc = 0; + if (!rewrite_error) + { + // handler::table is the place for the result set + // Skip execution for EXPLAIN queries + if (!thd->lex->describe) + { + mcs_handler_info mhi= mcs_handler_info( + reinterpret_cast(this), SELECT); + rc= ha_cs_impl_pushdown_init(&mhi, this->table); + } + } + else + { + my_printf_error(ER_INTERNAL_ERROR, "%s", MYF(0), err_msg.c_str()); + sql_print_error("%s", err_msg.c_str()); + rc= ER_INTERNAL_ERROR; + } + DBUG_RETURN(rc); } @@ -936,6 +949,3 @@ int ha_columnstore_select_handler::end_scan() DBUG_RETURN(rc); } - -void ha_columnstore_select_handler::print_error(int, unsigned long) -{} diff --git a/dbcon/mysql/ha_mcs_pushdown.h b/dbcon/mysql/ha_mcs_pushdown.h index 268d6b5b6..e8d5d23e3 100644 --- a/dbcon/mysql/ha_mcs_pushdown.h +++ b/dbcon/mysql/ha_mcs_pushdown.h @@ -80,9 +80,9 @@ class ha_mcs_group_by_handler: public group_by_handler public: ha_mcs_group_by_handler(THD* thd_arg, Query* query); ~ha_mcs_group_by_handler(); - int init_scan(); - int next_row(); - int end_scan(); + int init_scan() override; + int next_row() override; + int end_scan() override; List* select; TABLE_LIST* table_list; @@ -114,9 +114,9 @@ private: public: ha_columnstore_derived_handler(THD* thd_arg, TABLE_LIST *tbl); ~ha_columnstore_derived_handler(); - int init_scan(); - int next_row(); - int end_scan(); + int init_scan() override; + int next_row() override; + int end_scan() override; void print_error(int, unsigned long); }; @@ -139,12 +139,13 @@ private: COLUMNSTORE_SHARE *share; public: + bool rewrite_error; + std::string err_msg; ha_columnstore_select_handler(THD* thd_arg, SELECT_LEX* sel); ~ha_columnstore_select_handler(); - int init_scan(); - int next_row(); - int end_scan(); - void print_error(int, unsigned long); + int init_scan() override; + int next_row() override; + int end_scan() override; }; #endif From 17f1539f89bfb1929642f18ed4b1b1317260edca Mon Sep 17 00:00:00 2001 From: Patrick LeBlanc Date: Thu, 19 Mar 2020 12:39:16 -0400 Subject: [PATCH 10/21] MCOL-3356. bit_{or,xor,and} job creation error Checkpointing. Found it, fixed it. It was an extra assignment that was messing up RowGroup creation. --- dbcon/joblist/tupleaggregatestep.cpp | 2 +- utils/rowgroup/rowgroup.cpp | 9 +++++++++ 2 files changed, 10 insertions(+), 1 deletion(-) diff --git a/dbcon/joblist/tupleaggregatestep.cpp b/dbcon/joblist/tupleaggregatestep.cpp index 24c1efa7e..9e7be1d62 100644 --- a/dbcon/joblist/tupleaggregatestep.cpp +++ b/dbcon/joblist/tupleaggregatestep.cpp @@ -1946,7 +1946,7 @@ void TupleAggregateStep::prep1PhaseDistinctAggregate( keysAgg.push_back(aggKey); scaleAgg.push_back(0); precisionAgg.push_back(-16); // for connector to skip null check - typeAgg.push_back(CalpontSystemCatalog::BIGINT); + //typeAgg.push_back(CalpontSystemCatalog::BIGINT); if (isUnsigned(typeProj[colProj])) { diff --git a/utils/rowgroup/rowgroup.cpp b/utils/rowgroup/rowgroup.cpp index a8bae1086..433ce04d7 100644 --- a/utils/rowgroup/rowgroup.cpp +++ b/utils/rowgroup/rowgroup.cpp @@ -1047,6 +1047,15 @@ RowGroup::RowGroup(uint32_t colCount, useStringTable = (stringTable && hasLongStringField); offsets = (useStringTable ? &stOffsets[0] : &oldOffsets[0]); + + for (i = 0; i < columnCount; i++) + { + if (types[i] == CalpontSystemCatalog::BIGINT && colWidths[i] != 8) + { + cout << "WTF?" << endl; + } + + } } RowGroup::RowGroup(const RowGroup& r) : From 163b4a5bb768cf2d3b5f47d260ae09d209519007 Mon Sep 17 00:00:00 2001 From: Patrick LeBlanc Date: Thu, 19 Mar 2020 15:31:50 -0400 Subject: [PATCH 11/21] MCOL-3356. code cleanup. --- dbcon/joblist/tupleaggregatestep.cpp | 1 - utils/rowgroup/rowgroup.cpp | 9 --------- 2 files changed, 10 deletions(-) diff --git a/dbcon/joblist/tupleaggregatestep.cpp b/dbcon/joblist/tupleaggregatestep.cpp index 9e7be1d62..eb4cc52d9 100644 --- a/dbcon/joblist/tupleaggregatestep.cpp +++ b/dbcon/joblist/tupleaggregatestep.cpp @@ -1946,7 +1946,6 @@ void TupleAggregateStep::prep1PhaseDistinctAggregate( keysAgg.push_back(aggKey); scaleAgg.push_back(0); precisionAgg.push_back(-16); // for connector to skip null check - //typeAgg.push_back(CalpontSystemCatalog::BIGINT); if (isUnsigned(typeProj[colProj])) { diff --git a/utils/rowgroup/rowgroup.cpp b/utils/rowgroup/rowgroup.cpp index 433ce04d7..a8bae1086 100644 --- a/utils/rowgroup/rowgroup.cpp +++ b/utils/rowgroup/rowgroup.cpp @@ -1047,15 +1047,6 @@ RowGroup::RowGroup(uint32_t colCount, useStringTable = (stringTable && hasLongStringField); offsets = (useStringTable ? &stOffsets[0] : &oldOffsets[0]); - - for (i = 0; i < columnCount; i++) - { - if (types[i] == CalpontSystemCatalog::BIGINT && colWidths[i] != 8) - { - cout << "WTF?" << endl; - } - - } } RowGroup::RowGroup(const RowGroup& r) : From 7ca4e8403aa690fd4c05e3c4da27352d609ed77c Mon Sep 17 00:00:00 2001 From: Gagan Goel Date: Fri, 20 Mar 2020 21:54:31 +0000 Subject: [PATCH 12/21] MCOL-3756 Implement IS(NOT)TRUE() and IS(NOT)FALSE() functions. --- utils/funcexp/funcexp.cpp | 4 ++ utils/funcexp/functor_bool.h | 83 ++++++++++++++++++++++++++++++++++++ 2 files changed, 87 insertions(+) diff --git a/utils/funcexp/funcexp.cpp b/utils/funcexp/funcexp.cpp index f96f85b45..5f15ec991 100644 --- a/utils/funcexp/funcexp.cpp +++ b/utils/funcexp/funcexp.cpp @@ -141,6 +141,10 @@ FuncExp::FuncExp() fFuncMap["instr"] = new Func_instr(); fFuncMap["isnull"] = new Func_isnull(false); fFuncMap["isnotnull"] = new Func_isnull(true); + fFuncMap["istrue"] = new Func_IsTrue(); + fFuncMap["isnottrue"] = new Func_IsNotTrue(); + fFuncMap["isfalse"] = new Func_IsFalse(); + fFuncMap["isnotfalse"] = new Func_IsNotFalse(); fFuncMap["last_day"] = new Func_last_day(); fFuncMap["lcase"] = new Func_lcase(); //dlh fFuncMap["least"] = new Func_least(); //dlh diff --git a/utils/funcexp/functor_bool.h b/utils/funcexp/functor_bool.h index 28077e878..7467f57d4 100644 --- a/utils/funcexp/functor_bool.h +++ b/utils/funcexp/functor_bool.h @@ -1,4 +1,5 @@ /* Copyright (C) 2014 InfiniDB, Inc. + Copyright (C) 2020 MariaDB Corporation This program is free software; you can redistribute it and/or modify it under the terms of the GNU General Public License @@ -241,6 +242,88 @@ private: }; +/** @brief Func_Truth class + */ +class Func_Truth : public Func_Bool +{ +public: + Func_Truth(const std::string& funcName, bool a_value, bool a_affirmative) : + Func_Bool(funcName), value(a_value), affirmative(a_affirmative) {} + + virtual ~Func_Truth() {} + + execplan::CalpontSystemCatalog::ColType operationType(FunctionParm& fp, execplan::CalpontSystemCatalog::ColType& resultType) + { + assert (fp.size() == 1); + return fp[0]->data()->resultType(); + } + + bool getBoolVal(rowgroup::Row& row, + FunctionParm& fp, + bool& isNull, + execplan::CalpontSystemCatalog::ColType& op_ct) + { + bool val = fp[0]->data()->getBoolVal(row, isNull); + + /* + NULL val IS {TRUE, FALSE} --> FALSE + NULL val IS NOT {TRUE, FALSE} --> TRUE + {TRUE, FALSE} val IS {TRUE, FALSE} value --> val == value + {TRUE, FALSE} val IS NOT {TRUE, FALSE} value --> val != value + These cases can be reduced to the following bitwise operation. + */ + bool ret = (isNull & (!affirmative)) | ((!isNull) & (affirmative ^ (value ^ val))); + + isNull = false; + + return ret; + } + +private: + const bool value, affirmative; +}; + + +/** @brief Func_IsTrue class + */ +class Func_IsTrue : public Func_Truth +{ +public: + Func_IsTrue() : Func_Truth("istrue", true, true) {} + ~Func_IsTrue() {} +}; + + +/** @brief Func_IsNotTrue class + */ +class Func_IsNotTrue : public Func_Truth +{ +public: + Func_IsNotTrue() : Func_Truth("isnottrue", true, false) {} + ~Func_IsNotTrue() {} +}; + + +/** @brief Func_IsFalse class + */ +class Func_IsFalse : public Func_Truth +{ +public: + Func_IsFalse() : Func_Truth("isfalse", false, true) {} + ~Func_IsFalse() {} +}; + + +/** @brief Func_IsNotFalse class + */ +class Func_IsNotFalse : public Func_Truth +{ +public: + Func_IsNotFalse() : Func_Truth("isnotfalse", false, false) {} + ~Func_IsNotFalse() {} +}; + + } #endif From ce0311aa7962171d9e248b2ce348a427cda1c37a Mon Sep 17 00:00:00 2001 From: Roman Nozdrin Date: Thu, 19 Mar 2020 15:39:13 +0000 Subject: [PATCH 13/21] MCOL-3828 This commit replaces the method that calls JOIN::optimise() internally(thus can remove equi-JOIN predicates) with a recursive f() to inject equi-JOIN conditions for IN + correlated subquery Item. This change both solves MCOL-3747 issue and remove issues I first faced in union.sql test. This commit also fixes a clean up for conditions pushed for table API execution path and replaces NULL with nullptr in ha_mcs_impl.cpp. --- dbcon/mysql/ha_mcs_impl.cpp | 98 +++++++++++++++-------------- dbcon/mysql/ha_mcs_opt_rewrites.cpp | 89 +++++++++++++++++++++++++- dbcon/mysql/ha_mcs_opt_rewrites.h | 1 + dbcon/mysql/ha_mcs_pushdown.cpp | 44 ++++--------- dbcon/mysql/ha_mcs_pushdown.h | 2 +- 5 files changed, 152 insertions(+), 82 deletions(-) diff --git a/dbcon/mysql/ha_mcs_impl.cpp b/dbcon/mysql/ha_mcs_impl.cpp index 7367bc10a..d04d5f186 100644 --- a/dbcon/mysql/ha_mcs_impl.cpp +++ b/dbcon/mysql/ha_mcs_impl.cpp @@ -1,4 +1,4 @@ - /* Copyright (C) 2014 InfiniDB, Inc. +/* Copyright (C) 2014 InfiniDB, Inc. Copyright (C) 2019 MariaDB Corporation This program is free software; you can redistribute it and/or @@ -1196,7 +1196,7 @@ vector getOnUpdateTimestampColumns(string& schema, string& tableName, in uint32_t doUpdateDelete(THD* thd, gp_walk_info& gwi) { - if (get_fe_conn_info_ptr() == NULL) + if (get_fe_conn_info_ptr() == nullptr) set_fe_conn_info_ptr((void*)new cal_connection_info()); cal_connection_info* ci = reinterpret_cast(get_fe_conn_info_ptr()); @@ -2009,7 +2009,7 @@ uint32_t doUpdateDelete(THD* thd, gp_walk_info& gwi) else { delete ci->dmlProc; - ci->dmlProc = NULL; + ci->dmlProc = nullptr; ci->dmlProc = new MessageQueueClient("DMLProc"); } @@ -2069,7 +2069,7 @@ uint32_t doUpdateDelete(THD* thd, gp_walk_info& gwi) //cout << "line 1442. received 0 byte from DMLProc and retry = "<< retry << endl; // Seems dmlProc isn't playing. Reset it and try again. delete ci->dmlProc; - ci->dmlProc = NULL; + ci->dmlProc = nullptr; isTimeOut = true; //@Bug 4742 } } @@ -2117,7 +2117,7 @@ uint32_t doUpdateDelete(THD* thd, gp_walk_info& gwi) cout << ex.what() << endl; b = 1; delete ci->dmlProc; - ci->dmlProc = NULL; + ci->dmlProc = nullptr; errorMsg = ex.what(); } catch ( ... ) @@ -2125,7 +2125,7 @@ uint32_t doUpdateDelete(THD* thd, gp_walk_info& gwi) //cout << "... exception while writing to DMLProc" << endl; b = 1; delete ci->dmlProc; - ci->dmlProc = NULL; + ci->dmlProc = nullptr; errorMsg = "Unknown error caught"; } @@ -2189,7 +2189,7 @@ uint32_t doUpdateDelete(THD* thd, gp_walk_info& gwi) errorMsg = "Lost connection to DMLProc"; b = 1; delete ci->dmlProc; - ci->dmlProc = NULL; + ci->dmlProc = nullptr; } catch (...) { @@ -2247,7 +2247,7 @@ uint32_t doUpdateDelete(THD* thd, gp_walk_info& gwi) } delete ci->dmlProc; - ci->dmlProc = NULL; + ci->dmlProc = nullptr; return rc; } @@ -2375,7 +2375,7 @@ int ha_mcs_impl_rnd_init(TABLE* table) boost::shared_ptr csc = CalpontSystemCatalog::makeCalpontSystemCatalog(sessionID); csc->identity(CalpontSystemCatalog::FE); - if (get_fe_conn_info_ptr() == NULL) + if (get_fe_conn_info_ptr() == nullptr) set_fe_conn_info_ptr((void*)new cal_connection_info()); idbassert(ci != 0); @@ -2581,7 +2581,7 @@ int ha_mcs_impl_rnd_init(TABLE* table) ti = ci->tableMap[table]; ti.msTablePtr = table; - if (ti.tpl_ctx == 0) + if (ti.tpl_ctx == nullptr) { ti.tpl_ctx = new sm::cpsm_tplh_t(); ti.tpl_scan_ctx = sm::sp_cpsm_tplsch_t(new sm::cpsm_tplsch_t()); @@ -2589,7 +2589,7 @@ int ha_mcs_impl_rnd_init(TABLE* table) // make sure rowgroup is null so the new meta data can be taken. This is for some case mysql // call rnd_init for a table more than once. - ti.tpl_scan_ctx->rowGroup = NULL; + ti.tpl_scan_ctx->rowGroup = nullptr; try { @@ -2707,7 +2707,7 @@ int ha_mcs_impl_rnd_next(uchar* buf, TABLE* table) // if (MIGR::infinidb_vtable.impossibleWhereOnUnion) // return HA_ERR_END_OF_FILE; - if (get_fe_conn_info_ptr() == NULL) + if (get_fe_conn_info_ptr() == nullptr) set_fe_conn_info_ptr((void*)new cal_connection_info()); // @bug 3078 @@ -2772,7 +2772,7 @@ int ha_mcs_impl_rnd_end(TABLE* table, bool is_pushdown_hand) { int rc = 0; THD* thd = current_thd; - cal_connection_info* ci = NULL; + cal_connection_info* ci = nullptr; if (get_fe_conn_info_ptr() != NULL) ci = reinterpret_cast(get_fe_conn_info_ptr()); @@ -2911,7 +2911,7 @@ int ha_mcs_impl_create(const char* name, TABLE* table_arg, HA_CREATE_INFO* creat { THD* thd = current_thd; - if (get_fe_conn_info_ptr() == NULL) + if (get_fe_conn_info_ptr() == nullptr) set_fe_conn_info_ptr((void*)new cal_connection_info()); cal_connection_info* ci = reinterpret_cast(get_fe_conn_info_ptr()); @@ -2940,7 +2940,7 @@ int ha_mcs_impl_create(const char* name, TABLE* table_arg, HA_CREATE_INFO* creat int ha_mcs_impl_delete_table(const char* name) { THD* thd = current_thd; - char* dbName = NULL; + char* dbName = nullptr; if (!name) { @@ -2951,7 +2951,7 @@ int ha_mcs_impl_delete_table(const char* name) //if this is an InfiniDB tmp table ('#sql*.frm') just leave... if (!memcmp((uchar*)name, tmp_file_prefix, tmp_file_prefix_length)) return 0; - if (get_fe_conn_info_ptr() == NULL) + if (get_fe_conn_info_ptr() == nullptr) set_fe_conn_info_ptr((void*)new cal_connection_info()); cal_connection_info* ci = reinterpret_cast(get_fe_conn_info_ptr()); @@ -3010,7 +3010,7 @@ int ha_mcs_impl_write_row(const uchar* buf, TABLE* table, uint64_t rows_changed) } } - if (get_fe_conn_info_ptr() == NULL) + if (get_fe_conn_info_ptr() == nullptr) set_fe_conn_info_ptr((void*)new cal_connection_info()); cal_connection_info* ci = reinterpret_cast(get_fe_conn_info_ptr()); @@ -3057,7 +3057,7 @@ int ha_mcs_impl_write_row(const uchar* buf, TABLE* table, uint64_t rows_changed) int ha_mcs_impl_update_row() { - if (get_fe_conn_info_ptr() == NULL) + if (get_fe_conn_info_ptr() == nullptr) set_fe_conn_info_ptr((void*)new cal_connection_info()); cal_connection_info* ci = reinterpret_cast(get_fe_conn_info_ptr()); @@ -3071,7 +3071,7 @@ int ha_mcs_impl_update_row() int ha_mcs_impl_delete_row() { - if (get_fe_conn_info_ptr() == NULL) + if (get_fe_conn_info_ptr() == nullptr) set_fe_conn_info_ptr((void*)new cal_connection_info()); cal_connection_info* ci = reinterpret_cast(get_fe_conn_info_ptr()); @@ -3087,7 +3087,7 @@ void ha_mcs_impl_start_bulk_insert(ha_rows rows, TABLE* table) { THD* thd = current_thd; - if (get_fe_conn_info_ptr() == NULL) + if (get_fe_conn_info_ptr() == nullptr) set_fe_conn_info_ptr((void*)new cal_connection_info()); cal_connection_info* ci = reinterpret_cast(get_fe_conn_info_ptr()); @@ -3300,7 +3300,7 @@ void ha_mcs_impl_start_bulk_insert(ha_rows rows, TABLE* table) SECURITY_ATTRIBUTES saAttr; saAttr.nLength = sizeof(SECURITY_ATTRIBUTES); saAttr.bInheritHandle = TRUE; - saAttr.lpSecurityDescriptor = NULL; + saAttr.lpSecurityDescriptor = nullptr; HANDLE handleList[2]; const char* pSectionMsg; bSuccess = true; @@ -3320,7 +3320,7 @@ void ha_mcs_impl_start_bulk_insert(ha_rows rows, TABLE* table) } // Launch cpimport - LPPROC_THREAD_ATTRIBUTE_LIST lpAttributeList = NULL; + LPPROC_THREAD_ATTRIBUTE_LIST lpAttributeList = nullptr; SIZE_T attrSize = 0; STARTUPINFOEX siStartInfo; @@ -3338,7 +3338,7 @@ void ha_mcs_impl_start_bulk_insert(ha_rows rows, TABLE* table) pSectionMsg = "HeapAlloc for AttrList"; lpAttributeList = reinterpret_cast (HeapAlloc(GetProcessHeap(), 0, attrSize)); - bSuccess = lpAttributeList != NULL; + bSuccess = lpAttributeList != nullptr; } if (bSuccess) @@ -3373,8 +3373,8 @@ void ha_mcs_impl_start_bulk_insert(ha_rows rows, TABLE* table) memset(&siStartInfo, 0, sizeof(STARTUPINFOEX)); siStartInfo.StartupInfo.cb = sizeof(STARTUPINFOEX); siStartInfo.lpAttributeList = lpAttributeList; - siStartInfo.StartupInfo.hStdError = NULL; - siStartInfo.StartupInfo.hStdOutput = NULL; + siStartInfo.StartupInfo.hStdError = nullptr; + siStartInfo.StartupInfo.hStdOutput = nullptr; siStartInfo.StartupInfo.hStdInput = ci->cpimport_stdin_Rd; siStartInfo.StartupInfo.dwFlags |= STARTF_USESTDHANDLES; // Create the child process. @@ -3603,7 +3603,7 @@ int ha_mcs_impl_end_bulk_insert(bool abort, TABLE* table) std::string aTmpDir(startup::StartUp::tmpDir()); - if (get_fe_conn_info_ptr() == NULL) + if (get_fe_conn_info_ptr() == nullptr) set_fe_conn_info_ptr((void*)new cal_connection_info()); cal_connection_info* ci = reinterpret_cast(get_fe_conn_info_ptr()); @@ -3831,7 +3831,7 @@ int ha_mcs_impl_end_bulk_insert(bool abort, TABLE* table) int ha_mcs_impl_commit (handlerton* hton, THD* thd, bool all) { - if (get_fe_conn_info_ptr() == NULL) + if (get_fe_conn_info_ptr() == nullptr) set_fe_conn_info_ptr((void*)new cal_connection_info()); cal_connection_info* ci = reinterpret_cast(get_fe_conn_info_ptr()); @@ -3863,7 +3863,7 @@ int ha_mcs_impl_commit (handlerton* hton, THD* thd, bool all) int ha_mcs_impl_rollback (handlerton* hton, THD* thd, bool all) { - if (get_fe_conn_info_ptr() == NULL) + if (get_fe_conn_info_ptr() == nullptr) set_fe_conn_info_ptr((void*)new cal_connection_info()); cal_connection_info* ci = reinterpret_cast(get_fe_conn_info_ptr()); @@ -3897,7 +3897,7 @@ int ha_mcs_impl_close_connection (handlerton* hton, THD* thd) // from vtable(lock_type = 2) // An ugly way. I will use ha_data w/o external_lock. // This in MCOL-2178 - cal_connection_info* ci = NULL; + cal_connection_info* ci = nullptr; if(thd_get_ha_data(thd, hton) != (void*)0x42) // 0x42 is the magic CS sets when setup hton { ci = reinterpret_cast(thd_get_ha_data(thd, hton)); @@ -3911,7 +3911,7 @@ int ha_mcs_impl_close_connection (handlerton* hton, THD* thd) { rc = ha_mcs_impl_close_connection_(hton, thd, *ci); delete ci->dmlProc; - ci->dmlProc = NULL; + ci->dmlProc = nullptr; } if (ci->cal_conn_hndl) @@ -3927,7 +3927,7 @@ int ha_mcs_impl_rename_table(const char* from, const char* to) { IDEBUG( cout << "ha_mcs_impl_rename_table: " << from << " => " << to << endl ); - if (get_fe_conn_info_ptr() == NULL) + if (get_fe_conn_info_ptr() == nullptr) set_fe_conn_info_ptr((void*)new cal_connection_info()); cal_connection_info* ci = reinterpret_cast(get_fe_conn_info_ptr()); @@ -3970,7 +3970,7 @@ COND* ha_mcs_impl_cond_push(COND* cond, TABLE* table) alias.assign(table->alias.ptr(), table->alias.length()); IDEBUG( cout << "ha_mcs_impl_cond_push: " << alias << endl ); - if (get_fe_conn_info_ptr() == NULL) + if (get_fe_conn_info_ptr() == nullptr) set_fe_conn_info_ptr((void*)new cal_connection_info()); cal_connection_info* ci = reinterpret_cast(get_fe_conn_info_ptr()); @@ -4009,7 +4009,7 @@ COND* ha_mcs_impl_cond_push(COND* cond, TABLE* table) if (ti.condInfo) { delete ti.condInfo; - ti.condInfo = 0; + ti.condInfo = nullptr; ci->tableMap[table] = ti; } @@ -4022,7 +4022,7 @@ COND* ha_mcs_impl_cond_push(COND* cond, TABLE* table) } else { - return NULL; + return nullptr; } } @@ -4045,7 +4045,7 @@ int ha_mcs_impl_external_lock(THD* thd, TABLE* table, int lock_type) alias.assign(table->alias.ptr(), table->alias.length()); IDEBUG( cout << "external_lock for " << alias << endl ); - if (get_fe_conn_info_ptr() == NULL) + if (get_fe_conn_info_ptr() == nullptr) set_fe_conn_info_ptr((void*)new cal_connection_info()); cal_connection_info* ci = reinterpret_cast(get_fe_conn_info_ptr()); @@ -4062,7 +4062,9 @@ int ha_mcs_impl_external_lock(THD* thd, TABLE* table, int lock_type) CalTableMap::iterator mapiter = ci->tableMap.find(table); // make sure this is a release lock (2nd) call called in // the table mode. - if (mapiter != ci->tableMap.end() && mapiter->second.csep && lock_type == 2) + if (mapiter != ci->tableMap.end() + && (mapiter->second.condInfo || mapiter->second.csep) + && lock_type == 2) { // CS ends up processing query with handlers // table mode @@ -4075,13 +4077,13 @@ int ha_mcs_impl_external_lock(THD* thd, TABLE* table, int lock_type) ci->extendedStats = mapiter->second.conn_hndl->extendedStats; ci->miniStats = mapiter->second.conn_hndl->miniStats; sm::sm_cleanup(mapiter->second.conn_hndl); - mapiter->second.conn_hndl = 0; + mapiter->second.conn_hndl = nullptr; } if (mapiter->second.condInfo) { delete mapiter->second.condInfo; - mapiter->second.condInfo = 0; + mapiter->second.condInfo = nullptr; } // MCOL-2178 Check for tableMap size to set this only once. @@ -4132,7 +4134,7 @@ int ha_mcs_impl_external_lock(THD* thd, TABLE* table, int lock_type) if (tme.second.condInfo) { delete tme.second.condInfo; - tme.second.condInfo= 0; + tme.second.condInfo= nullptr; } } ci->tableMap.clear(); @@ -4200,7 +4202,7 @@ int ha_mcs_impl_group_by_init(mcs_handler_info *handler_info, TABLE* table) boost::shared_ptr csc = CalpontSystemCatalog::makeCalpontSystemCatalog(sessionID); csc->identity(CalpontSystemCatalog::FE); - if (get_fe_conn_info_ptr() == NULL) + if (get_fe_conn_info_ptr() == nullptr) set_fe_conn_info_ptr((void*)new cal_connection_info()); cal_connection_info* ci = reinterpret_cast(get_fe_conn_info_ptr()); @@ -4519,7 +4521,7 @@ int ha_mcs_impl_group_by_init(mcs_handler_info *handler_info, TABLE* table) // make sure rowgroup is null so the new meta data can be taken. This is for some case mysql // call rnd_init for a table more than once. - ti.tpl_scan_ctx->rowGroup = NULL; + ti.tpl_scan_ctx->rowGroup = nullptr; try { @@ -4621,7 +4623,7 @@ int ha_mcs_impl_group_by_next(TABLE* table) // if (MIGR::infinidb_vtable.impossibleWhereOnUnion) // return HA_ERR_END_OF_FILE; - if (get_fe_conn_info_ptr() == NULL) + if (get_fe_conn_info_ptr() == nullptr) set_fe_conn_info_ptr((void*)new cal_connection_info()); cal_connection_info* ci = reinterpret_cast(get_fe_conn_info_ptr()); @@ -4700,7 +4702,7 @@ int ha_mcs_impl_group_by_end(TABLE* table) { int rc = 0; THD* thd = current_thd; - cal_connection_info* ci = NULL; + cal_connection_info* ci = nullptr; if (thd->slave_thread && !get_replication_slave(thd) && ( thd->lex->sql_command == SQLCOM_INSERT || @@ -4962,8 +4964,8 @@ int ha_cs_impl_pushdown_init(mcs_handler_info* handler_info, TABLE* table) sm::cpsm_conhdl_t* hndl; SCSEP csep; // Declare handlers ptrs in this scope for future use. - select_handler* sh = NULL; - derived_handler* dh = NULL; + select_handler* sh = nullptr; + derived_handler* dh = nullptr; // update traceFlags according to the autoswitch state. ci->traceFlags = (ci->traceFlags | CalpontSelectExecutionPlan::TRACE_TUPLE_OFF)^ @@ -5237,7 +5239,7 @@ int ha_cs_impl_pushdown_init(mcs_handler_info* handler_info, TABLE* table) // make sure rowgroup is null so the new meta data can be taken. This is for some case mysql // call rnd_init for a table more than once. - ti.tpl_scan_ctx->rowGroup = NULL; + ti.tpl_scan_ctx->rowGroup = nullptr; try { @@ -5315,7 +5317,7 @@ int ha_cs_impl_select_next(uchar* buf, TABLE* table) int rc = HA_ERR_END_OF_FILE; THD* thd = current_thd; - if (get_fe_conn_info_ptr() == NULL) + if (get_fe_conn_info_ptr() == nullptr) set_fe_conn_info_ptr((void*)new cal_connection_info()); cal_connection_info* ci = reinterpret_cast(get_fe_conn_info_ptr()); @@ -5366,7 +5368,7 @@ int ha_cs_impl_select_next(uchar* buf, TABLE* table) // make sure rowgroup is null so the new meta data can be taken. This is for some case mysql // call rnd_init for a table more than once. - ti.tpl_scan_ctx->rowGroup = NULL; + ti.tpl_scan_ctx->rowGroup = nullptr; try { diff --git a/dbcon/mysql/ha_mcs_opt_rewrites.cpp b/dbcon/mysql/ha_mcs_opt_rewrites.cpp index 62c4c057a..a29366fed 100644 --- a/dbcon/mysql/ha_mcs_opt_rewrites.cpp +++ b/dbcon/mysql/ha_mcs_opt_rewrites.cpp @@ -1,4 +1,4 @@ -/* Copyright (C) 2019 MariaDB Corporation +/* Copyright (C) 2019-20 MariaDB Corporation This program is free software; you can redistribute it and/or modify it under the terms of the GNU General Public License @@ -14,6 +14,9 @@ along with this program; if not, write to the Free Software Foundation, Inc., 51 Franklin Street, Fifth Floor, Boston, MA 02110-1301, USA. */ + +#include + #include "ha_mcs_opt_rewrites.h" // Search simplify_joins() function in the server's code for detail @@ -243,3 +246,87 @@ simplify_joins_mcs(JOIN *join, List *join_list, COND *conds, bool to } DBUG_RETURN(conds); } + +/*@brief in_subselect_rewrite_walk - Rewrites Item_in_subselect*/ +/************************************************************ +* DESCRIPTION: +* It traverses filter predicates searching for +* Item_in_subselect and rewrites it adding equi-join predicate +* to finalise IN_2_EXISTS rewrite. +* PARAMETERS: +* item_arg - Item to check. +* arg - bool to early return if predicate injection fails. +* RETURN: +***********************************************************/ +void in_subselect_rewrite_walk(const Item* item_arg, void* arg) +{ + bool* result= reinterpret_cast(arg); + if (*result) return; + + Item* item= const_cast(item_arg); + + JOIN* join= nullptr; + if (typeid(*item) == typeid(Item_in_subselect)) + { + Item_in_subselect* sub= reinterpret_cast(item); + // MCS 1.4.3 doesn't support IN + subquery with UNION so + // we safe to take this JOIN. + join= sub->unit->first_select()->join; + // Inject equi-JOIN predicates if needed. + *result= sub->create_in_to_exists_cond(join); + *result= (*result) ? *result : + sub->inject_in_to_exists_cond(join); + } + else if (typeid(*item) == typeid(Item_singlerow_subselect)) + { + Item_singlerow_subselect* sub= + reinterpret_cast(item); + // MCS 1.4.3 doesn't support IN + subquery with UNION so + // we safe to take this JOIN. + join= sub->unit->first_select()->join; + } + else + { + // Exit for any but dedicated Items. + return; + } + + // Walk recursively to process nested IN ops. + if (join->conds) + { + join->conds->traverse_cond(in_subselect_rewrite_walk, + arg, Item::POSTFIX); + } +} + +/*@brief in_subselect_rewrite - Rewrites Item_in_subselect*/ +/************************************************************ +* DESCRIPTION: +* It traverses TABLE_LISTs running in_subselect_rewrite_walk +* PARAMETERS: +* select_lex +* RETURN: +* bool to to indicate predicate injection failures. +***********************************************************/ +bool in_subselect_rewrite(SELECT_LEX *select_lex) +{ + bool result = false; + TABLE_LIST *tbl; + List_iterator_fast li(select_lex->leaf_tables); + while (!result && (tbl= li++)) + { + if (tbl->is_view_or_derived()) + { + SELECT_LEX *dsl = tbl->derived->first_select(); + result = in_subselect_rewrite(dsl); + } + } + + if (select_lex->join && select_lex->join->conds) + { + select_lex->join->conds->traverse_cond(in_subselect_rewrite_walk, &result, + Item::POSTFIX); + } + + return result; +} diff --git a/dbcon/mysql/ha_mcs_opt_rewrites.h b/dbcon/mysql/ha_mcs_opt_rewrites.h index 2b221d1c5..3cf985b23 100644 --- a/dbcon/mysql/ha_mcs_opt_rewrites.h +++ b/dbcon/mysql/ha_mcs_opt_rewrites.h @@ -21,6 +21,7 @@ #include "idb_mysql.h" COND *simplify_joins_mcs(JOIN *join, List *join_list, COND *conds, bool top, bool in_sj); +bool in_subselect_rewrite(SELECT_LEX *select_lex); #endif diff --git a/dbcon/mysql/ha_mcs_pushdown.cpp b/dbcon/mysql/ha_mcs_pushdown.cpp index d73e8a861..5c5db65f8 100644 --- a/dbcon/mysql/ha_mcs_pushdown.cpp +++ b/dbcon/mysql/ha_mcs_pushdown.cpp @@ -42,26 +42,6 @@ void disable_indices_for_CEJ(THD *thd_) } } -bool optimize_unflattened_subqueries_mcs(SELECT_LEX *select_lex) -{ - bool result = false; - TABLE_LIST *tbl; - List_iterator_fast li(select_lex->leaf_tables); - while (!result && (tbl= li++)) - { - if (tbl->is_view_or_derived()) - { - SELECT_LEX *dsl = tbl->derived->first_select(); - result = optimize_unflattened_subqueries_mcs(dsl); - } - } - - result = (!result) ? - select_lex->optimize_unflattened_subqueries(false) : true; - - return result; -} - void mutate_optimizer_flags(THD *thd_) { // MCOL-2178 Disable all optimizer flags as it was in the fork. @@ -242,7 +222,6 @@ void save_join_predicates(const Item* item, void* arg) } } - /*@brief check_walk - It traverses filter conditions */ /************************************************************ * DESCRIPTION: @@ -823,18 +802,9 @@ create_columnstore_select_handler(THD* thd, SELECT_LEX* select_lex) COND *conds = nullptr; if (!unsupported_feature) { - conds = simplify_joins_mcs(join, select_lex->join_list, + conds= simplify_joins_mcs(join, select_lex->join_list, join->conds, TRUE, FALSE); } - // MCOL-3747 IN-TO-EXISTS rewrite inside MDB didn't add - // an equi-JOIN condition. - if (!unsupported_feature - && optimize_unflattened_subqueries_mcs(select_lex)) - { - unsupported_feature = true; - handler->err_msg.assign("create_columnstore_select_handler(): \ - Internal error occured in optimize_unflattened_subqueries_mcs()"); - } if (!unsupported_feature && conds) { @@ -843,9 +813,19 @@ create_columnstore_select_handler(THD* thd, SELECT_LEX* select_lex) #endif join->conds = conds; } + + // MCOL-3747 IN-TO-EXISTS rewrite inside MDB didn't add + // an equi-JOIN condition. + if (!unsupported_feature && in_subselect_rewrite(select_lex)) + { + unsupported_feature = true; + handler->err_msg.assign("create_columnstore_select_handler(): \ + Internal error occured in in_subselect_rewrite()"); + } + } - // We can't raise error now so set an error to raise it later in init_SH. + // We shouldn't raise error now so set an error to raise it later in init_SH. handler->rewrite_error= unsupported_feature; // Return SH even if init fails b/c CS changed SELECT_LEX structures diff --git a/dbcon/mysql/ha_mcs_pushdown.h b/dbcon/mysql/ha_mcs_pushdown.h index e8d5d23e3..4492f0470 100644 --- a/dbcon/mysql/ha_mcs_pushdown.h +++ b/dbcon/mysql/ha_mcs_pushdown.h @@ -1,5 +1,5 @@ /* - Copyright (c) 2019 MariaDB + Copyright (c) 2019-20 MariaDB This program is free software; you can redistribute it and/or modify it under the terms of the GNU General Public License as published by From 37acb583cc581d842c5c1ab95fc6d9e67600b6d0 Mon Sep 17 00:00:00 2001 From: jmrojas2332 Date: Tue, 24 Mar 2020 20:52:01 +0000 Subject: [PATCH 14/21] MCOL-2181 MessageQueueClient :: setup (): unknown name or service --- utils/messageqcpp/messagequeue.cpp | 5 +++++ 1 file changed, 5 insertions(+) diff --git a/utils/messageqcpp/messagequeue.cpp b/utils/messageqcpp/messagequeue.cpp index 142d4c19c..0d3999053 100644 --- a/utils/messageqcpp/messagequeue.cpp +++ b/utils/messageqcpp/messagequeue.cpp @@ -163,6 +163,11 @@ void MessageQueueClient::setup(bool syncProto) otherEndIPStr = fConfig->getConfig(fOtherEnd, "IPAddr"); otherEndPortStr = fConfig->getConfig(fOtherEnd, "Port"); + if (otherEndIPStr == "unassigned") + { + otherEndIPStr = "0.0.0.0"; + } + if (otherEndIPStr.length() == 0) otherEndIPStr = "127.0.0.1"; if (otherEndPortStr.length() == 0 || static_cast(strtol(otherEndPortStr.c_str(), 0, 0)) == 0) From 04a90277828fba1a114aa4ea93f76c883f0bc948 Mon Sep 17 00:00:00 2001 From: David Hall Date: Thu, 26 Mar 2020 13:35:15 -0500 Subject: [PATCH 15/21] MCOL-3904 group_concat with sum from subquery in order by --- dbcon/joblist/groupconcat.cpp | 16 +++++++++++----- 1 file changed, 11 insertions(+), 5 deletions(-) diff --git a/dbcon/joblist/groupconcat.cpp b/dbcon/joblist/groupconcat.cpp index 7f832946e..d48c5e289 100644 --- a/dbcon/joblist/groupconcat.cpp +++ b/dbcon/joblist/groupconcat.cpp @@ -365,12 +365,18 @@ void GroupConcatAgUM::applyMapping(const boost::shared_array& mapping, cons // For some reason the rowgroup mapping fcns don't work right in this class. for (uint64_t i = 0; i < fRow.getColumnCount(); i++) { - if (fRow.getColumnWidth(i) > 8 && - (fRow.getColTypes()[i] == execplan::CalpontSystemCatalog::CHAR || - fRow.getColTypes()[i] == execplan::CalpontSystemCatalog::VARCHAR || - fRow.getColTypes()[i] == execplan::CalpontSystemCatalog::TEXT)) + if (fRow.getColumnWidth(i) > 8) { - fRow.setStringField(row.getStringPointer(mapping[i]), row.getStringLength(mapping[i]), i); + if (fRow.getColTypes()[i] == execplan::CalpontSystemCatalog::CHAR || + fRow.getColTypes()[i] == execplan::CalpontSystemCatalog::VARCHAR || + fRow.getColTypes()[i] == execplan::CalpontSystemCatalog::TEXT) + { + fRow.setStringField(row.getStringPointer(mapping[i]), row.getStringLength(mapping[i]), i); + } + else if (fRow.getColTypes()[i] == execplan::CalpontSystemCatalog::LONGDOUBLE) + { + fRow.setLongDoubleField(row.getLongDoubleField(mapping[i]), i); + } } else { From eaf3c85ddf967d73d2e3c26febcba11e42dbdb4d Mon Sep 17 00:00:00 2001 From: Jose Date: Thu, 2 Apr 2020 19:31:19 +0000 Subject: [PATCH 16/21] MCOL-2096 Fix evaluation for derived table inside ConstantFilter --- dbcon/execplan/constantfilter.cpp | 13 ++++++++++++- 1 file changed, 12 insertions(+), 1 deletion(-) diff --git a/dbcon/execplan/constantfilter.cpp b/dbcon/execplan/constantfilter.cpp index 24998cd09..00155c499 100644 --- a/dbcon/execplan/constantfilter.cpp +++ b/dbcon/execplan/constantfilter.cpp @@ -272,8 +272,19 @@ void ConstantFilter::setDerivedTable() fDerivedTable = ""; return; } + for (unsigned i = 0; i < fFilterList.size(); i++) + { + fFilterList[i]->setDerivedTable(); + } - fDerivedTable = fCol->derivedTable(); + if (!fFilterList.empty()) + { + fDerivedTable = fFilterList[0]->derivedTable(); + } + else + { + fDerivedTable = ""; + } } void ConstantFilter::replaceRealCol(std::vector& derivedColList) From a1b8df55e4ab3cbf4ad6aaa671b9dcaa2155d034 Mon Sep 17 00:00:00 2001 From: Patrick LeBlanc Date: Tue, 7 Apr 2020 16:01:37 -0400 Subject: [PATCH 17/21] Fixed a typo in post-mysql-install re systemd detection. --- oam/install_scripts/post-mysql-install | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/oam/install_scripts/post-mysql-install b/oam/install_scripts/post-mysql-install index de2d920da..7960c7637 100755 --- a/oam/install_scripts/post-mysql-install +++ b/oam/install_scripts/post-mysql-install @@ -6,7 +6,7 @@ # check if running systemd running_systemd() { - if [ ps --no-headers -o comm 1 == "systemd" ]; then + if [ "$(ps --no-headers -o comm 1)" == "systemd" ]; then echo 0 else echo 1 From 3a5574d71f5e33fe7ee069be05f4c61887285c43 Mon Sep 17 00:00:00 2001 From: Patrick LeBlanc Date: Tue, 7 Apr 2020 16:23:13 -0400 Subject: [PATCH 18/21] Fixed a couple more instances of a typo. --- oam/install_scripts/columnstore-pre-uninstall.in | 2 +- oam/install_scripts/columnstore.in | 2 +- 2 files changed, 2 insertions(+), 2 deletions(-) diff --git a/oam/install_scripts/columnstore-pre-uninstall.in b/oam/install_scripts/columnstore-pre-uninstall.in index 204698d89..e1f107fd9 100755 --- a/oam/install_scripts/columnstore-pre-uninstall.in +++ b/oam/install_scripts/columnstore-pre-uninstall.in @@ -5,7 +5,7 @@ # pre-uninstall steps for columnstore install running_systemd() { - if [ ps --no-headers -o comm 1 == "systemd" ]; then + if [ "$(ps --no-headers -o comm 1)" == "systemd" ]; then echo 0 else echo 1 diff --git a/oam/install_scripts/columnstore.in b/oam/install_scripts/columnstore.in index 98355d255..c9f612af4 100644 --- a/oam/install_scripts/columnstore.in +++ b/oam/install_scripts/columnstore.in @@ -18,7 +18,7 @@ ### END INIT INFO running_systemd() { - if [ ps --no-headers -o comm 1 == "systemd" ]; then + if [ "$(ps --no-headers -o comm 1)" == "systemd" ]; then echo 0 else echo 1 From 35b59e81c52f3e9a2fa9147a33b23cd6bb9afbdd Mon Sep 17 00:00:00 2001 From: David Hall Date: Thu, 9 Apr 2020 10:53:27 -0500 Subject: [PATCH 19/21] MCOL-3934 Delete xml parse trees during shutdown --- dbcon/mysql/ha_mcs.h | 2 ++ 1 file changed, 2 insertions(+) diff --git a/dbcon/mysql/ha_mcs.h b/dbcon/mysql/ha_mcs.h index 8baf31e7e..c76aff81c 100644 --- a/dbcon/mysql/ha_mcs.h +++ b/dbcon/mysql/ha_mcs.h @@ -21,6 +21,7 @@ #include #include "idb_mysql.h" #include "ha_mcs_sysvars.h" +#include "configcpp.h" extern handlerton* mcs_hton; @@ -49,6 +50,7 @@ public: ha_mcs(handlerton* hton, TABLE_SHARE* table_arg); ~ha_mcs() { + config::Config::deleteInstanceMap(); } /** @brief From e7bc7e931300c3b2d9910e89dada1d33e63c6b47 Mon Sep 17 00:00:00 2001 From: David Hall Date: Thu, 9 Apr 2020 15:30:44 -0500 Subject: [PATCH 20/21] MCOL-3924 Use the first parse error found Often, after an error is found, it cascades into a bunch of errors, and each overwrites the previous. This sometimes obfuscates the true problem. --- dbcon/mysql/ha_mcs_execplan.cpp | 19 ++++++++++--------- 1 file changed, 10 insertions(+), 9 deletions(-) diff --git a/dbcon/mysql/ha_mcs_execplan.cpp b/dbcon/mysql/ha_mcs_execplan.cpp index 6a40e4992..150c3b427 100755 --- a/dbcon/mysql/ha_mcs_execplan.cpp +++ b/dbcon/mysql/ha_mcs_execplan.cpp @@ -6883,7 +6883,7 @@ int getSelectPlan(gp_walk_info& gwi, SELECT_LEX& select_lex, return -1; } } - else + else if ( !gwi.fatalParseError ) { Message::Args args; args.add(ifp->func_name()); @@ -7478,14 +7478,15 @@ int getSelectPlan(gp_walk_info& gwi, SELECT_LEX& select_lex, if (nonSupportItem) { - Message::Args args; - - if (nonSupportItem->name.length) - args.add("'" + string(nonSupportItem->name.str) + "'"); - else - args.add(""); - - gwi.parseErrorText = IDBErrorInfo::instance()->errorMsg(ERR_NON_SUPPORT_GROUP_BY, args); + if (gwi.parseErrorText.length() == 0) + { + Message::Args args; + if (nonSupportItem->name.length) + args.add("'" + string(nonSupportItem->name.str) + "'"); + else + args.add(""); + gwi.parseErrorText = IDBErrorInfo::instance()->errorMsg(ERR_NON_SUPPORT_GROUP_BY, args); + } setError(gwi.thd, ER_CHECK_NOT_IMPLEMENTED, gwi.parseErrorText, gwi); return ER_CHECK_NOT_IMPLEMENTED; } From 225789e171964db98a9484ef89bac2d535f5befb Mon Sep 17 00:00:00 2001 From: David Hall Date: Fri, 10 Apr 2020 13:33:16 -0500 Subject: [PATCH 21/21] MCOL-3934 free memory from XML parse at shutdown --- dbcon/mysql/ha_mcs.cpp | 1 + dbcon/mysql/ha_mcs.h | 2 -- 2 files changed, 1 insertion(+), 2 deletions(-) diff --git a/dbcon/mysql/ha_mcs.cpp b/dbcon/mysql/ha_mcs.cpp index 04fd9053a..5c2c3560c 100644 --- a/dbcon/mysql/ha_mcs.cpp +++ b/dbcon/mysql/ha_mcs.cpp @@ -167,6 +167,7 @@ static int columnstore_done_func(void* p) { DBUG_ENTER("columnstore_done_func"); + config::Config::deleteInstanceMap(); my_hash_free(&mcs_open_tables); #ifndef _MSC_VER pthread_mutex_destroy(&mcs_mutex); diff --git a/dbcon/mysql/ha_mcs.h b/dbcon/mysql/ha_mcs.h index c76aff81c..8baf31e7e 100644 --- a/dbcon/mysql/ha_mcs.h +++ b/dbcon/mysql/ha_mcs.h @@ -21,7 +21,6 @@ #include #include "idb_mysql.h" #include "ha_mcs_sysvars.h" -#include "configcpp.h" extern handlerton* mcs_hton; @@ -50,7 +49,6 @@ public: ha_mcs(handlerton* hton, TABLE_SHARE* table_arg); ~ha_mcs() { - config::Config::deleteInstanceMap(); } /** @brief