From 3c6443053e1909f2b8e8a09309d3d828a18a4b8c Mon Sep 17 00:00:00 2001 From: david hill Date: Thu, 15 Mar 2018 16:24:26 -0500 Subject: [PATCH 01/57] MCOL-1217 - setup msqyl rep on addmodule --- oamapps/postConfigure/helpers.cpp | 1 - procmgr/main.cpp | 2 +- procmgr/processmanager.cpp | 181 +++++++++++++----------------- procmgr/processmanager.h | 2 +- procmon/processmonitor.cpp | 48 +++++++- 5 files changed, 122 insertions(+), 112 deletions(-) diff --git a/oamapps/postConfigure/helpers.cpp b/oamapps/postConfigure/helpers.cpp index b53a17302..ba97915a9 100644 --- a/oamapps/postConfigure/helpers.cpp +++ b/oamapps/postConfigure/helpers.cpp @@ -489,7 +489,6 @@ int sendReplicationRequest(int IserverTypeInstall, std::string password, bool pm } else { - cout << endl << "ERROR: Module not Active, replication not done on " << (*pt).DeviceName << endl; pt++; } } diff --git a/procmgr/main.cpp b/procmgr/main.cpp index 434902138..118ac0d73 100644 --- a/procmgr/main.cpp +++ b/procmgr/main.cpp @@ -1803,7 +1803,7 @@ void pingDeviceThread() DeviceNetworkConfig devicenetworkconfig; devicenetworkconfig.DeviceName = moduleName; devicenetworklist.push_back(devicenetworkconfig); - processManager.setMySQLReplication(devicenetworklist, oam::UnassignedName, false, true); + processManager.setMySQLReplication(devicenetworklist, oam::UnassignedName, true); } } diff --git a/procmgr/processmanager.cpp b/procmgr/processmanager.cpp index fdb3f76b1..7c18d6ed8 100755 --- a/procmgr/processmanager.cpp +++ b/procmgr/processmanager.cpp @@ -2546,7 +2546,7 @@ void processMSG(messageqcpp::IOSocket* cfIos) // target = root password oam::DeviceNetworkList devicenetworklist; - status = processManager.setMySQLReplication(devicenetworklist, oam::UnassignedName, false, true, target); + status = processManager.setMySQLReplication(devicenetworklist, oam::UnassignedName, true, target); log.writeLog(__LINE__, "Enable MySQL Replication status: " + oam.itoa(status) ); @@ -2568,7 +2568,7 @@ void processMSG(messageqcpp::IOSocket* cfIos) // target = root password oam::DeviceNetworkList devicenetworklist; - status = processManager.setMySQLReplication(devicenetworklist, oam::UnassignedName, false, false, target, false); + status = processManager.setMySQLReplication(devicenetworklist, oam::UnassignedName, false, target, false); log.writeLog(__LINE__, "Disable MySQL Replication status: " + oam.itoa(status) ); @@ -3621,18 +3621,21 @@ int ProcessManager::startProcess(string moduleName, string processName, { Oam oam; - //skip if module is DISABLED - int opState; - bool degraded; - try{ - oam.getModuleStatus(moduleName, opState, degraded); - } - catch(...) - {} + if ( actionIndicator != oam::STATUS_UPDATE ) + { + //skip if module is DISABLED + int opState; + bool degraded; + try{ + oam.getModuleStatus(moduleName, opState, degraded); + } + catch(...) + {} - //check if disabled - if (opState == oam::MAN_DISABLED || opState == oam::AUTO_DISABLED) - return API_SUCCESS; + //check if disabled + if (opState == oam::MAN_DISABLED || opState == oam::AUTO_DISABLED) + return API_SUCCESS; + } ByteStream msg; ByteStream::byte requestID = START; @@ -4190,8 +4193,8 @@ int ProcessManager::startProcessType( std::string processName ) if ( systemprocessstatus.processstatus[i].ProcessName == processName) { // found one, request restart of it int retStatus = processManager.startProcess(systemprocessstatus.processstatus[i].Module, - processName, - FORCEFUL); + processName, + FORCEFUL); log.writeLog(__LINE__, "StartProcessType: Start ACK received from Process-Monitor, return status = " + oam.itoa(retStatus), LOG_TYPE_DEBUG); } } @@ -5426,9 +5429,23 @@ int ProcessManager::addModule(oam::DeviceNetworkList devicenetworklist, std::str log.writeLog(__LINE__, "addModule - sleep 60 - give ProcMon time to CONFIGURE and restart", LOG_TYPE_DEBUG); sleep(60); + //start mysqld on the new modules so mysql replication can be setup + listPT = devicenetworklist.begin(); + for( ; listPT != devicenetworklist.end() ; listPT++) + { + processManager.startProcess((*listPT).DeviceName, "mysqld", oam::STATUS_UPDATE); + } + log.writeLog(__LINE__, "Setup MySQL Replication for new Modules being Added", LOG_TYPE_DEBUG); - processManager.setMySQLReplication(devicenetworklist, oam::UnassignedName, false, true, password ); + processManager.setMySQLReplication(devicenetworklist, oam::UnassignedName, true, password, true, true ); + //stop mysqld + listPT = devicenetworklist.begin(); + for( ; listPT != devicenetworklist.end() ; listPT++) + { + processManager.stopProcess((*listPT).DeviceName, "mysqld", oam::FORCEFUL, true ); + } + return API_SUCCESS; } @@ -8728,7 +8745,7 @@ int ProcessManager::switchParentOAMModule(std::string newActiveModuleName) //change master MySQL Replication setup log.writeLog(__LINE__, "Setup MySQL Replication for new Parent Module during switch-over", LOG_TYPE_DEBUG); oam::DeviceNetworkList devicenetworklist; - processManager.setMySQLReplication(devicenetworklist, newActiveModuleName, false, false, oam::UnassignedName); + processManager.setMySQLReplication(devicenetworklist, newActiveModuleName, false, oam::UnassignedName); } catch (exception& ex) @@ -9530,7 +9547,7 @@ int ProcessManager::OAMParentModuleChange() //change master MySQL Replication setup log.writeLog(__LINE__, "Setup this node as MySQL Replication Master", LOG_TYPE_DEBUG); oam::DeviceNetworkList devicenetworklist; - processManager.setMySQLReplication(devicenetworklist, config.moduleName(), true); + processManager.setMySQLReplication(devicenetworklist, config.moduleName()); } //set query system state not ready @@ -10207,7 +10224,7 @@ void ProcessManager::flushInodeCache() * * ******************************************************************************************/ -int ProcessManager::setMySQLReplication(oam::DeviceNetworkList devicenetworklist, std::string masterModule, bool failover, bool distributeDB, std::string password, bool enable) +int ProcessManager::setMySQLReplication(oam::DeviceNetworkList devicenetworklist, std::string masterModule, bool distributeDB, std::string password, bool enable, bool addModule) { Oam oam; @@ -10226,57 +10243,6 @@ int ProcessManager::setMySQLReplication(oam::DeviceNetworkList devicenetworklist return oam::API_SUCCESS; } - //also skip if single-server, multi-node seperate with 1 UM, multi-node combo with 1 PM - -/* string SingleServerInstall = "n"; - try { - oam.getSystemConfig("SingleServerInstall", SingleServerInstall); - } - catch(...) { - SingleServerInstall = "n"; - } - - //single server check - if ( SingleServerInstall == "y" ) - { - log.writeLog(__LINE__, "setMySQLReplication: Single-Server, exiting", LOG_TYPE_DEBUG); - return oam::API_SUCCESS; - } - - //combined system check - if ( config.ServerInstallType() == oam::INSTALL_COMBINE_DM_UM_PM && !failover ) { - try { - Config* sysConfig = Config::makeConfig(); - if ( sysConfig->getConfig("DBRM_Controller", "NumWorkers") == "1" ) { - log.writeLog(__LINE__, "setMySQLReplication: 1 configured module, exiting", LOG_TYPE_DEBUG); - return oam::API_SUCCESS; - } - } - catch(...) - { - log.writeLog(__LINE__, "setMySQLReplication: makeConfig exception, exiting", LOG_TYPE_DEBUG); - return oam::API_SUCCESS; - } - } - - //seperate system check - if ( ( config.ServerInstallType() != oam::INSTALL_COMBINE_DM_UM_PM ) && - (PMwithUM == "n" ) ) - { - ModuleTypeConfig moduletypeconfig; - try{ - oam.getSystemConfig("um", moduletypeconfig); - } - catch(...) - {} - - if ( moduletypeconfig.ModuleCount < 2 ) - { - log.writeLog(__LINE__, "setMySQLReplication: moduleCount = 1, exiting", LOG_TYPE_DEBUG); - return oam::API_SUCCESS; - } - } -*/ log.writeLog(__LINE__, "Setup MySQL Replication", LOG_TYPE_DEBUG); //get master info @@ -10329,18 +10295,21 @@ int ProcessManager::setMySQLReplication(oam::DeviceNetworkList devicenetworklist if ( remoteModuleName == masterModule ) continue; - // skip disabled modules - int opState = oam::ACTIVE; - bool degraded; - try { - oam.getModuleStatus(remoteModuleName, opState, degraded); + if ( !addModule ) + { + // skip disabled modules + int opState = oam::ACTIVE; + bool degraded; + try { + oam.getModuleStatus(remoteModuleName, opState, degraded); + } + catch(...) + {} + + if (opState == oam::MAN_DISABLED || opState == oam::AUTO_DISABLED) + continue; } - catch(...) - {} - - if (opState == oam::MAN_DISABLED || opState == oam::AUTO_DISABLED) - continue; - + // don't do PMs unless PMwithUM flag is set if ( config.ServerInstallType() != oam::INSTALL_COMBINE_DM_UM_PM ) { string moduleType = remoteModuleName.substr(0,MAX_MODULE_TYPE_SIZE); @@ -10418,18 +10387,21 @@ int ProcessManager::setMySQLReplication(oam::DeviceNetworkList devicenetworklist if ( remoteModuleName == masterModule ) continue; - // skip disabled modules - int opState = oam::ACTIVE; - bool degraded; - try { - oam.getModuleStatus(remoteModuleName, opState, degraded); + if ( !addModule ) + { + // skip disabled modules + int opState = oam::ACTIVE; + bool degraded; + try { + oam.getModuleStatus(remoteModuleName, opState, degraded); + } + catch(...) + {} + + if (opState == oam::MAN_DISABLED || opState == oam::AUTO_DISABLED) + continue; } - catch(...) - {} - - if (opState == oam::MAN_DISABLED || opState == oam::AUTO_DISABLED) - continue; - + // don't do PMs unless PMwithUM flag is set if ( config.ServerInstallType() != oam::INSTALL_COMBINE_DM_UM_PM ) { string moduleType = remoteModuleName.substr(0,MAX_MODULE_TYPE_SIZE); @@ -10477,18 +10449,21 @@ int ProcessManager::setMySQLReplication(oam::DeviceNetworkList devicenetworklist if ( remoteModuleName == masterModule ) continue; - // skip disabled modules - int opState = oam::ACTIVE; - bool degraded; - try { - oam.getModuleStatus(remoteModuleName, opState, degraded); + if ( !addModule ) + { + // skip disabled modules + int opState = oam::ACTIVE; + bool degraded; + try { + oam.getModuleStatus(remoteModuleName, opState, degraded); + } + catch(...) + {} + + if (opState == oam::MAN_DISABLED || opState == oam::AUTO_DISABLED) + continue; } - catch(...) - {} - - if (opState == oam::MAN_DISABLED || opState == oam::AUTO_DISABLED) - continue; - + log.writeLog(__LINE__, "Setup Slave MySQL Replication on " + remoteModuleName, LOG_TYPE_DEBUG); ByteStream msg1; diff --git a/procmgr/processmanager.h b/procmgr/processmanager.h index d60a9eaf2..55d291b23 100644 --- a/procmgr/processmanager.h +++ b/procmgr/processmanager.h @@ -545,7 +545,7 @@ public: /** @brief Set MySQL Replication */ - int setMySQLReplication(oam::DeviceNetworkList devicenetworklist, std::string masterModule = oam::UnassignedName, bool failover = false, bool distributeDB = false, std::string password = oam::UnassignedName, bool enable = true); + int setMySQLReplication(oam::DeviceNetworkList devicenetworklist, std::string masterModule = oam::UnassignedName, bool distributeDB = false, std::string password = oam::UnassignedName, bool enable = true, bool addModule = false); /** @brief Gluster Assign dbroot to a module */ diff --git a/procmon/processmonitor.cpp b/procmon/processmonitor.cpp index 8f51cccf1..aa10f2666 100644 --- a/procmon/processmonitor.cpp +++ b/procmon/processmonitor.cpp @@ -457,6 +457,24 @@ void ProcessMonitor::processMessage(messageqcpp::ByteStream msg, messageqcpp::IO log.writeLog(__LINE__, "MSG RECEIVED: Stop process request on " + processName); int requestStatus = API_SUCCESS; + // check for mysql + if ( processName == "mysqld" ) { + try { + oam.actionMysqlCalpont(MYSQL_STOP); + } + catch(...) + {} + + ackMsg << (ByteStream::byte) ACK; + ackMsg << (ByteStream::byte) STOP; + ackMsg << (ByteStream::byte) API_SUCCESS; + mq.write(ackMsg); + + log.writeLog(__LINE__, "STOP: ACK back to ProcMgr, return status = " + oam.itoa((int) API_SUCCESS)); + + break; + } + processList::iterator listPtr; processList* aPtr = config.monitoredListPtr(); listPtr = aPtr->begin(); @@ -502,6 +520,24 @@ void ProcessMonitor::processMessage(messageqcpp::ByteStream msg, messageqcpp::IO msg >> manualFlag; log.writeLog(__LINE__, "MSG RECEIVED: Start process request on: " + processName); + // check for mysql + if ( processName == "mysqld" ) { + try { + oam.actionMysqlCalpont(MYSQL_START); + } + catch(...) + {} + + ackMsg << (ByteStream::byte) ACK; + ackMsg << (ByteStream::byte) START; + ackMsg << (ByteStream::byte) API_SUCCESS; + mq.write(ackMsg); + + log.writeLog(__LINE__, "START: ACK back to ProcMgr, return status = " + oam.itoa((int) API_SUCCESS)); + + break; + } + ProcessConfig processconfig; ProcessStatus processstatus; try { @@ -605,7 +641,7 @@ void ProcessMonitor::processMessage(messageqcpp::ByteStream msg, messageqcpp::IO int requestStatus = API_SUCCESS; // check for mysql restart - if ( processName == "mysql" ) { + if ( processName == "mysqld" ) { try { oam.actionMysqlCalpont(MYSQL_RESTART); } @@ -4952,11 +4988,11 @@ int ProcessMonitor::runMasterRep(std::string& masterLogFile, std::string& master //skip if module is not ACTIVE - int opState = oam::ACTIVE; - bool degraded; - oam.getModuleStatus(moduleName, opState, degraded); - if (opState != oam::ACTIVE) - continue; +// int opState = oam::ACTIVE; +// bool degraded; +// oam.getModuleStatus(moduleName, opState, degraded); +// if (opState != oam::ACTIVE) +// continue; bool passwordError = false; From e14b327a578efa49d52117fbf3de25fea4b9b8c0 Mon Sep 17 00:00:00 2001 From: Roman Nozdrin Date: Sat, 17 Mar 2018 14:06:23 +0300 Subject: [PATCH 02/57] MCOL-1052: Skeleton for group_by_handler for CS. --- dbcon/mysql/ha_calpont.cpp | 20 ++++++++++++++++++++ dbcon/mysql/ha_calpont.h | 19 +++++++++++++++++++ 2 files changed, 39 insertions(+) diff --git a/dbcon/mysql/ha_calpont.cpp b/dbcon/mysql/ha_calpont.cpp index 1a1b0b5ec..a4ace792f 100644 --- a/dbcon/mysql/ha_calpont.cpp +++ b/dbcon/mysql/ha_calpont.cpp @@ -123,6 +123,9 @@ static int calpont_rollback(handlerton* hton, THD* thd, bool all); static int calpont_close_connection ( handlerton* hton, THD* thd ); handlerton* calpont_hton; +static group_by_handler * +create_calpont_group_by_handler(THD *thd, Query *query); + /* Variables for example share methods */ /* @@ -218,6 +221,7 @@ static int columnstore_init_func(void* p) calpont_hton->commit = calpont_commit; calpont_hton->rollback = calpont_rollback; calpont_hton->close_connection = calpont_close_connection; + calpont_hton->create_group_by = create_calpont_group_by_handler; DBUG_RETURN(0); } @@ -1135,6 +1139,22 @@ static MYSQL_SYSVAR_ULONG( 0); #endif +static group_by_handler * +create_calpont_group_by_handler(THD *thd, Query *query) +{ + ha_calpont_group_by_handler *handler; + Item *item; + List_iterator_fast it(*query->select); + + handler= new ha_calpont_group_by_handler(thd, query->select, query->from); + return handler; +} + +int ha_calpont_group_by_handler::next_row() +{ + return(0); +} + static struct st_mysql_sys_var* calpont_system_variables[] = { // MYSQL_SYSVAR(enum_var), diff --git a/dbcon/mysql/ha_calpont.h b/dbcon/mysql/ha_calpont.h index 6abaf51ed..b187c03d4 100644 --- a/dbcon/mysql/ha_calpont.h +++ b/dbcon/mysql/ha_calpont.h @@ -40,6 +40,8 @@ #include #include "idb_mysql.h" +extern handlerton* calpont_hton; + /** @brief EXAMPLE_SHARE is a structure that will be shared among all open handlers. This example implements the minimum of what you will probably need. @@ -245,5 +247,22 @@ public: } }; + +class ha_calpont_group_by_handler: public group_by_handler +{ + List *fields; + TABLE_LIST *table_list; + bool first_row; + +public: + ha_calpont_group_by_handler(THD *thd_arg, List *fields_arg, +TABLE_LIST *table_list_arg) + : group_by_handler(thd_arg, calpont_hton), fields(fields_arg), + table_list(table_list_arg) {} + ~ha_calpont_group_by_handler() {} + int init_scan() { first_row= true ; return 0; } + int next_row(); + int end_scan() { return 0; } +}; #endif //HA_CALPONT_H__ From a03ecb7a8ef18243433b6ecc8492319ef124fb51 Mon Sep 17 00:00:00 2001 From: Roman Nozdrin Date: Mon, 19 Mar 2018 20:45:21 +0300 Subject: [PATCH 03/57] MCOL-1052. The handler returns fixed value. --- dbcon/mysql/ha_calpont.cpp | 10 ++++++++-- 1 file changed, 8 insertions(+), 2 deletions(-) diff --git a/dbcon/mysql/ha_calpont.cpp b/dbcon/mysql/ha_calpont.cpp index a4ace792f..533d99e76 100644 --- a/dbcon/mysql/ha_calpont.cpp +++ b/dbcon/mysql/ha_calpont.cpp @@ -1145,14 +1145,20 @@ create_calpont_group_by_handler(THD *thd, Query *query) ha_calpont_group_by_handler *handler; Item *item; List_iterator_fast it(*query->select); - + handler= new ha_calpont_group_by_handler(thd, query->select, query->from); return handler; } int ha_calpont_group_by_handler::next_row() { - return(0); + if (!first_row) + return(HA_ERR_END_OF_FILE); + first_row= 0; + Field *field = *(table->field); + field->store(5LL, 1); + field->set_notnull(); + return(0); } static struct st_mysql_sys_var* calpont_system_variables[] = From cff504c8bfbde147981e34199fdc60a7dbb1a365 Mon Sep 17 00:00:00 2001 From: Roman Nozdrin Date: Thu, 22 Mar 2018 17:12:56 +0300 Subject: [PATCH 04/57] MCOL-1052. init_scan() initial implementation. --- dbcon/mysql/ha_calpont.cpp | 59 +++++++++-- dbcon/mysql/ha_calpont.h | 29 +++--- dbcon/mysql/ha_calpont_impl.cpp | 173 ++++++++++++++++++++++++++++++++ dbcon/mysql/ha_calpont_impl.h | 4 + 4 files changed, 246 insertions(+), 19 deletions(-) diff --git a/dbcon/mysql/ha_calpont.cpp b/dbcon/mysql/ha_calpont.cpp index 533d99e76..3c5f9a41e 100644 --- a/dbcon/mysql/ha_calpont.cpp +++ b/dbcon/mysql/ha_calpont.cpp @@ -1139,28 +1139,73 @@ static MYSQL_SYSVAR_ULONG( 0); #endif +/*@brief ha_calpont_impl_group_by_init - Get data for MariaDB group_by + pushdown handler +*/ +/*********************************************************** + * DESCRIPTION: + * Creates a group_by pushdown handler. + * Details are in server/sql/group_by_handler.h + * PARAMETERS: + * thd - THD pointer. + * query - Query structure, that describes the pushdown query. + * RETURN: + * group_by_handler if success + * NULL in other case + ***********************************************************/ static group_by_handler * create_calpont_group_by_handler(THD *thd, Query *query) { - ha_calpont_group_by_handler *handler; - Item *item; - List_iterator_fast it(*query->select); + ha_calpont_group_by_handler *handler = NULL; + Item *item; + List_iterator_fast it(*query->select); - handler= new ha_calpont_group_by_handler(thd, query->select, query->from); - return handler; + if ( thd->infinidb_vtable.vtable_state != THD::INFINIDB_DISABLE_VTABLE ) + { + handler = new ha_calpont_group_by_handler(thd, query); + } + + return handler; +} + +int ha_calpont_group_by_handler::init_scan() +{ + DBUG_ENTER("ha_calpont_group_by_handler::init_scan"); + + int rc = ha_calpont_impl_group_by_init(query, table); + + DBUG_RETURN(rc); +// return 0; } int ha_calpont_group_by_handler::next_row() { - if (!first_row) - return(HA_ERR_END_OF_FILE); +// if (!first_row) +// return(HA_ERR_END_OF_FILE); + DBUG_ENTER("ha_calpont_group_by_handler::next_row"); + int rc = ha_calpont_impl_group_by_next(query, table); + + DBUG_RETURN(rc); +/* first_row= 0; Field *field = *(table->field); field->store(5LL, 1); field->set_notnull(); return(0); +*/ } +int ha_calpont_group_by_handler::end_scan() +{ + DBUG_ENTER("ha_calpont_group_by_handler::end_scan"); + + int rc = ha_calpont_impl_group_by_end(query, table); + + DBUG_RETURN(rc); +// return 0; +} + + static struct st_mysql_sys_var* calpont_system_variables[] = { // MYSQL_SYSVAR(enum_var), diff --git a/dbcon/mysql/ha_calpont.h b/dbcon/mysql/ha_calpont.h index b187c03d4..660af8d0a 100644 --- a/dbcon/mysql/ha_calpont.h +++ b/dbcon/mysql/ha_calpont.h @@ -250,19 +250,24 @@ public: class ha_calpont_group_by_handler: public group_by_handler { - List *fields; - TABLE_LIST *table_list; - bool first_row; + public: +// ha_calpont_group_by_handler(THD *thd_arg, List *fields_arg, +// TABLE_LIST *table_list_arg, Query *query) + ha_calpont_group_by_handler(THD *thd_arg, Query *query) + : group_by_handler(thd_arg, calpont_hton), fields(query->select), + table_list(query->from), query(query) {} + ~ha_calpont_group_by_handler() {} + int init_scan(); + int next_row(); + int end_scan(); + + private: + List *fields; + TABLE_LIST *table_list; + bool first_row; + Query *query; + -public: - ha_calpont_group_by_handler(THD *thd_arg, List *fields_arg, -TABLE_LIST *table_list_arg) - : group_by_handler(thd_arg, calpont_hton), fields(fields_arg), - table_list(table_list_arg) {} - ~ha_calpont_group_by_handler() {} - int init_scan() { first_row= true ; return 0; } - int next_row(); - int end_scan() { return 0; } }; #endif //HA_CALPONT_H__ diff --git a/dbcon/mysql/ha_calpont_impl.cpp b/dbcon/mysql/ha_calpont_impl.cpp index 8078d452e..6662b8394 100644 --- a/dbcon/mysql/ha_calpont_impl.cpp +++ b/dbcon/mysql/ha_calpont_impl.cpp @@ -4957,5 +4957,178 @@ int ha_calpont_impl_rnd_pos(uchar* buf, uchar* pos) return ER_INTERNAL_ERROR; } +/*@brief ha_calpont_impl_group_by_init - Get data for MariaDB group_by + pushdown handler +*/ +/*********************************************************** + * DESCRIPTION: + * Prepare data for group_by_handler::next_row() calls. + * PARAMETERS: + * query - Query structure, that describes the pushdown query. + * table - TABLE pointer The table to save the result set in. + * RETURN: + * 0 if success + * others if something went wrong whilst getting the result set + ***********************************************************/ +int ha_calpont_impl_group_by_init(Query* query, TABLE* table) +{ + //first_row= true; + + return(0); +} + +/*@brief ha_calpont_impl_group_by_next - Return result set for MariaDB group_by + pushdown handler +*/ +/*********************************************************** + * DESCRIPTION: + * Return a result record for each group_by_handler::next_row() call. + * PARAMETERS: + * query - Query structure, that describes the pushdown query. + * table - TABLE pointer The table to save the result set in. + * RETURN: + * 0 if success + * HA_ERR_END_OF_FILE if the record set has come to an end + * others if something went wrong whilst getting the result set + ***********************************************************/ +int ha_calpont_impl_group_by_next(Query* query, TABLE* table) +{ + //if (!first_row) + //return(HA_ERR_END_OF_FILE); + + //first_row = 0; + //Field *field = *(table->field); + //field->store(5LL, 1); + //field->set_notnull(); + //return(0); + THD* thd = current_thd; + + /* If this node is the slave, ignore DML to IDB tables */ + if (thd->slave_thread && ( + thd->lex->sql_command == SQLCOM_INSERT || + thd->lex->sql_command == SQLCOM_INSERT_SELECT || + thd->lex->sql_command == SQLCOM_UPDATE || + thd->lex->sql_command == SQLCOM_UPDATE_MULTI || + thd->lex->sql_command == SQLCOM_DELETE || + thd->lex->sql_command == SQLCOM_DELETE_MULTI || + thd->lex->sql_command == SQLCOM_TRUNCATE || + thd->lex->sql_command == SQLCOM_LOAD)) + return 0; + + + if (thd->infinidb_vtable.vtable_state == THD::INFINIDB_ERROR) + return ER_INTERNAL_ERROR; + + // @bug 3005 + if (thd->infinidb_vtable.vtable_state == THD::INFINIDB_SELECT_VTABLE && + string(table->s->table_name.str).find("$vtable") != 0) + return HA_ERR_END_OF_FILE; + + if (((thd->lex)->sql_command == SQLCOM_UPDATE) || ((thd->lex)->sql_command == SQLCOM_DELETE) || + ((thd->lex)->sql_command == SQLCOM_DELETE_MULTI) || ((thd->lex)->sql_command == SQLCOM_UPDATE_MULTI)) + return HA_ERR_END_OF_FILE; + + // @bug 2547 + if (thd->infinidb_vtable.impossibleWhereOnUnion) + return HA_ERR_END_OF_FILE; + + // @bug 2232. Basic SP support + // @bug 3939. Only error out for sp with select. Let pass for alter table in sp. + if (thd->infinidb_vtable.call_sp && (thd->lex)->sql_command != SQLCOM_ALTER_TABLE) + { + setError(thd, ER_CHECK_NOT_IMPLEMENTED, "This stored procedure syntax is not supported by Columnstore in this version"); + thd->infinidb_vtable.vtable_state = THD::INFINIDB_ERROR; + return ER_INTERNAL_ERROR; + } + + if (!thd->infinidb_vtable.cal_conn_info) + thd->infinidb_vtable.cal_conn_info = (void*)(new cal_connection_info()); + + cal_connection_info* ci = reinterpret_cast(thd->infinidb_vtable.cal_conn_info); + + // @bug 3078 + if (thd->killed == KILL_QUERY || thd->killed == KILL_QUERY_HARD) + { + if (ci->cal_conn_hndl) + { + // send ExeMgr a signal before cloing the connection + ByteStream msg; + ByteStream::quadbyte qb = 0; + msg << qb; + + try + { + ci->cal_conn_hndl->exeMgr->write(msg); + } + catch (...) + { + // cancel query. ignore connection failure. + } + + sm::sm_cleanup(ci->cal_conn_hndl); + ci->cal_conn_hndl = 0; + } + + return 0; + } + + if (ci->alterTableState > 0) return HA_ERR_END_OF_FILE; + + cal_table_info ti; + ti = ci->tableMap[table]; + int rc = HA_ERR_END_OF_FILE; + + if (!ti.tpl_ctx || !ti.tpl_scan_ctx) + { + CalpontSystemCatalog::removeCalpontSystemCatalog(tid2sid(thd->thread_id)); + return ER_INTERNAL_ERROR; + } + + idbassert(ti.msTablePtr == table); + + try + { + //rc = fetchNextRow(buf, ti, ci); + rc = 0; + } + catch (std::exception& e) + { + string emsg = string("Error while fetching from ExeMgr: ") + e.what(); + setError(thd, ER_INTERNAL_ERROR, emsg); + CalpontSystemCatalog::removeCalpontSystemCatalog(tid2sid(thd->thread_id)); + return ER_INTERNAL_ERROR; + } + + ci->tableMap[table] = ti; + + if (rc != 0 && rc != HA_ERR_END_OF_FILE) + { + string emsg; + + // remove this check when all error handling migrated to the new framework. + if (rc >= 1000) + emsg = ti.tpl_scan_ctx->errMsg; + else + { + logging::ErrorCodes errorcodes; + emsg = errorcodes.errorString(rc); + } + + setError(thd, ER_INTERNAL_ERROR, emsg); + //setError(thd, ER_INTERNAL_ERROR, "testing"); + ci->stats.fErrorNo = rc; + CalpontSystemCatalog::removeCalpontSystemCatalog(tid2sid(thd->thread_id)); + rc = ER_INTERNAL_ERROR; + } + + return rc; +} + +int ha_calpont_impl_group_by_end(Query* query, TABLE* table) +{ + return 0; +} + + // vim:sw=4 ts=4: diff --git a/dbcon/mysql/ha_calpont_impl.h b/dbcon/mysql/ha_calpont_impl.h index 1e7f7d5df..f6bea072c 100644 --- a/dbcon/mysql/ha_calpont_impl.h +++ b/dbcon/mysql/ha_calpont_impl.h @@ -48,6 +48,10 @@ extern int ha_calpont_impl_external_lock(THD* thd, TABLE* table, int lock_type); extern int ha_calpont_impl_update_row(); extern int ha_calpont_impl_delete_row(); extern int ha_calpont_impl_rnd_pos(uchar* buf, uchar* pos); +extern int ha_calpont_impl_group_by_init(Query* query, TABLE* table); +extern int ha_calpont_impl_group_by_next(Query* query, TABLE* table); +extern int ha_calpont_impl_group_by_end(Query* query, TABLE* table); + #endif #ifdef NEED_CALPONT_INTERFACE From fa4067b6f082faab4e18a1bf2019dcaa215d0d02 Mon Sep 17 00:00:00 2001 From: Roman Nozdrin Date: Tue, 27 Mar 2018 18:37:00 +0300 Subject: [PATCH 05/57] MCOL-1052 Generate execution plan for a aggregated function query call. --- dbcon/mysql/ha_calpont.cpp | 9 +- dbcon/mysql/ha_calpont.h | 4 - dbcon/mysql/ha_calpont_execplan.cpp | 130 +++- dbcon/mysql/ha_calpont_impl.cpp | 1024 ++++++++++++++++++++++++++- dbcon/mysql/ha_calpont_impl.h | 10 +- dbcon/mysql/ha_calpont_impl_if.h | 5 +- 6 files changed, 1159 insertions(+), 23 deletions(-) diff --git a/dbcon/mysql/ha_calpont.cpp b/dbcon/mysql/ha_calpont.cpp index 3c5f9a41e..e0e835c4b 100644 --- a/dbcon/mysql/ha_calpont.cpp +++ b/dbcon/mysql/ha_calpont.cpp @@ -1160,7 +1160,7 @@ create_calpont_group_by_handler(THD *thd, Query *query) Item *item; List_iterator_fast it(*query->select); - if ( thd->infinidb_vtable.vtable_state != THD::INFINIDB_DISABLE_VTABLE ) + if ( thd->infinidb_vtable.vtable_state == THD::INFINIDB_DISABLE_VTABLE ) { handler = new ha_calpont_group_by_handler(thd, query); } @@ -1172,10 +1172,9 @@ int ha_calpont_group_by_handler::init_scan() { DBUG_ENTER("ha_calpont_group_by_handler::init_scan"); - int rc = ha_calpont_impl_group_by_init(query, table); + int rc = ha_calpont_impl_group_by_init(this, table); DBUG_RETURN(rc); -// return 0; } int ha_calpont_group_by_handler::next_row() @@ -1183,7 +1182,7 @@ int ha_calpont_group_by_handler::next_row() // if (!first_row) // return(HA_ERR_END_OF_FILE); DBUG_ENTER("ha_calpont_group_by_handler::next_row"); - int rc = ha_calpont_impl_group_by_next(query, table); + int rc = ha_calpont_impl_group_by_next(this, table); DBUG_RETURN(rc); /* @@ -1199,7 +1198,7 @@ int ha_calpont_group_by_handler::end_scan() { DBUG_ENTER("ha_calpont_group_by_handler::end_scan"); - int rc = ha_calpont_impl_group_by_end(query, table); + int rc = ha_calpont_impl_group_by_end(this, table); DBUG_RETURN(rc); // return 0; diff --git a/dbcon/mysql/ha_calpont.h b/dbcon/mysql/ha_calpont.h index 660af8d0a..c3f2573e2 100644 --- a/dbcon/mysql/ha_calpont.h +++ b/dbcon/mysql/ha_calpont.h @@ -260,14 +260,10 @@ class ha_calpont_group_by_handler: public group_by_handler int init_scan(); int next_row(); int end_scan(); - - private: List *fields; TABLE_LIST *table_list; bool first_row; Query *query; - - }; #endif //HA_CALPONT_H__ diff --git a/dbcon/mysql/ha_calpont_execplan.cpp b/dbcon/mysql/ha_calpont_execplan.cpp index c4a37a77d..8c0934f4c 100644 --- a/dbcon/mysql/ha_calpont_execplan.cpp +++ b/dbcon/mysql/ha_calpont_execplan.cpp @@ -4440,7 +4440,7 @@ ReturnedColumn* buildAggregateColumn(Item* item, gp_walk_info& gwi) } else if (ac->constCol()) { - gwi.count_asterisk_list.push_back(ac); + gwi.count_asterisk_list.push_back(ac); } // For UDAF, populate the context and call the UDAF init() function. @@ -7831,6 +7831,10 @@ int cp_get_plan(THD* thd, SCSEP& csep) else if (status < 0) return status; + cerr << "---------------- cp_get_plan EXECUTION PLAN ----------------" << endl; + cerr << *csep << endl ; + cerr << "-------------- EXECUTION PLAN END --------------\n" << endl; + // Derived table projection and filter optimization. derivedTableOptimization(csep); @@ -7932,4 +7936,128 @@ int cp_get_table_plan(THD* thd, SCSEP& csep, cal_table_info& ti) return 0; } + +int cp_get_group_plan(THD* thd, SCSEP& csep, cal_table_info& ti ) +{ + gp_walk_info* gwi = ti.condInfo; + List_iterator_fast it(*ti.groupByFields); + Item* item; + + if (!gwi) + gwi = new gp_walk_info(); + + gwi->thd = thd; + LEX* lex = thd->lex; + idbassert(lex != 0); + uint32_t sessionID = csep->sessionID(); + gwi->sessionid = sessionID; + TABLE* table = ti.msTablePtr; + boost::shared_ptr csc = + CalpontSystemCatalog::makeCalpontSystemCatalog(sessionID); + csc->identity(CalpontSystemCatalog::FE); + + // get all columns that mysql needs to fetch + MY_BITMAP* read_set = table->read_set; + Field** f_ptr, *field; + gwi->columnMap.clear(); + + const CalpontSystemCatalog::TableAliasName tblAliasName= + make_aliastable(table->s->db.str, table->s->table_name.str, table->alias.c_ptr()); + + gwi->tbList.push_back(tblAliasName); + + while ((item = it++)) + { + Item *arg0; + Field *field; + ReturnedColumn* rc = buildAggregateColumn(item, *gwi); + //string alias(table->alias.c_ptr()); + //rc->tableAlias(lower(alias)); + assert (rc); + boost::shared_ptr sprc(rc); + gwi->returnedCols.push_back(sprc); + arg0=((Item_sum*) item)->get_arg(0); + field=((Item_field*) arg0)->field; + gwi->columnMap.insert(CalpontSelectExecutionPlan::ColumnMap::value_type(string(field->field_name), sprc)); + //arg0=((Item_sum*) item)->get_arg(0); + //field=((Item_field*) arg0)->field; + } + +/* + for (f_ptr = table->field ; (field = *f_ptr) ; f_ptr++) + { + if (bitmap_is_set(read_set, field->field_index)) + { + SimpleColumn* sc = new SimpleColumn(table->s->db.str, table->s->table_name.str, field->field_name, sessionID); + string alias(table->alias.c_ptr()); + sc->tableAlias(lower(alias)); + assert (sc); + boost::shared_ptr spsc(sc); + gwi->returnedCols.push_back(spsc); + gwi->columnMap.insert(CalpontSelectExecutionPlan::ColumnMap::value_type(string(field->field_name), spsc)); + } + } +*/ + if (gwi->columnMap.empty()) + { + CalpontSystemCatalog::TableName tn = make_table(table->s->db.str, table->s->table_name.str); + CalpontSystemCatalog::TableAliasName tan = make_aliastable(table->s->db.str, table->s->table_name.str, table->alias.c_ptr()); + SimpleColumn* sc = getSmallestColumn(csc, tn, tan, table, *gwi); + SRCP srcp(sc); + gwi->columnMap.insert(CalpontSelectExecutionPlan::ColumnMap::value_type(sc->columnName(), srcp)); + gwi->returnedCols.push_back(srcp); + } + + + // get filter + if (ti.condInfo) + { + gp_walk_info* gwi = ti.condInfo; + ParseTree* filters = 0; + ParseTree* ptp = 0; + ParseTree* rhs = 0; + + while (!gwi->ptWorkStack.empty()) + { + filters = gwi->ptWorkStack.top(); + gwi->ptWorkStack.pop(); + SimpleFilter* sf = dynamic_cast(filters->data()); + + if (sf && sf->op()->data() == "noop") + { + delete filters; + filters = 0; + + if (gwi->ptWorkStack.empty()) + break; + + continue; + } + + if (gwi->ptWorkStack.empty()) + break; + + ptp = new ParseTree(new LogicOperator("and")); + ptp->left(filters); + rhs = gwi->ptWorkStack.top(); + gwi->ptWorkStack.pop(); + ptp->right(rhs); + gwi->ptWorkStack.push(ptp); + } + + csep->filters(filters); + } + + csep->returnedCols(gwi->returnedCols); + csep->columnMap(gwi->columnMap); + CalpontSelectExecutionPlan::TableList tblist; + tblist.push_back(tblAliasName); + csep->tableList(tblist); + + // @bug 3321. Set max number of blocks in a dictionary file to be scanned for filtering + csep->stringScanThreshold(gwi->thd->variables.infinidb_string_scan_threshold); + + return 0; +} + } diff --git a/dbcon/mysql/ha_calpont_impl.cpp b/dbcon/mysql/ha_calpont_impl.cpp index 6662b8394..5d8e91e08 100644 --- a/dbcon/mysql/ha_calpont_impl.cpp +++ b/dbcon/mysql/ha_calpont_impl.cpp @@ -171,6 +171,7 @@ const unsigned NONSUPPORTED_ERR_THRESH = 2000; vector rmParms; ResourceManager* rm = ResourceManager::instance(); bool useHdfs = rm->useHdfs(); + //convenience fcn inline uint32_t tid2sid(const uint32_t tid) { @@ -760,6 +761,438 @@ int fetchNextRow(uchar* buf, cal_table_info& ti, cal_connection_info* ci) return rc; } +int fetchNextRowGrHand(cal_table_info& ti, cal_connection_info* ci) +{ + int rc = HA_ERR_END_OF_FILE; + int num_attr = ti.msTablePtr->s->fields; + sm::status_t sm_stat; + + try + { + if (ti.conn_hndl) + { + sm_stat = sm::tpl_scan_fetch(ti.tpl_scan_ctx, ti.conn_hndl); + } + else if (ci->cal_conn_hndl) + { + sm_stat = sm::tpl_scan_fetch(ti.tpl_scan_ctx, ci->cal_conn_hndl, (int*)(¤t_thd->killed)); + } + else + throw runtime_error("internal error"); + } + catch (std::exception& ex) + { +// @bug 2244. Always log this msg for now, as we try to track down when/why we are +// losing socket connection with ExeMgr +//#ifdef INFINIDB_DEBUG + tpl_scan_fetch_LogException( ti, ci, &ex); +//#endif + sm_stat = sm::CALPONT_INTERNAL_ERROR; + } + catch (...) + { +// @bug 2244. Always log this msg for now, as we try to track down when/why we are +// losing socket connection with ExeMgr +//#ifdef INFINIDB_DEBUG + tpl_scan_fetch_LogException( ti, ci, 0 ); +//#endif + sm_stat = sm::CALPONT_INTERNAL_ERROR; + } + + if (sm_stat == sm::STATUS_OK) + { + Field** f; + f = ti.msTablePtr->field; + //set all fields to null in null col bitmap + //memset(buf, -1, ti.msTablePtr->s->null_bytes); + std::vector& colTypes = ti.tpl_scan_ctx->ctp; + int64_t intColVal = 0; + uint64_t uintColVal = 0; + char tmp[256]; + + RowGroup* rowGroup = ti.tpl_scan_ctx->rowGroup; + + // table mode mysql expects all columns of the table. mapping between columnoid and position in rowgroup + // set coltype.position to be the position in rowgroup. only set once. + if (ti.tpl_scan_ctx->rowsreturned == 0 && + (ti.tpl_scan_ctx->traceFlags & execplan::CalpontSelectExecutionPlan::TRACE_TUPLE_OFF)) + { + for (uint32_t i = 0; i < rowGroup->getColumnCount(); i++) + { + int oid = rowGroup->getOIDs()[i]; + int j = 0; + + for (; j < num_attr; j++) + { + // mysql should haved eliminated duplicate projection columns + if (oid == colTypes[j].columnOID || oid == colTypes[j].ddn.dictOID) + { + colTypes[j].colPosition = i; + break; + } + } + } + } + + rowgroup::Row row; + rowGroup->initRow(&row); + rowGroup->getRow(ti.tpl_scan_ctx->rowsreturned, &row); + int s; + + for (int p = 0; p < num_attr; p++, f++) + { + //This col is going to be written + bitmap_set_bit(ti.msTablePtr->write_set, (*f)->field_index); + + // get coltype if not there yet + if (colTypes[0].colWidth == 0) + { + for (short c = 0; c < num_attr; c++) + { + colTypes[c].colPosition = c; + colTypes[c].colWidth = rowGroup->getColumnWidth(c); + colTypes[c].colDataType = rowGroup->getColTypes()[c]; + colTypes[c].columnOID = rowGroup->getOIDs()[c]; + colTypes[c].scale = rowGroup->getScale()[c]; + colTypes[c].precision = rowGroup->getPrecision()[c]; + } + } + + CalpontSystemCatalog::ColType colType(colTypes[p]); + + // table mode handling + if (ti.tpl_scan_ctx->traceFlags & execplan::CalpontSelectExecutionPlan::TRACE_TUPLE_OFF) + { + if (colType.colPosition == -1) // not projected by tuplejoblist + continue; + else + s = colType.colPosition; + } + else + { + s = p; + } + + // precision == -16 is borrowed as skip null check indicator for bit ops. + if (row.isNullValue(s) && colType.precision != -16) + { + // @2835. Handle empty string and null confusion. store empty string for string column + if (colType.colDataType == CalpontSystemCatalog::CHAR || + colType.colDataType == CalpontSystemCatalog::VARCHAR || + colType.colDataType == CalpontSystemCatalog::VARBINARY) + { + Field_varstring* f2 = (Field_varstring*)*f; + f2->store(tmp, 0, f2->charset()); + } + + continue; + } + + // fetch and store data + switch (colType.colDataType) + { + case CalpontSystemCatalog::DATE: + { + if ((*f)->null_ptr) + *(*f)->null_ptr &= ~(*f)->null_bit; + + intColVal = row.getUintField<4>(s); + DataConvert::dateToString(intColVal, tmp, 255); + Field_varstring* f2 = (Field_varstring*)*f; + f2->store(tmp, strlen(tmp), f2->charset()); + break; + } + + case CalpontSystemCatalog::DATETIME: + { + if ((*f)->null_ptr) + *(*f)->null_ptr &= ~(*f)->null_bit; + + intColVal = row.getUintField<8>(s); + DataConvert::datetimeToString(intColVal, tmp, 255); + + /* setting the field_length is a sort-of hack. The length + * at this point can be long enough to include mseconds. + * ColumnStore doesn't fully support mseconds yet so if + * they are requested, trim them off. + * At a later date we should set this more intelligently + * based on the result set. + */ + /* MCOL-683: UTF-8 datetime no msecs is 57, this sometimes happens! */ + if (((*f)->field_length > 19) && ((*f)->field_length != 57)) + (*f)->field_length = strlen(tmp); + + Field_varstring* f2 = (Field_varstring*)*f; + f2->store(tmp, strlen(tmp), f2->charset()); + break; + } + + case CalpontSystemCatalog::CHAR: + case CalpontSystemCatalog::VARCHAR: + { + Field_varstring* f2 = (Field_varstring*)*f; + + switch (colType.colWidth) + { + case 1: + intColVal = row.getUintField<1>(s); + f2->store((char*)(&intColVal), strlen((char*)(&intColVal)), f2->charset()); + break; + + case 2: + intColVal = row.getUintField<2>(s); + f2->store((char*)(&intColVal), strlen((char*)(&intColVal)), f2->charset()); + break; + + case 4: + intColVal = row.getUintField<4>(s); + f2->store((char*)(&intColVal), strlen((char*)(&intColVal)), f2->charset()); + break; + + case 8: + //make sure we don't send strlen off into the weeds... + intColVal = row.getUintField<8>(s); + memcpy(tmp, &intColVal, 8); + tmp[8] = 0; + f2->store(tmp, strlen(tmp), f2->charset()); + break; + + default: + f2->store((const char*)row.getStringPointer(s), row.getStringLength(s), f2->charset()); + } + + if ((*f)->null_ptr) + *(*f)->null_ptr &= ~(*f)->null_bit; + + break; + } + + case CalpontSystemCatalog::VARBINARY: + { + Field_varstring* f2 = (Field_varstring*)*f; + + if (current_thd->variables.infinidb_varbin_always_hex) + { + uint32_t l; + const uint8_t* p = row.getVarBinaryField(l, s); + uint32_t ll = l * 2; + boost::scoped_array sca(new char[ll]); + vbin2hex(p, l, sca.get()); + f2->store(sca.get(), ll, f2->charset()); + } + else + f2->store((const char*)row.getVarBinaryField(s), row.getVarBinaryLength(s), f2->charset()); + + if ((*f)->null_ptr) + *(*f)->null_ptr &= ~(*f)->null_bit; + + break; + } + + case CalpontSystemCatalog::BIGINT: + { + intColVal = row.getIntField<8>(s); + storeNumericField(f, intColVal, colType); + break; + } + + case CalpontSystemCatalog::UBIGINT: + { + uintColVal = row.getUintField<8>(s); + storeNumericField(f, uintColVal, colType); + break; + } + + case CalpontSystemCatalog::INT: + { + intColVal = row.getIntField<4>(s); + storeNumericField(f, intColVal, colType); + break; + } + + case CalpontSystemCatalog::UINT: + { + uintColVal = row.getUintField<4>(s); + storeNumericField(f, uintColVal, colType); + break; + } + + case CalpontSystemCatalog::SMALLINT: + { + intColVal = row.getIntField<2>(s); + storeNumericField(f, intColVal, colType); + break; + } + + case CalpontSystemCatalog::USMALLINT: + { + uintColVal = row.getUintField<2>(s); + storeNumericField(f, uintColVal, colType); + break; + } + + case CalpontSystemCatalog::TINYINT: + { + intColVal = row.getIntField<1>(s); + storeNumericField(f, intColVal, colType); + break; + } + + case CalpontSystemCatalog::UTINYINT: + { + uintColVal = row.getUintField<1>(s); + storeNumericField(f, uintColVal, colType); + break; + } + + //In this case, we're trying to load a double output column with float data. This is the + // case when you do sum(floatcol), e.g. + case CalpontSystemCatalog::FLOAT: + case CalpontSystemCatalog::UFLOAT: + { + float dl = row.getFloatField(s); + + if (dl == std::numeric_limits::infinity()) + continue; + + //int64_t* icvp = (int64_t*)&dl; + //intColVal = *icvp; + Field_float* f2 = (Field_float*)*f; + // bug 3485, reserve enough space for the longest float value + // -3.402823466E+38 to -1.175494351E-38, 0, and + // 1.175494351E-38 to 3.402823466E+38. + (*f)->field_length = 40; + + //float float_val = *(float*)(&value); + //f2->store(float_val); + if (f2->decimals() < (uint32_t)row.getScale(s)) + f2->dec = (uint32_t)row.getScale(s); + + f2->store(dl); + + if ((*f)->null_ptr) + *(*f)->null_ptr &= ~(*f)->null_bit; + + break; + + //storeNumericField(f, intColVal, colType); + //break; + } + + case CalpontSystemCatalog::DOUBLE: + case CalpontSystemCatalog::UDOUBLE: + { + double dl = row.getDoubleField(s); + + if (dl == std::numeric_limits::infinity()) + continue; + + Field_double* f2 = (Field_double*)*f; + // bug 3483, reserve enough space for the longest double value + // -1.7976931348623157E+308 to -2.2250738585072014E-308, 0, and + // 2.2250738585072014E-308 to 1.7976931348623157E+308. + (*f)->field_length = 310; + + //double double_val = *(double*)(&value); + //f2->store(double_val); + if (f2->decimals() < (uint32_t)row.getScale(s)) + f2->dec = (uint32_t)row.getScale(s); + + f2->store(dl); + + if ((*f)->null_ptr) + *(*f)->null_ptr &= ~(*f)->null_bit; + + break; + + + //int64_t* icvp = (int64_t*)&dl; + //intColVal = *icvp; + //storeNumericField(f, intColVal, colType); + //break; + } + + case CalpontSystemCatalog::DECIMAL: + case CalpontSystemCatalog::UDECIMAL: + { + intColVal = row.getIntField(s); + storeNumericField(f, intColVal, colType); + break; + } + + case CalpontSystemCatalog::BLOB: + case CalpontSystemCatalog::TEXT: + { + Field_blob* f2 = (Field_blob*)*f; + f2->set_ptr(row.getVarBinaryLength(s), (unsigned char*)row.getVarBinaryField(s)); + + if ((*f)->null_ptr) + *(*f)->null_ptr &= ~(*f)->null_bit; + + break; + } + + default: // treat as int64 + { + intColVal = row.getUintField<8>(s); + storeNumericField(f, intColVal, colType); + break; + } + } + } + + ti.tpl_scan_ctx->rowsreturned++; + ti.c++; +#ifdef INFINIDB_DEBUG + + if ((ti.c % 1000000) == 0) + cerr << "fetchNextRow so far table " << ti.msTablePtr->s->table_name.str << " rows = " << ti.c << endl; + +#endif + ti.moreRows = true; + rc = 0; + } + else if (sm_stat == sm::SQL_NOT_FOUND) + { + IDEBUG( cerr << "fetchNextRow done for table " << ti.msTablePtr->s->table_name.str << " rows = " << ti.c << endl ); + ti.c = 0; + ti.moreRows = false; + rc = HA_ERR_END_OF_FILE; + } + else if (sm_stat == sm::CALPONT_INTERNAL_ERROR) + { + ti.moreRows = false; + rc = ER_INTERNAL_ERROR; + ci->rc = rc; + } + else if ((uint32_t)sm_stat == logging::ERR_LOST_CONN_EXEMGR) + { + ti.moreRows = false; + rc = logging::ERR_LOST_CONN_EXEMGR; + sm::sm_init(tid2sid(current_thd->thread_id), &ci->cal_conn_hndl, + current_thd->variables.infinidb_local_query); + idbassert(ci->cal_conn_hndl != 0); + ci->rc = rc; + } + else if (sm_stat == sm::SQL_KILLED) + { + // query was aborted by the user. treat it the same as limit query. close + // connection after rnd_close. + ti.c = 0; + ti.moreRows = false; + rc = HA_ERR_END_OF_FILE; + ci->rc = rc; + } + else + { + ti.moreRows = false; + rc = sm_stat; + ci->rc = rc; + } + + return rc; +} + void makeUpdateScalarJoin(const ParseTree* n, void* obj) { TreeNode* tn = n->data(); @@ -4962,19 +5395,593 @@ int ha_calpont_impl_rnd_pos(uchar* buf, uchar* pos) */ /*********************************************************** * DESCRIPTION: - * Prepare data for group_by_handler::next_row() calls. + * Prepares data for group_by_handler::next_row() calls. * PARAMETERS: - * query - Query structure, that describes the pushdown query. + * group_hand - group by handler, that preserves initial table and items lists. . * table - TABLE pointer The table to save the result set in. * RETURN: * 0 if success * others if something went wrong whilst getting the result set ***********************************************************/ -int ha_calpont_impl_group_by_init(Query* query, TABLE* table) +int ha_calpont_impl_group_by_init(ha_calpont_group_by_handler* group_hand, TABLE* table) { //first_row= true; + string tableName = group_hand->table_list->table->s->table_name.str; + IDEBUG( cout << "group_by_init for table " << + group_hand->table_list->table->s->table_name.str << endl ); + THD* thd = current_thd; - return(0); + //check whether the system is ready to process statement. +#ifndef _MSC_VER + static DBRM dbrm(true); + bool bSystemQueryReady = dbrm.getSystemQueryReady(); + + if (bSystemQueryReady == 0) + { + // Still not ready + setError(thd, ER_INTERNAL_ERROR, "The system is not yet ready to accept queries"); + thd->infinidb_vtable.vtable_state = THD::INFINIDB_ERROR; + return ER_INTERNAL_ERROR; + } + else if (bSystemQueryReady < 0) + { + // Still not ready + setError(thd, ER_INTERNAL_ERROR, "DBRM is not responding. Cannot accept queries"); + thd->infinidb_vtable.vtable_state = THD::INFINIDB_ERROR; + return ER_INTERNAL_ERROR; + } + +#endif + // prevent "create table as select" from running on slave + thd->infinidb_vtable.hasInfiniDBTable = true; + + /* If this node is the slave, ignore DML to IDB tables */ + /*if (thd->slave_thread && ( + thd->lex->sql_command == SQLCOM_INSERT || + thd->lex->sql_command == SQLCOM_INSERT_SELECT || + thd->lex->sql_command == SQLCOM_UPDATE || + thd->lex->sql_command == SQLCOM_UPDATE_MULTI || + thd->lex->sql_command == SQLCOM_DELETE || + thd->lex->sql_command == SQLCOM_DELETE_MULTI || + thd->lex->sql_command == SQLCOM_TRUNCATE || + thd->lex->sql_command == SQLCOM_LOAD)) + return 0; + */ + + // @bug 3005. if the table is not $vtable, then this could be a UDF defined on the connector. + // watch this for other complications + //if (thd->infinidb_vtable.vtable_state == THD::INFINIDB_SELECT_VTABLE && + // string(table->s->table_name.str).find("$vtable") != 0) + // return 0; + + // return error if error status has been already set + if (thd->infinidb_vtable.vtable_state == THD::INFINIDB_ERROR) + return ER_INTERNAL_ERROR; + + // by pass the extra union trips. return 0 + //if (thd->infinidb_vtable.isUnion && thd->infinidb_vtable.vtable_state == THD::INFINIDB_CREATE_VTABLE) + // return 0; + + // @bug 2232. Basic SP support. Error out non support sp cases. + // @bug 3939. Only error out for sp with select. Let pass for alter table in sp. + /*if (thd->infinidb_vtable.call_sp && (thd->lex)->sql_command != SQLCOM_ALTER_TABLE) + { + setError(thd, ER_CHECK_NOT_IMPLEMENTED, "This stored procedure syntax is not supported by Columnstore in this version"); + thd->infinidb_vtable.vtable_state = THD::INFINIDB_ERROR; + return ER_INTERNAL_ERROR; + } + + // mysql reads table twice for order by + if (thd->infinidb_vtable.vtable_state == THD::INFINIDB_REDO_PHASE1 || + thd->infinidb_vtable.vtable_state == THD::INFINIDB_ORDER_BY) + return 0; + + if ( (thd->lex)->sql_command == SQLCOM_ALTER_TABLE ) + return 0; + + + //Update and delete code + if ( ((thd->lex)->sql_command == SQLCOM_UPDATE) || ((thd->lex)->sql_command == SQLCOM_DELETE) || ((thd->lex)->sql_command == SQLCOM_DELETE_MULTI) || ((thd->lex)->sql_command == SQLCOM_UPDATE_MULTI)) + return doUpdateDelete(thd); + */ + + uint32_t sessionID = tid2sid(thd->thread_id); + boost::shared_ptr csc = CalpontSystemCatalog::makeCalpontSystemCatalog(sessionID); + csc->identity(CalpontSystemCatalog::FE); + + if (!thd->infinidb_vtable.cal_conn_info) + thd->infinidb_vtable.cal_conn_info = (void*)(new cal_connection_info()); + + cal_connection_info* ci = reinterpret_cast(thd->infinidb_vtable.cal_conn_info); + + idbassert(ci != 0); + + /* + MySQL sometimes calls rnd_init multiple times, plan should only be + generated and sent once. + TO DO Check this statement. + */ + if (thd->infinidb_vtable.vtable_state == THD::INFINIDB_DISABLE_VTABLE && + !thd->infinidb_vtable.isNewQuery) + return 0; + + if (thd->killed == KILL_QUERY || thd->killed == KILL_QUERY_HARD) + { + if (ci->cal_conn_hndl) + { + // send ExeMgr a signal before closing the connection + ByteStream msg; + ByteStream::quadbyte qb = 0; + msg << qb; + + try + { + ci->cal_conn_hndl->exeMgr->write(msg); + } + catch (...) + { + // canceling query. ignore connection failure. + } + + sm::sm_cleanup(ci->cal_conn_hndl); + ci->cal_conn_hndl = 0; + } + + return 0; + } + + sm::tableid_t tableid = 0; + cal_table_info ti; + sm::cpsm_conhdl_t* hndl; + SCSEP csep; + + // update traceFlags according to the autoswitch state. replication query + // on slave are in table mode (create table as...) + if (thd->infinidb_vtable.vtable_state == THD::INFINIDB_DISABLE_VTABLE) + { + ci->traceFlags |= CalpontSelectExecutionPlan::TRACE_TUPLE_OFF; + } + + // MCOL-1052 TO DO Remove this + ci->traceFlags = CalpontSelectExecutionPlan::TRACE_LOG; + + bool localQuery = (thd->variables.infinidb_local_query > 0 ? true : false); + + // table mode + if (thd->infinidb_vtable.vtable_state == THD::INFINIDB_DISABLE_VTABLE) + { + ti = ci->tableMap[group_hand->table_list->table]; + + // get connection handle for this table handler + // re-establish table handle + if (ti.conn_hndl) + { + sm::sm_cleanup(ti.conn_hndl); + ti.conn_hndl = 0; + } + + sm::sm_init(sessionID, &ti.conn_hndl, localQuery); + ti.conn_hndl->csc = csc; + hndl = ti.conn_hndl; + + try + { + ti.conn_hndl->connect(); + } + catch (...) + { + setError(thd, ER_INTERNAL_ERROR, IDBErrorInfo::instance()->errorMsg(ERR_LOST_CONN_EXEMGR)); + CalpontSystemCatalog::removeCalpontSystemCatalog(sessionID); + goto error; + } + + // get filter plan for table + if (ti.csep.get() == 0) + { + ti.csep.reset(new CalpontSelectExecutionPlan()); + + SessionManager sm; + BRM::TxnID txnID; + txnID = sm.getTxnID(sessionID); + + if (!txnID.valid) + { + txnID.id = 0; + txnID.valid = true; + } + + QueryContext verID; + verID = sm.verID(); + + ti.csep->txnID(txnID.id); + ti.csep->verID(verID); + ti.csep->sessionID(sessionID); + + if (group_hand->table_list->db_length) + ti.csep->schemaName(group_hand->table_list->db); + + ti.csep->traceFlags(ci->traceFlags); + ti.msTablePtr = group_hand->table_list->table; + ti.groupByFields = group_hand->fields; + + // send plan whenever group_init is called + cp_get_group_plan(thd, ti.csep, ti); + } + + IDEBUG( cerr << tableName << " send plan:" << endl ); + IDEBUG( cerr << *ti.csep << endl ); + csep = ti.csep; + + // for ExeMgr logging sqltext. only log once for the query although multi plans may be sent + if (ci->tableMap.size() == 1) + ti.csep->data(idb_mysql_query_str(thd)); + } + // vtable mode + else + { + //if (!ci->cal_conn_hndl || thd->infinidb_vtable.vtable_state == THD::INFINIDB_CREATE_VTABLE) + if ( thd->infinidb_vtable.vtable_state == THD::INFINIDB_CREATE_VTABLE) + { + ci->stats.reset(); // reset query stats + ci->stats.setStartTime(); + ci->stats.fUser = thd->main_security_ctx.user; + + if (thd->main_security_ctx.host) + ci->stats.fHost = thd->main_security_ctx.host; + else if (thd->main_security_ctx.host_or_ip) + ci->stats.fHost = thd->main_security_ctx.host_or_ip; + else + ci->stats.fHost = "unknown"; + + try + { + ci->stats.userPriority(ci->stats.fHost, ci->stats.fUser); + } + catch (std::exception& e) + { + string msg = string("Columnstore User Priority - ") + e.what(); + ci->warningMsg = msg; + } + + // if the previous query has error, re-establish the connection + if (ci->queryState != 0) + { + sm::sm_cleanup(ci->cal_conn_hndl); + ci->cal_conn_hndl = 0; + } + } + + sm::sm_init(sessionID, &ci->cal_conn_hndl, localQuery); + idbassert(ci->cal_conn_hndl != 0); + ci->cal_conn_hndl->csc = csc; + idbassert(ci->cal_conn_hndl->exeMgr != 0); + + try + { + ci->cal_conn_hndl->connect(); + } + catch (...) + { + setError(thd, ER_INTERNAL_ERROR, IDBErrorInfo::instance()->errorMsg(ERR_LOST_CONN_EXEMGR)); + CalpontSystemCatalog::removeCalpontSystemCatalog(sessionID); + goto error; + } + + hndl = ci->cal_conn_hndl; + + if (thd->infinidb_vtable.vtable_state != THD::INFINIDB_SELECT_VTABLE) + { + //CalpontSelectExecutionPlan csep; + if (!csep) + csep.reset(new CalpontSelectExecutionPlan()); + + SessionManager sm; + BRM::TxnID txnID; + txnID = sm.getTxnID(sessionID); + + if (!txnID.valid) + { + txnID.id = 0; + txnID.valid = true; + } + + QueryContext verID; + verID = sm.verID(); + + csep->txnID(txnID.id); + csep->verID(verID); + csep->sessionID(sessionID); + + if (thd->db) + csep->schemaName(thd->db); + + csep->traceFlags(ci->traceFlags); + + if (thd->infinidb_vtable.isInsertSelect) + csep->queryType(CalpontSelectExecutionPlan::INSERT_SELECT); + + //get plan + int status = cp_get_plan(thd, csep); + + //if (cp_get_plan(thd, csep) != 0) + if (status > 0) + goto internal_error; + else if (status < 0) + return 0; + + // @bug 2547. don't need to send the plan if it's impossible where for all unions. + if (thd->infinidb_vtable.impossibleWhereOnUnion) + return 0; + + string query; + query.assign(thd->infinidb_vtable.original_query.ptr(), + thd->infinidb_vtable.original_query.length()); + csep->data(query); + + try + { + csep->priority( ci->stats.userPriority(ci->stats.fHost, ci->stats.fUser)); + } + catch (std::exception& e) + { + string msg = string("Columnstore User Priority - ") + e.what(); + push_warning(thd, Sql_condition::WARN_LEVEL_WARN, 9999, msg.c_str()); + } + +#ifdef PLAN_HEX_FILE + // plan serialization + ifstream ifs("/tmp/li1-plan.hex"); + ByteStream bs1; + ifs >> bs1; + ifs.close(); + csep->unserialize(bs1); +#endif + + if (ci->traceFlags & 1) + { + cerr << "---------------- EXECUTION PLAN ----------------" << endl; + cerr << *csep << endl ; + cerr << "-------------- EXECUTION PLAN END --------------\n" << endl; + } + else + { + IDEBUG( cout << "---------------- EXECUTION PLAN ----------------" << endl ); + IDEBUG( cerr << *csep << endl ); + IDEBUG( cout << "-------------- EXECUTION PLAN END --------------\n" << endl ); + } + } + }// end of execution plan generation + + if (thd->infinidb_vtable.vtable_state != THD::INFINIDB_SELECT_VTABLE) + { + ByteStream msg; + ByteStream emsgBs; + + while (true) + { + try + { + ByteStream::quadbyte qb = 4; + msg << qb; + hndl->exeMgr->write(msg); + msg.restart(); + csep->rmParms(rmParms); + + //send plan + csep->serialize(msg); + hndl->exeMgr->write(msg); + + //get ExeMgr status back to indicate a vtable joblist success or not + msg.restart(); + emsgBs.restart(); + msg = hndl->exeMgr->read(); + emsgBs = hndl->exeMgr->read(); + string emsg; + + if (msg.length() == 0 || emsgBs.length() == 0) + { + emsg = "Lost connection to ExeMgr. Please contact your administrator"; + setError(thd, ER_INTERNAL_ERROR, emsg); + return ER_INTERNAL_ERROR; + } + + string emsgStr; + emsgBs >> emsgStr; + bool err = false; + + if (msg.length() == 4) + { + msg >> qb; + + if (qb != 0) + { + err = true; + // for makejoblist error, stats contains only error code and insert from here + // because table fetch is not started + ci->stats.setEndTime(); + ci->stats.fQuery = csep->data(); + ci->stats.fQueryType = csep->queryType(); + ci->stats.fErrorNo = qb; + + try + { + ci->stats.insert(); + } + catch (std::exception& e) + { + string msg = string("Columnstore Query Stats - ") + e.what(); + push_warning(thd, Sql_condition::WARN_LEVEL_WARN, 9999, msg.c_str()); + } + } + } + else + { + err = true; + } + + if (err) + { + setError(thd, ER_INTERNAL_ERROR, emsgStr); + return ER_INTERNAL_ERROR; + } + + rmParms.clear(); + + if (thd->infinidb_vtable.vtable_state == THD::INFINIDB_DISABLE_VTABLE) + { + ci->tableMap[table] = ti; + } + else + { + ci->queryState = 1; + } + + break; + } + catch (...) + { + sm::sm_cleanup(hndl); + hndl = 0; + + sm::sm_init(sessionID, &hndl, localQuery); + idbassert(hndl != 0); + hndl->csc = csc; + + if (thd->infinidb_vtable.vtable_state == THD::INFINIDB_DISABLE_VTABLE) + ti.conn_hndl = hndl; + else + ci->cal_conn_hndl = hndl; + + try + { + hndl->connect(); + } + catch (...) + { + setError(thd, ER_INTERNAL_ERROR, IDBErrorInfo::instance()->errorMsg(ERR_LOST_CONN_EXEMGR)); + CalpontSystemCatalog::removeCalpontSystemCatalog(sessionID); + goto error; + } + + msg.restart(); + } + } + } + + // set query state to be in_process. Sometimes mysql calls rnd_init multiple + // times, this makes sure plan only being generated and sent once. It will be + // reset when query finishes in sm::end_query + thd->infinidb_vtable.isNewQuery = false; + + // common path for both vtable select phase and table mode -- open scan handle + ti = ci->tableMap[table]; + ti.msTablePtr = table; + + if ((thd->infinidb_vtable.vtable_state == THD::INFINIDB_SELECT_VTABLE) || + (thd->infinidb_vtable.vtable_state == THD::INFINIDB_DISABLE_VTABLE) || + (thd->infinidb_vtable.vtable_state == THD::INFINIDB_REDO_QUERY)) + { + if (ti.tpl_ctx == 0) + { + ti.tpl_ctx = new sm::cpsm_tplh_t(); + ti.tpl_scan_ctx = sm::sp_cpsm_tplsch_t(new sm::cpsm_tplsch_t()); + } + + // make sure rowgroup is null so the new meta data can be taken. This is for some case mysql + // call rnd_init for a table more than once. + ti.tpl_scan_ctx->rowGroup = NULL; + + try + { + tableid = execplan::IDB_VTABLE_ID; + } + catch (...) + { + string emsg = "No table ID found for table " + string(table->s->table_name.str); + setError(thd, ER_INTERNAL_ERROR, emsg); + CalpontSystemCatalog::removeCalpontSystemCatalog(sessionID); + goto internal_error; + } + + try + { + sm::tpl_open(tableid, ti.tpl_ctx, hndl); + sm::tpl_scan_open(tableid, ti.tpl_scan_ctx, hndl); + } + catch (std::exception& e) + { + string emsg = "table can not be opened: " + string(e.what()); + setError(thd, ER_INTERNAL_ERROR, emsg); + CalpontSystemCatalog::removeCalpontSystemCatalog(sessionID); + goto internal_error; + } + catch (...) + { + string emsg = "table can not be opened"; + setError(thd, ER_INTERNAL_ERROR, emsg); + CalpontSystemCatalog::removeCalpontSystemCatalog(sessionID); + goto internal_error; + } + + ti.tpl_scan_ctx->traceFlags = ci->traceFlags; + + if ((ti.tpl_scan_ctx->ctp).size() == 0) + { + uint32_t num_attr = table->s->fields; + + for (uint32_t i = 0; i < num_attr; i++) + { + CalpontSystemCatalog::ColType ctype; + ti.tpl_scan_ctx->ctp.push_back(ctype); + } + + // populate coltypes here for table mode because tableband gives treeoid for dictionary column + if (thd->infinidb_vtable.vtable_state == THD::INFINIDB_DISABLE_VTABLE) + { + CalpontSystemCatalog::RIDList oidlist = csc->columnRIDs(make_table(table->s->db.str, table->s->table_name.str), true); + + if (oidlist.size() != num_attr) + { + string emsg = "Size mismatch probably caused by front end out of sync"; + setError(thd, ER_INTERNAL_ERROR, emsg); + CalpontSystemCatalog::removeCalpontSystemCatalog(sessionID); + goto internal_error; + } + + for (unsigned int j = 0; j < oidlist.size(); j++) + { + CalpontSystemCatalog::ColType ctype = csc->colType(oidlist[j].objnum); + ti.tpl_scan_ctx->ctp[ctype.colPosition] = ctype; + ti.tpl_scan_ctx->ctp[ctype.colPosition].colPosition = -1; + } + } + } + } + + ci->tableMap[table] = ti; + return 0; + +error: + + if (ci->cal_conn_hndl) + { + sm::sm_cleanup(ci->cal_conn_hndl); + ci->cal_conn_hndl = 0; + } + + // do we need to close all connection handle of the table map? + return ER_INTERNAL_ERROR; + +internal_error: + + if (ci->cal_conn_hndl) + { + sm::sm_cleanup(ci->cal_conn_hndl); + ci->cal_conn_hndl = 0; + } + + return ER_INTERNAL_ERROR; + //return(0); } /*@brief ha_calpont_impl_group_by_next - Return result set for MariaDB group_by @@ -4984,14 +5991,14 @@ int ha_calpont_impl_group_by_init(Query* query, TABLE* table) * DESCRIPTION: * Return a result record for each group_by_handler::next_row() call. * PARAMETERS: - * query - Query structure, that describes the pushdown query. + * group_hand - group by handler, that preserves initial table and items lists. . * table - TABLE pointer The table to save the result set in. * RETURN: * 0 if success * HA_ERR_END_OF_FILE if the record set has come to an end * others if something went wrong whilst getting the result set ***********************************************************/ -int ha_calpont_impl_group_by_next(Query* query, TABLE* table) +int ha_calpont_impl_group_by_next(ha_calpont_group_by_handler* group_hand, TABLE* table) { //if (!first_row) //return(HA_ERR_END_OF_FILE); @@ -5088,7 +6095,7 @@ int ha_calpont_impl_group_by_next(Query* query, TABLE* table) try { - //rc = fetchNextRow(buf, ti, ci); + rc = fetchNextRowGrHand(ti, ci); rc = 0; } catch (std::exception& e) @@ -5124,11 +6131,10 @@ int ha_calpont_impl_group_by_next(Query* query, TABLE* table) return rc; } -int ha_calpont_impl_group_by_end(Query* query, TABLE* table) +int ha_calpont_impl_group_by_end(ha_calpont_group_by_handler* group_hand, TABLE* table) { return 0; } - // vim:sw=4 ts=4: diff --git a/dbcon/mysql/ha_calpont_impl.h b/dbcon/mysql/ha_calpont_impl.h index f6bea072c..7f7c43aae 100644 --- a/dbcon/mysql/ha_calpont_impl.h +++ b/dbcon/mysql/ha_calpont_impl.h @@ -48,15 +48,16 @@ extern int ha_calpont_impl_external_lock(THD* thd, TABLE* table, int lock_type); extern int ha_calpont_impl_update_row(); extern int ha_calpont_impl_delete_row(); extern int ha_calpont_impl_rnd_pos(uchar* buf, uchar* pos); -extern int ha_calpont_impl_group_by_init(Query* query, TABLE* table); -extern int ha_calpont_impl_group_by_next(Query* query, TABLE* table); -extern int ha_calpont_impl_group_by_end(Query* query, TABLE* table); +extern int ha_calpont_impl_group_by_init(ha_calpont_group_by_handler* group_hand, TABLE* table); +extern int ha_calpont_impl_group_by_next(ha_calpont_group_by_handler* group_hand, TABLE* table); +extern int ha_calpont_impl_group_by_end(ha_calpont_group_by_handler* group_hand, TABLE* table); #endif #ifdef NEED_CALPONT_INTERFACE #include "ha_calpont_impl_if.h" #include "calpontsystemcatalog.h" +#include "ha_calpont.h" extern int ha_calpont_impl_rename_table_(const char* from, const char* to, cal_impl_if::cal_connection_info& ci); extern int ha_calpont_impl_write_row_(uchar* buf, TABLE* table, cal_impl_if::cal_connection_info& ci, ha_rows& rowsInserted); extern int ha_calpont_impl_write_batch_row_(uchar* buf, TABLE* table, cal_impl_if::cal_connection_info& ci); @@ -73,6 +74,9 @@ extern std::string ha_calpont_impl_droppartition_ (execplan::CalpontSystemCatal extern std::string ha_calpont_impl_viewtablelock( cal_impl_if::cal_connection_info& ci, execplan::CalpontSystemCatalog::TableName& tablename); extern std::string ha_calpont_impl_cleartablelock( cal_impl_if::cal_connection_info& ci, uint64_t tableLockID); +extern int ha_calpont_impl_group_by_init(ha_calpont_group_by_handler* group_hand, TABLE* table); +extern int ha_calpont_impl_group_by_next(ha_calpont_group_by_handler* group_hand, TABLE* table); +extern int ha_calpont_impl_group_by_end(ha_calpont_group_by_handler* group_hand, TABLE* table); #endif #endif diff --git a/dbcon/mysql/ha_calpont_impl_if.h b/dbcon/mysql/ha_calpont_impl_if.h index 709e1b051..71fd0b35f 100644 --- a/dbcon/mysql/ha_calpont_impl_if.h +++ b/dbcon/mysql/ha_calpont_impl_if.h @@ -178,7 +178,8 @@ struct cal_table_info msTablePtr(0), conn_hndl(0), condInfo(0), - moreRows(false) + moreRows(false), + groupByFields(0) { } ~cal_table_info() {} sm::cpsm_tplh_t* tpl_ctx; @@ -189,6 +190,7 @@ struct cal_table_info gp_walk_info* condInfo; execplan::SCSEP csep; bool moreRows; //are there more rows to consume (b/c of limit) + List *groupByFields; // MCOL-1052 For CSEP generation }; typedef std::tr1::unordered_map CalTableMap; @@ -297,6 +299,7 @@ const std::string infinidb_err_msg = "\nThe query includes syntax that is not su int cp_get_plan(THD* thd, execplan::SCSEP& csep); int cp_get_table_plan(THD* thd, execplan::SCSEP& csep, cal_impl_if::cal_table_info& ti); +int cp_get_group_plan(THD* thd, execplan::SCSEP& csep, cal_impl_if::cal_table_info& ti); int getSelectPlan(gp_walk_info& gwi, SELECT_LEX& select_lex, execplan::SCSEP& csep, bool isUnion = false); void setError(THD* thd, uint32_t errcode, const std::string errmsg, gp_walk_info* gwi); void setError(THD* thd, uint32_t errcode, const std::string errmsg); From a47f16054d20a3cc0fcbe75e0580fc73eb62bcd8 Mon Sep 17 00:00:00 2001 From: Roman Nozdrin Date: Fri, 30 Mar 2018 19:07:33 +0300 Subject: [PATCH 06/57] MCOL-1052 Use existed getSelectPlan as a basis for group_by_handler plan generation. --- dbcon/mysql/CMakeLists.txt | 3 + dbcon/mysql/ha_calpont.h | 25 +- dbcon/mysql/ha_calpont_execplan.cpp | 2497 ++++++++++++++++++++++++++- dbcon/mysql/ha_calpont_impl.cpp | 14 +- dbcon/mysql/ha_calpont_impl_if.h | 28 +- 5 files changed, 2522 insertions(+), 45 deletions(-) diff --git a/dbcon/mysql/CMakeLists.txt b/dbcon/mysql/CMakeLists.txt index 104e6bf6e..7721aed80 100644 --- a/dbcon/mysql/CMakeLists.txt +++ b/dbcon/mysql/CMakeLists.txt @@ -20,6 +20,9 @@ SET ( libcalmysql_SRCS ha_pseudocolumn.cpp) add_definitions(-DMYSQL_DYNAMIC_PLUGIN) +add_definitions(-DDEBUG_WALK_COND) +add_definitions(-DINFINIDB_DEBUG) +#add_definitions(-DOUTER_JOIN_DEBUG) set_source_files_properties(ha_calpont.cpp PROPERTIES COMPILE_FLAGS "-fno-rtti -fno-implicit-templates") diff --git a/dbcon/mysql/ha_calpont.h b/dbcon/mysql/ha_calpont.h index c3f2573e2..f15078fad 100644 --- a/dbcon/mysql/ha_calpont.h +++ b/dbcon/mysql/ha_calpont.h @@ -254,16 +254,29 @@ class ha_calpont_group_by_handler: public group_by_handler // ha_calpont_group_by_handler(THD *thd_arg, List *fields_arg, // TABLE_LIST *table_list_arg, Query *query) ha_calpont_group_by_handler(THD *thd_arg, Query *query) - : group_by_handler(thd_arg, calpont_hton), fields(query->select), - table_list(query->from), query(query) {} - ~ha_calpont_group_by_handler() {} + : group_by_handler(thd_arg, calpont_hton), + select(query->select), + table_list(query->from), + distinct(query->distinct), + where(query->where), + group_by(query->group_by), + order_by(query->order_by), + having(query->having), + query(query) + { } + ~ha_calpont_group_by_handler() { } int init_scan(); int next_row(); int end_scan(); - List *fields; + List *select; TABLE_LIST *table_list; - bool first_row; - Query *query; + bool distinct; + Item *where; + ORDER *group_by; + ORDER *order_by; + Item *having; + bool first_row; // useless by now + Query *query; // useless by now }; #endif //HA_CALPONT_H__ diff --git a/dbcon/mysql/ha_calpont_execplan.cpp b/dbcon/mysql/ha_calpont_execplan.cpp index 8c0934f4c..5bed8e88c 100644 --- a/dbcon/mysql/ha_calpont_execplan.cpp +++ b/dbcon/mysql/ha_calpont_execplan.cpp @@ -7937,6 +7937,33 @@ int cp_get_table_plan(THD* thd, SCSEP& csep, cal_table_info& ti) return 0; } +int cp_get_group_plan(THD* thd, SCSEP& csep, cal_table_info& ti, cal_group_info& gi ) +{ + LEX* lex = thd->lex; + idbassert(lex != 0); + + SELECT_LEX select_lex = lex->select_lex; + gp_walk_info gwi; + gwi.thd = thd; + gwi.thd->infinidb_vtable.vtable_state = THD::INFINIDB_CREATE_VTABLE; + int status = getGroupPlan(gwi, select_lex, csep, gi); + + if (status > 0) + return ER_INTERNAL_ERROR; + else if (status < 0) + return status; + + cerr << "---------------- cp_get_group_plan EXECUTION PLAN ----------------" << endl; + cerr << *csep << endl ; + cerr << "-------------- EXECUTION PLAN END --------------\n" << endl; + + // Derived table projection and filter optimization. + //derivedTableOptimization(csep); + + return 0; +} + +/* int cp_get_group_plan(THD* thd, SCSEP& csep, cal_table_info& ti ) { gp_walk_info* gwi = ti.condInfo; @@ -7952,6 +7979,7 @@ int cp_get_group_plan(THD* thd, SCSEP& csep, cal_table_info& ti ) uint32_t sessionID = csep->sessionID(); gwi->sessionid = sessionID; TABLE* table = ti.msTablePtr; + TABLE_LIST* table_list = ti.groupByTables; boost::shared_ptr csc = CalpontSystemCatalog::makeCalpontSystemCatalog(sessionID); csc->identity(CalpontSystemCatalog::FE); @@ -7961,10 +7989,11 @@ int cp_get_group_plan(THD* thd, SCSEP& csep, cal_table_info& ti ) Field** f_ptr, *field; gwi->columnMap.clear(); - const CalpontSystemCatalog::TableAliasName tblAliasName= + CalpontSystemCatalog::TableAliasName tan= make_aliastable(table->s->db.str, table->s->table_name.str, table->alias.c_ptr()); - gwi->tbList.push_back(tblAliasName); + gwi->tbList.push_back(tan); + gwi->tableMap[tan] = make_pair(0, table_list); while ((item = it++)) { @@ -7976,38 +8005,59 @@ int cp_get_group_plan(THD* thd, SCSEP& csep, cal_table_info& ti ) assert (rc); boost::shared_ptr sprc(rc); gwi->returnedCols.push_back(sprc); - arg0=((Item_sum*) item)->get_arg(0); - field=((Item_field*) arg0)->field; - gwi->columnMap.insert(CalpontSelectExecutionPlan::ColumnMap::value_type(string(field->field_name), sprc)); - //arg0=((Item_sum*) item)->get_arg(0); - //field=((Item_field*) arg0)->field; - } - -/* - for (f_ptr = table->field ; (field = *f_ptr) ; f_ptr++) - { - if (bitmap_is_set(read_set, field->field_index)) - { - SimpleColumn* sc = new SimpleColumn(table->s->db.str, table->s->table_name.str, field->field_name, sessionID); - string alias(table->alias.c_ptr()); - sc->tableAlias(lower(alias)); - assert (sc); - boost::shared_ptr spsc(sc); - gwi->returnedCols.push_back(spsc); - gwi->columnMap.insert(CalpontSelectExecutionPlan::ColumnMap::value_type(string(field->field_name), spsc)); - } - } -*/ - if (gwi->columnMap.empty()) - { - CalpontSystemCatalog::TableName tn = make_table(table->s->db.str, table->s->table_name.str); - CalpontSystemCatalog::TableAliasName tan = make_aliastable(table->s->db.str, table->s->table_name.str, table->alias.c_ptr()); - SimpleColumn* sc = getSmallestColumn(csc, tn, tan, table, *gwi); - SRCP srcp(sc); - gwi->columnMap.insert(CalpontSelectExecutionPlan::ColumnMap::value_type(sc->columnName(), srcp)); - gwi->returnedCols.push_back(srcp); } + // TO DO add table into tableMap + // TO DO add the column only if there is no any for a table. + TableMap::iterator tb_iter = gwi->tableMap.begin(); + try + { + for (; tb_iter != gwi->tableMap.end(); tb_iter++) + { + if ((*tb_iter).second.first == 1) continue; + + CalpontSystemCatalog::TableAliasName tan = (*tb_iter).first; + CalpontSystemCatalog::TableName tn = make_table((*tb_iter).first.schema, (*tb_iter).first.table); + SimpleColumn* sc = getSmallestColumn(csc, tn, tan, (*tb_iter).second.second->table, *gwi); + SRCP srcp(sc); + gwi->columnMap.insert(CalpontSelectExecutionPlan::ColumnMap::value_type(sc->columnName(), srcp)); + (*tb_iter).second.first = 1; + } + } + catch (runtime_error& e) + { + setError(gwi->thd, ER_INTERNAL_ERROR, e.what(), *gwi); + CalpontSystemCatalog::removeCalpontSystemCatalog(sessionID); + return ER_INTERNAL_ERROR; + } + catch (...) + { + string emsg = IDBErrorInfo::instance()->errorMsg(ERR_LOST_CONN_EXEMGR); + setError(gwi->thd, ER_INTERNAL_ERROR, emsg, *gwi); + CalpontSystemCatalog::removeCalpontSystemCatalog(sessionID); + return ER_INTERNAL_ERROR; + } + + //if (gwi->columnMap.empty()) + //{ + // CalpontSystemCatalog::TableName tn = make_table(table->s->db.str, table->s->table_name.str); + // CalpontSystemCatalog::TableAliasName tan = make_aliastable(table->s->db.str, table->s->table_name.str, table->alias.c_ptr()); + // SimpleColumn* sc = getSmallestColumn(csc, tn, tan, table, *gwi); + // SRCP srcp(sc); + // gwi->columnMap.insert(CalpontSelectExecutionPlan::ColumnMap::value_type(sc->columnName(), srcp)); + // gwi->returnedCols.push_back(srcp); + //} + + + CalpontSystemCatalog::TableName tn = make_table(table->s->db.str, table->s->table_name.str); + //SimpleColumn* minSc = getSmallestColumn(csc, tn, tan, table, *gwi); + SRCP minSc; + minSc = gwi->returnedCols[0]; + std::vector::iterator coliter; + for (coliter = gwi->count_asterisk_list.begin(); coliter != gwi->count_asterisk_list.end(); ++coliter) + { + (*coliter)->functionParms(minSc); + } // get filter if (ti.condInfo) @@ -8051,7 +8101,7 @@ int cp_get_group_plan(THD* thd, SCSEP& csep, cal_table_info& ti ) csep->returnedCols(gwi->returnedCols); csep->columnMap(gwi->columnMap); CalpontSelectExecutionPlan::TableList tblist; - tblist.push_back(tblAliasName); + tblist.push_back(tan); csep->tableList(tblist); // @bug 3321. Set max number of blocks in a dictionary file to be scanned for filtering @@ -8059,5 +8109,2386 @@ int cp_get_group_plan(THD* thd, SCSEP& csep, cal_table_info& ti ) return 0; } +*/ + +int getGroupPlan(gp_walk_info& gwi, SELECT_LEX& select_lex, SCSEP& csep, cal_group_info& gi, bool isUnion) +{ +#ifdef DEBUG_WALK_COND + cerr << "getGroupPlan()" << endl; +#endif + + // by pass the derived table resolve phase of mysql + /*if (!(((gwi.thd->lex)->sql_command == SQLCOM_UPDATE ) || + ((gwi.thd->lex)->sql_command == SQLCOM_DELETE ) || + ((gwi.thd->lex)->sql_command == SQLCOM_UPDATE_MULTI ) || + ((gwi.thd->lex)->sql_command == SQLCOM_DELETE_MULTI ) ) && gwi.thd->derived_tables_processing) + { + gwi.thd->infinidb_vtable.isUnion = false; + return -1; + } + */ + // rollup is currently not supported + if (select_lex.olap == ROLLUP_TYPE) + { + gwi.fatalParseError = true; + gwi.parseErrorText = IDBErrorInfo::instance()->errorMsg(ERR_ROLLUP_NOT_SUPPORT); + setError(gwi.thd, ER_CHECK_NOT_IMPLEMENTED, gwi.parseErrorText, gwi); + return ER_CHECK_NOT_IMPLEMENTED; + } + + gwi.internalDecimalScale = (gwi.thd->variables.infinidb_use_decimal_scale ? gwi.thd->variables.infinidb_decimal_scale : -1); + gwi.subSelectType = csep->subType(); + + JOIN* join = select_lex.join; + Item_cond* icp = 0; + /* MCOL-1052 + if (join != 0) + icp = reinterpret_cast(join->conds); + + // if icp is null, try to find the where clause other where + if (!join && gwi.thd->lex->derived_tables) + { + if (select_lex.prep_where) + icp = (Item_cond*)(select_lex.prep_where); + else if (select_lex.where) + icp = (Item_cond*)(select_lex.where); + } + else if (!join && ( ((gwi.thd->lex)->sql_command == SQLCOM_UPDATE ) || + ((gwi.thd->lex)->sql_command == SQLCOM_DELETE ) || + ((gwi.thd->lex)->sql_command == SQLCOM_UPDATE_MULTI ) || + ((gwi.thd->lex)->sql_command == SQLCOM_DELETE_MULTI ))) + { + icp = reinterpret_cast(select_lex.where); + } + */ + // MCOL-1052 + if ( gi.groupByWhere ) + icp = reinterpret_cast(gi.groupByWhere); + + uint32_t sessionID = csep->sessionID(); + gwi.sessionid = sessionID; + boost::shared_ptr csc = CalpontSystemCatalog::makeCalpontSystemCatalog(sessionID); + csc->identity(CalpontSystemCatalog::FE); + gwi.csc = csc; + + // @bug 2123. Override large table estimate if infinidb_ordered hint was used. + // @bug 2404. Always override if the infinidb_ordered_only variable is turned on. + if (gwi.thd->infinidb_vtable.override_largeside_estimate || gwi.thd->variables.infinidb_ordered_only) + csep->overrideLargeSideEstimate(true); + + // @bug 5741. Set a flag when in Local PM only query mode + csep->localQuery(gwi.thd->variables.infinidb_local_query); + + // @bug 3321. Set max number of blocks in a dictionary file to be scanned for filtering + csep->stringScanThreshold(gwi.thd->variables.infinidb_string_scan_threshold); + + csep->stringTableThreshold(gwi.thd->variables.infinidb_stringtable_threshold); + + csep->djsSmallSideLimit(gwi.thd->variables.infinidb_diskjoin_smallsidelimit * 1024ULL * 1024); + csep->djsLargeSideLimit(gwi.thd->variables.infinidb_diskjoin_largesidelimit * 1024ULL * 1024); + csep->djsPartitionSize(gwi.thd->variables.infinidb_diskjoin_bucketsize * 1024ULL * 1024); + + if (gwi.thd->variables.infinidb_um_mem_limit == 0) + csep->umMemLimit(numeric_limits::max()); + else + csep->umMemLimit(gwi.thd->variables.infinidb_um_mem_limit * 1024ULL * 1024); + + // populate table map and trigger syscolumn cache for all the tables (@bug 1637). + // all tables on FROM list must have at least one col in colmap + TABLE_LIST* table_ptr = gi.groupByTables; + CalpontSelectExecutionPlan::SelectList derivedTbList; + +// DEBUG +#ifdef DEBUG_WALK_COND + List_iterator sj_list_it(select_lex.sj_nests); + TABLE_LIST* sj_nest; + + while ((sj_nest = sj_list_it++)) + { + cerr << sj_nest->db << "." << sj_nest->table_name << endl; + } + +#endif + + // @bug 1796. Remember table order on the FROM list. + gwi.clauseType = FROM; + + try + { + for (; table_ptr; table_ptr = table_ptr->next_local) + { + // mysql put vtable here for from sub. we ignore it + //if (string(table_ptr->table_name).find("$vtable") != string::npos) + // continue; + + // Until we handle recursive cte: + // Checking here ensures we catch all with clauses in the query. + if (table_ptr->is_recursive_with_table()) + { + gwi.fatalParseError = true; + gwi.parseErrorText = "Recursive CTE"; + setError(gwi.thd, ER_CHECK_NOT_IMPLEMENTED, gwi.parseErrorText, gwi); + return ER_CHECK_NOT_IMPLEMENTED; + } + + string viewName = getViewName(table_ptr); + + // @todo process from subquery + if (table_ptr->derived) + { + String str; + (table_ptr->derived->first_select())->print(gwi.thd, &str, QT_INFINIDB_DERIVED); + + SELECT_LEX* select_cursor = table_ptr->derived->first_select(); + FromSubQuery fromSub(gwi, select_cursor); + string alias(table_ptr->alias); + fromSub.alias(lower(alias)); + + CalpontSystemCatalog::TableAliasName tn = make_aliasview("", "", alias, viewName); + // @bug 3852. check return execplan + SCSEP plan = fromSub.transform(); + + if (!plan) + { + setError(gwi.thd, ER_INTERNAL_ERROR, fromSub.gwip().parseErrorText, gwi); + CalpontSystemCatalog::removeCalpontSystemCatalog(sessionID); + return ER_INTERNAL_ERROR; + } + + gwi.derivedTbList.push_back(plan); + gwi.tbList.push_back(tn); + CalpontSystemCatalog::TableAliasName tan = make_aliastable("", alias, alias); + gwi.tableMap[tan] = make_pair(0, table_ptr); + gwi.thd->infinidb_vtable.isUnion = true; //by-pass the 2nd pass of rnd_init + } + else if (table_ptr->view) + { + View* view = new View(table_ptr->view->select_lex, &gwi); + CalpontSystemCatalog::TableAliasName tn = make_aliastable(table_ptr->db, table_ptr->table_name, table_ptr->alias); + view->viewName(tn); + gwi.viewList.push_back(view); + view->transform(); + } + else + { + // check foreign engine tables + bool infiniDB = (table_ptr->table ? isInfiniDB(table_ptr->table) : true); + + // trigger system catalog cache + if (infiniDB) + csc->columnRIDs(make_table(table_ptr->db, table_ptr->table_name), true); + + string table_name = table_ptr->table_name; + + // @bug5523 + if (table_ptr->db && strcmp(table_ptr->db, "information_schema") == 0) + table_name = (table_ptr->schema_table_name ? table_ptr->schema_table_name : table_ptr->alias); + + CalpontSystemCatalog::TableAliasName tn = make_aliasview(table_ptr->db, table_name, table_ptr->alias, viewName, infiniDB); + gwi.tbList.push_back(tn); + CalpontSystemCatalog::TableAliasName tan = make_aliastable(table_ptr->db, table_name, table_ptr->alias, infiniDB); + gwi.tableMap[tan] = make_pair(0, table_ptr); +#ifdef DEBUG_WALK_COND + cerr << tn << endl; +#endif + } + } + + if (gwi.fatalParseError) + { + setError(gwi.thd, ER_INTERNAL_ERROR, gwi.parseErrorText, gwi); + return ER_INTERNAL_ERROR; + } + } + catch (IDBExcept& ie) + { + setError(gwi.thd, ER_INTERNAL_ERROR, ie.what(), gwi); + CalpontSystemCatalog::removeCalpontSystemCatalog(sessionID); + // @bug 3852. set error status for gwi. + gwi.fatalParseError = true; + gwi.parseErrorText = ie.what(); + return ER_INTERNAL_ERROR; + } + catch (...) + { + string emsg = IDBErrorInfo::instance()->errorMsg(ERR_LOST_CONN_EXEMGR); + // @bug3852 set error status for gwi. + gwi.fatalParseError = true; + gwi.parseErrorText = emsg; + setError(gwi.thd, ER_INTERNAL_ERROR, emsg, gwi); + CalpontSystemCatalog::removeCalpontSystemCatalog(sessionID); + return ER_INTERNAL_ERROR; + } + + csep->tableList(gwi.tbList); + + bool unionSel = false; + + if (!isUnion && select_lex.master_unit()->is_union()) + { + gwi.thd->infinidb_vtable.isUnion = true; + CalpontSelectExecutionPlan::SelectList unionVec; + SELECT_LEX* select_cursor = select_lex.master_unit()->first_select(); + unionSel = true; + uint8_t distUnionNum = 0; + + for (SELECT_LEX* sl = select_cursor; sl; sl = sl->next_select()) + { + SCSEP plan(new CalpontSelectExecutionPlan()); + plan->txnID(csep->txnID()); + plan->verID(csep->verID()); + plan->sessionID(csep->sessionID()); + plan->traceFlags(csep->traceFlags()); + plan->data(csep->data()); + + // @bug 3853. When one or more sides or union queries contain derived tables, + // sl->join->zero_result_cause is not trustable. Since we've already handled + // constant filter now (0/1), we can relax the following checking. + // @bug 2547. ignore union unit of zero result set case +// if (sl->join) +// { +// sl->join->optimize(); + // @bug 3067. not clear MySQL's behavior. when in subquery, this variable + // is not trustable. +// if (sl->join->zero_result_cause && !gwi.subQuery) +// continue; +// } + + // gwi for the union unit + gp_walk_info union_gwi; + union_gwi.thd = gwi.thd; + uint32_t err = 0; + + if ((err = getSelectPlan(union_gwi, *sl, plan, unionSel)) != 0) + return err; + + unionVec.push_back(SCEP(plan)); + + // distinct union num + if (sl == select_lex.master_unit()->union_distinct) + distUnionNum = unionVec.size(); + + /*#ifdef DEBUG_WALK_COND + IDEBUG( cout << ">>>> UNION DEBUG" << endl ); + JOIN* join = sl->join; + Item_cond* icp = 0; + if (join != 0) + icp = reinterpret_cast(join->conds); + if (icp) + icp->traverse_cond(debug_walk, &gwi, Item::POSTFIX); + IDEBUG ( cout << *plan << endl ); + IDEBUG ( cout << "<<<unionVec(unionVec); + csep->distinctUnionNum(distUnionNum); + + if (unionVec.empty()) + gwi.thd->infinidb_vtable.impossibleWhereOnUnion = true; + } + + gwi.clauseType = WHERE; + + if (icp) + { + // MCOL-1052 The condition could be useless. +// MariaDB bug 624 - without the fix_fields call, delete with join may error with "No query step". +//#if MYSQL_VERSION_ID < 50172 + //@bug 3039. fix fields for constants + if (!icp->fixed) + { + icp->fix_fields(gwi.thd, (Item**)&icp); + } + +//#endif + gwi.fatalParseError = false; +#ifdef DEBUG_WALK_COND + cerr << "------------------ WHERE -----------------------" << endl; + icp->traverse_cond(debug_walk, &gwi, Item::POSTFIX); + cerr << "------------------------------------------------\n" << endl; +#endif + + icp->traverse_cond(gp_walk, &gwi, Item::POSTFIX); + + if (gwi.fatalParseError) + { + // if this is dervied table process phase, mysql may have not developed the plan + // completely. Do not error and eventually mysql will call JOIN::exec() again. + // related to bug 2922. Need to find a way to skip calling rnd_init for derived table + // processing. + if (gwi.thd->derived_tables_processing) + { + gwi.thd->infinidb_vtable.isUnion = false; + gwi.thd->infinidb_vtable.isUpdateWithDerive = true; + return -1; + } + + setError(gwi.thd, ER_INTERNAL_ERROR, gwi.parseErrorText, gwi); + return ER_INTERNAL_ERROR; + } + } + else if (join && join->zero_result_cause) + { + gwi.rcWorkStack.push(new ConstantColumn((int64_t)0, ConstantColumn::NUM)); + } + + + // ZZ - the followinig debug shows the structure of nested outer join. should + // use a recursive function. +#ifdef OUTER_JOIN_DEBUG + List* tables = &(select_lex.top_join_list); + List_iterator_fast ti(*tables); + //TABLE_LIST *inner; + //TABLE_LIST **table= (TABLE_LIST **)gwi.thd->alloc(sizeof(TABLE_LIST*) * tables->elements); + //for (TABLE_LIST **t= table + (tables->elements - 1); t >= table; t--) + // *t= ti++; + + //DBUG_ASSERT(tables->elements >= 1); + + //TABLE_LIST **end= table + tables->elements; + //for (TABLE_LIST **tbl= table; tbl < end; tbl++) + TABLE_LIST* curr; + + while ((curr = ti++)) + { + TABLE_LIST* curr = *tbl; + + if (curr->table_name) + cout << curr->table_name << " "; + else + cout << curr->alias << endl; + + if (curr->outer_join) + cout << " is inner table" << endl; + else if (curr->straight) + cout << "straight_join" << endl; + else + cout << "join" << endl; + + if (curr->nested_join) + { + List* inners = &(curr->nested_join->join_list); + List_iterator_fast li(*inners); + TABLE_LIST** inner = (TABLE_LIST**)gwi.thd->alloc(sizeof(TABLE_LIST*) * inners->elements); + + for (TABLE_LIST** t = inner + (inners->elements - 1); t >= inner; t--) + *t = li++; + + TABLE_LIST** end1 = inner + inners->elements; + + for (TABLE_LIST** tb = inner; tb < end1; tb++) + { + TABLE_LIST* curr1 = *tb; + cout << curr1->alias << endl; + + if (curr1->sj_on_expr) + { + curr1->sj_on_expr->traverse_cond(debug_walk, &gwi, Item::POSTFIX); + } + } + } + + if (curr->sj_on_expr) + { + curr->sj_on_expr->traverse_cond(debug_walk, &gwi, Item::POSTFIX); + } + } + +#endif + // MCOL-1052 + uint32_t failed = buildOuterJoin(gwi, select_lex); + + if (failed) return failed; + + // @bug5764. build outer join for view, make sure outerjoin filter is appended + // to the end of the filter list. + for (uint i = 0; i < gwi.viewList.size(); i++) + { + failed = gwi.viewList[i]->processOuterJoin(gwi); + + if (failed) + break; + } + + if (failed != 0) + return failed; + + ParseTree* filters = NULL; + ParseTree* ptp = NULL; + ParseTree* lhs = NULL; + + // @bug 2932. for "select * from region where r_name" case. if icp not null and + // ptWorkStack empty, the item is in rcWorkStack. + // MySQL 5.6 (MariaDB?). when icp is null and zero_result_cause is set, a constant 0 + // is pushed to rcWorkStack. + if (/*icp && */gwi.ptWorkStack.empty() && !gwi.rcWorkStack.empty()) + { + filters = new ParseTree(gwi.rcWorkStack.top()); + gwi.rcWorkStack.pop(); + } + + while (!gwi.ptWorkStack.empty()) + { + filters = gwi.ptWorkStack.top(); + gwi.ptWorkStack.pop(); + + if (gwi.ptWorkStack.empty()) + break; + + ptp = new ParseTree(new LogicOperator("and")); + //ptp->left(filters); + ptp->right(filters); + lhs = gwi.ptWorkStack.top(); + gwi.ptWorkStack.pop(); + //ptp->right(rhs); + ptp->left(lhs); + gwi.ptWorkStack.push(ptp); + } + + if (filters) + { + csep->filters(filters); + filters->drawTree("/tmp/filter1.dot"); + } + + gwi.clauseType = SELECT; +#ifdef DEBUG_WALK_COND + { + cerr << "------------------- SELECT --------------------" << endl; + List_iterator_fast it(*gi.groupByFields); + Item* item; + + while ((item = it++)) + { + debug_walk(item, 0); + } + + cout << "-----------------------------------------------\n" << endl; + } +#endif + + // populate returnedcolumnlist and columnmap + List_iterator_fast it(*gi.groupByFields); + Item* item; + vector funcFieldVec; + string sel_cols_in_create; + string sel_cols_in_select; + bool redo = false; + + // empty rcWorkStack and ptWorkStack. They should all be empty by now. + clearStacks(gwi); + + // indicate the starting pos of scalar returned column, because some join column + // has been inserted to the returned column list. + if (gwi.subQuery) + { + ScalarSub* scalar = dynamic_cast(gwi.subQuery); + + if (scalar) + scalar->returnedColPos(gwi.additionalRetCols.size()); + } + + CalpontSelectExecutionPlan::SelectList selectSubList; + + while ((item = it++)) + { + string itemAlias = (item->name ? item->name : ""); + + // @bug 5916. Need to keep checking until getting concret item in case + // of nested view. + while (item->type() == Item::REF_ITEM) + { + Item_ref* ref = (Item_ref*)item; + item = (*(ref->ref)); + } + + Item::Type itype = item->type(); + + switch (itype) + { + case Item::FIELD_ITEM: + { + Item_field* ifp = (Item_field*)item; + SimpleColumn* sc = NULL; + + if (ifp->field_name && string(ifp->field_name) == "*") + { + collectAllCols(gwi, ifp); + break; + } + + sc = buildSimpleColumn(ifp, gwi); + + if (sc) + { + boost::shared_ptr spsc(sc); + + if (sel_cols_in_create.length() != 0) + sel_cols_in_create += ", "; + + string fullname; + String str; + ifp->print(&str, QT_INFINIDB_NO_QUOTE); + fullname = str.c_ptr(); + + //sel_cols_in_create += fullname; + if (ifp->is_autogenerated_name) // no alias + { + sel_cols_in_create += fullname + " `" + escapeBackTick(str.c_ptr()) + "`"; + sc->alias(fullname); + } + else // alias + { + if (!itemAlias.empty()) + sc->alias(itemAlias); + + sel_cols_in_create += fullname + " `" + escapeBackTick(sc->alias().c_str()) + "`"; + } + + if (ifp->is_autogenerated_name) + gwi.selectCols.push_back("`" + escapeBackTick(fullname.c_str()) + "`" + " `" + + escapeBackTick(itemAlias.empty() ? ifp->name : itemAlias.c_str()) + "`"); + else + gwi.selectCols.push_back("`" + escapeBackTick((itemAlias.empty() ? ifp->name : itemAlias.c_str())) + "`"); + + gwi.returnedCols.push_back(spsc); + + gwi.columnMap.insert(CalpontSelectExecutionPlan::ColumnMap::value_type(string(ifp->field_name), spsc)); + TABLE_LIST* tmp = 0; + + if (ifp->cached_table) + tmp = ifp->cached_table; + + gwi.tableMap[make_aliastable(sc->schemaName(), sc->tableName(), sc->tableAlias(), sc->isInfiniDB())] = + make_pair(1, tmp); + } + else + { + setError(gwi.thd, ER_INTERNAL_ERROR, gwi.parseErrorText, gwi); + delete sc; + return ER_INTERNAL_ERROR; + } + + break; + } + + //aggregate column + case Item::SUM_FUNC_ITEM: + { + ReturnedColumn* ac = buildAggregateColumn(item, gwi); + + if (gwi.fatalParseError) + { + // e.g., non-support ref column + setError(gwi.thd, ER_CHECK_NOT_IMPLEMENTED, gwi.parseErrorText, gwi); + delete ac; + return ER_CHECK_NOT_IMPLEMENTED; + } + + // add this agg col to returnedColumnList + boost::shared_ptr spac(ac); + gwi.returnedCols.push_back(spac); + gwi.selectCols.push_back('`' + escapeBackTick(spac->alias().c_str()) + '`'); + String str(256); + item->print(&str, QT_INFINIDB_NO_QUOTE); + + if (sel_cols_in_create.length() != 0) + sel_cols_in_create += ", "; + + sel_cols_in_create += string(str.c_ptr()) + " `" + escapeBackTick(spac->alias().c_str()) + "`"; + break; + } + + case Item::FUNC_ITEM: + { + Item_func* ifp = reinterpret_cast(item); + + // @bug4383. error out non-support stored function + if (ifp->functype() == Item_func::FUNC_SP) + { + gwi.fatalParseError = true; + gwi.parseErrorText = IDBErrorInfo::instance()->errorMsg(ERR_SP_FUNCTION_NOT_SUPPORT); + setError(gwi.thd, ER_CHECK_NOT_IMPLEMENTED, gwi.parseErrorText, gwi); + return ER_CHECK_NOT_IMPLEMENTED; + } + + if (string(ifp->func_name()) == "xor") + { + gwi.fatalParseError = true; + gwi.parseErrorText = IDBErrorInfo::instance()->errorMsg(ERR_FILTER_COND_EXP); + setError(gwi.thd, ER_CHECK_NOT_IMPLEMENTED, gwi.parseErrorText, gwi); + return ER_CHECK_NOT_IMPLEMENTED; + } + + uint16_t parseInfo = 0; + vector tmpVec; + bool hasNonSupportItem = false; + parse_item(ifp, tmpVec, hasNonSupportItem, parseInfo); + + if (ifp->has_subquery() || + string(ifp->func_name()) == string("") || + ifp->functype() == Item_func::NOT_ALL_FUNC || + parseInfo & SUB_BIT) + { + gwi.fatalParseError = true; + gwi.parseErrorText = IDBErrorInfo::instance()->errorMsg(ERR_NON_SUPPORT_SELECT_SUB); + setError(gwi.thd, ER_CHECK_NOT_IMPLEMENTED, gwi.parseErrorText, gwi); + return ER_CHECK_NOT_IMPLEMENTED; + } + + ReturnedColumn* rc = buildFunctionColumn(ifp, gwi, hasNonSupportItem); + SRCP srcp(rc); + + if (rc) + { + if (!hasNonSupportItem && !nonConstFunc(ifp) && !(parseInfo & AF_BIT) && tmpVec.size() == 0) + { + if (isUnion || unionSel || gwi.subSelectType != CalpontSelectExecutionPlan::MAIN_SELECT || + parseInfo & SUB_BIT ) //|| select_lex.group_list.elements != 0) + { + srcp.reset(buildReturnedColumn(item, gwi, gwi.fatalParseError)); + gwi.returnedCols.push_back(srcp); + + if (ifp->name) + srcp->alias(ifp->name); + + continue; + } + + if ( ((gwi.thd->lex)->sql_command == SQLCOM_UPDATE ) || + ((gwi.thd->lex)->sql_command == SQLCOM_DELETE ) || + ((gwi.thd->lex)->sql_command == SQLCOM_UPDATE_MULTI ) || + ((gwi.thd->lex)->sql_command == SQLCOM_DELETE_MULTI ) ) + { } + else + { + redo = true; + String str; + ifp->print(&str, QT_INFINIDB_NO_QUOTE); + gwi.selectCols.push_back(string(str.c_ptr()) + " " + "`" + escapeBackTick(item->name) + "`"); + } + + break; + } + + //SRCP srcp(rc); + gwi.returnedCols.push_back(srcp); + + if ( ((gwi.thd->lex)->sql_command == SQLCOM_UPDATE ) || ((gwi.thd->lex)->sql_command == SQLCOM_DELETE ) || ((gwi.thd->lex)->sql_command == SQLCOM_UPDATE_MULTI ) || ((gwi.thd->lex)->sql_command == SQLCOM_DELETE_MULTI )) + { } + else + { + String str(256); + ifp->print(&str, QT_INFINIDB_NO_QUOTE); + + if (sel_cols_in_create.length() != 0) + sel_cols_in_create += ", "; + + sel_cols_in_create += string(str.c_ptr()) + " `" + ifp->name + "`"; + gwi.selectCols.push_back("`" + escapeBackTick(ifp->name) + "`"); + } + } + else // InfiniDB Non support functions still go through post process for now + { + hasNonSupportItem = false; + uint32_t before_size = funcFieldVec.size(); + parse_item(ifp, funcFieldVec, hasNonSupportItem, parseInfo); + uint32_t after_size = funcFieldVec.size(); + + // group by func and func in subquery can not be post processed + // @bug3881. set_user_var can not be treated as constant function + // @bug5716. Try to avoid post process function for union query. + if ((gwi.subQuery /*|| select_lex.group_list.elements != 0 */ || + !csep->unionVec().empty() || isUnion) && + !hasNonSupportItem && (after_size - before_size) == 0 && + !(parseInfo & AGG_BIT) && !(parseInfo & SUB_BIT) && + string(ifp->func_name()) != "set_user_var") + { + String val, *str = ifp->val_str(&val); + string valStr; + + if (str) + valStr.assign(str->ptr(), str->length()); + + ConstantColumn* cc = NULL; + + if (!str) + { + cc = new ConstantColumn("", ConstantColumn::NULLDATA); + } + else if (ifp->result_type() == STRING_RESULT) + { + cc = new ConstantColumn(valStr, ConstantColumn::LITERAL); + } + else if (ifp->result_type() == DECIMAL_RESULT) + { + cc = buildDecimalColumn(ifp, gwi); + } + else + { + cc = new ConstantColumn(valStr, ConstantColumn::NUM); + cc->resultType(colType_MysqlToIDB(item)); + } + + SRCP srcp(cc); + + if (ifp->name) + cc->alias(ifp->name); + + gwi.returnedCols.push_back(srcp); + + // clear the error set by buildFunctionColumn + gwi.fatalParseError = false; + gwi.parseErrorText = ""; + break; + } + else if (hasNonSupportItem || parseInfo & AGG_BIT || parseInfo & SUB_BIT || + (gwi.fatalParseError && gwi.subQuery)) + { + if (gwi.parseErrorText.empty()) + { + Message::Args args; + args.add(ifp->func_name()); + gwi.parseErrorText = IDBErrorInfo::instance()->errorMsg(ERR_NON_SUPPORTED_FUNCTION, args); + } + + setError(gwi.thd, ER_CHECK_NOT_IMPLEMENTED, gwi.parseErrorText, gwi); + return ER_CHECK_NOT_IMPLEMENTED; + } + else if ( gwi.subQuery && (isPredicateFunction(ifp, &gwi) || ifp->type() == Item::COND_ITEM )) + { + gwi.fatalParseError = true; + gwi.parseErrorText = IDBErrorInfo::instance()->errorMsg(ERR_FILTER_COND_EXP); + setError(gwi.thd, ER_CHECK_NOT_IMPLEMENTED, gwi.parseErrorText, gwi); + return ER_CHECK_NOT_IMPLEMENTED; + } + + //@Bug 3030 Add error check for dml statement + if ( ((gwi.thd->lex)->sql_command == SQLCOM_UPDATE ) || ((gwi.thd->lex)->sql_command == SQLCOM_DELETE ) || ((gwi.thd->lex)->sql_command == SQLCOM_UPDATE_MULTI ) || ((gwi.thd->lex)->sql_command == SQLCOM_DELETE_MULTI ) ) + { + if ( after_size - before_size != 0 ) + { + gwi.parseErrorText = ifp->func_name(); + return -1; + } + } + + //@Bug 3021. Bypass postprocess for update and delete. + //if ( ((gwi.thd->lex)->sql_command == SQLCOM_UPDATE ) || ((gwi.thd->lex)->sql_command == SQLCOM_DELETE ) || ((gwi.thd->lex)->sql_command == SQLCOM_UPDATE_MULTI ) || ((gwi.thd->lex)->sql_command == SQLCOM_DELETE_MULTI )) + //{} + else + { + // @bug 3881. Here is the real redo part. + redo = true; + // @bug 1706 + String funcStr; + ifp->print(&funcStr, QT_INFINIDB); + gwi.selectCols.push_back(string(funcStr.c_ptr()) + " `" + escapeBackTick(ifp->name) + "`"); + // clear the error set by buildFunctionColumn + gwi.fatalParseError = false; + gwi.parseErrorText = ""; + } + } + + break; + } + + case Item::INT_ITEM: + { + if ( ((gwi.thd->lex)->sql_command == SQLCOM_UPDATE ) || ((gwi.thd->lex)->sql_command == SQLCOM_DELETE ) || ((gwi.thd->lex)->sql_command == SQLCOM_UPDATE_MULTI ) || ((gwi.thd->lex)->sql_command == SQLCOM_DELETE_MULTI )) + { } + else + { + // do not push the dummy column (mysql added) to returnedCol + if (item->name && string(item->name) == "Not_used") + continue; + + // @bug3509. Constant column is sent to ExeMgr now. + SRCP srcp(buildReturnedColumn(item, gwi, gwi.fatalParseError)); + + if (item->name) + srcp->alias(item->name); + + gwi.returnedCols.push_back(srcp); + + Item_int* isp = reinterpret_cast(item); + ostringstream oss; + oss << isp->value << " `" << escapeBackTick(srcp->alias().c_str()) << "`"; + + if (sel_cols_in_create.length() != 0) + sel_cols_in_create += ", "; + + sel_cols_in_create += oss.str(); + gwi.selectCols.push_back(oss.str()); + } + + break; + } + + case Item::STRING_ITEM: + { + if ( ((gwi.thd->lex)->sql_command == SQLCOM_UPDATE ) || ((gwi.thd->lex)->sql_command == SQLCOM_DELETE ) || ((gwi.thd->lex)->sql_command == SQLCOM_UPDATE_MULTI ) || ((gwi.thd->lex)->sql_command == SQLCOM_DELETE_MULTI )) + { } + else + { + SRCP srcp(buildReturnedColumn(item, gwi, gwi.fatalParseError)); + gwi.returnedCols.push_back(srcp); + + if (item->name) + srcp->alias(item->name); + + Item_string* isp = reinterpret_cast(item); + String val, *str = isp->val_str(&val); + string valStr; + valStr.assign(str->ptr(), str->length()); + string name = "'" + valStr + "'" + " " + "`" + escapeBackTick(srcp->alias().c_str()) + "`"; + + if (sel_cols_in_create.length() != 0) + sel_cols_in_create += ", "; + + sel_cols_in_create += name; + gwi.selectCols.push_back(name); + } + + break; + } + + case Item::DECIMAL_ITEM: + { + if ( ((gwi.thd->lex)->sql_command == SQLCOM_UPDATE ) || ((gwi.thd->lex)->sql_command == SQLCOM_DELETE ) || ((gwi.thd->lex)->sql_command == SQLCOM_UPDATE_MULTI ) || ((gwi.thd->lex)->sql_command == SQLCOM_DELETE_MULTI )) + { } + else + { + SRCP srcp(buildReturnedColumn(item, gwi, gwi.fatalParseError)); + gwi.returnedCols.push_back(srcp); + + if (item->name) + srcp->alias(item->name); + + Item_decimal* isp = reinterpret_cast(item); + String val, *str = isp->val_str(&val); + string valStr; + valStr.assign(str->ptr(), str->length()); + ostringstream oss; + oss << valStr.c_str() << " `" << escapeBackTick(srcp->alias().c_str()) << "`"; + + if (sel_cols_in_create.length() != 0) + sel_cols_in_create += ", "; + + sel_cols_in_create += oss.str(); + gwi.selectCols.push_back(oss.str()); + } + + break; + } + + case Item::NULL_ITEM: + { + if ( ((gwi.thd->lex)->sql_command == SQLCOM_UPDATE ) || ((gwi.thd->lex)->sql_command == SQLCOM_DELETE ) || ((gwi.thd->lex)->sql_command == SQLCOM_UPDATE_MULTI ) || ((gwi.thd->lex)->sql_command == SQLCOM_DELETE_MULTI ) ) + { } + else + { + SRCP srcp(buildReturnedColumn(item, gwi, gwi.fatalParseError)); + gwi.returnedCols.push_back(srcp); + + if (item->name) + srcp->alias(item->name); + + string name = string("null `") + escapeBackTick(srcp->alias().c_str()) + string("`") ; + + if (sel_cols_in_create.length() != 0) + sel_cols_in_create += ", "; + + sel_cols_in_create += name; + gwi.selectCols.push_back("null"); + } + + break; + } + + case Item::SUBSELECT_ITEM: + { + Item_subselect* sub = (Item_subselect*)item; + + if (sub->substype() != Item_subselect::SINGLEROW_SUBS) + { + gwi.fatalParseError = true; + gwi.parseErrorText = IDBErrorInfo::instance()->errorMsg(ERR_NON_SUPPORT_SELECT_SUB); + setError(gwi.thd, ER_CHECK_NOT_IMPLEMENTED, gwi.parseErrorText, gwi); + return ER_CHECK_NOT_IMPLEMENTED; + } + +#ifdef DEBUG_WALK_COND + cerr << "SELECT clause SUBSELECT Item: " << sub->substype() << endl; + JOIN* join = sub->get_select_lex()->join; + + if (join) + { + Item_cond* cond = reinterpret_cast(join->conds); + + if (cond) + cond->traverse_cond(debug_walk, &gwi, Item::POSTFIX); + } + + cerr << "Finish SELECT clause subselect item traversing" << endl; +#endif + SelectSubQuery* selectSub = new SelectSubQuery(gwi, sub); + //selectSub->gwip(&gwi); + SCSEP ssub = selectSub->transform(); + + if (!ssub || gwi.fatalParseError) + { + if (gwi.parseErrorText.empty()) + gwi.parseErrorText = "Unsupported Item in SELECT subquery."; + + setError(gwi.thd, ER_CHECK_NOT_IMPLEMENTED, gwi.parseErrorText, gwi); + return ER_CHECK_NOT_IMPLEMENTED; + } + + selectSubList.push_back(ssub); + SimpleColumn* rc = new SimpleColumn(); + rc->colSource(rc->colSource() | SELECT_SUB); + + if (sub->get_select_lex()->get_table_list()) + rc->viewName(lower(getViewName(sub->get_select_lex()->get_table_list()))); + + if (sub->name) + rc->alias(sub->name); + + gwi.returnedCols.push_back(SRCP(rc)); + String str; + sub->get_select_lex()->print(gwi.thd, &str, QT_INFINIDB_NO_QUOTE); + sel_cols_in_create += "(" + string(str.c_ptr()) + ")"; + + if (sub->name) + { + sel_cols_in_create += "`" + escapeBackTick(sub->name) + "`"; + gwi.selectCols.push_back(sub->name); + } + else + { + sel_cols_in_create += "`" + escapeBackTick(str.c_ptr()) + "`"; + gwi.selectCols.push_back("`" + escapeBackTick(str.c_ptr()) + "`"); + } + + break; + } + + case Item::COND_ITEM: + { + gwi.fatalParseError = true; + gwi.parseErrorText = IDBErrorInfo::instance()->errorMsg(ERR_FILTER_COND_EXP); + setError(gwi.thd, ER_CHECK_NOT_IMPLEMENTED, gwi.parseErrorText, gwi); + return ER_CHECK_NOT_IMPLEMENTED; + } + + case Item::EXPR_CACHE_ITEM: + { + printf("EXPR_CACHE_ITEM in getSelectPlan\n"); + gwi.fatalParseError = true; + gwi.parseErrorText = IDBErrorInfo::instance()->errorMsg(ERR_UNKNOWN_COL); + setError(gwi.thd, ER_CHECK_NOT_IMPLEMENTED, gwi.parseErrorText, gwi); + return ER_CHECK_NOT_IMPLEMENTED; + } + + case Item::WINDOW_FUNC_ITEM: + { + SRCP srcp(buildWindowFunctionColumn(item, gwi, gwi.fatalParseError)); + + if (!srcp || gwi.fatalParseError) + { + if (gwi.parseErrorText.empty()) + gwi.parseErrorText = "Unsupported Item in SELECT subquery."; + + setError(gwi.thd, ER_CHECK_NOT_IMPLEMENTED, gwi.parseErrorText, gwi); + return ER_CHECK_NOT_IMPLEMENTED; + } + + gwi.returnedCols.push_back(srcp); + break; + } + + default: + { + break; + } + } + } + + // @bug4388 normalize the project coltypes for union main select list + if (!csep->unionVec().empty()) + { + for (uint32_t i = 0; i < gwi.returnedCols.size(); i++) + { + vector coltypes; + + for (uint32_t j = 0; j < csep->unionVec().size(); j++) + { + coltypes.push_back( + dynamic_cast(csep->unionVec()[j].get())->returnedCols()[i]->resultType()); + + // @bug5976. set hasAggregate true for the main column if + // one corresponding union column has aggregate + if (dynamic_cast(csep->unionVec()[j].get())->returnedCols()[i]->hasAggregate()) + gwi.returnedCols[i]->hasAggregate(true); + } + + gwi.returnedCols[i]->resultType(dataconvert::DataConvert::convertUnionColType(coltypes)); + } + } + + // Having clause handling + gwi.clauseType = HAVING; + clearStacks(gwi); + ParseTree* havingFilter = 0; + // clear fatalParseError that may be left from post process functions + gwi.fatalParseError = false; + gwi.parseErrorText = ""; + + if (gi.groupByHaving != 0) + { + Item_cond* having = reinterpret_cast(gi.groupByHaving); +#ifdef DEBUG_WALK_COND + cerr << "------------------- HAVING ---------------------" << endl; + having->traverse_cond(debug_walk, &gwi, Item::POSTFIX); + cerr << "------------------------------------------------\n" << endl; +#endif + having->traverse_cond(gp_walk, &gwi, Item::POSTFIX); + + if (gwi.fatalParseError) + { + setError(gwi.thd, ER_INTERNAL_ERROR, gwi.parseErrorText, gwi); + return ER_INTERNAL_ERROR; + } + + ParseTree* ptp = 0; + ParseTree* rhs = 0; + + // @bug 4215. some function filter will be in the rcWorkStack. + if (gwi.ptWorkStack.empty() && !gwi.rcWorkStack.empty()) + { + havingFilter = new ParseTree(gwi.rcWorkStack.top()); + gwi.rcWorkStack.pop(); + } + + while (!gwi.ptWorkStack.empty()) + { + havingFilter = gwi.ptWorkStack.top(); + gwi.ptWorkStack.pop(); + + if (gwi.ptWorkStack.empty()) + break; + + ptp = new ParseTree(new LogicOperator("and")); + ptp->left(havingFilter); + rhs = gwi.ptWorkStack.top(); + gwi.ptWorkStack.pop(); + ptp->right(rhs); + gwi.ptWorkStack.push(ptp); + } + } + + // for post process expressions on the select list + // error out post process for union and sub select unit + if (isUnion || gwi.subSelectType != CalpontSelectExecutionPlan::MAIN_SELECT) + { + if (funcFieldVec.size() != 0 && !gwi.fatalParseError) + { + string emsg("Fatal parse error in vtable mode: Unsupported Items in union or sub select unit"); + setError(gwi.thd, ER_CHECK_NOT_IMPLEMENTED, emsg); + return ER_CHECK_NOT_IMPLEMENTED; + } + } + + for (uint32_t i = 0; i < funcFieldVec.size(); i++) + { + //SimpleColumn *sc = new SimpleColumn(funcFieldVec[i]->db_name, bestTableName(funcFieldVec[i])/*funcFieldVec[i]->table_name*/, funcFieldVec[i]->field_name, sessionID); + SimpleColumn* sc = buildSimpleColumn(funcFieldVec[i], gwi); + + if (!sc || gwi.fatalParseError) + { + string emsg; + + if (gwi.parseErrorText.empty()) + { + emsg = "un-recognized column"; + + if (funcFieldVec[i]->name) + emsg += string(funcFieldVec[i]->name); + } + else + { + emsg = gwi.parseErrorText; + } + + setError(gwi.thd, ER_INTERNAL_ERROR, emsg, gwi); + return ER_INTERNAL_ERROR; + } + + String str; + funcFieldVec[i]->print(&str, QT_INFINIDB_NO_QUOTE); + sc->alias(string(str.c_ptr())); + //sc->tableAlias(funcFieldVec[i]->table_name); + sc->tableAlias(sc->tableAlias()); + SRCP srcp(sc); + uint32_t j = 0; + + for (; j < gwi.returnedCols.size(); j++) + { + if (sc->sameColumn(gwi.returnedCols[j].get())) + { + SimpleColumn* field = dynamic_cast(gwi.returnedCols[j].get()); + + if (field && field->alias() == sc->alias()) + break; + } + } + + if (j == gwi.returnedCols.size()) + { + gwi.returnedCols.push_back(srcp); + gwi.columnMap.insert(CalpontSelectExecutionPlan::ColumnMap::value_type(string(funcFieldVec[i]->field_name), srcp)); + + if (sel_cols_in_create.length() != 0) + sel_cols_in_create += ", "; + + string fullname; + fullname = str.c_ptr(); + sel_cols_in_create += fullname + " `" + escapeBackTick(fullname.c_str()) + "`"; + TABLE_LIST* tmp = (funcFieldVec[i]->cached_table ? funcFieldVec[i]->cached_table : 0); + gwi.tableMap[make_aliastable(sc->schemaName(), sc->tableName(), sc->tableAlias(), sc->isInfiniDB())] = + make_pair(1, tmp); + } + } + + // post-process Order by list and expressions on select by redo phase1. only for vtable + // ignore ORDER BY clause for union select unit + string ord_cols = ""; // for normal select phase + SRCP minSc; // min width projected column. for count(*) use + + // Group by list. not valid for union main query + + /* + if (gwi.thd->infinidb_vtable.vtable_state == THD::INFINIDB_CREATE_VTABLE && !unionSel) + { + gwi.clauseType = GROUP_BY; + Item* nonSupportItem = NULL; + ORDER* groupcol = reinterpret_cast(gi.groupByGr.first); + + // check if window functions are in order by. InfiniDB process order by list if + // window functions are involved, either in order by or projection. + bool hasWindowFunc = gwi.hasWindowFunc; + gwi.hasWindowFunc = false; + + for (; groupcol; groupcol = groupcol->next) + { + if ((*(groupcol->item))->type() == Item::WINDOW_FUNC_ITEM) + gwi.hasWindowFunc = true; + } + + if (gwi.hasWindowFunc) + { + gwi.fatalParseError = true; + gwi.parseErrorText = IDBErrorInfo::instance()->errorMsg(ERR_WF_NOT_ALLOWED, "GROUP BY clause"); + setError(gwi.thd, ER_CHECK_NOT_IMPLEMENTED, gwi.parseErrorText, gwi); + return ER_CHECK_NOT_IMPLEMENTED; + } + + gwi.hasWindowFunc = hasWindowFunc; + groupcol = reinterpret_cast(gi.groupByGr.first); + + for (; groupcol; groupcol = groupcol->next) + { + Item* groupItem = *(groupcol->item); + + // @bug5993. Could be nested ref. + while (groupItem->type() == Item::REF_ITEM) + groupItem = (*((Item_ref*)groupItem)->ref); + + if (groupItem->type() == Item::FUNC_ITEM) + { + Item_func* ifp = (Item_func*)groupItem; + + // call buildFunctionColumn here mostly for finding out + // non-support column on GB list. Should be simplified. + ReturnedColumn* fc = buildFunctionColumn(ifp, gwi, gwi.fatalParseError); + + if (!fc || gwi.fatalParseError) + { + nonSupportItem = ifp; + break; + } + + if (groupcol->in_field_list && groupcol->counter_used) + { + delete fc; + fc = gwi.returnedCols[groupcol->counter - 1].get(); + SRCP srcp(fc->clone()); + + // check if no column parm + for (uint32_t i = 0; i < gwi.no_parm_func_list.size(); i++) + { + if (gwi.no_parm_func_list[i]->expressionId() == fc->expressionId()) + { + gwi.no_parm_func_list.push_back(dynamic_cast(srcp.get())); + break; + } + } + + srcp->orderPos(groupcol->counter - 1); + gwi.groupByCols.push_back(srcp); + continue; + } + else if (!groupItem->is_autogenerated_name) // alias + { + uint32_t i = 0; + + for (; i < gwi.returnedCols.size(); i++) + { + if (string(groupItem->name) == gwi.returnedCols[i]->alias()) + { + ReturnedColumn* rc = gwi.returnedCols[i]->clone(); + rc->orderPos(i); + gwi.groupByCols.push_back(SRCP(rc)); + delete fc; + break; + } + } + + if (i == gwi.returnedCols.size()) + { + nonSupportItem = groupItem; + break; + } + } + else + { + uint32_t i = 0; + + for (; i < gwi.returnedCols.size(); i++) + { + if (fc->operator==(gwi.returnedCols[i].get())) + { + ReturnedColumn* rc = gwi.returnedCols[i]->clone(); + rc->orderPos(i); + gwi.groupByCols.push_back(SRCP(rc)); + delete fc; + break; + } + } + + if (i == gwi.returnedCols.size()) + { + gwi.groupByCols.push_back(SRCP(fc)); + break; + } + } + } + else if (groupItem->type() == Item::FIELD_ITEM) + { + Item_field* ifp = (Item_field*)groupItem; + // this GB col could be an alias of F&E on the SELECT clause, not necessarily a field. + ReturnedColumn* rc = buildSimpleColumn(ifp, gwi); + SimpleColumn* sc = dynamic_cast(rc); + + for (uint32_t j = 0; j < gwi.returnedCols.size(); j++) + { + if (sc) + { + if (sc->sameColumn(gwi.returnedCols[j].get())) + { + sc->orderPos(j); + break; + } + else if (strcasecmp(sc->alias().c_str(), gwi.returnedCols[j]->alias().c_str()) == 0) + { + rc = gwi.returnedCols[j].get()->clone(); + rc->orderPos(j); + break; + } + } + else + { + if (ifp->name && string(ifp->name) == gwi.returnedCols[j].get()->alias()) + { + rc = gwi.returnedCols[j].get()->clone(); + rc->orderPos(j); + break; + } + } + } + + if (!rc) + { + nonSupportItem = ifp; + break; + } + + SRCP srcp(rc); + + // bug 3151 + AggregateColumn* ac = dynamic_cast(rc); + + if (ac) + { + nonSupportItem = ifp; + break; + } + + gwi.groupByCols.push_back(srcp); + gwi.columnMap.insert(CalpontSelectExecutionPlan::ColumnMap::value_type(string(ifp->field_name), srcp)); + } + // @bug5638. The group by column is constant but not counter, alias has to match a column + // on the select list + else if (!groupcol->counter_used && + (groupItem->type() == Item::INT_ITEM || + groupItem->type() == Item::STRING_ITEM || + groupItem->type() == Item::REAL_ITEM || + groupItem->type() == Item::DECIMAL_ITEM)) + { + ReturnedColumn* rc = 0; + + for (uint32_t j = 0; j < gwi.returnedCols.size(); j++) + { + if (groupItem->name && string(groupItem->name) == gwi.returnedCols[j].get()->alias()) + { + rc = gwi.returnedCols[j].get()->clone(); + rc->orderPos(j); + break; + } + } + + if (!rc) + { + nonSupportItem = groupItem; + break; + } + + gwi.groupByCols.push_back(SRCP(rc)); + } + else if ((*(groupcol->item))->type() == Item::SUBSELECT_ITEM) + { + if (!groupcol->in_field_list || !groupItem->name) + { + nonSupportItem = groupItem; + } + else + { + uint32_t i = 0; + + for (; i < gwi.returnedCols.size(); i++) + { + if (string(groupItem->name) == gwi.returnedCols[i]->alias()) + { + ReturnedColumn* rc = gwi.returnedCols[i]->clone(); + rc->orderPos(i); + gwi.groupByCols.push_back(SRCP(rc)); + break; + } + } + + if (i == gwi.returnedCols.size()) + { + nonSupportItem = groupItem; + } + } + } + // @bug 3761. + else if (groupcol->counter_used) + { + if (gwi.returnedCols.size() <= (uint32_t)(groupcol->counter - 1)) + { + nonSupportItem = groupItem; + } + else + { + gwi.groupByCols.push_back(SRCP(gwi.returnedCols[groupcol->counter - 1]->clone())); + } + } + else + { + nonSupportItem = groupItem; + } + + } + + // @bug 4756. Add internal groupby column for correlated join to the groupby list + if (gwi.aggOnSelect && !gwi.subGroupByCols.empty()) + gwi.groupByCols.insert(gwi.groupByCols.end(), gwi.subGroupByCols.begin(), gwi.subGroupByCols.end()); + + // this is window func on SELECT becuase ORDER BY has not been processed + if (!gwi.windowFuncList.empty() && !gwi.subGroupByCols.empty()) + { + for (uint32_t i = 0; i < gwi.windowFuncList.size(); i++) + { + if (gwi.windowFuncList[i]->hasWindowFunc()) + { + vector windowFunctions = gwi.windowFuncList[i]->windowfunctionColumnList(); + + for (uint32_t j = 0; j < windowFunctions.size(); j++) + windowFunctions[j]->addToPartition(gwi.subGroupByCols); + } + } + } + + if (nonSupportItem) + { + Message::Args args; + + if (nonSupportItem->name) + args.add("'" + string(nonSupportItem->name) + "'"); + else + args.add(""); + + gwi.parseErrorText = IDBErrorInfo::instance()->errorMsg(ERR_NON_SUPPORT_GROUP_BY, args); + setError(gwi.thd, ER_CHECK_NOT_IMPLEMENTED, gwi.parseErrorText, gwi); + return ER_CHECK_NOT_IMPLEMENTED; + } + } + */ + + + if (gwi.thd->infinidb_vtable.vtable_state == THD::INFINIDB_CREATE_VTABLE) + { + //SQL_I_List order_list = gi.groupByOrder; + ORDER* ordercol = reinterpret_cast(gi.groupByOrder); + string create_query(gwi.thd->infinidb_vtable.create_vtable_query.c_ptr()); + string select_query(gwi.thd->infinidb_vtable.select_vtable_query.c_ptr()); + string lower_create_query(gwi.thd->infinidb_vtable.create_vtable_query.c_ptr()); + string lower_select_query(gwi.thd->infinidb_vtable.select_vtable_query.c_ptr()); + algorithm::to_lower(lower_create_query); + algorithm::to_lower(lower_select_query); + + + // check if window functions are in order by. InfiniDB process order by list if + // window functions are involved, either in order by or projection. + for (; ordercol; ordercol = ordercol->next) + { + if ((*(ordercol->item))->type() == Item::WINDOW_FUNC_ITEM) + gwi.hasWindowFunc = true; + } + + // re-visit the first of ordercol list + ordercol = reinterpret_cast(gi.groupByOrder); + + // for subquery, order+limit by will be supported in infinidb. build order by columns + // @todo union order by and limit support + if (gwi.hasWindowFunc || gwi.subSelectType != CalpontSelectExecutionPlan::MAIN_SELECT) + { + for (; ordercol; ordercol = ordercol->next) + { + ReturnedColumn* rc = NULL; + + if (ordercol->in_field_list && ordercol->counter_used) + { + rc = gwi.returnedCols[ordercol->counter - 1]->clone(); + rc->orderPos(ordercol->counter - 1); + // can not be optimized off if used in order by with counter. + // set with self derived table alias if it's derived table + gwi.returnedCols[ordercol->counter - 1]->incRefCount(); + } + else + { + Item* ord_item = *(ordercol->item); + + // ignore not_used column on order by. + if (ord_item->type() == Item::INT_ITEM && ord_item->full_name() && string(ord_item->full_name()) == "Not_used") + continue; + else if (ord_item->type() == Item::INT_ITEM) + rc = gwi.returnedCols[((Item_int*)ord_item)->val_int() - 1]->clone(); + else if (ord_item->type() == Item::SUBSELECT_ITEM) + gwi.fatalParseError = true; + else + rc = buildReturnedColumn(ord_item, gwi, gwi.fatalParseError); + + // @bug5501 try item_ptr if item can not be fixed. For some + // weird dml statement state, item can not be fixed but the + // infomation is available in item_ptr. + if (!rc || gwi.fatalParseError) + { + Item* item_ptr = ordercol->item_ptr; + + while (item_ptr->type() == Item::REF_ITEM) + item_ptr = *(((Item_ref*)item_ptr)->ref); + + rc = buildReturnedColumn(item_ptr, gwi, gwi.fatalParseError); + } + + if (!rc) + { + string emsg = IDBErrorInfo::instance()->errorMsg(ERR_NON_SUPPORT_ORDER_BY); + gwi.parseErrorText = emsg; + setError(gwi.thd, ER_CHECK_NOT_IMPLEMENTED, emsg, gwi); + return ER_CHECK_NOT_IMPLEMENTED; + } + } + + if (ordercol->direction == ORDER::ORDER_ASC) + rc->asc(true); + else + rc->asc(false); + + gwi.orderByCols.push_back(SRCP(rc)); + } + } + else if (!isUnion) + { + vector fieldVec; + bool addToSel; + + // the following order by is just for redo phase + if (!unionSel) + { + for (; ordercol; ordercol = ordercol->next) + { + Item* ord_item = *(ordercol->item); + + // @bug5993. Could be nested ref. + while (ord_item->type() == Item::REF_ITEM) + ord_item = (*((Item_ref*)ord_item)->ref); + + // @bug 1706. re-construct the order by item one by one + //Item* ord_item = *(ordercol->item); + if (ord_cols.length() != 0) + ord_cols += ", "; + + addToSel = true; + string fullname; + + if (ordercol->in_field_list && ordercol->counter_used) + { + ostringstream oss; + oss << ordercol->counter; + ord_cols += oss.str(); + + if (ordercol->direction != ORDER::ORDER_ASC) + ord_cols += " desc"; + + continue; + } + + else if (ord_item->type() == Item::FUNC_ITEM) + { + // @bug 2621. order by alias + if (!ord_item->is_autogenerated_name && ord_item->name) + { + ord_cols += ord_item->name; + continue; + } + + // if there's group by clause or aggregate column, check to see + // if this item or the arguments is on the GB list. + ReturnedColumn* rc = 0; + // check if this order by column is on the select list + Item_func* ifp = (Item_func*)(*(ordercol->item)); + rc = buildFunctionColumn(ifp, gwi, gwi.fatalParseError); + + if (rc) + { + for (uint32_t i = 0; i < gwi.returnedCols.size(); i++) + { + if (rc && rc->operator==(gwi.returnedCols[i].get())) + { + ostringstream oss; + oss << i + 1; + ord_cols += oss.str(); + addToSel = false; + break; + } + } + } + + if (addToSel) + { + FunctionColumn* fc = dynamic_cast(rc); + + if (fc) + { + addToSel = false; + redo = true; + string ord_func = string(ifp->func_name()) + "("; + + for (uint32_t i = 0; i < fc->functionParms().size(); i++) + { + if (i != 0) + ord_func += ","; + + for (uint32_t j = 0; j < gwi.returnedCols.size(); j++) + { + if (fc->functionParms()[i]->data()->operator==(gwi.returnedCols[j].get())) + { + ord_func += "`" + escapeBackTick(gwi.returnedCols[j]->alias().c_str()) + "`"; + continue; + } + + AggregateColumn* ac = dynamic_cast(fc->functionParms()[i]->data()); + + if (ac) + { + gwi.parseErrorText = IDBErrorInfo::instance()->errorMsg(ERR_NON_SUPPORT_ORDER_BY); + setError(gwi.thd, ER_CHECK_NOT_IMPLEMENTED, gwi.parseErrorText, gwi); + return ER_CHECK_NOT_IMPLEMENTED; + } + + addToSel = true; + //continue; + + } + } + + ord_func += ")"; + + if (!addToSel) + ord_cols += ord_func; + } + } + } + else if (ord_item->type() == Item::SUBSELECT_ITEM) + { + string emsg = IDBErrorInfo::instance()->errorMsg(ERR_NON_SUPPORT_ORDER_BY); + setError(gwi.thd, ER_CHECK_NOT_IMPLEMENTED, emsg, gwi); + return ER_CHECK_NOT_IMPLEMENTED; + } + + else if (ord_item->type() == Item::SUM_FUNC_ITEM) + { + ReturnedColumn* ac = 0; + + Item_sum* ifp = (Item_sum*)(*(ordercol->item)); + // @bug3477. add aggregate column to the select list of the create phase. + ac = buildAggregateColumn(ifp, gwi); + + if (!ac) + { + setError(gwi.thd, ER_CHECK_NOT_IMPLEMENTED, + IDBErrorInfo::instance()->errorMsg(ERR_NON_SUPPORT_ORDER_BY), gwi); + return ER_CHECK_NOT_IMPLEMENTED; + } + + // check if this order by column is on the select list + for (uint32_t i = 0; i < gwi.returnedCols.size(); i++) + { + AggregateColumn* ret = dynamic_cast(gwi.returnedCols[i].get()); + + if (!ret) + continue; + + if (ac->operator==(gwi.returnedCols[i].get())) + { + ostringstream oss; + oss << i + 1; + ord_cols += oss.str(); + addToSel = false; + break; + } + } + + if (ac || !gwi.groupByCols.empty()) + { + if (addToSel) + { + redo = true; + // @bug 3076. do not add the argument of aggregate function to the SELECT list, + // instead, add the whole column + String str; + ord_item->print(&str, QT_INFINIDB_NO_QUOTE); + + if (sel_cols_in_create.length() != 0) + sel_cols_in_create += ", "; + + sel_cols_in_create += str.c_ptr(); + //gwi.selectCols.push_back(" `" + string(str.c_ptr()) + "`"); + SRCP srcp(ac); + gwi.returnedCols.push_back(srcp); + ord_cols += " `" + escapeBackTick(str.c_ptr()) + "`"; + } + + if (ordercol->direction != ORDER::ORDER_ASC) + ord_cols += " desc"; + + continue; + } + } + else if (ord_item->name && ord_item->type() == Item::FIELD_ITEM) + { + Item_field* field = reinterpret_cast(ord_item); + ReturnedColumn* rc = buildSimpleColumn(field, gwi); + fullname = field->full_name(); +// if (field->db_name) +// fullname += string(field->db_name) + "."; +// if (field->table_name) +// fullname += string(field->table_name) + "."; +// if (field->field_name) +// fullname += string(field->field_name); + + for (uint32_t i = 0; i < gwi.returnedCols.size(); i++) + { + SimpleColumn* sc = dynamic_cast(gwi.returnedCols[i].get()); + + if (sc && ((Item_field*)ord_item)->cached_table && + (strcasecmp(getViewName(((Item_field*)ord_item)->cached_table).c_str(), sc->viewName().c_str()) != 0)) + { + continue; + } + + if (strcasecmp(fullname.c_str(), gwi.returnedCols[i]->alias().c_str()) == 0 || + strcasecmp(ord_item->name, gwi.returnedCols[i]->alias().c_str()) == 0) + { + ord_cols += string(" `") + escapeBackTick(gwi.returnedCols[i]->alias().c_str()) + '`'; + addToSel = false; + break; + } + + if (sc && sc->sameColumn(rc)) + { + ostringstream oss; + oss << i + 1; + ord_cols += oss.str(); + addToSel = false; + break; + } + } + } + + if (addToSel) + { + // @bug 2719. Error out order by not on the distinct select list. + if ( gi.groupByDistinct ) + { + gwi.parseErrorText = IDBErrorInfo::instance()->errorMsg(ERR_ORDERBY_NOT_IN_DISTINCT); + setError(gwi.thd, ER_CHECK_NOT_IMPLEMENTED, gwi.parseErrorText, gwi); + return ER_CHECK_NOT_IMPLEMENTED; + } + + bool hasNonSupportItem = false; + uint16_t parseInfo = 0; + parse_item(ord_item, fieldVec, hasNonSupportItem, parseInfo); + + if (hasNonSupportItem) + { + string emsg = IDBErrorInfo::instance()->errorMsg(ERR_NON_SUPPORT_ORDER_BY); + setError(gwi.thd, ER_CHECK_NOT_IMPLEMENTED, emsg, gwi); + return ER_CHECK_NOT_IMPLEMENTED; + } + + String str; + ord_item->print(&str, QT_INFINIDB); + ord_cols += str.c_ptr(); + } + + if (ordercol->direction != ORDER::ORDER_ASC) + ord_cols += " desc"; + } + } + + redo = (redo || fieldVec.size() != 0); + + // populate string to be added to the select list for order by + for (uint32_t i = 0; i < fieldVec.size(); i++) + { + SimpleColumn* sc = buildSimpleColumn(fieldVec[i], gwi); + + if (!sc) + { + string emsg = IDBErrorInfo::instance()->errorMsg(ERR_NON_SUPPORT_ORDER_BY); + setError(gwi.thd, ER_CHECK_NOT_IMPLEMENTED, emsg, gwi); + return ER_CHECK_NOT_IMPLEMENTED; + } + + String str; + fieldVec[i]->print(&str, QT_INFINIDB_NO_QUOTE); + sc->alias(string(str.c_ptr())); + SRCP srcp(sc); + uint32_t j = 0; + + for (; j < gwi.returnedCols.size(); j++) + { + if (sc->sameColumn(gwi.returnedCols[j].get())) + { + SimpleColumn* field = dynamic_cast(gwi.returnedCols[j].get()); + + if (field && field->alias() == sc->alias()) + break; + } + } + + if (j == gwi.returnedCols.size()) + { + string fullname; + + if (sel_cols_in_create.length() != 0) + sel_cols_in_create += ", "; + + fullname = str.c_ptr(); + sel_cols_in_create += fullname + " `" + escapeBackTick(fullname.c_str()) + "`"; + + gwi.returnedCols.push_back(srcp); + gwi.columnMap.insert(CalpontSelectExecutionPlan::ColumnMap::value_type(string(fieldVec[i]->field_name), srcp)); + TABLE_LIST* tmp = (fieldVec[i]->cached_table ? fieldVec[i]->cached_table : 0); + gwi.tableMap[make_aliastable(sc->schemaName(), sc->tableName(), sc->tableAlias(), sc->isInfiniDB())] = + make_pair(1, tmp); + } + } + } + + // make sure columnmap, returnedcols and count(*) arg_list are not empty + TableMap::iterator tb_iter = gwi.tableMap.begin(); + + try + { + for (; tb_iter != gwi.tableMap.end(); tb_iter++) + { + if ((*tb_iter).second.first == 1) continue; + + CalpontSystemCatalog::TableAliasName tan = (*tb_iter).first; + CalpontSystemCatalog::TableName tn = make_table((*tb_iter).first.schema, (*tb_iter).first.table); + SimpleColumn* sc = getSmallestColumn(csc, tn, tan, (*tb_iter).second.second->table, gwi); + SRCP srcp(sc); + gwi.columnMap.insert(CalpontSelectExecutionPlan::ColumnMap::value_type(sc->columnName(), srcp)); + (*tb_iter).second.first = 1; + } + } + catch (runtime_error& e) + { + setError(gwi.thd, ER_INTERNAL_ERROR, e.what(), gwi); + CalpontSystemCatalog::removeCalpontSystemCatalog(sessionID); + return ER_INTERNAL_ERROR; + } + catch (...) + { + string emsg = IDBErrorInfo::instance()->errorMsg(ERR_LOST_CONN_EXEMGR); + setError(gwi.thd, ER_INTERNAL_ERROR, emsg, gwi); + CalpontSystemCatalog::removeCalpontSystemCatalog(sessionID); + return ER_INTERNAL_ERROR; + } + + if (!gwi.count_asterisk_list.empty() || !gwi.no_parm_func_list.empty() || + gwi.returnedCols.empty()) + { + // get the smallest column from colmap + CalpontSelectExecutionPlan::ColumnMap::const_iterator iter; + int minColWidth = 0; + CalpontSystemCatalog::ColType ct; + + try + { + for (iter = gwi.columnMap.begin(); iter != gwi.columnMap.end(); ++iter) + { + // should always not null + SimpleColumn* sc = dynamic_cast(iter->second.get()); + + if (sc && !(sc->joinInfo() & JOIN_CORRELATED)) + { + ct = csc->colType(sc->oid()); + + if (minColWidth == 0) + { + minColWidth = ct.colWidth; + minSc = iter->second; + } + else if (ct.colWidth < minColWidth) + { + minColWidth = ct.colWidth; + minSc = iter->second; + } + } + } + } + catch (...) + { + string emsg = IDBErrorInfo::instance()->errorMsg(ERR_LOST_CONN_EXEMGR); + setError(gwi.thd, ER_INTERNAL_ERROR, emsg, gwi); + CalpontSystemCatalog::removeCalpontSystemCatalog(sessionID); + return ER_INTERNAL_ERROR; + } + + if (gwi.returnedCols.empty() && gwi.additionalRetCols.empty()) + gwi.returnedCols.push_back(minSc); + } + + if (!isUnion && !gwi.hasWindowFunc && gwi.subSelectType == CalpontSelectExecutionPlan::MAIN_SELECT) + { + std::ostringstream vtb; + vtb << "infinidb_vtable.$vtable_" << gwi.thd->thread_id; + + //vtb << "$vtable_" << gwi.thd->thread_id; + // re-construct the select query and redo phase 1 + if (redo) + { + // select now() from region case. returnedCols should have minSc. + if (sel_cols_in_create.length() == 0) + { + SimpleColumn* sc = dynamic_cast(gwi.returnedCols[0].get()); + + if (sc) + sel_cols_in_create = dynamic_cast(gwi.returnedCols[0].get())->columnName(); + else + sel_cols_in_create = gwi.returnedCols[0]->alias(); + } + + // select * from derived table case + if (gwi.selectCols.empty()) + sel_cols_in_create = " * "; + + create_query = "create temporary table " + vtb.str() + " engine = aria as select " + sel_cols_in_create + " from "; + TABLE_LIST* table_ptr = gi.groupByTables; + + bool firstTb = true; + + // put all tables, derived tables and views on the list + //TABLE_LIST* table_ptr = select_lex.get_table_list(); + set aliasSet; // to avoid duplicate table alias + + for (; table_ptr; table_ptr = table_ptr->next_global) + { + if (string(table_ptr->table_name).find("$vtable") != string::npos) + continue; + + if (table_ptr->derived) + { + if (aliasSet.find(table_ptr->alias) != aliasSet.end()) + continue; + + String str; + (table_ptr->derived->first_select())->print(gwi.thd, &str, QT_INFINIDB_DERIVED); + + if (!firstTb) + create_query += ", "; + + create_query += "(" + string(str.c_ptr()) + ") " + string(table_ptr->alias); + firstTb = false; + aliasSet.insert(table_ptr->alias); + } + else if (table_ptr->view) + { + if (aliasSet.find(table_ptr->alias) != aliasSet.end()) + continue; + + if (!firstTb) + create_query += ", "; + + create_query += string(table_ptr->db) + "." + string(table_ptr->table_name) + + string(" `") + escapeBackTick(table_ptr->alias) + string("`"); + aliasSet.insert(table_ptr->alias); + firstTb = false; + } + else + { + // table referenced by view is represented by viewAlias_tableAlias. + // consistent with item.cc field print. + if (table_ptr->referencing_view) + { + if (aliasSet.find(string(table_ptr->referencing_view->alias) + "_" + + string(table_ptr->alias)) != aliasSet.end()) + continue; + + if (!firstTb) + create_query += ", "; + + create_query += string(table_ptr->db) + "." + string(table_ptr->table_name) + string(" "); + create_query += string(" `") + + escapeBackTick(table_ptr->referencing_view->alias) + "_" + + escapeBackTick(table_ptr->alias) + string("`"); + aliasSet.insert(string(table_ptr->referencing_view->alias) + "_" + + string(table_ptr->alias)); + } + else + { + if (aliasSet.find(table_ptr->alias) != aliasSet.end()) + continue; + + if (!firstTb) + create_query += ", "; + + create_query += string(table_ptr->db) + "." + string(table_ptr->table_name) + string(" "); + create_query += string("`") + escapeBackTick(table_ptr->alias) + string("`"); + aliasSet.insert(table_ptr->alias); + } + + firstTb = false; + } + } + + + gwi.thd->infinidb_vtable.create_vtable_query.free(); + gwi.thd->infinidb_vtable.create_vtable_query.append(create_query.c_str(), create_query.length()); + gwi.thd->infinidb_vtable.vtable_state = THD::INFINIDB_REDO_PHASE1; // redo phase 1 + + // turn off select distinct from post process unless there're post process functions + // on the select list. + string sel_query = "select "; + + if (gi.groupByDistinct && redo) + sel_query = "select distinct "; + else + sel_query = "select "; + + // select * from derived table... + if (gwi.selectCols.size() == 0) + sel_query += " * "; + + for (uint32_t i = 0; i < gwi.selectCols.size(); i++) + { + sel_query += gwi.selectCols[i]; + + if ( i + 1 != gwi.selectCols.size()) + sel_query += ", "; + } + + select_query.replace(lower_select_query.find("select *"), string("select *").length(), sel_query); + } + else + { + // remove order by clause in case this phase has been executed before. + // need a better fix later, like skip all the other non-optimized phase. + size_t pos = lower_select_query.find("order by"); + + if (pos != string::npos) + select_query.replace(pos, lower_select_query.length() - pos, ""); + + //select_query = "select * from " + vtb.str(); + // MCOL-1052 + if (unionSel) + { + ordercol = reinterpret_cast(gi.groupByOrder); + //order_list = gi.groupByOrder; + } + else + ordercol = 0; + + ord_cols = ""; + + for (; ordercol; ordercol = ordercol->next) + { + Item* ord_item = *(ordercol->item); + + // @bug 1706. re-construct the order by item one by one, because the ord_cols constucted so far + // is for REDO phase. + if (ord_cols.length() != 0) + ord_cols += ", "; + + if (ordercol->in_field_list && ordercol->counter_used) + { + ostringstream oss; + oss << ordercol->counter; + ord_cols += oss.str(); + } + else if (ord_item->type() == Item::NULL_ITEM) + { + // MCOL-793 Do nothing for an ORDER BY NULL + } + else if (ord_item->type() == Item::SUM_FUNC_ITEM) + { + Item_sum* ifp = (Item_sum*)(*(ordercol->item)); + ReturnedColumn* fc = buildAggregateColumn(ifp, gwi); + + for (uint32_t i = 0; i < gwi.returnedCols.size(); i++) + { + if (fc->operator==(gwi.returnedCols[i].get())) + { + ostringstream oss; + oss << i + 1; + ord_cols += oss.str(); + break; + } + } + + //continue; + } + // @bug 3518. if order by clause = selected column, use position. + else if (ord_item->name && ord_item->type() == Item::FIELD_ITEM) + { + Item_field* field = reinterpret_cast(ord_item); + string fullname; + + if (field->db_name) + fullname += string(field->db_name) + "."; + + if (field->table_name) + fullname += string(field->table_name) + "."; + + if (field->field_name) + fullname += string(field->field_name); + + uint32_t i = 0; + + for (i = 0; i < gwi.returnedCols.size(); i++) + { + SimpleColumn* sc = dynamic_cast(gwi.returnedCols[i].get()); + + if (sc && ((Item_field*)ord_item)->cached_table && + (strcasecmp(getViewName(((Item_field*)ord_item)->cached_table).c_str(), sc->viewName().c_str()) != 0)) + continue; + + if (strcasecmp(fullname.c_str(), gwi.returnedCols[i]->alias().c_str()) == 0 || + strcasecmp(ord_item->name, gwi.returnedCols[i]->alias().c_str()) == 0) + { + ostringstream oss; + oss << i + 1; + ord_cols += oss.str(); + break; + } + } + + if (i == gwi.returnedCols.size()) + ord_cols += string(" `") + escapeBackTick(ord_item->name) + '`'; + } + + else if (ord_item->name) + { + // for union order by 1 case. For unknown reason, it doesn't show in_field_list + if (ord_item->type() == Item::INT_ITEM) + { + ord_cols += ord_item->name; + } + else if (ord_item->type() == Item::SUBSELECT_ITEM) + { + string emsg = IDBErrorInfo::instance()->errorMsg(ERR_NON_SUPPORT_ORDER_BY); + setError(gwi.thd, ER_CHECK_NOT_IMPLEMENTED, emsg, gwi); + return ER_CHECK_NOT_IMPLEMENTED; + } + else + { + ord_cols += string(" `") + escapeBackTick(ord_item->name) + '`'; + } + } + else if (ord_item->type() == Item::FUNC_ITEM) + { + // @bug5636. check if this order by column is on the select list + ReturnedColumn* rc = buildFunctionColumn((Item_func*)(ord_item), gwi, gwi.fatalParseError); + + for (uint32_t i = 0; i < gwi.returnedCols.size(); i++) + { + if (rc && rc->operator==(gwi.returnedCols[i].get())) + { + ostringstream oss; + oss << i + 1; + ord_cols += oss.str(); + break; + } + } + } + else + { + String str; + ord_item->print(&str, QT_INFINIDB_NO_QUOTE); + ord_cols += string(str.c_ptr()); + } + + if (ordercol->direction != ORDER::ORDER_ASC) + ord_cols += " desc"; + } + } + + if (ord_cols.length() > 0) // has order by + { + gwi.thd->infinidb_vtable.has_order_by = true; + csep->hasOrderBy(true); + ord_cols = " order by " + ord_cols; + select_query += ord_cols; + } + } + +// MCOL-1052 LIMIT +/* + if (unionSel || gwi.subSelectType != CalpontSelectExecutionPlan::MAIN_SELECT) + { + if (select_lex.master_unit()->global_parameters()->explicit_limit) + { + if (select_lex.master_unit()->global_parameters()->offset_limit) + { + Item_int* offset = (Item_int*)select_lex.master_unit()->global_parameters()->offset_limit; + csep->limitStart(offset->val_int()); + } + + if (select_lex.master_unit()->global_parameters()->select_limit) + { + Item_int* select = (Item_int*)select_lex.master_unit()->global_parameters()->select_limit; + csep->limitNum(select->val_int()); + } + + if (unionSel && gwi.subSelectType == CalpontSelectExecutionPlan::MAIN_SELECT) + { + ostringstream limit; + limit << " limit "; + limit << csep->limitStart() << ", "; + limit << csep->limitNum(); + select_query += limit.str(); + } + } + } + else if (isUnion && select_lex.explicit_limit) + { + if (select_lex.braces) + { + if (select_lex.offset_limit) + csep->limitStart(((Item_int*)select_lex.offset_limit)->val_int()); + + if (select_lex.select_limit) + csep->limitNum(((Item_int*)select_lex.select_limit)->val_int()); + } + } + else if (select_lex.explicit_limit) + { + uint32_t limitOffset = 0; + uint32_t limitNum = std::numeric_limits::max(); + + if (join) + { +#if MYSQL_VERSION_ID >= 50172 + + // @bug5729. After upgrade, join->unit sometimes is uninitialized pointer + // (not null though) and will cause seg fault. Prefer checking + // select_lex->offset_limit if not null. + if (join->select_lex && + join->select_lex->offset_limit && + join->select_lex->offset_limit->fixed && + join->select_lex->select_limit && + join->select_lex->select_limit->fixed) + { + limitOffset = join->select_lex->offset_limit->val_int(); + limitNum = join->select_lex->select_limit->val_int(); + } + + else if (join->unit) + { + limitOffset = join->unit->offset_limit_cnt; + limitNum = join->unit->select_limit_cnt - limitOffset; + } + +#else + limitOffset = (join->unit)->offset_limit_cnt; + limitNum = (join->unit)->select_limit_cnt - (join->unit)->offset_limit_cnt; +#endif + } + else + { + if (select_lex.master_unit()->global_parameters()->offset_limit) + { + Item_int* offset = (Item_int*)select_lex.master_unit()->global_parameters()->offset_limit; + limitOffset = offset->val_int(); + } + + if (select_lex.master_unit()->global_parameters()->select_limit) + { + Item_int* select = (Item_int*)select_lex.master_unit()->global_parameters()->select_limit; + limitNum = select->val_int(); + } + } + + // relate to bug4848. let mysql drive limit when limit session variable set. + // do not set in csep. @bug5096. ignore session limit setting for dml + if ((gwi.thd->variables.select_limit == (uint64_t) - 1 || + (gwi.thd->variables.select_limit != (uint64_t) - 1 && + gwi.thd->infinidb_vtable.vtable_state != THD::INFINIDB_CREATE_VTABLE)) && + !csep->hasOrderBy()) + { + csep->limitStart(limitOffset); + csep->limitNum(limitNum); + } + else + { + ostringstream limit; + limit << " limit " << limitOffset << ", " << limitNum; + select_query += limit.str(); + } + } + gwi.thd->infinidb_vtable.select_vtable_query.free(); + gwi.thd->infinidb_vtable.select_vtable_query.append(select_query.c_str(), select_query.length()); + + // We don't currently support limit with correlated subquery + if (csep->limitNum() != (uint64_t) - 1 && + gwi.subQuery && !gwi.correlatedTbNameVec.empty()) + { + gwi.fatalParseError = true; + gwi.parseErrorText = IDBErrorInfo::instance()->errorMsg(ERR_NON_SUPPORT_LIMIT_SUB); + setError(gwi.thd, ER_INTERNAL_ERROR, gwi.parseErrorText, gwi); + return ER_CHECK_NOT_IMPLEMENTED; + } + */ + } + + + + if ( gi.groupByDistinct ) + csep->distinct(true); + + // add the smallest column to count(*) parm. + // select constant in subquery case + std::vector::iterator coliter; + + if (!minSc) + { + if (!gwi.returnedCols.empty()) + minSc = gwi.returnedCols[0]; + else if (!gwi.additionalRetCols.empty()) + minSc = gwi.additionalRetCols[0]; + } + + // @bug3523, count(*) on subquery always pick column[0]. + SimpleColumn* sc = dynamic_cast(minSc.get()); + + if (sc && sc->schemaName().empty()) + { + if (gwi.derivedTbList.size() >= 1) + { + SimpleColumn* sc1 = new SimpleColumn(); + sc1->columnName(sc->columnName()); + sc1->tableName(sc->tableName()); + sc1->tableAlias(sc->tableAlias()); + sc1->viewName(lower(sc->viewName())); + sc1->colPosition(0); + minSc.reset(sc1); + } + } + + for (coliter = gwi.count_asterisk_list.begin(); coliter != gwi.count_asterisk_list.end(); ++coliter) + { + // @bug5977 @note should never throw this, but checking just in case. + // When ExeMgr fix is ready, this should not error out... + if (dynamic_cast(minSc.get())) + { + gwi.fatalParseError = true; + gwi.parseErrorText = "No project column found for aggregate function"; + setError(gwi.thd, ER_INTERNAL_ERROR, gwi.parseErrorText, gwi); + return ER_CHECK_NOT_IMPLEMENTED; + } + + (*coliter)->functionParms(minSc); + } + + std::vector::iterator funciter; + + SPTP sptp(new ParseTree(minSc.get()->clone())); + + for (funciter = gwi.no_parm_func_list.begin(); funciter != gwi.no_parm_func_list.end(); ++funciter) + { + FunctionParm funcParms = (*funciter)->functionParms(); + funcParms.push_back(sptp); + (*funciter)->functionParms(funcParms); + } + + // set sequence# for subquery localCols + for (uint32_t i = 0; i < gwi.localCols.size(); i++) + gwi.localCols[i]->sequence(i); + + // append additionalRetCols to returnedCols + gwi.returnedCols.insert(gwi.returnedCols.begin(), gwi.additionalRetCols.begin(), + gwi.additionalRetCols.end()); + + csep->groupByCols(gwi.groupByCols); + csep->orderByCols(gwi.orderByCols); + csep->returnedCols(gwi.returnedCols); + csep->columnMap(gwi.columnMap); + csep->having(havingFilter); + csep->derivedTableList(gwi.derivedTbList); + csep->selectSubList(selectSubList); + csep->subSelectList(gwi.subselectList); + gwi.thd->infinidb_vtable.duplicate_field_name = false; + clearStacks(gwi); + return 0; +} + } diff --git a/dbcon/mysql/ha_calpont_impl.cpp b/dbcon/mysql/ha_calpont_impl.cpp index 5d8e91e08..c8a02713a 100644 --- a/dbcon/mysql/ha_calpont_impl.cpp +++ b/dbcon/mysql/ha_calpont_impl.cpp @@ -5532,6 +5532,7 @@ int ha_calpont_impl_group_by_init(ha_calpont_group_by_handler* group_hand, TABLE sm::tableid_t tableid = 0; cal_table_info ti; + cal_group_info gi; sm::cpsm_conhdl_t* hndl; SCSEP csep; @@ -5602,10 +5603,19 @@ int ha_calpont_impl_group_by_init(ha_calpont_group_by_handler* group_hand, TABLE ti.csep->traceFlags(ci->traceFlags); ti.msTablePtr = group_hand->table_list->table; - ti.groupByFields = group_hand->fields; + //ti.groupByTables = group_hand->table_list; + //ti.groupByFields = group_hand->fields; + gi.groupByTables = group_hand->table_list; + gi.groupByFields = group_hand->select; + gi.groupByWhere = group_hand->where; + gi.groupByGroup = group_hand->group_by; + gi.groupByOrder = group_hand->order_by; + gi.groupByHaving = group_hand->having; + gi.groupByDistinct = group_hand->distinct; + // send plan whenever group_init is called - cp_get_group_plan(thd, ti.csep, ti); + cp_get_group_plan(thd, ti.csep, ti, gi); } IDEBUG( cerr << tableName << " send plan:" << endl ); diff --git a/dbcon/mysql/ha_calpont_impl_if.h b/dbcon/mysql/ha_calpont_impl_if.h index 71fd0b35f..7c12c8b2c 100644 --- a/dbcon/mysql/ha_calpont_impl_if.h +++ b/dbcon/mysql/ha_calpont_impl_if.h @@ -178,8 +178,7 @@ struct cal_table_info msTablePtr(0), conn_hndl(0), condInfo(0), - moreRows(false), - groupByFields(0) + moreRows(false) { } ~cal_table_info() {} sm::cpsm_tplh_t* tpl_ctx; @@ -190,7 +189,27 @@ struct cal_table_info gp_walk_info* condInfo; execplan::SCSEP csep; bool moreRows; //are there more rows to consume (b/c of limit) - List *groupByFields; // MCOL-1052 For CSEP generation +}; + +struct cal_group_info +{ + cal_group_info() : groupByFields(0), + groupByTables(0), + groupByWhere(0), + groupByGroup(0), + groupByOrder(0), + groupByHaving(0), + groupByDistinct(false) + { } + ~cal_group_info() { } + + List* groupByFields; // MCOL-1052 SELECT + TABLE_LIST* groupByTables; // MCOL-1052 FROM + Item* groupByWhere; // MCOL-1052 WHERE + ORDER* groupByGroup; // MCOL-1052 GROUP BY + ORDER* groupByOrder; // MCOL-1052 ORDER BY + Item* groupByHaving; // MCOL-1052 HAVING + bool groupByDistinct; //MCOL-1052 DISTINCT }; typedef std::tr1::unordered_map CalTableMap; @@ -299,8 +318,9 @@ const std::string infinidb_err_msg = "\nThe query includes syntax that is not su int cp_get_plan(THD* thd, execplan::SCSEP& csep); int cp_get_table_plan(THD* thd, execplan::SCSEP& csep, cal_impl_if::cal_table_info& ti); -int cp_get_group_plan(THD* thd, execplan::SCSEP& csep, cal_impl_if::cal_table_info& ti); +int cp_get_group_plan(THD* thd, execplan::SCSEP& csep, cal_impl_if::cal_table_info& ti,cal_impl_if::cal_group_info& gi); int getSelectPlan(gp_walk_info& gwi, SELECT_LEX& select_lex, execplan::SCSEP& csep, bool isUnion = false); +int getGroupPlan(gp_walk_info& gwi, SELECT_LEX& select_lex, execplan::SCSEP& csep,cal_group_info& gi, bool isUnion = false); void setError(THD* thd, uint32_t errcode, const std::string errmsg, gp_walk_info* gwi); void setError(THD* thd, uint32_t errcode, const std::string errmsg); void gp_walk(const Item* item, void* arg); From d562caecbae4939304432d9d9d3e768047550612 Mon Sep 17 00:00:00 2001 From: Roman Nozdrin Date: Sat, 31 Mar 2018 22:38:18 +0300 Subject: [PATCH 07/57] MCOL-1052 Generate and send CSEP to and receive set data from ExeMgr. --- dbcon/mysql/ha_calpont.cpp | 19 +- dbcon/mysql/ha_calpont_execplan.cpp | 300 +++++------ dbcon/mysql/ha_calpont_impl.cpp | 762 +++++++++++++++++++++++++++- dbcon/mysql/ha_calpont_impl_if.h | 2 +- 4 files changed, 913 insertions(+), 170 deletions(-) diff --git a/dbcon/mysql/ha_calpont.cpp b/dbcon/mysql/ha_calpont.cpp index e0e835c4b..ae0850109 100644 --- a/dbcon/mysql/ha_calpont.cpp +++ b/dbcon/mysql/ha_calpont.cpp @@ -1172,7 +1172,11 @@ int ha_calpont_group_by_handler::init_scan() { DBUG_ENTER("ha_calpont_group_by_handler::init_scan"); + // MCOL-1052 + THD::infinidb_state oldState = thd->infinidb_vtable.vtable_state; + thd->infinidb_vtable.vtable_state = THD::INFINIDB_CREATE_VTABLE; int rc = ha_calpont_impl_group_by_init(this, table); + thd->infinidb_vtable.vtable_state = oldState; DBUG_RETURN(rc); } @@ -1184,14 +1188,15 @@ int ha_calpont_group_by_handler::next_row() DBUG_ENTER("ha_calpont_group_by_handler::next_row"); int rc = ha_calpont_impl_group_by_next(this, table); + + +// first_row= 0; + //Field *field = *(table->field); + //field->store(5LL, 1); + //field->set_notnull(); + //return(0); + DBUG_RETURN(rc); -/* - first_row= 0; - Field *field = *(table->field); - field->store(5LL, 1); - field->set_notnull(); - return(0); -*/ } int ha_calpont_group_by_handler::end_scan() diff --git a/dbcon/mysql/ha_calpont_execplan.cpp b/dbcon/mysql/ha_calpont_execplan.cpp index 5bed8e88c..6a8b1ac95 100644 --- a/dbcon/mysql/ha_calpont_execplan.cpp +++ b/dbcon/mysql/ha_calpont_execplan.cpp @@ -221,7 +221,7 @@ void debug_walk(const Item* item, void* arg) case Item::FIELD_ITEM: { Item_field* ifp = (Item_field*)item; - cout << "FIELD_ITEM: " << (ifp->db_name ? ifp->db_name : "") << '.' << bestTableName(ifp) << + cerr << "FIELD_ITEM: " << (ifp->db_name ? ifp->db_name : "") << '.' << bestTableName(ifp) << '.' << ifp->field_name << endl; break; } @@ -229,10 +229,10 @@ void debug_walk(const Item* item, void* arg) case Item::INT_ITEM: { Item_int* iip = (Item_int*)item; - cout << "INT_ITEM: "; + cerr << "INT_ITEM: "; - if (iip->name) cout << iip->name << " (from name string)" << endl; - else cout << iip->val_int() << endl; + if (iip->name) cerr << iip->name << " (from name string)" << endl; + else cerr << iip->val_int() << endl; break; } @@ -243,19 +243,19 @@ void debug_walk(const Item* item, void* arg) String val, *str = isp->val_str(&val); string valStr; valStr.assign(str->ptr(), str->length()); - cout << "STRING_ITEM: >" << valStr << '<' << endl; + cerr << "STRING_ITEM: >" << valStr << '<' << endl; break; } case Item::REAL_ITEM: { - cout << "REAL_ITEM" << endl; + cerr << "REAL_ITEM" << endl; break; } case Item::DECIMAL_ITEM: { - cout << "DECIMAL_ITEM" << endl; + cerr << "DECIMAL_ITEM" << endl; break; } @@ -263,85 +263,85 @@ void debug_walk(const Item* item, void* arg) { Item_func* ifp = (Item_func*)item; Item_func_opt_neg* inp; - cout << "FUNC_ITEM: "; + cerr << "FUNC_ITEM: "; switch (ifp->functype()) { case Item_func::UNKNOWN_FUNC: // 0 - cout << ifp->func_name() << " (" << ifp->functype() << ")" << endl; + cerr << ifp->func_name() << " (" << ifp->functype() << ")" << endl; break; case Item_func::GT_FUNC: // 7 - cout << '>' << " (" << ifp->functype() << ")" << endl; + cerr << '>' << " (" << ifp->functype() << ")" << endl; break; case Item_func::EQ_FUNC: // 1 - cout << '=' << " (" << ifp->functype() << ")" << endl; + cerr << '=' << " (" << ifp->functype() << ")" << endl; break; case Item_func::GE_FUNC: - cout << ">=" << " (" << ifp->functype() << ")" << endl; + cerr << ">=" << " (" << ifp->functype() << ")" << endl; break; case Item_func::LE_FUNC: - cout << "<=" << " (" << ifp->functype() << ")" << endl; + cerr << "<=" << " (" << ifp->functype() << ")" << endl; break; case Item_func::LT_FUNC: - cout << '<' << " (" << ifp->functype() << ")" << endl; + cerr << '<' << " (" << ifp->functype() << ")" << endl; break; case Item_func::NE_FUNC: - cout << "<>" << " (" << ifp->functype() << ")" << endl; + cerr << "<>" << " (" << ifp->functype() << ")" << endl; break; case Item_func::NEG_FUNC: // 45 - cout << "unary minus" << " (" << ifp->functype() << ")" << endl; + cerr << "unary minus" << " (" << ifp->functype() << ")" << endl; break; case Item_func::IN_FUNC: // 16 inp = (Item_func_opt_neg*)ifp; - if (inp->negated) cout << "not "; + if (inp->negated) cerr << "not "; - cout << "in" << " (" << ifp->functype() << ")" << endl; + cerr << "in" << " (" << ifp->functype() << ")" << endl; break; case Item_func::BETWEEN: inp = (Item_func_opt_neg*)ifp; - if (inp->negated) cout << "not "; + if (inp->negated) cerr << "not "; - cout << "between" << " (" << ifp->functype() << ")" << endl; + cerr << "between" << " (" << ifp->functype() << ")" << endl; break; case Item_func::ISNULL_FUNC: // 10 - cout << "is null" << " (" << ifp->functype() << ")" << endl; + cerr << "is null" << " (" << ifp->functype() << ")" << endl; break; case Item_func::ISNOTNULL_FUNC: // 11 - cout << "is not null" << " (" << ifp->functype() << ")" << endl; + cerr << "is not null" << " (" << ifp->functype() << ")" << endl; break; case Item_func::NOT_ALL_FUNC: - cout << "not_all" << " (" << ifp->functype() << ")" << endl; + cerr << "not_all" << " (" << ifp->functype() << ")" << endl; break; case Item_func::NOT_FUNC: - cout << "not_func" << " (" << ifp->functype() << ")" << endl; + cerr << "not_func" << " (" << ifp->functype() << ")" << endl; break; case Item_func::TRIG_COND_FUNC: - cout << "trig_cond_func" << " (" << ifp->functype() << ")" << endl; + cerr << "trig_cond_func" << " (" << ifp->functype() << ")" << endl; break; case Item_func::ISNOTNULLTEST_FUNC: - cout << "isnotnulltest_func" << " (" << ifp->functype() << ")" << endl; + cerr << "isnotnulltest_func" << " (" << ifp->functype() << ")" << endl; break; case Item_func::MULT_EQUAL_FUNC: { - cout << "mult_equal_func:" << " (" << ifp->functype() << ")" << endl; + cerr << "mult_equal_func:" << " (" << ifp->functype() << ")" << endl; Item_equal* item_eq = (Item_equal*)ifp; Item_equal_fields_iterator it(*item_eq); Item* item; @@ -349,142 +349,142 @@ void debug_walk(const Item* item, void* arg) while ((item = it++)) { Field* equal_field = it.get_curr_field(); - cout << equal_field->field_name << endl; + cerr << equal_field->field_name << endl; } break; } case Item_func::EQUAL_FUNC: - cout << "equal func" << " (" << ifp->functype() << ")" << endl; + cerr << "equal func" << " (" << ifp->functype() << ")" << endl; break; case Item_func::FT_FUNC: - cout << "ft func" << " (" << ifp->functype() << ")" << endl; + cerr << "ft func" << " (" << ifp->functype() << ")" << endl; break; case Item_func::LIKE_FUNC: - cout << "like func" << " (" << ifp->functype() << ")" << endl; + cerr << "like func" << " (" << ifp->functype() << ")" << endl; break; case Item_func::COND_AND_FUNC: - cout << "cond and func" << " (" << ifp->functype() << ")" << endl; + cerr << "cond and func" << " (" << ifp->functype() << ")" << endl; break; case Item_func::COND_OR_FUNC: - cout << "cond or func" << " (" << ifp->functype() << ")" << endl; + cerr << "cond or func" << " (" << ifp->functype() << ")" << endl; break; case Item_func::XOR_FUNC: - cout << "xor func" << " (" << ifp->functype() << ")" << endl; + cerr << "xor func" << " (" << ifp->functype() << ")" << endl; break; case Item_func::INTERVAL_FUNC: - cout << "interval func" << " (" << ifp->functype() << ")" << endl; + cerr << "interval func" << " (" << ifp->functype() << ")" << endl; break; case Item_func::SP_EQUALS_FUNC: - cout << "sp equals func" << " (" << ifp->functype() << ")" << endl; + cerr << "sp equals func" << " (" << ifp->functype() << ")" << endl; break; case Item_func::SP_DISJOINT_FUNC: - cout << "sp disjoint func" << " (" << ifp->functype() << ")" << endl; + cerr << "sp disjoint func" << " (" << ifp->functype() << ")" << endl; break; case Item_func::SP_INTERSECTS_FUNC: - cout << "sp intersects func" << " (" << ifp->functype() << ")" << endl; + cerr << "sp intersects func" << " (" << ifp->functype() << ")" << endl; break; case Item_func::SP_TOUCHES_FUNC: - cout << "sp touches func" << " (" << ifp->functype() << ")" << endl; + cerr << "sp touches func" << " (" << ifp->functype() << ")" << endl; break; case Item_func::SP_CROSSES_FUNC: - cout << "sp crosses func" << " (" << ifp->functype() << ")" << endl; + cerr << "sp crosses func" << " (" << ifp->functype() << ")" << endl; break; case Item_func::SP_WITHIN_FUNC: - cout << "sp within func" << " (" << ifp->functype() << ")" << endl; + cerr << "sp within func" << " (" << ifp->functype() << ")" << endl; break; case Item_func::SP_CONTAINS_FUNC: - cout << "sp contains func" << " (" << ifp->functype() << ")" << endl; + cerr << "sp contains func" << " (" << ifp->functype() << ")" << endl; break; case Item_func::SP_OVERLAPS_FUNC: - cout << "sp overlaps func" << " (" << ifp->functype() << ")" << endl; + cerr << "sp overlaps func" << " (" << ifp->functype() << ")" << endl; break; case Item_func::SP_STARTPOINT: - cout << "sp startpoint func" << " (" << ifp->functype() << ")" << endl; + cerr << "sp startpoint func" << " (" << ifp->functype() << ")" << endl; break; case Item_func::SP_ENDPOINT: - cout << "sp endpoint func" << " (" << ifp->functype() << ")" << endl; + cerr << "sp endpoint func" << " (" << ifp->functype() << ")" << endl; break; case Item_func::SP_EXTERIORRING: - cout << "sp exteriorring func" << " (" << ifp->functype() << ")" << endl; + cerr << "sp exteriorring func" << " (" << ifp->functype() << ")" << endl; break; case Item_func::SP_POINTN: - cout << "sp pointn func" << " (" << ifp->functype() << ")" << endl; + cerr << "sp pointn func" << " (" << ifp->functype() << ")" << endl; break; case Item_func::SP_GEOMETRYN: - cout << "sp geometryn func" << " (" << ifp->functype() << ")" << endl; + cerr << "sp geometryn func" << " (" << ifp->functype() << ")" << endl; break; case Item_func::SP_INTERIORRINGN: - cout << "sp exteriorringn func" << " (" << ifp->functype() << ")" << endl; + cerr << "sp exteriorringn func" << " (" << ifp->functype() << ")" << endl; break; case Item_func::SP_RELATE_FUNC: - cout << "sp relate func" << " (" << ifp->functype() << ")" << endl; + cerr << "sp relate func" << " (" << ifp->functype() << ")" << endl; break; case Item_func::NOW_FUNC: - cout << "now func" << " (" << ifp->functype() << ")" << endl; + cerr << "now func" << " (" << ifp->functype() << ")" << endl; break; case Item_func::SUSERVAR_FUNC: - cout << "suservar func" << " (" << ifp->functype() << ")" << endl; + cerr << "suservar func" << " (" << ifp->functype() << ")" << endl; break; case Item_func::GUSERVAR_FUNC: - cout << "guservar func" << " (" << ifp->functype() << ")" << endl; + cerr << "guservar func" << " (" << ifp->functype() << ")" << endl; break; case Item_func::COLLATE_FUNC: - cout << "collate func" << " (" << ifp->functype() << ")" << endl; + cerr << "collate func" << " (" << ifp->functype() << ")" << endl; break; case Item_func::EXTRACT_FUNC: - cout << "extract func" << " (" << ifp->functype() << ")" << endl; + cerr << "extract func" << " (" << ifp->functype() << ")" << endl; break; case Item_func::CHAR_TYPECAST_FUNC: - cout << "char typecast func" << " (" << ifp->functype() << ")" << endl; + cerr << "char typecast func" << " (" << ifp->functype() << ")" << endl; break; case Item_func::FUNC_SP: - cout << "func sp func" << " (" << ifp->functype() << ")" << endl; + cerr << "func sp func" << " (" << ifp->functype() << ")" << endl; break; case Item_func::UDF_FUNC: - cout << "udf func" << " (" << ifp->functype() << ")" << endl; + cerr << "udf func" << " (" << ifp->functype() << ")" << endl; break; case Item_func::GSYSVAR_FUNC: - cout << "gsysvar func" << " (" << ifp->functype() << ")" << endl; + cerr << "gsysvar func" << " (" << ifp->functype() << ")" << endl; break; case Item_func::DYNCOL_FUNC: - cout << "dyncol func" << " (" << ifp->functype() << ")" << endl; + cerr << "dyncol func" << " (" << ifp->functype() << ")" << endl; break; default: - cout << "type=" << ifp->functype() << endl; + cerr << "type=" << ifp->functype() << endl; break; } @@ -494,7 +494,7 @@ void debug_walk(const Item* item, void* arg) case Item::COND_ITEM: { Item_cond* icp = (Item_cond*)item; - cout << "COND_ITEM: " << icp->func_name() << endl; + cerr << "COND_ITEM: " << icp->func_name() << endl; break; } @@ -511,39 +511,39 @@ void debug_walk(const Item* item, void* arg) switch (isp->sum_func()) { case Item_sum::SUM_FUNC: - cout << "SUM_FUNC: " << item_name << endl; + cerr << "SUM_FUNC: " << item_name << endl; break; case Item_sum::SUM_DISTINCT_FUNC: - cout << "SUM_DISTINCT_FUNC: " << item_name << endl; + cerr << "SUM_DISTINCT_FUNC: " << item_name << endl; break; case Item_sum::AVG_FUNC: - cout << "AVG_FUNC: " << item_name << endl; + cerr << "AVG_FUNC: " << item_name << endl; break; case Item_sum::COUNT_FUNC: - cout << "COUNT_FUNC: " << item_name << endl; + cerr << "COUNT_FUNC: " << item_name << endl; break; case Item_sum::COUNT_DISTINCT_FUNC: - cout << "COUNT_DISTINCT_FUNC: " << item_name << endl; + cerr << "COUNT_DISTINCT_FUNC: " << item_name << endl; break; case Item_sum::MIN_FUNC: - cout << "MIN_FUNC: " << item_name << endl; + cerr << "MIN_FUNC: " << item_name << endl; break; case Item_sum::MAX_FUNC: - cout << "MAX_FUNC: " << item_name << endl; + cerr << "MAX_FUNC: " << item_name << endl; break; case Item_sum::UDF_SUM_FUNC: - cout << "UDAF_FUNC: " << item_name << endl; + cerr << "UDAF_FUNC: " << item_name << endl; break; default: - cout << "SUM_FUNC_ITEM type=" << isp->sum_func() << endl; + cerr << "SUM_FUNC_ITEM type=" << isp->sum_func() << endl; break; } @@ -553,24 +553,24 @@ void debug_walk(const Item* item, void* arg) case Item::SUBSELECT_ITEM: { Item_subselect* sub = (Item_subselect*)item; - cout << "SUBSELECT Item: "; + cerr << "SUBSELECT Item: "; switch (sub->substype()) { case Item_subselect::EXISTS_SUBS: - cout << "EXISTS"; + cerr << "EXISTS"; break; case Item_subselect::IN_SUBS: - cout << "IN"; + cerr << "IN"; break; default: - cout << sub->substype(); + cerr << sub->substype(); break; } - cout << endl; + cerr << endl; JOIN* join = sub->get_select_lex()->join; if (join) @@ -581,7 +581,7 @@ void debug_walk(const Item* item, void* arg) cond->traverse_cond(debug_walk, arg, Item::POSTFIX); } - cout << "Finish subselect item traversing" << endl; + cerr << "Finish subselect item traversing" << endl; break; } @@ -599,14 +599,14 @@ void debug_walk(const Item* item, void* arg) //ifp->cached_table->select_lex->select_number gives the select level. // could be used on alias. // could also be used to tell correlated join (equal level). - cout << "CACHED REF FIELD_ITEM: " << ifp->db_name << '.' << bestTableName(ifp) << + cerr << "CACHED REF FIELD_ITEM: " << ifp->db_name << '.' << bestTableName(ifp) << '.' << ifp->field_name << endl; break; } else if (field->type() == Item::FUNC_ITEM) { Item_func* ifp = (Item_func*)field; - cout << "CACHED REF FUNC_ITEM " << ifp->func_name() << endl; + cerr << "CACHED REF FUNC_ITEM " << ifp->func_name() << endl; } else if (field->type() == Item::REF_ITEM) { @@ -686,34 +686,34 @@ void debug_walk(const Item* item, void* arg) } } - cout << "CACHED REF_ITEM: ref type " << refType.c_str() << " real type " << realType.c_str() << endl; + cerr << "CACHED REF_ITEM: ref type " << refType.c_str() << " real type " << realType.c_str() << endl; break; } else { - cout << "REF_ITEM with CACHE_ITEM type unknown " << field->type() << endl; + cerr << "REF_ITEM with CACHE_ITEM type unknown " << field->type() << endl; } } else if (ref->real_item()->type() == Item::FIELD_ITEM) { Item_field* ifp = (Item_field*)ref->real_item(); - cout << "REF FIELD_ITEM: " << ifp->db_name << '.' << bestTableName(ifp) << '.' << + cerr << "REF FIELD_ITEM: " << ifp->db_name << '.' << bestTableName(ifp) << '.' << ifp->field_name << endl; break; } else if (ref->real_item()->type() == Item::FUNC_ITEM) { Item_func* ifp = (Item_func*)ref->real_item(); - cout << "REF FUNC_ITEM " << ifp->func_name() << endl; + cerr << "REF FUNC_ITEM " << ifp->func_name() << endl; } else if (ref->real_item()->type() == Item::WINDOW_FUNC_ITEM) { Item_window_func* ifp = (Item_window_func*)ref->real_item(); - cout << "REF WINDOW_FUNC_ITEM " << ifp->window_func()->func_name() << endl; + cerr << "REF WINDOW_FUNC_ITEM " << ifp->window_func()->func_name() << endl; } else { - cout << "UNKNOWN REF ITEM type " << ref->real_item()->type() << endl; + cerr << "UNKNOWN REF ITEM type " << ref->real_item()->type() << endl; } break; @@ -722,7 +722,7 @@ void debug_walk(const Item* item, void* arg) case Item::ROW_ITEM: { Item_row* row = (Item_row*)item; - cout << "ROW_ITEM: " << endl; + cerr << "ROW_ITEM: " << endl; for (uint32_t i = 0; i < row->cols(); i++) debug_walk(row->element_index(i), 0); @@ -732,7 +732,7 @@ void debug_walk(const Item* item, void* arg) case Item::EXPR_CACHE_ITEM: { - cout << "Expr Cache Item" << endl; + cerr << "Expr Cache Item" << endl; ((Item_cache_wrapper*)item)->get_orig_item()->traverse_cond(debug_walk, arg, Item::POSTFIX); break; } @@ -747,27 +747,27 @@ void debug_walk(const Item* item, void* arg) switch (item->result_type()) { case STRING_RESULT: - cout << "CACHE_STRING_ITEM" << endl; + cerr << "CACHE_STRING_ITEM" << endl; break; case REAL_RESULT: - cout << "CACHE_REAL_ITEM " << isp->val_real() << endl; + cerr << "CACHE_REAL_ITEM " << isp->val_real() << endl; break; case INT_RESULT: - cout << "CACHE_INT_ITEM " << isp->val_int() << endl; + cerr << "CACHE_INT_ITEM " << isp->val_int() << endl; break; case ROW_RESULT: - cout << "CACHE_ROW_ITEM" << endl; + cerr << "CACHE_ROW_ITEM" << endl; break; case DECIMAL_RESULT: - cout << "CACHE_DECIMAL_ITEM " << isp->val_decimal() << endl; + cerr << "CACHE_DECIMAL_ITEM " << isp->val_decimal() << endl; break; default: - cout << "CACHE_UNKNOWN_ITEM" << endl; + cerr << "CACHE_UNKNOWN_ITEM" << endl; break; } @@ -780,7 +780,7 @@ void debug_walk(const Item* item, void* arg) //ifp->cached_table->select_lex->select_number gives the select level. // could be used on alias. // could also be used to tell correlated join (equal level). - cout << "CACHED FIELD_ITEM: " << ifp->db_name << '.' << bestTableName(ifp) << + cerr << "CACHED FIELD_ITEM: " << ifp->db_name << '.' << bestTableName(ifp) << '.' << ifp->field_name << endl; break; } @@ -862,18 +862,18 @@ void debug_walk(const Item* item, void* arg) } } - cout << "CACHE_ITEM ref type " << refType.c_str() << " real type " << realType.c_str() << endl; + cerr << "CACHE_ITEM ref type " << refType.c_str() << " real type " << realType.c_str() << endl; break; } else if (field->type() == Item::FUNC_ITEM) { Item_func* ifp = (Item_func*)field; - cout << "CACHE_ITEM FUNC_ITEM " << ifp->func_name() << endl; + cerr << "CACHE_ITEM FUNC_ITEM " << ifp->func_name() << endl; break; } else { - cout << "CACHE_ITEM type unknown " << field->type() << endl; + cerr << "CACHE_ITEM type unknown " << field->type() << endl; } break; @@ -884,12 +884,12 @@ void debug_walk(const Item* item, void* arg) String val, *str = NULL; Item_temporal_literal* itp = (Item_temporal_literal*)item; str = itp->val_str(&val); - cout << "DATE ITEM: "; + cerr << "DATE ITEM: "; if (str) - cout << ": (" << str->ptr() << ')' << endl; + cerr << ": (" << str->ptr() << ')' << endl; else - cout << ": " << endl; + cerr << ": " << endl; break; } @@ -897,13 +897,13 @@ void debug_walk(const Item* item, void* arg) case Item::WINDOW_FUNC_ITEM: { Item_window_func* ifp = (Item_window_func*)item; - cout << "Window Function Item " << ifp->window_func()->func_name() << endl; + cerr << "Window Function Item " << ifp->window_func()->func_name() << endl; break; } default: { - cout << "UNKNOWN_ITEM type " << item->type() << endl; + cerr << "UNKNOWN_ITEM type " << item->type() << endl; break; } } @@ -1015,11 +1015,11 @@ uint32_t buildOuterJoin(gp_walk_info& gwi, SELECT_LEX& select_lex) #ifdef DEBUG_WALK_COND if (table_ptr->alias) - cout << table_ptr->alias ; + cerr << table_ptr->alias ; else if (table_ptr->alias) - cout << table_ptr->alias; + cerr << table_ptr->alias; - cout << " outer table expression: " << endl; + cerr << " outer table expression: " << endl; expr->traverse_cond(debug_walk, &gwi_outer, Item::POSTFIX); #endif expr->traverse_cond(gp_walk, &gwi_outer, Item::POSTFIX); @@ -1053,13 +1053,13 @@ uint32_t buildOuterJoin(gp_walk_info& gwi, SELECT_LEX& select_lex) Item_cond* expr = reinterpret_cast(table_ptr->embedding->on_expr); #ifdef DEBUG_WALK_COND - cout << "inner tables: " << endl; + cerr << "inner tables: " << endl; set::const_iterator it; for (it = gwi_outer.innerTables.begin(); it != gwi_outer.innerTables.end(); ++it) - cout << (*it) << " "; + cerr << (*it) << " "; - cout << endl; + cerr << endl; expr->traverse_cond(debug_walk, &gwi_outer, Item::POSTFIX); #endif expr->traverse_cond(gp_walk, &gwi_outer, Item::POSTFIX); @@ -1485,7 +1485,7 @@ bool buildPredicateItem(Item_func* ifp, gp_walk_info* gwip) // @bug5811. This filter string is for cross engine to use. // Use real table name. ifp->print(&str, QT_INFINIDB_DERIVED); - IDEBUG(cout << str.ptr() << endl); + IDEBUG(cerr << str.ptr() << endl); if (str.ptr()) cf->data(str.c_ptr()); @@ -1811,7 +1811,7 @@ bool buildPredicateItem(Item_func* ifp, gp_walk_info* gwip) (current_thd->lex->sql_command == SQLCOM_DELETE) || (current_thd->lex->sql_command == SQLCOM_DELETE_MULTI))) { - IDEBUG( cout << "deleted func with 2 const columns" << endl ); + IDEBUG( cerr << "deleted func with 2 const columns" << endl ); delete rhs; delete lhs; return false; @@ -1865,7 +1865,7 @@ bool buildPredicateItem(Item_func* ifp, gp_walk_info* gwip) if (notInner) { lhs->returnAll(true); - IDEBUG( cout << "setting returnAll on " << tan_lhs << endl); + IDEBUG( cerr << "setting returnAll on " << tan_lhs << endl); } } @@ -1882,7 +1882,7 @@ bool buildPredicateItem(Item_func* ifp, gp_walk_info* gwip) if (notInner) { rhs->returnAll(true); - IDEBUG( cout << "setting returnAll on " << tan_rhs << endl ); + IDEBUG( cerr << "setting returnAll on " << tan_rhs << endl ); } } @@ -2591,7 +2591,7 @@ CalpontSystemCatalog::ColType fieldType_MysqlToIDB (const Field* field) break; default: - IDEBUG( cout << "fieldType_MysqlToIDB:: Unknown result type of MySQL " + IDEBUG( cerr << "fieldType_MysqlToIDB:: Unknown result type of MySQL " << field->result_type() << endl ); break; } @@ -2699,7 +2699,7 @@ CalpontSystemCatalog::ColType colType_MysqlToIDB (const Item* item) break; default: - IDEBUG( cout << "colType_MysqlToIDB:: Unknown result type of MySQL " + IDEBUG( cerr << "colType_MysqlToIDB:: Unknown result type of MySQL " << item->result_type() << endl ); break; } @@ -2903,7 +2903,7 @@ ReturnedColumn* buildReturnedColumn(Item* item, gp_walk_info& gwi, bool& nonSupp { // TODO: item is a Item_cache_wrapper printf("EXPR_CACHE_ITEM in buildReturnedColumn\n"); - cout << "EXPR_CACHE_ITEM in buildReturnedColumn" << endl; + cerr << "EXPR_CACHE_ITEM in buildReturnedColumn" << endl; break; } @@ -3979,7 +3979,7 @@ ParseTree* buildParseTree(Item_func* item, gp_walk_info& gwi, bool& nonSupport) Item_cond* icp = (Item_cond*)item; #ifdef DEBUG_WALK_COND // debug - cout << "Build Parsetree: " << endl; + cerr << "Build Parsetree: " << endl; icp->traverse_cond(debug_walk, &gwi, Item::POSTFIX); #endif //@bug5044. PPSTFIX walking should always be treated as WHERE clause filter @@ -4826,7 +4826,7 @@ void gp_walk(const Item* item, void* arg) gwip->rcWorkStack.push(cc); if (str) - IDEBUG( cout << "Const F&E " << item->full_name() << " evaluate: " << valStr << endl ); + IDEBUG( cerr << "Const F&E " << item->full_name() << " evaluate: " << valStr << endl ); break; } @@ -5386,7 +5386,7 @@ void parse_item (Item* item, vector& field_vec, bool& hasNonSupport } else { - cout << "UNKNOWN REF Item" << endl; + cerr << "UNKNOWN REF Item" << endl; break; } } @@ -5458,7 +5458,7 @@ bool isInfiniDB(TABLE* table_ptr) int getSelectPlan(gp_walk_info& gwi, SELECT_LEX& select_lex, SCSEP& csep, bool isUnion) { #ifdef DEBUG_WALK_COND - cout << "getSelectPlan()" << endl; + cerr << "getSelectPlan()" << endl; #endif // by pass the derived table resolve phase of mysql @@ -5545,7 +5545,7 @@ int getSelectPlan(gp_walk_info& gwi, SELECT_LEX& select_lex, SCSEP& csep, bool i while ((sj_nest = sj_list_it++)) { - cout << sj_nest->db << "." << sj_nest->table_name << endl; + cerr << sj_nest->db << "." << sj_nest->table_name << endl; } #endif @@ -5629,7 +5629,7 @@ int getSelectPlan(gp_walk_info& gwi, SELECT_LEX& select_lex, SCSEP& csep, bool i CalpontSystemCatalog::TableAliasName tan = make_aliastable(table_ptr->db, table_name, table_ptr->alias, infiniDB); gwi.tableMap[tan] = make_pair(0, table_ptr); #ifdef DEBUG_WALK_COND - cout << tn << endl; + cerr << tn << endl; #endif } } @@ -5709,15 +5709,15 @@ int getSelectPlan(gp_walk_info& gwi, SELECT_LEX& select_lex, SCSEP& csep, bool i distUnionNum = unionVec.size(); /*#ifdef DEBUG_WALK_COND - IDEBUG( cout << ">>>> UNION DEBUG" << endl ); + IDEBUG( cerr << ">>>> UNION DEBUG" << endl ); JOIN* join = sl->join; Item_cond* icp = 0; if (join != 0) icp = reinterpret_cast(join->conds); if (icp) icp->traverse_cond(debug_walk, &gwi, Item::POSTFIX); - IDEBUG ( cout << *plan << endl ); - IDEBUG ( cout << "<<<traverse_cond(debug_walk, &gwi, Item::POSTFIX); - cout << "------------------------------------------------\n" << endl; + cerr << "------------------------------------------------\n" << endl; #endif icp->traverse_cond(gp_walk, &gwi, Item::POSTFIX); @@ -5794,16 +5794,16 @@ int getSelectPlan(gp_walk_info& gwi, SELECT_LEX& select_lex, SCSEP& csep, bool i TABLE_LIST* curr = *tbl; if (curr->table_name) - cout << curr->table_name << " "; + cerr << curr->table_name << " "; else - cout << curr->alias << endl; + cerr << curr->alias << endl; if (curr->outer_join) - cout << " is inner table" << endl; + cerr << " is inner table" << endl; else if (curr->straight) - cout << "straight_join" << endl; + cerr << "straight_join" << endl; else - cout << "join" << endl; + cerr << "join" << endl; if (curr->nested_join) { @@ -5819,7 +5819,7 @@ int getSelectPlan(gp_walk_info& gwi, SELECT_LEX& select_lex, SCSEP& csep, bool i for (TABLE_LIST** tb = inner; tb < end1; tb++) { TABLE_LIST* curr1 = *tb; - cout << curr1->alias << endl; + cerr << curr1->alias << endl; if (curr1->sj_on_expr) { @@ -5894,7 +5894,7 @@ int getSelectPlan(gp_walk_info& gwi, SELECT_LEX& select_lex, SCSEP& csep, bool i gwi.clauseType = SELECT; #ifdef DEBUG_WALK_COND { - cout << "------------------- SELECT --------------------" << endl; + cerr << "------------------- SELECT --------------------" << endl; List_iterator_fast it(select_lex.item_list); Item* item; @@ -5903,7 +5903,7 @@ int getSelectPlan(gp_walk_info& gwi, SELECT_LEX& select_lex, SCSEP& csep, bool i debug_walk(item, 0); } - cout << "-----------------------------------------------\n" << endl; + cerr << "-----------------------------------------------\n" << endl; } #endif @@ -6359,7 +6359,7 @@ int getSelectPlan(gp_walk_info& gwi, SELECT_LEX& select_lex, SCSEP& csep, bool i } #ifdef DEBUG_WALK_COND - cout << "SELECT clause SUBSELECT Item: " << sub->substype() << endl; + cerr << "SELECT clause SUBSELECT Item: " << sub->substype() << endl; JOIN* join = sub->get_select_lex()->join; if (join) @@ -6370,7 +6370,7 @@ int getSelectPlan(gp_walk_info& gwi, SELECT_LEX& select_lex, SCSEP& csep, bool i cond->traverse_cond(debug_walk, &gwi, Item::POSTFIX); } - cout << "Finish SELECT clause subselect item traversing" << endl; + cerr << "Finish SELECT clause subselect item traversing" << endl; #endif SelectSubQuery* selectSub = new SelectSubQuery(gwi, sub); //selectSub->gwip(&gwi); @@ -6489,9 +6489,9 @@ int getSelectPlan(gp_walk_info& gwi, SELECT_LEX& select_lex, SCSEP& csep, bool i { Item_cond* having = reinterpret_cast(select_lex.having); #ifdef DEBUG_WALK_COND - cout << "------------------- HAVING ---------------------" << endl; + cerr << "------------------- HAVING ---------------------" << endl; having->traverse_cond(debug_walk, &gwi, Item::POSTFIX); - cout << "------------------------------------------------\n" << endl; + cerr << "------------------------------------------------\n" << endl; #endif having->traverse_cond(gp_walk, &gwi, Item::POSTFIX); @@ -7937,7 +7937,8 @@ int cp_get_table_plan(THD* thd, SCSEP& csep, cal_table_info& ti) return 0; } -int cp_get_group_plan(THD* thd, SCSEP& csep, cal_table_info& ti, cal_group_info& gi ) +// ti is useless +int cp_get_group_plan(THD* thd, SCSEP& csep, cal_group_info& gi ) { LEX* lex = thd->lex; idbassert(lex != 0); @@ -7945,7 +7946,6 @@ int cp_get_group_plan(THD* thd, SCSEP& csep, cal_table_info& ti, cal_group_info& SELECT_LEX select_lex = lex->select_lex; gp_walk_info gwi; gwi.thd = thd; - gwi.thd->infinidb_vtable.vtable_state = THD::INFINIDB_CREATE_VTABLE; int status = getGroupPlan(gwi, select_lex, csep, gi); if (status > 0) @@ -8369,15 +8369,15 @@ int getGroupPlan(gp_walk_info& gwi, SELECT_LEX& select_lex, SCSEP& csep, cal_gro distUnionNum = unionVec.size(); /*#ifdef DEBUG_WALK_COND - IDEBUG( cout << ">>>> UNION DEBUG" << endl ); + IDEBUG( cerr << ">>>> UNION DEBUG" << endl ); JOIN* join = sl->join; Item_cond* icp = 0; if (join != 0) icp = reinterpret_cast(join->conds); if (icp) icp->traverse_cond(debug_walk, &gwi, Item::POSTFIX); - IDEBUG ( cout << *plan << endl ); - IDEBUG ( cout << "<<<table_name) - cout << curr->table_name << " "; + cerr << curr->table_name << " "; else - cout << curr->alias << endl; + cerr << curr->alias << endl; if (curr->outer_join) - cout << " is inner table" << endl; + cerr << " is inner table" << endl; else if (curr->straight) - cout << "straight_join" << endl; + cerr << "straight_join" << endl; else - cout << "join" << endl; + cerr << "join" << endl; if (curr->nested_join) { @@ -8480,7 +8480,7 @@ int getGroupPlan(gp_walk_info& gwi, SELECT_LEX& select_lex, SCSEP& csep, cal_gro for (TABLE_LIST** tb = inner; tb < end1; tb++) { TABLE_LIST* curr1 = *tb; - cout << curr1->alias << endl; + cerr << curr1->alias << endl; if (curr1->sj_on_expr) { @@ -8564,7 +8564,7 @@ int getGroupPlan(gp_walk_info& gwi, SELECT_LEX& select_lex, SCSEP& csep, cal_gro debug_walk(item, 0); } - cout << "-----------------------------------------------\n" << endl; + cerr << "-----------------------------------------------\n" << endl; } #endif diff --git a/dbcon/mysql/ha_calpont_impl.cpp b/dbcon/mysql/ha_calpont_impl.cpp index c8a02713a..d279d995e 100644 --- a/dbcon/mysql/ha_calpont_impl.cpp +++ b/dbcon/mysql/ha_calpont_impl.cpp @@ -276,6 +276,104 @@ void storeNumericField(Field** f, int64_t value, CalpontSystemCatalog::ColType& } } +void storeNumericFieldGroupBy(Field** f, int64_t value, CalpontSystemCatalog::ColType& ct) +{ + // unset null bit first + if ((*f)->null_ptr) + *(*f)->null_ptr &= ~(*f)->null_bit; + + // For unsigned, use the ColType returned in the row rather than the + // unsigned_flag set by mysql. This is because mysql gets it wrong for SUM() + // Hopefully, in all other cases we get it right. + switch ((*f)->type()) + { + case MYSQL_TYPE_NEWDECIMAL: + { + Field_new_decimal* f2 = (Field_new_decimal*)*f; + + // @bug4388 stick to InfiniDB's scale in case mysql gives wrong scale due + // to create vtable limitation. + if (f2->dec < ct.scale) + f2->dec = ct.scale; + + char buf[256]; + dataconvert::DataConvert::decimalToString(value, (unsigned)ct.scale, buf, 256, ct.colDataType); + f2->store(buf, strlen(buf), f2->charset()); + break; + } + + case MYSQL_TYPE_TINY: //TINYINT type + { + Field_tiny* f2 = (Field_tiny*)*f; + longlong int_val = (longlong)value; + f2->store(int_val, f2->unsigned_flag); + break; + } + + case MYSQL_TYPE_SHORT: //SMALLINT type + { + Field_short* f2 = (Field_short*)*f; + longlong int_val = (longlong)value; + f2->store(int_val, f2->unsigned_flag); + break; + } + + case MYSQL_TYPE_LONG: //INT type + { + Field_long* f2 = (Field_long*)*f; + longlong int_val = (longlong)value; + f2->store(int_val, f2->unsigned_flag); + break; + } + + case MYSQL_TYPE_LONGLONG: //BIGINT type + { + Field_longlong* f2 = (Field_longlong*)*f; + longlong int_val = (longlong)value; + f2->store(int_val, f2->unsigned_flag); + break; + } + + case MYSQL_TYPE_FLOAT: // FLOAT type + { + Field_float* f2 = (Field_float*)*f; + float float_val = *(float*)(&value); + f2->store(float_val); + break; + } + + case MYSQL_TYPE_DOUBLE: // DOUBLE type + { + Field_double* f2 = (Field_double*)*f; + double double_val = *(double*)(&value); + f2->store(double_val); + break; + } + + case MYSQL_TYPE_VARCHAR: + { + Field_varstring* f2 = (Field_varstring*)*f; + char tmp[25]; + + if (ct.colDataType == CalpontSystemCatalog::DECIMAL) + dataconvert::DataConvert::decimalToString(value, (unsigned)ct.scale, tmp, 25, ct.colDataType); + else + snprintf(tmp, 25, "%ld", value); + + f2->store(tmp, strlen(tmp), f2->charset()); + break; + } + + default: + { + Field_longlong* f2 = (Field_longlong*)*f; + longlong int_val = (longlong)value; + f2->store(int_val, f2->unsigned_flag); + break; + } + } +} + // // @bug 2244. Log exception related to lost connection to ExeMgr. // Log exception error from calls to sm::tpl_scan_fetch in fetchNextRow() @@ -761,7 +859,7 @@ int fetchNextRow(uchar* buf, cal_table_info& ti, cal_connection_info* ci) return rc; } -int fetchNextRowGrHand(cal_table_info& ti, cal_connection_info* ci) +int fetchNextRowGrHandGroupBy(cal_table_info& ti, cal_connection_info* ci) { int rc = HA_ERR_END_OF_FILE; int num_attr = ti.msTablePtr->s->fields; @@ -1193,6 +1291,442 @@ int fetchNextRowGrHand(cal_table_info& ti, cal_connection_info* ci) return rc; } +int fetchNextRowGrHand(cal_table_info& ti, cal_connection_info* ci) +{ + int rc = HA_ERR_END_OF_FILE; + int num_attr = ti.msTablePtr->s->fields; + sm::status_t sm_stat; + + try + { + if (ti.conn_hndl) + { + sm_stat = sm::tpl_scan_fetch(ti.tpl_scan_ctx, ti.conn_hndl); + } + else if (ci->cal_conn_hndl) + { + sm_stat = sm::tpl_scan_fetch(ti.tpl_scan_ctx, ci->cal_conn_hndl, (int*)(¤t_thd->killed)); + } + else + throw runtime_error("internal error"); + } + catch (std::exception& ex) + { +// @bug 2244. Always log this msg for now, as we try to track down when/why we are +// losing socket connection with ExeMgr +//#ifdef INFINIDB_DEBUG + tpl_scan_fetch_LogException( ti, ci, &ex); +//#endif + sm_stat = sm::CALPONT_INTERNAL_ERROR; + } + catch (...) + { +// @bug 2244. Always log this msg for now, as we try to track down when/why we are +// losing socket connection with ExeMgr +//#ifdef INFINIDB_DEBUG + tpl_scan_fetch_LogException( ti, ci, 0 ); +//#endif + sm_stat = sm::CALPONT_INTERNAL_ERROR; + } + + if (sm_stat == sm::STATUS_OK) + { + Field** f; + f = ti.msTablePtr->field; + //set all fields to null in null col bitmap + //memset(buf, -1, ti.msTablePtr->s->null_bytes); + std::vector& colTypes = ti.tpl_scan_ctx->ctp; + int64_t intColVal = 0; + uint64_t uintColVal = 0; + char tmp[256]; + + RowGroup* rowGroup = ti.tpl_scan_ctx->rowGroup; + + // table mode mysql expects all columns of the table. mapping between columnoid and position in rowgroup + // set coltype.position to be the position in rowgroup. only set once. + if (ti.tpl_scan_ctx->rowsreturned == 0 && + (ti.tpl_scan_ctx->traceFlags & execplan::CalpontSelectExecutionPlan::TRACE_TUPLE_OFF)) + { + for (uint32_t i = 0; i < rowGroup->getColumnCount(); i++) + { + int oid = rowGroup->getOIDs()[i]; + int j = 0; + + for (; j < num_attr; j++) + { + // mysql should haved eliminated duplicate projection columns + if (oid == colTypes[j].columnOID || oid == colTypes[j].ddn.dictOID) + { + colTypes[j].colPosition = i; + break; + } + } + } + } + + rowgroup::Row row; + rowGroup->initRow(&row); + rowGroup->getRow(ti.tpl_scan_ctx->rowsreturned, &row); + int s; + + for (int p = 0; p < num_attr; p++, f++) + { + //This col is going to be written + bitmap_set_bit(ti.msTablePtr->write_set, (*f)->field_index); + + // get coltype if not there yet + if (colTypes[0].colWidth == 0) + { + for (short c = 0; c < num_attr; c++) + { + colTypes[c].colPosition = c; + colTypes[c].colWidth = rowGroup->getColumnWidth(c); + colTypes[c].colDataType = rowGroup->getColTypes()[c]; + colTypes[c].columnOID = rowGroup->getOIDs()[c]; + colTypes[c].scale = rowGroup->getScale()[c]; + colTypes[c].precision = rowGroup->getPrecision()[c]; + } + } + + CalpontSystemCatalog::ColType colType(colTypes[p]); + + // table mode handling + if (ti.tpl_scan_ctx->traceFlags & execplan::CalpontSelectExecutionPlan::TRACE_TUPLE_OFF) + { + if (colType.colPosition == -1) // not projected by tuplejoblist + continue; + else + s = colType.colPosition; + } + else + { + s = p; + } + + // precision == -16 is borrowed as skip null check indicator for bit ops. + if (row.isNullValue(s) && colType.precision != -16) + { + // @2835. Handle empty string and null confusion. store empty string for string column + if (colType.colDataType == CalpontSystemCatalog::CHAR || + colType.colDataType == CalpontSystemCatalog::VARCHAR || + colType.colDataType == CalpontSystemCatalog::VARBINARY) + { + Field_varstring* f2 = (Field_varstring*)*f; + f2->store(tmp, 0, f2->charset()); + } + + continue; + } + + // fetch and store data + switch (colType.colDataType) + { + case CalpontSystemCatalog::DATE: + { + if ((*f)->null_ptr) + *(*f)->null_ptr &= ~(*f)->null_bit; + + intColVal = row.getUintField<4>(s); + DataConvert::dateToString(intColVal, tmp, 255); + Field_varstring* f2 = (Field_varstring*)*f; + f2->store(tmp, strlen(tmp), f2->charset()); + f2->set_notnull(); + break; + } + + case CalpontSystemCatalog::DATETIME: + { + if ((*f)->null_ptr) + *(*f)->null_ptr &= ~(*f)->null_bit; + + intColVal = row.getUintField<8>(s); + DataConvert::datetimeToString(intColVal, tmp, 255); + + /* setting the field_length is a sort-of hack. The length + * at this point can be long enough to include mseconds. + * ColumnStore doesn't fully support mseconds yet so if + * they are requested, trim them off. + * At a later date we should set this more intelligently + * based on the result set. + */ + /* MCOL-683: UTF-8 datetime no msecs is 57, this sometimes happens! */ + if (((*f)->field_length > 19) && ((*f)->field_length != 57)) + (*f)->field_length = strlen(tmp); + + Field_varstring* f2 = (Field_varstring*)*f; + f2->store(tmp, strlen(tmp), f2->charset()); + f2->set_notnull(); + break; + } + + case CalpontSystemCatalog::CHAR: + case CalpontSystemCatalog::VARCHAR: + { + Field_varstring* f2 = (Field_varstring*)*f; + + switch (colType.colWidth) + { + case 1: + intColVal = row.getUintField<1>(s); + f2->store((char*)(&intColVal), strlen((char*)(&intColVal)), f2->charset()); + break; + + case 2: + intColVal = row.getUintField<2>(s); + f2->store((char*)(&intColVal), strlen((char*)(&intColVal)), f2->charset()); + break; + + case 4: + intColVal = row.getUintField<4>(s); + f2->store((char*)(&intColVal), strlen((char*)(&intColVal)), f2->charset()); + break; + + case 8: + //make sure we don't send strlen off into the weeds... + intColVal = row.getUintField<8>(s); + memcpy(tmp, &intColVal, 8); + tmp[8] = 0; + f2->store(tmp, strlen(tmp), f2->charset()); + break; + + default: + f2->store((const char*)row.getStringPointer(s), row.getStringLength(s), f2->charset()); + } + + if ((*f)->null_ptr) + *(*f)->null_ptr &= ~(*f)->null_bit; + f2->set_notnull(); + break; + } + + case CalpontSystemCatalog::VARBINARY: + { + Field_varstring* f2 = (Field_varstring*)*f; + + if (current_thd->variables.infinidb_varbin_always_hex) + { + uint32_t l; + const uint8_t* p = row.getVarBinaryField(l, s); + uint32_t ll = l * 2; + boost::scoped_array sca(new char[ll]); + vbin2hex(p, l, sca.get()); + f2->store(sca.get(), ll, f2->charset()); + } + else + f2->store((const char*)row.getVarBinaryField(s), row.getVarBinaryLength(s), f2->charset()); + f2->set_notnull(); + + if ((*f)->null_ptr) + *(*f)->null_ptr &= ~(*f)->null_bit; + + break; + } + + case CalpontSystemCatalog::BIGINT: + { + intColVal = row.getIntField<8>(s); + storeNumericFieldGroupBy(f, intColVal, colType); + break; + } + + case CalpontSystemCatalog::UBIGINT: + { + uintColVal = row.getUintField<8>(s); + storeNumericFieldGroupBy(f, uintColVal, colType); + break; + } + + case CalpontSystemCatalog::INT: + { + intColVal = row.getIntField<4>(s); + storeNumericFieldGroupBy(f, intColVal, colType); + break; + } + + case CalpontSystemCatalog::UINT: + { + uintColVal = row.getUintField<4>(s); + storeNumericFieldGroupBy(f, uintColVal, colType); + break; + } + + case CalpontSystemCatalog::SMALLINT: + { + intColVal = row.getIntField<2>(s); + storeNumericFieldGroupBy(f, intColVal, colType); + break; + } + + case CalpontSystemCatalog::USMALLINT: + { + uintColVal = row.getUintField<2>(s); + storeNumericFieldGroupBy(f, uintColVal, colType); + break; + } + + case CalpontSystemCatalog::TINYINT: + { + intColVal = row.getIntField<1>(s); + storeNumericFieldGroupBy(f, intColVal, colType); + break; + } + + case CalpontSystemCatalog::UTINYINT: + { + uintColVal = row.getUintField<1>(s); + storeNumericFieldGroupBy(f, uintColVal, colType); + break; + } + + //In this case, we're trying to load a double output column with float data. This is the + // case when you do sum(floatcol), e.g. + case CalpontSystemCatalog::FLOAT: + case CalpontSystemCatalog::UFLOAT: + { + float dl = row.getFloatField(s); + + if (dl == std::numeric_limits::infinity()) + continue; + + //int64_t* icvp = (int64_t*)&dl; + //intColVal = *icvp; + Field_float* f2 = (Field_float*)*f; + // bug 3485, reserve enough space for the longest float value + // -3.402823466E+38 to -1.175494351E-38, 0, and + // 1.175494351E-38 to 3.402823466E+38. + (*f)->field_length = 40; + + //float float_val = *(float*)(&value); + //f2->store(float_val); + if (f2->decimals() < (uint32_t)row.getScale(s)) + f2->dec = (uint32_t)row.getScale(s); + + f2->store(dl); + f2->set_notnull(); + if ((*f)->null_ptr) + *(*f)->null_ptr &= ~(*f)->null_bit; + + break; + + //storeNumericField(f, intColVal, colType); + //break; + } + + case CalpontSystemCatalog::DOUBLE: + case CalpontSystemCatalog::UDOUBLE: + { + double dl = row.getDoubleField(s); + + if (dl == std::numeric_limits::infinity()) + continue; + + Field_double* f2 = (Field_double*)*f; + // bug 3483, reserve enough space for the longest double value + // -1.7976931348623157E+308 to -2.2250738585072014E-308, 0, and + // 2.2250738585072014E-308 to 1.7976931348623157E+308. + (*f)->field_length = 310; + + //double double_val = *(double*)(&value); + //f2->store(double_val); + if (f2->decimals() < (uint32_t)row.getScale(s)) + f2->dec = (uint32_t)row.getScale(s); + + f2->store(dl); + f2->set_notnull(); + + if ((*f)->null_ptr) + *(*f)->null_ptr &= ~(*f)->null_bit; + + break; + + + //int64_t* icvp = (int64_t*)&dl; + //intColVal = *icvp; + //storeNumericField(f, intColVal, colType); + //break; + } + + case CalpontSystemCatalog::DECIMAL: + case CalpontSystemCatalog::UDECIMAL: + { + intColVal = row.getIntField(s); + storeNumericFieldGroupBy(f, intColVal, colType); + break; + } + + case CalpontSystemCatalog::BLOB: + case CalpontSystemCatalog::TEXT: + { + Field_blob* f2 = (Field_blob*)*f; + f2->set_ptr(row.getVarBinaryLength(s), (unsigned char*)row.getVarBinaryField(s)); + + if ((*f)->null_ptr) + *(*f)->null_ptr &= ~(*f)->null_bit; + + break; + } + + default: // treat as int64 + { + intColVal = row.getUintField<8>(s); + storeNumericFieldGroupBy(f, intColVal, colType); + break; + } + } + } + + ti.tpl_scan_ctx->rowsreturned++; + ti.c++; +#ifdef INFINIDB_DEBUG + + if ((ti.c % 1000000) == 0) + cerr << "fetchNextRow so far table " << ti.msTablePtr->s->table_name.str << " rows = " << ti.c << endl; + +#endif + ti.moreRows = true; + rc = 0; + } + else if (sm_stat == sm::SQL_NOT_FOUND) + { + IDEBUG( cerr << "fetchNextRow done for table " << ti.msTablePtr->s->table_name.str << " rows = " << ti.c << endl ); + ti.c = 0; + ti.moreRows = false; + rc = HA_ERR_END_OF_FILE; + } + else if (sm_stat == sm::CALPONT_INTERNAL_ERROR) + { + ti.moreRows = false; + rc = ER_INTERNAL_ERROR; + ci->rc = rc; + } + else if ((uint32_t)sm_stat == logging::ERR_LOST_CONN_EXEMGR) + { + ti.moreRows = false; + rc = logging::ERR_LOST_CONN_EXEMGR; + sm::sm_init(tid2sid(current_thd->thread_id), &ci->cal_conn_hndl, + current_thd->variables.infinidb_local_query); + idbassert(ci->cal_conn_hndl != 0); + ci->rc = rc; + } + else if (sm_stat == sm::SQL_KILLED) + { + // query was aborted by the user. treat it the same as limit query. close + // connection after rnd_close. + ti.c = 0; + ti.moreRows = false; + rc = HA_ERR_END_OF_FILE; + ci->rc = rc; + } + else + { + ti.moreRows = false; + rc = sm_stat; + ci->rc = rc; + } + + return rc; +} + void makeUpdateScalarJoin(const ParseTree* n, void* obj) { TreeNode* tn = n->data(); @@ -5405,7 +5939,7 @@ int ha_calpont_impl_rnd_pos(uchar* buf, uchar* pos) ***********************************************************/ int ha_calpont_impl_group_by_init(ha_calpont_group_by_handler* group_hand, TABLE* table) { - //first_row= true; + //first_row= true; string tableName = group_hand->table_list->table->s->table_name.str; IDEBUG( cout << "group_by_init for table " << group_hand->table_list->table->s->table_name.str << endl ); @@ -5548,6 +6082,7 @@ int ha_calpont_impl_group_by_init(ha_calpont_group_by_handler* group_hand, TABLE bool localQuery = (thd->variables.infinidb_local_query > 0 ? true : false); + /* // table mode if (thd->infinidb_vtable.vtable_state == THD::INFINIDB_DISABLE_VTABLE) { @@ -5626,8 +6161,9 @@ int ha_calpont_impl_group_by_init(ha_calpont_group_by_handler* group_hand, TABLE if (ci->tableMap.size() == 1) ti.csep->data(idb_mysql_query_str(thd)); } + */ // vtable mode - else + //else { //if (!ci->cal_conn_hndl || thd->infinidb_vtable.vtable_state == THD::INFINIDB_CREATE_VTABLE) if ( thd->infinidb_vtable.vtable_state == THD::INFINIDB_CREATE_VTABLE) @@ -5702,6 +6238,23 @@ int ha_calpont_impl_group_by_init(ha_calpont_group_by_handler* group_hand, TABLE csep->verID(verID); csep->sessionID(sessionID); + if (group_hand->table_list->db_length) + csep->schemaName(group_hand->table_list->db); + + csep->traceFlags(ci->traceFlags); + //ti.msTablePtr = group_hand->table_list->table; + + gi.groupByTables = group_hand->table_list; + gi.groupByFields = group_hand->select; + gi.groupByWhere = group_hand->where; + gi.groupByGroup = group_hand->group_by; + gi.groupByOrder = group_hand->order_by; + gi.groupByHaving = group_hand->having; + gi.groupByDistinct = group_hand->distinct; + + // send plan whenever group_init is called + int status = cp_get_group_plan(thd, csep, gi); +/* if (thd->db) csep->schemaName(thd->db); @@ -5712,7 +6265,7 @@ int ha_calpont_impl_group_by_init(ha_calpont_group_by_handler* group_hand, TABLE //get plan int status = cp_get_plan(thd, csep); - +*/ //if (cp_get_plan(thd, csep) != 0) if (status > 0) goto internal_error; @@ -5887,6 +6440,9 @@ int ha_calpont_impl_group_by_init(ha_calpont_group_by_handler* group_hand, TABLE ti = ci->tableMap[table]; ti.msTablePtr = table; + // MCOL-1052 + thd->infinidb_vtable.vtable_state = THD::INFINIDB_SELECT_VTABLE; + if ((thd->infinidb_vtable.vtable_state == THD::INFINIDB_SELECT_VTABLE) || (thd->infinidb_vtable.vtable_state == THD::INFINIDB_DISABLE_VTABLE) || (thd->infinidb_vtable.vtable_state == THD::INFINIDB_REDO_QUERY)) @@ -6037,9 +6593,8 @@ int ha_calpont_impl_group_by_next(ha_calpont_group_by_handler* group_hand, TABLE return ER_INTERNAL_ERROR; // @bug 3005 - if (thd->infinidb_vtable.vtable_state == THD::INFINIDB_SELECT_VTABLE && - string(table->s->table_name.str).find("$vtable") != 0) - return HA_ERR_END_OF_FILE; + //if (thd->infinidb_vtable.vtable_state == THD::INFINIDB_SELECT_VTABLE) + // return HA_ERR_END_OF_FILE; if (((thd->lex)->sql_command == SQLCOM_UPDATE) || ((thd->lex)->sql_command == SQLCOM_DELETE) || ((thd->lex)->sql_command == SQLCOM_DELETE_MULTI) || ((thd->lex)->sql_command == SQLCOM_UPDATE_MULTI)) @@ -6051,13 +6606,13 @@ int ha_calpont_impl_group_by_next(ha_calpont_group_by_handler* group_hand, TABLE // @bug 2232. Basic SP support // @bug 3939. Only error out for sp with select. Let pass for alter table in sp. - if (thd->infinidb_vtable.call_sp && (thd->lex)->sql_command != SQLCOM_ALTER_TABLE) + /*if (thd->infinidb_vtable.call_sp && (thd->lex)->sql_command != SQLCOM_ALTER_TABLE) { setError(thd, ER_CHECK_NOT_IMPLEMENTED, "This stored procedure syntax is not supported by Columnstore in this version"); thd->infinidb_vtable.vtable_state = THD::INFINIDB_ERROR; return ER_INTERNAL_ERROR; } - + */ if (!thd->infinidb_vtable.cal_conn_info) thd->infinidb_vtable.cal_conn_info = (void*)(new cal_connection_info()); @@ -6105,8 +6660,9 @@ int ha_calpont_impl_group_by_next(ha_calpont_group_by_handler* group_hand, TABLE try { - rc = fetchNextRowGrHand(ti, ci); - rc = 0; + rc = fetchNextRowGrHandGroupBy(ti, ci); + // MCOL-1052 + //rc = 0; } catch (std::exception& e) { @@ -6143,7 +6699,189 @@ int ha_calpont_impl_group_by_next(ha_calpont_group_by_handler* group_hand, TABLE int ha_calpont_impl_group_by_end(ha_calpont_group_by_handler* group_hand, TABLE* table) { - return 0; + int rc = 0; + THD* thd = current_thd; + cal_connection_info* ci = NULL; + + + if (thd->slave_thread && ( + thd->lex->sql_command == SQLCOM_INSERT || + thd->lex->sql_command == SQLCOM_INSERT_SELECT || + thd->lex->sql_command == SQLCOM_UPDATE || + thd->lex->sql_command == SQLCOM_UPDATE_MULTI || + thd->lex->sql_command == SQLCOM_DELETE || + thd->lex->sql_command == SQLCOM_DELETE_MULTI || + thd->lex->sql_command == SQLCOM_TRUNCATE || + thd->lex->sql_command == SQLCOM_LOAD)) + return 0; + + thd->infinidb_vtable.isNewQuery = true; + + if (thd->infinidb_vtable.cal_conn_info) + ci = reinterpret_cast(thd->infinidb_vtable.cal_conn_info); + + if (thd->infinidb_vtable.vtable_state == THD::INFINIDB_ORDER_BY ) + { + thd->infinidb_vtable.vtable_state = THD::INFINIDB_SELECT_VTABLE;// flip back to normal state + return rc; + } + +// if (thd->infinidb_vtable.vtable_state == THD::INFINIDB_REDO_PHASE1) +// return rc; +/* + if ( (thd->lex)->sql_command == SQLCOM_ALTER_TABLE ) + return rc; + + if (((thd->lex)->sql_command == SQLCOM_UPDATE) || + ((thd->lex)->sql_command == SQLCOM_DELETE) || + ((thd->lex)->sql_command == SQLCOM_DELETE_MULTI) || + ((thd->lex)->sql_command == SQLCOM_UPDATE_MULTI)) + return rc; +*/ + if (((thd->lex)->sql_command == SQLCOM_INSERT) || + ((thd->lex)->sql_command == SQLCOM_INSERT_SELECT) ) + { + // @bug 4022. error handling for select part of dml + if (ci->cal_conn_hndl && ci->rc) + { + // send ExeMgr a signal before closing the connection + ByteStream msg; + ByteStream::quadbyte qb = 0; + msg << qb; + + try + { + ci->cal_conn_hndl->exeMgr->write(msg); + } + catch (...) + { + // this is error handling, so ignore connection failure. + } + + sm::sm_cleanup(ci->cal_conn_hndl); + ci->cal_conn_hndl = 0; + return rc; + } + } + + if (!ci) + { + thd->infinidb_vtable.cal_conn_info = (void*)(new cal_connection_info()); + ci = reinterpret_cast(thd->infinidb_vtable.cal_conn_info); + } + + // @bug 3078. Also session limit variable works the same as ctrl+c + if (thd->killed == KILL_QUERY || thd->killed == KILL_QUERY_HARD || + ((thd->lex)->sql_command != SQLCOM_INSERT && + (thd->lex)->sql_command != SQLCOM_INSERT_SELECT && + thd->variables.select_limit != (uint64_t) - 1)) + { + if (ci->cal_conn_hndl) + { + // send ExeMgr a signal before closing the connection + ByteStream msg; + ByteStream::quadbyte qb = 0; + msg << qb; + + try + { + ci->cal_conn_hndl->exeMgr->write(msg); + } + catch (...) + { + // this is the end of query. Ignore the exception if exemgr connection failed + // for whatever reason. + } + + sm::sm_cleanup(ci->cal_conn_hndl); + ci->cal_conn_hndl = 0; + // clear querystats because no query stats available for cancelled query + ci->queryStats = ""; + } + + return 0; + } + + IDEBUG( cerr << "group_by_end for table " << table->s->table_name.str << endl ); + + cal_table_info ti = ci->tableMap[table]; + sm::cpsm_conhdl_t* hndl; + + //if (thd->infinidb_vtable.vtable_state == THD::INFINIDB_DISABLE_VTABLE) + // hndl = ti.conn_hndl; + //else + hndl = ci->cal_conn_hndl; + + if (ti.tpl_ctx) + { + if (ti.tpl_scan_ctx.get()) + { + try + { + sm::tpl_scan_close(ti.tpl_scan_ctx); + } + catch (...) + { + rc = ER_INTERNAL_ERROR; + } + } + + ti.tpl_scan_ctx.reset(); + + try + { + sm::tpl_close(ti.tpl_ctx, &hndl, ci->stats); + + // set conn hndl back. could be changed in tpl_close + //if (thd->infinidb_vtable.vtable_state == THD::INFINIDB_DISABLE_VTABLE) + // ti.conn_hndl = hndl; + //else + ci->cal_conn_hndl = hndl; + + ti.tpl_ctx = 0; + } + catch (IDBExcept& e) + { + if (e.errorCode() == ERR_CROSS_ENGINE_CONNECT || e.errorCode() == ERR_CROSS_ENGINE_CONFIG) + { + string msg = string("Columnstore Query Stats - ") + e.what(); + push_warning(thd, Sql_condition::WARN_LEVEL_WARN, 9999, msg.c_str()); + } + else + { + setError(thd, ER_INTERNAL_ERROR, e.what()); + rc = ER_INTERNAL_ERROR; + } + } + catch (std::exception& e) + { + setError(thd, ER_INTERNAL_ERROR, e.what()); + rc = ER_INTERNAL_ERROR; + } + catch (...) + { + setError(thd, ER_INTERNAL_ERROR, "Internal error throwed in rnd_end"); + rc = ER_INTERNAL_ERROR; + } + } + + ti.tpl_ctx = 0; + + /* + if (thd->infinidb_vtable.vtable_state == THD::INFINIDB_SELECT_VTABLE && + thd->infinidb_vtable.has_order_by) + thd->infinidb_vtable.vtable_state = THD::INFINIDB_ORDER_BY; + */ + ci->tableMap[table] = ti; + + // push warnings from CREATE phase + if (!ci->warningMsg.empty()) + push_warning(thd, Sql_condition::WARN_LEVEL_WARN, 9999, ci->warningMsg.c_str()); + + ci->warningMsg.clear(); + // reset expressionId just in case + ci->expressionId = 0; + return rc; } // vim:sw=4 ts=4: diff --git a/dbcon/mysql/ha_calpont_impl_if.h b/dbcon/mysql/ha_calpont_impl_if.h index 7c12c8b2c..21560a4a0 100644 --- a/dbcon/mysql/ha_calpont_impl_if.h +++ b/dbcon/mysql/ha_calpont_impl_if.h @@ -318,7 +318,7 @@ const std::string infinidb_err_msg = "\nThe query includes syntax that is not su int cp_get_plan(THD* thd, execplan::SCSEP& csep); int cp_get_table_plan(THD* thd, execplan::SCSEP& csep, cal_impl_if::cal_table_info& ti); -int cp_get_group_plan(THD* thd, execplan::SCSEP& csep, cal_impl_if::cal_table_info& ti,cal_impl_if::cal_group_info& gi); +int cp_get_group_plan(THD* thd, execplan::SCSEP& csep, cal_impl_if::cal_group_info& gi); int getSelectPlan(gp_walk_info& gwi, SELECT_LEX& select_lex, execplan::SCSEP& csep, bool isUnion = false); int getGroupPlan(gp_walk_info& gwi, SELECT_LEX& select_lex, execplan::SCSEP& csep,cal_group_info& gi, bool isUnion = false); void setError(THD* thd, uint32_t errcode, const std::string errmsg, gp_walk_info* gwi); From 78aeedb16210e66bec2a28b555c26fea7568e9d0 Mon Sep 17 00:00:00 2001 From: Roman Nozdrin Date: Mon, 2 Apr 2018 15:03:59 +0300 Subject: [PATCH 08/57] MCOL-1052 WIP Process UNION subqueries separetely. --- dbcon/mysql/ha_calpont.cpp | 28 +++++++++------------------- dbcon/mysql/ha_calpont.h | 25 ++++++++++--------------- dbcon/mysql/ha_calpont_execplan.cpp | 16 +++++++++------- dbcon/mysql/ha_calpont_impl.cpp | 8 +++----- dbcon/mysql/sm.cpp | 2 +- 5 files changed, 32 insertions(+), 47 deletions(-) diff --git a/dbcon/mysql/ha_calpont.cpp b/dbcon/mysql/ha_calpont.cpp index ae0850109..a780a72e0 100644 --- a/dbcon/mysql/ha_calpont.cpp +++ b/dbcon/mysql/ha_calpont.cpp @@ -1148,7 +1148,7 @@ static MYSQL_SYSVAR_ULONG( * Details are in server/sql/group_by_handler.h * PARAMETERS: * thd - THD pointer. - * query - Query structure, that describes the pushdown query. + * query - Query structure, that describes the pushdowned query. * RETURN: * group_by_handler if success * NULL in other case @@ -1157,13 +1157,13 @@ static group_by_handler * create_calpont_group_by_handler(THD *thd, Query *query) { ha_calpont_group_by_handler *handler = NULL; - Item *item; - List_iterator_fast it(*query->select); - - if ( thd->infinidb_vtable.vtable_state == THD::INFINIDB_DISABLE_VTABLE ) - { - handler = new ha_calpont_group_by_handler(thd, query); - } + + handler = new ha_calpont_group_by_handler(thd, query); + + // Notify the server, that CS handles GROUP BY, ORDER BY and HAVING clauses. + query->group_by = NULL; + query->order_by = NULL; + query->having = NULL; return handler; } @@ -1172,7 +1172,7 @@ int ha_calpont_group_by_handler::init_scan() { DBUG_ENTER("ha_calpont_group_by_handler::init_scan"); - // MCOL-1052 + // Save vtable_state to restore the after we inited. THD::infinidb_state oldState = thd->infinidb_vtable.vtable_state; thd->infinidb_vtable.vtable_state = THD::INFINIDB_CREATE_VTABLE; int rc = ha_calpont_impl_group_by_init(this, table); @@ -1183,19 +1183,9 @@ int ha_calpont_group_by_handler::init_scan() int ha_calpont_group_by_handler::next_row() { -// if (!first_row) -// return(HA_ERR_END_OF_FILE); DBUG_ENTER("ha_calpont_group_by_handler::next_row"); int rc = ha_calpont_impl_group_by_next(this, table); - - -// first_row= 0; - //Field *field = *(table->field); - //field->store(5LL, 1); - //field->set_notnull(); - //return(0); - DBUG_RETURN(rc); } diff --git a/dbcon/mysql/ha_calpont.h b/dbcon/mysql/ha_calpont.h index f15078fad..0e475e615 100644 --- a/dbcon/mysql/ha_calpont.h +++ b/dbcon/mysql/ha_calpont.h @@ -250,20 +250,17 @@ public: class ha_calpont_group_by_handler: public group_by_handler { - public: -// ha_calpont_group_by_handler(THD *thd_arg, List *fields_arg, -// TABLE_LIST *table_list_arg, Query *query) + public: ha_calpont_group_by_handler(THD *thd_arg, Query *query) - : group_by_handler(thd_arg, calpont_hton), - select(query->select), - table_list(query->from), - distinct(query->distinct), - where(query->where), - group_by(query->group_by), - order_by(query->order_by), - having(query->having), - query(query) - { } + : group_by_handler(thd_arg, calpont_hton), + select(query->select), + table_list(query->from), + distinct(query->distinct), + where(query->where), + group_by(query->group_by), + order_by(query->order_by), + having(query->having) + { } ~ha_calpont_group_by_handler() { } int init_scan(); int next_row(); @@ -275,8 +272,6 @@ class ha_calpont_group_by_handler: public group_by_handler ORDER *group_by; ORDER *order_by; Item *having; - bool first_row; // useless by now - Query *query; // useless by now }; #endif //HA_CALPONT_H__ diff --git a/dbcon/mysql/ha_calpont_execplan.cpp b/dbcon/mysql/ha_calpont_execplan.cpp index 6a8b1ac95..7ed3c0e69 100644 --- a/dbcon/mysql/ha_calpont_execplan.cpp +++ b/dbcon/mysql/ha_calpont_execplan.cpp @@ -8259,7 +8259,7 @@ int getGroupPlan(gp_walk_info& gwi, SELECT_LEX& select_lex, SCSEP& csep, cal_gro gwi.tbList.push_back(tn); CalpontSystemCatalog::TableAliasName tan = make_aliastable("", alias, alias); gwi.tableMap[tan] = make_pair(0, table_ptr); - gwi.thd->infinidb_vtable.isUnion = true; //by-pass the 2nd pass of rnd_init +// gwi.thd->infinidb_vtable.isUnion = true; //by-pass the 2nd pass of rnd_init } else if (table_ptr->view) { @@ -8324,9 +8324,11 @@ int getGroupPlan(gp_walk_info& gwi, SELECT_LEX& select_lex, SCSEP& csep, cal_gro bool unionSel = false; + // MCOL-1052 + /* if (!isUnion && select_lex.master_unit()->is_union()) { - gwi.thd->infinidb_vtable.isUnion = true; +// gwi.thd->infinidb_vtable.isUnion = true; CalpontSelectExecutionPlan::SelectList unionVec; SELECT_LEX* select_cursor = select_lex.master_unit()->first_select(); unionSel = true; @@ -8359,7 +8361,7 @@ int getGroupPlan(gp_walk_info& gwi, SELECT_LEX& select_lex, SCSEP& csep, cal_gro union_gwi.thd = gwi.thd; uint32_t err = 0; - if ((err = getSelectPlan(union_gwi, *sl, plan, unionSel)) != 0) + if ((err = getGroupPlan(union_gwi, *sl, plan, gi, unionSel)) != 0) return err; unionVec.push_back(SCEP(plan)); @@ -8368,7 +8370,7 @@ int getGroupPlan(gp_walk_info& gwi, SELECT_LEX& select_lex, SCSEP& csep, cal_gro if (sl == select_lex.master_unit()->union_distinct) distUnionNum = unionVec.size(); - /*#ifdef DEBUG_WALK_COND + *//*#ifdef DEBUG_WALK_COND IDEBUG( cerr << ">>>> UNION DEBUG" << endl ); JOIN* join = sl->join; Item_cond* icp = 0; @@ -8379,7 +8381,7 @@ int getGroupPlan(gp_walk_info& gwi, SELECT_LEX& select_lex, SCSEP& csep, cal_gro IDEBUG ( cerr << *plan << endl ); IDEBUG ( cerr << "<<<unionVec(unionVec); csep->distinctUnionNum(distUnionNum); @@ -8387,7 +8389,7 @@ int getGroupPlan(gp_walk_info& gwi, SELECT_LEX& select_lex, SCSEP& csep, cal_gro if (unionVec.empty()) gwi.thd->infinidb_vtable.impossibleWhereOnUnion = true; } - + */ gwi.clauseType = WHERE; if (icp) @@ -10040,7 +10042,7 @@ int getGroupPlan(gp_walk_info& gwi, SELECT_LEX& select_lex, SCSEP& csep, cal_gro //TABLE_LIST* table_ptr = select_lex.get_table_list(); set aliasSet; // to avoid duplicate table alias - for (; table_ptr; table_ptr = table_ptr->next_global) + for (; table_ptr; table_ptr = table_ptr->next_local) { if (string(table_ptr->table_name).find("$vtable") != string::npos) continue; diff --git a/dbcon/mysql/ha_calpont_impl.cpp b/dbcon/mysql/ha_calpont_impl.cpp index d279d995e..09e125421 100644 --- a/dbcon/mysql/ha_calpont_impl.cpp +++ b/dbcon/mysql/ha_calpont_impl.cpp @@ -5939,10 +5939,8 @@ int ha_calpont_impl_rnd_pos(uchar* buf, uchar* pos) ***********************************************************/ int ha_calpont_impl_group_by_init(ha_calpont_group_by_handler* group_hand, TABLE* table) { - //first_row= true; string tableName = group_hand->table_list->table->s->table_name.str; - IDEBUG( cout << "group_by_init for table " << - group_hand->table_list->table->s->table_name.str << endl ); + IDEBUG( cout << "group_by_init for table " << tableName << endl ); THD* thd = current_thd; //check whether the system is ready to process statement. @@ -5993,8 +5991,8 @@ int ha_calpont_impl_group_by_init(ha_calpont_group_by_handler* group_hand, TABLE return ER_INTERNAL_ERROR; // by pass the extra union trips. return 0 - //if (thd->infinidb_vtable.isUnion && thd->infinidb_vtable.vtable_state == THD::INFINIDB_CREATE_VTABLE) - // return 0; + if (thd->infinidb_vtable.isUnion && thd->infinidb_vtable.vtable_state == THD::INFINIDB_CREATE_VTABLE) + return 0; // @bug 2232. Basic SP support. Error out non support sp cases. // @bug 3939. Only error out for sp with select. Let pass for alter table in sp. diff --git a/dbcon/mysql/sm.cpp b/dbcon/mysql/sm.cpp index 569fee6a5..fe1a23e76 100644 --- a/dbcon/mysql/sm.cpp +++ b/dbcon/mysql/sm.cpp @@ -145,7 +145,7 @@ status_t tpl_scan_fetch_getband(cpsm_conhdl_t* hndl, sp_cpsm_tplsch_t& ntplsch, if (ntplsch->bs.length() != 0) { ntplsch->deserializeTable(ntplsch->bs); - + if (ntplsch->rowGroup && ntplsch->rowGroup->getRGData() == NULL) { ntplsch->bs.restart(); From dbcbd6c7e74f0bf4e3d4f757f0b9bff7f2666e57 Mon Sep 17 00:00:00 2001 From: David Hall Date: Wed, 4 Apr 2018 12:34:17 -0500 Subject: [PATCH 09/57] MCOL-1323 cpimport Splitter has incorrect SIGPIPE mapping --- writeengine/splitter/we_splitterapp.cpp | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/writeengine/splitter/we_splitterapp.cpp b/writeengine/splitter/we_splitterapp.cpp index 7ecbca854..0077ebd60 100644 --- a/writeengine/splitter/we_splitterapp.cpp +++ b/writeengine/splitter/we_splitterapp.cpp @@ -159,7 +159,7 @@ void WESplitterApp::setupSignalHandlers() sa.sa_handler = SIG_IGN; sigaction(SIGPIPE, &sa, 0); sa.sa_handler = WESplitterApp::onSigHup; - sigaction(SIGPIPE, &sa, 0); + sigaction(SIGHUP, &sa, 0); sa.sa_handler = WESplitterApp::onSigInterrupt; sigaction(SIGUSR1, &sa, 0); /* From 617ff85b94461967d4283770132b0dfae583894e Mon Sep 17 00:00:00 2001 From: Roman Nozdrin Date: Thu, 5 Apr 2018 17:20:26 +0300 Subject: [PATCH 10/57] MCOL-1052 WIP Working with HAVING over agg functions problem. --- dbcon/mysql/CMakeLists.txt | 4 +- dbcon/mysql/ha_calpont.cpp | 38 +- dbcon/mysql/ha_calpont_execplan.cpp | 644 ++++++++++++---------------- dbcon/mysql/ha_calpont_impl.cpp | 129 +----- dbcon/mysql/ha_calpont_impl_if.h | 2 +- 5 files changed, 313 insertions(+), 504 deletions(-) diff --git a/dbcon/mysql/CMakeLists.txt b/dbcon/mysql/CMakeLists.txt index 7721aed80..a1786481d 100644 --- a/dbcon/mysql/CMakeLists.txt +++ b/dbcon/mysql/CMakeLists.txt @@ -20,8 +20,8 @@ SET ( libcalmysql_SRCS ha_pseudocolumn.cpp) add_definitions(-DMYSQL_DYNAMIC_PLUGIN) -add_definitions(-DDEBUG_WALK_COND) -add_definitions(-DINFINIDB_DEBUG) +add_definitions(-DDEBUG_WALK_COND=1) +add_definitions(-DINFINIDB_DEBUG=1) #add_definitions(-DOUTER_JOIN_DEBUG) set_source_files_properties(ha_calpont.cpp PROPERTIES COMPILE_FLAGS "-fno-rtti -fno-implicit-templates") diff --git a/dbcon/mysql/ha_calpont.cpp b/dbcon/mysql/ha_calpont.cpp index a780a72e0..e07beade3 100644 --- a/dbcon/mysql/ha_calpont.cpp +++ b/dbcon/mysql/ha_calpont.cpp @@ -1139,9 +1139,7 @@ static MYSQL_SYSVAR_ULONG( 0); #endif -/*@brief ha_calpont_impl_group_by_init - Get data for MariaDB group_by - pushdown handler -*/ +/*@brief create_calpont_group_by_handler- Creates handler*/ /*********************************************************** * DESCRIPTION: * Creates a group_by pushdown handler. @@ -1158,16 +1156,27 @@ create_calpont_group_by_handler(THD *thd, Query *query) { ha_calpont_group_by_handler *handler = NULL; - handler = new ha_calpont_group_by_handler(thd, query); + if ( thd->infinidb_vtable.vtable_state == THD::INFINIDB_DISABLE_VTABLE ) + { + handler = new ha_calpont_group_by_handler(thd, query); + + // Notify the server, that CS handles GROUP BY, ORDER BY and HAVING clauses. + query->group_by = NULL; + query->order_by = NULL; + query->having = NULL; + } - // Notify the server, that CS handles GROUP BY, ORDER BY and HAVING clauses. - query->group_by = NULL; - query->order_by = NULL; - query->having = NULL; + return handler; } +/*********************************************************** + * DESCRIPTION: + * Makes the plan and prepares the data + * RETURN: + * int rc + ***********************************************************/ int ha_calpont_group_by_handler::init_scan() { DBUG_ENTER("ha_calpont_group_by_handler::init_scan"); @@ -1181,6 +1190,12 @@ int ha_calpont_group_by_handler::init_scan() DBUG_RETURN(rc); } +/*********************************************************** + * DESCRIPTION: + * Fetches a row and saves it to a temporary table. + * RETURN: + * int rc + ***********************************************************/ int ha_calpont_group_by_handler::next_row() { DBUG_ENTER("ha_calpont_group_by_handler::next_row"); @@ -1189,6 +1204,12 @@ int ha_calpont_group_by_handler::next_row() DBUG_RETURN(rc); } +/*********************************************************** + * DESCRIPTION: + * Shuts the scan down. + * RETURN: + * int rc + ***********************************************************/ int ha_calpont_group_by_handler::end_scan() { DBUG_ENTER("ha_calpont_group_by_handler::end_scan"); @@ -1196,7 +1217,6 @@ int ha_calpont_group_by_handler::end_scan() int rc = ha_calpont_impl_group_by_end(this, table); DBUG_RETURN(rc); -// return 0; } diff --git a/dbcon/mysql/ha_calpont_execplan.cpp b/dbcon/mysql/ha_calpont_execplan.cpp index 7ed3c0e69..78b65bd18 100644 --- a/dbcon/mysql/ha_calpont_execplan.cpp +++ b/dbcon/mysql/ha_calpont_execplan.cpp @@ -2713,7 +2713,7 @@ ReturnedColumn* buildReturnedColumn(Item* item, gp_walk_info& gwi, bool& nonSupp if ( gwi.thd) { - if ( ((gwi.thd->lex)->sql_command == SQLCOM_UPDATE ) || ((gwi.thd->lex)->sql_command == SQLCOM_UPDATE_MULTI )) + //if ( ((gwi.thd->lex)->sql_command == SQLCOM_UPDATE ) || ((gwi.thd->lex)->sql_command == SQLCOM_UPDATE_MULTI )) { if ( !item->fixed) { @@ -2959,6 +2959,260 @@ ReturnedColumn* buildReturnedColumn(Item* item, gp_walk_info& gwi, bool& nonSupp return rc; } +ReturnedColumn* buildReturnedColumnGr(Item* item, gp_walk_info& gwi, bool& nonSupport) +{ + ReturnedColumn* rc = NULL; + + if ( gwi.thd) + { + //if ( ((gwi.thd->lex)->sql_command == SQLCOM_UPDATE ) || ((gwi.thd->lex)->sql_command == SQLCOM_UPDATE_MULTI )) + { + if ( !item->fixed) + { + item->fix_fields(gwi.thd, (Item**)&item); + } + } + } + + Item::Type itype = Item::SUM_FUNC_ITEM; + + switch (itype) + { + case Item::FIELD_ITEM: + { + Item_field* ifp = (Item_field*)item; + return buildSimpleColumn(ifp, gwi); + } + + case Item::INT_ITEM: + case Item::VARBIN_ITEM: + { + String val, *str = item->val_str(&val); + string valStr; + valStr.assign(str->ptr(), str->length()); + + if (item->unsigned_flag) + { + //cc = new ConstantColumn(valStr, (uint64_t)item->val_uint(), ConstantColumn::NUM); + // It seems that str at this point is crap if val_uint() is > MAX_BIGINT. By using + // this constructor, ConstantColumn is built with the proper string. For whatever reason, + // ExeMgr converts the fConstval member to numeric, rather than using the existing numeric + // values available, so it's important to have fConstval correct. + rc = new ConstantColumn((uint64_t)item->val_uint(), ConstantColumn::NUM); + } + else + { + rc = new ConstantColumn(valStr, (int64_t)item->val_int(), ConstantColumn::NUM); + } + + //return cc; + break; + } + + case Item::STRING_ITEM: + { + String val, *str = item->val_str(&val); + string valStr; + valStr.assign(str->ptr(), str->length()); + rc = new ConstantColumn(valStr); + break; + } + + case Item::REAL_ITEM: + { + String val, *str = item->val_str(&val); + string valStr; + valStr.assign(str->ptr(), str->length()); + rc = new ConstantColumn(valStr, item->val_real()); + break; + } + + case Item::DECIMAL_ITEM: + { + rc = buildDecimalColumn(item, gwi); + break; + } + + case Item::FUNC_ITEM: + { + Item_func* ifp = (Item_func*)item; + string func_name = ifp->func_name(); + + // try to evaluate const F&E. only for select clause + vector tmpVec; + //bool hasAggColumn = false; + uint16_t parseInfo = 0; + parse_item(ifp, tmpVec, gwi.fatalParseError, parseInfo); + + if (parseInfo & SUB_BIT) + { + gwi.fatalParseError = true; + gwi.parseErrorText = IDBErrorInfo::instance()->errorMsg(ERR_NON_SUPPORT_SELECT_SUB); + setError(gwi.thd, ER_CHECK_NOT_IMPLEMENTED, gwi.parseErrorText, gwi); + break; + } + + if (!gwi.fatalParseError && + !nonConstFunc(ifp) && + !(parseInfo & AF_BIT) && + (tmpVec.size() == 0)) + { + String val, *str = ifp->val_str(&val); + string valStr; + + if (str) + { + valStr.assign(str->ptr(), str->length()); + } + + if (!str) + { + rc = new ConstantColumn("", ConstantColumn::NULLDATA); + } + else if (ifp->result_type() == STRING_RESULT) + { + rc = new ConstantColumn(valStr, ConstantColumn::LITERAL); + rc->resultType(colType_MysqlToIDB(item)); + } + else if (ifp->result_type() == DECIMAL_RESULT) + rc = buildDecimalColumn(ifp, gwi); + else + { + rc = new ConstantColumn(valStr, ConstantColumn::NUM); + rc->resultType(colType_MysqlToIDB(item)); + } + + break; + } + + if (func_name == "+" || func_name == "-" || func_name == "*" || func_name == "/" ) + return buildArithmeticColumn(ifp, gwi, nonSupport); + else + return buildFunctionColumn(ifp, gwi, nonSupport); + } + + case Item::SUM_FUNC_ITEM: + { + return buildAggregateColumn(item, gwi); + } + + case Item::REF_ITEM: + { + Item_ref* ref = (Item_ref*)item; + + switch ((*(ref->ref))->type()) + { + case Item::SUM_FUNC_ITEM: + return buildAggregateColumn(*(ref->ref), gwi); + + case Item::FIELD_ITEM: + return buildReturnedColumn(*(ref->ref), gwi, nonSupport); + + case Item::REF_ITEM: + return buildReturnedColumn(*(((Item_ref*)(*(ref->ref)))->ref), gwi, nonSupport); + + case Item::FUNC_ITEM: + return buildFunctionColumn((Item_func*)(*(ref->ref)), gwi, nonSupport); + + case Item::WINDOW_FUNC_ITEM: + return buildWindowFunctionColumn(*(ref->ref), gwi, nonSupport); + + default: + gwi.fatalParseError = true; + gwi.parseErrorText = "Unknown REF item"; + break; + } + } + + case Item::NULL_ITEM: + { + if (gwi.condPush) + return new SimpleColumn("noop"); + + return new ConstantColumn("", ConstantColumn::NULLDATA); + } + + case Item::CACHE_ITEM: + { + Item* col = ((Item_cache*)item)->get_example(); + rc = buildReturnedColumn(col, gwi, nonSupport); + + if (rc) + { + ConstantColumn* cc = dynamic_cast(rc); + + if (!cc) + { + rc->joinInfo(rc->joinInfo() | JOIN_CORRELATED); + + if (gwi.subQuery) + gwi.subQuery->correlated(true); + } + } + + break; + } + + case Item::EXPR_CACHE_ITEM: + { + // TODO: item is a Item_cache_wrapper + printf("EXPR_CACHE_ITEM in buildReturnedColumn\n"); + cerr << "EXPR_CACHE_ITEM in buildReturnedColumn" << endl; + break; + } + + case Item::DATE_ITEM: + { + String val, *str = item->val_str(&val); + string valStr; + valStr.assign(str->ptr(), str->length()); + rc = new ConstantColumn(valStr); + break; + } + + case Item::WINDOW_FUNC_ITEM: + { + return buildWindowFunctionColumn(item, gwi, nonSupport); + } + +#if INTERVAL_ITEM + + case Item::INTERVAL_ITEM: + { + Item_interval* interval = (Item_interval*)item; + SRCP srcp; + srcp.reset(buildReturnedColumn(interval->item, gwi, nonSupport)); + + if (!srcp) + return NULL; + + rc = new IntervalColumn(srcp, (int)interval->interval); + rc->resultType(srcp->resultType()); + break; + } + +#endif + + case Item::SUBSELECT_ITEM: + { + gwi.hasSubSelect = true; + break; + } + + default: + { + gwi.fatalParseError = true; + gwi.parseErrorText = "Unknown item type"; + break; + } + } + + if (rc && item->name) + rc->alias(item->name); + + return rc; +} + ArithmeticColumn* buildArithmeticColumn(Item_func* item, gp_walk_info& gwi, bool& nonSupport) { if (!(gwi.thd->infinidb_vtable.cal_conn_info)) @@ -5029,7 +5283,13 @@ void gp_walk(const Item* item, void* arg) gwip->clauseType = SELECT; if (col->type() != Item::COND_ITEM) - rc = buildReturnedColumn(col, *gwip, gwip->fatalParseError); + { + /*if ( col->type() == Item::FIELD_ITEM ) + { + rc = buildReturnedColumnGr(col, *gwip, gwip->fatalParseError); + } else*/ + rc = buildReturnedColumn(col, *gwip, gwip->fatalParseError); + } SimpleColumn* sc = dynamic_cast(rc); @@ -7937,8 +8197,7 @@ int cp_get_table_plan(THD* thd, SCSEP& csep, cal_table_info& ti) return 0; } -// ti is useless -int cp_get_group_plan(THD* thd, SCSEP& csep, cal_group_info& gi ) +int cp_get_group_plan(THD* thd, SCSEP& csep, cal_impl_if::cal_group_info& gi) { LEX* lex = thd->lex; idbassert(lex != 0); @@ -7957,176 +8216,15 @@ int cp_get_group_plan(THD* thd, SCSEP& csep, cal_group_info& gi ) cerr << *csep << endl ; cerr << "-------------- EXECUTION PLAN END --------------\n" << endl; - // Derived table projection and filter optimization. - //derivedTableOptimization(csep); - return 0; } -/* -int cp_get_group_plan(THD* thd, SCSEP& csep, cal_table_info& ti ) -{ - gp_walk_info* gwi = ti.condInfo; - List_iterator_fast it(*ti.groupByFields); - Item* item; - - if (!gwi) - gwi = new gp_walk_info(); - - gwi->thd = thd; - LEX* lex = thd->lex; - idbassert(lex != 0); - uint32_t sessionID = csep->sessionID(); - gwi->sessionid = sessionID; - TABLE* table = ti.msTablePtr; - TABLE_LIST* table_list = ti.groupByTables; - boost::shared_ptr csc = - CalpontSystemCatalog::makeCalpontSystemCatalog(sessionID); - csc->identity(CalpontSystemCatalog::FE); - - // get all columns that mysql needs to fetch - MY_BITMAP* read_set = table->read_set; - Field** f_ptr, *field; - gwi->columnMap.clear(); - - CalpontSystemCatalog::TableAliasName tan= - make_aliastable(table->s->db.str, table->s->table_name.str, table->alias.c_ptr()); - - gwi->tbList.push_back(tan); - gwi->tableMap[tan] = make_pair(0, table_list); - - while ((item = it++)) - { - Item *arg0; - Field *field; - ReturnedColumn* rc = buildAggregateColumn(item, *gwi); - //string alias(table->alias.c_ptr()); - //rc->tableAlias(lower(alias)); - assert (rc); - boost::shared_ptr sprc(rc); - gwi->returnedCols.push_back(sprc); - } - - // TO DO add table into tableMap - // TO DO add the column only if there is no any for a table. - TableMap::iterator tb_iter = gwi->tableMap.begin(); - try - { - for (; tb_iter != gwi->tableMap.end(); tb_iter++) - { - if ((*tb_iter).second.first == 1) continue; - - CalpontSystemCatalog::TableAliasName tan = (*tb_iter).first; - CalpontSystemCatalog::TableName tn = make_table((*tb_iter).first.schema, (*tb_iter).first.table); - SimpleColumn* sc = getSmallestColumn(csc, tn, tan, (*tb_iter).second.second->table, *gwi); - SRCP srcp(sc); - gwi->columnMap.insert(CalpontSelectExecutionPlan::ColumnMap::value_type(sc->columnName(), srcp)); - (*tb_iter).second.first = 1; - } - } - catch (runtime_error& e) - { - setError(gwi->thd, ER_INTERNAL_ERROR, e.what(), *gwi); - CalpontSystemCatalog::removeCalpontSystemCatalog(sessionID); - return ER_INTERNAL_ERROR; - } - catch (...) - { - string emsg = IDBErrorInfo::instance()->errorMsg(ERR_LOST_CONN_EXEMGR); - setError(gwi->thd, ER_INTERNAL_ERROR, emsg, *gwi); - CalpontSystemCatalog::removeCalpontSystemCatalog(sessionID); - return ER_INTERNAL_ERROR; - } - - //if (gwi->columnMap.empty()) - //{ - // CalpontSystemCatalog::TableName tn = make_table(table->s->db.str, table->s->table_name.str); - // CalpontSystemCatalog::TableAliasName tan = make_aliastable(table->s->db.str, table->s->table_name.str, table->alias.c_ptr()); - // SimpleColumn* sc = getSmallestColumn(csc, tn, tan, table, *gwi); - // SRCP srcp(sc); - // gwi->columnMap.insert(CalpontSelectExecutionPlan::ColumnMap::value_type(sc->columnName(), srcp)); - // gwi->returnedCols.push_back(srcp); - //} - - - CalpontSystemCatalog::TableName tn = make_table(table->s->db.str, table->s->table_name.str); - //SimpleColumn* minSc = getSmallestColumn(csc, tn, tan, table, *gwi); - SRCP minSc; - minSc = gwi->returnedCols[0]; - std::vector::iterator coliter; - for (coliter = gwi->count_asterisk_list.begin(); coliter != gwi->count_asterisk_list.end(); ++coliter) - { - (*coliter)->functionParms(minSc); - } - - // get filter - if (ti.condInfo) - { - gp_walk_info* gwi = ti.condInfo; - ParseTree* filters = 0; - ParseTree* ptp = 0; - ParseTree* rhs = 0; - - while (!gwi->ptWorkStack.empty()) - { - filters = gwi->ptWorkStack.top(); - gwi->ptWorkStack.pop(); - SimpleFilter* sf = dynamic_cast(filters->data()); - - if (sf && sf->op()->data() == "noop") - { - delete filters; - filters = 0; - - if (gwi->ptWorkStack.empty()) - break; - - continue; - } - - if (gwi->ptWorkStack.empty()) - break; - - ptp = new ParseTree(new LogicOperator("and")); - ptp->left(filters); - rhs = gwi->ptWorkStack.top(); - gwi->ptWorkStack.pop(); - ptp->right(rhs); - gwi->ptWorkStack.push(ptp); - } - - csep->filters(filters); - } - - csep->returnedCols(gwi->returnedCols); - csep->columnMap(gwi->columnMap); - CalpontSelectExecutionPlan::TableList tblist; - tblist.push_back(tan); - csep->tableList(tblist); - - // @bug 3321. Set max number of blocks in a dictionary file to be scanned for filtering - csep->stringScanThreshold(gwi->thd->variables.infinidb_string_scan_threshold); - - return 0; -} -*/ - int getGroupPlan(gp_walk_info& gwi, SELECT_LEX& select_lex, SCSEP& csep, cal_group_info& gi, bool isUnion) { #ifdef DEBUG_WALK_COND cerr << "getGroupPlan()" << endl; #endif - // by pass the derived table resolve phase of mysql - /*if (!(((gwi.thd->lex)->sql_command == SQLCOM_UPDATE ) || - ((gwi.thd->lex)->sql_command == SQLCOM_DELETE ) || - ((gwi.thd->lex)->sql_command == SQLCOM_UPDATE_MULTI ) || - ((gwi.thd->lex)->sql_command == SQLCOM_DELETE_MULTI ) ) && gwi.thd->derived_tables_processing) - { - gwi.thd->infinidb_vtable.isUnion = false; - return -1; - } - */ // rollup is currently not supported if (select_lex.olap == ROLLUP_TYPE) { @@ -8324,74 +8422,9 @@ int getGroupPlan(gp_walk_info& gwi, SELECT_LEX& select_lex, SCSEP& csep, cal_gro bool unionSel = false; - // MCOL-1052 - /* - if (!isUnion && select_lex.master_unit()->is_union()) - { -// gwi.thd->infinidb_vtable.isUnion = true; - CalpontSelectExecutionPlan::SelectList unionVec; - SELECT_LEX* select_cursor = select_lex.master_unit()->first_select(); - unionSel = true; - uint8_t distUnionNum = 0; - - for (SELECT_LEX* sl = select_cursor; sl; sl = sl->next_select()) - { - SCSEP plan(new CalpontSelectExecutionPlan()); - plan->txnID(csep->txnID()); - plan->verID(csep->verID()); - plan->sessionID(csep->sessionID()); - plan->traceFlags(csep->traceFlags()); - plan->data(csep->data()); - - // @bug 3853. When one or more sides or union queries contain derived tables, - // sl->join->zero_result_cause is not trustable. Since we've already handled - // constant filter now (0/1), we can relax the following checking. - // @bug 2547. ignore union unit of zero result set case -// if (sl->join) -// { -// sl->join->optimize(); - // @bug 3067. not clear MySQL's behavior. when in subquery, this variable - // is not trustable. -// if (sl->join->zero_result_cause && !gwi.subQuery) -// continue; -// } - - // gwi for the union unit - gp_walk_info union_gwi; - union_gwi.thd = gwi.thd; - uint32_t err = 0; - - if ((err = getGroupPlan(union_gwi, *sl, plan, gi, unionSel)) != 0) - return err; - - unionVec.push_back(SCEP(plan)); - - // distinct union num - if (sl == select_lex.master_unit()->union_distinct) - distUnionNum = unionVec.size(); - - *//*#ifdef DEBUG_WALK_COND - IDEBUG( cerr << ">>>> UNION DEBUG" << endl ); - JOIN* join = sl->join; - Item_cond* icp = 0; - if (join != 0) - icp = reinterpret_cast(join->conds); - if (icp) - icp->traverse_cond(debug_walk, &gwi, Item::POSTFIX); - IDEBUG ( cerr << *plan << endl ); - IDEBUG ( cerr << "<<<unionVec(unionVec); - csep->distinctUnionNum(distUnionNum); - - if (unionVec.empty()) - gwi.thd->infinidb_vtable.impossibleWhereOnUnion = true; - } - */ gwi.clauseType = WHERE; + if (icp) { // MCOL-1052 The condition could be useless. @@ -8434,6 +8467,7 @@ int getGroupPlan(gp_walk_info& gwi, SELECT_LEX& select_lex, SCSEP& csep, cal_gro { gwi.rcWorkStack.push(new ConstantColumn((int64_t)0, ConstantColumn::NUM)); } + // ZZ - the followinig debug shows the structure of nested outer join. should @@ -9148,9 +9182,11 @@ int getGroupPlan(gp_walk_info& gwi, SELECT_LEX& select_lex, SCSEP& csep, cal_gro gwi.fatalParseError = false; gwi.parseErrorText = ""; - if (gi.groupByHaving != 0) + //if (gi.groupByHaving != 0) + if ( select_lex.having != 0 ) { - Item_cond* having = reinterpret_cast(gi.groupByHaving); + Item_cond* having = reinterpret_cast(select_lex.having); + //Item_cond* having = reinterpret_cast(gi.groupByHaving); #ifdef DEBUG_WALK_COND cerr << "------------------- HAVING ---------------------" << endl; having->traverse_cond(debug_walk, &gwi, Item::POSTFIX); @@ -9270,13 +9306,11 @@ int getGroupPlan(gp_walk_info& gwi, SELECT_LEX& select_lex, SCSEP& csep, cal_gro SRCP minSc; // min width projected column. for count(*) use // Group by list. not valid for union main query - - /* if (gwi.thd->infinidb_vtable.vtable_state == THD::INFINIDB_CREATE_VTABLE && !unionSel) { gwi.clauseType = GROUP_BY; Item* nonSupportItem = NULL; - ORDER* groupcol = reinterpret_cast(gi.groupByGr.first); + ORDER* groupcol = reinterpret_cast(gi.groupByGroup); // check if window functions are in order by. InfiniDB process order by list if // window functions are involved, either in order by or projection. @@ -9298,7 +9332,7 @@ int getGroupPlan(gp_walk_info& gwi, SELECT_LEX& select_lex, SCSEP& csep, cal_gro } gwi.hasWindowFunc = hasWindowFunc; - groupcol = reinterpret_cast(gi.groupByGr.first); + groupcol = reinterpret_cast(gi.groupByGroup); for (; groupcol; groupcol = groupcol->next) { @@ -9547,13 +9581,11 @@ int getGroupPlan(gp_walk_info& gwi, SELECT_LEX& select_lex, SCSEP& csep, cal_gro setError(gwi.thd, ER_CHECK_NOT_IMPLEMENTED, gwi.parseErrorText, gwi); return ER_CHECK_NOT_IMPLEMENTED; } - } - */ + } // GROUP processing ends here if (gwi.thd->infinidb_vtable.vtable_state == THD::INFINIDB_CREATE_VTABLE) { - //SQL_I_List order_list = gi.groupByOrder; ORDER* ordercol = reinterpret_cast(gi.groupByOrder); string create_query(gwi.thd->infinidb_vtable.create_vtable_query.c_ptr()); string select_query(gwi.thd->infinidb_vtable.select_vtable_query.c_ptr()); @@ -9576,7 +9608,7 @@ int getGroupPlan(gp_walk_info& gwi, SELECT_LEX& select_lex, SCSEP& csep, cal_gro // for subquery, order+limit by will be supported in infinidb. build order by columns // @todo union order by and limit support - if (gwi.hasWindowFunc || gwi.subSelectType != CalpontSelectExecutionPlan::MAIN_SELECT) + //if (gwi.hasWindowFunc || gwi.subSelectType != CalpontSelectExecutionPlan::MAIN_SELECT) { for (; ordercol; ordercol = ordercol->next) { @@ -9634,6 +9666,7 @@ int getGroupPlan(gp_walk_info& gwi, SELECT_LEX& select_lex, SCSEP& csep, cal_gro gwi.orderByCols.push_back(SRCP(rc)); } } + /* else if (!isUnion) { vector fieldVec; @@ -9817,12 +9850,6 @@ int getGroupPlan(gp_walk_info& gwi, SELECT_LEX& select_lex, SCSEP& csep, cal_gro Item_field* field = reinterpret_cast(ord_item); ReturnedColumn* rc = buildSimpleColumn(field, gwi); fullname = field->full_name(); -// if (field->db_name) -// fullname += string(field->db_name) + "."; -// if (field->table_name) -// fullname += string(field->table_name) + "."; -// if (field->field_name) -// fullname += string(field->field_name); for (uint32_t i = 0; i < gwi.returnedCols.size(); i++) { @@ -9933,6 +9960,7 @@ int getGroupPlan(gp_walk_info& gwi, SELECT_LEX& select_lex, SCSEP& csep, cal_gro } } } + */ // make sure columnmap, returnedcols and count(*) arg_list are not empty TableMap::iterator tb_iter = gwi.tableMap.begin(); @@ -10291,127 +10319,7 @@ int getGroupPlan(gp_walk_info& gwi, SELECT_LEX& select_lex, SCSEP& csep, cal_gro select_query += ord_cols; } } - -// MCOL-1052 LIMIT -/* - if (unionSel || gwi.subSelectType != CalpontSelectExecutionPlan::MAIN_SELECT) - { - if (select_lex.master_unit()->global_parameters()->explicit_limit) - { - if (select_lex.master_unit()->global_parameters()->offset_limit) - { - Item_int* offset = (Item_int*)select_lex.master_unit()->global_parameters()->offset_limit; - csep->limitStart(offset->val_int()); - } - - if (select_lex.master_unit()->global_parameters()->select_limit) - { - Item_int* select = (Item_int*)select_lex.master_unit()->global_parameters()->select_limit; - csep->limitNum(select->val_int()); - } - - if (unionSel && gwi.subSelectType == CalpontSelectExecutionPlan::MAIN_SELECT) - { - ostringstream limit; - limit << " limit "; - limit << csep->limitStart() << ", "; - limit << csep->limitNum(); - select_query += limit.str(); - } - } - } - else if (isUnion && select_lex.explicit_limit) - { - if (select_lex.braces) - { - if (select_lex.offset_limit) - csep->limitStart(((Item_int*)select_lex.offset_limit)->val_int()); - - if (select_lex.select_limit) - csep->limitNum(((Item_int*)select_lex.select_limit)->val_int()); - } - } - else if (select_lex.explicit_limit) - { - uint32_t limitOffset = 0; - uint32_t limitNum = std::numeric_limits::max(); - - if (join) - { -#if MYSQL_VERSION_ID >= 50172 - - // @bug5729. After upgrade, join->unit sometimes is uninitialized pointer - // (not null though) and will cause seg fault. Prefer checking - // select_lex->offset_limit if not null. - if (join->select_lex && - join->select_lex->offset_limit && - join->select_lex->offset_limit->fixed && - join->select_lex->select_limit && - join->select_lex->select_limit->fixed) - { - limitOffset = join->select_lex->offset_limit->val_int(); - limitNum = join->select_lex->select_limit->val_int(); - } - - else if (join->unit) - { - limitOffset = join->unit->offset_limit_cnt; - limitNum = join->unit->select_limit_cnt - limitOffset; - } - -#else - limitOffset = (join->unit)->offset_limit_cnt; - limitNum = (join->unit)->select_limit_cnt - (join->unit)->offset_limit_cnt; -#endif - } - else - { - if (select_lex.master_unit()->global_parameters()->offset_limit) - { - Item_int* offset = (Item_int*)select_lex.master_unit()->global_parameters()->offset_limit; - limitOffset = offset->val_int(); - } - - if (select_lex.master_unit()->global_parameters()->select_limit) - { - Item_int* select = (Item_int*)select_lex.master_unit()->global_parameters()->select_limit; - limitNum = select->val_int(); - } - } - - // relate to bug4848. let mysql drive limit when limit session variable set. - // do not set in csep. @bug5096. ignore session limit setting for dml - if ((gwi.thd->variables.select_limit == (uint64_t) - 1 || - (gwi.thd->variables.select_limit != (uint64_t) - 1 && - gwi.thd->infinidb_vtable.vtable_state != THD::INFINIDB_CREATE_VTABLE)) && - !csep->hasOrderBy()) - { - csep->limitStart(limitOffset); - csep->limitNum(limitNum); - } - else - { - ostringstream limit; - limit << " limit " << limitOffset << ", " << limitNum; - select_query += limit.str(); - } - } - gwi.thd->infinidb_vtable.select_vtable_query.free(); - gwi.thd->infinidb_vtable.select_vtable_query.append(select_query.c_str(), select_query.length()); - - // We don't currently support limit with correlated subquery - if (csep->limitNum() != (uint64_t) - 1 && - gwi.subQuery && !gwi.correlatedTbNameVec.empty()) - { - gwi.fatalParseError = true; - gwi.parseErrorText = IDBErrorInfo::instance()->errorMsg(ERR_NON_SUPPORT_LIMIT_SUB); - setError(gwi.thd, ER_INTERNAL_ERROR, gwi.parseErrorText, gwi); - return ER_CHECK_NOT_IMPLEMENTED; - } - */ - } - - + } // ORDER BY processing ends here if ( gi.groupByDistinct ) csep->distinct(true); diff --git a/dbcon/mysql/ha_calpont_impl.cpp b/dbcon/mysql/ha_calpont_impl.cpp index 09e125421..4232d4985 100644 --- a/dbcon/mysql/ha_calpont_impl.cpp +++ b/dbcon/mysql/ha_calpont_impl.cpp @@ -5932,7 +5932,7 @@ int ha_calpont_impl_rnd_pos(uchar* buf, uchar* pos) * Prepares data for group_by_handler::next_row() calls. * PARAMETERS: * group_hand - group by handler, that preserves initial table and items lists. . - * table - TABLE pointer The table to save the result set in. + * table - TABLE pointer The table to save the result set into. * RETURN: * 0 if success * others if something went wrong whilst getting the result set @@ -5967,25 +5967,6 @@ int ha_calpont_impl_group_by_init(ha_calpont_group_by_handler* group_hand, TABLE // prevent "create table as select" from running on slave thd->infinidb_vtable.hasInfiniDBTable = true; - /* If this node is the slave, ignore DML to IDB tables */ - /*if (thd->slave_thread && ( - thd->lex->sql_command == SQLCOM_INSERT || - thd->lex->sql_command == SQLCOM_INSERT_SELECT || - thd->lex->sql_command == SQLCOM_UPDATE || - thd->lex->sql_command == SQLCOM_UPDATE_MULTI || - thd->lex->sql_command == SQLCOM_DELETE || - thd->lex->sql_command == SQLCOM_DELETE_MULTI || - thd->lex->sql_command == SQLCOM_TRUNCATE || - thd->lex->sql_command == SQLCOM_LOAD)) - return 0; - */ - - // @bug 3005. if the table is not $vtable, then this could be a UDF defined on the connector. - // watch this for other complications - //if (thd->infinidb_vtable.vtable_state == THD::INFINIDB_SELECT_VTABLE && - // string(table->s->table_name.str).find("$vtable") != 0) - // return 0; - // return error if error status has been already set if (thd->infinidb_vtable.vtable_state == THD::INFINIDB_ERROR) return ER_INTERNAL_ERROR; @@ -6007,14 +5988,6 @@ int ha_calpont_impl_group_by_init(ha_calpont_group_by_handler* group_hand, TABLE if (thd->infinidb_vtable.vtable_state == THD::INFINIDB_REDO_PHASE1 || thd->infinidb_vtable.vtable_state == THD::INFINIDB_ORDER_BY) return 0; - - if ( (thd->lex)->sql_command == SQLCOM_ALTER_TABLE ) - return 0; - - - //Update and delete code - if ( ((thd->lex)->sql_command == SQLCOM_UPDATE) || ((thd->lex)->sql_command == SQLCOM_DELETE) || ((thd->lex)->sql_command == SQLCOM_DELETE_MULTI) || ((thd->lex)->sql_command == SQLCOM_UPDATE_MULTI)) - return doUpdateDelete(thd); */ uint32_t sessionID = tid2sid(thd->thread_id); @@ -6032,10 +6005,11 @@ int ha_calpont_impl_group_by_init(ha_calpont_group_by_handler* group_hand, TABLE MySQL sometimes calls rnd_init multiple times, plan should only be generated and sent once. TO DO Check this statement. - */ + if (thd->infinidb_vtable.vtable_state == THD::INFINIDB_DISABLE_VTABLE && !thd->infinidb_vtable.isNewQuery) return 0; + */ if (thd->killed == KILL_QUERY || thd->killed == KILL_QUERY_HARD) { @@ -6068,101 +6042,9 @@ int ha_calpont_impl_group_by_init(ha_calpont_group_by_handler* group_hand, TABLE sm::cpsm_conhdl_t* hndl; SCSEP csep; - // update traceFlags according to the autoswitch state. replication query - // on slave are in table mode (create table as...) - if (thd->infinidb_vtable.vtable_state == THD::INFINIDB_DISABLE_VTABLE) - { - ci->traceFlags |= CalpontSelectExecutionPlan::TRACE_TUPLE_OFF; - } - - // MCOL-1052 TO DO Remove this - ci->traceFlags = CalpontSelectExecutionPlan::TRACE_LOG; - bool localQuery = (thd->variables.infinidb_local_query > 0 ? true : false); - /* - // table mode - if (thd->infinidb_vtable.vtable_state == THD::INFINIDB_DISABLE_VTABLE) - { - ti = ci->tableMap[group_hand->table_list->table]; - - // get connection handle for this table handler - // re-establish table handle - if (ti.conn_hndl) - { - sm::sm_cleanup(ti.conn_hndl); - ti.conn_hndl = 0; - } - - sm::sm_init(sessionID, &ti.conn_hndl, localQuery); - ti.conn_hndl->csc = csc; - hndl = ti.conn_hndl; - - try - { - ti.conn_hndl->connect(); - } - catch (...) - { - setError(thd, ER_INTERNAL_ERROR, IDBErrorInfo::instance()->errorMsg(ERR_LOST_CONN_EXEMGR)); - CalpontSystemCatalog::removeCalpontSystemCatalog(sessionID); - goto error; - } - - // get filter plan for table - if (ti.csep.get() == 0) - { - ti.csep.reset(new CalpontSelectExecutionPlan()); - - SessionManager sm; - BRM::TxnID txnID; - txnID = sm.getTxnID(sessionID); - - if (!txnID.valid) - { - txnID.id = 0; - txnID.valid = true; - } - - QueryContext verID; - verID = sm.verID(); - - ti.csep->txnID(txnID.id); - ti.csep->verID(verID); - ti.csep->sessionID(sessionID); - - if (group_hand->table_list->db_length) - ti.csep->schemaName(group_hand->table_list->db); - - ti.csep->traceFlags(ci->traceFlags); - ti.msTablePtr = group_hand->table_list->table; - //ti.groupByTables = group_hand->table_list; - //ti.groupByFields = group_hand->fields; - - gi.groupByTables = group_hand->table_list; - gi.groupByFields = group_hand->select; - gi.groupByWhere = group_hand->where; - gi.groupByGroup = group_hand->group_by; - gi.groupByOrder = group_hand->order_by; - gi.groupByHaving = group_hand->having; - gi.groupByDistinct = group_hand->distinct; - - // send plan whenever group_init is called - cp_get_group_plan(thd, ti.csep, ti, gi); - } - - IDEBUG( cerr << tableName << " send plan:" << endl ); - IDEBUG( cerr << *ti.csep << endl ); - csep = ti.csep; - - // for ExeMgr logging sqltext. only log once for the query although multi plans may be sent - if (ci->tableMap.size() == 1) - ti.csep->data(idb_mysql_query_str(thd)); - } - */ - // vtable mode - //else - { + //if (!ci->cal_conn_hndl || thd->infinidb_vtable.vtable_state == THD::INFINIDB_CREATE_VTABLE) if ( thd->infinidb_vtable.vtable_state == THD::INFINIDB_CREATE_VTABLE) { @@ -6240,7 +6122,6 @@ int ha_calpont_impl_group_by_init(ha_calpont_group_by_handler* group_hand, TABLE csep->schemaName(group_hand->table_list->db); csep->traceFlags(ci->traceFlags); - //ti.msTablePtr = group_hand->table_list->table; gi.groupByTables = group_hand->table_list; gi.groupByFields = group_hand->select; @@ -6311,7 +6192,7 @@ int ha_calpont_impl_group_by_init(ha_calpont_group_by_handler* group_hand, TABLE IDEBUG( cout << "-------------- EXECUTION PLAN END --------------\n" << endl ); } } - }// end of execution plan generation + // end of execution plan generation if (thd->infinidb_vtable.vtable_state != THD::INFINIDB_SELECT_VTABLE) { diff --git a/dbcon/mysql/ha_calpont_impl_if.h b/dbcon/mysql/ha_calpont_impl_if.h index 21560a4a0..b74025d54 100644 --- a/dbcon/mysql/ha_calpont_impl_if.h +++ b/dbcon/mysql/ha_calpont_impl_if.h @@ -320,7 +320,7 @@ int cp_get_plan(THD* thd, execplan::SCSEP& csep); int cp_get_table_plan(THD* thd, execplan::SCSEP& csep, cal_impl_if::cal_table_info& ti); int cp_get_group_plan(THD* thd, execplan::SCSEP& csep, cal_impl_if::cal_group_info& gi); int getSelectPlan(gp_walk_info& gwi, SELECT_LEX& select_lex, execplan::SCSEP& csep, bool isUnion = false); -int getGroupPlan(gp_walk_info& gwi, SELECT_LEX& select_lex, execplan::SCSEP& csep,cal_group_info& gi, bool isUnion = false); +int getGroupPlan(gp_walk_info& gwi, SELECT_LEX& select_lex, execplan::SCSEP& csep, cal_group_info& gi, bool isUnion = false); void setError(THD* thd, uint32_t errcode, const std::string errmsg, gp_walk_info* gwi); void setError(THD* thd, uint32_t errcode, const std::string errmsg); void gp_walk(const Item* item, void* arg); From 6bd13849b290ac490a7980c734030113123a8159 Mon Sep 17 00:00:00 2001 From: Roman Nozdrin Date: Sat, 7 Apr 2018 04:46:34 +0300 Subject: [PATCH 11/57] MCOL-1052 WIP Use SELECT fields referenced in HAVING items list. --- dbcon/mysql/ha_calpont_execplan.cpp | 22 ++++++++++++++++++++++ dbcon/mysql/ha_calpont_impl_if.h | 1 + 2 files changed, 23 insertions(+) diff --git a/dbcon/mysql/ha_calpont_execplan.cpp b/dbcon/mysql/ha_calpont_execplan.cpp index 78b65bd18..5663f2871 100644 --- a/dbcon/mysql/ha_calpont_execplan.cpp +++ b/dbcon/mysql/ha_calpont_execplan.cpp @@ -5289,6 +5289,8 @@ void gp_walk(const Item* item, void* arg) rc = buildReturnedColumnGr(col, *gwip, gwip->fatalParseError); } else*/ rc = buildReturnedColumn(col, *gwip, gwip->fatalParseError); + if ( col->type() == Item::FIELD_ITEM ) + gwip->fatalParseError = false; } SimpleColumn* sc = dynamic_cast(rc); @@ -5361,6 +5363,19 @@ void gp_walk(const Item* item, void* arg) Item_func* ifp = (Item_func*)col; gwip->ptWorkStack.push(buildParseTree(ifp, *gwip, gwip->fatalParseError)); } + else if (col->type() == Item::FIELD_ITEM && gwip->clauseType == HAVING) + { + // Найти item в списке groupByFields + Item_func_or_sum* isfp = reinterpret_cast(gwip->havingAggColsItems[0]); + if ( isfp->type() == Item::SUM_FUNC_ITEM ) + { + ReturnedColumn* rc = buildAggregateColumn(isfp, *gwip); + } + + if (rc) + gwip->rcWorkStack.push(rc); + break; + } else cando = false; @@ -8725,6 +8740,13 @@ int getGroupPlan(gp_walk_info& gwi, SELECT_LEX& select_lex, SCSEP& csep, cal_gro // add this agg col to returnedColumnList boost::shared_ptr spac(ac); gwi.returnedCols.push_back(spac); + // This item will be used in HAVING clause later. + Item_func_or_sum* isfp = reinterpret_cast(item); + if ( ! isfp->name_length ) + { + gwi.havingAggColsItems.push_back(item); + } + gwi.selectCols.push_back('`' + escapeBackTick(spac->alias().c_str()) + '`'); String str(256); item->print(&str, QT_INFINIDB_NO_QUOTE); diff --git a/dbcon/mysql/ha_calpont_impl_if.h b/dbcon/mysql/ha_calpont_impl_if.h index b74025d54..83f6aaf41 100644 --- a/dbcon/mysql/ha_calpont_impl_if.h +++ b/dbcon/mysql/ha_calpont_impl_if.h @@ -99,6 +99,7 @@ struct gp_walk_info execplan::CalpontSelectExecutionPlan::ReturnedColumnList groupByCols; execplan::CalpontSelectExecutionPlan::ReturnedColumnList subGroupByCols; execplan::CalpontSelectExecutionPlan::ReturnedColumnList orderByCols; + std::vector havingAggColsItems; execplan::CalpontSelectExecutionPlan::ColumnMap columnMap; // This vector temporarily hold the projection columns to be added // to the returnedCols vector for subquery processing. It will be appended From bd8387202fccaf90dcdd706b52bc56e58e296120 Mon Sep 17 00:00:00 2001 From: david hill Date: Mon, 9 Apr 2018 08:55:45 -0500 Subject: [PATCH 12/57] MCOL-912 - added code to restart dbrm/ddml process when enable module is done --- procmgr/processmanager.cpp | 19 +++++++++++++++++-- procmgr/processmanager.h | 2 +- 2 files changed, 18 insertions(+), 3 deletions(-) diff --git a/procmgr/processmanager.cpp b/procmgr/processmanager.cpp index fdb3f76b1..2172b6153 100755 --- a/procmgr/processmanager.cpp +++ b/procmgr/processmanager.cpp @@ -3385,12 +3385,12 @@ int ProcessManager::disableModule(string target, bool manualFlag) * purpose: recyle process, generally after some disable module is run * ******************************************************************************************/ -void ProcessManager::recycleProcess(string module) +void ProcessManager::recycleProcess(string module, bool enableModule) { Oam oam; ModuleConfig moduleconfig; - log.writeLog(__LINE__, "recycleProcess request after module was disabled: " + module, LOG_TYPE_DEBUG); + log.writeLog(__LINE__, "recycleProcess request after module statsu update: " + module, LOG_TYPE_DEBUG); string moduleType = module.substr(0,MAX_MODULE_TYPE_SIZE); @@ -3399,11 +3399,23 @@ void ProcessManager::recycleProcess(string module) oam.getSystemConfig("PrimaryUMModuleName", PrimaryUMModuleName); } catch(...) {} + + // restart DMLProc and return if enable module is being done + if (enableModule) + { + //recycle DBRM processes in all cases + restartProcessType("DBRMControllerNode"); + restartProcessType("DBRMWorkerNode"); + + restartProcessType("DMLProc"); + return; + } //recycle DBRM processes in all cases restartProcessType("DBRMControllerNode", module); restartProcessType("DBRMWorkerNode"); + // only recycle dmlproc, if down/up module is non-parent UM if ( ( moduleType == "um" ) && ( PrimaryUMModuleName != module) ) @@ -3483,6 +3495,9 @@ int ProcessManager::enableModule(string target, int state) if ( newStandbyModule == target) setStandbyModule(newStandbyModule); + //set recycle process + recycleProcess(target); + log.writeLog(__LINE__, "enableModule request for " + target + " completed", LOG_TYPE_DEBUG); return API_SUCCESS; diff --git a/procmgr/processmanager.h b/procmgr/processmanager.h index d60a9eaf2..a9ba826eb 100644 --- a/procmgr/processmanager.h +++ b/procmgr/processmanager.h @@ -302,7 +302,7 @@ public: /** *@brief recycle Processes */ - void recycleProcess(std::string module); + void recycleProcess(std::string module, bool enableModule = false); /** *@brief Enable a specified module From afb8e4706d58fffbae0631add8c90cf21343d121 Mon Sep 17 00:00:00 2001 From: david hill Date: Mon, 9 Apr 2018 11:35:37 -0500 Subject: [PATCH 13/57] MCOL-912 - fix spelling error --- procmgr/processmanager.cpp | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/procmgr/processmanager.cpp b/procmgr/processmanager.cpp index 2172b6153..6db1a1660 100755 --- a/procmgr/processmanager.cpp +++ b/procmgr/processmanager.cpp @@ -3390,7 +3390,7 @@ void ProcessManager::recycleProcess(string module, bool enableModule) Oam oam; ModuleConfig moduleconfig; - log.writeLog(__LINE__, "recycleProcess request after module statsu update: " + module, LOG_TYPE_DEBUG); + log.writeLog(__LINE__, "recycleProcess request after module status update: " + module, LOG_TYPE_DEBUG); string moduleType = module.substr(0,MAX_MODULE_TYPE_SIZE); @@ -3400,7 +3400,7 @@ void ProcessManager::recycleProcess(string module, bool enableModule) } catch(...) {} - // restart DMLProc and return if enable module is being done + // restart DBRM Process and DMLProc and return if enable module is being done if (enableModule) { //recycle DBRM processes in all cases From 05bca5382dab2ffa375163ce01ea65745bca44d0 Mon Sep 17 00:00:00 2001 From: Roman Nozdrin Date: Tue, 10 Apr 2018 04:27:08 +0300 Subject: [PATCH 14/57] MCOL-1052 WIP HAVING filter works. Decline non GROUP BY fields not presented in the ORDER BY list with error. --- dbcon/mysql/ha_calpont.cpp | 2 +- dbcon/mysql/ha_calpont_execplan.cpp | 72 +++++++++++++++++++++-------- 2 files changed, 55 insertions(+), 19 deletions(-) diff --git a/dbcon/mysql/ha_calpont.cpp b/dbcon/mysql/ha_calpont.cpp index e07beade3..b649942d6 100644 --- a/dbcon/mysql/ha_calpont.cpp +++ b/dbcon/mysql/ha_calpont.cpp @@ -1162,7 +1162,7 @@ create_calpont_group_by_handler(THD *thd, Query *query) // Notify the server, that CS handles GROUP BY, ORDER BY and HAVING clauses. query->group_by = NULL; - query->order_by = NULL; + //query->order_by = NULL; query->having = NULL; } diff --git a/dbcon/mysql/ha_calpont_execplan.cpp b/dbcon/mysql/ha_calpont_execplan.cpp index 5663f2871..6013c77d4 100644 --- a/dbcon/mysql/ha_calpont_execplan.cpp +++ b/dbcon/mysql/ha_calpont_execplan.cpp @@ -503,7 +503,12 @@ void debug_walk(const Item* item, void* arg) Item_sum* isp = (Item_sum*)item; char* item_name = item->name; - if (!item_name) + // MCOL-1052 This is and item from extended SELECT list + if (!item_name && isp->get_arg_count() and isp->get_arg(0)->name) + { + item_name = isp->get_arg(0)->name; + } + else if (!item_name) { item_name = (char*)""; } @@ -697,8 +702,17 @@ void debug_walk(const Item* item, void* arg) else if (ref->real_item()->type() == Item::FIELD_ITEM) { Item_field* ifp = (Item_field*)ref->real_item(); - cerr << "REF FIELD_ITEM: " << ifp->db_name << '.' << bestTableName(ifp) << '.' << - ifp->field_name << endl; + // MCOL-1052 The field referenced presumable came from + // extended SELECT list. + if ( !ifp->field_name ) + { + cerr << "REF extra FIELD_ITEM: " << ifp->name << endl; + } + else + { + cerr << "REF FIELD_ITEM: " << ifp->db_name << '.' << bestTableName(ifp) << '.' << + ifp->field_name << endl; + } break; } else if (ref->real_item()->type() == Item::FUNC_ITEM) @@ -5283,14 +5297,10 @@ void gp_walk(const Item* item, void* arg) gwip->clauseType = SELECT; if (col->type() != Item::COND_ITEM) - { - /*if ( col->type() == Item::FIELD_ITEM ) - { - rc = buildReturnedColumnGr(col, *gwip, gwip->fatalParseError); - } else*/ - rc = buildReturnedColumn(col, *gwip, gwip->fatalParseError); - if ( col->type() == Item::FIELD_ITEM ) - gwip->fatalParseError = false; + { + rc = buildReturnedColumn(col, *gwip, gwip->fatalParseError); + if ( col->type() == Item::FIELD_ITEM ) + gwip->fatalParseError = false; } SimpleColumn* sc = dynamic_cast(rc); @@ -5365,15 +5375,22 @@ void gp_walk(const Item* item, void* arg) } else if (col->type() == Item::FIELD_ITEM && gwip->clauseType == HAVING) { - // Найти item в списке groupByFields - Item_func_or_sum* isfp = reinterpret_cast(gwip->havingAggColsItems[0]); - if ( isfp->type() == Item::SUM_FUNC_ITEM ) + Item_field* ifip = static_cast(col); + std::vector::iterator iter = gwip->havingAggColsItems.begin(); + Item_func_or_sum *isfp = NULL; + for( ;iter != gwip->havingAggColsItems.end(); iter++ ) { - ReturnedColumn* rc = buildAggregateColumn(isfp, *gwip); + Item *temp_isfp = *iter; + isfp = reinterpret_cast(temp_isfp); + if ( isfp->type() == Item::SUM_FUNC_ITEM && + isfp->result_field == ifip->field ) + { + ReturnedColumn* rc = buildAggregateColumn(isfp, *gwip); + if (rc) + gwip->rcWorkStack.push(rc); + break; + } } - - if (rc) - gwip->rcWorkStack.push(rc); break; } else @@ -9671,6 +9688,25 @@ int getGroupPlan(gp_walk_info& gwi, SELECT_LEX& select_lex, SCSEP& csep, cal_gro rc = buildReturnedColumn(item_ptr, gwi, gwi.fatalParseError); } + if ( ord_item->type() == Item::FIELD_ITEM ) + { + execplan::CalpontSelectExecutionPlan::ReturnedColumnList::iterator iter = gwi.groupByCols.begin(); + for( ; iter != gwi.groupByCols.end(); iter++ ) + { + if( rc->sameColumn((*iter).get()) ) + break; + } + // MCOL-1052 GROUP BY items list doesn't contain + // this ORDER BY item. + if ( iter == gwi.groupByCols.end() ) + { + string emsg = IDBErrorInfo::instance()->errorMsg(ERR_NOT_GROUPBY_EXPRESSION); + gwi.parseErrorText = emsg; + setError(gwi.thd, ER_INTERNAL_ERROR, emsg, gwi); + return ERR_NOT_GROUPBY_EXPRESSION; + } + } + if (!rc) { string emsg = IDBErrorInfo::instance()->errorMsg(ERR_NON_SUPPORT_ORDER_BY); From a80854c11f55be1882aaf798229f36e63353a966 Mon Sep 17 00:00:00 2001 From: Andrew Hutchings Date: Tue, 10 Apr 2018 14:14:22 +0100 Subject: [PATCH 15/57] MCOL-1182 Keep filters for cross engine step Cross engine step needs the filters for joining a constant with the cross engine table. Deleting them early causes a crash. --- dbcon/joblist/jlf_execplantojoblist.cpp | 21 ++++++++++++++++++--- 1 file changed, 18 insertions(+), 3 deletions(-) diff --git a/dbcon/joblist/jlf_execplantojoblist.cpp b/dbcon/joblist/jlf_execplantojoblist.cpp index 1ac770701..b6242ba77 100644 --- a/dbcon/joblist/jlf_execplantojoblist.cpp +++ b/dbcon/joblist/jlf_execplantojoblist.cpp @@ -1954,6 +1954,7 @@ const JobStepVector doOuterJoinOnFilter(OuterJoinOnFilter* oj, JobInfo& jobInfo) set doneNodes; // solved joins and simple filters map cpMap; // link for node removal JobStepVector join; // join step with its projection steps + bool keepFilters = false; // keep filters for cross engine step // To compromise the front end difficulty on setting outer attributes. set tablesInOuter; @@ -2162,6 +2163,14 @@ const JobStepVector doOuterJoinOnFilter(OuterJoinOnFilter* oj, JobInfo& jobInfo) jsv.insert(jsv.end(), sfv.begin(), sfv.end()); + // MCOL-1182 if we are doing a join between a cross engine + // step and a constant then keep the filter for the cross + // engine step instead of deleting it further down. + if (!sc->isInfiniDB()) + { + keepFilters = true; + } + doneNodes.insert(cn); } } @@ -2198,7 +2207,10 @@ const JobStepVector doOuterJoinOnFilter(OuterJoinOnFilter* oj, JobInfo& jobInfo) if (p == NULL) { filters = NULL; - delete c; + if (!keepFilters) + { + delete c; + } } else { @@ -2240,8 +2252,11 @@ const JobStepVector doOuterJoinOnFilter(OuterJoinOnFilter* oj, JobInfo& jobInfo) p->left(nullTree); p->right(nullTree); - delete p; - delete c; + if (!keepFilters) + { + delete p; + delete c; + } } } From 5a0ff2bb60b084621ea7d1dfbe21d37698ed4488 Mon Sep 17 00:00:00 2001 From: Roman Nozdrin Date: Tue, 10 Apr 2018 18:55:19 +0300 Subject: [PATCH 16/57] MCOL-1052 WIP group_by_handler processes agg functions in ORDER BY now. debug_walk() outputs information about extended SELECT list items. --- dbcon/mysql/ha_calpont.cpp | 2 +- dbcon/mysql/ha_calpont_execplan.cpp | 140 +++++++--------------------- dbcon/mysql/ha_calpont_impl_if.h | 1 + 3 files changed, 38 insertions(+), 105 deletions(-) diff --git a/dbcon/mysql/ha_calpont.cpp b/dbcon/mysql/ha_calpont.cpp index b649942d6..e07beade3 100644 --- a/dbcon/mysql/ha_calpont.cpp +++ b/dbcon/mysql/ha_calpont.cpp @@ -1162,7 +1162,7 @@ create_calpont_group_by_handler(THD *thd, Query *query) // Notify the server, that CS handles GROUP BY, ORDER BY and HAVING clauses. query->group_by = NULL; - //query->order_by = NULL; + query->order_by = NULL; query->having = NULL; } diff --git a/dbcon/mysql/ha_calpont_execplan.cpp b/dbcon/mysql/ha_calpont_execplan.cpp index 6013c77d4..9019f91cc 100644 --- a/dbcon/mysql/ha_calpont_execplan.cpp +++ b/dbcon/mysql/ha_calpont_execplan.cpp @@ -503,11 +503,16 @@ void debug_walk(const Item* item, void* arg) Item_sum* isp = (Item_sum*)item; char* item_name = item->name; - // MCOL-1052 This is and item from extended SELECT list - if (!item_name && isp->get_arg_count() and isp->get_arg(0)->name) + // MCOL-1052 This is an extended SELECT list item + if (!item_name && isp->get_arg_count() && isp->get_arg(0)->name) { item_name = isp->get_arg(0)->name; } + else if (!item_name && isp->get_arg_count() + && isp->get_arg(0)->type() == Item::INT_ITEM) + { + item_name = (char*)"INT||*"; + } else if (!item_name) { item_name = (char*)""; @@ -8269,28 +8274,9 @@ int getGroupPlan(gp_walk_info& gwi, SELECT_LEX& select_lex, SCSEP& csep, cal_gro gwi.internalDecimalScale = (gwi.thd->variables.infinidb_use_decimal_scale ? gwi.thd->variables.infinidb_decimal_scale : -1); gwi.subSelectType = csep->subType(); + // MCOL-1052 JOIN* join = select_lex.join; Item_cond* icp = 0; - /* MCOL-1052 - if (join != 0) - icp = reinterpret_cast(join->conds); - - // if icp is null, try to find the where clause other where - if (!join && gwi.thd->lex->derived_tables) - { - if (select_lex.prep_where) - icp = (Item_cond*)(select_lex.prep_where); - else if (select_lex.where) - icp = (Item_cond*)(select_lex.where); - } - else if (!join && ( ((gwi.thd->lex)->sql_command == SQLCOM_UPDATE ) || - ((gwi.thd->lex)->sql_command == SQLCOM_DELETE ) || - ((gwi.thd->lex)->sql_command == SQLCOM_UPDATE_MULTI ) || - ((gwi.thd->lex)->sql_command == SQLCOM_DELETE_MULTI ))) - { - icp = reinterpret_cast(select_lex.where); - } - */ // MCOL-1052 if ( gi.groupByWhere ) icp = reinterpret_cast(gi.groupByWhere); @@ -8460,8 +8446,8 @@ int getGroupPlan(gp_walk_info& gwi, SELECT_LEX& select_lex, SCSEP& csep, cal_gro if (icp) { // MCOL-1052 The condition could be useless. -// MariaDB bug 624 - without the fix_fields call, delete with join may error with "No query step". -//#if MYSQL_VERSION_ID < 50172 + // MariaDB bug 624 - without the fix_fields call, delete with join may error with "No query step". + //#if MYSQL_VERSION_ID < 50172 //@bug 3039. fix fields for constants if (!icp->fixed) { @@ -8499,71 +8485,6 @@ int getGroupPlan(gp_walk_info& gwi, SELECT_LEX& select_lex, SCSEP& csep, cal_gro { gwi.rcWorkStack.push(new ConstantColumn((int64_t)0, ConstantColumn::NUM)); } - - - - // ZZ - the followinig debug shows the structure of nested outer join. should - // use a recursive function. -#ifdef OUTER_JOIN_DEBUG - List* tables = &(select_lex.top_join_list); - List_iterator_fast ti(*tables); - //TABLE_LIST *inner; - //TABLE_LIST **table= (TABLE_LIST **)gwi.thd->alloc(sizeof(TABLE_LIST*) * tables->elements); - //for (TABLE_LIST **t= table + (tables->elements - 1); t >= table; t--) - // *t= ti++; - - //DBUG_ASSERT(tables->elements >= 1); - - //TABLE_LIST **end= table + tables->elements; - //for (TABLE_LIST **tbl= table; tbl < end; tbl++) - TABLE_LIST* curr; - - while ((curr = ti++)) - { - TABLE_LIST* curr = *tbl; - - if (curr->table_name) - cerr << curr->table_name << " "; - else - cerr << curr->alias << endl; - - if (curr->outer_join) - cerr << " is inner table" << endl; - else if (curr->straight) - cerr << "straight_join" << endl; - else - cerr << "join" << endl; - - if (curr->nested_join) - { - List* inners = &(curr->nested_join->join_list); - List_iterator_fast li(*inners); - TABLE_LIST** inner = (TABLE_LIST**)gwi.thd->alloc(sizeof(TABLE_LIST*) * inners->elements); - - for (TABLE_LIST** t = inner + (inners->elements - 1); t >= inner; t--) - *t = li++; - - TABLE_LIST** end1 = inner + inners->elements; - - for (TABLE_LIST** tb = inner; tb < end1; tb++) - { - TABLE_LIST* curr1 = *tb; - cerr << curr1->alias << endl; - - if (curr1->sj_on_expr) - { - curr1->sj_on_expr->traverse_cond(debug_walk, &gwi, Item::POSTFIX); - } - } - } - - if (curr->sj_on_expr) - { - curr->sj_on_expr->traverse_cond(debug_walk, &gwi, Item::POSTFIX); - } - } - -#endif // MCOL-1052 uint32_t failed = buildOuterJoin(gwi, select_lex); @@ -8757,11 +8678,12 @@ int getGroupPlan(gp_walk_info& gwi, SELECT_LEX& select_lex, SCSEP& csep, cal_gro // add this agg col to returnedColumnList boost::shared_ptr spac(ac); gwi.returnedCols.push_back(spac); - // This item will be used in HAVING clause later. + // This item will be used in HAVING|ORDER clause later. Item_func_or_sum* isfp = reinterpret_cast(item); if ( ! isfp->name_length ) { gwi.havingAggColsItems.push_back(item); + gwi.extSelectColsItems.push_back(item); } gwi.selectCols.push_back('`' + escapeBackTick(spac->alias().c_str()) + '`'); @@ -9675,20 +9597,8 @@ int getGroupPlan(gp_walk_info& gwi, SELECT_LEX& select_lex, SCSEP& csep, cal_gro else rc = buildReturnedColumn(ord_item, gwi, gwi.fatalParseError); - // @bug5501 try item_ptr if item can not be fixed. For some - // weird dml statement state, item can not be fixed but the - // infomation is available in item_ptr. - if (!rc || gwi.fatalParseError) - { - Item* item_ptr = ordercol->item_ptr; - - while (item_ptr->type() == Item::REF_ITEM) - item_ptr = *(((Item_ref*)item_ptr)->ref); - - rc = buildReturnedColumn(item_ptr, gwi, gwi.fatalParseError); - } - - if ( ord_item->type() == Item::FIELD_ITEM ) + // Looking for a match for this item in GROUP BY list. + if ( rc && ord_item->type() == Item::FIELD_ITEM ) { execplan::CalpontSelectExecutionPlan::ReturnedColumnList::iterator iter = gwi.groupByCols.begin(); for( ; iter != gwi.groupByCols.end(); iter++ ) @@ -9707,6 +9617,28 @@ int getGroupPlan(gp_walk_info& gwi, SELECT_LEX& select_lex, SCSEP& csep, cal_gro } } + // @bug5501 try item_ptr if item can not be fixed. For some + // weird dml statement state, item can not be fixed but the + // infomation is available in item_ptr. + if (!rc || gwi.fatalParseError) + { + gwi.fatalParseError = false; + Item* item_ptr = ordercol->item_ptr; + + while (item_ptr->type() == Item::REF_ITEM) + item_ptr = *(((Item_ref*)item_ptr)->ref); + + rc = buildReturnedColumn(item_ptr, gwi, gwi.fatalParseError); + } + + + // This ORDER BY item must be a agg function - try to + // find corresponding item in the exteded SELECT list. + if (!rc) + { + rc = buildReturnedColumn(gwi.extSelectColsItems[0], gwi, gwi.fatalParseError); + } + if (!rc) { string emsg = IDBErrorInfo::instance()->errorMsg(ERR_NON_SUPPORT_ORDER_BY); diff --git a/dbcon/mysql/ha_calpont_impl_if.h b/dbcon/mysql/ha_calpont_impl_if.h index 83f6aaf41..93d9cfa12 100644 --- a/dbcon/mysql/ha_calpont_impl_if.h +++ b/dbcon/mysql/ha_calpont_impl_if.h @@ -100,6 +100,7 @@ struct gp_walk_info execplan::CalpontSelectExecutionPlan::ReturnedColumnList subGroupByCols; execplan::CalpontSelectExecutionPlan::ReturnedColumnList orderByCols; std::vector havingAggColsItems; + std::vector extSelectColsItems; execplan::CalpontSelectExecutionPlan::ColumnMap columnMap; // This vector temporarily hold the projection columns to be added // to the returnedCols vector for subquery processing. It will be appended From ba38e392fc271568c392a5eb6607f0fe3cfe0cac Mon Sep 17 00:00:00 2001 From: Roman Nozdrin Date: Wed, 11 Apr 2018 15:52:26 +0300 Subject: [PATCH 17/57] MCOL-1052 WIP Uses item_ptr instead of auxilary vector; Changes fetchNextRow() interface and remove unused funcs; Uncomment extra checks in ha_calpont_impl_group_by_init(). --- dbcon/mysql/ha_calpont.cpp | 2 - dbcon/mysql/ha_calpont_execplan.cpp | 716 +++------------------ dbcon/mysql/ha_calpont_impl.cpp | 924 +--------------------------- dbcon/mysql/ha_calpont_impl_if.h | 1 - 4 files changed, 102 insertions(+), 1541 deletions(-) diff --git a/dbcon/mysql/ha_calpont.cpp b/dbcon/mysql/ha_calpont.cpp index e07beade3..6b5d6568f 100644 --- a/dbcon/mysql/ha_calpont.cpp +++ b/dbcon/mysql/ha_calpont.cpp @@ -1166,8 +1166,6 @@ create_calpont_group_by_handler(THD *thd, Query *query) query->having = NULL; } - - return handler; } diff --git a/dbcon/mysql/ha_calpont_execplan.cpp b/dbcon/mysql/ha_calpont_execplan.cpp index 9019f91cc..b17e0ab3e 100644 --- a/dbcon/mysql/ha_calpont_execplan.cpp +++ b/dbcon/mysql/ha_calpont_execplan.cpp @@ -2978,260 +2978,6 @@ ReturnedColumn* buildReturnedColumn(Item* item, gp_walk_info& gwi, bool& nonSupp return rc; } -ReturnedColumn* buildReturnedColumnGr(Item* item, gp_walk_info& gwi, bool& nonSupport) -{ - ReturnedColumn* rc = NULL; - - if ( gwi.thd) - { - //if ( ((gwi.thd->lex)->sql_command == SQLCOM_UPDATE ) || ((gwi.thd->lex)->sql_command == SQLCOM_UPDATE_MULTI )) - { - if ( !item->fixed) - { - item->fix_fields(gwi.thd, (Item**)&item); - } - } - } - - Item::Type itype = Item::SUM_FUNC_ITEM; - - switch (itype) - { - case Item::FIELD_ITEM: - { - Item_field* ifp = (Item_field*)item; - return buildSimpleColumn(ifp, gwi); - } - - case Item::INT_ITEM: - case Item::VARBIN_ITEM: - { - String val, *str = item->val_str(&val); - string valStr; - valStr.assign(str->ptr(), str->length()); - - if (item->unsigned_flag) - { - //cc = new ConstantColumn(valStr, (uint64_t)item->val_uint(), ConstantColumn::NUM); - // It seems that str at this point is crap if val_uint() is > MAX_BIGINT. By using - // this constructor, ConstantColumn is built with the proper string. For whatever reason, - // ExeMgr converts the fConstval member to numeric, rather than using the existing numeric - // values available, so it's important to have fConstval correct. - rc = new ConstantColumn((uint64_t)item->val_uint(), ConstantColumn::NUM); - } - else - { - rc = new ConstantColumn(valStr, (int64_t)item->val_int(), ConstantColumn::NUM); - } - - //return cc; - break; - } - - case Item::STRING_ITEM: - { - String val, *str = item->val_str(&val); - string valStr; - valStr.assign(str->ptr(), str->length()); - rc = new ConstantColumn(valStr); - break; - } - - case Item::REAL_ITEM: - { - String val, *str = item->val_str(&val); - string valStr; - valStr.assign(str->ptr(), str->length()); - rc = new ConstantColumn(valStr, item->val_real()); - break; - } - - case Item::DECIMAL_ITEM: - { - rc = buildDecimalColumn(item, gwi); - break; - } - - case Item::FUNC_ITEM: - { - Item_func* ifp = (Item_func*)item; - string func_name = ifp->func_name(); - - // try to evaluate const F&E. only for select clause - vector tmpVec; - //bool hasAggColumn = false; - uint16_t parseInfo = 0; - parse_item(ifp, tmpVec, gwi.fatalParseError, parseInfo); - - if (parseInfo & SUB_BIT) - { - gwi.fatalParseError = true; - gwi.parseErrorText = IDBErrorInfo::instance()->errorMsg(ERR_NON_SUPPORT_SELECT_SUB); - setError(gwi.thd, ER_CHECK_NOT_IMPLEMENTED, gwi.parseErrorText, gwi); - break; - } - - if (!gwi.fatalParseError && - !nonConstFunc(ifp) && - !(parseInfo & AF_BIT) && - (tmpVec.size() == 0)) - { - String val, *str = ifp->val_str(&val); - string valStr; - - if (str) - { - valStr.assign(str->ptr(), str->length()); - } - - if (!str) - { - rc = new ConstantColumn("", ConstantColumn::NULLDATA); - } - else if (ifp->result_type() == STRING_RESULT) - { - rc = new ConstantColumn(valStr, ConstantColumn::LITERAL); - rc->resultType(colType_MysqlToIDB(item)); - } - else if (ifp->result_type() == DECIMAL_RESULT) - rc = buildDecimalColumn(ifp, gwi); - else - { - rc = new ConstantColumn(valStr, ConstantColumn::NUM); - rc->resultType(colType_MysqlToIDB(item)); - } - - break; - } - - if (func_name == "+" || func_name == "-" || func_name == "*" || func_name == "/" ) - return buildArithmeticColumn(ifp, gwi, nonSupport); - else - return buildFunctionColumn(ifp, gwi, nonSupport); - } - - case Item::SUM_FUNC_ITEM: - { - return buildAggregateColumn(item, gwi); - } - - case Item::REF_ITEM: - { - Item_ref* ref = (Item_ref*)item; - - switch ((*(ref->ref))->type()) - { - case Item::SUM_FUNC_ITEM: - return buildAggregateColumn(*(ref->ref), gwi); - - case Item::FIELD_ITEM: - return buildReturnedColumn(*(ref->ref), gwi, nonSupport); - - case Item::REF_ITEM: - return buildReturnedColumn(*(((Item_ref*)(*(ref->ref)))->ref), gwi, nonSupport); - - case Item::FUNC_ITEM: - return buildFunctionColumn((Item_func*)(*(ref->ref)), gwi, nonSupport); - - case Item::WINDOW_FUNC_ITEM: - return buildWindowFunctionColumn(*(ref->ref), gwi, nonSupport); - - default: - gwi.fatalParseError = true; - gwi.parseErrorText = "Unknown REF item"; - break; - } - } - - case Item::NULL_ITEM: - { - if (gwi.condPush) - return new SimpleColumn("noop"); - - return new ConstantColumn("", ConstantColumn::NULLDATA); - } - - case Item::CACHE_ITEM: - { - Item* col = ((Item_cache*)item)->get_example(); - rc = buildReturnedColumn(col, gwi, nonSupport); - - if (rc) - { - ConstantColumn* cc = dynamic_cast(rc); - - if (!cc) - { - rc->joinInfo(rc->joinInfo() | JOIN_CORRELATED); - - if (gwi.subQuery) - gwi.subQuery->correlated(true); - } - } - - break; - } - - case Item::EXPR_CACHE_ITEM: - { - // TODO: item is a Item_cache_wrapper - printf("EXPR_CACHE_ITEM in buildReturnedColumn\n"); - cerr << "EXPR_CACHE_ITEM in buildReturnedColumn" << endl; - break; - } - - case Item::DATE_ITEM: - { - String val, *str = item->val_str(&val); - string valStr; - valStr.assign(str->ptr(), str->length()); - rc = new ConstantColumn(valStr); - break; - } - - case Item::WINDOW_FUNC_ITEM: - { - return buildWindowFunctionColumn(item, gwi, nonSupport); - } - -#if INTERVAL_ITEM - - case Item::INTERVAL_ITEM: - { - Item_interval* interval = (Item_interval*)item; - SRCP srcp; - srcp.reset(buildReturnedColumn(interval->item, gwi, nonSupport)); - - if (!srcp) - return NULL; - - rc = new IntervalColumn(srcp, (int)interval->interval); - rc->resultType(srcp->resultType()); - break; - } - -#endif - - case Item::SUBSELECT_ITEM: - { - gwi.hasSubSelect = true; - break; - } - - default: - { - gwi.fatalParseError = true; - gwi.parseErrorText = "Unknown item type"; - break; - } - } - - if (rc && item->name) - rc->alias(item->name); - - return rc; -} - ArithmeticColumn* buildArithmeticColumn(Item_func* item, gp_walk_info& gwi, bool& nonSupport) { if (!(gwi.thd->infinidb_vtable.cal_conn_info)) @@ -8249,10 +7995,6 @@ int cp_get_group_plan(THD* thd, SCSEP& csep, cal_impl_if::cal_group_info& gi) else if (status < 0) return status; - cerr << "---------------- cp_get_group_plan EXECUTION PLAN ----------------" << endl; - cerr << *csep << endl ; - cerr << "-------------- EXECUTION PLAN END --------------\n" << endl; - return 0; } @@ -8273,11 +8015,10 @@ int getGroupPlan(gp_walk_info& gwi, SELECT_LEX& select_lex, SCSEP& csep, cal_gro gwi.internalDecimalScale = (gwi.thd->variables.infinidb_use_decimal_scale ? gwi.thd->variables.infinidb_decimal_scale : -1); gwi.subSelectType = csep->subType(); - - // MCOL-1052 + JOIN* join = select_lex.join; Item_cond* icp = 0; - // MCOL-1052 + if ( gi.groupByWhere ) icp = reinterpret_cast(gi.groupByWhere); @@ -8485,7 +8226,7 @@ int getGroupPlan(gp_walk_info& gwi, SELECT_LEX& select_lex, SCSEP& csep, cal_gro { gwi.rcWorkStack.push(new ConstantColumn((int64_t)0, ConstantColumn::NUM)); } - // MCOL-1052 + uint32_t failed = buildOuterJoin(gwi, select_lex); if (failed) return failed; @@ -8678,12 +8419,11 @@ int getGroupPlan(gp_walk_info& gwi, SELECT_LEX& select_lex, SCSEP& csep, cal_gro // add this agg col to returnedColumnList boost::shared_ptr spac(ac); gwi.returnedCols.push_back(spac); - // This item will be used in HAVING|ORDER clause later. + // This item will be used in HAVING later. Item_func_or_sum* isfp = reinterpret_cast(item); if ( ! isfp->name_length ) { gwi.havingAggColsItems.push_back(item); - gwi.extSelectColsItems.push_back(item); } gwi.selectCols.push_back('`' + escapeBackTick(spac->alias().c_str()) + '`'); @@ -9143,11 +8883,9 @@ int getGroupPlan(gp_walk_info& gwi, SELECT_LEX& select_lex, SCSEP& csep, cal_gro gwi.fatalParseError = false; gwi.parseErrorText = ""; - //if (gi.groupByHaving != 0) - if ( select_lex.having != 0 ) + if (gi.groupByHaving != 0) { - Item_cond* having = reinterpret_cast(select_lex.having); - //Item_cond* having = reinterpret_cast(gi.groupByHaving); + Item_cond* having = reinterpret_cast(gi.groupByHaving); #ifdef DEBUG_WALK_COND cerr << "------------------- HAVING ---------------------" << endl; having->traverse_cond(debug_walk, &gwi, Item::POSTFIX); @@ -9202,7 +8940,6 @@ int getGroupPlan(gp_walk_info& gwi, SELECT_LEX& select_lex, SCSEP& csep, cal_gro for (uint32_t i = 0; i < funcFieldVec.size(); i++) { - //SimpleColumn *sc = new SimpleColumn(funcFieldVec[i]->db_name, bestTableName(funcFieldVec[i])/*funcFieldVec[i]->table_name*/, funcFieldVec[i]->field_name, sessionID); SimpleColumn* sc = buildSimpleColumn(funcFieldVec[i], gwi); if (!sc || gwi.fatalParseError) @@ -9570,388 +9307,113 @@ int getGroupPlan(gp_walk_info& gwi, SELECT_LEX& select_lex, SCSEP& csep, cal_gro // for subquery, order+limit by will be supported in infinidb. build order by columns // @todo union order by and limit support //if (gwi.hasWindowFunc || gwi.subSelectType != CalpontSelectExecutionPlan::MAIN_SELECT) + + for (; ordercol; ordercol = ordercol->next) { - for (; ordercol; ordercol = ordercol->next) + ReturnedColumn* rc = NULL; + + if (ordercol->in_field_list && ordercol->counter_used) { - ReturnedColumn* rc = NULL; + rc = gwi.returnedCols[ordercol->counter - 1]->clone(); + rc->orderPos(ordercol->counter - 1); + // can not be optimized off if used in order by with counter. + // set with self derived table alias if it's derived table + gwi.returnedCols[ordercol->counter - 1]->incRefCount(); + } + else + { + Item* ord_item = *(ordercol->item); - if (ordercol->in_field_list && ordercol->counter_used) - { - rc = gwi.returnedCols[ordercol->counter - 1]->clone(); - rc->orderPos(ordercol->counter - 1); - // can not be optimized off if used in order by with counter. - // set with self derived table alias if it's derived table - gwi.returnedCols[ordercol->counter - 1]->incRefCount(); - } + // ignore not_used column on order by. + if (ord_item->type() == Item::INT_ITEM && ord_item->full_name() && string(ord_item->full_name()) == "Not_used") + continue; + else if (ord_item->type() == Item::INT_ITEM) + rc = gwi.returnedCols[((Item_int*)ord_item)->val_int() - 1]->clone(); + else if (ord_item->type() == Item::SUBSELECT_ITEM) + gwi.fatalParseError = true; else + rc = buildReturnedColumn(ord_item, gwi, gwi.fatalParseError); + + // Looking for a match for this item in GROUP BY list. + if ( rc && ord_item->type() == Item::FIELD_ITEM ) { - Item* ord_item = *(ordercol->item); - - // ignore not_used column on order by. - if (ord_item->type() == Item::INT_ITEM && ord_item->full_name() && string(ord_item->full_name()) == "Not_used") - continue; - else if (ord_item->type() == Item::INT_ITEM) - rc = gwi.returnedCols[((Item_int*)ord_item)->val_int() - 1]->clone(); - else if (ord_item->type() == Item::SUBSELECT_ITEM) - gwi.fatalParseError = true; - else - rc = buildReturnedColumn(ord_item, gwi, gwi.fatalParseError); - - // Looking for a match for this item in GROUP BY list. - if ( rc && ord_item->type() == Item::FIELD_ITEM ) + execplan::CalpontSelectExecutionPlan::ReturnedColumnList::iterator iter = gwi.groupByCols.begin(); + for( ; iter != gwi.groupByCols.end(); iter++ ) { - execplan::CalpontSelectExecutionPlan::ReturnedColumnList::iterator iter = gwi.groupByCols.begin(); - for( ; iter != gwi.groupByCols.end(); iter++ ) - { - if( rc->sameColumn((*iter).get()) ) - break; - } - // MCOL-1052 GROUP BY items list doesn't contain - // this ORDER BY item. - if ( iter == gwi.groupByCols.end() ) - { - string emsg = IDBErrorInfo::instance()->errorMsg(ERR_NOT_GROUPBY_EXPRESSION); - gwi.parseErrorText = emsg; - setError(gwi.thd, ER_INTERNAL_ERROR, emsg, gwi); - return ERR_NOT_GROUPBY_EXPRESSION; - } + if( rc->sameColumn((*iter).get()) ) + break; } - - // @bug5501 try item_ptr if item can not be fixed. For some - // weird dml statement state, item can not be fixed but the - // infomation is available in item_ptr. - if (!rc || gwi.fatalParseError) + // MCOL-1052 GROUP BY items list doesn't contain + // this ORDER BY item. + if ( iter == gwi.groupByCols.end() ) { - gwi.fatalParseError = false; - Item* item_ptr = ordercol->item_ptr; - - while (item_ptr->type() == Item::REF_ITEM) - item_ptr = *(((Item_ref*)item_ptr)->ref); - - rc = buildReturnedColumn(item_ptr, gwi, gwi.fatalParseError); - } - - - // This ORDER BY item must be a agg function - try to - // find corresponding item in the exteded SELECT list. - if (!rc) - { - rc = buildReturnedColumn(gwi.extSelectColsItems[0], gwi, gwi.fatalParseError); - } - - if (!rc) - { - string emsg = IDBErrorInfo::instance()->errorMsg(ERR_NON_SUPPORT_ORDER_BY); + Item_ident *iip = reinterpret_cast(ord_item); + std::ostringstream ostream; + ostream << "'"; + if (iip->db_name) + ostream << iip->db_name << '.'; + else + ostream << "unknown db" << '.'; + if (iip->table_name) + ostream << iip->table_name << '.'; + else + ostream << "unknown table" << '.'; + if (iip->field_name) + ostream << iip->field_name; + else + ostream << "unknown field"; + ostream << "'"; + Message::Args args; + args.add(ostream.str()); + string emsg = IDBErrorInfo::instance()->errorMsg(ERR_NOT_GROUPBY_EXPRESSION, args); gwi.parseErrorText = emsg; - setError(gwi.thd, ER_CHECK_NOT_IMPLEMENTED, emsg, gwi); - return ER_CHECK_NOT_IMPLEMENTED; + setError(gwi.thd, ER_INTERNAL_ERROR, emsg, gwi); + return ERR_NOT_GROUPBY_EXPRESSION; } } - if (ordercol->direction == ORDER::ORDER_ASC) - rc->asc(true); - else - rc->asc(false); - - gwi.orderByCols.push_back(SRCP(rc)); - } - } - /* - else if (!isUnion) - { - vector fieldVec; - bool addToSel; - - // the following order by is just for redo phase - if (!unionSel) - { - for (; ordercol; ordercol = ordercol->next) + // @bug5501 try item_ptr if item can not be fixed. For some + // weird dml statement state, item can not be fixed but the + // infomation is available in item_ptr. + if (!rc || gwi.fatalParseError) { - Item* ord_item = *(ordercol->item); + gwi.fatalParseError = false; + Item* item_ptr = ordercol->item_ptr; - // @bug5993. Could be nested ref. - while (ord_item->type() == Item::REF_ITEM) - ord_item = (*((Item_ref*)ord_item)->ref); + while (item_ptr->type() == Item::REF_ITEM) + item_ptr = *(((Item_ref*)item_ptr)->ref); - // @bug 1706. re-construct the order by item one by one - //Item* ord_item = *(ordercol->item); - if (ord_cols.length() != 0) - ord_cols += ", "; - - addToSel = true; - string fullname; - - if (ordercol->in_field_list && ordercol->counter_used) - { - ostringstream oss; - oss << ordercol->counter; - ord_cols += oss.str(); - - if (ordercol->direction != ORDER::ORDER_ASC) - ord_cols += " desc"; - - continue; - } - - else if (ord_item->type() == Item::FUNC_ITEM) - { - // @bug 2621. order by alias - if (!ord_item->is_autogenerated_name && ord_item->name) - { - ord_cols += ord_item->name; - continue; - } - - // if there's group by clause or aggregate column, check to see - // if this item or the arguments is on the GB list. - ReturnedColumn* rc = 0; - // check if this order by column is on the select list - Item_func* ifp = (Item_func*)(*(ordercol->item)); - rc = buildFunctionColumn(ifp, gwi, gwi.fatalParseError); - - if (rc) - { - for (uint32_t i = 0; i < gwi.returnedCols.size(); i++) - { - if (rc && rc->operator==(gwi.returnedCols[i].get())) - { - ostringstream oss; - oss << i + 1; - ord_cols += oss.str(); - addToSel = false; - break; - } - } - } - - if (addToSel) - { - FunctionColumn* fc = dynamic_cast(rc); - - if (fc) - { - addToSel = false; - redo = true; - string ord_func = string(ifp->func_name()) + "("; - - for (uint32_t i = 0; i < fc->functionParms().size(); i++) - { - if (i != 0) - ord_func += ","; - - for (uint32_t j = 0; j < gwi.returnedCols.size(); j++) - { - if (fc->functionParms()[i]->data()->operator==(gwi.returnedCols[j].get())) - { - ord_func += "`" + escapeBackTick(gwi.returnedCols[j]->alias().c_str()) + "`"; - continue; - } - - AggregateColumn* ac = dynamic_cast(fc->functionParms()[i]->data()); - - if (ac) - { - gwi.parseErrorText = IDBErrorInfo::instance()->errorMsg(ERR_NON_SUPPORT_ORDER_BY); - setError(gwi.thd, ER_CHECK_NOT_IMPLEMENTED, gwi.parseErrorText, gwi); - return ER_CHECK_NOT_IMPLEMENTED; - } - - addToSel = true; - //continue; - - } - } - - ord_func += ")"; - - if (!addToSel) - ord_cols += ord_func; - } - } - } - else if (ord_item->type() == Item::SUBSELECT_ITEM) - { - string emsg = IDBErrorInfo::instance()->errorMsg(ERR_NON_SUPPORT_ORDER_BY); - setError(gwi.thd, ER_CHECK_NOT_IMPLEMENTED, emsg, gwi); - return ER_CHECK_NOT_IMPLEMENTED; - } - - else if (ord_item->type() == Item::SUM_FUNC_ITEM) - { - ReturnedColumn* ac = 0; - - Item_sum* ifp = (Item_sum*)(*(ordercol->item)); - // @bug3477. add aggregate column to the select list of the create phase. - ac = buildAggregateColumn(ifp, gwi); - - if (!ac) - { - setError(gwi.thd, ER_CHECK_NOT_IMPLEMENTED, - IDBErrorInfo::instance()->errorMsg(ERR_NON_SUPPORT_ORDER_BY), gwi); - return ER_CHECK_NOT_IMPLEMENTED; - } - - // check if this order by column is on the select list - for (uint32_t i = 0; i < gwi.returnedCols.size(); i++) - { - AggregateColumn* ret = dynamic_cast(gwi.returnedCols[i].get()); - - if (!ret) - continue; - - if (ac->operator==(gwi.returnedCols[i].get())) - { - ostringstream oss; - oss << i + 1; - ord_cols += oss.str(); - addToSel = false; - break; - } - } - - if (ac || !gwi.groupByCols.empty()) - { - if (addToSel) - { - redo = true; - // @bug 3076. do not add the argument of aggregate function to the SELECT list, - // instead, add the whole column - String str; - ord_item->print(&str, QT_INFINIDB_NO_QUOTE); - - if (sel_cols_in_create.length() != 0) - sel_cols_in_create += ", "; - - sel_cols_in_create += str.c_ptr(); - //gwi.selectCols.push_back(" `" + string(str.c_ptr()) + "`"); - SRCP srcp(ac); - gwi.returnedCols.push_back(srcp); - ord_cols += " `" + escapeBackTick(str.c_ptr()) + "`"; - } - - if (ordercol->direction != ORDER::ORDER_ASC) - ord_cols += " desc"; - - continue; - } - } - else if (ord_item->name && ord_item->type() == Item::FIELD_ITEM) - { - Item_field* field = reinterpret_cast(ord_item); - ReturnedColumn* rc = buildSimpleColumn(field, gwi); - fullname = field->full_name(); - - for (uint32_t i = 0; i < gwi.returnedCols.size(); i++) - { - SimpleColumn* sc = dynamic_cast(gwi.returnedCols[i].get()); - - if (sc && ((Item_field*)ord_item)->cached_table && - (strcasecmp(getViewName(((Item_field*)ord_item)->cached_table).c_str(), sc->viewName().c_str()) != 0)) - { - continue; - } - - if (strcasecmp(fullname.c_str(), gwi.returnedCols[i]->alias().c_str()) == 0 || - strcasecmp(ord_item->name, gwi.returnedCols[i]->alias().c_str()) == 0) - { - ord_cols += string(" `") + escapeBackTick(gwi.returnedCols[i]->alias().c_str()) + '`'; - addToSel = false; - break; - } - - if (sc && sc->sameColumn(rc)) - { - ostringstream oss; - oss << i + 1; - ord_cols += oss.str(); - addToSel = false; - break; - } - } - } - - if (addToSel) - { - // @bug 2719. Error out order by not on the distinct select list. - if ( gi.groupByDistinct ) - { - gwi.parseErrorText = IDBErrorInfo::instance()->errorMsg(ERR_ORDERBY_NOT_IN_DISTINCT); - setError(gwi.thd, ER_CHECK_NOT_IMPLEMENTED, gwi.parseErrorText, gwi); - return ER_CHECK_NOT_IMPLEMENTED; - } - - bool hasNonSupportItem = false; - uint16_t parseInfo = 0; - parse_item(ord_item, fieldVec, hasNonSupportItem, parseInfo); - - if (hasNonSupportItem) - { - string emsg = IDBErrorInfo::instance()->errorMsg(ERR_NON_SUPPORT_ORDER_BY); - setError(gwi.thd, ER_CHECK_NOT_IMPLEMENTED, emsg, gwi); - return ER_CHECK_NOT_IMPLEMENTED; - } - - String str; - ord_item->print(&str, QT_INFINIDB); - ord_cols += str.c_ptr(); - } - - if (ordercol->direction != ORDER::ORDER_ASC) - ord_cols += " desc"; + rc = buildReturnedColumn(item_ptr, gwi, gwi.fatalParseError); + } + + // This ORDER BY item must be an agg function - + // the ordercol->item_ptr and exteded SELECT list + // must contain the corresponding item. + if (!rc) + { + Item* item_ptr = ordercol->item_ptr; + if (item_ptr) + rc = buildReturnedColumn(item_ptr, gwi, gwi.fatalParseError); } - } - redo = (redo || fieldVec.size() != 0); - - // populate string to be added to the select list for order by - for (uint32_t i = 0; i < fieldVec.size(); i++) - { - SimpleColumn* sc = buildSimpleColumn(fieldVec[i], gwi); - - if (!sc) + if (!rc) { string emsg = IDBErrorInfo::instance()->errorMsg(ERR_NON_SUPPORT_ORDER_BY); + gwi.parseErrorText = emsg; setError(gwi.thd, ER_CHECK_NOT_IMPLEMENTED, emsg, gwi); return ER_CHECK_NOT_IMPLEMENTED; } - - String str; - fieldVec[i]->print(&str, QT_INFINIDB_NO_QUOTE); - sc->alias(string(str.c_ptr())); - SRCP srcp(sc); - uint32_t j = 0; - - for (; j < gwi.returnedCols.size(); j++) - { - if (sc->sameColumn(gwi.returnedCols[j].get())) - { - SimpleColumn* field = dynamic_cast(gwi.returnedCols[j].get()); - - if (field && field->alias() == sc->alias()) - break; - } - } - - if (j == gwi.returnedCols.size()) - { - string fullname; - - if (sel_cols_in_create.length() != 0) - sel_cols_in_create += ", "; - - fullname = str.c_ptr(); - sel_cols_in_create += fullname + " `" + escapeBackTick(fullname.c_str()) + "`"; - - gwi.returnedCols.push_back(srcp); - gwi.columnMap.insert(CalpontSelectExecutionPlan::ColumnMap::value_type(string(fieldVec[i]->field_name), srcp)); - TABLE_LIST* tmp = (fieldVec[i]->cached_table ? fieldVec[i]->cached_table : 0); - gwi.tableMap[make_aliastable(sc->schemaName(), sc->tableName(), sc->tableAlias(), sc->isInfiniDB())] = - make_pair(1, tmp); - } } - } - */ + if (ordercol->direction == ORDER::ORDER_ASC) + rc->asc(true); + else + rc->asc(false); + + gwi.orderByCols.push_back(SRCP(rc)); + } + + // make sure columnmap, returnedcols and count(*) arg_list are not empty TableMap::iterator tb_iter = gwi.tableMap.begin(); diff --git a/dbcon/mysql/ha_calpont_impl.cpp b/dbcon/mysql/ha_calpont_impl.cpp index 4232d4985..926e5cd93 100644 --- a/dbcon/mysql/ha_calpont_impl.cpp +++ b/dbcon/mysql/ha_calpont_impl.cpp @@ -427,7 +427,7 @@ int vbin2hex(const uint8_t* p, const unsigned l, char* o) return 0; } -int fetchNextRow(uchar* buf, cal_table_info& ti, cal_connection_info* ci) +int fetchNextRow(uchar* buf, cal_table_info& ti, cal_connection_info* ci, bool handler_flag=false) { int rc = HA_ERR_END_OF_FILE; int num_attr = ti.msTablePtr->s->fields; @@ -470,7 +470,8 @@ int fetchNextRow(uchar* buf, cal_table_info& ti, cal_connection_info* ci) Field** f; f = ti.msTablePtr->field; //set all fields to null in null col bitmap - memset(buf, -1, ti.msTablePtr->s->null_bytes); + if (!handler_flag) + memset(buf, -1, ti.msTablePtr->s->null_bytes); std::vector& colTypes = ti.tpl_scan_ctx->ctp; int64_t intColVal = 0; uint64_t uintColVal = 0; @@ -859,874 +860,6 @@ int fetchNextRow(uchar* buf, cal_table_info& ti, cal_connection_info* ci) return rc; } -int fetchNextRowGrHandGroupBy(cal_table_info& ti, cal_connection_info* ci) -{ - int rc = HA_ERR_END_OF_FILE; - int num_attr = ti.msTablePtr->s->fields; - sm::status_t sm_stat; - - try - { - if (ti.conn_hndl) - { - sm_stat = sm::tpl_scan_fetch(ti.tpl_scan_ctx, ti.conn_hndl); - } - else if (ci->cal_conn_hndl) - { - sm_stat = sm::tpl_scan_fetch(ti.tpl_scan_ctx, ci->cal_conn_hndl, (int*)(¤t_thd->killed)); - } - else - throw runtime_error("internal error"); - } - catch (std::exception& ex) - { -// @bug 2244. Always log this msg for now, as we try to track down when/why we are -// losing socket connection with ExeMgr -//#ifdef INFINIDB_DEBUG - tpl_scan_fetch_LogException( ti, ci, &ex); -//#endif - sm_stat = sm::CALPONT_INTERNAL_ERROR; - } - catch (...) - { -// @bug 2244. Always log this msg for now, as we try to track down when/why we are -// losing socket connection with ExeMgr -//#ifdef INFINIDB_DEBUG - tpl_scan_fetch_LogException( ti, ci, 0 ); -//#endif - sm_stat = sm::CALPONT_INTERNAL_ERROR; - } - - if (sm_stat == sm::STATUS_OK) - { - Field** f; - f = ti.msTablePtr->field; - //set all fields to null in null col bitmap - //memset(buf, -1, ti.msTablePtr->s->null_bytes); - std::vector& colTypes = ti.tpl_scan_ctx->ctp; - int64_t intColVal = 0; - uint64_t uintColVal = 0; - char tmp[256]; - - RowGroup* rowGroup = ti.tpl_scan_ctx->rowGroup; - - // table mode mysql expects all columns of the table. mapping between columnoid and position in rowgroup - // set coltype.position to be the position in rowgroup. only set once. - if (ti.tpl_scan_ctx->rowsreturned == 0 && - (ti.tpl_scan_ctx->traceFlags & execplan::CalpontSelectExecutionPlan::TRACE_TUPLE_OFF)) - { - for (uint32_t i = 0; i < rowGroup->getColumnCount(); i++) - { - int oid = rowGroup->getOIDs()[i]; - int j = 0; - - for (; j < num_attr; j++) - { - // mysql should haved eliminated duplicate projection columns - if (oid == colTypes[j].columnOID || oid == colTypes[j].ddn.dictOID) - { - colTypes[j].colPosition = i; - break; - } - } - } - } - - rowgroup::Row row; - rowGroup->initRow(&row); - rowGroup->getRow(ti.tpl_scan_ctx->rowsreturned, &row); - int s; - - for (int p = 0; p < num_attr; p++, f++) - { - //This col is going to be written - bitmap_set_bit(ti.msTablePtr->write_set, (*f)->field_index); - - // get coltype if not there yet - if (colTypes[0].colWidth == 0) - { - for (short c = 0; c < num_attr; c++) - { - colTypes[c].colPosition = c; - colTypes[c].colWidth = rowGroup->getColumnWidth(c); - colTypes[c].colDataType = rowGroup->getColTypes()[c]; - colTypes[c].columnOID = rowGroup->getOIDs()[c]; - colTypes[c].scale = rowGroup->getScale()[c]; - colTypes[c].precision = rowGroup->getPrecision()[c]; - } - } - - CalpontSystemCatalog::ColType colType(colTypes[p]); - - // table mode handling - if (ti.tpl_scan_ctx->traceFlags & execplan::CalpontSelectExecutionPlan::TRACE_TUPLE_OFF) - { - if (colType.colPosition == -1) // not projected by tuplejoblist - continue; - else - s = colType.colPosition; - } - else - { - s = p; - } - - // precision == -16 is borrowed as skip null check indicator for bit ops. - if (row.isNullValue(s) && colType.precision != -16) - { - // @2835. Handle empty string and null confusion. store empty string for string column - if (colType.colDataType == CalpontSystemCatalog::CHAR || - colType.colDataType == CalpontSystemCatalog::VARCHAR || - colType.colDataType == CalpontSystemCatalog::VARBINARY) - { - Field_varstring* f2 = (Field_varstring*)*f; - f2->store(tmp, 0, f2->charset()); - } - - continue; - } - - // fetch and store data - switch (colType.colDataType) - { - case CalpontSystemCatalog::DATE: - { - if ((*f)->null_ptr) - *(*f)->null_ptr &= ~(*f)->null_bit; - - intColVal = row.getUintField<4>(s); - DataConvert::dateToString(intColVal, tmp, 255); - Field_varstring* f2 = (Field_varstring*)*f; - f2->store(tmp, strlen(tmp), f2->charset()); - break; - } - - case CalpontSystemCatalog::DATETIME: - { - if ((*f)->null_ptr) - *(*f)->null_ptr &= ~(*f)->null_bit; - - intColVal = row.getUintField<8>(s); - DataConvert::datetimeToString(intColVal, tmp, 255); - - /* setting the field_length is a sort-of hack. The length - * at this point can be long enough to include mseconds. - * ColumnStore doesn't fully support mseconds yet so if - * they are requested, trim them off. - * At a later date we should set this more intelligently - * based on the result set. - */ - /* MCOL-683: UTF-8 datetime no msecs is 57, this sometimes happens! */ - if (((*f)->field_length > 19) && ((*f)->field_length != 57)) - (*f)->field_length = strlen(tmp); - - Field_varstring* f2 = (Field_varstring*)*f; - f2->store(tmp, strlen(tmp), f2->charset()); - break; - } - - case CalpontSystemCatalog::CHAR: - case CalpontSystemCatalog::VARCHAR: - { - Field_varstring* f2 = (Field_varstring*)*f; - - switch (colType.colWidth) - { - case 1: - intColVal = row.getUintField<1>(s); - f2->store((char*)(&intColVal), strlen((char*)(&intColVal)), f2->charset()); - break; - - case 2: - intColVal = row.getUintField<2>(s); - f2->store((char*)(&intColVal), strlen((char*)(&intColVal)), f2->charset()); - break; - - case 4: - intColVal = row.getUintField<4>(s); - f2->store((char*)(&intColVal), strlen((char*)(&intColVal)), f2->charset()); - break; - - case 8: - //make sure we don't send strlen off into the weeds... - intColVal = row.getUintField<8>(s); - memcpy(tmp, &intColVal, 8); - tmp[8] = 0; - f2->store(tmp, strlen(tmp), f2->charset()); - break; - - default: - f2->store((const char*)row.getStringPointer(s), row.getStringLength(s), f2->charset()); - } - - if ((*f)->null_ptr) - *(*f)->null_ptr &= ~(*f)->null_bit; - - break; - } - - case CalpontSystemCatalog::VARBINARY: - { - Field_varstring* f2 = (Field_varstring*)*f; - - if (current_thd->variables.infinidb_varbin_always_hex) - { - uint32_t l; - const uint8_t* p = row.getVarBinaryField(l, s); - uint32_t ll = l * 2; - boost::scoped_array sca(new char[ll]); - vbin2hex(p, l, sca.get()); - f2->store(sca.get(), ll, f2->charset()); - } - else - f2->store((const char*)row.getVarBinaryField(s), row.getVarBinaryLength(s), f2->charset()); - - if ((*f)->null_ptr) - *(*f)->null_ptr &= ~(*f)->null_bit; - - break; - } - - case CalpontSystemCatalog::BIGINT: - { - intColVal = row.getIntField<8>(s); - storeNumericField(f, intColVal, colType); - break; - } - - case CalpontSystemCatalog::UBIGINT: - { - uintColVal = row.getUintField<8>(s); - storeNumericField(f, uintColVal, colType); - break; - } - - case CalpontSystemCatalog::INT: - { - intColVal = row.getIntField<4>(s); - storeNumericField(f, intColVal, colType); - break; - } - - case CalpontSystemCatalog::UINT: - { - uintColVal = row.getUintField<4>(s); - storeNumericField(f, uintColVal, colType); - break; - } - - case CalpontSystemCatalog::SMALLINT: - { - intColVal = row.getIntField<2>(s); - storeNumericField(f, intColVal, colType); - break; - } - - case CalpontSystemCatalog::USMALLINT: - { - uintColVal = row.getUintField<2>(s); - storeNumericField(f, uintColVal, colType); - break; - } - - case CalpontSystemCatalog::TINYINT: - { - intColVal = row.getIntField<1>(s); - storeNumericField(f, intColVal, colType); - break; - } - - case CalpontSystemCatalog::UTINYINT: - { - uintColVal = row.getUintField<1>(s); - storeNumericField(f, uintColVal, colType); - break; - } - - //In this case, we're trying to load a double output column with float data. This is the - // case when you do sum(floatcol), e.g. - case CalpontSystemCatalog::FLOAT: - case CalpontSystemCatalog::UFLOAT: - { - float dl = row.getFloatField(s); - - if (dl == std::numeric_limits::infinity()) - continue; - - //int64_t* icvp = (int64_t*)&dl; - //intColVal = *icvp; - Field_float* f2 = (Field_float*)*f; - // bug 3485, reserve enough space for the longest float value - // -3.402823466E+38 to -1.175494351E-38, 0, and - // 1.175494351E-38 to 3.402823466E+38. - (*f)->field_length = 40; - - //float float_val = *(float*)(&value); - //f2->store(float_val); - if (f2->decimals() < (uint32_t)row.getScale(s)) - f2->dec = (uint32_t)row.getScale(s); - - f2->store(dl); - - if ((*f)->null_ptr) - *(*f)->null_ptr &= ~(*f)->null_bit; - - break; - - //storeNumericField(f, intColVal, colType); - //break; - } - - case CalpontSystemCatalog::DOUBLE: - case CalpontSystemCatalog::UDOUBLE: - { - double dl = row.getDoubleField(s); - - if (dl == std::numeric_limits::infinity()) - continue; - - Field_double* f2 = (Field_double*)*f; - // bug 3483, reserve enough space for the longest double value - // -1.7976931348623157E+308 to -2.2250738585072014E-308, 0, and - // 2.2250738585072014E-308 to 1.7976931348623157E+308. - (*f)->field_length = 310; - - //double double_val = *(double*)(&value); - //f2->store(double_val); - if (f2->decimals() < (uint32_t)row.getScale(s)) - f2->dec = (uint32_t)row.getScale(s); - - f2->store(dl); - - if ((*f)->null_ptr) - *(*f)->null_ptr &= ~(*f)->null_bit; - - break; - - - //int64_t* icvp = (int64_t*)&dl; - //intColVal = *icvp; - //storeNumericField(f, intColVal, colType); - //break; - } - - case CalpontSystemCatalog::DECIMAL: - case CalpontSystemCatalog::UDECIMAL: - { - intColVal = row.getIntField(s); - storeNumericField(f, intColVal, colType); - break; - } - - case CalpontSystemCatalog::BLOB: - case CalpontSystemCatalog::TEXT: - { - Field_blob* f2 = (Field_blob*)*f; - f2->set_ptr(row.getVarBinaryLength(s), (unsigned char*)row.getVarBinaryField(s)); - - if ((*f)->null_ptr) - *(*f)->null_ptr &= ~(*f)->null_bit; - - break; - } - - default: // treat as int64 - { - intColVal = row.getUintField<8>(s); - storeNumericField(f, intColVal, colType); - break; - } - } - } - - ti.tpl_scan_ctx->rowsreturned++; - ti.c++; -#ifdef INFINIDB_DEBUG - - if ((ti.c % 1000000) == 0) - cerr << "fetchNextRow so far table " << ti.msTablePtr->s->table_name.str << " rows = " << ti.c << endl; - -#endif - ti.moreRows = true; - rc = 0; - } - else if (sm_stat == sm::SQL_NOT_FOUND) - { - IDEBUG( cerr << "fetchNextRow done for table " << ti.msTablePtr->s->table_name.str << " rows = " << ti.c << endl ); - ti.c = 0; - ti.moreRows = false; - rc = HA_ERR_END_OF_FILE; - } - else if (sm_stat == sm::CALPONT_INTERNAL_ERROR) - { - ti.moreRows = false; - rc = ER_INTERNAL_ERROR; - ci->rc = rc; - } - else if ((uint32_t)sm_stat == logging::ERR_LOST_CONN_EXEMGR) - { - ti.moreRows = false; - rc = logging::ERR_LOST_CONN_EXEMGR; - sm::sm_init(tid2sid(current_thd->thread_id), &ci->cal_conn_hndl, - current_thd->variables.infinidb_local_query); - idbassert(ci->cal_conn_hndl != 0); - ci->rc = rc; - } - else if (sm_stat == sm::SQL_KILLED) - { - // query was aborted by the user. treat it the same as limit query. close - // connection after rnd_close. - ti.c = 0; - ti.moreRows = false; - rc = HA_ERR_END_OF_FILE; - ci->rc = rc; - } - else - { - ti.moreRows = false; - rc = sm_stat; - ci->rc = rc; - } - - return rc; -} - -int fetchNextRowGrHand(cal_table_info& ti, cal_connection_info* ci) -{ - int rc = HA_ERR_END_OF_FILE; - int num_attr = ti.msTablePtr->s->fields; - sm::status_t sm_stat; - - try - { - if (ti.conn_hndl) - { - sm_stat = sm::tpl_scan_fetch(ti.tpl_scan_ctx, ti.conn_hndl); - } - else if (ci->cal_conn_hndl) - { - sm_stat = sm::tpl_scan_fetch(ti.tpl_scan_ctx, ci->cal_conn_hndl, (int*)(¤t_thd->killed)); - } - else - throw runtime_error("internal error"); - } - catch (std::exception& ex) - { -// @bug 2244. Always log this msg for now, as we try to track down when/why we are -// losing socket connection with ExeMgr -//#ifdef INFINIDB_DEBUG - tpl_scan_fetch_LogException( ti, ci, &ex); -//#endif - sm_stat = sm::CALPONT_INTERNAL_ERROR; - } - catch (...) - { -// @bug 2244. Always log this msg for now, as we try to track down when/why we are -// losing socket connection with ExeMgr -//#ifdef INFINIDB_DEBUG - tpl_scan_fetch_LogException( ti, ci, 0 ); -//#endif - sm_stat = sm::CALPONT_INTERNAL_ERROR; - } - - if (sm_stat == sm::STATUS_OK) - { - Field** f; - f = ti.msTablePtr->field; - //set all fields to null in null col bitmap - //memset(buf, -1, ti.msTablePtr->s->null_bytes); - std::vector& colTypes = ti.tpl_scan_ctx->ctp; - int64_t intColVal = 0; - uint64_t uintColVal = 0; - char tmp[256]; - - RowGroup* rowGroup = ti.tpl_scan_ctx->rowGroup; - - // table mode mysql expects all columns of the table. mapping between columnoid and position in rowgroup - // set coltype.position to be the position in rowgroup. only set once. - if (ti.tpl_scan_ctx->rowsreturned == 0 && - (ti.tpl_scan_ctx->traceFlags & execplan::CalpontSelectExecutionPlan::TRACE_TUPLE_OFF)) - { - for (uint32_t i = 0; i < rowGroup->getColumnCount(); i++) - { - int oid = rowGroup->getOIDs()[i]; - int j = 0; - - for (; j < num_attr; j++) - { - // mysql should haved eliminated duplicate projection columns - if (oid == colTypes[j].columnOID || oid == colTypes[j].ddn.dictOID) - { - colTypes[j].colPosition = i; - break; - } - } - } - } - - rowgroup::Row row; - rowGroup->initRow(&row); - rowGroup->getRow(ti.tpl_scan_ctx->rowsreturned, &row); - int s; - - for (int p = 0; p < num_attr; p++, f++) - { - //This col is going to be written - bitmap_set_bit(ti.msTablePtr->write_set, (*f)->field_index); - - // get coltype if not there yet - if (colTypes[0].colWidth == 0) - { - for (short c = 0; c < num_attr; c++) - { - colTypes[c].colPosition = c; - colTypes[c].colWidth = rowGroup->getColumnWidth(c); - colTypes[c].colDataType = rowGroup->getColTypes()[c]; - colTypes[c].columnOID = rowGroup->getOIDs()[c]; - colTypes[c].scale = rowGroup->getScale()[c]; - colTypes[c].precision = rowGroup->getPrecision()[c]; - } - } - - CalpontSystemCatalog::ColType colType(colTypes[p]); - - // table mode handling - if (ti.tpl_scan_ctx->traceFlags & execplan::CalpontSelectExecutionPlan::TRACE_TUPLE_OFF) - { - if (colType.colPosition == -1) // not projected by tuplejoblist - continue; - else - s = colType.colPosition; - } - else - { - s = p; - } - - // precision == -16 is borrowed as skip null check indicator for bit ops. - if (row.isNullValue(s) && colType.precision != -16) - { - // @2835. Handle empty string and null confusion. store empty string for string column - if (colType.colDataType == CalpontSystemCatalog::CHAR || - colType.colDataType == CalpontSystemCatalog::VARCHAR || - colType.colDataType == CalpontSystemCatalog::VARBINARY) - { - Field_varstring* f2 = (Field_varstring*)*f; - f2->store(tmp, 0, f2->charset()); - } - - continue; - } - - // fetch and store data - switch (colType.colDataType) - { - case CalpontSystemCatalog::DATE: - { - if ((*f)->null_ptr) - *(*f)->null_ptr &= ~(*f)->null_bit; - - intColVal = row.getUintField<4>(s); - DataConvert::dateToString(intColVal, tmp, 255); - Field_varstring* f2 = (Field_varstring*)*f; - f2->store(tmp, strlen(tmp), f2->charset()); - f2->set_notnull(); - break; - } - - case CalpontSystemCatalog::DATETIME: - { - if ((*f)->null_ptr) - *(*f)->null_ptr &= ~(*f)->null_bit; - - intColVal = row.getUintField<8>(s); - DataConvert::datetimeToString(intColVal, tmp, 255); - - /* setting the field_length is a sort-of hack. The length - * at this point can be long enough to include mseconds. - * ColumnStore doesn't fully support mseconds yet so if - * they are requested, trim them off. - * At a later date we should set this more intelligently - * based on the result set. - */ - /* MCOL-683: UTF-8 datetime no msecs is 57, this sometimes happens! */ - if (((*f)->field_length > 19) && ((*f)->field_length != 57)) - (*f)->field_length = strlen(tmp); - - Field_varstring* f2 = (Field_varstring*)*f; - f2->store(tmp, strlen(tmp), f2->charset()); - f2->set_notnull(); - break; - } - - case CalpontSystemCatalog::CHAR: - case CalpontSystemCatalog::VARCHAR: - { - Field_varstring* f2 = (Field_varstring*)*f; - - switch (colType.colWidth) - { - case 1: - intColVal = row.getUintField<1>(s); - f2->store((char*)(&intColVal), strlen((char*)(&intColVal)), f2->charset()); - break; - - case 2: - intColVal = row.getUintField<2>(s); - f2->store((char*)(&intColVal), strlen((char*)(&intColVal)), f2->charset()); - break; - - case 4: - intColVal = row.getUintField<4>(s); - f2->store((char*)(&intColVal), strlen((char*)(&intColVal)), f2->charset()); - break; - - case 8: - //make sure we don't send strlen off into the weeds... - intColVal = row.getUintField<8>(s); - memcpy(tmp, &intColVal, 8); - tmp[8] = 0; - f2->store(tmp, strlen(tmp), f2->charset()); - break; - - default: - f2->store((const char*)row.getStringPointer(s), row.getStringLength(s), f2->charset()); - } - - if ((*f)->null_ptr) - *(*f)->null_ptr &= ~(*f)->null_bit; - f2->set_notnull(); - break; - } - - case CalpontSystemCatalog::VARBINARY: - { - Field_varstring* f2 = (Field_varstring*)*f; - - if (current_thd->variables.infinidb_varbin_always_hex) - { - uint32_t l; - const uint8_t* p = row.getVarBinaryField(l, s); - uint32_t ll = l * 2; - boost::scoped_array sca(new char[ll]); - vbin2hex(p, l, sca.get()); - f2->store(sca.get(), ll, f2->charset()); - } - else - f2->store((const char*)row.getVarBinaryField(s), row.getVarBinaryLength(s), f2->charset()); - f2->set_notnull(); - - if ((*f)->null_ptr) - *(*f)->null_ptr &= ~(*f)->null_bit; - - break; - } - - case CalpontSystemCatalog::BIGINT: - { - intColVal = row.getIntField<8>(s); - storeNumericFieldGroupBy(f, intColVal, colType); - break; - } - - case CalpontSystemCatalog::UBIGINT: - { - uintColVal = row.getUintField<8>(s); - storeNumericFieldGroupBy(f, uintColVal, colType); - break; - } - - case CalpontSystemCatalog::INT: - { - intColVal = row.getIntField<4>(s); - storeNumericFieldGroupBy(f, intColVal, colType); - break; - } - - case CalpontSystemCatalog::UINT: - { - uintColVal = row.getUintField<4>(s); - storeNumericFieldGroupBy(f, uintColVal, colType); - break; - } - - case CalpontSystemCatalog::SMALLINT: - { - intColVal = row.getIntField<2>(s); - storeNumericFieldGroupBy(f, intColVal, colType); - break; - } - - case CalpontSystemCatalog::USMALLINT: - { - uintColVal = row.getUintField<2>(s); - storeNumericFieldGroupBy(f, uintColVal, colType); - break; - } - - case CalpontSystemCatalog::TINYINT: - { - intColVal = row.getIntField<1>(s); - storeNumericFieldGroupBy(f, intColVal, colType); - break; - } - - case CalpontSystemCatalog::UTINYINT: - { - uintColVal = row.getUintField<1>(s); - storeNumericFieldGroupBy(f, uintColVal, colType); - break; - } - - //In this case, we're trying to load a double output column with float data. This is the - // case when you do sum(floatcol), e.g. - case CalpontSystemCatalog::FLOAT: - case CalpontSystemCatalog::UFLOAT: - { - float dl = row.getFloatField(s); - - if (dl == std::numeric_limits::infinity()) - continue; - - //int64_t* icvp = (int64_t*)&dl; - //intColVal = *icvp; - Field_float* f2 = (Field_float*)*f; - // bug 3485, reserve enough space for the longest float value - // -3.402823466E+38 to -1.175494351E-38, 0, and - // 1.175494351E-38 to 3.402823466E+38. - (*f)->field_length = 40; - - //float float_val = *(float*)(&value); - //f2->store(float_val); - if (f2->decimals() < (uint32_t)row.getScale(s)) - f2->dec = (uint32_t)row.getScale(s); - - f2->store(dl); - f2->set_notnull(); - if ((*f)->null_ptr) - *(*f)->null_ptr &= ~(*f)->null_bit; - - break; - - //storeNumericField(f, intColVal, colType); - //break; - } - - case CalpontSystemCatalog::DOUBLE: - case CalpontSystemCatalog::UDOUBLE: - { - double dl = row.getDoubleField(s); - - if (dl == std::numeric_limits::infinity()) - continue; - - Field_double* f2 = (Field_double*)*f; - // bug 3483, reserve enough space for the longest double value - // -1.7976931348623157E+308 to -2.2250738585072014E-308, 0, and - // 2.2250738585072014E-308 to 1.7976931348623157E+308. - (*f)->field_length = 310; - - //double double_val = *(double*)(&value); - //f2->store(double_val); - if (f2->decimals() < (uint32_t)row.getScale(s)) - f2->dec = (uint32_t)row.getScale(s); - - f2->store(dl); - f2->set_notnull(); - - if ((*f)->null_ptr) - *(*f)->null_ptr &= ~(*f)->null_bit; - - break; - - - //int64_t* icvp = (int64_t*)&dl; - //intColVal = *icvp; - //storeNumericField(f, intColVal, colType); - //break; - } - - case CalpontSystemCatalog::DECIMAL: - case CalpontSystemCatalog::UDECIMAL: - { - intColVal = row.getIntField(s); - storeNumericFieldGroupBy(f, intColVal, colType); - break; - } - - case CalpontSystemCatalog::BLOB: - case CalpontSystemCatalog::TEXT: - { - Field_blob* f2 = (Field_blob*)*f; - f2->set_ptr(row.getVarBinaryLength(s), (unsigned char*)row.getVarBinaryField(s)); - - if ((*f)->null_ptr) - *(*f)->null_ptr &= ~(*f)->null_bit; - - break; - } - - default: // treat as int64 - { - intColVal = row.getUintField<8>(s); - storeNumericFieldGroupBy(f, intColVal, colType); - break; - } - } - } - - ti.tpl_scan_ctx->rowsreturned++; - ti.c++; -#ifdef INFINIDB_DEBUG - - if ((ti.c % 1000000) == 0) - cerr << "fetchNextRow so far table " << ti.msTablePtr->s->table_name.str << " rows = " << ti.c << endl; - -#endif - ti.moreRows = true; - rc = 0; - } - else if (sm_stat == sm::SQL_NOT_FOUND) - { - IDEBUG( cerr << "fetchNextRow done for table " << ti.msTablePtr->s->table_name.str << " rows = " << ti.c << endl ); - ti.c = 0; - ti.moreRows = false; - rc = HA_ERR_END_OF_FILE; - } - else if (sm_stat == sm::CALPONT_INTERNAL_ERROR) - { - ti.moreRows = false; - rc = ER_INTERNAL_ERROR; - ci->rc = rc; - } - else if ((uint32_t)sm_stat == logging::ERR_LOST_CONN_EXEMGR) - { - ti.moreRows = false; - rc = logging::ERR_LOST_CONN_EXEMGR; - sm::sm_init(tid2sid(current_thd->thread_id), &ci->cal_conn_hndl, - current_thd->variables.infinidb_local_query); - idbassert(ci->cal_conn_hndl != 0); - ci->rc = rc; - } - else if (sm_stat == sm::SQL_KILLED) - { - // query was aborted by the user. treat it the same as limit query. close - // connection after rnd_close. - ti.c = 0; - ti.moreRows = false; - rc = HA_ERR_END_OF_FILE; - ci->rc = rc; - } - else - { - ti.moreRows = false; - rc = sm_stat; - ci->rc = rc; - } - - return rc; -} - void makeUpdateScalarJoin(const ParseTree* n, void* obj) { TreeNode* tn = n->data(); @@ -5925,8 +5058,7 @@ int ha_calpont_impl_rnd_pos(uchar* buf, uchar* pos) } /*@brief ha_calpont_impl_group_by_init - Get data for MariaDB group_by - pushdown handler -*/ + pushdown handler */ /*********************************************************** * DESCRIPTION: * Prepares data for group_by_handler::next_row() calls. @@ -5977,19 +5109,13 @@ int ha_calpont_impl_group_by_init(ha_calpont_group_by_handler* group_hand, TABLE // @bug 2232. Basic SP support. Error out non support sp cases. // @bug 3939. Only error out for sp with select. Let pass for alter table in sp. - /*if (thd->infinidb_vtable.call_sp && (thd->lex)->sql_command != SQLCOM_ALTER_TABLE) + if (thd->infinidb_vtable.call_sp && (thd->lex)->sql_command != SQLCOM_ALTER_TABLE) { setError(thd, ER_CHECK_NOT_IMPLEMENTED, "This stored procedure syntax is not supported by Columnstore in this version"); thd->infinidb_vtable.vtable_state = THD::INFINIDB_ERROR; return ER_INTERNAL_ERROR; - } + } - // mysql reads table twice for order by - if (thd->infinidb_vtable.vtable_state == THD::INFINIDB_REDO_PHASE1 || - thd->infinidb_vtable.vtable_state == THD::INFINIDB_ORDER_BY) - return 0; - */ - uint32_t sessionID = tid2sid(thd->thread_id); boost::shared_ptr csc = CalpontSystemCatalog::makeCalpontSystemCatalog(sessionID); csc->identity(CalpontSystemCatalog::FE); @@ -6001,15 +5127,12 @@ int ha_calpont_impl_group_by_init(ha_calpont_group_by_handler* group_hand, TABLE idbassert(ci != 0); - /* - MySQL sometimes calls rnd_init multiple times, plan should only be - generated and sent once. - TO DO Check this statement. + // MySQL sometimes calls rnd_init multiple times, plan should only be + // generated and sent once. if (thd->infinidb_vtable.vtable_state == THD::INFINIDB_DISABLE_VTABLE && !thd->infinidb_vtable.isNewQuery) return 0; - */ if (thd->killed == KILL_QUERY || thd->killed == KILL_QUERY_HARD) { @@ -6096,8 +5219,7 @@ int ha_calpont_impl_group_by_init(ha_calpont_group_by_handler* group_hand, TABLE hndl = ci->cal_conn_hndl; if (thd->infinidb_vtable.vtable_state != THD::INFINIDB_SELECT_VTABLE) - { - //CalpontSelectExecutionPlan csep; + { if (!csep) csep.reset(new CalpontSelectExecutionPlan()); @@ -6133,19 +5255,7 @@ int ha_calpont_impl_group_by_init(ha_calpont_group_by_handler* group_hand, TABLE // send plan whenever group_init is called int status = cp_get_group_plan(thd, csep, gi); -/* - if (thd->db) - csep->schemaName(thd->db); - csep->traceFlags(ci->traceFlags); - - if (thd->infinidb_vtable.isInsertSelect) - csep->queryType(CalpontSelectExecutionPlan::INSERT_SELECT); - - //get plan - int status = cp_get_plan(thd, csep); -*/ - //if (cp_get_plan(thd, csep) != 0) if (status > 0) goto internal_error; else if (status < 0) @@ -6445,14 +5555,6 @@ internal_error: ***********************************************************/ int ha_calpont_impl_group_by_next(ha_calpont_group_by_handler* group_hand, TABLE* table) { - //if (!first_row) - //return(HA_ERR_END_OF_FILE); - - //first_row = 0; - //Field *field = *(table->field); - //field->store(5LL, 1); - //field->set_notnull(); - //return(0); THD* thd = current_thd; /* If this node is the slave, ignore DML to IDB tables */ @@ -6539,9 +5641,9 @@ int ha_calpont_impl_group_by_next(ha_calpont_group_by_handler* group_hand, TABLE try { - rc = fetchNextRowGrHandGroupBy(ti, ci); - // MCOL-1052 - //rc = 0; + // fetchNextRow interface forces to use buf. + unsigned char buf; + rc = fetchNextRow(&buf, ti, ci, true); } catch (std::exception& e) { @@ -6739,7 +5841,7 @@ int ha_calpont_impl_group_by_end(ha_calpont_group_by_handler* group_hand, TABLE* } catch (...) { - setError(thd, ER_INTERNAL_ERROR, "Internal error throwed in rnd_end"); + setError(thd, ER_INTERNAL_ERROR, "Internal error throwed in group_by_end"); rc = ER_INTERNAL_ERROR; } } diff --git a/dbcon/mysql/ha_calpont_impl_if.h b/dbcon/mysql/ha_calpont_impl_if.h index 93d9cfa12..83f6aaf41 100644 --- a/dbcon/mysql/ha_calpont_impl_if.h +++ b/dbcon/mysql/ha_calpont_impl_if.h @@ -100,7 +100,6 @@ struct gp_walk_info execplan::CalpontSelectExecutionPlan::ReturnedColumnList subGroupByCols; execplan::CalpontSelectExecutionPlan::ReturnedColumnList orderByCols; std::vector havingAggColsItems; - std::vector extSelectColsItems; execplan::CalpontSelectExecutionPlan::ColumnMap columnMap; // This vector temporarily hold the projection columns to be added // to the returnedCols vector for subquery processing. It will be appended From 720e1d91b98dd551cee01f1771e5876d2bf2a38a Mon Sep 17 00:00:00 2001 From: Roman Nozdrin Date: Wed, 11 Apr 2018 16:15:12 +0300 Subject: [PATCH 18/57] MCOL-1052 WIP Clean up the code. --- dbcon/mysql/ha_calpont.cpp | 1 + dbcon/mysql/ha_calpont_impl.cpp | 260 ++++++++++++++------------------ 2 files changed, 112 insertions(+), 149 deletions(-) diff --git a/dbcon/mysql/ha_calpont.cpp b/dbcon/mysql/ha_calpont.cpp index 6b5d6568f..aed7ffafc 100644 --- a/dbcon/mysql/ha_calpont.cpp +++ b/dbcon/mysql/ha_calpont.cpp @@ -1181,6 +1181,7 @@ int ha_calpont_group_by_handler::init_scan() // Save vtable_state to restore the after we inited. THD::infinidb_state oldState = thd->infinidb_vtable.vtable_state; + // MCOL-1052 Should be removed after cleaning the code up. thd->infinidb_vtable.vtable_state = THD::INFINIDB_CREATE_VTABLE; int rc = ha_calpont_impl_group_by_init(this, table); thd->infinidb_vtable.vtable_state = oldState; diff --git a/dbcon/mysql/ha_calpont_impl.cpp b/dbcon/mysql/ha_calpont_impl.cpp index 926e5cd93..147c8ce79 100644 --- a/dbcon/mysql/ha_calpont_impl.cpp +++ b/dbcon/mysql/ha_calpont_impl.cpp @@ -5165,41 +5165,37 @@ int ha_calpont_impl_group_by_init(ha_calpont_group_by_handler* group_hand, TABLE sm::cpsm_conhdl_t* hndl; SCSEP csep; - bool localQuery = (thd->variables.infinidb_local_query > 0 ? true : false); + bool localQuery = (thd->variables.infinidb_local_query > 0 ? true : false); - - //if (!ci->cal_conn_hndl || thd->infinidb_vtable.vtable_state == THD::INFINIDB_CREATE_VTABLE) - if ( thd->infinidb_vtable.vtable_state == THD::INFINIDB_CREATE_VTABLE) + { + ci->stats.reset(); // reset query stats + ci->stats.setStartTime(); + ci->stats.fUser = thd->main_security_ctx.user; + + if (thd->main_security_ctx.host) + ci->stats.fHost = thd->main_security_ctx.host; + else if (thd->main_security_ctx.host_or_ip) + ci->stats.fHost = thd->main_security_ctx.host_or_ip; + else + ci->stats.fHost = "unknown"; + + try { - ci->stats.reset(); // reset query stats - ci->stats.setStartTime(); - ci->stats.fUser = thd->main_security_ctx.user; - - if (thd->main_security_ctx.host) - ci->stats.fHost = thd->main_security_ctx.host; - else if (thd->main_security_ctx.host_or_ip) - ci->stats.fHost = thd->main_security_ctx.host_or_ip; - else - ci->stats.fHost = "unknown"; - - try - { - ci->stats.userPriority(ci->stats.fHost, ci->stats.fUser); - } - catch (std::exception& e) - { - string msg = string("Columnstore User Priority - ") + e.what(); - ci->warningMsg = msg; - } - - // if the previous query has error, re-establish the connection - if (ci->queryState != 0) - { - sm::sm_cleanup(ci->cal_conn_hndl); - ci->cal_conn_hndl = 0; - } + ci->stats.userPriority(ci->stats.fHost, ci->stats.fUser); + } + catch (std::exception& e) + { + string msg = string("Columnstore User Priority - ") + e.what(); + ci->warningMsg = msg; } + // if the previous query has error, re-establish the connection + if (ci->queryState != 0) + { + sm::sm_cleanup(ci->cal_conn_hndl); + ci->cal_conn_hndl = 0; + } + sm::sm_init(sessionID, &ci->cal_conn_hndl, localQuery); idbassert(ci->cal_conn_hndl != 0); ci->cal_conn_hndl->csc = csc; @@ -5218,93 +5214,90 @@ int ha_calpont_impl_group_by_init(ha_calpont_group_by_handler* group_hand, TABLE hndl = ci->cal_conn_hndl; - if (thd->infinidb_vtable.vtable_state != THD::INFINIDB_SELECT_VTABLE) - { - if (!csep) - csep.reset(new CalpontSelectExecutionPlan()); + if (!csep) + csep.reset(new CalpontSelectExecutionPlan()); - SessionManager sm; - BRM::TxnID txnID; - txnID = sm.getTxnID(sessionID); + SessionManager sm; + BRM::TxnID txnID; + txnID = sm.getTxnID(sessionID); - if (!txnID.valid) - { - txnID.id = 0; - txnID.valid = true; - } - - QueryContext verID; - verID = sm.verID(); - - csep->txnID(txnID.id); - csep->verID(verID); - csep->sessionID(sessionID); - - if (group_hand->table_list->db_length) - csep->schemaName(group_hand->table_list->db); - - csep->traceFlags(ci->traceFlags); - - gi.groupByTables = group_hand->table_list; - gi.groupByFields = group_hand->select; - gi.groupByWhere = group_hand->where; - gi.groupByGroup = group_hand->group_by; - gi.groupByOrder = group_hand->order_by; - gi.groupByHaving = group_hand->having; - gi.groupByDistinct = group_hand->distinct; - - // send plan whenever group_init is called - int status = cp_get_group_plan(thd, csep, gi); - - if (status > 0) - goto internal_error; - else if (status < 0) - return 0; - - // @bug 2547. don't need to send the plan if it's impossible where for all unions. - if (thd->infinidb_vtable.impossibleWhereOnUnion) - return 0; - - string query; - query.assign(thd->infinidb_vtable.original_query.ptr(), - thd->infinidb_vtable.original_query.length()); - csep->data(query); - - try - { - csep->priority( ci->stats.userPriority(ci->stats.fHost, ci->stats.fUser)); - } - catch (std::exception& e) - { - string msg = string("Columnstore User Priority - ") + e.what(); - push_warning(thd, Sql_condition::WARN_LEVEL_WARN, 9999, msg.c_str()); - } - -#ifdef PLAN_HEX_FILE - // plan serialization - ifstream ifs("/tmp/li1-plan.hex"); - ByteStream bs1; - ifs >> bs1; - ifs.close(); - csep->unserialize(bs1); -#endif - - if (ci->traceFlags & 1) - { - cerr << "---------------- EXECUTION PLAN ----------------" << endl; - cerr << *csep << endl ; - cerr << "-------------- EXECUTION PLAN END --------------\n" << endl; - } - else - { - IDEBUG( cout << "---------------- EXECUTION PLAN ----------------" << endl ); - IDEBUG( cerr << *csep << endl ); - IDEBUG( cout << "-------------- EXECUTION PLAN END --------------\n" << endl ); - } + if (!txnID.valid) + { + txnID.id = 0; + txnID.valid = true; } - // end of execution plan generation - if (thd->infinidb_vtable.vtable_state != THD::INFINIDB_SELECT_VTABLE) + QueryContext verID; + verID = sm.verID(); + + csep->txnID(txnID.id); + csep->verID(verID); + csep->sessionID(sessionID); + + if (group_hand->table_list->db_length) + csep->schemaName(group_hand->table_list->db); + + csep->traceFlags(ci->traceFlags); + + // MCOL-1052 Send Items lists down to the optimizer. + gi.groupByTables = group_hand->table_list; + gi.groupByFields = group_hand->select; + gi.groupByWhere = group_hand->where; + gi.groupByGroup = group_hand->group_by; + gi.groupByOrder = group_hand->order_by; + gi.groupByHaving = group_hand->having; + gi.groupByDistinct = group_hand->distinct; + + // send plan whenever group_init is called + int status = cp_get_group_plan(thd, csep, gi); + + if (status > 0) + goto internal_error; + else if (status < 0) + return 0; + + // @bug 2547. don't need to send the plan if it's impossible where for all unions. + if (thd->infinidb_vtable.impossibleWhereOnUnion) + return 0; + + string query; + query.assign(thd->infinidb_vtable.original_query.ptr(), + thd->infinidb_vtable.original_query.length()); + csep->data(query); + + try + { + csep->priority( ci->stats.userPriority(ci->stats.fHost, ci->stats.fUser)); + } + catch (std::exception& e) + { + string msg = string("Columnstore User Priority - ") + e.what(); + push_warning(thd, Sql_condition::WARN_LEVEL_WARN, 9999, msg.c_str()); + } + + #ifdef PLAN_HEX_FILE + // plan serialization + ifstream ifs("/tmp/li1-plan.hex"); + ByteStream bs1; + ifs >> bs1; + ifs.close(); + csep->unserialize(bs1); + #endif + + if (ci->traceFlags & 1) + { + cerr << "---------------- EXECUTION PLAN ----------------" << endl; + cerr << *csep << endl ; + cerr << "-------------- EXECUTION PLAN END --------------\n" << endl; + } + else + { + IDEBUG( cout << "---------------- EXECUTION PLAN ----------------" << endl ); + IDEBUG( cerr << *csep << endl ); + IDEBUG( cout << "-------------- EXECUTION PLAN END --------------\n" << endl ); + } + }// end of execution plan generation + { ByteStream msg; ByteStream emsgBs; @@ -5535,8 +5528,7 @@ internal_error: ci->cal_conn_hndl = 0; } - return ER_INTERNAL_ERROR; - //return(0); + return ER_INTERNAL_ERROR; } /*@brief ha_calpont_impl_group_by_next - Return result set for MariaDB group_by @@ -5573,10 +5565,6 @@ int ha_calpont_impl_group_by_next(ha_calpont_group_by_handler* group_hand, TABLE if (thd->infinidb_vtable.vtable_state == THD::INFINIDB_ERROR) return ER_INTERNAL_ERROR; - // @bug 3005 - //if (thd->infinidb_vtable.vtable_state == THD::INFINIDB_SELECT_VTABLE) - // return HA_ERR_END_OF_FILE; - if (((thd->lex)->sql_command == SQLCOM_UPDATE) || ((thd->lex)->sql_command == SQLCOM_DELETE) || ((thd->lex)->sql_command == SQLCOM_DELETE_MULTI) || ((thd->lex)->sql_command == SQLCOM_UPDATE_MULTI)) return HA_ERR_END_OF_FILE; @@ -5668,8 +5656,7 @@ int ha_calpont_impl_group_by_next(ha_calpont_group_by_handler* group_hand, TABLE emsg = errorcodes.errorString(rc); } - setError(thd, ER_INTERNAL_ERROR, emsg); - //setError(thd, ER_INTERNAL_ERROR, "testing"); + setError(thd, ER_INTERNAL_ERROR, emsg); ci->stats.fErrorNo = rc; CalpontSystemCatalog::removeCalpontSystemCatalog(tid2sid(thd->thread_id)); rc = ER_INTERNAL_ERROR; @@ -5707,18 +5694,6 @@ int ha_calpont_impl_group_by_end(ha_calpont_group_by_handler* group_hand, TABLE* return rc; } -// if (thd->infinidb_vtable.vtable_state == THD::INFINIDB_REDO_PHASE1) -// return rc; -/* - if ( (thd->lex)->sql_command == SQLCOM_ALTER_TABLE ) - return rc; - - if (((thd->lex)->sql_command == SQLCOM_UPDATE) || - ((thd->lex)->sql_command == SQLCOM_DELETE) || - ((thd->lex)->sql_command == SQLCOM_DELETE_MULTI) || - ((thd->lex)->sql_command == SQLCOM_UPDATE_MULTI)) - return rc; -*/ if (((thd->lex)->sql_command == SQLCOM_INSERT) || ((thd->lex)->sql_command == SQLCOM_INSERT_SELECT) ) { @@ -5788,10 +5763,7 @@ int ha_calpont_impl_group_by_end(ha_calpont_group_by_handler* group_hand, TABLE* cal_table_info ti = ci->tableMap[table]; sm::cpsm_conhdl_t* hndl; - //if (thd->infinidb_vtable.vtable_state == THD::INFINIDB_DISABLE_VTABLE) - // hndl = ti.conn_hndl; - //else - hndl = ci->cal_conn_hndl; + hndl = ci->cal_conn_hndl; if (ti.tpl_ctx) { @@ -5813,11 +5785,7 @@ int ha_calpont_impl_group_by_end(ha_calpont_group_by_handler* group_hand, TABLE* { sm::tpl_close(ti.tpl_ctx, &hndl, ci->stats); - // set conn hndl back. could be changed in tpl_close - //if (thd->infinidb_vtable.vtable_state == THD::INFINIDB_DISABLE_VTABLE) - // ti.conn_hndl = hndl; - //else - ci->cal_conn_hndl = hndl; + ci->cal_conn_hndl = hndl; ti.tpl_ctx = 0; } @@ -5848,11 +5816,6 @@ int ha_calpont_impl_group_by_end(ha_calpont_group_by_handler* group_hand, TABLE* ti.tpl_ctx = 0; - /* - if (thd->infinidb_vtable.vtable_state == THD::INFINIDB_SELECT_VTABLE && - thd->infinidb_vtable.has_order_by) - thd->infinidb_vtable.vtable_state = THD::INFINIDB_ORDER_BY; - */ ci->tableMap[table] = ti; // push warnings from CREATE phase @@ -5866,4 +5829,3 @@ int ha_calpont_impl_group_by_end(ha_calpont_group_by_handler* group_hand, TABLE* } // vim:sw=4 ts=4: - From 8220a5103fd77f9f89a94bbb84411292ab6ba1bb Mon Sep 17 00:00:00 2001 From: david hill Date: Fri, 13 Apr 2018 10:29:17 -0500 Subject: [PATCH 19/57] MCOL-1333 - changed addmodule to look for rpm/deb, not tar.gz packages --- procmgr/processmanager.cpp | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/procmgr/processmanager.cpp b/procmgr/processmanager.cpp index fdb3f76b1..7343e6c7a 100755 --- a/procmgr/processmanager.cpp +++ b/procmgr/processmanager.cpp @@ -4530,10 +4530,10 @@ int ProcessManager::addModule(oam::DeviceNetworkList devicenetworklist, std::str } if ( packageType == "rpm") - calpontPackage = homedir + "/mariadb-columnstore*" + columnstore_version + "-" + columnstore_release + "*.rpm.tar.gz"; + calpontPackage = homedir + "/mariadb-columnstore*" + columnstore_version + "-" + columnstore_release + "*.rpm"; else if ( packageType == "deb") - calpontPackage = homedir + "/mariadb-columnstore*" + columnstore_version + "-" + columnstore_release + "*.deb.tar.gz"; + calpontPackage = homedir + "/mariadb-columnstore*" + columnstore_version + "-" + columnstore_release + "*.deb"; else calpontPackage = homedir + "/mariadb-columnstore*" + columnstore_version + "-" + columnstore_release + "*.bin.tar.gz"; From 4013c8b9b259903ba073b7353aa345e077b268a3 Mon Sep 17 00:00:00 2001 From: Roman Nozdrin Date: Fri, 13 Apr 2018 22:16:44 +0300 Subject: [PATCH 20/57] MCOL-1327 CS supports visible special ASCII symbols. --- dbcon/ddlpackage/ddl.l | 6 ++++-- 1 file changed, 4 insertions(+), 2 deletions(-) diff --git a/dbcon/ddlpackage/ddl.l b/dbcon/ddlpackage/ddl.l index 4812bbd0d..e922c08ca 100644 --- a/dbcon/ddlpackage/ddl.l +++ b/dbcon/ddlpackage/ddl.l @@ -60,6 +60,7 @@ double_quote \" grave_accent ` comment ("--"{non_newline}*) +extended_ident_cont [A-Za-z\200-\377_0-9\$#,()\[\].;\:\+\-\*\/\%\^\<\>\=!&|@\\] self [,()\[\].;\:\+\-\*\/\%\^\<\>\=] whitespace ({space}+|{comment}) @@ -67,10 +68,11 @@ digit [0-9] ident_start [A-Za-z\200-\377_0-9] ident_cont [A-Za-z\200-\377_0-9\$] identifier {ident_start}{ident_cont}* +extended_identifier {ident_start}{extended_ident_cont}* /* fully qualified names regexes */ fq_identifier {identifier}\.{identifier} -identifer_quoted {grave_accent}{identifier}{grave_accent} -identifer_double_quoted {double_quote}{identifier}{double_quote} +identifer_quoted {grave_accent}{extended_identifier}{grave_accent} +identifer_double_quoted {double_quote}{extended_identifier}{double_quote} integer [-+]?{digit}+ decimal ([-+]?({digit}*\.{digit}+)|({digit}+\.{digit}*)) From 5d48ddb158d33f2ff903a7448315a0c12bc71c2f Mon Sep 17 00:00:00 2001 From: Roman Nozdrin Date: Mon, 16 Apr 2018 16:51:42 +0300 Subject: [PATCH 21/57] MCOL-1052 CS group_by_handler now uses CS internal sorting. --- dbcon/joblist/jlf_subquery.cpp | 4 ++-- dbcon/joblist/jlf_tuplejoblist.cpp | 16 ++++++++-------- dbcon/joblist/joblistfactory.cpp | 2 +- dbcon/joblist/limitedorderby.cpp | 12 ++++++++++-- dbcon/mysql/CMakeLists.txt | 3 --- 5 files changed, 21 insertions(+), 16 deletions(-) diff --git a/dbcon/joblist/jlf_subquery.cpp b/dbcon/joblist/jlf_subquery.cpp index e1537d577..5c868bee3 100644 --- a/dbcon/joblist/jlf_subquery.cpp +++ b/dbcon/joblist/jlf_subquery.cpp @@ -751,8 +751,8 @@ int doFromSubquery(CalpontExecutionPlan* ep, const string& alias, const string& void addOrderByAndLimit(CalpontSelectExecutionPlan* csep, JobInfo& jobInfo) { // make sure there is a LIMIT - if (csep->orderByCols().size() > 0 && csep->limitNum() == (uint64_t) - 1) - return; +// if (csep->orderByCols().size() > 0 csep->limitNum() == (uint64_t) - 1) +// return; jobInfo.limitStart = csep->limitStart(); jobInfo.limitCount = csep->limitNum(); diff --git a/dbcon/joblist/jlf_tuplejoblist.cpp b/dbcon/joblist/jlf_tuplejoblist.cpp index aabc81486..480b23f91 100644 --- a/dbcon/joblist/jlf_tuplejoblist.cpp +++ b/dbcon/joblist/jlf_tuplejoblist.cpp @@ -483,28 +483,28 @@ void adjustLastStep(JobStepVector& querySteps, DeliveredTableMap& deliverySteps, deliverySteps[CNX_VTABLE_ID] = ws; } - if ((jobInfo.limitCount != (uint64_t) - 1) || - (jobInfo.constantCol == CONST_COL_EXIST) || - (jobInfo.hasDistinct)) - { +// if ((jobInfo.limitCount != (uint64_t) - 1) || +// (jobInfo.constantCol == CONST_COL_EXIST) || +// (jobInfo.hasDistinct)) +// { if (jobInfo.annexStep.get() == NULL) jobInfo.annexStep.reset(new TupleAnnexStep(jobInfo)); TupleAnnexStep* tas = dynamic_cast(jobInfo.annexStep.get()); tas->setLimit(jobInfo.limitStart, jobInfo.limitCount); - if (jobInfo.limitCount != (uint64_t) - 1) - { + // if (jobInfo.limitCount != (uint64_t) - 1) +// { if (jobInfo.orderByColVec.size() > 0) tas->addOrderBy(new LimitedOrderBy()); - } +// } if (jobInfo.constantCol == CONST_COL_EXIST) tas->addConstant(new TupleConstantStep(jobInfo)); if (jobInfo.hasDistinct) tas->setDistinct(); - } +// } if (jobInfo.annexStep) { diff --git a/dbcon/joblist/joblistfactory.cpp b/dbcon/joblist/joblistfactory.cpp index 812256869..a48ecd13a 100644 --- a/dbcon/joblist/joblistfactory.cpp +++ b/dbcon/joblist/joblistfactory.cpp @@ -1622,7 +1622,7 @@ void makeVtableModeSteps(CalpontSelectExecutionPlan* csep, JobInfo& jobInfo, JobStepVector& querySteps, JobStepVector& projectSteps, DeliveredTableMap& deliverySteps) { // @bug4848, enhance and unify limit handling. - if (csep->limitNum() != (uint64_t) - 1) +// if (csep->limitNum() != (uint64_t) - 1) { // special case for outer query order by limit -- return all if (jobInfo.subId == 0 && csep->hasOrderBy()) diff --git a/dbcon/joblist/limitedorderby.cpp b/dbcon/joblist/limitedorderby.cpp index e9bcaf95e..6701faf4b 100644 --- a/dbcon/joblist/limitedorderby.cpp +++ b/dbcon/joblist/limitedorderby.cpp @@ -76,7 +76,13 @@ void LimitedOrderBy::initialize(const RowGroup& rg, const JobInfo& jobInfo) { map::iterator j = keyToIndexMap.find(i->first); idbassert(j != keyToIndexMap.end()); - fOrderByCond.push_back(IdbSortSpec(j->second, i->second)); + // MCOL-1052 Ordering direction in CSEP differs from + // internal direction representation. + if (i->second) + fOrderByCond.push_back(IdbSortSpec(j->second, false)); + else + fOrderByCond.push_back(IdbSortSpec(j->second, true)); + //fOrderByCond.push_back(IdbSortSpec(j->second, i->second)); } // limit row count info @@ -174,7 +180,9 @@ void LimitedOrderBy::finalize() if (fRowGroup.getRowCount() > 0) fDataQueue.push(fData); - if (fStart != 0) + // MCOL-1052 The removed check effectivly disables sorting to happen, + // since fStart = 0; + if (true) { uint64_t newSize = fRowsPerRG * fRowGroup.getRowSize(); fMemSize += newSize; diff --git a/dbcon/mysql/CMakeLists.txt b/dbcon/mysql/CMakeLists.txt index a1786481d..104e6bf6e 100644 --- a/dbcon/mysql/CMakeLists.txt +++ b/dbcon/mysql/CMakeLists.txt @@ -20,9 +20,6 @@ SET ( libcalmysql_SRCS ha_pseudocolumn.cpp) add_definitions(-DMYSQL_DYNAMIC_PLUGIN) -add_definitions(-DDEBUG_WALK_COND=1) -add_definitions(-DINFINIDB_DEBUG=1) -#add_definitions(-DOUTER_JOIN_DEBUG) set_source_files_properties(ha_calpont.cpp PROPERTIES COMPILE_FLAGS "-fno-rtti -fno-implicit-templates") From 0c6dc5e15c97061d72df291a8c33100d3b345cfb Mon Sep 17 00:00:00 2001 From: Andrew Hutchings Date: Mon, 16 Apr 2018 19:21:28 +0100 Subject: [PATCH 22/57] MCOL-1341 Fix CASE handling with 10.2.14 MariaDB Server 10.2.14 changed the order that CASE items are processed. This broke the engine's CASE handling. This patch uses the new order instead since this is what will be used in 10.2 and 10.3 going forward. --- dbcon/mysql/ha_calpont_execplan.cpp | 16 +++- utils/funcexp/func_case.cpp | 136 ++++++++++++++++++---------- 2 files changed, 105 insertions(+), 47 deletions(-) diff --git a/dbcon/mysql/ha_calpont_execplan.cpp b/dbcon/mysql/ha_calpont_execplan.cpp index da80366b8..0f54d8bdb 100755 --- a/dbcon/mysql/ha_calpont_execplan.cpp +++ b/dbcon/mysql/ha_calpont_execplan.cpp @@ -3201,6 +3201,20 @@ FunctionColumn* buildCaseFunction(Item_func* item, gp_walk_info& gwi, bool& nonS // and position. We can't tell which at this point, so we // rebuild the item from the arguments directly and then try to // figure what to pop, if anything, in order to sync the stacks. + // + // MCOL-1341 - With MariaDB 10.2.14 onwards CASE is now in the order: + // [case,]when1,when2,...,then1,then2,...[,else] + // See server commit bf1ca14ff3f3faa9f7a018097b25aa0f66d068cd for more + // information. + int32_t arg_offset = 0; + if ((item->argument_count() - 1) % 2) + { + arg_offset = (item->argument_count() - 1) / 2; + } + else + { + arg_offset = item->argument_count() / 2; + } for (int32_t i = item->argument_count()-1; i >=0; i--) { // For case_searched, we know the items for the WHEN clause will @@ -3209,7 +3223,7 @@ FunctionColumn* buildCaseFunction(Item_func* item, gp_walk_info& gwi, bool& nonS // Every even numbered arg is a WHEN. In between are the THEN. // An odd number of args indicates an ELSE residing in the last spot. if (funcName == "case_searched" && - i % 2 == 0 && uint(i) != item->argument_count()-1) + (i < arg_offset)) { sptp.reset(buildParseTree((Item_func*)(item->arguments()[i]), gwi, nonSupport)); if (!gwi.ptWorkStack.empty() && *gwi.ptWorkStack.top()->data() == sptp->data()) diff --git a/utils/funcexp/func_case.cpp b/utils/funcexp/func_case.cpp index 3787947ab..2b5ae58c2 100644 --- a/utils/funcexp/func_case.cpp +++ b/utils/funcexp/func_case.cpp @@ -52,9 +52,10 @@ inline uint64_t simple_case_cmp(Row& row, CalpontSystemCatalog::ColType& operationColType) { uint64_t i = 0; // index to the parm list - uint64_t n = parm.size() - 1; // remove expression from count of expression_i + result_i - uint64_t hasElse = n % 2; // if 1, then ELSE exist - n -= hasElse; // index to expression + uint64_t n = 0; // remove expression from count of expression_i + result_i + uint64_t hasElse = (parm.size()-1) % 2; // if 1, then ELSE exist + uint64_t whereCount = hasElse ? (parm.size() - 2) / 2 : (parm.size() - 1) / 2; + bool foundIt = false; switch (operationColType.colDataType) { @@ -70,10 +71,13 @@ inline uint64_t simple_case_cmp(Row& row, if (isNull) break; - for (; i < n; i += 2) + for (i = 1; i <= whereCount; i++) { if (ev == parm[i]->data()->getIntVal(row, isNull) && !isNull) + { + foundIt = true; break; + } else isNull = false; } @@ -90,10 +94,13 @@ inline uint64_t simple_case_cmp(Row& row, if (isNull) break; - for (; i < n; i += 2) + for (i = 1; i <= whereCount; i++) { if (ev == parm[i]->data()->getUintVal(row, isNull) && !isNull) + { + foundIt = true; break; + } else isNull = false; } @@ -108,11 +115,14 @@ inline uint64_t simple_case_cmp(Row& row, if (isNull) break; - for (; i < n; i += 2) + for (i = 1; i <= whereCount; i++) { //BUG 5362 if (utf8::idb_strcoll(ev.c_str(), parm[i]->data()->getStrVal(row, isNull).c_str()) == 0 && !isNull) + { + foundIt = true; break; + } else isNull = false; } @@ -126,10 +136,13 @@ inline uint64_t simple_case_cmp(Row& row, if (isNull) break; - for (; i < n; i += 2) + for (i = 1; i <= whereCount; i++) { if (ev == parm[i]->data()->getDecimalVal(row, isNull) && !isNull) + { + foundIt = true; break; + } else isNull = false; } @@ -143,10 +156,13 @@ inline uint64_t simple_case_cmp(Row& row, if (isNull) break; - for (; i < n; i += 2) + for (i = 1; i <= whereCount; i++) { if (ev == parm[i]->data()->getDoubleVal(row, isNull) && !isNull) + { + foundIt = true; break; + } else isNull = false; } @@ -160,10 +176,13 @@ inline uint64_t simple_case_cmp(Row& row, if (isNull) break; - for (; i < n; i += 2) + for (i = 1; i <= whereCount; i++) { if (ev == parm[i]->data()->getFloatVal(row, isNull) && !isNull) + { + foundIt = true; break; + } else isNull = false; } @@ -178,16 +197,24 @@ inline uint64_t simple_case_cmp(Row& row, } } - if (i == n && !hasElse) + if (!foundIt && !hasElse) isNull = true; + else if (!foundIt && hasElse && !isNull) + { + i = parm.size() - 1; + } else if (isNull && hasElse) // BUG 5110. Only way we can exit above with isNull == true is when ev is NULL - // if so and we have else condition we need to use it by setting i = n + // if so and we have else condition we need to use it by setting i = else { - i = n; + i = parm.size() - 1; isNull = false; } + if (foundIt) + { + i += whereCount; + } return i; } @@ -198,21 +225,33 @@ inline uint64_t searched_case_cmp(Row& row, bool& isNull) { uint64_t i = 0; // index to the parm list - uint64_t n = parm.size(); // count of boolean_expression_i + result_i - uint64_t hasElse = n % 2; // if 1, then ELSE exist - n -= hasElse; // index to expression + uint64_t hasElse = parm.size() % 2; // if 1, then ELSE exist + uint64_t whereCount = hasElse ? (parm.size() - 1) / 2 : parm.size() / 2; + bool foundIt = false; - for (; i < n; i += 2) + + for (i = 0; i < whereCount; i++) { if (parm[i]->getBoolVal(row, isNull)) + { + foundIt = true; break; + } } isNull = false; - if (i == n && !hasElse) + if (!foundIt && !hasElse) isNull = true; + else if (!foundIt && hasElse) + { + i = parm.size() - 1; + } + if (foundIt) + { + i += whereCount; + } - return (i == n ? i-1 : i); + return i; } @@ -220,7 +259,6 @@ CalpontSystemCatalog::ColType caseOperationType(FunctionParm& fp, CalpontSystemCatalog::ColType& resultType, bool simpleCase) { - // ... expression_i + result_i + ... [[expression] + result_N] FunctionParm::size_type n = fp.size(); if (simpleCase) // simple case has an expression @@ -228,6 +266,9 @@ CalpontSystemCatalog::ColType caseOperationType(FunctionParm& fp, bool hasElse = ((n % 2) != 0); // if 1, then ELSE exist if (hasElse) --n; // n now is an even number + uint64_t parmCount = hasElse ? (fp.size() - 2) : (fp.size() - 1); + uint64_t whereCount = hasElse ? (fp.size() - 2 + simpleCase) / 2 : (fp.size() - 1) / 2 + simpleCase; + idbassert((n % 2) == 0); bool allStringO = true; @@ -238,10 +279,10 @@ CalpontSystemCatalog::ColType caseOperationType(FunctionParm& fp, CalpontSystemCatalog::ColType oct = fp[l]->data()->resultType(); CalpontSystemCatalog::ColType rct = resultType; bool operation = true; - for (uint64_t i = 0; i <= n; i++) + for (uint64_t i = 0; i <= parmCount; i++) { // operation or result type - operation = ((i % 2) == 0); + operation = ((i > 0) && (i <= whereCount)); // the result type of ELSE, if exists. if (i == n) @@ -334,8 +375,9 @@ namespace funcexp // END // // simple CASE parm order: -// expression1 result1 expression2 result2 ... expression [resultN] +// expression condition1 condition2 ... result1 result2 ... [resultN] // +// Note that this order changed in 10.2.14, see MCOL-1341 CalpontSystemCatalog::ColType Func_simple_case::operationType(FunctionParm& fp, CalpontSystemCatalog::ColType& resultType) { @@ -353,7 +395,7 @@ bool Func_simple_case::getBoolVal(Row& row, if (isNull) return joblist::BIGINTNULL; - return parm[i+1]->data()->getBoolVal(row, isNull); + return parm[i]->data()->getBoolVal(row, isNull); } @@ -367,7 +409,7 @@ int64_t Func_simple_case::getIntVal(Row& row, if (isNull) return joblist::BIGINTNULL; - return parm[i+1]->data()->getIntVal(row, isNull); + return parm[i]->data()->getIntVal(row, isNull); } @@ -381,7 +423,7 @@ string Func_simple_case::getStrVal(Row& row, if (isNull) return string(""); - return parm[i+1]->data()->getStrVal(row, isNull); + return parm[i]->data()->getStrVal(row, isNull); } @@ -395,7 +437,7 @@ IDB_Decimal Func_simple_case::getDecimalVal(Row& row, if (isNull) return IDB_Decimal(); // need a null value for IDB_Decimal?? - return parm[i+1]->data()->getDecimalVal(row, isNull); + return parm[i]->data()->getDecimalVal(row, isNull); } @@ -409,7 +451,7 @@ double Func_simple_case::getDoubleVal(Row& row, if (isNull) return doubleNullVal(); - return parm[i+1]->data()->getDoubleVal(row, isNull); + return parm[i]->data()->getDoubleVal(row, isNull); } @@ -423,9 +465,9 @@ int32_t Func_simple_case::getDateIntVal(rowgroup::Row& row, if (isNull) return joblist::DATENULL; - return parm[i+1]->data()->getDateIntVal(row, isNull); -} - + return parm[i]->data()->getDateIntVal(row, isNull); +} + int64_t Func_simple_case::getDatetimeIntVal(rowgroup::Row& row, FunctionParm& parm, @@ -437,8 +479,8 @@ int64_t Func_simple_case::getDatetimeIntVal(rowgroup::Row& row, if (isNull) return joblist::DATETIMENULL; - return parm[i+1]->data()->getDatetimeIntVal(row, isNull); -} + return parm[i]->data()->getDatetimeIntVal(row, isNull); +} @@ -451,8 +493,10 @@ int64_t Func_simple_case::getDatetimeIntVal(rowgroup::Row& row, // END // // searched CASE parm order: -// boolean_expression1 result1 boolean_expression2 result2 ... [resultN] +// boolean_expression1 boolean_expression2 ... result1 result2 ... [resultN] // +// Note that this order changed in 10.2.14, see MCOL-1341 + CalpontSystemCatalog::ColType Func_searched_case::operationType(FunctionParm& fp, CalpontSystemCatalog::ColType& resultType) { // operation type not used by this functor. @@ -470,14 +514,14 @@ bool Func_searched_case::getBoolVal(Row& row, if (isNull) return joblist::BIGINTNULL; - ParseTree* lop = parm[i+1]->left(); - ParseTree* rop = parm[i+1]->right(); + ParseTree* lop = parm[i]->left(); + ParseTree* rop = parm[i]->right(); if (lop && rop) { - return (reinterpret_cast(parm[i+1]->data()))->getBoolVal(row, isNull, lop, rop); + return (reinterpret_cast(parm[i]->data()))->getBoolVal(row, isNull, lop, rop); } - return parm[i+1]->data()->getBoolVal(row, isNull); + return parm[i]->data()->getBoolVal(row, isNull); } int64_t Func_searched_case::getIntVal(Row& row, @@ -490,7 +534,7 @@ int64_t Func_searched_case::getIntVal(Row& row, if (isNull) return joblist::BIGINTNULL; - return parm[i+1]->data()->getIntVal(row, isNull); + return parm[i]->data()->getIntVal(row, isNull); } @@ -504,7 +548,7 @@ string Func_searched_case::getStrVal(Row& row, if (isNull) return string(""); - return parm[i+1]->data()->getStrVal(row, isNull); + return parm[i]->data()->getStrVal(row, isNull); } @@ -518,7 +562,7 @@ IDB_Decimal Func_searched_case::getDecimalVal(Row& row, if (isNull) return IDB_Decimal(); // need a null value for IDB_Decimal?? - return parm[i+1]->data()->getDecimalVal(row, isNull); + return parm[i]->data()->getDecimalVal(row, isNull); } @@ -532,7 +576,7 @@ double Func_searched_case::getDoubleVal(Row& row, if (isNull) return doubleNullVal(); - return parm[i+1]->data()->getDoubleVal(row, isNull); + return parm[i]->data()->getDoubleVal(row, isNull); } @@ -546,10 +590,10 @@ int32_t Func_searched_case::getDateIntVal(rowgroup::Row& row, if (isNull) return joblist::DATENULL; - return parm[i+1]->data()->getDateIntVal(row, isNull); -} + return parm[i]->data()->getDateIntVal(row, isNull); +} + - int64_t Func_searched_case::getDatetimeIntVal(rowgroup::Row& row, FunctionParm& parm, bool& isNull, @@ -560,8 +604,8 @@ int64_t Func_searched_case::getDatetimeIntVal(rowgroup::Row& row, if (isNull) return joblist::DATETIMENULL; - return parm[i+1]->data()->getDatetimeIntVal(row, isNull); -} + return parm[i]->data()->getDatetimeIntVal(row, isNull); +} } // namespace funcexp From f2d748cff197656c7ae5d490d0298eabbabccc7d Mon Sep 17 00:00:00 2001 From: david hill Date: Mon, 16 Apr 2018 16:22:03 -0500 Subject: [PATCH 23/57] MCOL-1293 --- utils/clusterTester/columnstoreClusterTester.sh | 10 +++++----- 1 file changed, 5 insertions(+), 5 deletions(-) diff --git a/utils/clusterTester/columnstoreClusterTester.sh b/utils/clusterTester/columnstoreClusterTester.sh index 341e37748..9f8f3e9a3 100755 --- a/utils/clusterTester/columnstoreClusterTester.sh +++ b/utils/clusterTester/columnstoreClusterTester.sh @@ -736,7 +736,7 @@ checkPackages() echo "** Run MariaDB ColumnStore Dependent Package Check" echo "" - declare -a CENTOS_PKG=("expect" "perl" "perl-DBI" "openssl" "zlib" "file" "sudo" "libaio" "rsync" "snappy" "net-tools") + declare -a CENTOS_PKG=("expect" "perl" "perl-DBI" "openssl" "zlib" "file" "sudo" "libaio" "rsync" "snappy" "net-tools" "numactl-libs") declare -a CENTOS_PKG_NOT=("mariadb-libs") if [ "$OS" == "centos6" ] || [ "$OS" == "centos7" ]; then @@ -855,7 +855,7 @@ checkPackages() fi fi - declare -a SUSE_PKG=("boost-devel" "expect" "perl" "perl-DBI" "openssl" "file" "sudo" "libaio1" "rsync" "libsnappy1" "net-tools") + declare -a SUSE_PKG=("boost-devel" "expect" "perl" "perl-DBI" "openssl" "file" "sudo" "libaio1" "rsync" "libsnappy1" "net-tools" "libnuma1") declare -a SUSE_PKG_NOT=("mariadb" , "libmariadb18") if [ "$OS" == "suse12" ]; then @@ -946,7 +946,7 @@ checkPackages() fi fi - declare -a UBUNTU_PKG=("libboost-all-dev" "expect" "libdbi-perl" "perl" "openssl" "file" "sudo" "libreadline-dev" "rsync" "libsnappy1V5" "net-tools") + declare -a UBUNTU_PKG=("libboost-all-dev" "expect" "libdbi-perl" "perl" "openssl" "file" "sudo" "libreadline-dev" "rsync" "libsnappy1V5" "net-tools" "libnuma1" ) declare -a UBUNTU_PKG_NOT=("mariadb-server" "libmariadb18") if [ "$OS" == "ubuntu16" ] ; then @@ -1063,7 +1063,7 @@ checkPackages() fi fi - declare -a DEBIAN_PKG=("libboost-all-dev" "expect" "libdbi-perl" "perl" "openssl" "file" "sudo" "libreadline-dev" "rsync" "libsnappy1" "net-tools") + declare -a DEBIAN_PKG=("libboost-all-dev" "expect" "libdbi-perl" "perl" "openssl" "file" "sudo" "libreadline-dev" "rsync" "libsnappy1" "net-tools" "libnuma1") declare -a DEBIAN_PKG_NOT=("libmariadb18" "mariadb-server") if [ "$OS" == "debian8" ]; then @@ -1180,7 +1180,7 @@ checkPackages() fi fi - declare -a DEBIAN9_PKG=("libboost-all-dev" "expect" "libdbi-perl" "perl" "openssl" "file" "sudo" "libreadline5" "rsync" "libsnappy1V5" "net-tools" "libaio1") + declare -a DEBIAN9_PKG=("libboost-all-dev" "expect" "libdbi-perl" "perl" "openssl" "file" "sudo" "libreadline5" "rsync" "libsnappy1V5" "net-tools" "libaio1" "libnuma1") declare -a DEBIAN9_PKG_NOT=("libmariadb18" "mariadb-server") if [ "$OS" == "debian9" ]; then From 33a5969dc38ffe736066c55e89b4f43af02fae7d Mon Sep 17 00:00:00 2001 From: Roman Nozdrin Date: Tue, 17 Apr 2018 15:05:02 +0300 Subject: [PATCH 24/57] MCOL-1052 WIP Comment out useless check and restore isUnion value in the group_end(). --- dbcon/mysql/ha_calpont_impl.cpp | 11 ++++++----- 1 file changed, 6 insertions(+), 5 deletions(-) diff --git a/dbcon/mysql/ha_calpont_impl.cpp b/dbcon/mysql/ha_calpont_impl.cpp index 147c8ce79..c6730277f 100644 --- a/dbcon/mysql/ha_calpont_impl.cpp +++ b/dbcon/mysql/ha_calpont_impl.cpp @@ -5684,15 +5684,16 @@ int ha_calpont_impl_group_by_end(ha_calpont_group_by_handler* group_hand, TABLE* return 0; thd->infinidb_vtable.isNewQuery = true; + thd->infinidb_vtable.isUnion = false; if (thd->infinidb_vtable.cal_conn_info) ci = reinterpret_cast(thd->infinidb_vtable.cal_conn_info); - if (thd->infinidb_vtable.vtable_state == THD::INFINIDB_ORDER_BY ) - { - thd->infinidb_vtable.vtable_state = THD::INFINIDB_SELECT_VTABLE;// flip back to normal state - return rc; - } + //if (thd->infinidb_vtable.vtable_state == THD::INFINIDB_ORDER_BY ) + //{ + // thd->infinidb_vtable.vtable_state = THD::INFINIDB_SELECT_VTABLE;// flip back to normal state + // return rc; + //} if (((thd->lex)->sql_command == SQLCOM_INSERT) || ((thd->lex)->sql_command == SQLCOM_INSERT_SELECT) ) From b8f3cb2a9829f6837100e9b759e1b4dcc621b8e8 Mon Sep 17 00:00:00 2001 From: Roman Nozdrin Date: Fri, 20 Apr 2018 20:44:28 +0300 Subject: [PATCH 25/57] MCOL-1052 WIP Incorporate a fix for MCOL-1348. Comment on the group_by_handler class. --- dbcon/joblist/tupleaggregatestep.cpp | 96 ++++++++++++++++++---------- dbcon/mysql/ha_calpont.h | 25 ++++++++ dbcon/mysql/ha_calpont_impl.cpp | 6 +- dbcon/mysql/ha_window_function.cpp | 12 ++++ 4 files changed, 105 insertions(+), 34 deletions(-) diff --git a/dbcon/joblist/tupleaggregatestep.cpp b/dbcon/joblist/tupleaggregatestep.cpp index a77cbcdd1..6c8760d71 100644 --- a/dbcon/joblist/tupleaggregatestep.cpp +++ b/dbcon/joblist/tupleaggregatestep.cpp @@ -1091,6 +1091,7 @@ void TupleAggregateStep::prep1PhaseAggregate( vector functionVec; uint32_t bigIntWidth = sizeof(int64_t); uint32_t bigUintWidth = sizeof(uint64_t); + uint32_t projColsUDAFIndex = 0; mcsv1sdk::mcsv1_UDAF* pUDAFFunc = NULL; // for count column of average function @@ -1279,18 +1280,26 @@ void TupleAggregateStep::prep1PhaseAggregate( if (aggOp == ROWAGG_UDAF) { - UDAFColumn* udafc = dynamic_cast(jobInfo.projectionCols[i].get()); - - if (udafc) + std::vector::iterator it = jobInfo.projectionCols.begin() + projColsUDAFIndex; + for (; it != jobInfo.projectionCols.end(); it++) { - // Create a RowAggFunctionCol (UDAF subtype) with the context. - funct.reset(new RowUDAFFunctionCol(udafc->getContext(), colProj, i)); + UDAFColumn* udafc = dynamic_cast((*it).get()); + projColsUDAFIndex++; + if (udafc) + { + pUDAFFunc = udafc->getContext().getFunction(); + // Create a RowAggFunctionCol (UDAF subtype) with the context. + funct.reset(new RowUDAFFunctionCol(udafc->getContext(), colProj, i)); + break; + } + } - else + + if (it == jobInfo.projectionCols.end()) { - throw logic_error("prep1PhasesAggregate: A UDAF function is called but there's no UDAFColumn"); + throw logic_error("prep1PhaseAggregate: A UDAF function is called but there's no/not enough UDAFColumn/-s"); } - } + } else { funct.reset(new RowAggFunctionCol(aggOp, stats, colProj, i)); @@ -1649,6 +1658,7 @@ void TupleAggregateStep::prep1PhaseDistinctAggregate( AGG_MAP aggFuncMap; mcsv1sdk::mcsv1_UDAF* pUDAFFunc = NULL; set avgSet; + uint32_t projColsUDAFIndex = 0; // for count column of average function map avgFuncMap, avgDistFuncMap; @@ -1812,17 +1822,24 @@ void TupleAggregateStep::prep1PhaseDistinctAggregate( if (aggOp == ROWAGG_UDAF) { - UDAFColumn* udafc = dynamic_cast(jobInfo.projectionCols[i].get()); - - if (udafc) + std::vector::iterator it = jobInfo.projectionCols.begin() + projColsUDAFIndex; + for (; it != jobInfo.projectionCols.end(); it++) { - pUDAFFunc = udafc->getContext().getFunction(); - // Create a RowAggFunctionCol (UDAF subtype) with the context. - funct.reset(new RowUDAFFunctionCol(udafc->getContext(), colProj, colAgg)); + UDAFColumn* udafc = dynamic_cast((*it).get()); + projColsUDAFIndex++; + if (udafc) + { + pUDAFFunc = udafc->getContext().getFunction(); + // Create a RowAggFunctionCol (UDAF subtype) with the context. + funct.reset(new RowUDAFFunctionCol(udafc->getContext(), colProj, colAgg)); + break; + } + } - else + + if (it == jobInfo.projectionCols.end()) { - throw logic_error("prep1PhaseDistinctAggregate: A UDAF function is called but there's no UDAFColumn"); + throw logic_error("prep1PhaseDistinctAggregate: A UDAF function is called but there's no/not enough UDAFColumn/-s"); } } else @@ -2814,6 +2831,7 @@ void TupleAggregateStep::prep2PhasesAggregate( vector > aggColVec; set avgSet; vector >& returnedColVec = jobInfo.returnedColVec; + uint32_t projColsUDAFIndex = 0; for (uint64_t i = 0; i < returnedColVec.size(); i++) { @@ -2992,17 +3010,24 @@ void TupleAggregateStep::prep2PhasesAggregate( if (aggOp == ROWAGG_UDAF) { - UDAFColumn* udafc = dynamic_cast(jobInfo.projectionCols[i].get()); - - if (udafc) + std::vector::iterator it = jobInfo.projectionCols.begin() + projColsUDAFIndex; + for (; it != jobInfo.projectionCols.end(); it++) { - pUDAFFunc = udafc->getContext().getFunction(); - // Create a RowAggFunctionCol (UDAF subtype) with the context. - funct.reset(new RowUDAFFunctionCol(udafc->getContext(), colProj, colAggPm)); + UDAFColumn* udafc = dynamic_cast((*it).get()); + projColsUDAFIndex++; + if (udafc) + { + pUDAFFunc = udafc->getContext().getFunction(); + // Create a RowAggFunctionCol (UDAF subtype) with the context. + funct.reset(new RowUDAFFunctionCol(udafc->getContext(), colProj, colAggPm)); + break; + } + } - else + + if (it == jobInfo.projectionCols.end()) { - throw logic_error("prep2PhasesAggregate: A UDAF function is called but there's no UDAFColumn"); + throw logic_error("prep2PhasesAggregate: A UDAF function is called but there's no/not enough UDAFColumn/-s"); } } else @@ -3583,6 +3608,7 @@ void TupleAggregateStep::prep2PhasesDistinctAggregate( vector > aggColVec, aggNoDistColVec; set avgSet, avgDistSet; vector >& returnedColVec = jobInfo.returnedColVec; + uint32_t projColsUDAFIndex = 0; for (uint64_t i = 0; i < returnedColVec.size(); i++) { @@ -3796,17 +3822,23 @@ void TupleAggregateStep::prep2PhasesDistinctAggregate( if (aggOp == ROWAGG_UDAF) { - UDAFColumn* udafc = dynamic_cast(jobInfo.projectionCols[i].get()); - - if (udafc) + std::vector::iterator it = jobInfo.projectionCols.begin() + projColsUDAFIndex; + for (; it != jobInfo.projectionCols.end(); it++) { - pUDAFFunc = udafc->getContext().getFunction(); - // Create a RowAggFunctionCol (UDAF subtype) with the context. - funct.reset(new RowUDAFFunctionCol(udafc->getContext(), colProj, colAggPm)); + UDAFColumn* udafc = dynamic_cast((*it).get()); + projColsUDAFIndex++; + if (udafc) + { + pUDAFFunc = udafc->getContext().getFunction(); + // Create a RowAggFunctionCol (UDAF subtype) with the context. + funct.reset(new RowUDAFFunctionCol(udafc->getContext(), colProj, colAggPm)); + break; + } } - else + + if (it == jobInfo.projectionCols.end()) { - throw logic_error("prep2PhasesDistinctAggregate: A UDAF function is called but there's no UDAFColumn"); + throw logic_error("prep2PhasesDistinctAggregate: A UDAF function is called but there's no/not enough UDAFColumn/-s"); } } else diff --git a/dbcon/mysql/ha_calpont.h b/dbcon/mysql/ha_calpont.h index 0e475e615..d1e545fec 100644 --- a/dbcon/mysql/ha_calpont.h +++ b/dbcon/mysql/ha_calpont.h @@ -248,6 +248,30 @@ public: }; +/*@brief group_by_handler class*/ +/*********************************************************** + * DESCRIPTION: + * Provides server with group_by_handler API methods. + * One should read comments in server/sql/group_by_handler.h + * Attributes: + * select - attribute contains all GROUP BY, HAVING, ORDER items and calls it + * an extended SELECT list accordin to comments in + * server/sql/group_handler.cc. + * So the temporary table for + * select count(*) from b group by a having a > 3 order by a + * will have 4 columns not 1. + * However server ignores all NULLs used in GROUP BY, HAVING, ORDER. + * table_list - contains all tables involved. Must be CS tables only. + * distinct - looks like a useless thing for now. Couldn't get it set by server. + * where - where items. + * group_by - group by ORDER items. + * order_by - order by ORDER items. + * having - having Item. + * Methods: + * init_scan - get plan and send it to ExeMgr. Get the execution result. + * next_row - get a row back from sm. + * end_scan - finish and clean the things up. + ***********************************************************/ class ha_calpont_group_by_handler: public group_by_handler { public: @@ -265,6 +289,7 @@ class ha_calpont_group_by_handler: public group_by_handler int init_scan(); int next_row(); int end_scan(); + List *select; TABLE_LIST *table_list; bool distinct; diff --git a/dbcon/mysql/ha_calpont_impl.cpp b/dbcon/mysql/ha_calpont_impl.cpp index c6730277f..27e35e3d1 100644 --- a/dbcon/mysql/ha_calpont_impl.cpp +++ b/dbcon/mysql/ha_calpont_impl.cpp @@ -5103,9 +5103,10 @@ int ha_calpont_impl_group_by_init(ha_calpont_group_by_handler* group_hand, TABLE if (thd->infinidb_vtable.vtable_state == THD::INFINIDB_ERROR) return ER_INTERNAL_ERROR; + // MCOL-1052 // by pass the extra union trips. return 0 - if (thd->infinidb_vtable.isUnion && thd->infinidb_vtable.vtable_state == THD::INFINIDB_CREATE_VTABLE) - return 0; + //if (thd->infinidb_vtable.isUnion && thd->infinidb_vtable.vtable_state == THD::INFINIDB_CREATE_VTABLE) + // return 0; // @bug 2232. Basic SP support. Error out non support sp cases. // @bug 3939. Only error out for sp with select. Let pass for alter table in sp. @@ -5689,6 +5690,7 @@ int ha_calpont_impl_group_by_end(ha_calpont_group_by_handler* group_hand, TABLE* if (thd->infinidb_vtable.cal_conn_info) ci = reinterpret_cast(thd->infinidb_vtable.cal_conn_info); + // MCOL-1052 //if (thd->infinidb_vtable.vtable_state == THD::INFINIDB_ORDER_BY ) //{ // thd->infinidb_vtable.vtable_state = THD::INFINIDB_SELECT_VTABLE;// flip back to normal state diff --git a/dbcon/mysql/ha_window_function.cpp b/dbcon/mysql/ha_window_function.cpp index 7e30389c4..a861cbde1 100644 --- a/dbcon/mysql/ha_window_function.cpp +++ b/dbcon/mysql/ha_window_function.cpp @@ -512,6 +512,18 @@ ReturnedColumn* buildWindowFunctionColumn(Item* item, gp_walk_info& gwi, bool& n { Item* orderItem = *(orderCol->item); srcp.reset(buildReturnedColumn(orderItem, gwi, nonSupport)); + + // MCOL-1052 GROUP BY handler has all of query's agg Items + // as field and correlates them with its extended SELECT Items. + if (!srcp) + { + orderItem = orderCol->item_ptr; + if (orderItem) + { + gwi.fatalParseError = false; + srcp.reset(buildReturnedColumn(orderItem, gwi, nonSupport)); + } + } if (!srcp) return nullOnError(gwi); From f8c08f8b0bf46618275c3bc2511b059443515a32 Mon Sep 17 00:00:00 2001 From: Roman Nozdrin Date: Sun, 22 Apr 2018 23:45:36 +0300 Subject: [PATCH 26/57] MCOL-1052 WIP Fixed ORDER BY NULL query processing by group_by_handler. --- dbcon/mysql/ha_calpont_execplan.cpp | 11 ++++++++--- 1 file changed, 8 insertions(+), 3 deletions(-) diff --git a/dbcon/mysql/ha_calpont_execplan.cpp b/dbcon/mysql/ha_calpont_execplan.cpp index b17e0ab3e..18f5aaaee 100644 --- a/dbcon/mysql/ha_calpont_execplan.cpp +++ b/dbcon/mysql/ha_calpont_execplan.cpp @@ -919,7 +919,12 @@ void debug_walk(const Item* item, void* arg) cerr << "Window Function Item " << ifp->window_func()->func_name() << endl; break; } - + + case Item::NULL_ITEM: + { + cerr << "NULL item" << endl; + break; + } default: { cerr << "UNKNOWN_ITEM type " << item->type() << endl; @@ -8722,7 +8727,7 @@ int getGroupPlan(gp_walk_info& gwi, SELECT_LEX& select_lex, SCSEP& csep, cal_gro case Item::NULL_ITEM: { - if ( ((gwi.thd->lex)->sql_command == SQLCOM_UPDATE ) || ((gwi.thd->lex)->sql_command == SQLCOM_DELETE ) || ((gwi.thd->lex)->sql_command == SQLCOM_UPDATE_MULTI ) || ((gwi.thd->lex)->sql_command == SQLCOM_DELETE_MULTI ) ) + /*if ( ((gwi.thd->lex)->sql_command == SQLCOM_UPDATE ) || ((gwi.thd->lex)->sql_command == SQLCOM_DELETE ) || ((gwi.thd->lex)->sql_command == SQLCOM_UPDATE_MULTI ) || ((gwi.thd->lex)->sql_command == SQLCOM_DELETE_MULTI ) ) { } else { @@ -8739,7 +8744,7 @@ int getGroupPlan(gp_walk_info& gwi, SELECT_LEX& select_lex, SCSEP& csep, cal_gro sel_cols_in_create += name; gwi.selectCols.push_back("null"); - } + }*/ break; } From 51715c76eee3992d59970e1b755f64c1cc7ff82d Mon Sep 17 00:00:00 2001 From: Roman Nozdrin Date: Tue, 24 Apr 2018 15:14:08 +0300 Subject: [PATCH 27/57] MCOL-1348 Add multiply UDAF calls support. --- dbcon/joblist/tupleaggregatestep.cpp | 134 +++++++++++++++++---------- 1 file changed, 85 insertions(+), 49 deletions(-) diff --git a/dbcon/joblist/tupleaggregatestep.cpp b/dbcon/joblist/tupleaggregatestep.cpp index b5cde967b..e68bc9f18 100755 --- a/dbcon/joblist/tupleaggregatestep.cpp +++ b/dbcon/joblist/tupleaggregatestep.cpp @@ -958,6 +958,7 @@ void TupleAggregateStep::prep1PhaseAggregate( vector functionVec; uint32_t bigIntWidth = sizeof(int64_t); uint32_t bigUintWidth = sizeof(uint64_t); + uint32_t projColsUDAFIndex = 0; mcsv1sdk::mcsv1_UDAF* pUDAFFunc = NULL; // for count column of average function @@ -1135,18 +1136,27 @@ void TupleAggregateStep::prep1PhaseAggregate( SP_ROWAGG_FUNC_t funct; if (aggOp == ROWAGG_UDAF) - { - UDAFColumn* udafc = dynamic_cast(jobInfo.projectionCols[i].get()); - if (udafc) - { - // Create a RowAggFunctionCol (UDAF subtype) with the context. - funct.reset(new RowUDAFFunctionCol(udafc->getContext(), colProj, i)); - } - else - { - throw logic_error("prep1PhasesAggregate: A UDAF function is called but there's no UDAFColumn"); - } - } + { + std::vector::iterator it = jobInfo.projectionCols.begin() + projColsUDAFIndex; + for (; it != jobInfo.projectionCols.end(); it++) + { + UDAFColumn* udafc = dynamic_cast((*it).get()); + projColsUDAFIndex++; + if (udafc) + { + pUDAFFunc = udafc->getContext().getFunction(); + // Create a RowAggFunctionCol (UDAF subtype) with the context. + funct.reset(new RowUDAFFunctionCol(udafc->getContext(), colProj, i)); + break; + } + + } + + if (it == jobInfo.projectionCols.end()) + { + throw logic_error("prep1PhaseAggregate: A UDAF function is called but there's no/not enough UDAFColumn/-s"); + } + } else { funct.reset(new RowAggFunctionCol(aggOp, stats, colProj, i)); @@ -1484,6 +1494,7 @@ void TupleAggregateStep::prep1PhaseDistinctAggregate( AGG_MAP aggFuncMap; mcsv1sdk::mcsv1_UDAF* pUDAFFunc = NULL; set avgSet; + uint32_t projColsUDAFIndex = 0; // for count column of average function map avgFuncMap, avgDistFuncMap; @@ -1636,19 +1647,27 @@ void TupleAggregateStep::prep1PhaseDistinctAggregate( SP_ROWAGG_FUNC_t funct; if (aggOp == ROWAGG_UDAF) - { - UDAFColumn* udafc = dynamic_cast(jobInfo.projectionCols[i].get()); - if (udafc) - { - pUDAFFunc = udafc->getContext().getFunction(); - // Create a RowAggFunctionCol (UDAF subtype) with the context. - funct.reset(new RowUDAFFunctionCol(udafc->getContext(), colProj, colAgg)); - } - else - { - throw logic_error("prep1PhaseDistinctAggregate: A UDAF function is called but there's no UDAFColumn"); - } - } + { + std::vector::iterator it = jobInfo.projectionCols.begin() + projColsUDAFIndex; + for (; it != jobInfo.projectionCols.end(); it++) + { + UDAFColumn* udafc = dynamic_cast((*it).get()); + projColsUDAFIndex++; + if (udafc) + { + pUDAFFunc = udafc->getContext().getFunction(); + // Create a RowAggFunctionCol (UDAF subtype) with the context. + funct.reset(new RowUDAFFunctionCol(udafc->getContext(), colProj, colAgg)); + break; + } + + } + + if (it == jobInfo.projectionCols.end()) + { + throw logic_error("prep1PhaseDistinctAggregate: A UDAF function is called but there's no/not enough UDAFColumn/-s"); + } + } else { funct.reset(new RowAggFunctionCol(aggOp, stats, colProj, colAgg)); @@ -2579,6 +2598,7 @@ void TupleAggregateStep::prep2PhasesAggregate( vector > aggColVec; set avgSet; vector >& returnedColVec = jobInfo.returnedColVec; + uint32_t projColsUDAFIndex = 0; for (uint64_t i = 0; i < returnedColVec.size(); i++) { // skip if not an aggregation column @@ -2746,18 +2766,26 @@ void TupleAggregateStep::prep2PhasesAggregate( SP_ROWAGG_FUNC_t funct; if (aggOp == ROWAGG_UDAF) { - UDAFColumn* udafc = dynamic_cast(jobInfo.projectionCols[i].get()); - if (udafc) - { - pUDAFFunc = udafc->getContext().getFunction(); - // Create a RowAggFunctionCol (UDAF subtype) with the context. - funct.reset(new RowUDAFFunctionCol(udafc->getContext(), colProj, colAggPm)); - } - else - { - throw logic_error("prep2PhasesAggregate: A UDAF function is called but there's no UDAFColumn"); - } - } + std::vector::iterator it = jobInfo.projectionCols.begin() + projColsUDAFIndex; + for (; it != jobInfo.projectionCols.end(); it++) + { + UDAFColumn* udafc = dynamic_cast((*it).get()); + projColsUDAFIndex++; + if (udafc) + { + pUDAFFunc = udafc->getContext().getFunction(); + // Create a RowAggFunctionCol (UDAF subtype) with the context. + funct.reset(new RowUDAFFunctionCol(udafc->getContext(), colProj, colAggPm)); + break; + } + + } + + if (it == jobInfo.projectionCols.end()) + { + throw logic_error("prep2PhasesAggregate: A UDAF function is called but there's no/not enough UDAFColumn/-s"); + } + } else { funct.reset(new RowAggFunctionCol(aggOp, stats, colProj, colAggPm)); @@ -3301,6 +3329,7 @@ void TupleAggregateStep::prep2PhasesDistinctAggregate( vector > aggColVec, aggNoDistColVec; set avgSet, avgDistSet; vector >& returnedColVec = jobInfo.returnedColVec; + uint32_t projColsUDAFIndex = 0; for (uint64_t i = 0; i < returnedColVec.size(); i++) { // col should be an aggregate or groupBy or window function @@ -3498,18 +3527,25 @@ void TupleAggregateStep::prep2PhasesDistinctAggregate( SP_ROWAGG_FUNC_t funct; if (aggOp == ROWAGG_UDAF) { - UDAFColumn* udafc = dynamic_cast(jobInfo.projectionCols[i].get()); - if (udafc) - { - pUDAFFunc = udafc->getContext().getFunction(); - // Create a RowAggFunctionCol (UDAF subtype) with the context. - funct.reset(new RowUDAFFunctionCol(udafc->getContext(), colProj, colAggPm)); - } - else - { - throw logic_error("prep2PhasesDistinctAggregate: A UDAF function is called but there's no UDAFColumn"); - } - } + std::vector::iterator it = jobInfo.projectionCols.begin() + projColsUDAFIndex; + for (; it != jobInfo.projectionCols.end(); it++) + { + UDAFColumn* udafc = dynamic_cast((*it).get()); + projColsUDAFIndex++; + if (udafc) + { + pUDAFFunc = udafc->getContext().getFunction(); + // Create a RowAggFunctionCol (UDAF subtype) with the context. + funct.reset(new RowUDAFFunctionCol(udafc->getContext(), colProj, colAggPm)); + break; + } + } + + if (it == jobInfo.projectionCols.end()) + { + throw logic_error("prep2PhasesDistinctAggregate: A UDAF function is called but there's no/not enough UDAFColumn/-s"); + } + } else { funct.reset(new RowAggFunctionCol(aggOp, stats, colProj, colAggPm)); From 863e18744bb2da10c881be21667e5bfef23b1dee Mon Sep 17 00:00:00 2001 From: david hill Date: Wed, 25 Apr 2018 14:07:18 -0500 Subject: [PATCH 28/57] update to 1.1.5 --- VERSION | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/VERSION b/VERSION index 445a70a7a..aef96eefc 100644 --- a/VERSION +++ b/VERSION @@ -1,4 +1,4 @@ COLUMNSTORE_VERSION_MAJOR=1 COLUMNSTORE_VERSION_MINOR=1 -COLUMNSTORE_VERSION_PATCH=4 +COLUMNSTORE_VERSION_PATCH=5 COLUMNSTORE_VERSION_RELEASE=1 From 1bcc8a3052d9dd53a14b457121216d86e3db41c1 Mon Sep 17 00:00:00 2001 From: Ravi Prakash Date: Sat, 28 Apr 2018 14:48:13 -0700 Subject: [PATCH 29/57] MCOL-1229 - IS.columnstore_columns crashes when DDL is simultaneously executing. The crash was due to an attempt to iterate over the columns of a recently dropped table. Such a table will now be ignored. --- dbcon/mysql/is_columnstore_columns.cpp | 19 ++++++++++++++++++- 1 file changed, 18 insertions(+), 1 deletion(-) diff --git a/dbcon/mysql/is_columnstore_columns.cpp b/dbcon/mysql/is_columnstore_columns.cpp index 21c9e748e..13f9b9485 100644 --- a/dbcon/mysql/is_columnstore_columns.cpp +++ b/dbcon/mysql/is_columnstore_columns.cpp @@ -27,6 +27,8 @@ #include #include "calpontsystemcatalog.h" #include "dataconvert.h" +#include "exceptclasses.h" +using namespace logging; // Required declaration as it isn't in a MairaDB include @@ -70,7 +72,22 @@ static int is_columnstore_columns_fill(THD *thd, TABLE_LIST *tables, COND *cond) for (std::vector >::const_iterator it = catalog_tables.begin(); it != catalog_tables.end(); ++it) { - execplan::CalpontSystemCatalog::RIDList column_rid_list = systemCatalogPtr->columnRIDs((*it).second, true); + execplan::CalpontSystemCatalog::RIDList column_rid_list; + // Note a table may get dropped as you iterate over the list of tables. + // So simply ignore the dropped table. + try { + column_rid_list = systemCatalogPtr->columnRIDs((*it).second, true); + } + catch (IDBExcept& ex) + { + if (ex.errorCode() == ERR_TABLE_NOT_IN_CATALOG) { + continue; + } + else { + throw; + } + } + for (size_t col_num = 0; col_num < column_rid_list.size(); col_num++) { execplan::CalpontSystemCatalog::TableColName tcn = systemCatalogPtr->colName(column_rid_list[col_num].objnum); From fdbf82e48d0b4b9e446fc19ba095c0573e03be7c Mon Sep 17 00:00:00 2001 From: Roman Nozdrin Date: Sun, 29 Apr 2018 21:13:11 +0300 Subject: [PATCH 30/57] MCOL-1052 WIP Replace SimpleColumn with ConstantColumn when server pushes equal predicate using cond_push. --- dbcon/mysql/ha_calpont_execplan.cpp | 81 +++++++++++++++++++++++++++-- dbcon/mysql/ha_calpont_impl.cpp | 23 ++++++++ dbcon/mysql/ha_calpont_impl_if.h | 1 + 3 files changed, 100 insertions(+), 5 deletions(-) diff --git a/dbcon/mysql/ha_calpont_execplan.cpp b/dbcon/mysql/ha_calpont_execplan.cpp index 18f5aaaee..f5ba036f1 100644 --- a/dbcon/mysql/ha_calpont_execplan.cpp +++ b/dbcon/mysql/ha_calpont_execplan.cpp @@ -8003,6 +8003,55 @@ int cp_get_group_plan(THD* thd, SCSEP& csep, cal_impl_if::cal_group_info& gi) return 0; } +/*@brief buildConstColFromFilter- change SimpleColumn into ConstColumn*/ +/*********************************************************** + * DESCRIPTION: + * Server could optimize out fields from GROUP BY list, when certain + * filter predicate is used, e.g. + * field = 'AIR', field IN ('AIR'). This utility function tries to + * replace such fields with ConstantColumns using cond_pushed filters. + * PARAMETERS: + * originalSC SimpleColumn* removed field + * gwi main strucutre + * gi auxilary group_by handler structure + * RETURNS + * ConstantColumn* if originalSC equals with cond_pushed columns. + * NULL otherwise + ***********************************************************/ +ConstantColumn* buildConstColFromFilter(SimpleColumn* originalSC, +gp_walk_info& gwi, cal_group_info& gi) +{ + execplan::SimpleColumn* simpleCol; + execplan::ConstantColumn* constCol; + execplan::SOP op; + execplan::SimpleFilter* simpFilter; + execplan::ConstantColumn* result = NULL; + std::vector::iterator ptIt = gi.pushedPts.begin(); + for(; ptIt != gi.pushedPts.end(); ptIt++) + { + simpFilter = dynamic_cast((*ptIt)->data()); + if (simpFilter == NULL) + continue; + simpleCol = dynamic_cast(simpFilter->lhs()); + constCol = dynamic_cast(simpFilter->rhs()); + if(simpleCol == NULL || constCol == NULL) + continue; + op = simpFilter->op(); + if ( originalSC->sameColumn(dynamic_cast(simpleCol)) + && op.get()->op() == OP_EQ && constCol) + { +#ifdef DEBUG_WALK_COND + cerr << "buildConstColFromFilter() replaced " << endl; + cerr << simpleCol->toString() << endl; + cerr << " with " << endl; + cerr << constCol << endl; +#endif + result = constCol; + } + } + return result; +} + int getGroupPlan(gp_walk_info& gwi, SELECT_LEX& select_lex, SCSEP& csep, cal_group_info& gi, bool isUnion) { #ifdef DEBUG_WALK_COND @@ -8284,7 +8333,9 @@ int getGroupPlan(gp_walk_info& gwi, SELECT_LEX& select_lex, SCSEP& csep, cal_gro if (filters) { csep->filters(filters); +#ifdef DEBUG_WALK_COND filters->drawTree("/tmp/filter1.dot"); +#endif } gwi.clauseType = SELECT; @@ -8346,6 +8397,7 @@ int getGroupPlan(gp_walk_info& gwi, SELECT_LEX& select_lex, SCSEP& csep, cal_gro { Item_field* ifp = (Item_field*)item; SimpleColumn* sc = NULL; + ConstantColumn* constCol = NULL; if (ifp->field_name && string(ifp->field_name) == "*") { @@ -8357,6 +8409,8 @@ int getGroupPlan(gp_walk_info& gwi, SELECT_LEX& select_lex, SCSEP& csep, cal_gro if (sc) { + constCol = buildConstColFromFilter(sc, gwi, gi); + boost::shared_ptr spcc(constCol); boost::shared_ptr spsc(sc); if (sel_cols_in_create.length() != 0) @@ -8386,10 +8440,19 @@ int getGroupPlan(gp_walk_info& gwi, SELECT_LEX& select_lex, SCSEP& csep, cal_gro escapeBackTick(itemAlias.empty() ? ifp->name : itemAlias.c_str()) + "`"); else gwi.selectCols.push_back("`" + escapeBackTick((itemAlias.empty() ? ifp->name : itemAlias.c_str())) + "`"); - - gwi.returnedCols.push_back(spsc); - - gwi.columnMap.insert(CalpontSelectExecutionPlan::ColumnMap::value_type(string(ifp->field_name), spsc)); + + // MCOL-1052 Replace SimpleColumn with ConstantColumn, + // since it must have a single value only. + if(constCol) + { + gwi.returnedCols.push_back(spcc); + gwi.columnMap.insert(CalpontSelectExecutionPlan::ColumnMap::value_type(string(ifp->field_name), spcc)); + } + else + { + gwi.returnedCols.push_back(spsc); + gwi.columnMap.insert(CalpontSelectExecutionPlan::ColumnMap::value_type(string(ifp->field_name), spsc)); + } TABLE_LIST* tmp = 0; if (ifp->cached_table) @@ -9348,11 +9411,19 @@ int getGroupPlan(gp_walk_info& gwi, SELECT_LEX& select_lex, SCSEP& csep, cal_gro if( rc->sameColumn((*iter).get()) ) break; } + + // MCOL-1052 Find and remove the optimized field + // from ORDER using cond_pushed filters. + if(buildConstColFromFilter( + dynamic_cast(rc),gwi, gi)) + { + break; + } // MCOL-1052 GROUP BY items list doesn't contain // this ORDER BY item. if ( iter == gwi.groupByCols.end() ) { - Item_ident *iip = reinterpret_cast(ord_item); + Item_ident *iip = reinterpret_cast(ord_item); std::ostringstream ostream; ostream << "'"; if (iip->db_name) diff --git a/dbcon/mysql/ha_calpont_impl.cpp b/dbcon/mysql/ha_calpont_impl.cpp index 27e35e3d1..4fc21b148 100644 --- a/dbcon/mysql/ha_calpont_impl.cpp +++ b/dbcon/mysql/ha_calpont_impl.cpp @@ -5249,6 +5249,29 @@ int ha_calpont_impl_group_by_init(ha_calpont_group_by_handler* group_hand, TABLE gi.groupByHaving = group_hand->having; gi.groupByDistinct = group_hand->distinct; + // MCOL-1052 Send pushed conditions here, since server could omit GROUP BY + // items in case of = or IN functions used on GROUP BY columns. + { + CalTableMap::iterator mapiter; + execplan::CalpontSelectExecutionPlan::ColumnMap::iterator colMapIter; + execplan::CalpontSelectExecutionPlan::ColumnMap::iterator condColMapIter; + execplan::ParseTree* ptIt; + execplan::ReturnedColumn* rcIt; + for(TABLE_LIST* tl = gi.groupByTables; tl; tl=tl->next_local) + { + mapiter = ci->tableMap.find(tl->table); + if(mapiter != ci->tableMap.end() && mapiter->second.condInfo != NULL + && mapiter->second.condInfo->condPush) + { + while(!mapiter->second.condInfo->ptWorkStack.empty()) + { + ptIt=mapiter->second.condInfo->ptWorkStack.top(); + mapiter->second.condInfo->ptWorkStack.pop(); + gi.pushedPts.push_back(ptIt); + } + } + } + } // send plan whenever group_init is called int status = cp_get_group_plan(thd, csep, gi); diff --git a/dbcon/mysql/ha_calpont_impl_if.h b/dbcon/mysql/ha_calpont_impl_if.h index 83f6aaf41..b1e5ec443 100644 --- a/dbcon/mysql/ha_calpont_impl_if.h +++ b/dbcon/mysql/ha_calpont_impl_if.h @@ -211,6 +211,7 @@ struct cal_group_info ORDER* groupByOrder; // MCOL-1052 ORDER BY Item* groupByHaving; // MCOL-1052 HAVING bool groupByDistinct; //MCOL-1052 DISTINCT + std::vector pushedPts; }; typedef std::tr1::unordered_map CalTableMap; From b584a7f555a2f40fd5e41fc2c547896e77643910 Mon Sep 17 00:00:00 2001 From: Andrew Hutchings Date: Thu, 19 Apr 2018 18:51:18 +0100 Subject: [PATCH 31/57] MCOL-392 Add DATETIME microscond support Add initial microsecond support for DATETIME --- dbcon/ddlpackage/ddl.y | 3 ++- dbcon/mysql/ha_calpont_dml.cpp | 18 ++++++++++--- dbcon/mysql/ha_calpont_impl.cpp | 4 +-- utils/dataconvert/dataconvert.cpp | 18 +++++++++++-- utils/dataconvert/dataconvert.h | 33 +++++++++++++++++------- writeengine/server/we_dmlcommandproc.cpp | 4 +-- 6 files changed, 60 insertions(+), 20 deletions(-) diff --git a/dbcon/ddlpackage/ddl.y b/dbcon/ddlpackage/ddl.y index 8f6e713fc..c084f44fe 100644 --- a/dbcon/ddlpackage/ddl.y +++ b/dbcon/ddlpackage/ddl.y @@ -1114,10 +1114,11 @@ literal: ; datetime_type: - DATETIME + DATETIME opt_time_precision { $$ = new ColumnType(DDL_DATETIME); $$->fLength = DDLDatatypeLength[DDL_DATETIME]; + $$->fPrecision = $2; } | DATE diff --git a/dbcon/mysql/ha_calpont_dml.cpp b/dbcon/mysql/ha_calpont_dml.cpp index 1cef4d4da..c4ca59862 100644 --- a/dbcon/mysql/ha_calpont_dml.cpp +++ b/dbcon/mysql/ha_calpont_dml.cpp @@ -838,11 +838,21 @@ int ha_calpont_impl_write_batch_row_(uchar* buf, TABLE* table, cal_impl_if::cal_ // mariadb 10.1 compatibility -- MYSQL_TYPE_DATETIME2 introduced in mysql 5.6 MYSQL_TIME ltime; const uchar* pos = buf; - longlong tmp = my_datetime_packed_from_binary(pos, 0); + longlong tmp = my_datetime_packed_from_binary(pos, table->field[colpos]->decimals()); TIME_from_longlong_datetime_packed(<ime, tmp); - fprintf(ci.filePtr, "%04d-%02d-%02d %02d:%02d:%02d%c", - ltime.year, ltime.month, ltime.day, - ltime.hour, ltime.minute, ltime.second, ci.delimiter); + if (!ltime.second_part) + { + fprintf(ci.filePtr, "%04d-%02d-%02d %02d:%02d:%02d%c", + ltime.year, ltime.month, ltime.day, + ltime.hour, ltime.minute, ltime.second, ci.delimiter); + } + else + { + fprintf(ci.filePtr, "%04d-%02d-%02d %02d:%02d:%02d.%ld%c", + ltime.year, ltime.month, ltime.day, + ltime.hour, ltime.minute, ltime.second, + ltime.second_part, ci.delimiter); + } buf += table->field[colpos]->pack_length(); } else diff --git a/dbcon/mysql/ha_calpont_impl.cpp b/dbcon/mysql/ha_calpont_impl.cpp index 8078d452e..6bd887332 100644 --- a/dbcon/mysql/ha_calpont_impl.cpp +++ b/dbcon/mysql/ha_calpont_impl.cpp @@ -486,8 +486,8 @@ int fetchNextRow(uchar* buf, cal_table_info& ti, cal_connection_info* ci) * based on the result set. */ /* MCOL-683: UTF-8 datetime no msecs is 57, this sometimes happens! */ - if (((*f)->field_length > 19) && ((*f)->field_length != 57)) - (*f)->field_length = strlen(tmp); +// if (((*f)->field_length > 19) && ((*f)->field_length != 57)) +// (*f)->field_length = strlen(tmp); Field_varstring* f2 = (Field_varstring*)*f; f2->store(tmp, strlen(tmp), f2->charset()); diff --git a/utils/dataconvert/dataconvert.cpp b/utils/dataconvert/dataconvert.cpp index d0689fb0d..6e8a33dc1 100644 --- a/utils/dataconvert/dataconvert.cpp +++ b/utils/dataconvert/dataconvert.cpp @@ -1717,14 +1717,28 @@ std::string DataConvert::dateToString( int datevalue ) return buf; } -std::string DataConvert::datetimeToString( long long datetimevalue ) +std::string DataConvert::datetimeToString( long long datetimevalue, long decimals ) { + // 10 is default which means we don't need microseconds + if (decimals > 6 || decimals < 0) + { + decimals = 0; + } // @bug 4703 abandon multiple ostringstream's for conversion DateTime dt(datetimevalue); - const int DATETIMETOSTRING_LEN = 21; // YYYY-MM-DD HH:MM:SS\0 + const int DATETIMETOSTRING_LEN = 28; // YYYY-MM-DD HH:MM:SS.mmmmmm\0 char buf[DATETIMETOSTRING_LEN]; sprintf(buf, "%04d-%02d-%02d %02d:%02d:%02d", dt.year, dt.month, dt.day, dt.hour, dt.minute, dt.second); + if (dt.msecond && decimals) + { + snprintf(buf + strlen(buf), 21 + decimals, ".%d", dt.msecond); + // Pad end with zeros + if (strlen(buf) < (21 + decimals)) + { + sprintf(buf + strlen(buf), "%0*d", 21 + decimals - strlen(buf), 0); + } + } return buf; } diff --git a/utils/dataconvert/dataconvert.h b/utils/dataconvert/dataconvert.h index 1debe1bcc..d2d9b834f 100644 --- a/utils/dataconvert/dataconvert.h +++ b/utils/dataconvert/dataconvert.h @@ -361,7 +361,7 @@ public: * @param type the columns database type * @param data the columns string representation of it's data */ - EXPORT static std::string datetimeToString( long long datetimevalue ); + EXPORT static std::string datetimeToString( long long datetimevalue, long decimals = 0 ); static inline void datetimeToString( long long datetimevalue, char* buf, unsigned int buflen ); /** @@ -459,14 +459,29 @@ inline void DataConvert::dateToString( int datevalue, char* buf, unsigned int bu inline void DataConvert::datetimeToString( long long datetimevalue, char* buf, unsigned int buflen ) { - snprintf( buf, buflen, "%04d-%02d-%02d %02d:%02d:%02d", - (unsigned)((datetimevalue >> 48) & 0xffff), - (unsigned)((datetimevalue >> 44) & 0xf), - (unsigned)((datetimevalue >> 38) & 0x3f), - (unsigned)((datetimevalue >> 32) & 0x3f), - (unsigned)((datetimevalue >> 26) & 0x3f), - (unsigned)((datetimevalue >> 20) & 0x3f) - ); + if ((datetimevalue & 0xfffff) > 0) + { + snprintf( buf, buflen, "%04d-%02d-%02d %02d:%02d:%02d.%d", + (unsigned)((datetimevalue >> 48) & 0xffff), + (unsigned)((datetimevalue >> 44) & 0xf), + (unsigned)((datetimevalue >> 38) & 0x3f), + (unsigned)((datetimevalue >> 32) & 0x3f), + (unsigned)((datetimevalue >> 26) & 0x3f), + (unsigned)((datetimevalue >> 20) & 0x3f), + (unsigned)((datetimevalue) & 0xfffff) + ); + } + else + { + snprintf( buf, buflen, "%04d-%02d-%02d %02d:%02d:%02d", + (unsigned)((datetimevalue >> 48) & 0xffff), + (unsigned)((datetimevalue >> 44) & 0xf), + (unsigned)((datetimevalue >> 38) & 0x3f), + (unsigned)((datetimevalue >> 32) & 0x3f), + (unsigned)((datetimevalue >> 26) & 0x3f), + (unsigned)((datetimevalue >> 20) & 0x3f) + ); + } } inline void DataConvert::dateToString1( int datevalue, char* buf, unsigned int buflen) diff --git a/writeengine/server/we_dmlcommandproc.cpp b/writeengine/server/we_dmlcommandproc.cpp index bf5adf906..07037a1e3 100644 --- a/writeengine/server/we_dmlcommandproc.cpp +++ b/writeengine/server/we_dmlcommandproc.cpp @@ -2977,7 +2977,7 @@ uint8_t WE_DMLCommandProc::processUpdate(messageqcpp::ByteStream& bs, case CalpontSystemCatalog::DATETIME: { intColVal = row.getUintField<8>(fetchColPos); - value = DataConvert::datetimeToString(intColVal); + value = DataConvert::datetimeToString(intColVal, colType.precision); break; } @@ -3305,7 +3305,7 @@ uint8_t WE_DMLCommandProc::processUpdate(messageqcpp::ByteStream& bs, case CalpontSystemCatalog::DATETIME: { intColVal = row.getUintField<8>(fetchColPos); - value = DataConvert::datetimeToString(intColVal); + value = DataConvert::datetimeToString(intColVal, colType.precision); break; } From 3c1ebd8b94c7e87c0d7f8956a693e971b4c2142d Mon Sep 17 00:00:00 2001 From: Andrew Hutchings Date: Mon, 23 Apr 2018 19:20:31 +0100 Subject: [PATCH 32/57] MCOL-392 Add initial TIME datatype support --- dbcon/ddlpackage/ddl.l | 1 + dbcon/ddlpackage/ddl.y | 13 +- dbcon/ddlpackage/ddlpkg.h | 3 + dbcon/ddlpackageproc/altertableprocessor.cpp | 5 + dbcon/ddlpackageproc/ddlindexpopulator.cpp | 2 + dbcon/ddlpackageproc/ddlpackageprocessor.cpp | 11 + dbcon/execplan/aggregatecolumn.cpp | 8 + dbcon/execplan/aggregatecolumn.h | 4 +- dbcon/execplan/arithmeticcolumn.h | 5 + dbcon/execplan/arithmeticoperator.h | 5 + dbcon/execplan/calpontsystemcatalog.cpp | 4 + dbcon/execplan/calpontsystemcatalog.h | 1 + dbcon/execplan/constantcolumn.h | 15 + dbcon/execplan/functioncolumn.h | 5 +- dbcon/execplan/operator.h | 4 + dbcon/execplan/parsetree.h | 8 + dbcon/execplan/predicateoperator.cpp | 12 + dbcon/execplan/predicateoperator.h | 31 ++ dbcon/execplan/simplecolumn.cpp | 1 + dbcon/execplan/simplecolumn.h | 6 + dbcon/execplan/simplefilter.cpp | 31 +- dbcon/execplan/treenode.h | 50 +++ dbcon/execplan/windowfunctioncolumn.cpp | 10 + dbcon/execplan/windowfunctioncolumn.h | 6 +- dbcon/joblist/crossenginestep.cpp | 4 + dbcon/joblist/expressionstep.cpp | 5 +- dbcon/joblist/groupconcat.cpp | 24 ++ dbcon/joblist/jlf_common.cpp | 5 + dbcon/joblist/jlf_execplantojoblist.cpp | 8 + dbcon/joblist/jlf_subquery.cpp | 5 + dbcon/joblist/joblisttypes.h | 2 + dbcon/joblist/lbidlist.cpp | 1 + dbcon/joblist/pcolscan.cpp | 1 + dbcon/joblist/subquerytransformer.cpp | 5 +- dbcon/joblist/tupleaggregatestep.cpp | 33 +- dbcon/joblist/tupleconstantstep.cpp | 1 + dbcon/joblist/tupleunion.cpp | 37 +- dbcon/joblist/windowfunctionstep.cpp | 2 + dbcon/mysql/ha_calpont_ddl.cpp | 4 + dbcon/mysql/ha_calpont_dml.cpp | 37 +- dbcon/mysql/ha_calpont_execplan.cpp | 12 + dbcon/mysql/ha_calpont_impl.cpp | 13 + dbcon/mysql/ha_calpont_partition.cpp | 11 + dbcon/mysql/ha_window_function.cpp | 2 + primitives/linux-port/column.cpp | 9 + primitives/primproc/columncommand.cpp | 1 + tools/pingproc/pingproc.cpp | 1 + utils/common/nullvaluemanip.cpp | 6 + utils/dataconvert/dataconvert.cpp | 378 ++++++++++++++++++ utils/dataconvert/dataconvert.h | 116 +++++- utils/funcexp/func_add_time.cpp | 124 ++++++ utils/funcexp/func_between.cpp | 18 + utils/funcexp/func_bitand.cpp | 25 +- utils/funcexp/func_bitwise.cpp | 9 + utils/funcexp/func_case.cpp | 27 ++ utils/funcexp/func_cast.cpp | 116 ++++++ utils/funcexp/func_ceil.cpp | 18 + utils/funcexp/func_char_length.cpp | 6 + utils/funcexp/func_coalesce.cpp | 24 ++ utils/funcexp/func_extract.cpp | 1 + utils/funcexp/func_floor.cpp | 26 ++ utils/funcexp/func_from_unixtime.cpp | 16 + utils/funcexp/func_greatest.cpp | 20 + utils/funcexp/func_hour.cpp | 6 + utils/funcexp/func_if.cpp | 16 +- utils/funcexp/func_ifnull.cpp | 19 + utils/funcexp/func_in.cpp | 21 + utils/funcexp/func_inet_aton.cpp | 20 + utils/funcexp/func_inet_ntoa.cpp | 14 + utils/funcexp/func_least.cpp | 19 + utils/funcexp/func_math.cpp | 225 +++++++++++ utils/funcexp/func_minute.cpp | 1 + utils/funcexp/func_nullif.cpp | 111 +++++ utils/funcexp/func_regexp.cpp | 16 + utils/funcexp/func_round.cpp | 1 + utils/funcexp/func_second.cpp | 6 + utils/funcexp/func_sysdate.cpp | 9 + utils/funcexp/func_time.cpp | 1 + utils/funcexp/func_timediff.cpp | 10 + utils/funcexp/func_timestampdiff.cpp | 7 + utils/funcexp/func_truncate.cpp | 45 +++ utils/funcexp/funcexp.cpp | 12 + utils/funcexp/functor.cpp | 20 + utils/funcexp/functor.h | 10 + utils/funcexp/functor_all.h | 44 +- utils/funcexp/functor_bool.h | 9 + utils/funcexp/functor_dtm.h | 31 ++ utils/funcexp/functor_real.h | 6 + utils/funcexp/functor_str.h | 12 + utils/rowgroup/rowaggregation.cpp | 59 ++- utils/rowgroup/rowgroup.cpp | 7 + .../docs/source/reference/ColumnDatum.rst | 2 + utils/udfsdk/mcsv1_udaf.cpp | 1 + utils/udfsdk/udfsdk.cpp | 4 +- utils/windowfunction/idborderby.cpp | 2 + utils/windowfunction/wf_lead_lag.cpp | 1 + utils/windowfunction/wf_min_max.cpp | 1 + utils/windowfunction/wf_nth_value.cpp | 1 + utils/windowfunction/wf_percentile.cpp | 1 + utils/windowfunction/wf_udaf.cpp | 1 + utils/windowfunction/windowfunctiontype.cpp | 6 + writeengine/bulk/we_bulkloadbuffer.cpp | 61 ++- writeengine/bulk/we_tableinfo.cpp | 5 + writeengine/server/we_ddlcommon.h | 11 + writeengine/server/we_dmlcommandproc.cpp | 15 + writeengine/shared/we_convertor.cpp | 3 + writeengine/splitter/we_sdhandler.cpp | 4 + writeengine/wrapper/writeengine.cpp | 2 + writeengine/xml/we_xmljob.cpp | 16 + 109 files changed, 2241 insertions(+), 47 deletions(-) diff --git a/dbcon/ddlpackage/ddl.l b/dbcon/ddlpackage/ddl.l index e922c08ca..7e7400451 100644 --- a/dbcon/ddlpackage/ddl.l +++ b/dbcon/ddlpackage/ddl.l @@ -118,6 +118,7 @@ CREATE {return CREATE;} CURRENT_USER {return CURRENT_USER;} DATE {ddlget_lval(yyscanner)->str=strdup("date"); return DATE;} DATETIME {return DATETIME;} +TIME {return TIME;} DECIMAL {return DECIMAL;} DEC {return DECIMAL;} DEFAULT {return DEFAULT;} diff --git a/dbcon/ddlpackage/ddl.y b/dbcon/ddlpackage/ddl.y index c084f44fe..68747dfc9 100644 --- a/dbcon/ddlpackage/ddl.y +++ b/dbcon/ddlpackage/ddl.y @@ -177,7 +177,6 @@ VARYING WITH ZONE DOUBLE IDB_FLOAT REAL CHARSET IDB_IF EXISTS CHANGE TRUNCATE %type opt_precision_scale %type opt_referential_triggered_action %type opt_time_precision -%type opt_with_time_zone %type qualified_name %type referential_action %type referential_triggered_action @@ -1127,12 +1126,11 @@ datetime_type: $$->fLength = DDLDatatypeLength[DDL_DATE]; } | - TIME opt_time_precision opt_with_time_zone + TIME opt_time_precision { - $$ = new ColumnType(DDL_DATETIME); - $$->fLength = DDLDatatypeLength[DDL_DATETIME]; + $$ = new ColumnType(DDL_TIME); + $$->fLength = DDLDatatypeLength[DDL_TIME]; $$->fPrecision = $2; - $$->fWithTimezone = $3; } opt_time_precision: @@ -1140,11 +1138,6 @@ opt_time_precision: | {$$ = -1;} ; -opt_with_time_zone: - WITH TIME ZONE {$$ = true;} - | {$$ = false;} - ; - drop_column_def: DROP column_name drop_behavior {$$ = new AtaDropColumn($2, $3);} | DROP COLUMN column_name drop_behavior {$$ = new AtaDropColumn($3, $4);} diff --git a/dbcon/ddlpackage/ddlpkg.h b/dbcon/ddlpackage/ddlpkg.h index 97e971b6b..779577f83 100644 --- a/dbcon/ddlpackage/ddlpkg.h +++ b/dbcon/ddlpackage/ddlpkg.h @@ -236,6 +236,7 @@ enum DDL_DATATYPES DDL_UNSIGNED_DOUBLE, DDL_UNSIGNED_NUMERIC, DDL_TEXT, + DDL_TIME, DDL_INVALID_DATATYPE }; @@ -273,6 +274,7 @@ const std::string DDLDatatypeString[] = "unsigned-double", "unsigned-numeric", "text", + "time" "" }; @@ -328,6 +330,7 @@ const int DDLDatatypeLength[] = 8, // UNSIGNED_DOUBLE, 2, // UNSIGNED_NUMERIC, 8, // TEXT + 8, // TIME -1 // INVALID LENGTH }; diff --git a/dbcon/ddlpackageproc/altertableprocessor.cpp b/dbcon/ddlpackageproc/altertableprocessor.cpp index c6edff71c..a398e656f 100644 --- a/dbcon/ddlpackageproc/altertableprocessor.cpp +++ b/dbcon/ddlpackageproc/altertableprocessor.cpp @@ -209,6 +209,11 @@ bool typesAreSame(const CalpontSystemCatalog::ColType& colType, const ColumnType break; + case (CalpontSystemCatalog::TIME): + if (newType.fType == DDL_TIME) return true; + + break; + case (CalpontSystemCatalog::VARCHAR): if (newType.fType == DDL_VARCHAR && colType.colWidth == newType.fLength) return true; diff --git a/dbcon/ddlpackageproc/ddlindexpopulator.cpp b/dbcon/ddlpackageproc/ddlindexpopulator.cpp index 12d6c9854..cb58df5bc 100644 --- a/dbcon/ddlpackageproc/ddlindexpopulator.cpp +++ b/dbcon/ddlpackageproc/ddlindexpopulator.cpp @@ -330,6 +330,7 @@ boost::any DDLIndexPopulator::convertData(const CalpontSystemCatalog::ColType& return *reinterpret_cast(&data); case execplan::CalpontSystemCatalog::DATETIME: // @bug 375 + case execplan::CalpontSystemCatalog::TIME: case execplan::CalpontSystemCatalog::BIGINT: return *reinterpret_cast(&data); @@ -524,6 +525,7 @@ bool DDLIndexPopulator::checkNotNull(const IdxTuple& data, const CalpontSystemCa break; case execplan::CalpontSystemCatalog::DATETIME: + case execplan::CalpontSystemCatalog::TIME: isNull = any_cast(data.data) == any_cast(nullvalue); break; diff --git a/dbcon/ddlpackageproc/ddlpackageprocessor.cpp b/dbcon/ddlpackageproc/ddlpackageprocessor.cpp index 9b8c66658..2a64dd1f7 100644 --- a/dbcon/ddlpackageproc/ddlpackageprocessor.cpp +++ b/dbcon/ddlpackageproc/ddlpackageprocessor.cpp @@ -232,6 +232,10 @@ execplan::CalpontSystemCatalog::ColDataType DDLPackageProcessor::convertDataType colDataType = CalpontSystemCatalog::DATETIME; break; + case ddlpackage::DDL_TIME: + colDataType = CalpontSystemCatalog::TIME; + break; + case ddlpackage::DDL_CLOB: colDataType = CalpontSystemCatalog::CLOB; break; @@ -476,6 +480,13 @@ DDLPackageProcessor::getNullValueForType(const execplan::CalpontSystemCatalog::C } break; + case execplan::CalpontSystemCatalog::TIME: + { + long long d = joblist::TIMENULL; + value = d; + } + break; + case execplan::CalpontSystemCatalog::CHAR: { std::string charnull; diff --git a/dbcon/execplan/aggregatecolumn.cpp b/dbcon/execplan/aggregatecolumn.cpp index e0914beb3..18cba2607 100644 --- a/dbcon/execplan/aggregatecolumn.cpp +++ b/dbcon/execplan/aggregatecolumn.cpp @@ -353,6 +353,14 @@ void AggregateColumn::evaluate(Row& row, bool& isNull) break; + case CalpontSystemCatalog::TIME: + if (row.equals<8>(TIMENULL, fInputIndex)) + isNull = true; + else + fResult.intVal = row.getIntField<8>(fInputIndex); + + break; + case CalpontSystemCatalog::CHAR: case CalpontSystemCatalog::VARCHAR: case CalpontSystemCatalog::STRINT: diff --git a/dbcon/execplan/aggregatecolumn.h b/dbcon/execplan/aggregatecolumn.h index a70189e50..028f1ee54 100644 --- a/dbcon/execplan/aggregatecolumn.h +++ b/dbcon/execplan/aggregatecolumn.h @@ -411,10 +411,10 @@ public: /** * F&E */ - virtual int64_t getDatetimeIntVal(rowgroup::Row& row, bool& isNull) + virtual int64_t getTimeIntVal(rowgroup::Row& row, bool& isNull) { evaluate(row, isNull); - return TreeNode::getDatetimeIntVal(); + return TreeNode::getTimeIntVal(); } private: diff --git a/dbcon/execplan/arithmeticcolumn.h b/dbcon/execplan/arithmeticcolumn.h index b6ca823c4..191416fbf 100644 --- a/dbcon/execplan/arithmeticcolumn.h +++ b/dbcon/execplan/arithmeticcolumn.h @@ -248,6 +248,11 @@ public: return fExpression->getDatetimeIntVal(row, isNull); } + virtual int64_t getTimeIntVal(rowgroup::Row& row, bool& isNull) + { + return fExpression->getTimeIntVal(row, isNull); + } + virtual bool getBoolVal(rowgroup::Row& row, bool& isNull) { return fExpression->getBoolVal(row, isNull); diff --git a/dbcon/execplan/arithmeticoperator.h b/dbcon/execplan/arithmeticoperator.h index ef7b538fa..ca254075b 100644 --- a/dbcon/execplan/arithmeticoperator.h +++ b/dbcon/execplan/arithmeticoperator.h @@ -151,6 +151,11 @@ public: evaluate(row, isNull, lop, rop); return TreeNode::getDatetimeIntVal(); } + virtual int64_t getTimeIntVal(rowgroup::Row& row, bool& isNull, ParseTree* lop, ParseTree* rop) + { + evaluate(row, isNull, lop, rop); + return TreeNode::getTimeIntVal(); + } virtual bool getBoolVal(rowgroup::Row& row, bool& isNull, ParseTree* lop, ParseTree* rop) { evaluate(row, isNull, lop, rop); diff --git a/dbcon/execplan/calpontsystemcatalog.cpp b/dbcon/execplan/calpontsystemcatalog.cpp index 501f488ce..15ccd73aa 100644 --- a/dbcon/execplan/calpontsystemcatalog.cpp +++ b/dbcon/execplan/calpontsystemcatalog.cpp @@ -156,6 +156,10 @@ const string colDataTypeToString(CalpontSystemCatalog::ColDataType cdt) return "datetime"; break; + case CalpontSystemCatalog::TIME: + return "time"; + break; + case CalpontSystemCatalog::VARCHAR: return "varchar"; break; diff --git a/dbcon/execplan/calpontsystemcatalog.h b/dbcon/execplan/calpontsystemcatalog.h index 5918f7f8c..7b828a297 100644 --- a/dbcon/execplan/calpontsystemcatalog.h +++ b/dbcon/execplan/calpontsystemcatalog.h @@ -154,6 +154,7 @@ public: UBIGINT, /*!< Unsigned BIGINT type */ UDOUBLE, /*!< Unsigned DOUBLE type */ TEXT, /*!< TEXT type */ + TIME, /*!< TIME type */ NUM_OF_COL_DATA_TYPE, /* NEW TYPES ABOVE HERE */ LONGDOUBLE, /* @bug3241, dev and variance calculation only */ STRINT, /* @bug3532, string as int for fast comparison */ diff --git a/dbcon/execplan/constantcolumn.h b/dbcon/execplan/constantcolumn.h index 8e7f178aa..04098faae 100644 --- a/dbcon/execplan/constantcolumn.h +++ b/dbcon/execplan/constantcolumn.h @@ -308,6 +308,21 @@ public: return fResult.intVal; } + /** + * F&E + */ + virtual int64_t getTimeIntVal(rowgroup::Row& row, bool& isNull) + { + isNull = isNull || (fType == NULLDATA); + + if (!fResult.valueConverted) + { + fResult.intVal = dataconvert::DataConvert::stringToTime(fResult.strVal); + fResult.valueConverted = true; + } + + return fResult.intVal; + } /** * F&E */ diff --git a/dbcon/execplan/functioncolumn.h b/dbcon/execplan/functioncolumn.h index 8f27cad75..5a099d1d2 100644 --- a/dbcon/execplan/functioncolumn.h +++ b/dbcon/execplan/functioncolumn.h @@ -255,7 +255,10 @@ public: { return fFunctor->getDatetimeIntVal(row, fFunctionParms, isNull, fOperationType); } - + virtual int64_t getTimeIntVal(rowgroup::Row& row, bool& isNull) + { + return fFunctor->getTimeIntVal(row, fFunctionParms, isNull, fOperationType); + } private: funcexp::FunctionParm fFunctionParms; diff --git a/dbcon/execplan/operator.h b/dbcon/execplan/operator.h index e0e16be2f..1a33dcb94 100644 --- a/dbcon/execplan/operator.h +++ b/dbcon/execplan/operator.h @@ -190,6 +190,10 @@ public: { return fResult.intVal; } + virtual int64_t getTimeIntVal(rowgroup::Row& row, bool& isNull, ParseTree* lop, ParseTree* rop) + { + return fResult.intVal; + } virtual bool getBoolVal(rowgroup::Row& row, bool& isNull, ParseTree* lop, ParseTree* rop) { return fResult.boolVal; diff --git a/dbcon/execplan/parsetree.h b/dbcon/execplan/parsetree.h index 313f2906a..2ad41b6bb 100644 --- a/dbcon/execplan/parsetree.h +++ b/dbcon/execplan/parsetree.h @@ -282,6 +282,14 @@ public: return fData->getDatetimeIntVal(row, isNull); } + inline int64_t getTimeIntVal(rowgroup::Row& row, bool& isNull) + { + if (fLeft && fRight) + return (reinterpret_cast(fData))->getTimeIntVal(row, isNull, fLeft, fRight); + else + return fData->getTimeIntVal(row, isNull); + } + private: /** draw the tree * diff --git a/dbcon/execplan/predicateoperator.cpp b/dbcon/execplan/predicateoperator.cpp index 84579df22..166a8edc0 100644 --- a/dbcon/execplan/predicateoperator.cpp +++ b/dbcon/execplan/predicateoperator.cpp @@ -196,6 +196,7 @@ bool PredicateOperator::operator!=(const TreeNode* t) const void PredicateOperator::setOpType(Type& l, Type& r) { if ( l.colDataType == execplan::CalpontSystemCatalog::DATETIME || + l.colDataType == execplan::CalpontSystemCatalog::TIME || l.colDataType == execplan::CalpontSystemCatalog::DATE ) { switch (r.colDataType) @@ -210,6 +211,11 @@ void PredicateOperator::setOpType(Type& l, Type& r) fOperationType.colWidth = 8; break; + case execplan::CalpontSystemCatalog::TIME: + fOperationType.colDataType = execplan::CalpontSystemCatalog::TIME; + fOperationType.colWidth = 8; + break; + case execplan::CalpontSystemCatalog::DATE: fOperationType = l; break; @@ -221,6 +227,7 @@ void PredicateOperator::setOpType(Type& l, Type& r) } } else if ( r.colDataType == execplan::CalpontSystemCatalog::DATETIME || + r.colDataType == execplan::CalpontSystemCatalog::TIME || r.colDataType == execplan::CalpontSystemCatalog::DATE ) { switch (l.colDataType) @@ -236,6 +243,11 @@ void PredicateOperator::setOpType(Type& l, Type& r) fOperationType.colWidth = 8; break; + case execplan::CalpontSystemCatalog::TIME: + fOperationType.colDataType = execplan::CalpontSystemCatalog::TIME; + fOperationType.colWidth = 8; + break; + case execplan::CalpontSystemCatalog::DATE: fOperationType = r; break; diff --git a/dbcon/execplan/predicateoperator.h b/dbcon/execplan/predicateoperator.h index 2b63548d5..6253d9389 100644 --- a/dbcon/execplan/predicateoperator.h +++ b/dbcon/execplan/predicateoperator.h @@ -344,6 +344,37 @@ inline bool PredicateOperator::getBoolVal(rowgroup::Row& row, bool& isNull, Retu return numericCompare(val1, rop->getDatetimeIntVal(row, isNull)) && !isNull; } + case execplan::CalpontSystemCatalog::TIME: + { + if (fOp == OP_ISNULL) + { + lop->getTimeIntVal(row, isNull); + bool ret = isNull; + isNull = false; + return ret; + } + + if (fOp == OP_ISNOTNULL) + { + lop->getTimeIntVal(row, isNull); + bool ret = isNull; + isNull = false; + return !ret; + } + + if (isNull) + return false; + + int64_t val1 = lop->getTimeIntVal(row, isNull); + + if (isNull) + return false; + + return numericCompare(val1, rop->getTimeIntVal(row, isNull)) && !isNull; + } + + + case execplan::CalpontSystemCatalog::VARCHAR: case execplan::CalpontSystemCatalog::CHAR: case execplan::CalpontSystemCatalog::TEXT: diff --git a/dbcon/execplan/simplecolumn.cpp b/dbcon/execplan/simplecolumn.cpp index 429e4f883..64955401e 100644 --- a/dbcon/execplan/simplecolumn.cpp +++ b/dbcon/execplan/simplecolumn.cpp @@ -502,6 +502,7 @@ void SimpleColumn::evaluate(Row& row, bool& isNull) } case CalpontSystemCatalog::DATETIME: + case CalpontSystemCatalog::TIME: { fResult.intVal = row.getUintField<8>(fInputIndex); break; diff --git a/dbcon/execplan/simplecolumn.h b/dbcon/execplan/simplecolumn.h index 81571e057..60eff939b 100644 --- a/dbcon/execplan/simplecolumn.h +++ b/dbcon/execplan/simplecolumn.h @@ -331,6 +331,12 @@ public: return TreeNode::getDatetimeIntVal(); } + inline int64_t getTimeIntVal(rowgroup::Row& row, bool& isNull) + { + evaluate(row, isNull); + return TreeNode::getTimeIntVal(); + } + }; typedef boost::shared_ptr SSC; diff --git a/dbcon/execplan/simplefilter.cpp b/dbcon/execplan/simplefilter.cpp index 1df673dcd..1da14449c 100644 --- a/dbcon/execplan/simplefilter.cpp +++ b/dbcon/execplan/simplefilter.cpp @@ -209,7 +209,8 @@ const string SimpleFilter::data() const fRhs->resultType().colDataType == CalpontSystemCatalog::TEXT || fRhs->resultType().colDataType == CalpontSystemCatalog::VARBINARY || fRhs->resultType().colDataType == CalpontSystemCatalog::DATE || - fRhs->resultType().colDataType == CalpontSystemCatalog::DATETIME)) + fRhs->resultType().colDataType == CalpontSystemCatalog::DATETIME || + fRhs->resultType().colDataType == CalpontSystemCatalog::TIME)) rhs = "'" + SimpleFilter::escapeString(fRhs->data()) + "'"; else rhs = fRhs->data(); @@ -221,6 +222,7 @@ const string SimpleFilter::data() const fLhs->resultType().colDataType == CalpontSystemCatalog::TEXT || fLhs->resultType().colDataType == CalpontSystemCatalog::VARBINARY || fLhs->resultType().colDataType == CalpontSystemCatalog::DATE || + fLhs->resultType().colDataType == CalpontSystemCatalog::TIME || fLhs->resultType().colDataType == CalpontSystemCatalog::DATETIME)) lhs = "'" + SimpleFilter::escapeString(fLhs->data()) + "'"; else @@ -544,6 +546,19 @@ void SimpleFilter::convertConstant() result.intVal = dataconvert::DataConvert::datetimeToInt(result.strVal); } } + else if (fRhs->resultType().colDataType == CalpontSystemCatalog::TIME) + { + if (lcc->constval().empty()) + { + lcc->constval("00:00:00"); + result.intVal = 0; + result.strVal = lcc->constval(); + } + else + { + result.intVal = dataconvert::DataConvert::timeToInt(result.strVal); + } + } lcc->result(result); } @@ -578,7 +593,19 @@ void SimpleFilter::convertConstant() result.intVal = dataconvert::DataConvert::datetimeToInt(result.strVal); } } - + else if (fLhs->resultType().colDataType == CalpontSystemCatalog::TIME) + { + if (rcc->constval().empty()) + { + rcc->constval("00:00:00"); + result.intVal = 0; + result.strVal = rcc->constval(); + } + else + { + result.intVal = dataconvert::DataConvert::timeToInt(result.strVal); + } + } rcc->result(result); } } diff --git a/dbcon/execplan/treenode.h b/dbcon/execplan/treenode.h index 92420da3f..27795ea9d 100644 --- a/dbcon/execplan/treenode.h +++ b/dbcon/execplan/treenode.h @@ -388,6 +388,10 @@ public: { return fResult.intVal; } + virtual int64_t getTimeIntVal(rowgroup::Row& row, bool& isNull) + { + return fResult.intVal; + } virtual void evaluate(rowgroup::Row& row, bool& isNull) {} inline bool getBoolVal(); @@ -399,6 +403,7 @@ public: inline IDB_Decimal getDecimalVal(); inline int32_t getDateIntVal(); inline int64_t getDatetimeIntVal(); + inline int64_t getTimeIntVal(); virtual const execplan::CalpontSystemCatalog::ColType& resultType() const { @@ -490,6 +495,7 @@ inline bool TreeNode::getBoolVal() case CalpontSystemCatalog::INT: case CalpontSystemCatalog::DATE: case CalpontSystemCatalog::DATETIME: + case CalpontSystemCatalog::TIME: return (fResult.intVal != 0); case CalpontSystemCatalog::UBIGINT: @@ -665,6 +671,13 @@ inline const std::string& TreeNode::getStrVal() break; } + case CalpontSystemCatalog::TIME: + { + dataconvert::DataConvert::timeToString(fResult.intVal, tmp, 255); + fResult.strVal = std::string(tmp); + break; + } + default: throw logging::InvalidConversionExcept("TreeNode::getStrVal: Invalid conversion."); } @@ -727,6 +740,7 @@ inline int64_t TreeNode::getIntVal() case CalpontSystemCatalog::DATE: case CalpontSystemCatalog::DATETIME: + case CalpontSystemCatalog::TIME: return fResult.intVal; default: @@ -769,6 +783,7 @@ inline uint64_t TreeNode::getUintVal() case CalpontSystemCatalog::DATE: case CalpontSystemCatalog::DATETIME: + case CalpontSystemCatalog::TIME: return fResult.intVal; default: @@ -831,6 +846,7 @@ inline float TreeNode::getFloatVal() case CalpontSystemCatalog::DATE: case CalpontSystemCatalog::DATETIME: + case CalpontSystemCatalog::TIME: return (float)fResult.intVal; default: @@ -895,6 +911,7 @@ inline double TreeNode::getDoubleVal() case CalpontSystemCatalog::DATE: case CalpontSystemCatalog::DATETIME: + case CalpontSystemCatalog::TIME: return (double)fResult.intVal; default: @@ -937,9 +954,14 @@ inline IDB_Decimal TreeNode::getDecimalVal() break; case CalpontSystemCatalog::DATE: + throw logging::InvalidConversionExcept("TreeNode::getDecimalVal: Invalid conversion from date."); + case CalpontSystemCatalog::DATETIME: throw logging::InvalidConversionExcept("TreeNode::getDecimalVal: Invalid conversion from datetime."); + case CalpontSystemCatalog::TIME: + throw logging::InvalidConversionExcept("TreeNode::getDecimalVal: Invalid conversion from time."); + case CalpontSystemCatalog::FLOAT: throw logging::InvalidConversionExcept("TreeNode::getDecimalVal: non-support conversion from float"); @@ -967,6 +989,17 @@ inline int64_t TreeNode::getDatetimeIntVal() { if (fResultType.colDataType == execplan::CalpontSystemCatalog::DATE) return (fResult.intVal & 0x00000000FFFFFFC0LL) << 32; + else if (fResultType.colDataType == execplan::CalpontSystemCatalog::TIME) + { + dataconvert::Time tt; + + memcpy(&tt, &fResult.intVal, 8); + if (tt.hour > 23 || tt.hour < 0) + throw logging::InvalidConversionExcept("TreeNode::getDecimalVal: Invalid conversion from time (out of range)."); + dataconvert::DateTime dt(0, 0, 0, tt.hour, tt.minute, tt.second, tt.msecond); + memcpy(&fResult.intVal, &dt, 8); + return fResult.intVal; + } else if (fResultType.colDataType == execplan::CalpontSystemCatalog::DATETIME) //return (fResult.intVal & 0xFFFFFFFFFFF00000LL); return (fResult.intVal); @@ -974,6 +1007,23 @@ inline int64_t TreeNode::getDatetimeIntVal() return getIntVal(); } +inline int64_t TreeNode::getTimeIntVal() +{ + if (fResultType.colDataType == execplan::CalpontSystemCatalog::DATETIME) + { + dataconvert::DateTime dt; + + memcpy(&dt, &fResult.intVal, 8); + dataconvert::Time tt(0, dt.hour, dt.minute, dt.second, dt.msecond); + memcpy(&fResult.intVal, &tt, 8); + return fResult.intVal; + } + else if (fResultType.colDataType == execplan::CalpontSystemCatalog::TIME) + return (fResult.intVal); + else + return getIntVal(); +} + inline int32_t TreeNode::getDateIntVal() { if (fResultType.colDataType == execplan::CalpontSystemCatalog::DATETIME) diff --git a/dbcon/execplan/windowfunctioncolumn.cpp b/dbcon/execplan/windowfunctioncolumn.cpp index f81a6a5d4..5c84ff2d1 100644 --- a/dbcon/execplan/windowfunctioncolumn.cpp +++ b/dbcon/execplan/windowfunctioncolumn.cpp @@ -403,6 +403,16 @@ void WindowFunctionColumn::evaluate(Row& row, bool& isNull) break; } + case CalpontSystemCatalog::TIME: + { + if (row.equals<8>(TIMENULL, fInputIndex)) + isNull = true; + else + fResult.intVal = row.getIntField<8>(fInputIndex); + + break; + } + case CalpontSystemCatalog::CHAR: case CalpontSystemCatalog::VARCHAR: case CalpontSystemCatalog::STRINT: diff --git a/dbcon/execplan/windowfunctioncolumn.h b/dbcon/execplan/windowfunctioncolumn.h index 8b427947c..3fc983e55 100644 --- a/dbcon/execplan/windowfunctioncolumn.h +++ b/dbcon/execplan/windowfunctioncolumn.h @@ -213,7 +213,11 @@ public: evaluate(row, isNull); return TreeNode::getDatetimeIntVal(); } - + virtual int64_t getTimeIntVal(rowgroup::Row& row, bool& isNull) + { + evaluate(row, isNull); + return TreeNode::getTimeIntVal(); + } private: void evaluate(rowgroup::Row& row, bool& isNull); }; diff --git a/dbcon/joblist/crossenginestep.cpp b/dbcon/joblist/crossenginestep.cpp index 483249067..789e58cd4 100644 --- a/dbcon/joblist/crossenginestep.cpp +++ b/dbcon/joblist/crossenginestep.cpp @@ -351,6 +351,10 @@ int64_t CrossEngineStep::convertValueNum( rv = boost::any_cast(anyVal); break; + case CalpontSystemCatalog::TIME: + rv = boost::any_cast(anyVal); + break; + case CalpontSystemCatalog::DECIMAL: case CalpontSystemCatalog::UDECIMAL: if (ct.colWidth == CalpontSystemCatalog::ONE_BYTE) diff --git a/dbcon/joblist/expressionstep.cpp b/dbcon/joblist/expressionstep.cpp index 4a169e081..0e064c359 100644 --- a/dbcon/joblist/expressionstep.cpp +++ b/dbcon/joblist/expressionstep.cpp @@ -403,8 +403,9 @@ void ExpressionStep::populateColumnInfo(SimpleColumn* sc, JobInfo& jobInfo) TupleInfo ti(setTupleInfo(ct, sc->oid(), jobInfo, tblOid, sc, alias)); fColumnKeys.push_back(ti.key); - // @bug 2990, MySQL date/datetime type is different from IDB type - if (ti.dtype == CalpontSystemCatalog::DATE || ti.dtype == CalpontSystemCatalog::DATETIME) + // @bug 2990, MySQL time/date/datetime type is different from IDB type + if (ti.dtype == CalpontSystemCatalog::DATE || ti.dtype == CalpontSystemCatalog::DATETIME || + ti.dtype == CalpontSystemCatalog::TIME) { if (ti.dtype != ct.colDataType) { diff --git a/dbcon/joblist/groupconcat.cpp b/dbcon/joblist/groupconcat.cpp index fd122fe8d..1e84585b3 100644 --- a/dbcon/joblist/groupconcat.cpp +++ b/dbcon/joblist/groupconcat.cpp @@ -507,6 +507,12 @@ void GroupConcator::outputRow(std::ostringstream& oss, const rowgroup::Row& row) break; } + case CalpontSystemCatalog::TIME: + { + oss << DataConvert::timeToString(row.getUintField(*i)); + break; + } + default: { break; @@ -640,6 +646,24 @@ int64_t GroupConcator::lengthEstimate(const rowgroup::Row& row) case CalpontSystemCatalog::DATETIME: { fieldLen = 19; // YYYY-MM-DD HH24:MI:SS + // Decimal point and milliseconds + uint64_t colPrecision = row.getPrecision(*i); + if (colPrecision > 0 && colPrecision < 7) + { + fieldLen += colPrecision + 1; + } + break; + } + + case CalpontSystemCatalog::TIME: + { + fieldLen = 10; // -HHH:MI:SS + // Decimal point and milliseconds + uint64_t colPrecision = row.getPrecision(*i); + if (colPrecision > 0 && colPrecision < 7) + { + fieldLen += colPrecision + 1; + } break; } diff --git a/dbcon/joblist/jlf_common.cpp b/dbcon/joblist/jlf_common.cpp index 063e24dfc..f5dbeee17 100644 --- a/dbcon/joblist/jlf_common.cpp +++ b/dbcon/joblist/jlf_common.cpp @@ -792,6 +792,11 @@ bool compatibleColumnTypes(const CalpontSystemCatalog::ColDataType& dt1, uint32_ break; + case CalpontSystemCatalog::TIME: + if (dt2 != CalpontSystemCatalog::TIME) return false; + + break; + case CalpontSystemCatalog::CHAR: case CalpontSystemCatalog::VARCHAR: case CalpontSystemCatalog::TEXT: diff --git a/dbcon/joblist/jlf_execplantojoblist.cpp b/dbcon/joblist/jlf_execplantojoblist.cpp index 0ff2f7c93..b81a2ec8d 100644 --- a/dbcon/joblist/jlf_execplantojoblist.cpp +++ b/dbcon/joblist/jlf_execplantojoblist.cpp @@ -282,6 +282,10 @@ int64_t valueNullNum(const CalpontSystemCatalog::ColType& ct) n = boost::any_cast(anyVal); break; + case CalpontSystemCatalog::TIME: + n = boost::any_cast(anyVal); + break; + case CalpontSystemCatalog::DECIMAL: case CalpontSystemCatalog::UDECIMAL: if (ct.colWidth == execplan::CalpontSystemCatalog::ONE_BYTE) @@ -421,6 +425,10 @@ int64_t convertValueNum(const string& str, const CalpontSystemCatalog::ColType& v = boost::any_cast(anyVal); break; + case CalpontSystemCatalog::TIME: + v = boost::any_cast(anyVal); + break; + case CalpontSystemCatalog::DECIMAL: case CalpontSystemCatalog::UDECIMAL: if (ct.colWidth == execplan::CalpontSystemCatalog::ONE_BYTE) diff --git a/dbcon/joblist/jlf_subquery.cpp b/dbcon/joblist/jlf_subquery.cpp index e1537d577..1e4eaeeec 100644 --- a/dbcon/joblist/jlf_subquery.cpp +++ b/dbcon/joblist/jlf_subquery.cpp @@ -135,6 +135,11 @@ void getColumnValue(ConstantColumn** cc, uint64_t i, const Row& row) *cc = new ConstantColumn(oss.str()); break; + case CalpontSystemCatalog::TIME: + oss << dataconvert::DataConvert::timeToString(row.getUintField<8>(i)); + *cc = new ConstantColumn(oss.str()); + break; + case CalpontSystemCatalog::CHAR: case CalpontSystemCatalog::VARCHAR: case CalpontSystemCatalog::TEXT: diff --git a/dbcon/joblist/joblisttypes.h b/dbcon/joblist/joblisttypes.h index 3f3755a71..37d52f9f4 100644 --- a/dbcon/joblist/joblisttypes.h +++ b/dbcon/joblist/joblisttypes.h @@ -56,6 +56,8 @@ const uint32_t DATENULL = 0xFFFFFFFE; const uint32_t DATEEMPTYROW = 0xFFFFFFFF; const uint64_t DATETIMENULL = 0xFFFFFFFFFFFFFFFEULL; const uint64_t DATETIMEEMPTYROW = 0xFFFFFFFFFFFFFFFFULL; +const uint64_t TIMENULL = 0xFFFFFFFFFFFFFFFEULL; +const uint64_t TIMEEMPTYROW = 0xFFFFFFFFFFFFFFFFULL; const uint8_t CHAR1NULL = 0xFE; const uint8_t CHAR1EMPTYROW = 0xFF; diff --git a/dbcon/joblist/lbidlist.cpp b/dbcon/joblist/lbidlist.cpp index bd012941e..c317defc9 100644 --- a/dbcon/joblist/lbidlist.cpp +++ b/dbcon/joblist/lbidlist.cpp @@ -510,6 +510,7 @@ bool LBIDList::CasualPartitionDataType(const CalpontSystemCatalog::ColDataType t case CalpontSystemCatalog::DECIMAL: case CalpontSystemCatalog::DATE: case CalpontSystemCatalog::DATETIME: + case CalpontSystemCatalog::TIME: case CalpontSystemCatalog::UTINYINT: case CalpontSystemCatalog::USMALLINT: case CalpontSystemCatalog::UDECIMAL: diff --git a/dbcon/joblist/pcolscan.cpp b/dbcon/joblist/pcolscan.cpp index cd19ce34c..6cea4fc03 100644 --- a/dbcon/joblist/pcolscan.cpp +++ b/dbcon/joblist/pcolscan.cpp @@ -1167,6 +1167,7 @@ bool pColScanStep::isEmptyVal(const uint8_t* val8) const case CalpontSystemCatalog::VARCHAR: case CalpontSystemCatalog::DATE: case CalpontSystemCatalog::DATETIME: + case CalpontSystemCatalog::TIME: if (width == 1) { return (*val8 == joblist::CHAR1EMPTYROW); diff --git a/dbcon/joblist/subquerytransformer.cpp b/dbcon/joblist/subquerytransformer.cpp index a5e0759fa..c212c701d 100644 --- a/dbcon/joblist/subquerytransformer.cpp +++ b/dbcon/joblist/subquerytransformer.cpp @@ -237,9 +237,10 @@ SJSTEP& SubQueryTransformer::makeSubQueryStep(execplan::CalpontSelectExecutionPl fVtable.columnType(ct, i); } } - // MySQL date/datetime type is different from IDB type + // MySQL time/date/datetime type is different from IDB type else if (colDataTypeInRg == CalpontSystemCatalog::DATE || - colDataTypeInRg == CalpontSystemCatalog::DATETIME) + colDataTypeInRg == CalpontSystemCatalog::DATETIME || + colDataTypeInRg == CalpontSystemCatalog::TIME) { ct.colWidth = row.getColumnWidth(i); ct.colDataType = row.getColTypes()[i]; diff --git a/dbcon/joblist/tupleaggregatestep.cpp b/dbcon/joblist/tupleaggregatestep.cpp index ca44cebbc..9e23ac17b 100644 --- a/dbcon/joblist/tupleaggregatestep.cpp +++ b/dbcon/joblist/tupleaggregatestep.cpp @@ -232,6 +232,9 @@ inline string colTypeIdString(CalpontSystemCatalog::ColDataType type) case CalpontSystemCatalog::DATETIME: return string("DATETIME"); + case CalpontSystemCatalog::TIME: + return string("TIME"); + case CalpontSystemCatalog::VARCHAR: return string("VARCHAR"); @@ -1333,7 +1336,8 @@ void TupleAggregateStep::prep1PhaseAggregate( typeProj[colProj] == CalpontSystemCatalog::BLOB || typeProj[colProj] == CalpontSystemCatalog::TEXT || typeProj[colProj] == CalpontSystemCatalog::DATE || - typeProj[colProj] == CalpontSystemCatalog::DATETIME) + typeProj[colProj] == CalpontSystemCatalog::DATETIME || + typeProj[colProj] == CalpontSystemCatalog::TIME) { Message::Args args; args.add("sum/average"); @@ -1415,7 +1419,8 @@ void TupleAggregateStep::prep1PhaseAggregate( typeProj[colProj] == CalpontSystemCatalog::TEXT || typeProj[colProj] == CalpontSystemCatalog::BLOB || typeProj[colProj] == CalpontSystemCatalog::DATE || - typeProj[colProj] == CalpontSystemCatalog::DATETIME) + typeProj[colProj] == CalpontSystemCatalog::DATETIME || + typeProj[colProj] == CalpontSystemCatalog::TIME) { Message::Args args; args.add("variance/standard deviation"); @@ -1881,7 +1886,8 @@ void TupleAggregateStep::prep1PhaseDistinctAggregate( typeProj[colProj] == CalpontSystemCatalog::BLOB || typeProj[colProj] == CalpontSystemCatalog::TEXT || typeProj[colProj] == CalpontSystemCatalog::DATE || - typeProj[colProj] == CalpontSystemCatalog::DATETIME) + typeProj[colProj] == CalpontSystemCatalog::DATETIME || + typeProj[colProj] == CalpontSystemCatalog::TIME) { Message::Args args; args.add("sum/average"); @@ -1966,7 +1972,8 @@ void TupleAggregateStep::prep1PhaseDistinctAggregate( typeProj[colProj] == CalpontSystemCatalog::BLOB || typeProj[colProj] == CalpontSystemCatalog::TEXT || typeProj[colProj] == CalpontSystemCatalog::DATE || - typeProj[colProj] == CalpontSystemCatalog::DATETIME) + typeProj[colProj] == CalpontSystemCatalog::DATETIME || + typeProj[colProj] == CalpontSystemCatalog::TIME) { Message::Args args; args.add("variance/standard deviation"); @@ -2149,7 +2156,8 @@ void TupleAggregateStep::prep1PhaseDistinctAggregate( typeAgg[colAgg] == CalpontSystemCatalog::BLOB || typeAgg[colAgg] == CalpontSystemCatalog::TEXT || typeAgg[colAgg] == CalpontSystemCatalog::DATE || - typeAgg[colAgg] == CalpontSystemCatalog::DATETIME) + typeAgg[colAgg] == CalpontSystemCatalog::DATETIME || + typeAgg[colAgg] == CalpontSystemCatalog::TIME) { Message::Args args; args.add("sum/average"); @@ -3071,7 +3079,8 @@ void TupleAggregateStep::prep2PhasesAggregate( typeProj[colProj] == CalpontSystemCatalog::BLOB || typeProj[colProj] == CalpontSystemCatalog::TEXT || typeProj[colProj] == CalpontSystemCatalog::DATE || - typeProj[colProj] == CalpontSystemCatalog::DATETIME) + typeProj[colProj] == CalpontSystemCatalog::DATETIME || + typeProj[colProj] == CalpontSystemCatalog::TIME) { Message::Args args; args.add("sum/average"); @@ -3160,7 +3169,8 @@ void TupleAggregateStep::prep2PhasesAggregate( typeProj[colProj] == CalpontSystemCatalog::BLOB || typeProj[colProj] == CalpontSystemCatalog::TEXT || typeProj[colProj] == CalpontSystemCatalog::DATE || - typeProj[colProj] == CalpontSystemCatalog::DATETIME) + typeProj[colProj] == CalpontSystemCatalog::DATETIME || + typeProj[colProj] == CalpontSystemCatalog::TIME) { Message::Args args; args.add("variance/standard deviation"); @@ -3884,7 +3894,8 @@ void TupleAggregateStep::prep2PhasesDistinctAggregate( typeProj[colProj] == CalpontSystemCatalog::BLOB || typeProj[colProj] == CalpontSystemCatalog::TEXT || typeProj[colProj] == CalpontSystemCatalog::DATE || - typeProj[colProj] == CalpontSystemCatalog::DATETIME) + typeProj[colProj] == CalpontSystemCatalog::DATETIME || + typeProj[colProj] == CalpontSystemCatalog::TIME) { Message::Args args; args.add("sum/average"); @@ -3969,7 +3980,8 @@ void TupleAggregateStep::prep2PhasesDistinctAggregate( typeProj[colProj] == CalpontSystemCatalog::BLOB || typeProj[colProj] == CalpontSystemCatalog::TEXT || typeProj[colProj] == CalpontSystemCatalog::DATE || - typeProj[colProj] == CalpontSystemCatalog::DATETIME) + typeProj[colProj] == CalpontSystemCatalog::DATETIME || + typeProj[colProj] == CalpontSystemCatalog::TIME) { Message::Args args; args.add("variance/standard deviation"); @@ -4184,7 +4196,8 @@ void TupleAggregateStep::prep2PhasesDistinctAggregate( typeAggUm[colUm] == CalpontSystemCatalog::BLOB || typeAggUm[colUm] == CalpontSystemCatalog::TEXT || typeAggUm[colUm] == CalpontSystemCatalog::DATE || - typeAggUm[colUm] == CalpontSystemCatalog::DATETIME) + typeAggUm[colUm] == CalpontSystemCatalog::DATETIME || + typeAggUm[colUm] == CalpontSystemCatalog::TIME) { Message::Args args; args.add("sum/average"); diff --git a/dbcon/joblist/tupleconstantstep.cpp b/dbcon/joblist/tupleconstantstep.cpp index 6e75ad350..309bb0058 100644 --- a/dbcon/joblist/tupleconstantstep.cpp +++ b/dbcon/joblist/tupleconstantstep.cpp @@ -223,6 +223,7 @@ void TupleConstantStep::constructContanstRow(const JobInfo& jobInfo) case CalpontSystemCatalog::BIGINT: case CalpontSystemCatalog::DATE: case CalpontSystemCatalog::DATETIME: + case CalpontSystemCatalog::TIME: { fRowConst.setIntField(c.intVal, *i); break; diff --git a/dbcon/joblist/tupleunion.cpp b/dbcon/joblist/tupleunion.cpp index 51b5cde97..77665ae90 100644 --- a/dbcon/joblist/tupleunion.cpp +++ b/dbcon/joblist/tupleunion.cpp @@ -473,7 +473,8 @@ void TupleUnion::normalize(const Row& in, Row* out) case CalpontSystemCatalog::DATE: case CalpontSystemCatalog::DATETIME: - throw logic_error("TupleUnion::normalize(): tried to normalize an int to a date or datetime"); + case CalpontSystemCatalog::TIME: + throw logic_error("TupleUnion::normalize(): tried to normalize an int to a time, date or datetime"); case CalpontSystemCatalog::FLOAT: case CalpontSystemCatalog::UFLOAT: @@ -582,7 +583,8 @@ dec1: case CalpontSystemCatalog::DATE: case CalpontSystemCatalog::DATETIME: - throw logic_error("TupleUnion::normalize(): tried to normalize an int to a date or datetime"); + case CalpontSystemCatalog::TIME: + throw logic_error("TupleUnion::normalize(): tried to normalize an int to a time, date or datetime"); case CalpontSystemCatalog::FLOAT: case CalpontSystemCatalog::UFLOAT: @@ -736,6 +738,33 @@ dec2: break; + case CalpontSystemCatalog::TIME: + switch (out->getColTypes()[i]) + { + case CalpontSystemCatalog::TIME: + out->setIntField(in.getIntField(i), i); + break; + + case CalpontSystemCatalog::CHAR: + case CalpontSystemCatalog::TEXT: + case CalpontSystemCatalog::VARCHAR: + { + string d = DataConvert::timeToString(in.getIntField(i)); + out->setStringField(d, i); + break; + } + + default: + { + ostringstream os; + os << "TupleUnion::normalize(): tried an illegal conversion: time to " + << out->getColTypes()[i]; + throw logic_error(os.str()); + } + } + + break; + case CalpontSystemCatalog::FLOAT: case CalpontSystemCatalog::UFLOAT: case CalpontSystemCatalog::DOUBLE: @@ -1069,6 +1098,10 @@ void TupleUnion::writeNull(Row* out, uint32_t col) out->setUintField<8>(joblist::DATETIMENULL, col); break; + case CalpontSystemCatalog::TIME: + out->setUintField<8>(joblist::TIMENULL, col); + break; + case CalpontSystemCatalog::CHAR: case CalpontSystemCatalog::TEXT: case CalpontSystemCatalog::VARCHAR: diff --git a/dbcon/joblist/windowfunctionstep.cpp b/dbcon/joblist/windowfunctionstep.cpp index ed382ee4a..4d24f0b4b 100644 --- a/dbcon/joblist/windowfunctionstep.cpp +++ b/dbcon/joblist/windowfunctionstep.cpp @@ -1201,6 +1201,7 @@ boost::shared_ptr WindowFunctionStep::parseFrameBoundRows( case execplan::CalpontSystemCatalog::UDECIMAL: case execplan::CalpontSystemCatalog::DATE: case execplan::CalpontSystemCatalog::DATETIME: + case execplan::CalpontSystemCatalog::TIME: { fb.reset(new FrameBoundExpressionRow(type, id, idx)); break; @@ -1351,6 +1352,7 @@ boost::shared_ptr WindowFunctionStep::parseFrameBoundRange(const exe case execplan::CalpontSystemCatalog::UDECIMAL: case execplan::CalpontSystemCatalog::DATE: case execplan::CalpontSystemCatalog::DATETIME: + case execplan::CalpontSystemCatalog::TIME: { if (isConstant) { diff --git a/dbcon/mysql/ha_calpont_ddl.cpp b/dbcon/mysql/ha_calpont_ddl.cpp index 221155592..ba9b2c299 100644 --- a/dbcon/mysql/ha_calpont_ddl.cpp +++ b/dbcon/mysql/ha_calpont_ddl.cpp @@ -174,6 +174,10 @@ uint32_t convertDataType(int dataType) calpontDataType = CalpontSystemCatalog::DATETIME; break; + case ddlpackage::DDL_TIME: + calpontDataType = CalpontSystemCatalog::TIME; + break; + case ddlpackage::DDL_CLOB: calpontDataType = CalpontSystemCatalog::CLOB; break; diff --git a/dbcon/mysql/ha_calpont_dml.cpp b/dbcon/mysql/ha_calpont_dml.cpp index c4ca59862..57a91bd89 100644 --- a/dbcon/mysql/ha_calpont_dml.cpp +++ b/dbcon/mysql/ha_calpont_dml.cpp @@ -136,7 +136,8 @@ int buildBuffer(uchar* buf, string& buffer, int& columns, TABLE* table) (*field)->type() == MYSQL_TYPE_STRING || (*field)->type() == MYSQL_TYPE_DATE || (*field)->type() == MYSQL_TYPE_DATETIME || - (*field)->type() == MYSQL_TYPE_DATETIME2 ) + (*field)->type() == MYSQL_TYPE_DATETIME2 || + (*field)->type() == MYSQL_TYPE_TIME ) vals.append("'"); while (ptr < end_ptr) @@ -166,7 +167,8 @@ int buildBuffer(uchar* buf, string& buffer, int& columns, TABLE* table) (*field)->type() == MYSQL_TYPE_STRING || (*field)->type() == MYSQL_TYPE_DATE || (*field)->type() == MYSQL_TYPE_DATETIME || - (*field)->type() == MYSQL_TYPE_DATETIME2 ) + (*field)->type() == MYSQL_TYPE_DATETIME2 || + (*field)->type() == MYSQL_TYPE_TIME ) vals.append("'"); } } @@ -876,6 +878,37 @@ int ha_calpont_impl_write_batch_row_(uchar* buf, TABLE* table, cal_impl_if::cal_ break; } + case CalpontSystemCatalog::TIME: + { + if (nullVal && (ci.columnTypes[colpos].constraintType != CalpontSystemCatalog::NOTNULL_CONSTRAINT)) + { + fprintf(ci.filePtr, "%c", ci.delimiter); + + buf += table->field[colpos]->pack_length(); + } + else + { + MYSQL_TIME ltime; + const uchar* pos = buf; + longlong tmp = my_time_packed_from_binary(pos, table->field[colpos]->decimals()); + TIME_from_longlong_time_packed(<ime, tmp); + if (!ltime.second_part) + { + fprintf(ci.filePtr, "%02d:%02d:%02d%c", + ltime.hour, ltime.minute, ltime.second, ci.delimiter); + } + else + { + fprintf(ci.filePtr, "%02d:%02d:%02d.%ld%c", + ltime.hour, ltime.minute, ltime.second, + ltime.second_part, ci.delimiter); + } + buf += table->field[colpos]->pack_length(); + } + + break; + } + case CalpontSystemCatalog::CHAR: { if (nullVal && (ci.columnTypes[colpos].constraintType != CalpontSystemCatalog::NOTNULL_CONSTRAINT)) diff --git a/dbcon/mysql/ha_calpont_execplan.cpp b/dbcon/mysql/ha_calpont_execplan.cpp index 1079efe02..fe2f96e52 100644 --- a/dbcon/mysql/ha_calpont_execplan.cpp +++ b/dbcon/mysql/ha_calpont_execplan.cpp @@ -2657,6 +2657,11 @@ CalpontSystemCatalog::ColType colType_MysqlToIDB (const Item* item) ct.colDataType = CalpontSystemCatalog::DATETIME; ct.colWidth = 8; } + else if (item->field_type() == MYSQL_TYPE_TIME) + { + ct.colDataType = CalpontSystemCatalog::TIME; + ct.colWidth = 8; + } if (item->field_type() == MYSQL_TYPE_BLOB) { @@ -3528,6 +3533,13 @@ ReturnedColumn* buildFunctionColumn(Item_func* ifp, gp_walk_info& gwi, bool& non ct.colWidth = 4; fc->resultType(ct); } + else if (ifp->field_type() == MYSQL_TYPE_TIME) + { + CalpontSystemCatalog::ColType ct; + ct.colDataType = CalpontSystemCatalog::TIME; + ct.colWidth = 8; + fc->resultType(ct); + } #if 0 diff --git a/dbcon/mysql/ha_calpont_impl.cpp b/dbcon/mysql/ha_calpont_impl.cpp index 6bd887332..e59f7c7d5 100644 --- a/dbcon/mysql/ha_calpont_impl.cpp +++ b/dbcon/mysql/ha_calpont_impl.cpp @@ -494,6 +494,19 @@ int fetchNextRow(uchar* buf, cal_table_info& ti, cal_connection_info* ci) break; } + case CalpontSystemCatalog::TIME: + { + if ((*f)->null_ptr) + *(*f)->null_ptr &= ~(*f)->null_bit; + + intColVal = row.getUintField<8>(s); + DataConvert::timeToString(intColVal, tmp, 255); + + Field_varstring* f2 = (Field_varstring*)*f; + f2->store(tmp, strlen(tmp), f2->charset()); + break; + } + case CalpontSystemCatalog::CHAR: case CalpontSystemCatalog::VARCHAR: { diff --git a/dbcon/mysql/ha_calpont_partition.cpp b/dbcon/mysql/ha_calpont_partition.cpp index 84ea65d82..99940262b 100644 --- a/dbcon/mysql/ha_calpont_partition.cpp +++ b/dbcon/mysql/ha_calpont_partition.cpp @@ -125,6 +125,9 @@ string name(CalpontSystemCatalog::ColType& ct) case CalpontSystemCatalog::DATETIME: return "DATETIME"; + case CalpontSystemCatalog::TIME: + return "TIME"; + case CalpontSystemCatalog::DECIMAL: return "DECIMAL"; @@ -201,6 +204,7 @@ bool CP_type(CalpontSystemCatalog::ColType& ct) ct.colDataType == CalpontSystemCatalog::BIGINT || ct.colDataType == CalpontSystemCatalog::DATE || ct.colDataType == CalpontSystemCatalog::DATETIME || + ct.colDataType == CalpontSystemCatalog::TIME || ct.colDataType == CalpontSystemCatalog::DECIMAL || ct.colDataType == CalpontSystemCatalog::UTINYINT || ct.colDataType == CalpontSystemCatalog::USMALLINT || @@ -261,6 +265,9 @@ const string format(int64_t v, CalpontSystemCatalog::ColType& ct) oss << DataConvert::datetimeToString(v); break; + case CalpontSystemCatalog::TIME: + oss << DataConvert::timeToString(v); + case CalpontSystemCatalog::CHAR: case CalpontSystemCatalog::VARCHAR: { @@ -387,6 +394,10 @@ const int64_t IDB_format(char* str, CalpontSystemCatalog::ColType& ct, uint8_t& v = boost::any_cast(anyVal); break; + case CalpontSystemCatalog::TIME: + v = boost::any_cast(anyVal); + break; + case CalpontSystemCatalog::DECIMAL: case CalpontSystemCatalog::UDECIMAL: if (ct.colWidth == execplan::CalpontSystemCatalog::ONE_BYTE) diff --git a/dbcon/mysql/ha_window_function.cpp b/dbcon/mysql/ha_window_function.cpp index 7e30389c4..5724b231c 100644 --- a/dbcon/mysql/ha_window_function.cpp +++ b/dbcon/mysql/ha_window_function.cpp @@ -590,6 +590,7 @@ ReturnedColumn* buildWindowFunctionColumn(Item* item, gp_walk_info& gwi, bool& n case CalpontSystemCatalog::DATE: case CalpontSystemCatalog::DATETIME: + case CalpontSystemCatalog::TIME: if (!frm.fIsRange) boundTypeErr = true; else if (dynamic_cast(frm.fStart.fVal.get()) == NULL) @@ -641,6 +642,7 @@ ReturnedColumn* buildWindowFunctionColumn(Item* item, gp_walk_info& gwi, bool& n case CalpontSystemCatalog::DATE: case CalpontSystemCatalog::DATETIME: + case CalpontSystemCatalog::TIME: if (!frm.fIsRange) boundTypeErr = true; else if (dynamic_cast(frm.fEnd.fVal.get()) == NULL) diff --git a/primitives/linux-port/column.cpp b/primitives/linux-port/column.cpp index 879859384..f31273b1c 100644 --- a/primitives/linux-port/column.cpp +++ b/primitives/linux-port/column.cpp @@ -290,6 +290,7 @@ inline bool isEmptyVal<8>(uint8_t type, const uint8_t* ival) case CalpontSystemCatalog::VARCHAR: case CalpontSystemCatalog::DATE: case CalpontSystemCatalog::DATETIME: + case CalpontSystemCatalog::TIME: case CalpontSystemCatalog::VARBINARY: case CalpontSystemCatalog::BLOB: case CalpontSystemCatalog::TEXT: @@ -322,6 +323,7 @@ inline bool isEmptyVal<4>(uint8_t type, const uint8_t* ival) case CalpontSystemCatalog::TEXT: case CalpontSystemCatalog::DATE: case CalpontSystemCatalog::DATETIME: + case CalpontSystemCatalog::TIME: return (joblist::CHAR4EMPTYROW == *val); case CalpontSystemCatalog::UINT: @@ -347,6 +349,7 @@ inline bool isEmptyVal<2>(uint8_t type, const uint8_t* ival) case CalpontSystemCatalog::TEXT: case CalpontSystemCatalog::DATE: case CalpontSystemCatalog::DATETIME: + case CalpontSystemCatalog::TIME: return (joblist::CHAR2EMPTYROW == *val); case CalpontSystemCatalog::USMALLINT: @@ -372,6 +375,7 @@ inline bool isEmptyVal<1>(uint8_t type, const uint8_t* ival) case CalpontSystemCatalog::TEXT: case CalpontSystemCatalog::DATE: case CalpontSystemCatalog::DATETIME: + case CalpontSystemCatalog::TIME: return (*val == joblist::CHAR1EMPTYROW); case CalpontSystemCatalog::UTINYINT: @@ -402,6 +406,7 @@ inline bool isNullVal<8>(uint8_t type, const uint8_t* ival) case CalpontSystemCatalog::VARCHAR: case CalpontSystemCatalog::DATE: case CalpontSystemCatalog::DATETIME: + case CalpontSystemCatalog::TIME: case CalpontSystemCatalog::VARBINARY: case CalpontSystemCatalog::BLOB: case CalpontSystemCatalog::TEXT: @@ -438,6 +443,7 @@ inline bool isNullVal<4>(uint8_t type, const uint8_t* ival) case CalpontSystemCatalog::DATE: case CalpontSystemCatalog::DATETIME: + case CalpontSystemCatalog::TIME: return (joblist::DATENULL == *val); case CalpontSystemCatalog::UINT: @@ -463,6 +469,7 @@ inline bool isNullVal<2>(uint8_t type, const uint8_t* ival) case CalpontSystemCatalog::TEXT: case CalpontSystemCatalog::DATE: case CalpontSystemCatalog::DATETIME: + case CalpontSystemCatalog::TIME: return (joblist::CHAR2NULL == *val); case CalpontSystemCatalog::USMALLINT: @@ -488,6 +495,7 @@ inline bool isNullVal<1>(uint8_t type, const uint8_t* ival) case CalpontSystemCatalog::TEXT: case CalpontSystemCatalog::DATE: case CalpontSystemCatalog::DATETIME: + case CalpontSystemCatalog::TIME: return (*val == joblist::CHAR1NULL); case CalpontSystemCatalog::UTINYINT: @@ -548,6 +556,7 @@ inline bool isMinMaxValid(const NewColRequestHeader* in) case CalpontSystemCatalog::DATE: case CalpontSystemCatalog::BIGINT: case CalpontSystemCatalog::DATETIME: + case CalpontSystemCatalog::TIME: case CalpontSystemCatalog::UTINYINT: case CalpontSystemCatalog::USMALLINT: case CalpontSystemCatalog::UINT: diff --git a/primitives/primproc/columncommand.cpp b/primitives/primproc/columncommand.cpp index 19e5e9a1f..ed4b26a44 100644 --- a/primitives/primproc/columncommand.cpp +++ b/primitives/primproc/columncommand.cpp @@ -980,6 +980,7 @@ const uint64_t ColumnCommand::getEmptyRowValue( const execplan::CalpontSystemCat case execplan::CalpontSystemCatalog::VARCHAR : case execplan::CalpontSystemCatalog::DATE : case execplan::CalpontSystemCatalog::DATETIME : + case execplan::CalpontSystemCatalog::TIME : case execplan::CalpontSystemCatalog::VARBINARY : case execplan::CalpontSystemCatalog::BLOB : case execplan::CalpontSystemCatalog::TEXT : diff --git a/tools/pingproc/pingproc.cpp b/tools/pingproc/pingproc.cpp index cb1a7ab11..b40eec905 100644 --- a/tools/pingproc/pingproc.cpp +++ b/tools/pingproc/pingproc.cpp @@ -308,6 +308,7 @@ bool OidOperation::isIntegralDataType() DataType() == CalpontSystemCatalog::DATE || DataType() == CalpontSystemCatalog::BIGINT || DataType() == CalpontSystemCatalog::DATETIME || + DataType() == CalpontSystemCatalog::TIME || DataType() == CalpontSystemCatalog::UTINYINT || DataType() == CalpontSystemCatalog::USMALLINT || DataType() == CalpontSystemCatalog::UMEDINT || diff --git a/utils/common/nullvaluemanip.cpp b/utils/common/nullvaluemanip.cpp index 4eb96f77c..475f495cc 100644 --- a/utils/common/nullvaluemanip.cpp +++ b/utils/common/nullvaluemanip.cpp @@ -56,6 +56,9 @@ uint64_t getNullValue(CalpontSystemCatalog::ColDataType t, uint32_t colWidth) case CalpontSystemCatalog::DATETIME: return joblist::DATETIMENULL; + case CalpontSystemCatalog::TIME: + return joblist::TIMENULL; + case CalpontSystemCatalog::CHAR: case CalpontSystemCatalog::VARCHAR: case CalpontSystemCatalog::STRINT: @@ -163,6 +166,9 @@ int64_t getSignedNullValue(CalpontSystemCatalog::ColDataType t, uint32_t colWidt case CalpontSystemCatalog::DATETIME: return joblist::DATETIMENULL; + case CalpontSystemCatalog::TIME: + return joblist::TIMENULL; + case CalpontSystemCatalog::CHAR: case CalpontSystemCatalog::VARCHAR: case CalpontSystemCatalog::STRINT: diff --git a/utils/dataconvert/dataconvert.cpp b/utils/dataconvert/dataconvert.cpp index 6e8a33dc1..b5b1ae619 100644 --- a/utils/dataconvert/dataconvert.cpp +++ b/utils/dataconvert/dataconvert.cpp @@ -859,6 +859,153 @@ bool mysql_str_to_datetime( const string& input, DateTime& output, bool& isDate return true; } +bool mysql_str_to_time( const string& input, Time& output ) +{ + int32_t datesepct = 0; + uint32_t dtend = 0; + + /** + * We need to deal with the time portion. + * The rules are: + * - Time portion may be empty + * - Time portion may start with 'T' + * - Time portion always ends with '\0' + * - Time portion always starts with hour + * - Without time separators (':'): + * HHMMSS + * - All Times can end with option .[microseconds] + * - With time separators there are no specific field length + * requirements + */ + while ( input[dtend] == ' ' && dtend < input.length() ) + { + ++dtend; + } + + if ( dtend == input.length() ) + { + return false; + } + + uint32_t timesep_ct = 0; + bool has_usec = false; + uint32_t len_before_msec = 0; + uint32_t tmstart = ( input[dtend] == ' ' || input[dtend] == 'T' ) ? dtend + 1 : dtend; + uint32_t tmend = tmstart; + + for ( ; tmend < input.length(); ++tmend ) + { + char c = input[tmend]; + + if ( isdigit( c ) ) + { + // digits always ok + continue; + } +// else if( c == ':' ) +// { +// timesep_ct++; +// } +// else if( c == '.' ) +// { +// len_before_msec = ( tmend - tmstart ); +// has_usec = true; +// } + else if ( ispunct(c) ) + { + if ( c == '.' && timesep_ct == 2 ) + { + len_before_msec = ( tmend - tmstart ); + has_usec = true; + } + else + { + timesep_ct++; + } + } + else + { + // some other character showed up + output.reset(); + return false; + } + } + + if ( !len_before_msec ) + len_before_msec = ( tmend - tmstart ); + + int32_t hour = -1; + int32_t min = 0; + int32_t sec = 0; + int32_t usec = 0; + const char* tstart = input.c_str() + tmstart; + + if ( timesep_ct == 2 ) + { + readDecimal(tstart, hour); + ++tstart; // skip one separator + readDecimal(tstart, min); + ++tstart; // skip one separator + readDecimal(tstart, sec); + } + else if ( timesep_ct == 1 ) + { + readDecimal(tstart, hour); + ++tstart; // skip one separator + readDecimal(tstart, min); + } + else if ( timesep_ct == 0 && len_before_msec == 6 ) + { + readDecimal(tstart, hour, 2); + readDecimal(tstart, min, 2); + readDecimal(tstart, sec, 2); + } + else if ( timesep_ct == 0 && len_before_msec == 4 ) + { + readDecimal(tstart, hour, 2); + readDecimal(tstart, min, 2); + } + else if ( timesep_ct == 0 && len_before_msec == 2 ) + { + readDecimal(tstart, hour, 2); + } + else + { + output.reset(); + return false; + } + + if ( has_usec ) + { + ++tstart; // skip '.' character. We could error check if we wanted to + uint32_t numread = readDecimal(tstart, usec); + + if ( numread > 6 || numread < 1 ) + { + // don't allow more than 6 digits when specifying microseconds + output.reset(); + return false; + } + + // usec have to be scaled up so that it always represents microseconds + for ( int i = numread; i < 6; i++ ) + usec *= 10; + } + + if ( !isTimeValid( hour, min, sec, usec ) ) + { + output.reset(); + return false; + } + + output.hour = hour; + output.minute = min; + output.second = sec; + output.msecond = usec; + return true; +} + + bool stringToDateStruct( const string& data, Date& date ) { bool isDate; @@ -894,6 +1041,14 @@ bool stringToDatetimeStruct(const string& data, DateTime& dtime, bool* date) return true; } +bool stringToTimeStruct(const string& data, Time& dtime) +{ + if ( !mysql_str_to_time( data, dtime ) ) + return false; + + return true; +} + boost::any DataConvert::convertColumnData(const CalpontSystemCatalog::ColType& colType, const std::string& dataOrig, bool& pushWarning, bool nulFlag, bool noRoundup, bool isUpdate ) @@ -1229,6 +1384,23 @@ DataConvert::convertColumnData(const CalpontSystemCatalog::ColType& colType, } break; + case CalpontSystemCatalog::TIME: + { + Time aTime; + + if (stringToTimeStruct(data, aTime)) + { + value = (int64_t) *(reinterpret_cast(&aTime)); + } + else + { + value = (int64_t) 0; + pushWarning = true; + } + } + break; + + case CalpontSystemCatalog::BLOB: case CalpontSystemCatalog::CLOB: value = data; @@ -1345,6 +1517,13 @@ DataConvert::convertColumnData(const CalpontSystemCatalog::ColType& colType, } break; + case CalpontSystemCatalog::TIME: + { + uint64_t d = joblist::TIMENULL; + value = d; + } + break; + case CalpontSystemCatalog::CHAR: { std::string charnull; @@ -1692,6 +1871,91 @@ int64_t DataConvert::convertColumnDatetime( return value; } +//------------------------------------------------------------------------------ +// Convert time string to binary time. Used by BulkLoad. +// Most of this is taken from str_to_time in sql-common/my_time.c +//------------------------------------------------------------------------------ +int64_t DataConvert::convertColumnTime( + const char* dataOrg, + CalpontDateTimeFormat datetimeFormat, + int& status, + unsigned int dataOrgLen ) +{ + status = 0; + const char* p; + p = dataOrg; + char fld[10]; + int16_t value = 0; + int inYear, inMonth, inDay, inHour, inMinute, inSecond, inMicrosecond; + inHour = 0; + inMinute = 0; + inSecond = 0; + inMicrosecond = 0; + if ( datetimeFormat != CALPONTTIME_ENUM ) + { + status = -1; + return value; + } + + memcpy( fld, p, 2); + fld[2] = '\0'; + + inHour = strtol(fld, 0, 10); + + if (!isdigit(p[2]) || !isdigit(p[3])) + { + status = -1; + return value; + } + + memcpy( fld, p + 2, 2); + fld[2] = '\0'; + + inMinute = strtol(fld, 0, 10); + + if (!isdigit(p[4]) || !isdigit(p[5])) + { + status = -1; + return value; + } + + memcpy( fld, p + 4, 2); + fld[2] = '\0'; + + inSecond = strtol(fld, 0, 10); + + if (dataOrgLen > 9) + { + unsigned int microFldLen = dataOrgLen - 9; + + if (microFldLen > (sizeof(fld) - 1)) + microFldLen = sizeof(fld) - 1; + + memcpy( fld, p + 9, microFldLen); + fld[microFldLen] = '\0'; + inMicrosecond = strtol(fld, 0, 10); + } + + if ( isTimeValid (inHour, inMinute, inSecond, inMicrosecond) ) + { + Time atime; + atime.hour = inHour; + atime.minute = inMinute; + atime.second = inSecond; + atime.msecond = inMicrosecond; + + memcpy( &value, &atime, 8); + } + else + { + status = -1; + } + + return value; + +} + + //------------------------------------------------------------------------------ // Verify that specified datetime is valid //------------------------------------------------------------------------------ @@ -1706,6 +1970,14 @@ bool DataConvert::isColumnDateTimeValid( int64_t dateTime ) return false; } +bool DataConvert::isColumnTimeValid( int64_t time ) +{ + Time dt; + memcpy(&dt, &time, sizeof(uint64_t)); + + return isTimeValid(dt.hour, dt.minute, dt.second, dt.msecond); +} + std::string DataConvert::dateToString( int datevalue ) { // @bug 4703 abandon multiple ostringstream's for conversion @@ -1742,6 +2014,32 @@ std::string DataConvert::datetimeToString( long long datetimevalue, long decima return buf; } +std::string DataConvert::timeToString( long long timevalue, long decimals ) +{ + // 10 is default which means we don't need microseconds + if (decimals > 6 || decimals < 0) + { + decimals = 0; + } + // @bug 4703 abandon multiple ostringstream's for conversion + Time dt(timevalue); + const int TIMETOSTRING_LEN = 19; // (-H)HH:MM:SS.mmmmmm\0 + char buf[TIMETOSTRING_LEN]; + + sprintf(buf, "%02d:%02d:%02d", dt.hour, dt.minute, dt.second); + if (dt.msecond && decimals) + { + size_t start = strlen(buf); + snprintf(buf + strlen(buf), 12 + decimals, ".%d", dt.msecond); + // Pad end with zeros + if (strlen(buf) - start < decimals) + { + sprintf(buf + strlen(buf), "%0*d", decimals - (strlen(buf) - start), 0); + } + } + return buf; +} + std::string DataConvert::dateToString1( int datevalue ) { // @bug 4703 abandon multiple ostringstream's for conversion @@ -1763,6 +2061,18 @@ std::string DataConvert::datetimeToString1( long long datetimevalue ) sprintf(buf, "%04d%02d%02d%02d%02d%02d%06d", dt.year, dt.month, dt.day, dt.hour, dt.minute, dt.second, dt.msecond); return buf; } + +std::string DataConvert::timeToString1( long long datetimevalue ) +{ + // @bug 4703 abandon multiple ostringstream's for conversion + DateTime dt(datetimevalue); + const int TIMETOSTRING1_LEN = 14; // HHMMSSmmmmmm\0 + char buf[TIMETOSTRING1_LEN]; + + sprintf(buf, "%02d%02d%02d%06d", dt.hour, dt.minute, dt.second, dt.msecond); + return buf; +} + #if 0 bool DataConvert::isNullData(ColumnResult* cr, int rownum, CalpontSystemCatalog::ColType colType) { @@ -1983,6 +2293,11 @@ int64_t DataConvert::datetimeToInt(const string& datetime) return stringToDatetime(datetime); } +int64_t DataConvert::timeToInt(const string& time) +{ + return stringToTime(time); +} + int64_t DataConvert::stringToDate(const string& data) { Date aDay; @@ -2256,6 +2571,66 @@ int64_t DataConvert::intToDatetime(int64_t data, bool* date) return *(reinterpret_cast(&adaytime)); } +int64_t DataConvert::intToTime(int64_t data) +{ + char buf[21] = {0}; + Time atime; + + if (data == 0) + { + atime.hour = 0; + atime.minute = 0; + atime.second = 0; + atime.msecond = 0; + + return *(reinterpret_cast(&atime)); + } + + snprintf( buf, 15, "%llu", (long long unsigned int)data); + //string date = buf; + string hour, min, sec, msec; + int64_t h = 0, minute = 0, s = 0, ms = 0; + + switch (strlen(buf)) + { + case 6: + hour = string(buf, 2); + min = string(buf + 2, 2); + sec = string(buf + 4, 2); + msec = string(buf + 6, 6); + break; + + case 4: + min = string(buf, 2); + sec = string(buf + 2, 2); + msec = string(buf + 4, 6); + break; + + case 2: + sec = string(buf, 2); + msec = string(buf + 2, 6); + break; + + default: + return -1; + } + + h = atoi(hour.c_str()); + minute = atoi(min.c_str()); + s = atoi(sec.c_str()); + ms = atoi(msec.c_str()); + + if (!isTimeValid(h, minute, s, ms)) + return -1; + + atime.hour = h; + atime.minute = minute; + atime.second = s; + atime.msecond = ms; + + return *(reinterpret_cast(&atime)); +} + int64_t DataConvert::stringToTime(const string& data) { // MySQL supported time value format 'D HHH:MM:SS.fraction' @@ -2402,6 +2777,7 @@ CalpontSystemCatalog::ColType DataConvert::convertUnionColType(vector> 40) & 0xfff), day((val >> 52) & 0xfff) {} + + Time(signed d, signed h, signed min, signed sec, signed msec) : + msecond(sec), second(sec), minute(min), hour(h), day(d) {} + + int64_t convertToMySQLint() const; + void reset(); }; +inline +void Time::reset() +{ + msecond = 0xFFFFFE; + second = 0xFF; + minute = 0xFF; + hour = 0xFFF; + day = 0xFFF; +} + +inline +int64_t Time::convertToMySQLint() const +{ + return (int64_t) (hour * 10000000000) + (minute * 100000000) + (second * 1000000) + msecond; +} + static uint32_t daysInMonth[13] = {31, 28, 31, 30, 31, 30, 31, 31, 30, 31, 30, 31, 0}; inline uint32_t getDaysInMonth(uint32_t month) @@ -281,6 +304,28 @@ bool isDateTimeValid ( int hour, int minute, int second, int microSecond) return valid; } +inline +bool isTimeValid ( int hour, int minute, int second, int microSecond) +{ + bool valid = false; + + if ( hour >= -838 && hour <= 838 ) + { + if ( minute >= 0 && minute < 60 ) + { + if ( second >= 0 && second < 60 ) + { + if ( microSecond >= 0 && microSecond <= 999999 ) + { + valid = true; + } + } + } + } + + return valid; +} + inline int64_t string_to_ll( const std::string& data, bool& bSaturate ) { @@ -364,6 +409,15 @@ public: EXPORT static std::string datetimeToString( long long datetimevalue, long decimals = 0 ); static inline void datetimeToString( long long datetimevalue, char* buf, unsigned int buflen ); + /** + * @brief convert a columns data from native format to a string + * + * @param type the columns database type + * @param data the columns string representation of it's data + */ + EXPORT static std::string timeToString( long long timevalue, long decimals = 0 ); + static inline void timeToString( long long timevalue, char* buf, unsigned int buflen ); + /** * @brief convert a columns data from native format to a string * @@ -382,6 +436,15 @@ public: EXPORT static std::string datetimeToString1( long long datetimevalue ); static inline void datetimeToString1( long long datetimevalue, char* buf, unsigned int buflen ); + /** + * @brief convert a columns data from native format to a string + * + * @param type the columns database type + * @param data the columns string representation of it's data + */ + EXPORT static std::string timeToString1( long long timevalue ); + static inline void timeToString1( long long timevalue, char* buf, unsigned int buflen ); + /** * @brief convert a date column data, represnted as a string, to it's native * format. This function is for bulkload to use. @@ -415,10 +478,25 @@ public: CalpontDateTimeFormat datetimeFormat, int& status, unsigned int dataOrgLen ); + /** + * @brief convert a time column data, represented as a string, + * to it's native format. This function is for bulkload to use. + * + * @param type the columns data type + * @param dataOrig the columns string representation of it's data + * @param timeFormat the format the time value in + * @param status 0 - success, -1 - fail + * @param dataOrgLen length specification of dataOrg + */ + EXPORT static int64_t convertColumnTime( const char* dataOrg, + CalpontDateTimeFormat datetimeFormat, + int& status, unsigned int dataOrgLen ); + /** * @brief Is specified datetime valid; used by binary bulk load */ EXPORT static bool isColumnDateTimeValid( int64_t dateTime ); + EXPORT static bool isColumnTimeValid( int64_t time ); EXPORT static bool isNullData(execplan::ColumnResult* cr, int rownum, execplan::CalpontSystemCatalog::ColType colType); static inline std::string decimalToString(int64_t value, uint8_t scale, execplan::CalpontSystemCatalog::ColDataType colDataType); @@ -438,11 +516,13 @@ public: EXPORT static int64_t intToDate(int64_t data); // convert integer to datetime EXPORT static int64_t intToDatetime(int64_t data, bool* isDate = NULL); - + // convert integer to date + EXPORT static int64_t intToTime(int64_t data); // convert string to date. alias to stringToDate EXPORT static int64_t dateToInt(const std::string& date); // convert string to datetime. alias to datetimeToInt EXPORT static int64_t datetimeToInt(const std::string& datetime); + EXPORT static int64_t timeToInt(const std::string& time); EXPORT static int64_t stringToTime (const std::string& data); // bug4388, union type conversion EXPORT static execplan::CalpontSystemCatalog::ColType convertUnionColType(std::vector&); @@ -484,6 +564,27 @@ inline void DataConvert::datetimeToString( long long datetimevalue, char* buf, u } } +inline void DataConvert::timeToString( long long timevalue, char* buf, unsigned int buflen ) +{ + if ((timevalue & 0xffffff) > 0) + { + snprintf( buf, buflen, "%02d:%02d:%02d.%d", + (unsigned)((timevalue >> 40) & 0xfff), + (unsigned)((timevalue >> 32) & 0xff), + (unsigned)((timevalue >> 24) & 0xff), + (unsigned)((timevalue) & 0xffffff) + ); + } + else + { + snprintf( buf, buflen, "%02d:%02d:%02d", + (unsigned)((timevalue >> 40) & 0xfff), + (unsigned)((timevalue >> 32) & 0xff), + (unsigned)((timevalue >> 24) & 0xff) + ); + } +} + inline void DataConvert::dateToString1( int datevalue, char* buf, unsigned int buflen) { snprintf( buf, buflen, "%04d%02d%02d", @@ -505,6 +606,15 @@ inline void DataConvert::datetimeToString1( long long datetimevalue, char* buf, ); } +inline void DataConvert::timeToString1( long long timevalue, char* buf, unsigned int buflen ) +{ + snprintf( buf, buflen, "%02d%02d%02d", + (unsigned)((timevalue >> 40) & 0xfff), + (unsigned)((timevalue >> 32) & 0xff), + (unsigned)((timevalue >> 14) & 0xff) + ); +} + inline std::string DataConvert::decimalToString(int64_t value, uint8_t scale, execplan::CalpontSystemCatalog::ColDataType colDataType) { char buf[80]; diff --git a/utils/funcexp/func_add_time.cpp b/utils/funcexp/func_add_time.cpp index fb4469eec..6fa7a5c19 100644 --- a/utils/funcexp/func_add_time.cpp +++ b/utils/funcexp/func_add_time.cpp @@ -138,6 +138,50 @@ int64_t addTime(DateTime& dt1, Time& dt2) return *(reinterpret_cast(&dt)); } + +int64_t addTime(Time& dt1, Time& dt2) +{ + Time dt; + dt.hour = 0; + dt.minute = 0; + dt.second = 0; + dt.msecond = 0; + + int64_t hour, min, sec, msec, tmp; + msec = (signed)(dt1.msecond + dt2.msecond); + dt.msecond = tmp = msec % 1000000; + + if (tmp < 0) + { + dt.msecond = tmp + 1000000; + dt2.second--; + } + + sec = (signed)(dt1.second + dt2.second + msec / 1000000); + dt.second = tmp = sec % 60; + + if (tmp < 0) + { + dt.second = tmp + 60; + dt2.minute--; + } + + min = (signed)(dt1.minute + dt2.minute + sec / 60); + dt.minute = tmp = min % 60; + + if (tmp < 0) + { + dt.minute = tmp + 60; + dt2.hour--; + } + + hour = (signed)(dt1.hour + dt2.hour + min / 60); + dt.hour = tmp = hour % 838; + + return *(reinterpret_cast(&dt)); +} + + } namespace funcexp @@ -254,6 +298,86 @@ int64_t Func_add_time::getDatetimeIntVal(rowgroup::Row& row, return addTime(dt1, t2); } +int64_t Func_add_time::getTimeIntVal(rowgroup::Row& row, + FunctionParm& parm, + bool& isNull, + CalpontSystemCatalog::ColType& ct) +{ + int64_t val1 = parm[0]->data()->getTimeIntVal(row, isNull); + + if (isNull) + return -1; + + const string& val2 = parm[1]->data()->getStrVal(row, isNull); + int sign = parm[2]->data()->getIntVal(row, isNull); + Time dt1; + dt1.hour = (val1 >> 40) & 0xff; + dt1.minute = (val1 >> 32) & 0xff; + dt1.second = (val1 >> 24) & 0xff; + dt1.msecond = val1 & 0xffffff; + + int64_t time = DataConvert::stringToTime(val2); + + if (time == -1) + { + isNull = true; + return -1; + } + + Time t2 = *(reinterpret_cast(&time)); + + // MySQL TIME type range '-838:59:59' and '838:59:59' + if (t2.minute > 59 || t2.second > 59 || t2.msecond > 999999) + { + isNull = true; + return -1; + } + + int val_sign = 1; + + if (t2.day != 0 && t2.hour < 0) + { + isNull = true; + return -1; + } + else if (t2.day < 0 || t2.hour < 0) + { + val_sign = -1; + } + + if ((abs(t2.day) * 24 + abs(t2.hour)) > 838) + { + t2.hour = 838; + t2.minute = 59; + t2.second = 59; + t2.msecond = 999999; + } + else + { + t2.hour = abs(t2.day) * 24 + t2.hour; + } + + t2.day = 0; + + if (val_sign * sign < 0) + { + t2.hour = -abs(t2.hour); + t2.minute = -abs(t2.minute); + t2.second = -abs(t2.second); + t2.msecond = -abs(t2.msecond); + } + else + { + t2.hour = abs(t2.hour); + t2.minute = abs(t2.minute); + t2.second = abs(t2.second); + t2.msecond = abs(t2.msecond); + } + + return addTime(dt1, t2); +} + + } // namespace funcexp // vim:ts=4 sw=4: diff --git a/utils/funcexp/func_between.cpp b/utils/funcexp/func_between.cpp index af950cc95..8398593c4 100644 --- a/utils/funcexp/func_between.cpp +++ b/utils/funcexp/func_between.cpp @@ -156,6 +156,24 @@ inline bool getBool(rowgroup::Row& row, numericLE(val, pm[2]->data()->getDatetimeIntVal(row, isNull)); } + case execplan::CalpontSystemCatalog::TIME: + { + int64_t val = pm[0]->data()->getTimeIntVal(row, isNull); + + if (notBetween) + { + if (!numericGE(val, pm[1]->data()->getTimeIntVal(row, isNull)) && !isNull) + return true; + + isNull = false; + return (!numericLE(val, pm[2]->data()->getTimeIntVal(row, isNull)) && !isNull); + } + + return !isNull && + numericGE(val, pm[1]->data()->getTimeIntVal(row, isNull)) && + numericLE(val, pm[2]->data()->getTimeIntVal(row, isNull)); + } + case execplan::CalpontSystemCatalog::DOUBLE: case execplan::CalpontSystemCatalog::UDOUBLE: case execplan::CalpontSystemCatalog::FLOAT: diff --git a/utils/funcexp/func_bitand.cpp b/utils/funcexp/func_bitand.cpp index 89aadf39b..eceb4e4c0 100644 --- a/utils/funcexp/func_bitand.cpp +++ b/utils/funcexp/func_bitand.cpp @@ -141,7 +141,8 @@ int64_t Func_bitand::getIntVal(Row& row, day = 0, hour = 0, min = 0, - sec = 0; + sec = 0, + msec = 0; year = (uint32_t)((time >> 48) & 0xffff); month = (uint32_t)((time >> 44) & 0xf); @@ -149,9 +150,29 @@ int64_t Func_bitand::getIntVal(Row& row, hour = (uint32_t)((time >> 32) & 0x3f); min = (uint32_t)((time >> 26) & 0x3f); sec = (uint32_t)((time >> 20) & 0x3f); + msec = (uint32_t)(time & 0xfffff); // return (int64_t) (year*1000000000000)+(month*100000000)+(day*1000000)+(hour*10000)+(min*100)+sec; - values.push_back((month * 100000000) + (day * 1000000) + (hour * 10000) + (min * 100) + sec); + values.push_back((month * 100000000000000) + (day * 1000000000000) + (hour * 10000000000) + (min * 100000000) + (sec * 1000000) + msec); + } + break; + + case execplan::CalpontSystemCatalog::TIME: + { + int64_t time = parm[i]->data()->getTimeIntVal(row, isNull); + + int32_t hour = 0, + min = 0, + sec = 0, + msec = 0; + + hour = (uint32_t)((time >> 40) & 0xfff); + min = (uint32_t)((time >> 32) & 0xff); + sec = (uint32_t)((time >> 24) & 0xff); + msec = (uint32_t)(time & 0xffffff); + + // return (int64_t) (year*1000000000000)+(month*100000000)+(day*1000000)+(hour*10000)+(min*100)+sec; + values.push_back((hour * 10000000000) + (min * 100000000) + (sec * 1000000) + msec); } break; diff --git a/utils/funcexp/func_bitwise.cpp b/utils/funcexp/func_bitwise.cpp index 84c83f231..752c28561 100644 --- a/utils/funcexp/func_bitwise.cpp +++ b/utils/funcexp/func_bitwise.cpp @@ -143,6 +143,15 @@ bool getUIntValFromParm( } break; + case execplan::CalpontSystemCatalog::TIME: + { + int64_t time = parm->data()->getTimeIntVal(row, isNull); + + Time dt(time); + value = dt.convertToMySQLint(); + } + break; + default: { return false; diff --git a/utils/funcexp/func_case.cpp b/utils/funcexp/func_case.cpp index 4eec004ca..46a479491 100644 --- a/utils/funcexp/func_case.cpp +++ b/utils/funcexp/func_case.cpp @@ -66,6 +66,7 @@ inline uint64_t simple_case_cmp(Row& row, case execplan::CalpontSystemCatalog::BIGINT: case execplan::CalpontSystemCatalog::DATE: case execplan::CalpontSystemCatalog::DATETIME: + case execplan::CalpontSystemCatalog::TIME: { int64_t ev = parm[n]->data()->getIntVal(row, isNull); @@ -503,6 +504,19 @@ int64_t Func_simple_case::getDatetimeIntVal(rowgroup::Row& row, } +int64_t Func_simple_case::getTimeIntVal(rowgroup::Row& row, + FunctionParm& parm, + bool& isNull, + execplan::CalpontSystemCatalog::ColType& op_ct) +{ + uint64_t i = simple_case_cmp(row, parm, isNull, op_ct); + + if (isNull) + return joblist::TIMENULL; + + return parm[i + 1]->data()->getTimeIntVal(row, isNull); +} + // searched CASE: // SELECT CASE @@ -629,5 +643,18 @@ int64_t Func_searched_case::getDatetimeIntVal(rowgroup::Row& row, } +int64_t Func_searched_case::getTimeIntVal(rowgroup::Row& row, + FunctionParm& parm, + bool& isNull, + execplan::CalpontSystemCatalog::ColType& op_ct) +{ + uint64_t i = simple_case_cmp(row, parm, isNull, op_ct); + + if (isNull) + return joblist::TIMENULL; + + return parm[i + 1]->data()->getTimeIntVal(row, isNull); +} + } // namespace funcexp // vim:ts=4 sw=4: diff --git a/utils/funcexp/func_cast.cpp b/utils/funcexp/func_cast.cpp index 9d2423293..3542e341d 100644 --- a/utils/funcexp/func_cast.cpp +++ b/utils/funcexp/func_cast.cpp @@ -191,6 +191,15 @@ int64_t Func_cast_signed::getIntVal(Row& row, } break; + case execplan::CalpontSystemCatalog::TIME: + { + int64_t time = parm[0]->data()->getTimeIntVal(row, isNull); + + Time dt(time); + return dt.convertToMySQLint(); + } + break; + default: { std::ostringstream oss; @@ -313,6 +322,15 @@ uint64_t Func_cast_unsigned::getUintVal(Row& row, } break; + case execplan::CalpontSystemCatalog::TIME: + { + int64_t time = parm[0]->data()->getTimeIntVal(row, isNull); + + Time dt(time); + return dt.convertToMySQLint(); + } + break; + default: { std::ostringstream oss; @@ -805,6 +823,85 @@ int64_t Func_cast_datetime::getDatetimeIntVal(rowgroup::Row& row, return -1; } + +int64_t Func_cast_datetime::getTimeIntVal(rowgroup::Row& row, + FunctionParm& parm, + bool& isNull, + execplan::CalpontSystemCatalog::ColType& operationColType) +{ + int64_t val; + + switch (parm[0]->data()->resultType().colDataType) + { + case execplan::CalpontSystemCatalog::BIGINT: + case execplan::CalpontSystemCatalog::INT: + case execplan::CalpontSystemCatalog::MEDINT: + case execplan::CalpontSystemCatalog::TINYINT: + case execplan::CalpontSystemCatalog::SMALLINT: + case execplan::CalpontSystemCatalog::UBIGINT: + case execplan::CalpontSystemCatalog::UINT: + case execplan::CalpontSystemCatalog::UMEDINT: + case execplan::CalpontSystemCatalog::UTINYINT: + case execplan::CalpontSystemCatalog::USMALLINT: + { + val = dataconvert::DataConvert::intToTime(parm[0]->data()->getIntVal(row, isNull)); + + if (val == -1) + isNull = true; + else + return val; + + break; + } + + case execplan::CalpontSystemCatalog::DECIMAL: + case execplan::CalpontSystemCatalog::UDECIMAL: + { + val = dataconvert::DataConvert::intToTime(parm[0]->data()->getIntVal(row, isNull)); + + if (val == -1) + isNull = true; + else + return val; + + break; + } + + case execplan::CalpontSystemCatalog::VARCHAR: + case execplan::CalpontSystemCatalog::CHAR: + case execplan::CalpontSystemCatalog::TEXT: + { + val = dataconvert::DataConvert::stringToTime(parm[0]->data()->getStrVal(row, isNull)); + + if (val == -1) + isNull = true; + else + return val; + + break; + } + + case execplan::CalpontSystemCatalog::DATE: + { + return parm[0]->data()->getTimeIntVal(row, isNull); + } + + case execplan::CalpontSystemCatalog::DATETIME: + { + return parm[0]->data()->getTimeIntVal(row, isNull); + } + + default: + { + isNull = true; + } + } + + return -1; +} + + + // // Func_cast_decimal // @@ -1138,6 +1235,25 @@ IDB_Decimal Func_cast_decimal::getDecimalVal(Row& row, } break; + case execplan::CalpontSystemCatalog::TIME: + { + int32_t s = 0; + + string value = dataconvert::DataConvert::timeToString1(parm[0]->data()->getTimeIntVal(row, isNull)); + + //strip off micro seconds + string date = value.substr(0, 14); + + int64_t x = atoll(date.c_str()); + + if (!isNull) + { + decimal.value = x; + decimal.scale = s; + } + } + break; + default: { std::ostringstream oss; diff --git a/utils/funcexp/func_ceil.cpp b/utils/funcexp/func_ceil.cpp index 7c1cafdfb..7c2f7f571 100644 --- a/utils/funcexp/func_ceil.cpp +++ b/utils/funcexp/func_ceil.cpp @@ -152,6 +152,15 @@ int64_t Func_ceil::getIntVal(Row& row, } break; + case CalpontSystemCatalog::TIME: + { + Time dt(parm[0]->data()->getTimeIntVal(row, isNull)); + + if (!isNull) + ret = dt.convertToMySQLint(); + } + break; + default: { std::ostringstream oss; @@ -233,6 +242,15 @@ uint64_t Func_ceil::getUintVal(Row& row, } break; + case CalpontSystemCatalog::TIME: + { + Time dt(parm[0]->data()->getTimeIntVal(row, isNull)); + + if (!isNull) + ret = dt.convertToMySQLint(); + } + break; + default: { std::ostringstream oss; diff --git a/utils/funcexp/func_char_length.cpp b/utils/funcexp/func_char_length.cpp index 9421554f8..b47afd58a 100644 --- a/utils/funcexp/func_char_length.cpp +++ b/utils/funcexp/func_char_length.cpp @@ -101,6 +101,12 @@ int64_t Func_char_length::getIntVal(rowgroup::Row& row, return (int64_t)date.size(); } + case execplan::CalpontSystemCatalog::TIME: + { + string date = dataconvert::DataConvert::timeToString(parm[0]->data()->getTimeIntVal(row, isNull)); + return (int64_t)date.size(); + } + default: { std::ostringstream oss; diff --git a/utils/funcexp/func_coalesce.cpp b/utils/funcexp/func_coalesce.cpp index 33a1312bc..30091a81e 100644 --- a/utils/funcexp/func_coalesce.cpp +++ b/utils/funcexp/func_coalesce.cpp @@ -139,6 +139,30 @@ int64_t Func_coalesce::getDatetimeIntVal(rowgroup::Row& row, return val; } +int64_t Func_coalesce::getTimeIntVal(rowgroup::Row& row, + FunctionParm& parm, + bool& isNull, + CalpontSystemCatalog::ColType& ct) +{ + int64_t val = 0; + + for (uint32_t i = 0; i < parm.size(); i++) + { + val = parm[i]->data()->getTimeIntVal(row, isNull); + + if (isNull) + { + isNull = false; + continue; + } + + return val; + } + + isNull = true; + return val; +} + double Func_coalesce::getDoubleVal(rowgroup::Row& row, FunctionParm& parm, bool& isNull, diff --git a/utils/funcexp/func_extract.cpp b/utils/funcexp/func_extract.cpp index 08a545fde..d5a6cc5b5 100644 --- a/utils/funcexp/func_extract.cpp +++ b/utils/funcexp/func_extract.cpp @@ -154,6 +154,7 @@ int64_t Func_extract::getIntVal(rowgroup::Row& row, { case CalpontSystemCatalog::DATE: case CalpontSystemCatalog::DATETIME: + case CalpontSystemCatalog::TIME: time = parm[0]->data()->getDatetimeIntVal(row, isNull); break; diff --git a/utils/funcexp/func_floor.cpp b/utils/funcexp/func_floor.cpp index ad47dc988..eb05b405f 100644 --- a/utils/funcexp/func_floor.cpp +++ b/utils/funcexp/func_floor.cpp @@ -151,6 +151,19 @@ int64_t Func_floor::getIntVal(Row& row, } break; + case execplan::CalpontSystemCatalog::TIME: + { + string str = + DataConvert::timeToString1(parm[0]->data()->getTimeIntVal(row, isNull)); + + // strip off micro seconds + str = str.substr(0, 14); + + if (!isNull) + ret = atoll(str.c_str()); + } + break; + default: { std::ostringstream oss; @@ -236,6 +249,19 @@ uint64_t Func_floor::getUintVal(Row& row, } break; + case execplan::CalpontSystemCatalog::TIME: + { + string str = + DataConvert::timeToString1(parm[0]->data()->getTimeIntVal(row, isNull)); + + // strip off micro seconds + str = str.substr(0, 14); + + if (!isNull) + ret = strtoull(str.c_str(), NULL, 10); + } + break; + default: { std::ostringstream oss; diff --git a/utils/funcexp/func_from_unixtime.cpp b/utils/funcexp/func_from_unixtime.cpp index 183a75a56..9968ba1f0 100644 --- a/utils/funcexp/func_from_unixtime.cpp +++ b/utils/funcexp/func_from_unixtime.cpp @@ -148,6 +148,22 @@ int64_t Func_from_unixtime::getDatetimeIntVal(rowgroup::Row& row, return *reinterpret_cast(&dt); } +int64_t Func_from_unixtime::getTimeIntVal(rowgroup::Row& row, + FunctionParm& parm, + bool& isNull, + CalpontSystemCatalog::ColType& ct) +{ + DateTime dt = getDateTime(row, parm, isNull); + + if (*reinterpret_cast(&dt) == 0) + { + isNull = true; + return 0; + } + + return *reinterpret_cast(&dt); +} + int64_t Func_from_unixtime::getIntVal(rowgroup::Row& row, FunctionParm& parm, bool& isNull, diff --git a/utils/funcexp/func_greatest.cpp b/utils/funcexp/func_greatest.cpp index 8309456ae..50ff4204a 100644 --- a/utils/funcexp/func_greatest.cpp +++ b/utils/funcexp/func_greatest.cpp @@ -206,6 +206,26 @@ int64_t Func_greatest::getDatetimeIntVal(rowgroup::Row& row, return greatestStr; } +int64_t Func_greatest::getTimeIntVal(rowgroup::Row& row, + FunctionParm& fp, + bool& isNull, + execplan::CalpontSystemCatalog::ColType& ct) +{ + int64_t str = fp[0]->data()->getTimeIntVal(row, isNull); + + int64_t greatestStr = str; + + for (uint32_t i = 1; i < fp.size(); i++) + { + int64_t str1 = fp[i]->data()->getTimeIntVal(row, isNull); + + if ( greatestStr < str1 ) + greatestStr = str1; + } + + return greatestStr; +} + } // namespace funcexp // vim:ts=4 sw=4: diff --git a/utils/funcexp/func_hour.cpp b/utils/funcexp/func_hour.cpp index f5058a084..b9e3da9ec 100644 --- a/utils/funcexp/func_hour.cpp +++ b/utils/funcexp/func_hour.cpp @@ -108,6 +108,12 @@ int64_t Func_hour::getIntVal(rowgroup::Row& row, break; } + case execplan::CalpontSystemCatalog::TIME: + { + val = parm[0]->data()->getDatetimeIntVal(row, isNull); + break; + } + default: { isNull = true; diff --git a/utils/funcexp/func_if.cpp b/utils/funcexp/func_if.cpp index 378c6d74a..4cc8cd164 100644 --- a/utils/funcexp/func_if.cpp +++ b/utils/funcexp/func_if.cpp @@ -75,6 +75,7 @@ bool boolVal(SPTP& parm, Row& row) case CalpontSystemCatalog::UINT: case CalpontSystemCatalog::DATE: case CalpontSystemCatalog::DATETIME: + case CalpontSystemCatalog::TIME: default: ret = (parm->data()->getIntVal(row, isNull) != 0); } @@ -219,6 +220,19 @@ int64_t Func_if::getDatetimeIntVal(Row& row, } } - +int64_t Func_if::getTimeIntVal(Row& row, + FunctionParm& parm, + bool& isNull, + CalpontSystemCatalog::ColType&) +{ + if (boolVal(parm[0], row)) + { + return parm[1]->data()->getTimeIntVal(row, isNull); + } + else + { + return parm[2]->data()->getTimeIntVal(row, isNull); + } +} } // namespace funcexp // vim:ts=4 sw=4: diff --git a/utils/funcexp/func_ifnull.cpp b/utils/funcexp/func_ifnull.cpp index 41db7e3b4..18887f00e 100644 --- a/utils/funcexp/func_ifnull.cpp +++ b/utils/funcexp/func_ifnull.cpp @@ -168,6 +168,25 @@ int64_t Func_ifnull::getDatetimeIntVal(Row& row, return r; } +int64_t Func_ifnull::getTimeIntVal(Row& row, + FunctionParm& parm, + bool& isNull, + CalpontSystemCatalog::ColType&) +{ + if (isNull) + return 0; + + int64_t r = parm[0]->data()->getTimeIntVal(row, isNull); + + if (isNull) + { + isNull = false; + return parm[1]->data()->getTimeIntVal(row, isNull); + } + + return r; +} + bool Func_ifnull::getBoolVal(Row& row, FunctionParm& parm, bool& isNull, diff --git a/utils/funcexp/func_in.cpp b/utils/funcexp/func_in.cpp index 404c0f39f..e271fa0f7 100644 --- a/utils/funcexp/func_in.cpp +++ b/utils/funcexp/func_in.cpp @@ -158,6 +158,27 @@ inline bool getBoolForIn(rowgroup::Row& row, return false; } + case execplan::CalpontSystemCatalog::TIME: + { + int64_t val = pm[0]->data()->getTimeIntVal(row, isNull); + + if (isNull) + return false; + + for (uint32_t i = 1; i < pm.size(); i++) + { + isNull = false; + + if ( val == pm[i]->data()->getTimeIntVal(row, isNull) && !isNull ) + return true; + + if (isNull && isNotIn) + return true; // will be reversed to false by the caller + } + + return false; + } + case execplan::CalpontSystemCatalog::DOUBLE: case execplan::CalpontSystemCatalog::UDOUBLE: case execplan::CalpontSystemCatalog::FLOAT: diff --git a/utils/funcexp/func_inet_aton.cpp b/utils/funcexp/func_inet_aton.cpp index bd2d2a2d7..d6e3465cf 100644 --- a/utils/funcexp/func_inet_aton.cpp +++ b/utils/funcexp/func_inet_aton.cpp @@ -222,6 +222,26 @@ int64_t Func_inet_aton::getDatetimeIntVal(rowgroup::Row& row, return iValue; } +int64_t Func_inet_aton::getTimeIntVal(rowgroup::Row& row, + FunctionParm& fp, + bool& isNull, + execplan::CalpontSystemCatalog::ColType& op_ct) +{ + int64_t iValue = joblist::TIMENULL; + + const std::string& sValue = fp[0]->data()->getStrVal(row, isNull); + + if (!isNull) + { + int64_t iVal = convertAton( sValue, isNull ); + + if (!isNull) + iValue = iVal; + } + + return iValue; +} + //------------------------------------------------------------------------------ // Convert an ascii IP address string to it's integer equivalent. // isNull is set to true if the IP address string has invalid content. diff --git a/utils/funcexp/func_inet_ntoa.cpp b/utils/funcexp/func_inet_ntoa.cpp index 84c476eb6..e25f56097 100644 --- a/utils/funcexp/func_inet_ntoa.cpp +++ b/utils/funcexp/func_inet_ntoa.cpp @@ -256,6 +256,20 @@ int64_t Func_inet_ntoa::getDatetimeIntVal(rowgroup::Row& row, return iValue; } +int64_t Func_inet_ntoa::getTimeIntVal(rowgroup::Row& row, + FunctionParm& fp, + bool& isNull, + execplan::CalpontSystemCatalog::ColType& op_ct) +{ +// std::cout << "In Func_inet_ntoa::getTimeVal" << std::endl; + +// int64t iValue = fp[0]->data()->getTimeIntVal(row, isNull); + int64_t iValue = joblist::TIMENULL; + isNull = true; + + return iValue; +} + //------------------------------------------------------------------------------ // Convert an integer IP address to its equivalent IP address string. // Source code based on MySQL source (Item_func_inet_ntoa() in item_strfunc.cc). diff --git a/utils/funcexp/func_least.cpp b/utils/funcexp/func_least.cpp index e273a4618..ef71e44e6 100644 --- a/utils/funcexp/func_least.cpp +++ b/utils/funcexp/func_least.cpp @@ -182,6 +182,25 @@ int64_t Func_least::getDatetimeIntVal(rowgroup::Row& row, return leastStr; } +int64_t Func_least::getTimeIntVal(rowgroup::Row& row, + FunctionParm& fp, + bool& isNull, + execplan::CalpontSystemCatalog::ColType& op_ct) +{ + int64_t str = fp[0]->data()->getTimeIntVal(row, isNull); + + int64_t leastStr = str; + + for (uint32_t i = 1; i < fp.size(); i++) + { + int64_t str1 = fp[i]->data()->getTimeIntVal(row, isNull); + + if ( leastStr > str1 ) + leastStr = str1; + } + + return leastStr; +} } // namespace funcexp diff --git a/utils/funcexp/func_math.cpp b/utils/funcexp/func_math.cpp index 8e5b84def..8592ec8b0 100644 --- a/utils/funcexp/func_math.cpp +++ b/utils/funcexp/func_math.cpp @@ -154,6 +154,20 @@ double Func_acos::getDoubleVal(Row& row, } break; + case execplan::CalpontSystemCatalog::TIME: + { + int64_t value = parm[0]->data()->getTimeIntVal(row, isNull); + + if (isNull || (value < -1.0 || value > 1.0)) + { + isNull = true; + return doubleNullVal(); + } + + return acos((double)value); + } + break; + default: { std::ostringstream oss; @@ -244,6 +258,20 @@ double Func_asin::getDoubleVal(Row& row, } break; + case execplan::CalpontSystemCatalog::TIME: + { + int64_t value = parm[0]->data()->getTimeIntVal(row, isNull); + + if (isNull || (value < -1.0 || value > 1.0)) + { + isNull = true; + return doubleNullVal(); + } + + return asin((double)value); + } + break; + default: { std::ostringstream oss; @@ -373,6 +401,34 @@ double Func_atan::getDoubleVal(Row& row, } break; + + case execplan::CalpontSystemCatalog::TIME: + { + int64_t value = parm[0]->data()->getTimeIntVal(row, isNull); + + if (isNull) + { + isNull = true; + return doubleNullVal(); + } + + if (parm.size() > 1 ) + { + double value2 = parm[1]->data()->getDoubleVal(row, isNull); + + if (isNull) + { + isNull = true; + return doubleNullVal(); + } + + return atan2(value, value2); + } + + return atan((double)value); + } + break; + default: { std::ostringstream oss; @@ -462,6 +518,20 @@ double Func_cos::getDoubleVal(Row& row, } break; + case execplan::CalpontSystemCatalog::TIME: + { + int64_t value = parm[0]->data()->getTimeIntVal(row, isNull); + + if (isNull) + { + isNull = true; + return doubleNullVal(); + } + + return cos((double)value); + } + break; + default: { std::ostringstream oss; @@ -578,6 +648,30 @@ double Func_cot::getDoubleVal(Row& row, } break; + case execplan::CalpontSystemCatalog::TIME: + { + int64_t value = parm[0]->data()->getTimeIntVal(row, isNull); + + if (value == 0) + { + Message::Args args; + args.add("cot"); + args.add((uint64_t)value); + unsigned errcode = ERR_FUNC_OUT_OF_RANGE_RESULT; + throw IDBExcept(IDBErrorInfo::instance()->errorMsg(errcode, args), errcode); + } + + if (isNull) + { + isNull = true; + return doubleNullVal(); + } + + return 1.0 / tan((double)value); + } + break; + + default: { std::ostringstream oss; @@ -703,6 +797,33 @@ double Func_log::getDoubleVal(Row& row, } break; + case execplan::CalpontSystemCatalog::TIME: + { + int64_t value = parm[0]->data()->getTimeIntVal(row, isNull); + + if (isNull || value <= 0.0) + { + isNull = true; + return doubleNullVal(); + } + + if (parm.size() > 1 ) + { + double value2 = parm[1]->data()->getDoubleVal(row, isNull); + + if (isNull || (value2 <= 0.0 || value == 1.0) ) + { + isNull = true; + return doubleNullVal(); + } + + return log(value2) / log((double)value); + } + + return log((double)value); + } + break; + case execplan::CalpontSystemCatalog::VARCHAR: case execplan::CalpontSystemCatalog::CHAR: case execplan::CalpontSystemCatalog::TEXT: @@ -797,6 +918,20 @@ double Func_log2::getDoubleVal(Row& row, } break; + case execplan::CalpontSystemCatalog::TIME: + { + int64_t value = parm[0]->data()->getTimeIntVal(row, isNull); + + if (isNull || value <= 0.0) + { + isNull = true; + return doubleNullVal(); + } + + return log2(value); + } + break; + case execplan::CalpontSystemCatalog::VARCHAR: case execplan::CalpontSystemCatalog::CHAR: case execplan::CalpontSystemCatalog::TEXT: @@ -891,6 +1026,20 @@ double Func_log10::getDoubleVal(Row& row, } break; + case execplan::CalpontSystemCatalog::TIME: + { + int64_t value = parm[0]->data()->getTimeIntVal(row, isNull); + + if (isNull || value <= 0.0) + { + isNull = true; + return doubleNullVal(); + } + + return log10((double)value); + } + break; + case execplan::CalpontSystemCatalog::VARCHAR: case execplan::CalpontSystemCatalog::CHAR: case execplan::CalpontSystemCatalog::TEXT: @@ -988,6 +1137,20 @@ double Func_sin::getDoubleVal(Row& row, } break; + case execplan::CalpontSystemCatalog::TIME: + { + int64_t value = parm[0]->data()->getTimeIntVal(row, isNull); + + if (isNull) + { + isNull = true; + return doubleNullVal(); + } + + return sin((double)value); + } + break; + default: { std::ostringstream oss; @@ -1077,6 +1240,20 @@ double Func_sqrt::getDoubleVal(Row& row, } break; + case execplan::CalpontSystemCatalog::TIME: + { + int64_t value = parm[0]->data()->getTimeIntVal(row, isNull); + + if (isNull || value < 0) + { + isNull = true; + return doubleNullVal(); + } + + return sqrt((double)value); + } + break; + default: { std::ostringstream oss; @@ -1166,6 +1343,20 @@ double Func_tan::getDoubleVal(Row& row, } break; + case execplan::CalpontSystemCatalog::TIME: + { + int64_t value = parm[0]->data()->getTimeIntVal(row, isNull); + + if (isNull) + { + isNull = true; + return doubleNullVal(); + } + + return tan((double)value); + } + break; + default: { std::ostringstream oss; @@ -1253,6 +1444,12 @@ string Func_format::getStrVal(Row& row, } break; + case execplan::CalpontSystemCatalog::TIME: + { + value = dataconvert::DataConvert::timeToString1(parm[0]->data()->getTimeIntVal(row, isNull)); + } + break; + case execplan::CalpontSystemCatalog::DECIMAL: case execplan::CalpontSystemCatalog::UDECIMAL: { @@ -1474,6 +1671,20 @@ double Func_radians::getDoubleVal(Row& row, } break; + case execplan::CalpontSystemCatalog::TIME: + { + int64_t value = parm[0]->data()->getTimeIntVal(row, isNull); + + if (isNull) + { + isNull = true; + return doubleNullVal(); + } + + return radians((double)value); + } + break; + default: { std::ostringstream oss; @@ -1563,6 +1774,20 @@ double Func_degrees::getDoubleVal(Row& row, } break; + case execplan::CalpontSystemCatalog::TIME: + { + int64_t value = parm[0]->data()->getTimeIntVal(row, isNull); + + if (isNull) + { + isNull = true; + return doubleNullVal(); + } + + return degrees((double)value); + } + break; + default: { std::ostringstream oss; diff --git a/utils/funcexp/func_minute.cpp b/utils/funcexp/func_minute.cpp index f70af29e3..299dd6ae2 100644 --- a/utils/funcexp/func_minute.cpp +++ b/utils/funcexp/func_minute.cpp @@ -102,6 +102,7 @@ int64_t Func_minute::getIntVal(rowgroup::Row& row, break; } + case execplan::CalpontSystemCatalog::TIME: case execplan::CalpontSystemCatalog::DATETIME: { val = parm[0]->data()->getDatetimeIntVal(row, isNull); diff --git a/utils/funcexp/func_nullif.cpp b/utils/funcexp/func_nullif.cpp index 7d1647d1d..23362f4ef 100644 --- a/utils/funcexp/func_nullif.cpp +++ b/utils/funcexp/func_nullif.cpp @@ -155,6 +155,38 @@ int64_t Func_nullif::getIntVal(rowgroup::Row& row, break; } + case execplan::CalpontSystemCatalog::TIME: + { + exp2 = parm[1]->data()->getTimeIntVal(row, isNull); + + if (parm[0]->data()->resultType().colDataType == + execplan::CalpontSystemCatalog::DATETIME) + { + // NULLIF arg0 is DATETIME, arg1 is TIME, + // Upgrade arg1 to time + // When comparing exp1 as a Date, we can't simply promote. We have + // to be careful of the return value in case of not null return. + int64_t exp1 = parm[0]->data()->getTimeIntVal(row, isNull); + + if ( exp1 == exp2 ) + { + isNull = true; + return 0; + } + + // since exp1 here is inside the block, when we leave the block, the + // original (Date) value is restored. + } + + if (isNull) + { + isNull = false; + return exp1; + } + + break; + } + default: { isNull = true; @@ -244,6 +276,19 @@ uint64_t Func_nullif::getUintVal(rowgroup::Row& row, break; } + case execplan::CalpontSystemCatalog::TIME: + { + exp2 = parm[1]->data()->getTimeIntVal(row, isNull); + + if (isNull) + { + isNull = false; + return exp1; + } + + break; + } + default: { isNull = true; @@ -427,6 +472,7 @@ int64_t Func_nullif::getDatetimeIntVal(rowgroup::Row& row, } break; + case execplan::CalpontSystemCatalog::TIME: case execplan::CalpontSystemCatalog::DATETIME: { exp2 = parm[1]->data()->getDatetimeIntVal(row, isNull); @@ -455,6 +501,69 @@ int64_t Func_nullif::getDatetimeIntVal(rowgroup::Row& row, return exp1; } +int64_t Func_nullif::getTimeIntVal(rowgroup::Row& row, + FunctionParm& parm, + bool& isNull, + CalpontSystemCatalog::ColType& ct) +{ + int64_t exp1 = parm[0]->data()->getTimeIntVal(row, isNull); + int64_t exp2 = 0; + + switch (parm[1]->data()->resultType().colDataType) + { + case execplan::CalpontSystemCatalog::BIGINT: + case execplan::CalpontSystemCatalog::INT: + case execplan::CalpontSystemCatalog::MEDINT: + case execplan::CalpontSystemCatalog::TINYINT: + case execplan::CalpontSystemCatalog::SMALLINT: + case execplan::CalpontSystemCatalog::DOUBLE: + case execplan::CalpontSystemCatalog::FLOAT: + case execplan::CalpontSystemCatalog::DECIMAL: + case execplan::CalpontSystemCatalog::VARCHAR: + case execplan::CalpontSystemCatalog::CHAR: + case execplan::CalpontSystemCatalog::TEXT: + { + exp2 = parm[1]->data()->getIntVal(row, isNull); + + if (isNull) + { + isNull = false; + return exp1; + } + + break; + } + + case execplan::CalpontSystemCatalog::TIME: + case execplan::CalpontSystemCatalog::DATETIME: + { + exp2 = parm[1]->data()->getTimeIntVal(row, isNull); + + if (isNull) + { + isNull = false; + return exp1; + } + + break; + } + + default: + { + isNull = true; + } + } + + if ( exp1 == exp2 ) + { + isNull = true; + return 0; + } + + return exp1; +} + + double Func_nullif::getDoubleVal(rowgroup::Row& row, FunctionParm& parm, bool& isNull, @@ -502,6 +611,7 @@ double Func_nullif::getDoubleVal(rowgroup::Row& row, } break; + case execplan::CalpontSystemCatalog::TIME: case execplan::CalpontSystemCatalog::DATETIME: { exp2 = parm[1]->data()->getDatetimeIntVal(row, isNull); @@ -644,6 +754,7 @@ execplan::IDB_Decimal Func_nullif::getDecimalVal(rowgroup::Row& row, } break; + case execplan::CalpontSystemCatalog::TIME: case execplan::CalpontSystemCatalog::DATETIME: { int32_t s = 0; diff --git a/utils/funcexp/func_regexp.cpp b/utils/funcexp/func_regexp.cpp index 11f702a60..420c792cf 100644 --- a/utils/funcexp/func_regexp.cpp +++ b/utils/funcexp/func_regexp.cpp @@ -94,6 +94,14 @@ inline bool getBool(rowgroup::Row& row, break; } + case execplan::CalpontSystemCatalog::TIME: + { + expr = dataconvert::DataConvert::timeToString(pm[0]->data()->getTimeIntVal(row, isNull)); + //strip off micro seconds + expr = expr.substr(0, 19); + break; + } + case execplan::CalpontSystemCatalog::DECIMAL: case execplan::CalpontSystemCatalog::UDECIMAL: { @@ -153,6 +161,14 @@ inline bool getBool(rowgroup::Row& row, break; } + case execplan::CalpontSystemCatalog::TIME: + { + pattern = dataconvert::DataConvert::timeToString(pm[1]->data()->getTimeIntVal(row, isNull)); + //strip off micro seconds + pattern = pattern.substr(0, 19); + break; + } + case execplan::CalpontSystemCatalog::DECIMAL: case execplan::CalpontSystemCatalog::UDECIMAL: { diff --git a/utils/funcexp/func_round.cpp b/utils/funcexp/func_round.cpp index 18b888b2c..c107fc0fc 100644 --- a/utils/funcexp/func_round.cpp +++ b/utils/funcexp/func_round.cpp @@ -390,6 +390,7 @@ IDB_Decimal Func_round::getDecimalVal(Row& row, } break; + case execplan::CalpontSystemCatalog::TIME: case execplan::CalpontSystemCatalog::DATETIME: { int32_t s = 0; diff --git a/utils/funcexp/func_second.cpp b/utils/funcexp/func_second.cpp index 509633c46..678a1fa61 100644 --- a/utils/funcexp/func_second.cpp +++ b/utils/funcexp/func_second.cpp @@ -108,6 +108,12 @@ int64_t Func_second::getIntVal(rowgroup::Row& row, break; } + case execplan::CalpontSystemCatalog::TIME: + { + val = parm[0]->data()->getTimeIntVal(row, isNull); + break; + } + default: { isNull = true; diff --git a/utils/funcexp/func_sysdate.cpp b/utils/funcexp/func_sysdate.cpp index f8a5a98bd..d8e2580bb 100644 --- a/utils/funcexp/func_sysdate.cpp +++ b/utils/funcexp/func_sysdate.cpp @@ -100,6 +100,15 @@ int64_t Func_sysdate::getDatetimeIntVal(rowgroup::Row& row, return getIntVal(row, parm, isNull, operationColType); } +int64_t Func_sysdate::getTimeIntVal(rowgroup::Row& row, + FunctionParm& parm, + bool& isNull, + CalpontSystemCatalog::ColType& operationColType) +{ + return getIntVal(row, parm, isNull, operationColType); +} + + } // namespace funcexp // vim:ts=4 sw=4: diff --git a/utils/funcexp/func_time.cpp b/utils/funcexp/func_time.cpp index 509caa777..8aab1389b 100644 --- a/utils/funcexp/func_time.cpp +++ b/utils/funcexp/func_time.cpp @@ -112,6 +112,7 @@ string Func_time::getStrVal(rowgroup::Row& row, break; } + case execplan::CalpontSystemCatalog::TIME: case execplan::CalpontSystemCatalog::DATETIME: { val = parm[0]->data()->getDatetimeIntVal(row, isNull); diff --git a/utils/funcexp/func_timediff.cpp b/utils/funcexp/func_timediff.cpp index c87e99b64..3c9ce96a8 100644 --- a/utils/funcexp/func_timediff.cpp +++ b/utils/funcexp/func_timediff.cpp @@ -116,6 +116,7 @@ string Func_timediff::getStrVal(rowgroup::Row& row, isDate1 = true; break; + case execplan::CalpontSystemCatalog::TIME: case execplan::CalpontSystemCatalog::DATETIME: val1 = parm[0]->data()->getDatetimeIntVal(row, isNull); break; @@ -157,6 +158,7 @@ string Func_timediff::getStrVal(rowgroup::Row& row, isDate2 = true; break; + case execplan::CalpontSystemCatalog::TIME: case execplan::CalpontSystemCatalog::DATETIME: val2 = parm[1]->data()->getDatetimeIntVal(row, isNull); break; @@ -213,6 +215,14 @@ int64_t Func_timediff::getDatetimeIntVal(rowgroup::Row& row, return dataconvert::DataConvert::datetimeToInt(getStrVal(row, parm, isNull, ct)); } +int64_t Func_timediff::getTimeIntVal(rowgroup::Row& row, + FunctionParm& parm, + bool& isNull, + CalpontSystemCatalog::ColType& ct) +{ + return dataconvert::DataConvert::timeToInt(getStrVal(row, parm, isNull, ct)); +} + int64_t Func_timediff::getIntVal(rowgroup::Row& row, FunctionParm& parm, bool& isNull, diff --git a/utils/funcexp/func_timestampdiff.cpp b/utils/funcexp/func_timestampdiff.cpp index f8d1fc7ac..9394203d7 100644 --- a/utils/funcexp/func_timestampdiff.cpp +++ b/utils/funcexp/func_timestampdiff.cpp @@ -152,6 +152,13 @@ int64_t Func_timestampdiff::getDatetimeIntVal(rowgroup::Row& row, return getIntVal(row, parm, isNull, ct); } +int64_t Func_timestampdiff::getTimeIntVal(rowgroup::Row& row, + FunctionParm& parm, + bool& isNull, + CalpontSystemCatalog::ColType& ct) +{ + return getIntVal(row, parm, isNull, ct); +} } // namespace funcexp // vim:ts=4 sw=4: diff --git a/utils/funcexp/func_truncate.cpp b/utils/funcexp/func_truncate.cpp index c8be27de0..b822caacd 100644 --- a/utils/funcexp/func_truncate.cpp +++ b/utils/funcexp/func_truncate.cpp @@ -387,6 +387,51 @@ IDB_Decimal Func_truncate::getDecimalVal(Row& row, } break; + case execplan::CalpontSystemCatalog::TIME: + { + int32_t s = 0; + int64_t x = 0; + + string value = + DataConvert::timeToString1(parm[0]->data()->getTimeIntVal(row, isNull)); + + s = parm[1]->data()->getIntVal(row, isNull); + + if (!isNull) + { + //strip off micro seconds + value = value.substr(0, 14); + int64_t x = atoll(value.c_str()); + + if ( s > 5 ) + s = 0; + + if ( s > 0 ) + { + x *= helpers::powerOf10_c[s]; + } + else if (s < 0) + { + s = -s; + + if ( s >= (int32_t) value.size() ) + { + x = 0; + } + else + { + x /= helpers::powerOf10_c[s]; + x *= helpers::powerOf10_c[s]; + } + + s = 0; + } + } + + decimal.value = x; + decimal.scale = s; + } + break; default: { diff --git a/utils/funcexp/funcexp.cpp b/utils/funcexp/funcexp.cpp index a59be9a7c..3c530f381 100644 --- a/utils/funcexp/funcexp.cpp +++ b/utils/funcexp/funcexp.cpp @@ -280,6 +280,18 @@ void FuncExp::evaluate(rowgroup::Row& row, std::vector& expressi break; } + case CalpontSystemCatalog::TIME: + { + int64_t val = expression[i]->getTimeIntVal(row, isNull); + + if (isNull) + row.setIntField<8>(TIMENULL, expression[i]->outputIndex()); + else + row.setIntField<8>(val, expression[i]->outputIndex()); + + break; + } + case CalpontSystemCatalog::CHAR: case CalpontSystemCatalog::VARCHAR: diff --git a/utils/funcexp/functor.cpp b/utils/funcexp/functor.cpp index 662db6448..b74812ee9 100644 --- a/utils/funcexp/functor.cpp +++ b/utils/funcexp/functor.cpp @@ -106,6 +106,21 @@ uint64_t Func::stringToDatetime(const string str) return ret; } +int64_t Func::stringToTime(const string str) +{ + int64_t ret = DataConvert::stringToTime(str); + + if (ret == -1) + { + Message::Args args; + args.add("time"); + args.add(str); + unsigned errcode = ERR_INCORRECT_VALUE; + throw IDBExcept(IDBErrorInfo::instance()->errorMsg(errcode, args), errcode); + } + + return ret; +} uint32_t Func::intToDate(int64_t i) { @@ -124,6 +139,11 @@ uint64_t Func::intToDatetime(int64_t i) return i; } +int64_t Func::intToTime(int64_t i) +{ + // Don't think we need to do anything here? + return i; +} string Func::intToString(int64_t i) { diff --git a/utils/funcexp/functor.h b/utils/funcexp/functor.h index 9e03e4af2..065eb7865 100644 --- a/utils/funcexp/functor.h +++ b/utils/funcexp/functor.h @@ -116,6 +116,14 @@ public: return intToDatetime(getIntVal(row, fp, isNull, op_ct)); } + virtual int64_t getTimeIntVal(rowgroup::Row& row, + FunctionParm& fp, + bool& isNull, + execplan::CalpontSystemCatalog::ColType& op_ct) + { + return intToTime(getIntVal(row, fp, isNull, op_ct)); + } + virtual bool getBoolVal(rowgroup::Row& row, FunctionParm& fp, bool& isNull, @@ -145,9 +153,11 @@ public: protected: virtual uint32_t stringToDate(std::string); virtual uint64_t stringToDatetime(std::string); + virtual int64_t stringToTime(std::string); virtual uint32_t intToDate(int64_t); virtual uint64_t intToDatetime(int64_t); + virtual int64_t intToTime(int64_t); virtual std::string intToString(int64_t); virtual std::string doubleToString(double); diff --git a/utils/funcexp/functor_all.h b/utils/funcexp/functor_all.h index 6300084cd..0eb0df845 100644 --- a/utils/funcexp/functor_all.h +++ b/utils/funcexp/functor_all.h @@ -104,6 +104,11 @@ public: FunctionParm& fp, bool& isNull, execplan::CalpontSystemCatalog::ColType& op_ct); + + int64_t getTimeIntVal(rowgroup::Row& row, + FunctionParm& fp, + bool& isNull, + execplan::CalpontSystemCatalog::ColType& op_ct); }; @@ -151,6 +156,12 @@ public: FunctionParm& fp, bool& isNull, execplan::CalpontSystemCatalog::ColType& op_ct); + + int64_t getTimeIntVal(rowgroup::Row& row, + FunctionParm& fp, + bool& isNull, + execplan::CalpontSystemCatalog::ColType& op_ct); + }; @@ -193,7 +204,11 @@ public: FunctionParm& fp, bool& isNull, execplan::CalpontSystemCatalog::ColType& op_ct); -}; + + int64_t getTimeIntVal(rowgroup::Row& row, + FunctionParm& fp, + bool& isNull, + execplan::CalpontSystemCatalog::ColType& op_ct);}; /** @brief Func_ifnull class @@ -236,6 +251,11 @@ public: bool& isNull, execplan::CalpontSystemCatalog::ColType& op_ct); + int64_t getTimeIntVal(rowgroup::Row& row, + FunctionParm& fp, + bool& isNull, + execplan::CalpontSystemCatalog::ColType& op_ct); + bool getBoolVal(rowgroup::Row& row, FunctionParm& fp, bool& isNull, @@ -287,6 +307,12 @@ public: FunctionParm& fp, bool& isNull, execplan::CalpontSystemCatalog::ColType& op_ct); + + int64_t getTimeIntVal(rowgroup::Row& row, + FunctionParm& fp, + bool& isNull, + execplan::CalpontSystemCatalog::ColType& op_ct); + }; @@ -329,7 +355,11 @@ public: FunctionParm& fp, bool& isNull, execplan::CalpontSystemCatalog::ColType& op_ct); -}; + + int64_t getTimeIntVal(rowgroup::Row& row, + FunctionParm& fp, + bool& isNull, + execplan::CalpontSystemCatalog::ColType& op_ct);}; /** @brief Func_coalesce class @@ -371,6 +401,11 @@ public: FunctionParm& fp, bool& isNull, execplan::CalpontSystemCatalog::ColType& op_ct); + + int64_t getTimeIntVal(rowgroup::Row& row, + FunctionParm& fp, + bool& isNull, + execplan::CalpontSystemCatalog::ColType& op_ct); }; @@ -399,6 +434,11 @@ public: bool& isNull, execplan::CalpontSystemCatalog::ColType& op_ct); + int64_t getTimeIntVal(rowgroup::Row& row, + FunctionParm& fp, + bool& isNull, + execplan::CalpontSystemCatalog::ColType& op_ct); + int64_t getIntVal(rowgroup::Row& row, FunctionParm& fp, bool& isNull, diff --git a/utils/funcexp/functor_bool.h b/utils/funcexp/functor_bool.h index a20f925b3..f67f405f1 100644 --- a/utils/funcexp/functor_bool.h +++ b/utils/funcexp/functor_bool.h @@ -97,6 +97,15 @@ public: isNull = true; return 0; } + + int64_t getTimeIntVal(rowgroup::Row& row, + FunctionParm& fp, + bool& isNull, + execplan::CalpontSystemCatalog::ColType& op_ct) + { + isNull = true; + return 0; + } }; diff --git a/utils/funcexp/functor_dtm.h b/utils/funcexp/functor_dtm.h index dc662a176..74aa41e38 100644 --- a/utils/funcexp/functor_dtm.h +++ b/utils/funcexp/functor_dtm.h @@ -137,6 +137,11 @@ public: bool& isNull, execplan::CalpontSystemCatalog::ColType& op_ct); + int64_t getTimeIntVal(rowgroup::Row& row, + FunctionParm& fp, + bool& isNull, + execplan::CalpontSystemCatalog::ColType& op_ct); + int64_t getIntVal(rowgroup::Row& row, FunctionParm& fp, bool& isNull, @@ -247,6 +252,12 @@ public: FunctionParm& fp, bool& isNull, execplan::CalpontSystemCatalog::ColType& op_ct); + + int64_t getTimeIntVal(rowgroup::Row& row, + FunctionParm& fp, + bool& isNull, + execplan::CalpontSystemCatalog::ColType& op_ct); + }; @@ -311,6 +322,11 @@ public: FunctionParm& fp, bool& isNull, execplan::CalpontSystemCatalog::ColType& op_ct); + + int64_t getTimeIntVal(rowgroup::Row& row, + FunctionParm& fp, + bool& isNull, + execplan::CalpontSystemCatalog::ColType& op_ct); }; @@ -343,6 +359,11 @@ public: FunctionParm& fp, bool& isNull, execplan::CalpontSystemCatalog::ColType& op_ct); + + int64_t getTimeIntVal(rowgroup::Row& row, + FunctionParm& fp, + bool& isNull, + execplan::CalpontSystemCatalog::ColType& op_ct); }; @@ -371,6 +392,11 @@ public: bool& isNull, execplan::CalpontSystemCatalog::ColType& op_ct); + int64_t getTimeIntVal(rowgroup::Row& row, + FunctionParm& fp, + bool& isNull, + execplan::CalpontSystemCatalog::ColType& op_ct); + int64_t getIntVal(rowgroup::Row& row, FunctionParm& fp, bool& isNull, @@ -411,6 +437,11 @@ public: FunctionParm& fp, bool& isNull, execplan::CalpontSystemCatalog::ColType& op_ct); + + int64_t getTimeIntVal(rowgroup::Row& row, + FunctionParm& fp, + bool& isNull, + execplan::CalpontSystemCatalog::ColType& op_ct); }; /** @brief Func_str_to_date class diff --git a/utils/funcexp/functor_real.h b/utils/funcexp/functor_real.h index 936401f80..779f4d108 100644 --- a/utils/funcexp/functor_real.h +++ b/utils/funcexp/functor_real.h @@ -654,6 +654,12 @@ public: bool& isNull, execplan::CalpontSystemCatalog::ColType& op_ct); + int64_t getTimeIntVal(rowgroup::Row& row, + FunctionParm& fp, + bool& isNull, + execplan::CalpontSystemCatalog::ColType& op_ct); + + private: int64_t convertAton(const std::string& ipString, bool& isNull); }; diff --git a/utils/funcexp/functor_str.h b/utils/funcexp/functor_str.h index 0b40685f6..895101211 100644 --- a/utils/funcexp/functor_str.h +++ b/utils/funcexp/functor_str.h @@ -89,6 +89,14 @@ public: return (isNull ? 0 : stringToDatetime(str)); } + int64_t getTimeIntVal(rowgroup::Row& row, + FunctionParm& fp, + bool& isNull, + execplan::CalpontSystemCatalog::ColType& op_ct) + { + std::string str = getStrVal(row, fp, isNull, op_ct); + return (isNull ? 0 : stringToTime(str)); + } protected: const std::string& stringValue(execplan::SPTP& fp, rowgroup::Row& row, bool& isNull) @@ -676,6 +684,10 @@ public: bool& isNull, execplan::CalpontSystemCatalog::ColType& op_ct); + int64_t getTimeIntVal(rowgroup::Row& row, + FunctionParm& fp, + bool& isNull, + execplan::CalpontSystemCatalog::ColType& op_ct); private: void convertNtoa(int64_t ipNum, std::string& ipString); }; diff --git a/utils/rowgroup/rowaggregation.cpp b/utils/rowgroup/rowaggregation.cpp index 0a98f6870..a7edd9248 100644 --- a/utils/rowgroup/rowaggregation.cpp +++ b/utils/rowgroup/rowaggregation.cpp @@ -125,6 +125,11 @@ inline uint64_t getUintNullValue(int colType, int colWidth = 0) return joblist::DATETIMENULL; } + case execplan::CalpontSystemCatalog::TIME: + { + return joblist::TIMENULL; + } + case execplan::CalpontSystemCatalog::DECIMAL: case execplan::CalpontSystemCatalog::UDECIMAL: { @@ -640,6 +645,12 @@ inline bool RowAggregation::isNull(const RowGroup* pRowGroup, const Row& row, in break; } + case execplan::CalpontSystemCatalog::TIME: + { + ret = ((uint64_t)row.getUintField(col) == joblist::TIMENULL); + break; + } + case execplan::CalpontSystemCatalog::VARBINARY: case execplan::CalpontSystemCatalog::BLOB: { @@ -1117,6 +1128,7 @@ void RowAggregation::initMapData(const Row& rowIn) case execplan::CalpontSystemCatalog::DATE: case execplan::CalpontSystemCatalog::DATETIME: + case execplan::CalpontSystemCatalog::TIME: { fRow.setUintField(rowIn.getUintField(colIn), colOut); break; @@ -1247,6 +1259,7 @@ void RowAggregation::makeAggFieldsNull(Row& row) case execplan::CalpontSystemCatalog::DATE: case execplan::CalpontSystemCatalog::DATETIME: + case execplan::CalpontSystemCatalog::TIME: { row.setUintField(getUintNullValue(colDataType), colOut); break; @@ -1375,6 +1388,7 @@ void RowAggregation::doMinMaxSum(const Row& rowIn, int64_t colIn, int64_t colOut case execplan::CalpontSystemCatalog::DATE: case execplan::CalpontSystemCatalog::DATETIME: + case execplan::CalpontSystemCatalog::TIME: { if (funcType == ROWAGG_SUM) { @@ -1518,9 +1532,17 @@ void RowAggregation::doBitOp(const Row& rowIn, int64_t colIn, int64_t colOut, in case execplan::CalpontSystemCatalog::DATETIME: { uint64_t dtm = rowIn.getUintField(colIn); - valIn = ((dtm >> 48) * 10000000000LL) + (((dtm >> 44) & 0xF) * 100000000) + - (((dtm >> 38) & 077) * 1000000) + (((dtm >> 32) & 077) * 10000) + - (((dtm >> 26) & 077) * 100) + ((dtm >> 20) & 077); + valIn = ((dtm >> 48) * 10000000000000000LL) + (((dtm >> 44) & 0xF) * 100000000000000) + + (((dtm >> 38) & 077) * 1000000000000) + (((dtm >> 32) & 077) * 10000000000) + + (((dtm >> 26) & 077) * 100000000) + (((dtm >> 20) & 077) * 1000000) + (dtm & 0xfffff); + break; + } + + case execplan::CalpontSystemCatalog::TIME: + { + int64_t dtm = rowIn.getUintField(colIn); + valIn = (((dtm >> 40) & 0xfff) * 10000000000) + + (((dtm >> 32) & 0xff) * 100000000) + (((dtm >> 24) & 0xff) * 1000000) + (dtm & 0xffffff); break; } @@ -2047,6 +2069,12 @@ void RowAggregation::doUDAF(const Row& rowIn, int64_t colIn, int64_t colOut, int datum.columnData = rowIn.getUintField(colIn); break; } + case execplan::CalpontSystemCatalog::TIME: + { + datum.dataType = execplan::CalpontSystemCatalog::BIGINT; + datum.columnData = rowIn.getIntField(colIn); + break; + } case execplan::CalpontSystemCatalog::CHAR: case execplan::CalpontSystemCatalog::VARCHAR: @@ -2723,6 +2751,11 @@ void RowAggregationUM::SetUDAFValue(static_any::any& valOut, int64_t colOut) fRow.setUintField<8>(uintOut, colOut); break; + case execplan::CalpontSystemCatalog::TIME: + + fRow.setIntField<8>(intOut, colOut); + break; + case execplan::CalpontSystemCatalog::FLOAT: case execplan::CalpontSystemCatalog::UFLOAT: fRow.setFloatField(floatOut, colOut); @@ -3031,6 +3064,12 @@ void RowAggregationUM::doNullConstantAggregate(const ConstantAggData& aggData, u } break; + case execplan::CalpontSystemCatalog::TIME: + { + fRow.setIntField(getIntNullValue(colDataType), colOut); + } + break; + case execplan::CalpontSystemCatalog::CHAR: case execplan::CalpontSystemCatalog::VARCHAR: case execplan::CalpontSystemCatalog::TEXT: @@ -3207,6 +3246,12 @@ void RowAggregationUM::doNotNullConstantAggregate(const ConstantAggData& aggData } break; + case execplan::CalpontSystemCatalog::TIME: + { + fRow.setIntField(DataConvert::stringToTime(aggData.fConstValue), colOut); + } + break; + case execplan::CalpontSystemCatalog::CHAR: case execplan::CalpontSystemCatalog::VARCHAR: case execplan::CalpontSystemCatalog::TEXT: @@ -3295,6 +3340,7 @@ void RowAggregationUM::doNotNullConstantAggregate(const ConstantAggData& aggData case execplan::CalpontSystemCatalog::DATE: case execplan::CalpontSystemCatalog::DATETIME: + case execplan::CalpontSystemCatalog::TIME: case execplan::CalpontSystemCatalog::CHAR: case execplan::CalpontSystemCatalog::VARCHAR: case execplan::CalpontSystemCatalog::TEXT: @@ -3355,6 +3401,7 @@ void RowAggregationUM::doNotNullConstantAggregate(const ConstantAggData& aggData break; case execplan::CalpontSystemCatalog::DATETIME: + case execplan::CalpontSystemCatalog::TIME: { fRow.setUintField(0, colOut); } @@ -3492,6 +3539,12 @@ void RowAggregationUM::doNotNullConstantAggregate(const ConstantAggData& aggData } break; + case execplan::CalpontSystemCatalog::TIME: + { + datum.columnData = DataConvert::stringToTime(aggData.fConstValue); + } + break; + case execplan::CalpontSystemCatalog::CHAR: case execplan::CalpontSystemCatalog::VARCHAR: case execplan::CalpontSystemCatalog::TEXT: diff --git a/utils/rowgroup/rowgroup.cpp b/utils/rowgroup/rowgroup.cpp index c35129d03..185d3de67 100644 --- a/utils/rowgroup/rowgroup.cpp +++ b/utils/rowgroup/rowgroup.cpp @@ -699,6 +699,10 @@ void Row::initToNull() *((uint64_t*) &data[offsets[i]]) = joblist::DATETIMENULL; break; + case CalpontSystemCatalog::TIME: + *((uint64_t*) &data[offsets[i]]) = joblist::TIMENULL; + break; + case CalpontSystemCatalog::CHAR: case CalpontSystemCatalog::VARCHAR: case CalpontSystemCatalog::TEXT: @@ -841,6 +845,9 @@ bool Row::isNullValue(uint32_t colIndex) const case CalpontSystemCatalog::DATETIME: return (*((uint64_t*) &data[offsets[colIndex]]) == joblist::DATETIMENULL); + case CalpontSystemCatalog::TIME: + return (*((uint64_t*) &data[offsets[colIndex]]) == joblist::TIMENULL); + case CalpontSystemCatalog::CHAR: case CalpontSystemCatalog::VARCHAR: case CalpontSystemCatalog::STRINT: diff --git a/utils/udfsdk/docs/source/reference/ColumnDatum.rst b/utils/udfsdk/docs/source/reference/ColumnDatum.rst index 564b24e44..dd1006363 100644 --- a/utils/udfsdk/docs/source/reference/ColumnDatum.rst +++ b/utils/udfsdk/docs/source/reference/ColumnDatum.rst @@ -74,6 +74,8 @@ The provided values are: - A floating point number. Represented as a C++ double type. * - DATETIME - A Columnstore date-time stored as an eight byte unsigned integer. + * - TIME + - A Columnstore time stored as an eight byte unsigned integer. * - VARCHAR - A mariadb variable length string. Represented a std::string * - VARBINARY diff --git a/utils/udfsdk/mcsv1_udaf.cpp b/utils/udfsdk/mcsv1_udaf.cpp index a31ae71c0..349a642ec 100644 --- a/utils/udfsdk/mcsv1_udaf.cpp +++ b/utils/udfsdk/mcsv1_udaf.cpp @@ -95,6 +95,7 @@ int32_t mcsv1Context::getColWidth() case CalpontSystemCatalog::DOUBLE: case CalpontSystemCatalog::UDOUBLE: case CalpontSystemCatalog::DATETIME: + case CalpontSystemCatalog::TIME: case CalpontSystemCatalog::STRINT: fColWidth = 8; break; diff --git a/utils/udfsdk/udfsdk.cpp b/utils/udfsdk/udfsdk.cpp index ea206a167..9a8973232 100644 --- a/utils/udfsdk/udfsdk.cpp +++ b/utils/udfsdk/udfsdk.cpp @@ -101,7 +101,9 @@ CalpontSystemCatalog::ColType MCS_add::operationType (FunctionParm& fp, else if (fp[0]->data()->resultType().colDataType == CalpontSystemCatalog::DATE || fp[1]->data()->resultType().colDataType == CalpontSystemCatalog::DATE || fp[0]->data()->resultType().colDataType == CalpontSystemCatalog::DATETIME || - fp[1]->data()->resultType().colDataType == CalpontSystemCatalog::DATETIME) + fp[1]->data()->resultType().colDataType == CalpontSystemCatalog::DATETIME || + fp[0]->data()->resultType().colDataType == CalpontSystemCatalog::TIME || + fp[1]->data()->resultType().colDataType == CalpontSystemCatalog::TIME) { rt.colDataType = CalpontSystemCatalog::BIGINT; rt.colWidth = 8; diff --git a/utils/windowfunction/idborderby.cpp b/utils/windowfunction/idborderby.cpp index 5764afba5..8a021f8d8 100644 --- a/utils/windowfunction/idborderby.cpp +++ b/utils/windowfunction/idborderby.cpp @@ -281,6 +281,7 @@ void CompareRule::compileRules(const std::vector& spec, const rowgr case CalpontSystemCatalog::DATE: case CalpontSystemCatalog::DATETIME: + case CalpontSystemCatalog::TIME: { Compare* c = new UintCompare(*i); fCompares.push_back(c); @@ -413,6 +414,7 @@ bool EqualCompData::operator()(Row::Pointer a, Row::Pointer b) case CalpontSystemCatalog::UBIGINT: case CalpontSystemCatalog::DATE: case CalpontSystemCatalog::DATETIME: + case CalpontSystemCatalog::TIME: { // equal compare. ignore sign and null eq = (fRow1.getUintField(*i) == fRow2.getUintField(*i)); diff --git a/utils/windowfunction/wf_lead_lag.cpp b/utils/windowfunction/wf_lead_lag.cpp index dddfb91c6..5160b75bf 100644 --- a/utils/windowfunction/wf_lead_lag.cpp +++ b/utils/windowfunction/wf_lead_lag.cpp @@ -77,6 +77,7 @@ boost::shared_ptr WF_lead_lag::makeFunction(int id, const case CalpontSystemCatalog::UDECIMAL: case CalpontSystemCatalog::DATE: case CalpontSystemCatalog::DATETIME: + case CalpontSystemCatalog::TIME: { func.reset(new WF_lead_lag(id, name)); break; diff --git a/utils/windowfunction/wf_min_max.cpp b/utils/windowfunction/wf_min_max.cpp index 99a8fef84..b0c1fe033 100644 --- a/utils/windowfunction/wf_min_max.cpp +++ b/utils/windowfunction/wf_min_max.cpp @@ -76,6 +76,7 @@ boost::shared_ptr WF_min_max::makeFunction(int id, const case CalpontSystemCatalog::UDECIMAL: case CalpontSystemCatalog::DATE: case CalpontSystemCatalog::DATETIME: + case CalpontSystemCatalog::TIME: { func.reset(new WF_min_max(id, name)); break; diff --git a/utils/windowfunction/wf_nth_value.cpp b/utils/windowfunction/wf_nth_value.cpp index 5362ee8ec..eacd1202a 100644 --- a/utils/windowfunction/wf_nth_value.cpp +++ b/utils/windowfunction/wf_nth_value.cpp @@ -77,6 +77,7 @@ boost::shared_ptr WF_nth_value::makeFunction(int id, cons case CalpontSystemCatalog::UDECIMAL: case CalpontSystemCatalog::DATE: case CalpontSystemCatalog::DATETIME: + case CalpontSystemCatalog::TIME: { func.reset(new WF_nth_value(id, name)); break; diff --git a/utils/windowfunction/wf_percentile.cpp b/utils/windowfunction/wf_percentile.cpp index 9ccc07b9c..83d6f57d8 100644 --- a/utils/windowfunction/wf_percentile.cpp +++ b/utils/windowfunction/wf_percentile.cpp @@ -82,6 +82,7 @@ boost::shared_ptr WF_percentile::makeFunction(int id, con case CalpontSystemCatalog::UDECIMAL: case CalpontSystemCatalog::DATE: case CalpontSystemCatalog::DATETIME: + case CalpontSystemCatalog::TIME: { func.reset(new WF_percentile(id, name)); break; diff --git a/utils/windowfunction/wf_udaf.cpp b/utils/windowfunction/wf_udaf.cpp index fe394c028..f302c49cd 100644 --- a/utils/windowfunction/wf_udaf.cpp +++ b/utils/windowfunction/wf_udaf.cpp @@ -392,6 +392,7 @@ void WF_udaf::SetUDAFValue(static_any::any& valOut, int64_t colOut, case execplan::CalpontSystemCatalog::UBIGINT: case execplan::CalpontSystemCatalog::DATE: case execplan::CalpontSystemCatalog::DATETIME: + case execplan::CalpontSystemCatalog::TIME: setValue(colDataType, b, e, c, &uintOut); break; diff --git a/utils/windowfunction/windowfunctiontype.cpp b/utils/windowfunction/windowfunctiontype.cpp index 5a101f870..950045899 100644 --- a/utils/windowfunction/windowfunctiontype.cpp +++ b/utils/windowfunction/windowfunctiontype.cpp @@ -91,6 +91,7 @@ map colType2String = assign::map_list_of (CalpontSystemCatalog::LONGDOUBLE, "INTERNAL LONG DOUBLE") (CalpontSystemCatalog::STRINT, "INTERNAL SHORT STRING") (CalpontSystemCatalog::TEXT, "TEXT") + (CalpontSystemCatalog::TIME, "TIME") ; @@ -490,6 +491,7 @@ void* WindowFunctionType::getNullValueByType(int ct, int pos) static uint64_t doubleNull = joblist::DOUBLENULL; static uint64_t dateNull = joblist::DATENULL; static uint64_t datetimeNull = joblist::DATETIMENULL; + static uint64_t timeNull = joblist::TIMENULL; static uint64_t char1Null = joblist::CHAR1NULL; static uint64_t char2Null = joblist::CHAR2NULL; static uint64_t char4Null = joblist::CHAR4NULL; @@ -525,6 +527,10 @@ void* WindowFunctionType::getNullValueByType(int ct, int pos) v = &datetimeNull; break; + case CalpontSystemCatalog::TIME: + v = &timeNull; + break; + case CalpontSystemCatalog::FLOAT: case CalpontSystemCatalog::UFLOAT: v = &floatNull; diff --git a/writeengine/bulk/we_bulkloadbuffer.cpp b/writeengine/bulk/we_bulkloadbuffer.cpp index 68bf71027..79ea553f1 100644 --- a/writeengine/bulk/we_bulkloadbuffer.cpp +++ b/writeengine/bulk/we_bulkloadbuffer.cpp @@ -902,7 +902,8 @@ void BulkLoadBuffer::convert(char* field, int fieldLength, { bool bSatVal = false; - if ( column.dataType != CalpontSystemCatalog::DATETIME ) + if ( column.dataType != CalpontSystemCatalog::DATETIME && + column.dataType != CalpontSystemCatalog::TIME ) { if (nullFlag) { @@ -976,6 +977,59 @@ void BulkLoadBuffer::convert(char* field, int fieldLength, pVal = &llVal; } + else if (column.dataType == CalpontSystemCatalog::TIME) + { + // time conversion + int rc = 0; + + if (nullFlag) + { + if (column.fWithDefault) + { + llDate = column.fDefaultInt; + // fall through to update saturation and min/max + } + else + { + llDate = joblist::TIMENULL; + pVal = &llDate; + break; + } + } + else + { + if (fImportDataMode != IMPORT_DATA_TEXT) + { + memcpy(&llDate, field, sizeof(llDate)); + + if (!dataconvert::DataConvert::isColumnTimeValid( + llDate)) + rc = -1; + } + else + { + llDate = dataconvert::DataConvert::convertColumnTime( + field, dataconvert::CALPONTTIME_ENUM, + rc, fieldLength ); + } + } + + if (rc == 0) + { + if (llDate < bufStats.minBufferVal) + bufStats.minBufferVal = llDate; + + if (llDate > bufStats.maxBufferVal) + bufStats.maxBufferVal = llDate; + } + else + { + llDate = 0; + bufStats.satCount++; + } + + pVal = &llDate; + } else { // datetime conversion @@ -2973,6 +3027,11 @@ bool BulkLoadBuffer::isBinaryFieldNull(void* val, if ((*(uint64_t*)val) == joblist::DATETIMENULL) isNullFlag = true; } + else if (dt == execplan::CalpontSystemCatalog::TIME) + { + if ((*(uint64_t*)val) == joblist::TIMENULL) + isNullFlag = true; + } else { if ((*(uint64_t*)val) == joblist::BIGINTNULL) diff --git a/writeengine/bulk/we_tableinfo.cpp b/writeengine/bulk/we_tableinfo.cpp index 8da9ce968..da28204d2 100644 --- a/writeengine/bulk/we_tableinfo.cpp +++ b/writeengine/bulk/we_tableinfo.cpp @@ -978,6 +978,11 @@ void TableInfo::reportTotals(double elapsedTime) ossSatCnt << "invalid date/times replaced with zero value : "; } + else if (fColumns[i].column.dataType == CalpontSystemCatalog::TIME) + { + ossSatCnt << + "invalid times replaced with zero value : "; + } else if (fColumns[i].column.dataType == CalpontSystemCatalog::CHAR) ossSatCnt << "character strings truncated: "; diff --git a/writeengine/server/we_ddlcommon.h b/writeengine/server/we_ddlcommon.h index aca25dc39..1d482b442 100644 --- a/writeengine/server/we_ddlcommon.h +++ b/writeengine/server/we_ddlcommon.h @@ -274,6 +274,13 @@ inline boost::any getNullValueForType(const execplan::CalpontSystemCatalog::ColT } break; + case execplan::CalpontSystemCatalog::TIME: + { + long long d = joblist::TIMENULL; + value = d; + } + break; + case execplan::CalpontSystemCatalog::CHAR: { std::string charnull; @@ -433,6 +440,10 @@ inline int convertDataType(int dataType) calpontDataType = CalpontSystemCatalog::DATETIME; break; + case ddlpackage::DDL_TIME: + calpontDataType = CalpontSystemCatalog::TIME; + break; + case ddlpackage::DDL_CLOB: calpontDataType = CalpontSystemCatalog::CLOB; break; diff --git a/writeengine/server/we_dmlcommandproc.cpp b/writeengine/server/we_dmlcommandproc.cpp index 07037a1e3..86625d013 100644 --- a/writeengine/server/we_dmlcommandproc.cpp +++ b/writeengine/server/we_dmlcommandproc.cpp @@ -1781,6 +1781,7 @@ uint8_t WE_DMLCommandProc::processBatchInsertBinary(messageqcpp::ByteStream& bs, case execplan::CalpontSystemCatalog::BIGINT: case execplan::CalpontSystemCatalog::DATETIME: + case execplan::CalpontSystemCatalog::TIME: case execplan::CalpontSystemCatalog::UBIGINT: bs >> val64; @@ -2981,6 +2982,13 @@ uint8_t WE_DMLCommandProc::processUpdate(messageqcpp::ByteStream& bs, break; } + case CalpontSystemCatalog::TIME: + { + intColVal = row.getIntField<8>(fetchColPos); + value = DataConvert::timeToString(intColVal, colType.precision); + break; + } + case CalpontSystemCatalog::CHAR: case CalpontSystemCatalog::VARCHAR: { @@ -3309,6 +3317,13 @@ uint8_t WE_DMLCommandProc::processUpdate(messageqcpp::ByteStream& bs, break; } + case CalpontSystemCatalog::TIME: + { + intColVal = row.getIntField<8>(fetchColPos); + value = DataConvert::timeToString(intColVal, colType.precision); + break; + } + case CalpontSystemCatalog::CHAR: case CalpontSystemCatalog::VARCHAR: { diff --git a/writeengine/shared/we_convertor.cpp b/writeengine/shared/we_convertor.cpp index dc9754355..8050426f3 100644 --- a/writeengine/shared/we_convertor.cpp +++ b/writeengine/shared/we_convertor.cpp @@ -420,6 +420,7 @@ void Convertor::convertColType(CalpontSystemCatalog::ColDataType dataType, // Map BIGINT and DATETIME to WR_LONGLONG case CalpontSystemCatalog::BIGINT : case CalpontSystemCatalog::DATETIME : + case CalpontSystemCatalog::TIME : internalType = WriteEngine::WR_LONGLONG; break; @@ -629,6 +630,7 @@ void Convertor::convertColType(ColStruct* curStruct) // Map BIGINT and DATETIME to WR_LONGLONG case CalpontSystemCatalog::BIGINT : case CalpontSystemCatalog::DATETIME : + case CalpontSystemCatalog::TIME : *internalType = WriteEngine::WR_LONGLONG; break; @@ -792,6 +794,7 @@ int Convertor::getCorrectRowWidth(CalpontSystemCatalog::ColDataType dataType, in break; case CalpontSystemCatalog::DATETIME: + case CalpontSystemCatalog::TIME: newWidth = 8; break; diff --git a/writeengine/splitter/we_sdhandler.cpp b/writeengine/splitter/we_sdhandler.cpp index 68d5bf784..802fd108b 100644 --- a/writeengine/splitter/we_sdhandler.cpp +++ b/writeengine/splitter/we_sdhandler.cpp @@ -1907,6 +1907,10 @@ void WESDHandler::onCleanupResult(int PmId, messageqcpp::SBS& Sbs) ossSatCnt << "invalid date/times replaced with zero value: "; break; + case CalpontSystemCatalog::TIME: + ossSatCnt << "invalid times replaced with zero value: "; + break; + case CalpontSystemCatalog::CHAR: ossSatCnt << "character strings truncated: "; break; diff --git a/writeengine/wrapper/writeengine.cpp b/writeengine/wrapper/writeengine.cpp index cb6eb3745..5d3dfec85 100644 --- a/writeengine/wrapper/writeengine.cpp +++ b/writeengine/wrapper/writeengine.cpp @@ -443,6 +443,8 @@ void WriteEngineWrapper::convertValue(const ColType colType, void* valArray, con case WriteEngine::WR_LONGLONG: if (data.type() == typeid(long long)) ((long long*)valArray)[pos] = boost::any_cast(data); + else if (data.type() == typeid(long)) + ((long long*)valArray)[pos] = (long long)boost::any_cast(data); else ((long long*)valArray)[pos] = boost::any_cast(data); diff --git a/writeengine/xml/we_xmljob.cpp b/writeengine/xml/we_xmljob.cpp index 135754ac1..8d755b824 100644 --- a/writeengine/xml/we_xmljob.cpp +++ b/writeengine/xml/we_xmljob.cpp @@ -1116,6 +1116,22 @@ void XMLJob::fillInXMLDataNotNullDefault( break; } + case execplan::CalpontSystemCatalog::TIME: + { + int convertStatus; + int64_t dt = + dataconvert::DataConvert::convertColumnTime( + col_defaultValue.c_str(), + dataconvert::CALPONTTIME_ENUM, convertStatus, + col_defaultValue.length() ); + + if (convertStatus != 0) + bDefaultConvertError = true; + + col.fDefaultInt = dt; + break; + } + case execplan::CalpontSystemCatalog::FLOAT: case execplan::CalpontSystemCatalog::DOUBLE: case execplan::CalpontSystemCatalog::UFLOAT: From edb2e2f36d2f9ff40e7b4c6413cdc1b5b29e151f Mon Sep 17 00:00:00 2001 From: Andrew Hutchings Date: Thu, 26 Apr 2018 14:28:17 +0100 Subject: [PATCH 33/57] MCOL-392 Fix negative time handling --- utils/dataconvert/dataconvert.cpp | 12 ++++++++---- utils/dataconvert/dataconvert.h | 24 +++++++++++++++++++++--- utils/funcexp/func_bitand.cpp | 7 ++++++- utils/rowgroup/rowaggregation.cpp | 10 +++++++++- 4 files changed, 44 insertions(+), 9 deletions(-) diff --git a/utils/dataconvert/dataconvert.cpp b/utils/dataconvert/dataconvert.cpp index b5b1ae619..013e3f05c 100644 --- a/utils/dataconvert/dataconvert.cpp +++ b/utils/dataconvert/dataconvert.cpp @@ -863,12 +863,11 @@ bool mysql_str_to_time( const string& input, Time& output ) { int32_t datesepct = 0; uint32_t dtend = 0; + bool isNeg = false; /** * We need to deal with the time portion. * The rules are: - * - Time portion may be empty - * - Time portion may start with 'T' * - Time portion always ends with '\0' * - Time portion always starts with hour * - Without time separators (':'): @@ -890,7 +889,7 @@ bool mysql_str_to_time( const string& input, Time& output ) uint32_t timesep_ct = 0; bool has_usec = false; uint32_t len_before_msec = 0; - uint32_t tmstart = ( input[dtend] == ' ' || input[dtend] == 'T' ) ? dtend + 1 : dtend; + uint32_t tmstart = dtend; uint32_t tmend = tmstart; for ( ; tmend < input.length(); ++tmend ) @@ -918,6 +917,11 @@ bool mysql_str_to_time( const string& input, Time& output ) len_before_msec = ( tmend - tmstart ); has_usec = true; } + else if (c == '-' && (tmend == tmstart)) + { + isNeg = true; + ++tmstart; + } else { timesep_ct++; @@ -998,7 +1002,7 @@ bool mysql_str_to_time( const string& input, Time& output ) return false; } - output.hour = hour; + output.hour = isNeg ? 0-hour : hour; output.minute = min; output.second = sec; output.msecond = usec; diff --git a/utils/dataconvert/dataconvert.h b/utils/dataconvert/dataconvert.h index 7f148daf8..1c7b7f663 100644 --- a/utils/dataconvert/dataconvert.h +++ b/utils/dataconvert/dataconvert.h @@ -566,10 +566,19 @@ inline void DataConvert::datetimeToString( long long datetimevalue, char* buf, u inline void DataConvert::timeToString( long long timevalue, char* buf, unsigned int buflen ) { + // Handle negative correctly + int hour = 0; + if ((timevalue >> 40) & 0xf00) + { + hour = 0xfffff000; + } + + hour |= ((timevalue >> 40) & 0xfff); + if ((timevalue & 0xffffff) > 0) { snprintf( buf, buflen, "%02d:%02d:%02d.%d", - (unsigned)((timevalue >> 40) & 0xfff), + hour, (unsigned)((timevalue >> 32) & 0xff), (unsigned)((timevalue >> 24) & 0xff), (unsigned)((timevalue) & 0xffffff) @@ -578,7 +587,7 @@ inline void DataConvert::timeToString( long long timevalue, char* buf, unsigned else { snprintf( buf, buflen, "%02d:%02d:%02d", - (unsigned)((timevalue >> 40) & 0xfff), + hour, (unsigned)((timevalue >> 32) & 0xff), (unsigned)((timevalue >> 24) & 0xff) ); @@ -608,8 +617,17 @@ inline void DataConvert::datetimeToString1( long long datetimevalue, char* buf, inline void DataConvert::timeToString1( long long timevalue, char* buf, unsigned int buflen ) { + // Handle negative correctly + int hour = 0; + if ((timevalue >> 40) & 0xf00) + { + hour = 0xfffff000; + } + + hour |= ((timevalue >> 40) & 0xfff); + snprintf( buf, buflen, "%02d%02d%02d", - (unsigned)((timevalue >> 40) & 0xfff), + hour, (unsigned)((timevalue >> 32) & 0xff), (unsigned)((timevalue >> 14) & 0xff) ); diff --git a/utils/funcexp/func_bitand.cpp b/utils/funcexp/func_bitand.cpp index eceb4e4c0..3966d7a42 100644 --- a/utils/funcexp/func_bitand.cpp +++ b/utils/funcexp/func_bitand.cpp @@ -165,8 +165,13 @@ int64_t Func_bitand::getIntVal(Row& row, min = 0, sec = 0, msec = 0; + // Handle negative correctly + if ((time >> 40) & 0xf00) + { + hour = 0xfffff000; + } - hour = (uint32_t)((time >> 40) & 0xfff); + hour |= ((time >> 40) & 0xfff); min = (uint32_t)((time >> 32) & 0xff); sec = (uint32_t)((time >> 24) & 0xff); msec = (uint32_t)(time & 0xffffff); diff --git a/utils/rowgroup/rowaggregation.cpp b/utils/rowgroup/rowaggregation.cpp index a7edd9248..b6d72fede 100644 --- a/utils/rowgroup/rowaggregation.cpp +++ b/utils/rowgroup/rowaggregation.cpp @@ -1541,7 +1541,15 @@ void RowAggregation::doBitOp(const Row& rowIn, int64_t colIn, int64_t colOut, in case execplan::CalpontSystemCatalog::TIME: { int64_t dtm = rowIn.getUintField(colIn); - valIn = (((dtm >> 40) & 0xfff) * 10000000000) + + // Handle negative correctly + int hour = 0; + if ((dtm >> 40) & 0xf00) + { + hour = 0xfffff000; + } + + hour |= ((dtm >> 40) & 0xfff); + valIn = (hour * 10000000000) + (((dtm >> 32) & 0xff) * 100000000) + (((dtm >> 24) & 0xff) * 1000000) + (dtm & 0xffffff); break; } From dba04e8b72d31c955d4371f41d48d982832654c4 Mon Sep 17 00:00:00 2001 From: Andrew Hutchings Date: Thu, 26 Apr 2018 17:13:24 +0100 Subject: [PATCH 34/57] MCOL-392 Fix cpimport and >8bit positive hour --- utils/dataconvert/dataconvert.cpp | 81 ++++++++++++++++++------------- utils/dataconvert/dataconvert.h | 2 +- utils/funcexp/func_bitand.cpp | 2 +- utils/rowgroup/rowaggregation.cpp | 2 +- writeengine/shared/we_type.h | 3 +- 5 files changed, 53 insertions(+), 37 deletions(-) diff --git a/utils/dataconvert/dataconvert.cpp b/utils/dataconvert/dataconvert.cpp index 013e3f05c..ea0a4f2c9 100644 --- a/utils/dataconvert/dataconvert.cpp +++ b/utils/dataconvert/dataconvert.cpp @@ -1886,11 +1886,11 @@ int64_t DataConvert::convertColumnTime( unsigned int dataOrgLen ) { status = 0; - const char* p; - p = dataOrg; - char fld[10]; - int16_t value = 0; - int inYear, inMonth, inDay, inHour, inMinute, inSecond, inMicrosecond; + char* p; + char* savePoint = NULL; + p = const_cast(dataOrg); + int64_t value = 0; + int inHour, inMinute, inSecond, inMicrosecond; inHour = 0; inMinute = 0; inSecond = 0; @@ -1901,44 +1901,59 @@ int64_t DataConvert::convertColumnTime( return value; } - memcpy( fld, p, 2); - fld[2] = '\0'; + errno = 0; - inHour = strtol(fld, 0, 10); + p = strtok_r(p, ":.", &savePoint); + inHour = strtol(p, 0, 10); - if (!isdigit(p[2]) || !isdigit(p[3])) - { - status = -1; - return value; - } + if (errno) + { + status = -1; + return value; + } - memcpy( fld, p + 2, 2); - fld[2] = '\0'; + p = strtok_r(NULL, ":.", &savePoint); + if (p == NULL) + { + status = -1; + return value; + } - inMinute = strtol(fld, 0, 10); + inMinute = strtol(p, 0, 10); - if (!isdigit(p[4]) || !isdigit(p[5])) - { - status = -1; - return value; - } + if (errno) + { + status = -1; + return value; + } - memcpy( fld, p + 4, 2); - fld[2] = '\0'; + p = strtok_r(NULL, ":.", &savePoint); - inSecond = strtol(fld, 0, 10); + if (p == NULL) + { + status = -1; + return value; + } - if (dataOrgLen > 9) - { - unsigned int microFldLen = dataOrgLen - 9; + inSecond = strtol(p, 0, 10); - if (microFldLen > (sizeof(fld) - 1)) - microFldLen = sizeof(fld) - 1; + if (errno) + { + status = -1; + return value; + } - memcpy( fld, p + 9, microFldLen); - fld[microFldLen] = '\0'; - inMicrosecond = strtol(fld, 0, 10); - } + p = strtok_r(NULL, ":.", &savePoint); + + if (p != NULL) + { + inMicrosecond = strtol(p, 0, 10); + if (errno) + { + status = -1; + return value; + } + } if ( isTimeValid (inHour, inMinute, inSecond, inMicrosecond) ) { diff --git a/utils/dataconvert/dataconvert.h b/utils/dataconvert/dataconvert.h index 1c7b7f663..057d617ef 100644 --- a/utils/dataconvert/dataconvert.h +++ b/utils/dataconvert/dataconvert.h @@ -568,7 +568,7 @@ inline void DataConvert::timeToString( long long timevalue, char* buf, unsigned { // Handle negative correctly int hour = 0; - if ((timevalue >> 40) & 0xf00) + if ((timevalue >> 40) & 0x800) { hour = 0xfffff000; } diff --git a/utils/funcexp/func_bitand.cpp b/utils/funcexp/func_bitand.cpp index 3966d7a42..32cb846fe 100644 --- a/utils/funcexp/func_bitand.cpp +++ b/utils/funcexp/func_bitand.cpp @@ -166,7 +166,7 @@ int64_t Func_bitand::getIntVal(Row& row, sec = 0, msec = 0; // Handle negative correctly - if ((time >> 40) & 0xf00) + if ((time >> 40) & 0x800) { hour = 0xfffff000; } diff --git a/utils/rowgroup/rowaggregation.cpp b/utils/rowgroup/rowaggregation.cpp index b6d72fede..c496d91e6 100644 --- a/utils/rowgroup/rowaggregation.cpp +++ b/utils/rowgroup/rowaggregation.cpp @@ -1543,7 +1543,7 @@ void RowAggregation::doBitOp(const Row& rowIn, int64_t colIn, int64_t colOut, in int64_t dtm = rowIn.getUintField(colIn); // Handle negative correctly int hour = 0; - if ((dtm >> 40) & 0xf00) + if ((dtm >> 40) & 0x800) { hour = 0xfffff000; } diff --git a/writeengine/shared/we_type.h b/writeengine/shared/we_type.h index 21fad3e08..06a367ecb 100644 --- a/writeengine/shared/we_type.h +++ b/writeengine/shared/we_type.h @@ -161,7 +161,8 @@ const char ColDataTypeStr[execplan::CalpontSystemCatalog::NUM_OF_COL_DATA_TYPE] "unsigned-float", "unsigned-bigint", "unsigned-double", - "text" + "text", + "time" }; enum FuncType { FUNC_WRITE_ENGINE, FUNC_INDEX, FUNC_DICTIONARY }; From bd50bbb8bb37f3d2d9e15f601c56b9126c67d273 Mon Sep 17 00:00:00 2001 From: Andrew Hutchings Date: Thu, 26 Apr 2018 17:48:16 +0100 Subject: [PATCH 35/57] MCOL-392 Fix saturation handling --- utils/dataconvert/dataconvert.cpp | 42 +++++++++++++++++++++++++- writeengine/bulk/we_bulkloadbuffer.cpp | 1 - 2 files changed, 41 insertions(+), 2 deletions(-) diff --git a/utils/dataconvert/dataconvert.cpp b/utils/dataconvert/dataconvert.cpp index ea0a4f2c9..bfdb25239 100644 --- a/utils/dataconvert/dataconvert.cpp +++ b/utils/dataconvert/dataconvert.cpp @@ -998,7 +998,26 @@ bool mysql_str_to_time( const string& input, Time& output ) if ( !isTimeValid( hour, min, sec, usec ) ) { - output.reset(); + // Emulate MariaDB's time saturation + if (hour > 838) + { + output.hour = 838; + output.minute = 59; + output.second = 59; + output.msecond = 999999; + } + else if (hour < -838) + { + output.hour = -838; + output.minute = 59; + output.second = 59; + output.msecond = 999999; + } + // If neither of the above match then we return a 0 time + else + { + output.reset(); + } return false; } @@ -1967,6 +1986,27 @@ int64_t DataConvert::convertColumnTime( } else { + // Emulate MariaDB's time saturation + if (inHour > 838) + { + Time atime; + atime.hour = 838; + atime.minute = 59; + atime.second = 59; + atime.msecond = 999999; + memcpy( &value, &atime, 8); + } + else if (inHour < -838) + { + Time atime; + atime.hour = -838; + atime.minute = 59; + atime.second = 59; + atime.msecond = 999999; + memcpy( &value, &atime, 8); + } + // If neither of the above match then we return a 0 time + status = -1; } diff --git a/writeengine/bulk/we_bulkloadbuffer.cpp b/writeengine/bulk/we_bulkloadbuffer.cpp index 79ea553f1..ccf73363f 100644 --- a/writeengine/bulk/we_bulkloadbuffer.cpp +++ b/writeengine/bulk/we_bulkloadbuffer.cpp @@ -1024,7 +1024,6 @@ void BulkLoadBuffer::convert(char* field, int fieldLength, } else { - llDate = 0; bufStats.satCount++; } From 957dc446151d340abe783cebf9a415a2d03256de Mon Sep 17 00:00:00 2001 From: Andrew Hutchings Date: Fri, 27 Apr 2018 21:18:14 +0100 Subject: [PATCH 36/57] MCOL-392 Function fixes Fixes most of the functions in funcexp so that time and datetime's microseconds are handled correctly --- dbcon/execplan/treenode.h | 19 +++-- dbcon/mysql/ha_calpont_execplan.cpp | 3 +- utils/dataconvert/dataconvert.cpp | 12 +-- utils/dataconvert/dataconvert.h | 111 ++++++++++++++++------------ utils/funcexp/func_add_time.cpp | 49 ++++++++---- utils/funcexp/func_between.cpp | 31 ++++++-- utils/funcexp/func_extract.cpp | 97 +++++++++++++++++++++++- utils/funcexp/func_greatest.cpp | 5 +- utils/funcexp/func_hour.cpp | 20 ++++- utils/funcexp/func_least.cpp | 5 +- utils/funcexp/func_time.cpp | 20 +++-- 11 files changed, 271 insertions(+), 101 deletions(-) diff --git a/dbcon/execplan/treenode.h b/dbcon/execplan/treenode.h index 27795ea9d..08b8b43c8 100644 --- a/dbcon/execplan/treenode.h +++ b/dbcon/execplan/treenode.h @@ -666,14 +666,14 @@ inline const std::string& TreeNode::getStrVal() case CalpontSystemCatalog::DATETIME: { - dataconvert::DataConvert::datetimeToString(fResult.intVal, tmp, 255); + dataconvert::DataConvert::datetimeToString(fResult.intVal, tmp, 255, fResultType.precision); fResult.strVal = std::string(tmp); break; } case CalpontSystemCatalog::TIME: { - dataconvert::DataConvert::timeToString(fResult.intVal, tmp, 255); + dataconvert::DataConvert::timeToString(fResult.intVal, tmp, 255, fResultType.precision); fResult.strVal = std::string(tmp); break; } @@ -992,11 +992,20 @@ inline int64_t TreeNode::getDatetimeIntVal() else if (fResultType.colDataType == execplan::CalpontSystemCatalog::TIME) { dataconvert::Time tt; + int day = 0; memcpy(&tt, &fResult.intVal, 8); - if (tt.hour > 23 || tt.hour < 0) - throw logging::InvalidConversionExcept("TreeNode::getDecimalVal: Invalid conversion from time (out of range)."); - dataconvert::DateTime dt(0, 0, 0, tt.hour, tt.minute, tt.second, tt.msecond); + // Note, this should probably be current date +/- time + if (tt.hour > 23) + { + day = tt.hour / 24; + tt.hour = tt.hour % 24; + } + else if (tt.hour < 0) + { + tt.hour = 0; + } + dataconvert::DateTime dt(0, 0, day, tt.hour, tt.minute, tt.second, tt.msecond); memcpy(&fResult.intVal, &dt, 8); return fResult.intVal; } diff --git a/dbcon/mysql/ha_calpont_execplan.cpp b/dbcon/mysql/ha_calpont_execplan.cpp index fe2f96e52..210da9992 100644 --- a/dbcon/mysql/ha_calpont_execplan.cpp +++ b/dbcon/mysql/ha_calpont_execplan.cpp @@ -3518,8 +3518,7 @@ ReturnedColumn* buildFunctionColumn(Item_func* ifp, gp_walk_info& gwi, bool& non if (ifp->field_type() == MYSQL_TYPE_DATETIME || ifp->field_type() == MYSQL_TYPE_DATETIME2 || ifp->field_type() == MYSQL_TYPE_TIMESTAMP || - ifp->field_type() == MYSQL_TYPE_TIMESTAMP2 || - funcName == "add_time") + ifp->field_type() == MYSQL_TYPE_TIMESTAMP2) { CalpontSystemCatalog::ColType ct; ct.colDataType = CalpontSystemCatalog::DATETIME; diff --git a/utils/dataconvert/dataconvert.cpp b/utils/dataconvert/dataconvert.cpp index bfdb25239..41dff9aa9 100644 --- a/utils/dataconvert/dataconvert.cpp +++ b/utils/dataconvert/dataconvert.cpp @@ -2065,9 +2065,9 @@ std::string DataConvert::datetimeToString( long long datetimevalue, long decima { snprintf(buf + strlen(buf), 21 + decimals, ".%d", dt.msecond); // Pad end with zeros - if (strlen(buf) < (21 + decimals)) + if (strlen(buf) < (size_t)(21 + decimals)) { - sprintf(buf + strlen(buf), "%0*d", 21 + decimals - strlen(buf), 0); + sprintf(buf + strlen(buf), "%0*d", (int)(21 + decimals - strlen(buf)), 0); } } return buf; @@ -2091,9 +2091,9 @@ std::string DataConvert::timeToString( long long timevalue, long decimals ) size_t start = strlen(buf); snprintf(buf + strlen(buf), 12 + decimals, ".%d", dt.msecond); // Pad end with zeros - if (strlen(buf) - start < decimals) + if (strlen(buf) - start < (size_t)decimals) { - sprintf(buf + strlen(buf), "%0*d", decimals - (strlen(buf) - start), 0); + sprintf(buf + strlen(buf), "%0*d", (int)(decimals - (strlen(buf) - start) + 1), 0); } } return buf; @@ -2642,7 +2642,7 @@ int64_t DataConvert::intToTime(int64_t data) atime.second = 0; atime.msecond = 0; - return *(reinterpret_cast(&atime)); + return *(reinterpret_cast(&atime)); } snprintf( buf, 15, "%llu", (long long unsigned int)data); @@ -2696,7 +2696,7 @@ int64_t DataConvert::stringToTime(const string& data) // -34 <= D <= 34 // -838 <= H <= 838 uint64_t min = 0, sec = 0, msec = 0; - int64_t day = 0, hour = 0; + int64_t day = -1, hour = 0; string time, hms, ms; char* end = NULL; diff --git a/utils/dataconvert/dataconvert.h b/utils/dataconvert/dataconvert.h index 057d617ef..8c35bec4b 100644 --- a/utils/dataconvert/dataconvert.h +++ b/utils/dataconvert/dataconvert.h @@ -173,7 +173,7 @@ struct DateTime inline int64_t DateTime::convertToMySQLint() const { - return (int64_t) (year * 10000000000000000LL) + (month * 100000000000000) + (day * 1000000000000) + (hour * 10000000000) + (minute * 100000000) + (second * 1000000) + msecond; + return (int64_t) (year * 10000000000LL) + (month * 100000000) + (day * 1000000) + (hour * 10000) + (minute * 100) + second; } inline @@ -235,16 +235,11 @@ void Time::reset() inline int64_t Time::convertToMySQLint() const { - return (int64_t) (hour * 10000000000) + (minute * 100000000) + (second * 1000000) + msecond; + return (int64_t) (hour * 10000) + (minute * 100) + second; } static uint32_t daysInMonth[13] = {31, 28, 31, 30, 31, 30, 31, 31, 30, 31, 30, 31, 0}; -inline uint32_t getDaysInMonth(uint32_t month) -{ - return ( (month < 1 || month > 12) ? 0 : daysInMonth[month - 1]); -} - inline bool isLeapYear ( int year) { if ( year % 400 == 0 ) @@ -256,6 +251,18 @@ inline bool isLeapYear ( int year) return false; } +inline uint32_t getDaysInMonth(uint32_t month, int year) +{ + if (month < 1 || month > 12) + return 0; + + uint32_t days = daysInMonth[month - 1]; + if ((month == 2) && isLeapYear(year)) + days++; + + return days; +} + inline bool isDateValid ( int day, int month, int year) { @@ -266,11 +273,7 @@ bool isDateValid ( int day, int month, int year) return true; } - int daycheck = getDaysInMonth( month ); - - if ( month == 2 && isLeapYear( year ) ) - // 29 days in February in a leap year - daycheck = 29; + int daycheck = getDaysInMonth( month, year ); if ( ( year < 1000 ) || ( year > 9999 ) ) valid = false; @@ -407,7 +410,7 @@ public: * @param data the columns string representation of it's data */ EXPORT static std::string datetimeToString( long long datetimevalue, long decimals = 0 ); - static inline void datetimeToString( long long datetimevalue, char* buf, unsigned int buflen ); + static inline void datetimeToString( long long datetimevalue, char* buf, unsigned int buflen, long decimals = 0 ); /** * @brief convert a columns data from native format to a string @@ -416,7 +419,7 @@ public: * @param data the columns string representation of it's data */ EXPORT static std::string timeToString( long long timevalue, long decimals = 0 ); - static inline void timeToString( long long timevalue, char* buf, unsigned int buflen ); + static inline void timeToString( long long timevalue, char* buf, unsigned int buflen, long decimals = 0); /** * @brief convert a columns data from native format to a string @@ -537,37 +540,48 @@ inline void DataConvert::dateToString( int datevalue, char* buf, unsigned int bu ); } -inline void DataConvert::datetimeToString( long long datetimevalue, char* buf, unsigned int buflen ) +inline void DataConvert::datetimeToString( long long datetimevalue, char* buf, unsigned int buflen, long decimals ) { + // 10 is default which means we don't need microseconds + if (decimals > 6 || decimals < 0) + { + decimals = 0; + } + int msec = 0; if ((datetimevalue & 0xfffff) > 0) { - snprintf( buf, buflen, "%04d-%02d-%02d %02d:%02d:%02d.%d", - (unsigned)((datetimevalue >> 48) & 0xffff), - (unsigned)((datetimevalue >> 44) & 0xf), - (unsigned)((datetimevalue >> 38) & 0x3f), - (unsigned)((datetimevalue >> 32) & 0x3f), - (unsigned)((datetimevalue >> 26) & 0x3f), - (unsigned)((datetimevalue >> 20) & 0x3f), - (unsigned)((datetimevalue) & 0xfffff) - ); + msec = (unsigned)((datetimevalue) & 0xfffff); } - else + snprintf( buf, buflen, "%04d-%02d-%02d %02d:%02d:%02d", + (unsigned)((datetimevalue >> 48) & 0xffff), + (unsigned)((datetimevalue >> 44) & 0xf), + (unsigned)((datetimevalue >> 38) & 0x3f), + (unsigned)((datetimevalue >> 32) & 0x3f), + (unsigned)((datetimevalue >> 26) & 0x3f), + (unsigned)((datetimevalue >> 20) & 0x3f) + ); + + if (msec || decimals) { - snprintf( buf, buflen, "%04d-%02d-%02d %02d:%02d:%02d", - (unsigned)((datetimevalue >> 48) & 0xffff), - (unsigned)((datetimevalue >> 44) & 0xf), - (unsigned)((datetimevalue >> 38) & 0x3f), - (unsigned)((datetimevalue >> 32) & 0x3f), - (unsigned)((datetimevalue >> 26) & 0x3f), - (unsigned)((datetimevalue >> 20) & 0x3f) - ); + size_t start = strlen(buf); + snprintf(buf + strlen(buf), buflen - start, ".%d", msec); + // Pad end with zeros + if (strlen(buf) - start < (size_t)decimals) + { + snprintf(buf + strlen(buf), buflen - strlen(buf), "%0*d", (int)(decimals - (strlen(buf) - start) + 1), 0); + } } } -inline void DataConvert::timeToString( long long timevalue, char* buf, unsigned int buflen ) +inline void DataConvert::timeToString( long long timevalue, char* buf, unsigned int buflen, long decimals ) { + // 10 is default which means we don't need microseconds + if (decimals > 6 || decimals < 0) + { + decimals = 0; + } // Handle negative correctly - int hour = 0; + int hour = 0, msec = 0; if ((timevalue >> 40) & 0x800) { hour = 0xfffff000; @@ -577,20 +591,23 @@ inline void DataConvert::timeToString( long long timevalue, char* buf, unsigned if ((timevalue & 0xffffff) > 0) { - snprintf( buf, buflen, "%02d:%02d:%02d.%d", - hour, - (unsigned)((timevalue >> 32) & 0xff), - (unsigned)((timevalue >> 24) & 0xff), - (unsigned)((timevalue) & 0xffffff) - ); + msec = (unsigned)((timevalue) & 0xffffff); } - else + snprintf( buf, buflen, "%02d:%02d:%02d", + hour, + (unsigned)((timevalue >> 32) & 0xff), + (unsigned)((timevalue >> 24) & 0xff) + ); + + if (msec || decimals) { - snprintf( buf, buflen, "%02d:%02d:%02d", - hour, - (unsigned)((timevalue >> 32) & 0xff), - (unsigned)((timevalue >> 24) & 0xff) - ); + size_t start = strlen(buf); + snprintf(buf + strlen(buf), buflen - start, ".%d", msec); + // Pad end with zeros + if (strlen(buf) - start < (size_t)decimals) + { + snprintf(buf + strlen(buf), buflen - strlen(buf), "%0*d", (int)(decimals - (strlen(buf) - start) + 1), 0); + } } } diff --git a/utils/funcexp/func_add_time.cpp b/utils/funcexp/func_add_time.cpp index 6fa7a5c19..0fa379bfd 100644 --- a/utils/funcexp/func_add_time.cpp +++ b/utils/funcexp/func_add_time.cpp @@ -80,16 +80,15 @@ int64_t addTime(DateTime& dt1, Time& dt2) } hour = (signed)(dt1.hour + dt2.hour + min / 60); - dt.hour = tmp = hour % 24; -// if (tmp < -1) - if (tmp < 0) // fix for subtime dlh + if ((hour < 0) || (hour > 23)) { - dt.hour = tmp + 24; - dt2.day--; + dt.hour = hour % 24; + dt2.day = hour / 24; } - day = (signed)(dt1.day + dt2.day + hour / 24); + day = (signed)(dt1.day + dt2.day); + if (isLeapYear(dt1.year) && dt1.month == 2) day--; @@ -97,7 +96,7 @@ int64_t addTime(DateTime& dt1, Time& dt2) month = dt1.month; int addyear = 0; - if (dt2.day < 0 || dt2.hour < 0) + if (day < 0) { int monthSave = month; @@ -106,7 +105,7 @@ int64_t addTime(DateTime& dt1, Time& dt2) month = (month == 1 ? 12 : month - 1); for (; day <= 0 && month > 0; month--) - day += getDaysInMonth(month); + day += getDaysInMonth(month, dt1.year); month++; // month=12; @@ -119,10 +118,10 @@ int64_t addTime(DateTime& dt1, Time& dt2) { int monthSave = month; - while (day > getDaysInMonth(month)) + while (day > getDaysInMonth(month, dt1.year)) { - for (; day > getDaysInMonth(month) && month <= 12; month++) - day -= getDaysInMonth(month); + for (; day > getDaysInMonth(month, dt1.year) && month <= 12; month++) + day -= getDaysInMonth(month, dt1.year); if (month > 12) month = 1; @@ -175,8 +174,23 @@ int64_t addTime(Time& dt1, Time& dt2) dt2.hour--; } - hour = (signed)(dt1.hour + dt2.hour + min / 60); - dt.hour = tmp = hour % 838; + dt.hour = (signed)(dt1.hour + dt2.hour + min / 60); + + // Saturation + if (dt.hour > 838) + { + dt.hour = 838; + dt.minute = 59; + dt.second = 59; + dt.msecond = 999999; + } + else if (dt.hour < -838) + { + dt.hour = -838; + dt.minute = 59; + dt.second = 59; + dt.msecond = 999999; + } return *(reinterpret_cast(&dt)); } @@ -226,6 +240,13 @@ int64_t Func_add_time::getDatetimeIntVal(rowgroup::Row& row, if (isNull) return -1; + // Adding a zero date to a time is always NULL + if (val1 == 0) + { + isNull = true; + return -1; + } + const string& val2 = parm[1]->data()->getStrVal(row, isNull); int sign = parm[2]->data()->getIntVal(row, isNull); DateTime dt1; @@ -311,7 +332,7 @@ int64_t Func_add_time::getTimeIntVal(rowgroup::Row& row, const string& val2 = parm[1]->data()->getStrVal(row, isNull); int sign = parm[2]->data()->getIntVal(row, isNull); Time dt1; - dt1.hour = (val1 >> 40) & 0xff; + dt1.hour = (val1 >> 40) & 0xfff; dt1.minute = (val1 >> 32) & 0xff; dt1.second = (val1 >> 24) & 0xff; dt1.msecond = val1 & 0xffffff; diff --git a/utils/funcexp/func_between.cpp b/utils/funcexp/func_between.cpp index 8398593c4..f23d67826 100644 --- a/utils/funcexp/func_between.cpp +++ b/utils/funcexp/func_between.cpp @@ -158,20 +158,21 @@ inline bool getBool(rowgroup::Row& row, case execplan::CalpontSystemCatalog::TIME: { - int64_t val = pm[0]->data()->getTimeIntVal(row, isNull); + // Shift out unused day for compare + int64_t val = pm[0]->data()->getTimeIntVal(row, isNull) << 12; if (notBetween) { - if (!numericGE(val, pm[1]->data()->getTimeIntVal(row, isNull)) && !isNull) + if (!numericGE(val, pm[1]->data()->getTimeIntVal(row, isNull) << 12) && !isNull) return true; isNull = false; - return (!numericLE(val, pm[2]->data()->getTimeIntVal(row, isNull)) && !isNull); + return (!numericLE(val, pm[2]->data()->getTimeIntVal(row, isNull) << 12) && !isNull); } return !isNull && - numericGE(val, pm[1]->data()->getTimeIntVal(row, isNull)) && - numericLE(val, pm[2]->data()->getTimeIntVal(row, isNull)); + numericGE(val, pm[1]->data()->getTimeIntVal(row, isNull) << 12) && + numericLE(val, pm[2]->data()->getTimeIntVal(row, isNull) << 12); } case execplan::CalpontSystemCatalog::DOUBLE: @@ -265,7 +266,8 @@ CalpontSystemCatalog::ColType Func_between::operationType( FunctionParm& fp, Cal fp[i]->data()->resultType().colDataType != CalpontSystemCatalog::TEXT && fp[i]->data()->resultType().colDataType != CalpontSystemCatalog::VARCHAR) || ct.colDataType == CalpontSystemCatalog::DATE || - ct.colDataType == CalpontSystemCatalog::DATETIME) + ct.colDataType == CalpontSystemCatalog::DATETIME || + ct.colDataType == CalpontSystemCatalog::TIME) { allString = false; } @@ -293,6 +295,23 @@ CalpontSystemCatalog::ColType Func_between::operationType( FunctionParm& fp, Cal } } } + else if (op.operationType().colDataType == CalpontSystemCatalog::TIME) + { + ConstantColumn* cc = NULL; + + for (uint32_t i = 1; i < fp.size(); i++) + { + cc = dynamic_cast(fp[i]->data()); + + if (cc) + { + Result result = cc->result(); + result.intVal = dataconvert::DataConvert::timeToInt(result.strVal); + cc->result(result); + } + } + } + return ct; } diff --git a/utils/funcexp/func_extract.cpp b/utils/funcexp/func_extract.cpp index d5a6cc5b5..67fc70a4d 100644 --- a/utils/funcexp/func_extract.cpp +++ b/utils/funcexp/func_extract.cpp @@ -131,6 +131,90 @@ long long dateGet( uint64_t time, IntervalColumn::interval_type unit, bool dateT throw runtime_error("unit type is not supported: " + unit); }; } + +long long timeGet( uint64_t time, IntervalColumn::interval_type unit ) +{ + int32_t hour = 0, + min = 0, + sec = 0, + msec = 0, + day = 0; + + min = (int32_t)((time >> 32) & 0xff); + sec = (int32_t)((time >> 24) & 0xff); + msec = (int32_t)((time & 0xfffff)); + + // If negative, mask so it doesn't turn positive + int64_t mask = 0; + if ((time >> 40) & 0x800) + mask = 0xfffffffffffff000; + hour = mask | ((time >> 40) & 0xfff); + + // Always positive! + day = abs(hour / 24); + + switch ( unit ) + { + case IntervalColumn::INTERVAL_YEAR: + case IntervalColumn::INTERVAL_MONTH: + return 0; + + case IntervalColumn::INTERVAL_DAY: + return day; + + case IntervalColumn::INTERVAL_HOUR: + return hour; + + case IntervalColumn::INTERVAL_MINUTE: + return min; + + case IntervalColumn::INTERVAL_SECOND: + return sec; + + case IntervalColumn::INTERVAL_MICROSECOND: + return msec; + + case IntervalColumn::INTERVAL_QUARTER: + case IntervalColumn::INTERVAL_WEEK: + case IntervalColumn::INTERVAL_YEAR_MONTH: + return 0; + + case IntervalColumn::INTERVAL_DAY_HOUR: + return (day * 100) + hour; + + case IntervalColumn::INTERVAL_DAY_MINUTE: + return (day * 10000) + (hour * 100) + min; + + case IntervalColumn::INTERVAL_DAY_SECOND: + return (day * 1000000) + (hour * 10000) + (min * 100) + sec; + + case IntervalColumn::INTERVAL_HOUR_MINUTE: + return (hour * 100) + min; + + case IntervalColumn::INTERVAL_HOUR_SECOND: + return (hour * 10000) + (min * 100) + sec; + + case IntervalColumn::INTERVAL_MINUTE_SECOND: + return (min * 100) + sec; + + case IntervalColumn::INTERVAL_DAY_MICROSECOND: + return (((day * 1000000) + (hour * 10000) + (min * 100) + sec) * 1000000) + msec; + + case IntervalColumn::INTERVAL_HOUR_MICROSECOND: + return (((hour * 10000) + (min * 100) + sec) * 1000000) + msec; + + case IntervalColumn::INTERVAL_MINUTE_MICROSECOND: + return (((min * 100) + sec) * 1000000) + msec; + + case IntervalColumn::INTERVAL_SECOND_MICROSECOND: + return (sec * 1000000) + msec; + + default: + throw runtime_error("unit type is not supported: " + unit); + }; +} + + } namespace funcexp @@ -148,16 +232,21 @@ int64_t Func_extract::getIntVal(rowgroup::Row& row, { IntervalColumn::interval_type unit = static_cast(parm[1]->data()->getIntVal(row, isNull)); uint64_t time; + bool isTime = false; //@bug4678 handle conversion from non date/datetime datatype switch (parm[0]->data()->resultType().colDataType) { case CalpontSystemCatalog::DATE: case CalpontSystemCatalog::DATETIME: - case CalpontSystemCatalog::TIME: time = parm[0]->data()->getDatetimeIntVal(row, isNull); break; + case CalpontSystemCatalog::TIME: + time = parm[0]->data()->getTimeIntVal(row, isNull); + isTime = true; + break; + case CalpontSystemCatalog::VARCHAR: case CalpontSystemCatalog::CHAR: case CalpontSystemCatalog::TEXT: @@ -182,7 +271,11 @@ int64_t Func_extract::getIntVal(rowgroup::Row& row, time = parm[0]->data()->getIntVal(row, isNull); } - long long value = dateGet( time, unit, false ); + long long value; + if (isTime) + value = timeGet( time, unit ); + else + value = dateGet( time, unit, false ); return value; } diff --git a/utils/funcexp/func_greatest.cpp b/utils/funcexp/func_greatest.cpp index 50ff4204a..11650ddca 100644 --- a/utils/funcexp/func_greatest.cpp +++ b/utils/funcexp/func_greatest.cpp @@ -211,13 +211,14 @@ int64_t Func_greatest::getTimeIntVal(rowgroup::Row& row, bool& isNull, execplan::CalpontSystemCatalog::ColType& ct) { - int64_t str = fp[0]->data()->getTimeIntVal(row, isNull); + // Strip off unused day + int64_t str = fp[0]->data()->getTimeIntVal(row, isNull) << 12; int64_t greatestStr = str; for (uint32_t i = 1; i < fp.size(); i++) { - int64_t str1 = fp[i]->data()->getTimeIntVal(row, isNull); + int64_t str1 = fp[i]->data()->getTimeIntVal(row, isNull) << 12; if ( greatestStr < str1 ) greatestStr = str1; diff --git a/utils/funcexp/func_hour.cpp b/utils/funcexp/func_hour.cpp index b9e3da9ec..353d0cc4e 100644 --- a/utils/funcexp/func_hour.cpp +++ b/utils/funcexp/func_hour.cpp @@ -48,6 +48,7 @@ int64_t Func_hour::getIntVal(rowgroup::Row& row, CalpontSystemCatalog::ColType& op_ct) { int64_t val = 0; + bool isTime = false; switch (parm[0]->data()->resultType().colDataType) { @@ -110,7 +111,8 @@ int64_t Func_hour::getIntVal(rowgroup::Row& row, case execplan::CalpontSystemCatalog::TIME: { - val = parm[0]->data()->getDatetimeIntVal(row, isNull); + isTime = true; + val = parm[0]->data()->getTimeIntVal(row, isNull); break; } @@ -123,10 +125,20 @@ int64_t Func_hour::getIntVal(rowgroup::Row& row, if (isNull) return -1; - if ( val < 1000000000 ) - return 0; + if (isTime) + { + // If negative, mask so it doesn't turn positive + int64_t mask = 0; + if ((val >> 40) & 0x800) + mask = 0xfffffffffffff000; + val = mask | ((val >> 40) & 0xfff); + } + else + { + val = (val >> 32) & 0x3f; + } - return (uint32_t)((val >> 32) & 0x3f); + return val; } diff --git a/utils/funcexp/func_least.cpp b/utils/funcexp/func_least.cpp index ef71e44e6..70a64f267 100644 --- a/utils/funcexp/func_least.cpp +++ b/utils/funcexp/func_least.cpp @@ -187,13 +187,14 @@ int64_t Func_least::getTimeIntVal(rowgroup::Row& row, bool& isNull, execplan::CalpontSystemCatalog::ColType& op_ct) { - int64_t str = fp[0]->data()->getTimeIntVal(row, isNull); + // Strip off unused day + int64_t str = fp[0]->data()->getTimeIntVal(row, isNull) << 12; int64_t leastStr = str; for (uint32_t i = 1; i < fp.size(); i++) { - int64_t str1 = fp[i]->data()->getTimeIntVal(row, isNull); + int64_t str1 = fp[i]->data()->getTimeIntVal(row, isNull) << 12; if ( leastStr > str1 ) leastStr = str1; diff --git a/utils/funcexp/func_time.cpp b/utils/funcexp/func_time.cpp index 8aab1389b..bd28d6e6c 100644 --- a/utils/funcexp/func_time.cpp +++ b/utils/funcexp/func_time.cpp @@ -59,7 +59,7 @@ string Func_time::getStrVal(rowgroup::Row& row, case execplan::CalpontSystemCatalog::TINYINT: case execplan::CalpontSystemCatalog::SMALLINT: { - val = dataconvert::DataConvert::intToDatetime(parm[0]->data()->getIntVal(row, isNull)); + val = dataconvert::DataConvert::intToTime(parm[0]->data()->getIntVal(row, isNull)); if (val == -1) isNull = true; @@ -73,7 +73,7 @@ string Func_time::getStrVal(rowgroup::Row& row, { if (parm[0]->data()->resultType().scale) { - val = dataconvert::DataConvert::intToDatetime(parm[0]->data()->getIntVal(row, isNull)); + val = dataconvert::DataConvert::intToTime(parm[0]->data()->getIntVal(row, isNull)); if (val == -1) isNull = true; @@ -96,7 +96,7 @@ string Func_time::getStrVal(rowgroup::Row& row, case execplan::CalpontSystemCatalog::CHAR: case execplan::CalpontSystemCatalog::TEXT: { - val = dataconvert::DataConvert::stringToDatetime(parm[0]->data()->getStrVal(row, isNull)); + val = dataconvert::DataConvert::stringToTime(parm[0]->data()->getStrVal(row, isNull)); if (val == -1) isNull = true; @@ -106,16 +106,14 @@ string Func_time::getStrVal(rowgroup::Row& row, break; } - case execplan::CalpontSystemCatalog::DATE: + case execplan::CalpontSystemCatalog::TIME: { - val = parm[0]->data()->getDatetimeIntVal(row, isNull); + val = parm[0]->data()->getTimeIntVal(row, isNull); break; } - - case execplan::CalpontSystemCatalog::TIME: case execplan::CalpontSystemCatalog::DATETIME: { - val = parm[0]->data()->getDatetimeIntVal(row, isNull); + val = parm[0]->data()->getTimeIntVal(row, isNull); break; } @@ -129,9 +127,9 @@ string Func_time::getStrVal(rowgroup::Row& row, return ""; char buf[30] = {'\0'}; - dataconvert::DataConvert::datetimeToString(val, buf, sizeof(buf)); + dataconvert::DataConvert::timeToString(val, buf, sizeof(buf)); string time(buf); - return time.substr(11, 80); + return time; } int64_t Func_time::getIntVal(rowgroup::Row& row, @@ -139,7 +137,7 @@ int64_t Func_time::getIntVal(rowgroup::Row& row, bool& isNull, execplan::CalpontSystemCatalog::ColType& op_ct) { - return dataconvert::DataConvert::datetimeToInt(getStrVal(row, fp, isNull, op_ct)); + return dataconvert::DataConvert::timeToInt(getStrVal(row, fp, isNull, op_ct)); } double Func_time::getDoubleVal(rowgroup::Row& row, From aae2810d73099a7d33b674a8df384ab868ca0632 Mon Sep 17 00:00:00 2001 From: Andrew Hutchings Date: Mon, 30 Apr 2018 07:56:47 +0100 Subject: [PATCH 37/57] MCOL-392 Fix time() for a DATETIME value --- utils/dataconvert/dataconvert.h | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/utils/dataconvert/dataconvert.h b/utils/dataconvert/dataconvert.h index 8c35bec4b..0992cd843 100644 --- a/utils/dataconvert/dataconvert.h +++ b/utils/dataconvert/dataconvert.h @@ -216,7 +216,7 @@ struct Time {} Time(signed d, signed h, signed min, signed sec, signed msec) : - msecond(sec), second(sec), minute(min), hour(h), day(d) {} + msecond(msec), second(sec), minute(min), hour(h), day(d) {} int64_t convertToMySQLint() const; void reset(); From fc05a9c6c230fb30b14725e4ec1aae1c73e7388a Mon Sep 17 00:00:00 2001 From: Andrew Hutchings Date: Mon, 30 Apr 2018 08:08:42 +0100 Subject: [PATCH 38/57] MCOL-392 Fix greatest() and least() for time --- utils/funcexp/func_greatest.cpp | 12 ++++++++---- utils/funcexp/func_least.cpp | 12 ++++++++---- 2 files changed, 16 insertions(+), 8 deletions(-) diff --git a/utils/funcexp/func_greatest.cpp b/utils/funcexp/func_greatest.cpp index 11650ddca..2d7dd15d6 100644 --- a/utils/funcexp/func_greatest.cpp +++ b/utils/funcexp/func_greatest.cpp @@ -212,16 +212,20 @@ int64_t Func_greatest::getTimeIntVal(rowgroup::Row& row, execplan::CalpontSystemCatalog::ColType& ct) { // Strip off unused day - int64_t str = fp[0]->data()->getTimeIntVal(row, isNull) << 12; + int64_t greatestStr = fp[0]->data()->getTimeIntVal(row, isNull); - int64_t greatestStr = str; + int64_t str = greatestStr << 12; for (uint32_t i = 1; i < fp.size(); i++) { - int64_t str1 = fp[i]->data()->getTimeIntVal(row, isNull) << 12; + int64_t str1 = fp[i]->data()->getTimeIntVal(row, isNull); + int64_t str2 = str1 << 12; - if ( greatestStr < str1 ) + if ( str < str1 ) + { greatestStr = str1; + str = str2; + } } return greatestStr; diff --git a/utils/funcexp/func_least.cpp b/utils/funcexp/func_least.cpp index 70a64f267..2f645af5f 100644 --- a/utils/funcexp/func_least.cpp +++ b/utils/funcexp/func_least.cpp @@ -188,16 +188,20 @@ int64_t Func_least::getTimeIntVal(rowgroup::Row& row, execplan::CalpontSystemCatalog::ColType& op_ct) { // Strip off unused day - int64_t str = fp[0]->data()->getTimeIntVal(row, isNull) << 12; + int64_t leastStr = fp[0]->data()->getTimeIntVal(row, isNull); - int64_t leastStr = str; + int64_t str = leastStr << 12; for (uint32_t i = 1; i < fp.size(); i++) { - int64_t str1 = fp[i]->data()->getTimeIntVal(row, isNull) << 12; + int64_t str1 = fp[i]->data()->getTimeIntVal(row, isNull); + int64_t str2 = str1 << 12; - if ( leastStr > str1 ) + if ( str > str1 ) + { leastStr = str1; + str = str2; + } } return leastStr; From a5f2f80bed670c9e2f07fa0c56c7899ad417793c Mon Sep 17 00:00:00 2001 From: Andrew Hutchings Date: Mon, 30 Apr 2018 10:41:52 +0100 Subject: [PATCH 39/57] MCOL-392 Fix case --- utils/funcexp/func_add_time.cpp | 2 +- utils/funcexp/func_case.cpp | 49 ++++++++++++++++++++++++++++++--- 2 files changed, 46 insertions(+), 5 deletions(-) diff --git a/utils/funcexp/func_add_time.cpp b/utils/funcexp/func_add_time.cpp index 0fa379bfd..bb32dc79d 100644 --- a/utils/funcexp/func_add_time.cpp +++ b/utils/funcexp/func_add_time.cpp @@ -146,7 +146,7 @@ int64_t addTime(Time& dt1, Time& dt2) dt.second = 0; dt.msecond = 0; - int64_t hour, min, sec, msec, tmp; + int64_t min, sec, msec, tmp; msec = (signed)(dt1.msecond + dt2.msecond); dt.msecond = tmp = msec % 1000000; diff --git a/utils/funcexp/func_case.cpp b/utils/funcexp/func_case.cpp index 46a479491..f9d70cb75 100644 --- a/utils/funcexp/func_case.cpp +++ b/utils/funcexp/func_case.cpp @@ -65,8 +65,6 @@ inline uint64_t simple_case_cmp(Row& row, case execplan::CalpontSystemCatalog::INT: case execplan::CalpontSystemCatalog::BIGINT: case execplan::CalpontSystemCatalog::DATE: - case execplan::CalpontSystemCatalog::DATETIME: - case execplan::CalpontSystemCatalog::TIME: { int64_t ev = parm[n]->data()->getIntVal(row, isNull); @@ -87,6 +85,49 @@ inline uint64_t simple_case_cmp(Row& row, break; } + case execplan::CalpontSystemCatalog::DATETIME: + { + int64_t ev = parm[n]->data()->getDatetimeIntVal(row, isNull); + + if (isNull) + break; + + for (i = 1; i <= whereCount; i++) + { + if (ev == parm[i]->data()->getDatetimeIntVal(row, isNull) && !isNull) + { + foundIt = true; + break; + } + else + isNull = false; + } + + break; + } + + case execplan::CalpontSystemCatalog::TIME: + { + int64_t ev = parm[n]->data()->getTimeIntVal(row, isNull); + + if (isNull) + break; + + for (i = 1; i <= whereCount; i++) + { + if (ev == parm[i]->data()->getTimeIntVal(row, isNull) && !isNull) + { + foundIt = true; + break; + } + else + isNull = false; + } + + break; + } + + case execplan::CalpontSystemCatalog::UBIGINT: case execplan::CalpontSystemCatalog::UINT: case execplan::CalpontSystemCatalog::UMEDINT: @@ -514,7 +555,7 @@ int64_t Func_simple_case::getTimeIntVal(rowgroup::Row& row, if (isNull) return joblist::TIMENULL; - return parm[i + 1]->data()->getTimeIntVal(row, isNull); + return parm[i]->data()->getTimeIntVal(row, isNull); } @@ -653,7 +694,7 @@ int64_t Func_searched_case::getTimeIntVal(rowgroup::Row& row, if (isNull) return joblist::TIMENULL; - return parm[i + 1]->data()->getTimeIntVal(row, isNull); + return parm[i]->data()->getTimeIntVal(row, isNull); } } // namespace funcexp From 3c090647af01dd3c0c5a9de25c79babadf160438 Mon Sep 17 00:00:00 2001 From: Andrew Hutchings Date: Mon, 30 Apr 2018 11:34:58 +0100 Subject: [PATCH 40/57] MCOL-392 Fix microsecond and nullif for TIME --- utils/funcexp/func_microsecond.cpp | 5 ++++ utils/funcexp/func_nullif.cpp | 37 ++++++++---------------------- 2 files changed, 14 insertions(+), 28 deletions(-) diff --git a/utils/funcexp/func_microsecond.cpp b/utils/funcexp/func_microsecond.cpp index e0a8e6d11..f292e26a5 100644 --- a/utils/funcexp/func_microsecond.cpp +++ b/utils/funcexp/func_microsecond.cpp @@ -63,6 +63,11 @@ int64_t Func_microsecond::getIntVal(rowgroup::Row& row, microSecond = (uint32_t)((val & 0xfffff)); break; + case CalpontSystemCatalog::TIME: + val = parm[0]->data()->getIntVal(row, isNull); + microSecond = (uint32_t)((val & 0xffffff)); + break; + case CalpontSystemCatalog::CHAR: case CalpontSystemCatalog::TEXT: case CalpontSystemCatalog::VARCHAR: diff --git a/utils/funcexp/func_nullif.cpp b/utils/funcexp/func_nullif.cpp index 23362f4ef..30d4682e5 100644 --- a/utils/funcexp/func_nullif.cpp +++ b/utils/funcexp/func_nullif.cpp @@ -127,8 +127,11 @@ int64_t Func_nullif::getIntVal(rowgroup::Row& row, { exp2 = parm[1]->data()->getDatetimeIntVal(row, isNull); - if (parm[0]->data()->resultType().colDataType == - execplan::CalpontSystemCatalog::DATE) + if ((parm[0]->data()->resultType().colDataType == + execplan::CalpontSystemCatalog::DATE) || + (parm[0]->data()->resultType().colDataType == + execplan::CalpontSystemCatalog::DATETIME)) + { // NULLIF arg0 is DATE, arg1 is DATETIME, // Upgrade arg1 to datetime @@ -159,8 +162,10 @@ int64_t Func_nullif::getIntVal(rowgroup::Row& row, { exp2 = parm[1]->data()->getTimeIntVal(row, isNull); - if (parm[0]->data()->resultType().colDataType == - execplan::CalpontSystemCatalog::DATETIME) + if ((parm[0]->data()->resultType().colDataType == + execplan::CalpontSystemCatalog::DATETIME) || + (parm[0]->data()->resultType().colDataType == + execplan::CalpontSystemCatalog::TIME)) { // NULLIF arg0 is DATETIME, arg1 is TIME, // Upgrade arg1 to time @@ -445,18 +450,6 @@ int64_t Func_nullif::getDatetimeIntVal(rowgroup::Row& row, case execplan::CalpontSystemCatalog::VARCHAR: case execplan::CalpontSystemCatalog::CHAR: case execplan::CalpontSystemCatalog::TEXT: - { - exp2 = parm[1]->data()->getIntVal(row, isNull); - - if (isNull) - { - isNull = false; - return exp1; - } - - break; - } - case execplan::CalpontSystemCatalog::DATE: { // Upgrade to datetime for proper comparison @@ -522,18 +515,6 @@ int64_t Func_nullif::getTimeIntVal(rowgroup::Row& row, case execplan::CalpontSystemCatalog::VARCHAR: case execplan::CalpontSystemCatalog::CHAR: case execplan::CalpontSystemCatalog::TEXT: - { - exp2 = parm[1]->data()->getIntVal(row, isNull); - - if (isNull) - { - isNull = false; - return exp1; - } - - break; - } - case execplan::CalpontSystemCatalog::TIME: case execplan::CalpontSystemCatalog::DATETIME: { From fda6b35243649673a7a3d9f9d082b796280e9016 Mon Sep 17 00:00:00 2001 From: Ravi Prakash Date: Mon, 30 Apr 2018 10:39:13 -0700 Subject: [PATCH 41/57] More change for MCOL-1229. Do not throw an exception but return an error code. --- dbcon/mysql/is_columnstore_columns.cpp | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/dbcon/mysql/is_columnstore_columns.cpp b/dbcon/mysql/is_columnstore_columns.cpp index 13f9b9485..b446eed4e 100644 --- a/dbcon/mysql/is_columnstore_columns.cpp +++ b/dbcon/mysql/is_columnstore_columns.cpp @@ -84,7 +84,7 @@ static int is_columnstore_columns_fill(THD *thd, TABLE_LIST *tables, COND *cond) continue; } else { - throw; + return 1; } } From dfc351b73068b5b7018ceeca4fe05436cf788016 Mon Sep 17 00:00:00 2001 From: Andrew Hutchings Date: Mon, 30 Apr 2018 22:08:10 +0100 Subject: [PATCH 42/57] MCOL-392 fix negative zero hours Also fix some functions that were not behaving correctly --- dbcon/execplan/treenode.h | 6 +-- utils/dataconvert/dataconvert.cpp | 74 ++++++++++++++++++++++++------ utils/dataconvert/dataconvert.h | 49 ++++++++++++++++---- utils/funcexp/func_bitand.cpp | 2 + utils/funcexp/func_extract.cpp | 3 ++ utils/funcexp/func_greatest.cpp | 2 +- utils/funcexp/func_hour.cpp | 8 ++++ utils/funcexp/func_least.cpp | 2 +- utils/funcexp/func_second.cpp | 1 + utils/funcexp/func_time_to_sec.cpp | 37 +++++++++++---- 10 files changed, 148 insertions(+), 36 deletions(-) diff --git a/dbcon/execplan/treenode.h b/dbcon/execplan/treenode.h index 08b8b43c8..3b2dd7200 100644 --- a/dbcon/execplan/treenode.h +++ b/dbcon/execplan/treenode.h @@ -996,12 +996,12 @@ inline int64_t TreeNode::getDatetimeIntVal() memcpy(&tt, &fResult.intVal, 8); // Note, this should probably be current date +/- time - if (tt.hour > 23) + if ((tt.hour > 23) && (!tt.is_neg)) { day = tt.hour / 24; tt.hour = tt.hour % 24; } - else if (tt.hour < 0) + else if ((tt.hour < 0) || (tt.is_neg)) { tt.hour = 0; } @@ -1023,7 +1023,7 @@ inline int64_t TreeNode::getTimeIntVal() dataconvert::DateTime dt; memcpy(&dt, &fResult.intVal, 8); - dataconvert::Time tt(0, dt.hour, dt.minute, dt.second, dt.msecond); + dataconvert::Time tt(0, dt.hour, dt.minute, dt.second, dt.msecond, false); memcpy(&fResult.intVal, &tt, 8); return fResult.intVal; } diff --git a/utils/dataconvert/dataconvert.cpp b/utils/dataconvert/dataconvert.cpp index 41dff9aa9..0ba1e4a87 100644 --- a/utils/dataconvert/dataconvert.cpp +++ b/utils/dataconvert/dataconvert.cpp @@ -1005,6 +1005,7 @@ bool mysql_str_to_time( const string& input, Time& output ) output.minute = 59; output.second = 59; output.msecond = 999999; + output.is_neg = 0; } else if (hour < -838) { @@ -1012,6 +1013,7 @@ bool mysql_str_to_time( const string& input, Time& output ) output.minute = 59; output.second = 59; output.msecond = 999999; + output.is_neg = 1; } // If neither of the above match then we return a 0 time else @@ -1025,6 +1027,7 @@ bool mysql_str_to_time( const string& input, Time& output ) output.minute = min; output.second = sec; output.msecond = usec; + output.is_neg = isNeg; return true; } @@ -1914,12 +1917,18 @@ int64_t DataConvert::convertColumnTime( inMinute = 0; inSecond = 0; inMicrosecond = 0; + bool isNeg = false; if ( datetimeFormat != CALPONTTIME_ENUM ) { status = -1; return value; } + if (p[0] == '-') + { + isNeg = true; + } + errno = 0; p = strtok_r(p, ":.", &savePoint); @@ -1981,6 +1990,7 @@ int64_t DataConvert::convertColumnTime( atime.minute = inMinute; atime.second = inSecond; atime.msecond = inMicrosecond; + atime.is_neg = isNeg; memcpy( &value, &atime, 8); } @@ -1994,6 +2004,7 @@ int64_t DataConvert::convertColumnTime( atime.minute = 59; atime.second = 59; atime.msecond = 999999; + atime.is_neg = false; memcpy( &value, &atime, 8); } else if (inHour < -838) @@ -2003,6 +2014,7 @@ int64_t DataConvert::convertColumnTime( atime.minute = 59; atime.second = 59; atime.msecond = 999999; + atime.is_neg = false; memcpy( &value, &atime, 8); } // If neither of the above match then we return a 0 time @@ -2084,8 +2096,15 @@ std::string DataConvert::timeToString( long long timevalue, long decimals ) Time dt(timevalue); const int TIMETOSTRING_LEN = 19; // (-H)HH:MM:SS.mmmmmm\0 char buf[TIMETOSTRING_LEN]; + char* outbuf = buf; - sprintf(buf, "%02d:%02d:%02d", dt.hour, dt.minute, dt.second); + if ((dt.hour >= 0) && dt.is_neg) + { + outbuf[0] = '-'; + outbuf++; + } + + sprintf(outbuf, "%02d:%02d:%02d", dt.hour, dt.minute, dt.second); if (dt.msecond && decimals) { size_t start = strlen(buf); @@ -2128,7 +2147,9 @@ std::string DataConvert::timeToString1( long long datetimevalue ) const int TIMETOSTRING1_LEN = 14; // HHMMSSmmmmmm\0 char buf[TIMETOSTRING1_LEN]; - sprintf(buf, "%02d%02d%02d%06d", dt.hour, dt.minute, dt.second, dt.msecond); + char* outbuf = buf; + + sprintf(outbuf, "%02d%02d%02d%06d", dt.hour, dt.minute, dt.second, dt.msecond); return buf; } @@ -2633,7 +2654,9 @@ int64_t DataConvert::intToDatetime(int64_t data, bool* date) int64_t DataConvert::intToTime(int64_t data) { char buf[21] = {0}; + char* bufread = buf; Time atime; + bool isNeg = false; if (data == 0) { @@ -2641,6 +2664,7 @@ int64_t DataConvert::intToTime(int64_t data) atime.minute = 0; atime.second = 0; atime.msecond = 0; + atime.is_neg = 0; return *(reinterpret_cast(&atime)); } @@ -2650,24 +2674,37 @@ int64_t DataConvert::intToTime(int64_t data) string hour, min, sec, msec; int64_t h = 0, minute = 0, s = 0, ms = 0; - switch (strlen(buf)) + if (bufread[0] == '-') { + isNeg = true; + bufread++; + } + + switch (strlen(bufread)) + { + case 7: + hour = string(bufread, 3); + min = string(bufread + 2, 2); + sec = string(bufread + 4, 2); + msec = string(bufread + 6, 6); + break; + case 6: - hour = string(buf, 2); - min = string(buf + 2, 2); - sec = string(buf + 4, 2); - msec = string(buf + 6, 6); + hour = string(bufread, 2); + min = string(bufread + 2, 2); + sec = string(bufread + 4, 2); + msec = string(bufread + 6, 6); break; case 4: - min = string(buf, 2); - sec = string(buf + 2, 2); - msec = string(buf + 4, 6); + min = string(bufread, 2); + sec = string(bufread + 2, 2); + msec = string(bufread + 4, 6); break; case 2: - sec = string(buf, 2); - msec = string(buf + 2, 6); + sec = string(bufread, 2); + msec = string(bufread + 2, 6); break; default: @@ -2686,6 +2723,7 @@ int64_t DataConvert::intToTime(int64_t data) atime.minute = minute; atime.second = s; atime.msecond = ms; + atime.is_neg = isNeg; return *(reinterpret_cast(&atime)); } @@ -2697,11 +2735,20 @@ int64_t DataConvert::stringToTime(const string& data) // -838 <= H <= 838 uint64_t min = 0, sec = 0, msec = 0; int64_t day = -1, hour = 0; + bool isNeg = false; string time, hms, ms; char* end = NULL; + + size_t pos = data.find("-"); + + if (pos != string::npos) + { + isNeg = true; + } + // Day - size_t pos = data.find(" "); + pos = data.find(" "); if (pos != string::npos) { @@ -2762,6 +2809,7 @@ int64_t DataConvert::stringToTime(const string& data) atime.minute = min; atime.second = sec; atime.msecond = msec; + atime.is_neg = isNeg; return *(reinterpret_cast(&atime)); } diff --git a/utils/dataconvert/dataconvert.h b/utils/dataconvert/dataconvert.h index 0992cd843..01e530709 100644 --- a/utils/dataconvert/dataconvert.h +++ b/utils/dataconvert/dataconvert.h @@ -197,14 +197,17 @@ struct Time signed second : 8; signed minute : 8; signed hour : 12; - signed day : 12; + signed day : 11; + signed is_neg : 1; // NULL column value = 0xFFFFFFFFFFFFFFFE Time() : msecond (0xFFFFFE), second (0xFF), minute (0xFF), hour (0xFFF), - day (0xFFF) {} + day (0x7FF), + is_neg (0b1) + {} // Construct a Time from a 64 bit integer InfiniDB time. Time(int64_t val) : @@ -212,11 +215,16 @@ struct Time second((val >> 24) & 0xff), minute((val >> 32) & 0xff), hour((val >> 40) & 0xfff), - day((val >> 52) & 0xfff) + day((val >> 52) & 0x7ff), + is_neg(val >> 63) {} - Time(signed d, signed h, signed min, signed sec, signed msec) : - msecond(msec), second(sec), minute(min), hour(h), day(d) {} + Time(signed d, signed h, signed min, signed sec, signed msec, bool neg) : + msecond(msec), second(sec), minute(min), hour(h), day(d), is_neg(neg) + { + if (h < 0) + is_neg = 0b1; + } int64_t convertToMySQLint() const; void reset(); @@ -229,13 +237,25 @@ void Time::reset() second = 0xFF; minute = 0xFF; hour = 0xFFF; - day = 0xFFF; + is_neg = 0b1; + day = 0x7FF; } inline int64_t Time::convertToMySQLint() const { - return (int64_t) (hour * 10000) + (minute * 100) + second; + if ((hour >= 0) && is_neg) + { + return (int64_t) ((hour * 10000) + (minute * 100) + second) * -1; + } + else if (hour >= 0) + { + return (int64_t) (hour * 10000) + (minute * 100) + second; + } + else + { + return (int64_t) (hour * 10000) - (minute * 100) - second; + } } static uint32_t daysInMonth[13] = {31, 28, 31, 30, 31, 30, 31, 31, 30, 31, 30, 31, 0}; @@ -593,6 +613,12 @@ inline void DataConvert::timeToString( long long timevalue, char* buf, unsigned { msec = (unsigned)((timevalue) & 0xffffff); } + if ((hour >= 0) && (timevalue >> 63)) + { + buf[0] = '-'; + buf++; + buflen--; + } snprintf( buf, buflen, "%02d:%02d:%02d", hour, (unsigned)((timevalue >> 32) & 0xff), @@ -636,13 +662,20 @@ inline void DataConvert::timeToString1( long long timevalue, char* buf, unsigned { // Handle negative correctly int hour = 0; - if ((timevalue >> 40) & 0xf00) + if ((timevalue >> 40) & 0x800) { hour = 0xfffff000; } hour |= ((timevalue >> 40) & 0xfff); + if ((hour >= 0) && (timevalue >> 63)) + { + buf[0] = '-'; + buf++; + buflen--; + } + snprintf( buf, buflen, "%02d%02d%02d", hour, (unsigned)((timevalue >> 32) & 0xff), diff --git a/utils/funcexp/func_bitand.cpp b/utils/funcexp/func_bitand.cpp index 32cb846fe..6906a928a 100644 --- a/utils/funcexp/func_bitand.cpp +++ b/utils/funcexp/func_bitand.cpp @@ -172,6 +172,8 @@ int64_t Func_bitand::getIntVal(Row& row, } hour |= ((time >> 40) & 0xfff); + if ((hour >= 0) && (time >> 63)) + hour*= -1; min = (uint32_t)((time >> 32) & 0xff); sec = (uint32_t)((time >> 24) & 0xff); msec = (uint32_t)(time & 0xffffff); diff --git a/utils/funcexp/func_extract.cpp b/utils/funcexp/func_extract.cpp index 67fc70a4d..c77a23d70 100644 --- a/utils/funcexp/func_extract.cpp +++ b/utils/funcexp/func_extract.cpp @@ -150,6 +150,9 @@ long long timeGet( uint64_t time, IntervalColumn::interval_type unit ) mask = 0xfffffffffffff000; hour = mask | ((time >> 40) & 0xfff); + if ((hour >= 0) && (time >> 63)) + hour*= -1; + // Always positive! day = abs(hour / 24); diff --git a/utils/funcexp/func_greatest.cpp b/utils/funcexp/func_greatest.cpp index 2d7dd15d6..80dbd251f 100644 --- a/utils/funcexp/func_greatest.cpp +++ b/utils/funcexp/func_greatest.cpp @@ -221,7 +221,7 @@ int64_t Func_greatest::getTimeIntVal(rowgroup::Row& row, int64_t str1 = fp[i]->data()->getTimeIntVal(row, isNull); int64_t str2 = str1 << 12; - if ( str < str1 ) + if ( str < str2 ) { greatestStr = str1; str = str2; diff --git a/utils/funcexp/func_hour.cpp b/utils/funcexp/func_hour.cpp index 353d0cc4e..631b070c8 100644 --- a/utils/funcexp/func_hour.cpp +++ b/utils/funcexp/func_hour.cpp @@ -128,10 +128,18 @@ int64_t Func_hour::getIntVal(rowgroup::Row& row, if (isTime) { // If negative, mask so it doesn't turn positive + bool isNeg = false; int64_t mask = 0; if ((val >> 40) & 0x800) mask = 0xfffffffffffff000; + + if (!mask && (val >> 63)) + { + isNeg = true; + } val = mask | ((val >> 40) & 0xfff); + if (isNeg) + val*= -1; } else { diff --git a/utils/funcexp/func_least.cpp b/utils/funcexp/func_least.cpp index 2f645af5f..ee5b2afc0 100644 --- a/utils/funcexp/func_least.cpp +++ b/utils/funcexp/func_least.cpp @@ -197,7 +197,7 @@ int64_t Func_least::getTimeIntVal(rowgroup::Row& row, int64_t str1 = fp[i]->data()->getTimeIntVal(row, isNull); int64_t str2 = str1 << 12; - if ( str > str1 ) + if ( str > str2 ) { leastStr = str1; str = str2; diff --git a/utils/funcexp/func_second.cpp b/utils/funcexp/func_second.cpp index 678a1fa61..ae12b157c 100644 --- a/utils/funcexp/func_second.cpp +++ b/utils/funcexp/func_second.cpp @@ -111,6 +111,7 @@ int64_t Func_second::getIntVal(rowgroup::Row& row, case execplan::CalpontSystemCatalog::TIME: { val = parm[0]->data()->getTimeIntVal(row, isNull); + return (uint32_t)((val >> 24) & 0xff); break; } diff --git a/utils/funcexp/func_time_to_sec.cpp b/utils/funcexp/func_time_to_sec.cpp index b87c2995a..2b8d6e3cc 100644 --- a/utils/funcexp/func_time_to_sec.cpp +++ b/utils/funcexp/func_time_to_sec.cpp @@ -49,12 +49,13 @@ int64_t Func_time_to_sec::getIntVal(rowgroup::Row& row, CalpontSystemCatalog::ColType& op_ct) { // assume 256 is enough. assume not allowing incomplete date - uint32_t hour = 0, + int32_t hour = 0, min = 0, sec = 0; bool bIsNegative = false; // Only set to true if CHAR or VARCHAR with a '-' int64_t val = 0; + int64_t mask = 0; dataconvert::Time tval; switch (parm[0]->data()->resultType().colDataType) @@ -64,9 +65,25 @@ int64_t Func_time_to_sec::getIntVal(rowgroup::Row& row, case CalpontSystemCatalog::DATETIME: val = parm[0]->data()->getIntVal(row, isNull); - hour = (uint32_t)((val >> 32) & 0x3f); - min = (uint32_t)((val >> 26) & 0x3f); - sec = (uint32_t)((val >> 20) & 0x3f); + hour = (int32_t)((val >> 32) & 0x3f); + min = (int32_t)((val >> 26) & 0x3f); + sec = (int32_t)((val >> 20) & 0x3f); + break; + + case CalpontSystemCatalog::TIME: + val = parm[0]->data()->getTimeIntVal(row, isNull); + // If negative, mask so it doesn't turn positive + if ((val >> 40) & 0x800) + mask = 0xfffffffffffff000; + + bIsNegative = val >> 63; + hour = (int32_t)(mask | ((val >> 40) & 0xfff)); + if ((hour >= 0) && bIsNegative) + hour*= -1; + else + bIsNegative = false; + min = (int32_t)((val >> 32) & 0xff); + sec = (int32_t)((val >> 24) & 0xff); break; case CalpontSystemCatalog::CHAR: @@ -112,9 +129,9 @@ int64_t Func_time_to_sec::getIntVal(rowgroup::Row& row, } else { - hour = (uint32_t)((val >> 32) & 0x3f); - min = (uint32_t)((val >> 26) & 0x3f); - sec = (uint32_t)((val >> 20) & 0x3f); + hour = (int32_t)((val >> 32) & 0x3f); + min = (int32_t)((val >> 26) & 0x3f); + sec = (int32_t)((val >> 20) & 0x3f); } break; @@ -131,9 +148,9 @@ int64_t Func_time_to_sec::getIntVal(rowgroup::Row& row, } else { - hour = (uint32_t)((val >> 32) & 0x3f); - min = (uint32_t)((val >> 26) & 0x3f); - sec = (uint32_t)((val >> 20) & 0x3f); + hour = (int32_t)((val >> 32) & 0x3f); + min = (int32_t)((val >> 26) & 0x3f); + sec = (int32_t)((val >> 20) & 0x3f); } } From 4ef42860220231e3d6adbba7581b352705ed2939 Mon Sep 17 00:00:00 2001 From: Andrew Hutchings Date: Tue, 1 May 2018 07:25:25 +0100 Subject: [PATCH 43/57] MCOL-392 Fix time_to_sec() for negative time --- utils/funcexp/func_time_to_sec.cpp | 10 +++++++++- 1 file changed, 9 insertions(+), 1 deletion(-) diff --git a/utils/funcexp/func_time_to_sec.cpp b/utils/funcexp/func_time_to_sec.cpp index 2b8d6e3cc..0d804242c 100644 --- a/utils/funcexp/func_time_to_sec.cpp +++ b/utils/funcexp/func_time_to_sec.cpp @@ -161,7 +161,15 @@ int64_t Func_time_to_sec::getIntVal(rowgroup::Row& row, return -1; } - int64_t rtn = (int64_t)(hour * 60 * 60) + (min * 60) + sec; + int64_t rtn; + if (hour < 0) + { + rtn = (int64_t)(hour * 60 * 60) - (min * 60) - sec; + } + else + { + rtn = (int64_t)(hour * 60 * 60) + (min * 60) + sec; + } if (bIsNegative) { From c40903de9bf421c006714543a4927d708ecf07c6 Mon Sep 17 00:00:00 2001 From: Andrew Hutchings Date: Tue, 1 May 2018 09:52:26 +0100 Subject: [PATCH 44/57] MCOL-392 Apply astyle Make this branch apply our style guidelines --- dbcon/execplan/simplefilter.cpp | 1 + dbcon/execplan/treenode.h | 2 + dbcon/joblist/groupconcat.cpp | 4 ++ dbcon/mysql/ha_calpont_dml.cpp | 4 ++ utils/dataconvert/dataconvert.cpp | 17 ++++++++- utils/dataconvert/dataconvert.h | 11 ++++++ utils/funcexp/func_add_time.cpp | 6 +-- utils/funcexp/func_bitand.cpp | 5 ++- utils/funcexp/func_case.cpp | 6 +-- utils/funcexp/func_coalesce.cpp | 6 +-- utils/funcexp/func_extract.cpp | 35 +++++++++-------- utils/funcexp/func_greatest.cpp | 6 +-- utils/funcexp/func_hour.cpp | 5 ++- utils/funcexp/func_if.cpp | 6 +-- utils/funcexp/func_ifnull.cpp | 6 +-- utils/funcexp/func_inet_aton.cpp | 6 +-- utils/funcexp/func_inet_ntoa.cpp | 6 +-- utils/funcexp/func_least.cpp | 6 +-- utils/funcexp/func_nullif.cpp | 14 +++---- utils/funcexp/func_sysdate.cpp | 6 +-- utils/funcexp/func_time.cpp | 1 + utils/funcexp/func_time_to_sec.cpp | 10 +++-- utils/funcexp/func_timediff.cpp | 6 +-- utils/funcexp/functor.h | 6 +-- utils/funcexp/functor_all.h | 52 +++++++++++++------------- utils/funcexp/functor_bool.h | 6 +-- utils/funcexp/functor_dtm.h | 36 +++++++++--------- utils/funcexp/functor_real.h | 6 +-- utils/funcexp/functor_str.h | 12 +++--- utils/rowgroup/rowaggregation.cpp | 2 + writeengine/bulk/we_bulkloadbuffer.cpp | 4 +- 31 files changed, 176 insertions(+), 123 deletions(-) diff --git a/dbcon/execplan/simplefilter.cpp b/dbcon/execplan/simplefilter.cpp index 1da14449c..639a04945 100644 --- a/dbcon/execplan/simplefilter.cpp +++ b/dbcon/execplan/simplefilter.cpp @@ -606,6 +606,7 @@ void SimpleFilter::convertConstant() result.intVal = dataconvert::DataConvert::timeToInt(result.strVal); } } + rcc->result(result); } } diff --git a/dbcon/execplan/treenode.h b/dbcon/execplan/treenode.h index 3b2dd7200..cef9579e9 100644 --- a/dbcon/execplan/treenode.h +++ b/dbcon/execplan/treenode.h @@ -995,6 +995,7 @@ inline int64_t TreeNode::getDatetimeIntVal() int day = 0; memcpy(&tt, &fResult.intVal, 8); + // Note, this should probably be current date +/- time if ((tt.hour > 23) && (!tt.is_neg)) { @@ -1005,6 +1006,7 @@ inline int64_t TreeNode::getDatetimeIntVal() { tt.hour = 0; } + dataconvert::DateTime dt(0, 0, day, tt.hour, tt.minute, tt.second, tt.msecond); memcpy(&fResult.intVal, &dt, 8); return fResult.intVal; diff --git a/dbcon/joblist/groupconcat.cpp b/dbcon/joblist/groupconcat.cpp index 1e84585b3..234fc0a8e 100644 --- a/dbcon/joblist/groupconcat.cpp +++ b/dbcon/joblist/groupconcat.cpp @@ -648,10 +648,12 @@ int64_t GroupConcator::lengthEstimate(const rowgroup::Row& row) fieldLen = 19; // YYYY-MM-DD HH24:MI:SS // Decimal point and milliseconds uint64_t colPrecision = row.getPrecision(*i); + if (colPrecision > 0 && colPrecision < 7) { fieldLen += colPrecision + 1; } + break; } @@ -660,10 +662,12 @@ int64_t GroupConcator::lengthEstimate(const rowgroup::Row& row) fieldLen = 10; // -HHH:MI:SS // Decimal point and milliseconds uint64_t colPrecision = row.getPrecision(*i); + if (colPrecision > 0 && colPrecision < 7) { fieldLen += colPrecision + 1; } + break; } diff --git a/dbcon/mysql/ha_calpont_dml.cpp b/dbcon/mysql/ha_calpont_dml.cpp index 57a91bd89..cf103a801 100644 --- a/dbcon/mysql/ha_calpont_dml.cpp +++ b/dbcon/mysql/ha_calpont_dml.cpp @@ -842,6 +842,7 @@ int ha_calpont_impl_write_batch_row_(uchar* buf, TABLE* table, cal_impl_if::cal_ const uchar* pos = buf; longlong tmp = my_datetime_packed_from_binary(pos, table->field[colpos]->decimals()); TIME_from_longlong_datetime_packed(<ime, tmp); + if (!ltime.second_part) { fprintf(ci.filePtr, "%04d-%02d-%02d %02d:%02d:%02d%c", @@ -855,6 +856,7 @@ int ha_calpont_impl_write_batch_row_(uchar* buf, TABLE* table, cal_impl_if::cal_ ltime.hour, ltime.minute, ltime.second, ltime.second_part, ci.delimiter); } + buf += table->field[colpos]->pack_length(); } else @@ -892,6 +894,7 @@ int ha_calpont_impl_write_batch_row_(uchar* buf, TABLE* table, cal_impl_if::cal_ const uchar* pos = buf; longlong tmp = my_time_packed_from_binary(pos, table->field[colpos]->decimals()); TIME_from_longlong_time_packed(<ime, tmp); + if (!ltime.second_part) { fprintf(ci.filePtr, "%02d:%02d:%02d%c", @@ -903,6 +906,7 @@ int ha_calpont_impl_write_batch_row_(uchar* buf, TABLE* table, cal_impl_if::cal_ ltime.hour, ltime.minute, ltime.second, ltime.second_part, ci.delimiter); } + buf += table->field[colpos]->pack_length(); } diff --git a/utils/dataconvert/dataconvert.cpp b/utils/dataconvert/dataconvert.cpp index 0ba1e4a87..b1cf8bcb2 100644 --- a/utils/dataconvert/dataconvert.cpp +++ b/utils/dataconvert/dataconvert.cpp @@ -1020,10 +1020,11 @@ bool mysql_str_to_time( const string& input, Time& output ) { output.reset(); } + return false; } - output.hour = isNeg ? 0-hour : hour; + output.hour = isNeg ? 0 - hour : hour; output.minute = min; output.second = sec; output.msecond = usec; @@ -1416,7 +1417,7 @@ DataConvert::convertColumnData(const CalpontSystemCatalog::ColType& colType, if (stringToTimeStruct(data, aTime)) { - value = (int64_t) *(reinterpret_cast(&aTime)); + value = (int64_t) * (reinterpret_cast(&aTime)); } else { @@ -1918,6 +1919,7 @@ int64_t DataConvert::convertColumnTime( inSecond = 0; inMicrosecond = 0; bool isNeg = false; + if ( datetimeFormat != CALPONTTIME_ENUM ) { status = -1; @@ -1941,6 +1943,7 @@ int64_t DataConvert::convertColumnTime( } p = strtok_r(NULL, ":.", &savePoint); + if (p == NULL) { status = -1; @@ -1976,6 +1979,7 @@ int64_t DataConvert::convertColumnTime( if (p != NULL) { inMicrosecond = strtol(p, 0, 10); + if (errno) { status = -1; @@ -2017,6 +2021,7 @@ int64_t DataConvert::convertColumnTime( atime.is_neg = false; memcpy( &value, &atime, 8); } + // If neither of the above match then we return a 0 time status = -1; @@ -2067,21 +2072,25 @@ std::string DataConvert::datetimeToString( long long datetimevalue, long decima { decimals = 0; } + // @bug 4703 abandon multiple ostringstream's for conversion DateTime dt(datetimevalue); const int DATETIMETOSTRING_LEN = 28; // YYYY-MM-DD HH:MM:SS.mmmmmm\0 char buf[DATETIMETOSTRING_LEN]; sprintf(buf, "%04d-%02d-%02d %02d:%02d:%02d", dt.year, dt.month, dt.day, dt.hour, dt.minute, dt.second); + if (dt.msecond && decimals) { snprintf(buf + strlen(buf), 21 + decimals, ".%d", dt.msecond); + // Pad end with zeros if (strlen(buf) < (size_t)(21 + decimals)) { sprintf(buf + strlen(buf), "%0*d", (int)(21 + decimals - strlen(buf)), 0); } } + return buf; } @@ -2092,6 +2101,7 @@ std::string DataConvert::timeToString( long long timevalue, long decimals ) { decimals = 0; } + // @bug 4703 abandon multiple ostringstream's for conversion Time dt(timevalue); const int TIMETOSTRING_LEN = 19; // (-H)HH:MM:SS.mmmmmm\0 @@ -2105,16 +2115,19 @@ std::string DataConvert::timeToString( long long timevalue, long decimals ) } sprintf(outbuf, "%02d:%02d:%02d", dt.hour, dt.minute, dt.second); + if (dt.msecond && decimals) { size_t start = strlen(buf); snprintf(buf + strlen(buf), 12 + decimals, ".%d", dt.msecond); + // Pad end with zeros if (strlen(buf) - start < (size_t)decimals) { sprintf(buf + strlen(buf), "%0*d", (int)(decimals - (strlen(buf) - start) + 1), 0); } } + return buf; } diff --git a/utils/dataconvert/dataconvert.h b/utils/dataconvert/dataconvert.h index 01e530709..0cf3480c5 100644 --- a/utils/dataconvert/dataconvert.h +++ b/utils/dataconvert/dataconvert.h @@ -277,6 +277,7 @@ inline uint32_t getDaysInMonth(uint32_t month, int year) return 0; uint32_t days = daysInMonth[month - 1]; + if ((month == 2) && isLeapYear(year)) days++; @@ -567,11 +568,14 @@ inline void DataConvert::datetimeToString( long long datetimevalue, char* buf, u { decimals = 0; } + int msec = 0; + if ((datetimevalue & 0xfffff) > 0) { msec = (unsigned)((datetimevalue) & 0xfffff); } + snprintf( buf, buflen, "%04d-%02d-%02d %02d:%02d:%02d", (unsigned)((datetimevalue >> 48) & 0xffff), (unsigned)((datetimevalue >> 44) & 0xf), @@ -585,6 +589,7 @@ inline void DataConvert::datetimeToString( long long datetimevalue, char* buf, u { size_t start = strlen(buf); snprintf(buf + strlen(buf), buflen - start, ".%d", msec); + // Pad end with zeros if (strlen(buf) - start < (size_t)decimals) { @@ -600,8 +605,10 @@ inline void DataConvert::timeToString( long long timevalue, char* buf, unsigned { decimals = 0; } + // Handle negative correctly int hour = 0, msec = 0; + if ((timevalue >> 40) & 0x800) { hour = 0xfffff000; @@ -613,12 +620,14 @@ inline void DataConvert::timeToString( long long timevalue, char* buf, unsigned { msec = (unsigned)((timevalue) & 0xffffff); } + if ((hour >= 0) && (timevalue >> 63)) { buf[0] = '-'; buf++; buflen--; } + snprintf( buf, buflen, "%02d:%02d:%02d", hour, (unsigned)((timevalue >> 32) & 0xff), @@ -629,6 +638,7 @@ inline void DataConvert::timeToString( long long timevalue, char* buf, unsigned { size_t start = strlen(buf); snprintf(buf + strlen(buf), buflen - start, ".%d", msec); + // Pad end with zeros if (strlen(buf) - start < (size_t)decimals) { @@ -662,6 +672,7 @@ inline void DataConvert::timeToString1( long long timevalue, char* buf, unsigned { // Handle negative correctly int hour = 0; + if ((timevalue >> 40) & 0x800) { hour = 0xfffff000; diff --git a/utils/funcexp/func_add_time.cpp b/utils/funcexp/func_add_time.cpp index bb32dc79d..ced9beaf2 100644 --- a/utils/funcexp/func_add_time.cpp +++ b/utils/funcexp/func_add_time.cpp @@ -320,9 +320,9 @@ int64_t Func_add_time::getDatetimeIntVal(rowgroup::Row& row, } int64_t Func_add_time::getTimeIntVal(rowgroup::Row& row, - FunctionParm& parm, - bool& isNull, - CalpontSystemCatalog::ColType& ct) + FunctionParm& parm, + bool& isNull, + CalpontSystemCatalog::ColType& ct) { int64_t val1 = parm[0]->data()->getTimeIntVal(row, isNull); diff --git a/utils/funcexp/func_bitand.cpp b/utils/funcexp/func_bitand.cpp index 6906a928a..69429bfb8 100644 --- a/utils/funcexp/func_bitand.cpp +++ b/utils/funcexp/func_bitand.cpp @@ -165,6 +165,7 @@ int64_t Func_bitand::getIntVal(Row& row, min = 0, sec = 0, msec = 0; + // Handle negative correctly if ((time >> 40) & 0x800) { @@ -172,8 +173,10 @@ int64_t Func_bitand::getIntVal(Row& row, } hour |= ((time >> 40) & 0xfff); + if ((hour >= 0) && (time >> 63)) - hour*= -1; + hour *= -1; + min = (uint32_t)((time >> 32) & 0xff); sec = (uint32_t)((time >> 24) & 0xff); msec = (uint32_t)(time & 0xffffff); diff --git a/utils/funcexp/func_case.cpp b/utils/funcexp/func_case.cpp index f9d70cb75..4c416360c 100644 --- a/utils/funcexp/func_case.cpp +++ b/utils/funcexp/func_case.cpp @@ -546,9 +546,9 @@ int64_t Func_simple_case::getDatetimeIntVal(rowgroup::Row& row, int64_t Func_simple_case::getTimeIntVal(rowgroup::Row& row, - FunctionParm& parm, - bool& isNull, - execplan::CalpontSystemCatalog::ColType& op_ct) + FunctionParm& parm, + bool& isNull, + execplan::CalpontSystemCatalog::ColType& op_ct) { uint64_t i = simple_case_cmp(row, parm, isNull, op_ct); diff --git a/utils/funcexp/func_coalesce.cpp b/utils/funcexp/func_coalesce.cpp index 30091a81e..e03040350 100644 --- a/utils/funcexp/func_coalesce.cpp +++ b/utils/funcexp/func_coalesce.cpp @@ -140,9 +140,9 @@ int64_t Func_coalesce::getDatetimeIntVal(rowgroup::Row& row, } int64_t Func_coalesce::getTimeIntVal(rowgroup::Row& row, - FunctionParm& parm, - bool& isNull, - CalpontSystemCatalog::ColType& ct) + FunctionParm& parm, + bool& isNull, + CalpontSystemCatalog::ColType& ct) { int64_t val = 0; diff --git a/utils/funcexp/func_extract.cpp b/utils/funcexp/func_extract.cpp index c77a23d70..332c05441 100644 --- a/utils/funcexp/func_extract.cpp +++ b/utils/funcexp/func_extract.cpp @@ -135,26 +135,28 @@ long long dateGet( uint64_t time, IntervalColumn::interval_type unit, bool dateT long long timeGet( uint64_t time, IntervalColumn::interval_type unit ) { int32_t hour = 0, - min = 0, - sec = 0, - msec = 0, - day = 0; + min = 0, + sec = 0, + msec = 0, + day = 0; - min = (int32_t)((time >> 32) & 0xff); - sec = (int32_t)((time >> 24) & 0xff); - msec = (int32_t)((time & 0xfffff)); + min = (int32_t)((time >> 32) & 0xff); + sec = (int32_t)((time >> 24) & 0xff); + msec = (int32_t)((time & 0xfffff)); - // If negative, mask so it doesn't turn positive - int64_t mask = 0; - if ((time >> 40) & 0x800) - mask = 0xfffffffffffff000; - hour = mask | ((time >> 40) & 0xfff); + // If negative, mask so it doesn't turn positive + int64_t mask = 0; - if ((hour >= 0) && (time >> 63)) - hour*= -1; + if ((time >> 40) & 0x800) + mask = 0xfffffffffffff000; - // Always positive! - day = abs(hour / 24); + hour = mask | ((time >> 40) & 0xfff); + + if ((hour >= 0) && (time >> 63)) + hour *= -1; + + // Always positive! + day = abs(hour / 24); switch ( unit ) { @@ -275,6 +277,7 @@ int64_t Func_extract::getIntVal(rowgroup::Row& row, } long long value; + if (isTime) value = timeGet( time, unit ); else diff --git a/utils/funcexp/func_greatest.cpp b/utils/funcexp/func_greatest.cpp index 80dbd251f..de63c283e 100644 --- a/utils/funcexp/func_greatest.cpp +++ b/utils/funcexp/func_greatest.cpp @@ -207,9 +207,9 @@ int64_t Func_greatest::getDatetimeIntVal(rowgroup::Row& row, } int64_t Func_greatest::getTimeIntVal(rowgroup::Row& row, - FunctionParm& fp, - bool& isNull, - execplan::CalpontSystemCatalog::ColType& ct) + FunctionParm& fp, + bool& isNull, + execplan::CalpontSystemCatalog::ColType& ct) { // Strip off unused day int64_t greatestStr = fp[0]->data()->getTimeIntVal(row, isNull); diff --git a/utils/funcexp/func_hour.cpp b/utils/funcexp/func_hour.cpp index 631b070c8..685a264db 100644 --- a/utils/funcexp/func_hour.cpp +++ b/utils/funcexp/func_hour.cpp @@ -130,6 +130,7 @@ int64_t Func_hour::getIntVal(rowgroup::Row& row, // If negative, mask so it doesn't turn positive bool isNeg = false; int64_t mask = 0; + if ((val >> 40) & 0x800) mask = 0xfffffffffffff000; @@ -137,9 +138,11 @@ int64_t Func_hour::getIntVal(rowgroup::Row& row, { isNeg = true; } + val = mask | ((val >> 40) & 0xfff); + if (isNeg) - val*= -1; + val *= -1; } else { diff --git a/utils/funcexp/func_if.cpp b/utils/funcexp/func_if.cpp index 4cc8cd164..ad65d10c9 100644 --- a/utils/funcexp/func_if.cpp +++ b/utils/funcexp/func_if.cpp @@ -221,9 +221,9 @@ int64_t Func_if::getDatetimeIntVal(Row& row, } int64_t Func_if::getTimeIntVal(Row& row, - FunctionParm& parm, - bool& isNull, - CalpontSystemCatalog::ColType&) + FunctionParm& parm, + bool& isNull, + CalpontSystemCatalog::ColType&) { if (boolVal(parm[0], row)) { diff --git a/utils/funcexp/func_ifnull.cpp b/utils/funcexp/func_ifnull.cpp index 18887f00e..4b8606d4e 100644 --- a/utils/funcexp/func_ifnull.cpp +++ b/utils/funcexp/func_ifnull.cpp @@ -169,9 +169,9 @@ int64_t Func_ifnull::getDatetimeIntVal(Row& row, } int64_t Func_ifnull::getTimeIntVal(Row& row, - FunctionParm& parm, - bool& isNull, - CalpontSystemCatalog::ColType&) + FunctionParm& parm, + bool& isNull, + CalpontSystemCatalog::ColType&) { if (isNull) return 0; diff --git a/utils/funcexp/func_inet_aton.cpp b/utils/funcexp/func_inet_aton.cpp index d6e3465cf..9692cf850 100644 --- a/utils/funcexp/func_inet_aton.cpp +++ b/utils/funcexp/func_inet_aton.cpp @@ -223,9 +223,9 @@ int64_t Func_inet_aton::getDatetimeIntVal(rowgroup::Row& row, } int64_t Func_inet_aton::getTimeIntVal(rowgroup::Row& row, - FunctionParm& fp, - bool& isNull, - execplan::CalpontSystemCatalog::ColType& op_ct) + FunctionParm& fp, + bool& isNull, + execplan::CalpontSystemCatalog::ColType& op_ct) { int64_t iValue = joblist::TIMENULL; diff --git a/utils/funcexp/func_inet_ntoa.cpp b/utils/funcexp/func_inet_ntoa.cpp index e25f56097..ff8c889ed 100644 --- a/utils/funcexp/func_inet_ntoa.cpp +++ b/utils/funcexp/func_inet_ntoa.cpp @@ -257,9 +257,9 @@ int64_t Func_inet_ntoa::getDatetimeIntVal(rowgroup::Row& row, } int64_t Func_inet_ntoa::getTimeIntVal(rowgroup::Row& row, - FunctionParm& fp, - bool& isNull, - execplan::CalpontSystemCatalog::ColType& op_ct) + FunctionParm& fp, + bool& isNull, + execplan::CalpontSystemCatalog::ColType& op_ct) { // std::cout << "In Func_inet_ntoa::getTimeVal" << std::endl; diff --git a/utils/funcexp/func_least.cpp b/utils/funcexp/func_least.cpp index ee5b2afc0..9ba5c98de 100644 --- a/utils/funcexp/func_least.cpp +++ b/utils/funcexp/func_least.cpp @@ -183,9 +183,9 @@ int64_t Func_least::getDatetimeIntVal(rowgroup::Row& row, } int64_t Func_least::getTimeIntVal(rowgroup::Row& row, - FunctionParm& fp, - bool& isNull, - execplan::CalpontSystemCatalog::ColType& op_ct) + FunctionParm& fp, + bool& isNull, + execplan::CalpontSystemCatalog::ColType& op_ct) { // Strip off unused day int64_t leastStr = fp[0]->data()->getTimeIntVal(row, isNull); diff --git a/utils/funcexp/func_nullif.cpp b/utils/funcexp/func_nullif.cpp index 30d4682e5..a268b0ea1 100644 --- a/utils/funcexp/func_nullif.cpp +++ b/utils/funcexp/func_nullif.cpp @@ -129,8 +129,8 @@ int64_t Func_nullif::getIntVal(rowgroup::Row& row, if ((parm[0]->data()->resultType().colDataType == execplan::CalpontSystemCatalog::DATE) || - (parm[0]->data()->resultType().colDataType == - execplan::CalpontSystemCatalog::DATETIME)) + (parm[0]->data()->resultType().colDataType == + execplan::CalpontSystemCatalog::DATETIME)) { // NULLIF arg0 is DATE, arg1 is DATETIME, @@ -164,8 +164,8 @@ int64_t Func_nullif::getIntVal(rowgroup::Row& row, if ((parm[0]->data()->resultType().colDataType == execplan::CalpontSystemCatalog::DATETIME) || - (parm[0]->data()->resultType().colDataType == - execplan::CalpontSystemCatalog::TIME)) + (parm[0]->data()->resultType().colDataType == + execplan::CalpontSystemCatalog::TIME)) { // NULLIF arg0 is DATETIME, arg1 is TIME, // Upgrade arg1 to time @@ -495,9 +495,9 @@ int64_t Func_nullif::getDatetimeIntVal(rowgroup::Row& row, } int64_t Func_nullif::getTimeIntVal(rowgroup::Row& row, - FunctionParm& parm, - bool& isNull, - CalpontSystemCatalog::ColType& ct) + FunctionParm& parm, + bool& isNull, + CalpontSystemCatalog::ColType& ct) { int64_t exp1 = parm[0]->data()->getTimeIntVal(row, isNull); int64_t exp2 = 0; diff --git a/utils/funcexp/func_sysdate.cpp b/utils/funcexp/func_sysdate.cpp index d8e2580bb..bf71e0415 100644 --- a/utils/funcexp/func_sysdate.cpp +++ b/utils/funcexp/func_sysdate.cpp @@ -101,9 +101,9 @@ int64_t Func_sysdate::getDatetimeIntVal(rowgroup::Row& row, } int64_t Func_sysdate::getTimeIntVal(rowgroup::Row& row, - FunctionParm& parm, - bool& isNull, - CalpontSystemCatalog::ColType& operationColType) + FunctionParm& parm, + bool& isNull, + CalpontSystemCatalog::ColType& operationColType) { return getIntVal(row, parm, isNull, operationColType); } diff --git a/utils/funcexp/func_time.cpp b/utils/funcexp/func_time.cpp index bd28d6e6c..4a90e0fe2 100644 --- a/utils/funcexp/func_time.cpp +++ b/utils/funcexp/func_time.cpp @@ -111,6 +111,7 @@ string Func_time::getStrVal(rowgroup::Row& row, val = parm[0]->data()->getTimeIntVal(row, isNull); break; } + case execplan::CalpontSystemCatalog::DATETIME: { val = parm[0]->data()->getTimeIntVal(row, isNull); diff --git a/utils/funcexp/func_time_to_sec.cpp b/utils/funcexp/func_time_to_sec.cpp index 0d804242c..ee5c4b669 100644 --- a/utils/funcexp/func_time_to_sec.cpp +++ b/utils/funcexp/func_time_to_sec.cpp @@ -50,8 +50,8 @@ int64_t Func_time_to_sec::getIntVal(rowgroup::Row& row, { // assume 256 is enough. assume not allowing incomplete date int32_t hour = 0, - min = 0, - sec = 0; + min = 0, + sec = 0; bool bIsNegative = false; // Only set to true if CHAR or VARCHAR with a '-' int64_t val = 0; @@ -72,16 +72,19 @@ int64_t Func_time_to_sec::getIntVal(rowgroup::Row& row, case CalpontSystemCatalog::TIME: val = parm[0]->data()->getTimeIntVal(row, isNull); + // If negative, mask so it doesn't turn positive if ((val >> 40) & 0x800) mask = 0xfffffffffffff000; bIsNegative = val >> 63; hour = (int32_t)(mask | ((val >> 40) & 0xfff)); + if ((hour >= 0) && bIsNegative) - hour*= -1; + hour *= -1; else bIsNegative = false; + min = (int32_t)((val >> 32) & 0xff); sec = (int32_t)((val >> 24) & 0xff); break; @@ -162,6 +165,7 @@ int64_t Func_time_to_sec::getIntVal(rowgroup::Row& row, } int64_t rtn; + if (hour < 0) { rtn = (int64_t)(hour * 60 * 60) - (min * 60) - sec; diff --git a/utils/funcexp/func_timediff.cpp b/utils/funcexp/func_timediff.cpp index 3c9ce96a8..742e8faf7 100644 --- a/utils/funcexp/func_timediff.cpp +++ b/utils/funcexp/func_timediff.cpp @@ -216,9 +216,9 @@ int64_t Func_timediff::getDatetimeIntVal(rowgroup::Row& row, } int64_t Func_timediff::getTimeIntVal(rowgroup::Row& row, - FunctionParm& parm, - bool& isNull, - CalpontSystemCatalog::ColType& ct) + FunctionParm& parm, + bool& isNull, + CalpontSystemCatalog::ColType& ct) { return dataconvert::DataConvert::timeToInt(getStrVal(row, parm, isNull, ct)); } diff --git a/utils/funcexp/functor.h b/utils/funcexp/functor.h index 065eb7865..9edb9bf62 100644 --- a/utils/funcexp/functor.h +++ b/utils/funcexp/functor.h @@ -117,9 +117,9 @@ public: } virtual int64_t getTimeIntVal(rowgroup::Row& row, - FunctionParm& fp, - bool& isNull, - execplan::CalpontSystemCatalog::ColType& op_ct) + FunctionParm& fp, + bool& isNull, + execplan::CalpontSystemCatalog::ColType& op_ct) { return intToTime(getIntVal(row, fp, isNull, op_ct)); } diff --git a/utils/funcexp/functor_all.h b/utils/funcexp/functor_all.h index 0eb0df845..553abc40f 100644 --- a/utils/funcexp/functor_all.h +++ b/utils/funcexp/functor_all.h @@ -105,10 +105,10 @@ public: bool& isNull, execplan::CalpontSystemCatalog::ColType& op_ct); - int64_t getTimeIntVal(rowgroup::Row& row, - FunctionParm& fp, - bool& isNull, - execplan::CalpontSystemCatalog::ColType& op_ct); + int64_t getTimeIntVal(rowgroup::Row& row, + FunctionParm& fp, + bool& isNull, + execplan::CalpontSystemCatalog::ColType& op_ct); }; @@ -158,9 +158,9 @@ public: execplan::CalpontSystemCatalog::ColType& op_ct); int64_t getTimeIntVal(rowgroup::Row& row, - FunctionParm& fp, - bool& isNull, - execplan::CalpontSystemCatalog::ColType& op_ct); + FunctionParm& fp, + bool& isNull, + execplan::CalpontSystemCatalog::ColType& op_ct); }; @@ -206,9 +206,10 @@ public: execplan::CalpontSystemCatalog::ColType& op_ct); int64_t getTimeIntVal(rowgroup::Row& row, - FunctionParm& fp, - bool& isNull, - execplan::CalpontSystemCatalog::ColType& op_ct);}; + FunctionParm& fp, + bool& isNull, + execplan::CalpontSystemCatalog::ColType& op_ct); +}; /** @brief Func_ifnull class @@ -252,9 +253,9 @@ public: execplan::CalpontSystemCatalog::ColType& op_ct); int64_t getTimeIntVal(rowgroup::Row& row, - FunctionParm& fp, - bool& isNull, - execplan::CalpontSystemCatalog::ColType& op_ct); + FunctionParm& fp, + bool& isNull, + execplan::CalpontSystemCatalog::ColType& op_ct); bool getBoolVal(rowgroup::Row& row, FunctionParm& fp, @@ -309,9 +310,9 @@ public: execplan::CalpontSystemCatalog::ColType& op_ct); int64_t getTimeIntVal(rowgroup::Row& row, - FunctionParm& fp, - bool& isNull, - execplan::CalpontSystemCatalog::ColType& op_ct); + FunctionParm& fp, + bool& isNull, + execplan::CalpontSystemCatalog::ColType& op_ct); }; @@ -357,9 +358,10 @@ public: execplan::CalpontSystemCatalog::ColType& op_ct); int64_t getTimeIntVal(rowgroup::Row& row, - FunctionParm& fp, - bool& isNull, - execplan::CalpontSystemCatalog::ColType& op_ct);}; + FunctionParm& fp, + bool& isNull, + execplan::CalpontSystemCatalog::ColType& op_ct); +}; /** @brief Func_coalesce class @@ -403,9 +405,9 @@ public: execplan::CalpontSystemCatalog::ColType& op_ct); int64_t getTimeIntVal(rowgroup::Row& row, - FunctionParm& fp, - bool& isNull, - execplan::CalpontSystemCatalog::ColType& op_ct); + FunctionParm& fp, + bool& isNull, + execplan::CalpontSystemCatalog::ColType& op_ct); }; @@ -435,9 +437,9 @@ public: execplan::CalpontSystemCatalog::ColType& op_ct); int64_t getTimeIntVal(rowgroup::Row& row, - FunctionParm& fp, - bool& isNull, - execplan::CalpontSystemCatalog::ColType& op_ct); + FunctionParm& fp, + bool& isNull, + execplan::CalpontSystemCatalog::ColType& op_ct); int64_t getIntVal(rowgroup::Row& row, FunctionParm& fp, diff --git a/utils/funcexp/functor_bool.h b/utils/funcexp/functor_bool.h index f67f405f1..b7b85106b 100644 --- a/utils/funcexp/functor_bool.h +++ b/utils/funcexp/functor_bool.h @@ -99,9 +99,9 @@ public: } int64_t getTimeIntVal(rowgroup::Row& row, - FunctionParm& fp, - bool& isNull, - execplan::CalpontSystemCatalog::ColType& op_ct) + FunctionParm& fp, + bool& isNull, + execplan::CalpontSystemCatalog::ColType& op_ct) { isNull = true; return 0; diff --git a/utils/funcexp/functor_dtm.h b/utils/funcexp/functor_dtm.h index 74aa41e38..d7837a4fe 100644 --- a/utils/funcexp/functor_dtm.h +++ b/utils/funcexp/functor_dtm.h @@ -138,9 +138,9 @@ public: execplan::CalpontSystemCatalog::ColType& op_ct); int64_t getTimeIntVal(rowgroup::Row& row, - FunctionParm& fp, - bool& isNull, - execplan::CalpontSystemCatalog::ColType& op_ct); + FunctionParm& fp, + bool& isNull, + execplan::CalpontSystemCatalog::ColType& op_ct); int64_t getIntVal(rowgroup::Row& row, FunctionParm& fp, @@ -254,9 +254,9 @@ public: execplan::CalpontSystemCatalog::ColType& op_ct); int64_t getTimeIntVal(rowgroup::Row& row, - FunctionParm& fp, - bool& isNull, - execplan::CalpontSystemCatalog::ColType& op_ct); + FunctionParm& fp, + bool& isNull, + execplan::CalpontSystemCatalog::ColType& op_ct); }; @@ -324,9 +324,9 @@ public: execplan::CalpontSystemCatalog::ColType& op_ct); int64_t getTimeIntVal(rowgroup::Row& row, - FunctionParm& fp, - bool& isNull, - execplan::CalpontSystemCatalog::ColType& op_ct); + FunctionParm& fp, + bool& isNull, + execplan::CalpontSystemCatalog::ColType& op_ct); }; @@ -361,9 +361,9 @@ public: execplan::CalpontSystemCatalog::ColType& op_ct); int64_t getTimeIntVal(rowgroup::Row& row, - FunctionParm& fp, - bool& isNull, - execplan::CalpontSystemCatalog::ColType& op_ct); + FunctionParm& fp, + bool& isNull, + execplan::CalpontSystemCatalog::ColType& op_ct); }; @@ -393,9 +393,9 @@ public: execplan::CalpontSystemCatalog::ColType& op_ct); int64_t getTimeIntVal(rowgroup::Row& row, - FunctionParm& fp, - bool& isNull, - execplan::CalpontSystemCatalog::ColType& op_ct); + FunctionParm& fp, + bool& isNull, + execplan::CalpontSystemCatalog::ColType& op_ct); int64_t getIntVal(rowgroup::Row& row, FunctionParm& fp, @@ -439,9 +439,9 @@ public: execplan::CalpontSystemCatalog::ColType& op_ct); int64_t getTimeIntVal(rowgroup::Row& row, - FunctionParm& fp, - bool& isNull, - execplan::CalpontSystemCatalog::ColType& op_ct); + FunctionParm& fp, + bool& isNull, + execplan::CalpontSystemCatalog::ColType& op_ct); }; /** @brief Func_str_to_date class diff --git a/utils/funcexp/functor_real.h b/utils/funcexp/functor_real.h index 779f4d108..5d40ce589 100644 --- a/utils/funcexp/functor_real.h +++ b/utils/funcexp/functor_real.h @@ -655,9 +655,9 @@ public: execplan::CalpontSystemCatalog::ColType& op_ct); int64_t getTimeIntVal(rowgroup::Row& row, - FunctionParm& fp, - bool& isNull, - execplan::CalpontSystemCatalog::ColType& op_ct); + FunctionParm& fp, + bool& isNull, + execplan::CalpontSystemCatalog::ColType& op_ct); private: diff --git a/utils/funcexp/functor_str.h b/utils/funcexp/functor_str.h index 895101211..c71cdec91 100644 --- a/utils/funcexp/functor_str.h +++ b/utils/funcexp/functor_str.h @@ -90,9 +90,9 @@ public: } int64_t getTimeIntVal(rowgroup::Row& row, - FunctionParm& fp, - bool& isNull, - execplan::CalpontSystemCatalog::ColType& op_ct) + FunctionParm& fp, + bool& isNull, + execplan::CalpontSystemCatalog::ColType& op_ct) { std::string str = getStrVal(row, fp, isNull, op_ct); return (isNull ? 0 : stringToTime(str)); @@ -685,9 +685,9 @@ public: execplan::CalpontSystemCatalog::ColType& op_ct); int64_t getTimeIntVal(rowgroup::Row& row, - FunctionParm& fp, - bool& isNull, - execplan::CalpontSystemCatalog::ColType& op_ct); + FunctionParm& fp, + bool& isNull, + execplan::CalpontSystemCatalog::ColType& op_ct); private: void convertNtoa(int64_t ipNum, std::string& ipString); }; diff --git a/utils/rowgroup/rowaggregation.cpp b/utils/rowgroup/rowaggregation.cpp index c496d91e6..8d110cfc8 100644 --- a/utils/rowgroup/rowaggregation.cpp +++ b/utils/rowgroup/rowaggregation.cpp @@ -1543,6 +1543,7 @@ void RowAggregation::doBitOp(const Row& rowIn, int64_t colIn, int64_t colOut, in int64_t dtm = rowIn.getUintField(colIn); // Handle negative correctly int hour = 0; + if ((dtm >> 40) & 0x800) { hour = 0xfffff000; @@ -2077,6 +2078,7 @@ void RowAggregation::doUDAF(const Row& rowIn, int64_t colIn, int64_t colOut, int datum.columnData = rowIn.getUintField(colIn); break; } + case execplan::CalpontSystemCatalog::TIME: { datum.dataType = execplan::CalpontSystemCatalog::BIGINT; diff --git a/writeengine/bulk/we_bulkloadbuffer.cpp b/writeengine/bulk/we_bulkloadbuffer.cpp index ccf73363f..506bff2eb 100644 --- a/writeengine/bulk/we_bulkloadbuffer.cpp +++ b/writeengine/bulk/we_bulkloadbuffer.cpp @@ -902,8 +902,8 @@ void BulkLoadBuffer::convert(char* field, int fieldLength, { bool bSatVal = false; - if ( column.dataType != CalpontSystemCatalog::DATETIME && - column.dataType != CalpontSystemCatalog::TIME ) + if ( column.dataType != CalpontSystemCatalog::DATETIME && + column.dataType != CalpontSystemCatalog::TIME ) { if (nullFlag) { From f68f80eb0068890f0a35bdadbf40a496ac922f91 Mon Sep 17 00:00:00 2001 From: Andrew Hutchings Date: Tue, 1 May 2018 20:56:52 +0100 Subject: [PATCH 45/57] MCOL-392 Add back accidental removal of function --- dbcon/execplan/aggregatecolumn.h | 8 ++++++++ 1 file changed, 8 insertions(+) diff --git a/dbcon/execplan/aggregatecolumn.h b/dbcon/execplan/aggregatecolumn.h index 028f1ee54..d1db7e5a4 100644 --- a/dbcon/execplan/aggregatecolumn.h +++ b/dbcon/execplan/aggregatecolumn.h @@ -416,6 +416,14 @@ public: evaluate(row, isNull); return TreeNode::getTimeIntVal(); } + /** + * F&E + */ + virtual int64_t getDatetimeIntVal(rowgroup::Row& row, bool& isNull) + { + evaluate(row, isNull); + return TreeNode::getDatetimeIntVal(); + } private: void evaluate(rowgroup::Row& row, bool& isNull); From b83c21d891ba9ef7ae86155936ad6fc9e0777850 Mon Sep 17 00:00:00 2001 From: Andrew Hutchings Date: Wed, 2 May 2018 09:40:27 +0100 Subject: [PATCH 46/57] MCOL-1376 Fix compiler errors in Ubuntu 18.04 Ubuntu 18.04 uses GCC 7.3 which is a little stricter than before. Fixes a few errors due to implicit includes that are no longer implicit and a ton of warnings about the implied alignment of code in utils/common/any.hpp --- dbcon/joblist/tupleunion.cpp | 6 +++--- utils/common/any.hpp | 28 +++++++++++++++++++--------- utils/common/cgroupconfigurator.cpp | 1 + utils/threadpool/threadpool.cpp | 1 + 4 files changed, 24 insertions(+), 12 deletions(-) diff --git a/dbcon/joblist/tupleunion.cpp b/dbcon/joblist/tupleunion.cpp index 77665ae90..d0892e45f 100644 --- a/dbcon/joblist/tupleunion.cpp +++ b/dbcon/joblist/tupleunion.cpp @@ -48,7 +48,7 @@ using namespace dataconvert; namespace { //returns the value of 10 raised to the power x. -inline double pow10(double x) +inline double exp10(double x) { return exp(x * M_LN10); } @@ -460,7 +460,7 @@ void TupleUnion::normalize(const Row& in, Row* out) if (in.getScale(i)) { double d = in.getIntField(i); - d /= pow10(in.getScale(i)); + d /= exp10(in.getScale(i)); os.precision(15); os << d; } @@ -570,7 +570,7 @@ dec1: if (in.getScale(i)) { double d = in.getUintField(i); - d /= pow10(in.getScale(i)); + d /= exp10(in.getScale(i)); os.precision(15); os << d; } diff --git a/utils/common/any.hpp b/utils/common/any.hpp index 5265015f1..be0ca679b 100755 --- a/utils/common/any.hpp +++ b/utils/common/any.hpp @@ -54,15 +54,25 @@ namespace anyimpl template struct big_any_policy : typed_base_any_policy { - virtual void static_delete(void** x) { if (*x) - delete(*reinterpret_cast(x)); *x = NULL; } - virtual void copy_from_value(void const* src, void** dest) { - *dest = new T(*reinterpret_cast(src)); } - virtual void clone(void* const* src, void** dest) { - *dest = new T(**reinterpret_cast(src)); } - virtual void move(void* const* src, void** dest) { - (*reinterpret_cast(dest))->~T(); - **reinterpret_cast(dest) = **reinterpret_cast(src); } + virtual void static_delete(void** x) + { + if (*x) + delete(*reinterpret_cast(x)); + *x = NULL; + } + virtual void copy_from_value(void const* src, void** dest) + { + *dest = new T(*reinterpret_cast(src)); + } + virtual void clone(void* const* src, void** dest) + { + *dest = new T(**reinterpret_cast(src)); + } + virtual void move(void* const* src, void** dest) + { + (*reinterpret_cast(dest))->~T(); + **reinterpret_cast(dest) = **reinterpret_cast(src); + } virtual void* get_value(void** src) { return *src; } }; diff --git a/utils/common/cgroupconfigurator.cpp b/utils/common/cgroupconfigurator.cpp index a5c736027..2be36b3a8 100644 --- a/utils/common/cgroupconfigurator.cpp +++ b/utils/common/cgroupconfigurator.cpp @@ -19,6 +19,7 @@ #include "configcpp.h" #include "logger.h" #include +#include #include #ifdef _MSC_VER #include "unistd.h" diff --git a/utils/threadpool/threadpool.cpp b/utils/threadpool/threadpool.cpp index 8b3b40ecd..f1aa4ec19 100644 --- a/utils/threadpool/threadpool.cpp +++ b/utils/threadpool/threadpool.cpp @@ -21,6 +21,7 @@ * ***********************************************************************/ #include +#include using namespace std; #include "messageobj.h" From 0b32f95dac0a69dd740ac5c68e02782bd6d9c5f9 Mon Sep 17 00:00:00 2001 From: Andrew Hutchings Date: Wed, 2 May 2018 15:26:01 +0100 Subject: [PATCH 47/57] MCOL-1349 Fix outer joins in views Outer join handling inside views was broken due to the joins being processed twice. This patch brings back the code so that outer joins in views are only processed once. --- dbcon/mysql/ha_calpont_execplan.cpp | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/dbcon/mysql/ha_calpont_execplan.cpp b/dbcon/mysql/ha_calpont_execplan.cpp index 0f54d8bdb..9cc3a99c8 100755 --- a/dbcon/mysql/ha_calpont_execplan.cpp +++ b/dbcon/mysql/ha_calpont_execplan.cpp @@ -801,8 +801,8 @@ uint32_t buildOuterJoin(gp_walk_info& gwi, SELECT_LEX& select_lex) // View is already processed in view::transform // @bug5319. view is sometimes treated as derived table and // fromSub::transform does not build outer join filters. - //if (!table_ptr->derived && table_ptr->view) - // continue; + if (!table_ptr->derived && table_ptr->view) + continue; CalpontSystemCatalog:: TableAliasName tan = make_aliasview( (table_ptr->db ? table_ptr->db : ""), From 543f6cb8dd9987bf064737e2f8dd4401fb033834 Mon Sep 17 00:00:00 2001 From: david hill Date: Thu, 3 May 2018 09:06:45 -0500 Subject: [PATCH 48/57] MCOL-1377 - enchance the user/group setting for syslog --- oam/install_scripts/syslogSetup.sh | 20 ++++++++++++++++++-- 1 file changed, 18 insertions(+), 2 deletions(-) diff --git a/oam/install_scripts/syslogSetup.sh b/oam/install_scripts/syslogSetup.sh index 1f4235a30..78b292dac 100755 --- a/oam/install_scripts/syslogSetup.sh +++ b/oam/install_scripts/syslogSetup.sh @@ -13,6 +13,7 @@ syslog_conf=nofile rsyslog7=0 user=`whoami 2>/dev/null` +group=user SUDO=" " if [ "$user" != "root" ]; then @@ -167,9 +168,24 @@ if [ ! -z "$syslog_conf" ] ; then # remove older version incase it was installed by previous build $SUDO rm -rf /etc/rsyslog.d/columnstore.conf + // determine username/groupname + + if [ -f /var/log/messages ]; then + user=`stat -c "%U %G" /var/log/messages | awk '{print $1}'` + group=`stat -c "%U %G" /var/log/messages | awk '{print $2}'` + fi + + if [ -f /var/log/syslog ]; then + user=`stat -c "%U %G" /var/log/syslog | awk '{print $1}'` + group=`stat -c "%U %G" /var/log/syslog | awk '{print $2}'` + fi + + //set permissions + $SUDO chown $user:$group -R /var/log/mariadb > /dev/null 2>&1 + if [ $rsyslog7 == 1 ]; then - sed -i -e s/groupname/adm/g ${columnstoreSyslogFile7} - sed -i -e s/username/syslog/g ${columnstoreSyslogFile7} + sed -i -e s/groupname/$group/g ${columnstoreSyslogFile7} + sed -i -e s/username/$user/g ${columnstoreSyslogFile7} $SUDO rm -f /etc/rsyslog.d/49-columnstore.conf $SUDO cp ${columnstoreSyslogFile7} ${syslog_conf} From 00990ca5bdb813492f3c1b83ae08e103e7415879 Mon Sep 17 00:00:00 2001 From: Roman Nozdrin Date: Thu, 3 May 2018 22:35:41 +0300 Subject: [PATCH 49/57] MCOL-1052 GROUP BY handler returns NULL values properly. --- dbcon/mysql/ha_calpont_impl.cpp | 4 ++++ 1 file changed, 4 insertions(+) diff --git a/dbcon/mysql/ha_calpont_impl.cpp b/dbcon/mysql/ha_calpont_impl.cpp index 4fc21b148..f29c24383 100644 --- a/dbcon/mysql/ha_calpont_impl.cpp +++ b/dbcon/mysql/ha_calpont_impl.cpp @@ -472,6 +472,10 @@ int fetchNextRow(uchar* buf, cal_table_info& ti, cal_connection_info* ci, bool h //set all fields to null in null col bitmap if (!handler_flag) memset(buf, -1, ti.msTablePtr->s->null_bytes); + else + { + memset(ti.msTablePtr->null_flags, -1, ti.msTablePtr->s->null_bytes); + } std::vector& colTypes = ti.tpl_scan_ctx->ctp; int64_t intColVal = 0; uint64_t uintColVal = 0; From b9f2b554847f81e29e2ba6f0b9616d2baf96613f Mon Sep 17 00:00:00 2001 From: Roman Nozdrin Date: Wed, 24 Jan 2018 09:40:44 +0300 Subject: [PATCH 50/57] MCOL-1384 Parser accepts quotes with qualified dbobject identifiers. --- dbcon/ddlpackage/CMakeLists.txt | 4 ++ dbcon/ddlpackage/ddl.l | 100 +++++++++++++++++++++++++++++--- dbcon/ddlpackage/ddl.y | 17 +++++- dbcon/mysql/ha_calpont_ddl.cpp | 74 +++++++++++------------ 4 files changed, 149 insertions(+), 46 deletions(-) diff --git a/dbcon/ddlpackage/CMakeLists.txt b/dbcon/ddlpackage/CMakeLists.txt index 27d2a3015..ae2f82fa9 100644 --- a/dbcon/ddlpackage/CMakeLists.txt +++ b/dbcon/ddlpackage/CMakeLists.txt @@ -9,6 +9,10 @@ ADD_CUSTOM_COMMAND( DEPENDS ddl.y ddl.l ) +# Parser puts extra info to stderr. +INCLUDE(../../check_compiler_flag.cmake) +MY_CHECK_AND_SET_COMPILER_FLAG("-DYYDEBUG" DEBUG) + ########### next target ############### set(ddlpackage_LIB_SRCS diff --git a/dbcon/ddlpackage/ddl.l b/dbcon/ddlpackage/ddl.l index ac51fe020..f65ef161d 100644 --- a/dbcon/ddlpackage/ddl.l +++ b/dbcon/ddlpackage/ddl.l @@ -18,6 +18,7 @@ /* $Id: ddl.l 9341 2013-03-27 14:10:35Z chao $ */ %{ +#include #include #include #include @@ -31,10 +32,11 @@ #endif using namespace ddlpackage; +typedef enum { NOOP, STRIP_QUOTES, STRIP_QUOTES_FQ } copy_action_t; int lineno = 1; void ddlerror(struct pass_to_bison* x, char const *s); -static char* scanner_copy(char *str, yyscan_t yyscanner); +static char* scanner_copy(char *str, yyscan_t yyscanner, copy_action_t action = NOOP ); %} @@ -54,6 +56,10 @@ horiz_space [ \t\f] newline [\n\r] non_newline [^\n\r] +quote ' +double_quote \" +grave_accent ` + comment ("--"{non_newline}*) self [,()\[\].;\:\+\-\*\/\%\^\<\>\=] whitespace ({space}+|{comment}) @@ -62,6 +68,12 @@ digit [0-9] ident_start [A-Za-z\200-\377_] ident_cont [A-Za-z\200-\377_0-9\$] identifier {ident_start}{ident_cont}* +/* fully qualified names regexes */ +fq_identifier {identifier}\.{identifier} +identifier_quoted {grave_accent}{identifier}{grave_accent} +identifier_double_quoted {double_quote}{identifier}{double_quote} +fq_quoted ({identifier_quoted}|{identifier})\.({identifier_quoted}|{identifier}) +fq_double_quoted ({identifier_double_quoted}|{identifier})\.({identifier_double_quoted}|{identifier}) integer [-+]?{digit}+ decimal ([-+]?({digit}*\.{digit}+)|({digit}+\.{digit}*)) @@ -69,11 +81,16 @@ real ({integer}|{decimal})[Ee][-+]?{digit}+ realfail1 ({integer}|{decimal})[Ee] realfail2 ({integer}|{decimal})[Ee][-+] -quote ' -grave_accent ` %% + +{identifier_quoted} { ddlget_lval(yyscanner)->str = scanner_copy( ddlget_text(yyscanner), yyscanner, STRIP_QUOTES ); return IDENT; } +{identifier_double_quoted} { ddlget_lval(yyscanner)->str = scanner_copy( ddlget_text(yyscanner), yyscanner, STRIP_QUOTES ); return IDENT; } +{fq_identifier} { ddlget_lval(yyscanner)->str = scanner_copy(ddlget_text(yyscanner), yyscanner); return FQ_IDENT; } +{fq_quoted} { ddlget_lval(yyscanner)->str = scanner_copy(ddlget_text(yyscanner), yyscanner, STRIP_QUOTES_FQ); return FQ_IDENT; } +{fq_double_quoted} { ddlget_lval(yyscanner)->str = scanner_copy(ddlget_text(yyscanner), yyscanner, STRIP_QUOTES_FQ); return FQ_IDENT; } + ACTION {return ACTION;} ADD {return ADD;} ALTER {return ALTER;} @@ -198,6 +215,11 @@ using namespace ddlpackage; */ void scanner_init(const char* str, yyscan_t yyscanner) { +#ifdef YYDEBUG + extern int ddldebug; + ddldebug = 1; +#endif + size_t slen = strlen(str); scan_data* pScanData = (scan_data*)ddlget_extra(yyscanner); @@ -246,10 +268,72 @@ void scanner_finish(yyscan_t yyscanner) pScanData->valbuf.clear(); } -char* scanner_copy (char *str, yyscan_t yyscanner) +char* scanner_copy (char *str, yyscan_t yyscanner, copy_action_t action) { - char* nv = strdup(str); - if(nv) - ((scan_data*)ddlget_extra(yyscanner))->valbuf.push_back(nv); - return nv; + char* result; + char* nv = strdup(str); + result = nv; + + // free strduped memory later to prevent possible memory leak + if(nv) + ((scan_data*)ddlget_extra(yyscanner))->valbuf.push_back(nv); + + if(action == STRIP_QUOTES) + { + nv[strlen(str) - 1] = '\0'; + result = nv + 1; + } + else if (action == STRIP_QUOTES_FQ) + { + bool move_left = false; + bool move_right = false; + char* left = nv; + char* tmp_first = nv; + // MCOL-1384 Loop through all comas in this quoted fq id + // looking for $quote_sign.$quote_sign sequence. + char* fq_delimiter; + int tmp_pos = 0; + while((fq_delimiter = strchr(tmp_first, '.')) != NULL) + { + if( (*(fq_delimiter -1) == '`' && *(fq_delimiter + 1) == '`') || + (*(fq_delimiter -1) == '"' && *(fq_delimiter + 1) == '"') ) + { + tmp_pos += fq_delimiter - tmp_first; + break; + } + tmp_first = fq_delimiter; + } + + char* fq_delimiter_orig = str + tmp_pos; + char* right = fq_delimiter + 1; + char* right_orig = fq_delimiter_orig + 1; + // MCOL-1384 Strip quotes from the left part. + if(*left == '"' || *left == '`') + { + result = left + 1; + *(fq_delimiter - 1) = '.'; + move_left = true; + } + else + { + fq_delimiter += 1; + } + + int right_length = strlen(right); + // MCOL-1384 Strip quotes from the right part. + if(*right == '`' || *right == '"') + { + right += 1; right_orig += 1; + right_length -= 2; + move_right = true; + *(fq_delimiter + right_length) = '\0'; + } + + if(move_left || move_right) + { + strncpy(fq_delimiter, right_orig, right_length); + } + } + + return result; } diff --git a/dbcon/ddlpackage/ddl.y b/dbcon/ddlpackage/ddl.y index 8d36b2c2b..982167287 100644 --- a/dbcon/ddlpackage/ddl.y +++ b/dbcon/ddlpackage/ddl.y @@ -48,6 +48,7 @@ */ %{ +#include "string.h" #include "sqlparser.h" #ifdef _MSC_VER @@ -121,7 +122,7 @@ REFERENCES RENAME RESTRICT SET SMALLINT TABLE TEXT TIME TINYBLOB TINYTEXT TINYINT TO UNIQUE UNSIGNED UPDATE USER SESSION_USER SYSTEM_USER VARCHAR VARBINARY VARYING WITH ZONE DOUBLE IDB_FLOAT REAL CHARSET IDB_IF EXISTS CHANGE TRUNCATE -%token IDENT FCONST SCONST CP_SEARCH_CONDITION_TEXT ICONST DATE +%token FQ_IDENT IDENT FCONST SCONST CP_SEARCH_CONDITION_TEXT ICONST DATE /* Notes: * 1. "ata" stands for alter_table_action @@ -611,7 +612,19 @@ table_name: ; qualified_name: - IDENT '.' IDENT {$$ = new QualifiedName($1, $3);} + FQ_IDENT { + char* delimeterPosition = strchr(const_cast($1), '.'); + if( delimeterPosition ) + { + *delimeterPosition = '\0'; + char* schemaName = const_cast($1); + char* tableName = delimeterPosition + 1; + $$ = new QualifiedName(schemaName, tableName); + *delimeterPosition = '.'; + } + else + $$ = new QualifiedName($1); + } | IDENT { if (x->fDBSchema.size()) $$ = new QualifiedName((char*)x->fDBSchema.c_str(), $1); diff --git a/dbcon/mysql/ha_calpont_ddl.cpp b/dbcon/mysql/ha_calpont_ddl.cpp index 611f1da3b..74b413667 100755 --- a/dbcon/mysql/ha_calpont_ddl.cpp +++ b/dbcon/mysql/ha_calpont_ddl.cpp @@ -2039,51 +2039,53 @@ int ha_calpont_impl_delete_table_(const char *db, const char *name, cal_connecti int ha_calpont_impl_rename_table_(const char* from, const char* to, cal_connection_info& ci) { - THD *thd = current_thd; - string emsg; + THD* thd = current_thd; + string emsg; - ostringstream stmt1; - pair fromPair; - pair toPair; - string stmt; + pair fromPair; + pair toPair; + string stmt; - //this is replcated DDL, treat it just like SSO - if (thd->slave_thread) - return 0; + //this is replcated DDL, treat it just like SSO + if (thd->slave_thread) + return 0; - //@bug 5660. Error out REAL DDL/DML on slave node. - // When the statement gets here, it's NOT SSO or RESTRICT - if (ci.isSlaveNode) - { - string emsg = logging::IDBErrorInfo::instance()->errorMsg(ERR_DML_DDL_SLAVE); - setError(current_thd, ER_CHECK_NOT_IMPLEMENTED, emsg); - return 1; - } + //@bug 5660. Error out REAL DDL/DML on slave node. + // When the statement gets here, it's NOT SSO or RESTRICT + if (ci.isSlaveNode) + { + string emsg = logging::IDBErrorInfo::instance()->errorMsg(ERR_DML_DDL_SLAVE); + setError(current_thd, ER_CHECK_NOT_IMPLEMENTED, emsg); + return 1; + } - fromPair = parseTableName(from); - toPair = parseTableName(to); + fromPair = parseTableName(from); + toPair = parseTableName(to); - if (fromPair.first != toPair.first) - { - thd->get_stmt_da()->set_overwrite_status(true); - thd->raise_error_printf(ER_CHECK_NOT_IMPLEMENTED, "Both tables must be in the same database to use RENAME TABLE"); - return -1; - } + if (fromPair.first != toPair.first) + { + thd->get_stmt_da()->set_overwrite_status(true); + thd->raise_error_printf(ER_CHECK_NOT_IMPLEMENTED, "Both tables must be in the same database to use RENAME TABLE"); + return -1; + } - stmt1 << "alter table " << fromPair.second << " rename to " << toPair.second << ";"; + // This explicitely shields both db objects with quotes that the lexer strips down later. + stmt = "alter table `" + fromPair.second + "` rename to `" + toPair.second + "`;"; + string db; - stmt = stmt1.str(); - string db; - if ( fromPair.first.length() !=0 ) - db = fromPair.first; - else if ( thd->db ) - db = thd->db; + if ( thd->db ) + db = thd->db; + else if ( fromPair.first.length() != 0 ) + db = fromPair.first; + else + db = toPair.first; - int rc = ProcessDDLStatement(stmt, db, "", tid2sid(thd->thread_id), emsg); - if (rc != 0) - push_warning(thd, Sql_condition::WARN_LEVEL_WARN, 9999, emsg.c_str()); + int rc = ProcessDDLStatement(stmt, db, "", tid2sid(thd->thread_id), emsg); - return rc; + if (rc != 0) + push_warning(thd, Sql_condition::WARN_LEVEL_ERROR, 9999, emsg.c_str()); + + return rc; } From ac3e702a3e038765772f662da029e4e7c61333c5 Mon Sep 17 00:00:00 2001 From: Andrew Hutchings Date: Tue, 8 May 2018 19:38:06 +0100 Subject: [PATCH 51/57] MCOL-1396 Allow StringStore to hold more than 2GB StringStore originally worked by returning a 32bit pointer to a memory location and storing the length with that pointer. This allowed 4GB to be stored in 64KB blocks. With 1.1 we used the high bit to signify a TEXT/BLOB string of > 64KB reducing the max capacity to 2GB but without any bounds checking. So, if you went over the 2GB mark the getter would think you are trying to get a long string instead of a short one and come up empty. It would then return NULL. This patch uses 64bit memory points still retaining the high bit to signify long strings. It also now stores the length with the string rather than with the pointer to allow the full 64bits for pointers. It also adds a bounds check for small strings. --- utils/rowgroup/rowgroup.cpp | 64 ++++++++-------- utils/rowgroup/rowgroup.h | 147 ++++++++++++++++++++++-------------- 2 files changed, 123 insertions(+), 88 deletions(-) diff --git a/utils/rowgroup/rowgroup.cpp b/utils/rowgroup/rowgroup.cpp index 48bdd7031..ba64e3596 100755 --- a/utils/rowgroup/rowgroup.cpp +++ b/utils/rowgroup/rowgroup.cpp @@ -79,10 +79,10 @@ StringStore::~StringStore() #endif } -uint32_t StringStore::storeString(const uint8_t *data, uint32_t len) +uint64_t StringStore::storeString(const uint8_t *data, uint32_t len) { MemChunk *lastMC = NULL; - uint32_t ret = 0; + uint64_t ret = 0; empty = false; // At least a NULL is being stored. @@ -92,7 +92,7 @@ uint32_t StringStore::storeString(const uint8_t *data, uint32_t len) if ((len == 8 || len == 9) && *((uint64_t *) data) == *((uint64_t *) joblist::CPNULLSTRMARK.c_str())) - return numeric_limits::max(); + return numeric_limits::max(); //@bug6065, make StringStore::storeString() thread safe boost::mutex::scoped_lock lk(fMutex, defer_lock); @@ -102,20 +102,21 @@ uint32_t StringStore::storeString(const uint8_t *data, uint32_t len) if (mem.size() > 0) lastMC = (MemChunk *) mem.back().get(); - if (len >= CHUNK_SIZE) + if ((len+4) >= CHUNK_SIZE) { - shared_array newOne(new uint8_t[len + sizeof(MemChunk)]); + shared_array newOne(new uint8_t[len + sizeof(MemChunk) + 4]); longStrings.push_back(newOne); lastMC = (MemChunk*) longStrings.back().get(); - lastMC->capacity = lastMC->currentSize = len; - memcpy(lastMC->data, data, len); + lastMC->capacity = lastMC->currentSize = len + 4; + memcpy(lastMC->data, &len, 4); + memcpy(lastMC->data + 4, data, len); // High bit to mark a long string - ret = 0x80000000; + ret = 0x8000000000000000; ret += longStrings.size() - 1; } else { - if ((lastMC == NULL) || (lastMC->capacity - lastMC->currentSize < len)) + if ((lastMC == NULL) || (lastMC->capacity - lastMC->currentSize < (len + 4))) { // mem usage debugging //if (lastMC) @@ -130,7 +131,11 @@ uint32_t StringStore::storeString(const uint8_t *data, uint32_t len) ret = ((mem.size()-1) * CHUNK_SIZE) + lastMC->currentSize; - memcpy(&(lastMC->data[lastMC->currentSize]), data, len); + // If this ever happens then we have big problems + if (ret & 0x8000000000000000) + throw logic_error("StringStore memory exceeded."); + memcpy(&(lastMC->data[lastMC->currentSize]), &len, 4); + memcpy(&(lastMC->data[lastMC->currentSize]) + 4, data, len); /* cout << "stored: '" << hex; for (uint32_t i = 0; i < len ; i++) { @@ -138,7 +143,7 @@ uint32_t StringStore::storeString(const uint8_t *data, uint32_t len) } cout << "' at position " << lastMC->currentSize << " len " << len << dec << endl; */ - lastMC->currentSize += len; + lastMC->currentSize += len + 4; } return ret; @@ -146,31 +151,31 @@ uint32_t StringStore::storeString(const uint8_t *data, uint32_t len) void StringStore::serialize(ByteStream &bs) const { - uint32_t i; + uint64_t i; MemChunk *mc; - bs << (uint32_t) mem.size(); + bs << (uint64_t) mem.size(); bs << (uint8_t) empty; for (i = 0; i < mem.size(); i++) { mc = (MemChunk *) mem[i].get(); - bs << (uint32_t) mc->currentSize; + bs << (uint64_t) mc->currentSize; //cout << "serialized " << mc->currentSize << " bytes\n"; bs.append(mc->data, mc->currentSize); } - bs << (uint32_t) longStrings.size(); + bs << (uint64_t) longStrings.size(); for (i = 0; i < longStrings.size(); i++) { mc = (MemChunk *) longStrings[i].get(); - bs << (uint32_t) mc->currentSize; + bs << (uint64_t) mc->currentSize; bs.append(mc->data, mc->currentSize); } } void StringStore::deserialize(ByteStream &bs) { - uint32_t i; - uint32_t count; - uint32_t size; + uint64_t i; + uint64_t count; + uint64_t size; uint8_t *buf; MemChunk *mc; uint8_t tmp8; @@ -718,10 +723,9 @@ bool Row::isNullValue(uint32_t colIndex) const case CalpontSystemCatalog::STRINT: { uint32_t len = getColumnWidth(colIndex); if (inStringTable(colIndex)) { - uint32_t offset, length; - offset = *((uint32_t *) &data[offsets[colIndex]]); - length = *((uint32_t *) &data[offsets[colIndex] + 4]); - return strings->isNullValue(offset, length); + uint64_t offset; + offset = *((uint64_t *) &data[offsets[colIndex]]); + return strings->isNullValue(offset); } if (data[offsets[colIndex]] == 0) // empty string return true; @@ -757,10 +761,9 @@ bool Row::isNullValue(uint32_t colIndex) const case CalpontSystemCatalog::VARBINARY: { uint32_t pos = offsets[colIndex]; if (inStringTable(colIndex)) { - uint32_t offset, length; - offset = *((uint32_t *) &data[pos]); - length = *((uint32_t *) &data[pos+4]); - return strings->isNullValue(offset, length); + uint64_t offset; + offset = *((uint64_t *) &data[pos]); + return strings->isNullValue(offset); } if (*((uint16_t*) &data[pos]) == 0) return true; @@ -1416,8 +1419,8 @@ RGData RowGroup::duplicate() void Row::setStringField(const std::string &val, uint32_t colIndex) { - uint32_t length; - uint32_t offset; + uint64_t offset; + uint64_t length; //length = strlen(val.c_str()) + 1; length = val.length(); @@ -1426,8 +1429,7 @@ void Row::setStringField(const std::string &val, uint32_t colIndex) if (inStringTable(colIndex)) { offset = strings->storeString((const uint8_t *) val.data(), length); - *((uint32_t *) &data[offsets[colIndex]]) = offset; - *((uint32_t *) &data[offsets[colIndex] + 4]) = length; + *((uint64_t *) &data[offsets[colIndex]]) = offset; // cout << " -- stored offset " << *((uint32_t *) &data[offsets[colIndex]]) // << " length " << *((uint32_t *) &data[offsets[colIndex] + 4]) // << endl; diff --git a/utils/rowgroup/rowgroup.h b/utils/rowgroup/rowgroup.h index 8b5ea75d7..7aca0c93f 100755 --- a/utils/rowgroup/rowgroup.h +++ b/utils/rowgroup/rowgroup.h @@ -92,13 +92,14 @@ public: StringStore(); virtual ~StringStore(); - inline std::string getString(uint32_t offset, uint32_t length) const; - uint32_t storeString(const uint8_t *data, uint32_t length); //returns the offset - inline const uint8_t * getPointer(uint32_t offset) const; + inline std::string getString(uint64_t offset) const; + uint64_t storeString(const uint8_t *data, uint32_t length); //returns the offset + inline const uint8_t * getPointer(uint64_t offset) const; + inline uint32_t getStringLength(uint64_t offset); inline bool isEmpty() const; inline uint64_t getSize() const; - inline bool isNullValue(uint32_t offset, uint32_t length) const; - inline bool equals(const std::string &str, uint32_t offset, uint32_t length) const; + inline bool isNullValue(uint64_t offset) const; + inline bool equals(const std::string &str, uint64_t offset) const; void clear(); @@ -541,9 +542,8 @@ inline bool Row::equals(uint64_t val, uint32_t colIndex) const inline bool Row::equals(const std::string &val, uint32_t colIndex) const { if (inStringTable(colIndex)) { - uint32_t offset = *((uint32_t *) &data[offsets[colIndex]]); - uint32_t length = *((uint32_t *) &data[offsets[colIndex] + 4]); - return strings->equals(val, offset, length); + uint64_t offset = *((uint64_t *) &data[offsets[colIndex]]); + return strings->equals(val, offset); } else return (strncmp(val.c_str(), (char *) &data[offsets[colIndex]], getColumnWidth(colIndex)) == 0); @@ -609,28 +609,27 @@ inline int64_t Row::getIntField(uint32_t colIndex) const inline const uint8_t * Row::getStringPointer(uint32_t colIndex) const { if (inStringTable(colIndex)) - return strings->getPointer(*((uint32_t *) &data[offsets[colIndex]])); + return strings->getPointer(*((uint64_t *) &data[offsets[colIndex]])); return &data[offsets[colIndex]]; } inline uint32_t Row::getStringLength(uint32_t colIndex) const { if (inStringTable(colIndex)) - return *((uint32_t *) &data[offsets[colIndex] + 4]); + return strings->getStringLength(*((uint64_t *) &data[offsets[colIndex]])); return strnlen((char *) &data[offsets[colIndex]], getColumnWidth(colIndex)); } inline void Row::setStringField(const uint8_t *strdata, uint32_t length, uint32_t colIndex) { - uint32_t offset; + uint64_t offset; if (length > getColumnWidth(colIndex)) length = getColumnWidth(colIndex); if (inStringTable(colIndex)) { offset = strings->storeString(strdata, length); - *((uint32_t *) &data[offsets[colIndex]]) = offset; - *((uint32_t *) &data[offsets[colIndex] + 4]) = length; + *((uint64_t *) &data[offsets[colIndex]]) = offset; // cout << " -- stored offset " << *((uint32_t *) &data[offsets[colIndex]]) // << " length " << *((uint32_t *) &data[offsets[colIndex] + 4]) // << endl; @@ -645,8 +644,7 @@ inline void Row::setStringField(const uint8_t *strdata, uint32_t length, uint32_ inline std::string Row::getStringField(uint32_t colIndex) const { if (inStringTable(colIndex)) - return strings->getString(*((uint32_t *) &data[offsets[colIndex]]), - *((uint32_t *) &data[offsets[colIndex] + 4])); + return strings->getString(*((uint64_t *) &data[offsets[colIndex]])); // Not all CHAR/VARCHAR are NUL terminated so use length return std::string((char *) &data[offsets[colIndex]], strnlen((char *) &data[offsets[colIndex]], getColumnWidth(colIndex))); @@ -662,21 +660,21 @@ inline std::string Row::getVarBinaryStringField(uint32_t colIndex) const inline uint32_t Row::getVarBinaryLength(uint32_t colIndex) const { if (inStringTable(colIndex)) - return *((uint32_t *) &data[offsets[colIndex] + 4]); + return strings->getStringLength(*((uint64_t *) &data[offsets[colIndex]]));; return *((uint16_t*) &data[offsets[colIndex]]); } inline const uint8_t* Row::getVarBinaryField(uint32_t colIndex) const { if (inStringTable(colIndex)) - return strings->getPointer(*((uint32_t *) &data[offsets[colIndex]])); + return strings->getPointer(*((uint64_t *) &data[offsets[colIndex]])); return &data[offsets[colIndex] + 2]; } inline const uint8_t* Row::getVarBinaryField(uint32_t& len, uint32_t colIndex) const { if (inStringTable(colIndex)) { - len = *((uint32_t *) &data[offsets[colIndex] + 4]); + len = strings->getStringLength(*((uint64_t *) &data[offsets[colIndex]])); return getVarBinaryField(colIndex); } else { @@ -854,9 +852,8 @@ inline void Row::setVarBinaryField(const uint8_t *val, uint32_t len, uint32_t co if (len > getColumnWidth(colIndex)) len = getColumnWidth(colIndex); if (inStringTable(colIndex)) { - uint32_t offset = strings->storeString(val, len); - *((uint32_t *) &data[offsets[colIndex]]) = offset; - *((uint32_t *) &data[offsets[colIndex] + 4]) = len; + uint64_t offset = strings->storeString(val, len); + *((uint64_t *) &data[offsets[colIndex]]) = offset; } else { *((uint16_t*) &data[offsets[colIndex]]) = len; @@ -1535,49 +1532,53 @@ inline void copyRow(const Row &in, Row *out) copyRow(in, out, std::min(in.getColumnCount(), out->getColumnCount())); } -inline std::string StringStore::getString(uint32_t off, uint32_t len) const +inline std::string StringStore::getString(uint64_t off) const { - if (off == std::numeric_limits::max()) + uint32_t length; + if (off == std::numeric_limits::max()) return joblist::CPNULLSTRMARK; MemChunk *mc; - if (off & 0x80000000) + if (off & 0x8000000000000000) { - off = off - 0x80000000; + off = off - 0x8000000000000000; if (longStrings.size() <= off) return joblist::CPNULLSTRMARK; mc = (MemChunk*) longStrings[off].get(); - return std::string((char *) mc->data, len); + memcpy(&length, mc->data, 4); + return std::string((char *) mc->data+4, length); } - uint32_t chunk = off / CHUNK_SIZE; - uint32_t offset = off % CHUNK_SIZE; + uint64_t chunk = off / CHUNK_SIZE; + uint64_t offset = off % CHUNK_SIZE; // this has to handle uninitialized data as well. If it's uninitialized it doesn't matter // what gets returned, it just can't go out of bounds. if (mem.size() <= chunk) return joblist::CPNULLSTRMARK; mc = (MemChunk *) mem[chunk].get(); - if ((offset + len) > mc->currentSize) + + memcpy(&length, &mc->data[offset], 4); + if ((offset + length) > mc->currentSize) return joblist::CPNULLSTRMARK; - - return std::string((char *) &(mc->data[offset]), len); + + return std::string((char *) &(mc->data[offset])+4, length); } -inline const uint8_t * StringStore::getPointer(uint32_t off) const +inline const uint8_t * StringStore::getPointer(uint64_t off) const { - if (off == std::numeric_limits::max()) + if (off == std::numeric_limits::max()) return (const uint8_t *) joblist::CPNULLSTRMARK.c_str(); - uint32_t chunk = off / CHUNK_SIZE; - uint32_t offset = off % CHUNK_SIZE; + uint64_t chunk = off / CHUNK_SIZE; + uint64_t offset = off % CHUNK_SIZE; MemChunk *mc; - if (off & 0x80000000) + if (off & 0x8000000000000000) { - off = off - 0x80000000; + off = off - 0x8000000000000000; if (longStrings.size() <= off) return (const uint8_t *) joblist::CPNULLSTRMARK.c_str(); mc = (MemChunk*) longStrings[off].get(); - return mc->data; + return mc->data+4; } // this has to handle uninitialized data as well. If it's uninitialized it doesn't matter // what gets returned, it just can't go out of bounds. @@ -1587,19 +1588,17 @@ inline const uint8_t * StringStore::getPointer(uint32_t off) const if (offset > mc->currentSize) return (const uint8_t *) joblist::CPNULLSTRMARK.c_str(); - return &(mc->data[offset]); + return &(mc->data[offset]) + 4; } -inline bool StringStore::isNullValue(uint32_t off, uint32_t len) const +inline bool StringStore::isNullValue(uint64_t off) const { - if (off == std::numeric_limits::max() || len == 0) + uint32_t length; + if (off == std::numeric_limits::max()) return true; - if (len < 8) - return false; - // Long strings won't be NULL - if (off & 0x80000000) + if (off & 0x8000000000000000) return false; uint32_t chunk = off / CHUNK_SIZE; @@ -1609,31 +1608,38 @@ inline bool StringStore::isNullValue(uint32_t off, uint32_t len) const return true; mc = (MemChunk *) mem[chunk].get(); - if ((offset + len) > mc->currentSize) + memcpy(&length, &mc->data[offset], 4); + if (length == 0) return true; - if (mc->data[offset] == 0) // "" = NULL string for some reason... + if (length < 8) + return false; + if ((offset + length) > mc->currentSize) return true; - return (*((uint64_t *) &mc->data[offset]) == *((uint64_t *) joblist::CPNULLSTRMARK.c_str())); + if (mc->data[offset+4] == 0) // "" = NULL string for some reason... + return true; + return (*((uint64_t *) &mc->data[offset]+4) == *((uint64_t *) joblist::CPNULLSTRMARK.c_str())); } -inline bool StringStore::equals(const std::string &str, uint32_t off, uint32_t len) const +inline bool StringStore::equals(const std::string &str, uint64_t off) const { - if (off == std::numeric_limits::max() || len == 0) + uint32_t length; + if (off == std::numeric_limits::max()) return str == joblist::CPNULLSTRMARK; MemChunk *mc; - if (off & 0x80000000) + if (off & 0x8000000000000000) { - if (longStrings.size() <= (off - 0x80000000)) + if (longStrings.size() <= (off - 0x8000000000000000)) return false; - mc = (MemChunk *) longStrings[off - 0x80000000].get(); + mc = (MemChunk *) longStrings[off - 0x8000000000000000].get(); + memcpy(&length, mc->data, 4); // Not sure if this check it needed, but adds safety - if (len > mc->currentSize) + if (length > mc->currentSize) return false; - return (strncmp(str.c_str(), (const char*) mc->data, len) == 0); + return (strncmp(str.c_str(), (const char*) mc->data+4, length) == 0); } uint32_t chunk = off / CHUNK_SIZE; uint32_t offset = off % CHUNK_SIZE; @@ -1641,10 +1647,37 @@ inline bool StringStore::equals(const std::string &str, uint32_t off, uint32_t l return false; mc = (MemChunk *) mem[chunk].get(); - if ((offset + len) > mc->currentSize) + memcpy(&length, &mc->data[offset], 4); + if ((offset + length) > mc->currentSize) return false; - return (strncmp(str.c_str(), (const char *) &mc->data[offset], len) == 0); + return (strncmp(str.c_str(), (const char *) &mc->data[offset]+4, length) == 0); +} +inline uint32_t StringStore::getStringLength(uint64_t off) +{ + uint32_t length; + MemChunk *mc; + if (off == std::numeric_limits::max()) + return 0; + if (off & 0x8000000000000000) + { + off = off - 0x8000000000000000; + if (longStrings.size() <= off) + return 0; + mc = (MemChunk*) longStrings[off].get(); + memcpy(&length, mc->data, 4); + } + else + { + uint64_t chunk = off / CHUNK_SIZE; + uint64_t offset = off % CHUNK_SIZE; + if (mem.size() <= chunk) + return 0; + mc = (MemChunk *) mem[chunk].get(); + memcpy(&length, &mc->data[offset], 4); + } + + return length; } inline bool StringStore::isEmpty() const From 1ac28c4f26db92d883f31f94f4c2a72dcf1c50dc Mon Sep 17 00:00:00 2001 From: Andrew Hutchings Date: Wed, 9 May 2018 09:28:30 +0100 Subject: [PATCH 52/57] Fix indentation --- dbcon/mysql/ha_calpont_execplan.cpp | 5711 +++++++++++++-------------- 1 file changed, 2852 insertions(+), 2859 deletions(-) diff --git a/dbcon/mysql/ha_calpont_execplan.cpp b/dbcon/mysql/ha_calpont_execplan.cpp index e15a5eb97..fd8ae8922 100644 --- a/dbcon/mysql/ha_calpont_execplan.cpp +++ b/dbcon/mysql/ha_calpont_execplan.cpp @@ -6912,2450 +6912,21 @@ int getSelectPlan(gp_walk_info& gwi, SELECT_LEX& select_lex, SCSEP& csep, bool i algorithm::to_lower(lower_create_query); algorithm::to_lower(lower_select_query); - // Group by list. not valid for union main query - if (gwi.thd->infinidb_vtable.vtable_state == THD::INFINIDB_CREATE_VTABLE && !unionSel) + // check if window functions are in order by. InfiniDB process order by list if + // window functions are involved, either in order by or projection. + for (; ordercol; ordercol = ordercol->next) { - gwi.clauseType = GROUP_BY; - Item* nonSupportItem = NULL; - ORDER* groupcol = reinterpret_cast(select_lex.group_list.first); - - // check if window functions are in order by. InfiniDB process order by list if - // window functions are involved, either in order by or projection. - for (; ordercol; ordercol = ordercol->next) - { - if ((*(ordercol->item))->type() == Item::WINDOW_FUNC_ITEM) - gwi.hasWindowFunc = true; - } - - // re-visit the first of ordercol list - ordercol = reinterpret_cast(order_list.first); - - // for subquery, order+limit by will be supported in infinidb. build order by columns - // @todo union order by and limit support - if (gwi.hasWindowFunc || gwi.subSelectType != CalpontSelectExecutionPlan::MAIN_SELECT) - { - for (; ordercol; ordercol = ordercol->next) - { - ReturnedColumn* rc = NULL; - - if (ordercol->in_field_list && ordercol->counter_used) - { - rc = gwi.returnedCols[ordercol->counter - 1]->clone(); - rc->orderPos(ordercol->counter - 1); - // can not be optimized off if used in order by with counter. - // set with self derived table alias if it's derived table - gwi.returnedCols[ordercol->counter - 1]->incRefCount(); - } - else - { - Item* ord_item = *(ordercol->item); - - // ignore not_used column on order by. - if (ord_item->type() == Item::INT_ITEM && ord_item->full_name() && string(ord_item->full_name()) == "Not_used") - continue; - else if (ord_item->type() == Item::INT_ITEM) - rc = gwi.returnedCols[((Item_int*)ord_item)->val_int() - 1]->clone(); - else if (ord_item->type() == Item::SUBSELECT_ITEM) - gwi.fatalParseError = true; - else - rc = buildReturnedColumn(ord_item, gwi, gwi.fatalParseError); - - // @bug5501 try item_ptr if item can not be fixed. For some - // weird dml statement state, item can not be fixed but the - // infomation is available in item_ptr. - if (!rc || gwi.fatalParseError) - { - Item* item_ptr = ordercol->item_ptr; - - while (item_ptr->type() == Item::REF_ITEM) - item_ptr = *(((Item_ref*)item_ptr)->ref); - - rc = buildReturnedColumn(item_ptr, gwi, gwi.fatalParseError); - } - - if (!rc) - { - string emsg = IDBErrorInfo::instance()->errorMsg(ERR_NON_SUPPORT_ORDER_BY); - gwi.parseErrorText = emsg; - setError(gwi.thd, ER_CHECK_NOT_IMPLEMENTED, emsg, gwi); - return ER_CHECK_NOT_IMPLEMENTED; - } - } - - if (ordercol->direction == ORDER::ORDER_ASC) - rc->asc(true); - else - rc->asc(false); - - gwi.orderByCols.push_back(SRCP(rc)); - } - } - else if (!isUnion) - { - vector fieldVec; - bool addToSel; - - // the following order by is just for redo phase - if (!unionSel) - { - for (; ordercol; ordercol = ordercol->next) - { - Item* ord_item = *(ordercol->item); - - // @bug5993. Could be nested ref. - while (ord_item->type() == Item::REF_ITEM) - ord_item = (*((Item_ref*)ord_item)->ref); - - // @bug 1706. re-construct the order by item one by one - //Item* ord_item = *(ordercol->item); - if (ord_cols.length() != 0) - ord_cols += ", "; - - addToSel = true; - string fullname; - - if (ordercol->in_field_list && ordercol->counter_used) - { - ostringstream oss; - oss << ordercol->counter; - ord_cols += oss.str(); - - if (ordercol->direction != ORDER::ORDER_ASC) - ord_cols += " desc"; - - continue; - } - - else if (ord_item->type() == Item::FUNC_ITEM) - { - // @bug 2621. order by alias - if (!ord_item->is_autogenerated_name && ord_item->name) - { - ord_cols += ord_item->name; - continue; - } - - // if there's group by clause or aggregate column, check to see - // if this item or the arguments is on the GB list. - ReturnedColumn* rc = 0; - // check if this order by column is on the select list - Item_func* ifp = (Item_func*)(*(ordercol->item)); - rc = buildFunctionColumn(ifp, gwi, gwi.fatalParseError); - - if (rc) - { - for (uint32_t i = 0; i < gwi.returnedCols.size(); i++) - { - if (rc && rc->operator==(gwi.returnedCols[i].get())) - { - ostringstream oss; - oss << i + 1; - ord_cols += oss.str(); - addToSel = false; - break; - } - } - } - - if (addToSel) - { - FunctionColumn* fc = dynamic_cast(rc); - - if (fc) - { - addToSel = false; - redo = true; - string ord_func = string(ifp->func_name()) + "("; - - for (uint32_t i = 0; i < fc->functionParms().size(); i++) - { - if (i != 0) - ord_func += ","; - - for (uint32_t j = 0; j < gwi.returnedCols.size(); j++) - { - if (fc->functionParms()[i]->data()->operator==(gwi.returnedCols[j].get())) - { - ord_func += "`" + escapeBackTick(gwi.returnedCols[j]->alias().c_str()) + "`"; - continue; - } - - AggregateColumn* ac = dynamic_cast(fc->functionParms()[i]->data()); - - if (ac) - { - gwi.parseErrorText = IDBErrorInfo::instance()->errorMsg(ERR_NON_SUPPORT_ORDER_BY); - setError(gwi.thd, ER_CHECK_NOT_IMPLEMENTED, gwi.parseErrorText, gwi); - return ER_CHECK_NOT_IMPLEMENTED; - } - - addToSel = true; - //continue; - - } - } - - ord_func += ")"; - - if (!addToSel) - ord_cols += ord_func; - } - } - } - else if (ord_item->type() == Item::SUBSELECT_ITEM) - { - string emsg = IDBErrorInfo::instance()->errorMsg(ERR_NON_SUPPORT_ORDER_BY); - setError(gwi.thd, ER_CHECK_NOT_IMPLEMENTED, emsg, gwi); - return ER_CHECK_NOT_IMPLEMENTED; - } - - else if (ord_item->type() == Item::SUM_FUNC_ITEM) - { - ReturnedColumn* ac = 0; - - Item_sum* ifp = (Item_sum*)(*(ordercol->item)); - // @bug3477. add aggregate column to the select list of the create phase. - ac = buildAggregateColumn(ifp, gwi); - - if (!ac) - { - setError(gwi.thd, ER_CHECK_NOT_IMPLEMENTED, - IDBErrorInfo::instance()->errorMsg(ERR_NON_SUPPORT_ORDER_BY), gwi); - return ER_CHECK_NOT_IMPLEMENTED; - } - - // check if this order by column is on the select list - for (uint32_t i = 0; i < gwi.returnedCols.size(); i++) - { - AggregateColumn* ret = dynamic_cast(gwi.returnedCols[i].get()); - - if (!ret) - continue; - - if (ac->operator==(gwi.returnedCols[i].get())) - { - ostringstream oss; - oss << i + 1; - ord_cols += oss.str(); - addToSel = false; - break; - } - } - - if (ac || !gwi.groupByCols.empty()) - { - if (addToSel) - { - redo = true; - // @bug 3076. do not add the argument of aggregate function to the SELECT list, - // instead, add the whole column - String str; - ord_item->print(&str, QT_INFINIDB_NO_QUOTE); - - if (sel_cols_in_create.length() != 0) - sel_cols_in_create += ", "; - - sel_cols_in_create += str.c_ptr(); - //gwi.selectCols.push_back(" `" + string(str.c_ptr()) + "`"); - SRCP srcp(ac); - gwi.returnedCols.push_back(srcp); - ord_cols += " `" + escapeBackTick(str.c_ptr()) + "`"; - } - - if (ordercol->direction != ORDER::ORDER_ASC) - ord_cols += " desc"; - - continue; - } - } - else if (ord_item->name && ord_item->type() == Item::FIELD_ITEM) - { - Item_field* field = reinterpret_cast(ord_item); - ReturnedColumn* rc = buildSimpleColumn(field, gwi); - fullname = field->full_name(); -// if (field->db_name) -// fullname += string(field->db_name) + "."; -// if (field->table_name) -// fullname += string(field->table_name) + "."; -// if (field->field_name) -// fullname += string(field->field_name); - - for (uint32_t i = 0; i < gwi.returnedCols.size(); i++) - { - SimpleColumn* sc = dynamic_cast(gwi.returnedCols[i].get()); - - if (sc && ((Item_field*)ord_item)->cached_table && - (strcasecmp(getViewName(((Item_field*)ord_item)->cached_table).c_str(), sc->viewName().c_str()) != 0)) - { - continue; - } - - if (strcasecmp(fullname.c_str(), gwi.returnedCols[i]->alias().c_str()) == 0 || - strcasecmp(ord_item->name, gwi.returnedCols[i]->alias().c_str()) == 0) - { - ord_cols += string(" `") + escapeBackTick(gwi.returnedCols[i]->alias().c_str()) + '`'; - addToSel = false; - break; - } - - if (sc && sc->sameColumn(rc)) - { - ostringstream oss; - oss << i + 1; - ord_cols += oss.str(); - addToSel = false; - break; - } - } - } - - if (addToSel) - { - // @bug 2719. Error out order by not on the distinct select list. - if (select_lex.options & SELECT_DISTINCT) - { - gwi.parseErrorText = IDBErrorInfo::instance()->errorMsg(ERR_ORDERBY_NOT_IN_DISTINCT); - setError(gwi.thd, ER_CHECK_NOT_IMPLEMENTED, gwi.parseErrorText, gwi); - return ER_CHECK_NOT_IMPLEMENTED; - } - - bool hasNonSupportItem = false; - uint16_t parseInfo = 0; - parse_item(ord_item, fieldVec, hasNonSupportItem, parseInfo); - - if (hasNonSupportItem) - { - string emsg = IDBErrorInfo::instance()->errorMsg(ERR_NON_SUPPORT_ORDER_BY); - setError(gwi.thd, ER_CHECK_NOT_IMPLEMENTED, emsg, gwi); - return ER_CHECK_NOT_IMPLEMENTED; - } - - String str; - ord_item->print(&str, QT_INFINIDB); - ord_cols += str.c_ptr(); - } - - if (ordercol->direction != ORDER::ORDER_ASC) - ord_cols += " desc"; - } - } - - redo = (redo || fieldVec.size() != 0); - - // populate string to be added to the select list for order by - for (uint32_t i = 0; i < fieldVec.size(); i++) - { - SimpleColumn* sc = buildSimpleColumn(fieldVec[i], gwi); - - if (!sc) - { - string emsg = IDBErrorInfo::instance()->errorMsg(ERR_NON_SUPPORT_ORDER_BY); - setError(gwi.thd, ER_CHECK_NOT_IMPLEMENTED, emsg, gwi); - return ER_CHECK_NOT_IMPLEMENTED; - } - - String str; - fieldVec[i]->print(&str, QT_INFINIDB_NO_QUOTE); - sc->alias(string(str.c_ptr())); - SRCP srcp(sc); - uint32_t j = 0; - - for (; j < gwi.returnedCols.size(); j++) - { - if (sc->sameColumn(gwi.returnedCols[j].get())) - { - SimpleColumn* field = dynamic_cast(gwi.returnedCols[j].get()); - - if (field && field->alias() == sc->alias()) - break; - } - } - - if (j == gwi.returnedCols.size()) - { - string fullname; - - if (sel_cols_in_create.length() != 0) - sel_cols_in_create += ", "; - - fullname = str.c_ptr(); - sel_cols_in_create += fullname + " `" + escapeBackTick(fullname.c_str()) + "`"; - - gwi.returnedCols.push_back(srcp); - gwi.columnMap.insert(CalpontSelectExecutionPlan::ColumnMap::value_type(string(fieldVec[i]->field_name), srcp)); - TABLE_LIST* tmp = (fieldVec[i]->cached_table ? fieldVec[i]->cached_table : 0); - gwi.tableMap[make_aliastable(sc->schemaName(), sc->tableName(), sc->tableAlias(), sc->isInfiniDB())] = - make_pair(1, tmp); - } - } - } - - // make sure columnmap, returnedcols and count(*) arg_list are not empty - TableMap::iterator tb_iter = gwi.tableMap.begin(); - - try - { - for (; tb_iter != gwi.tableMap.end(); tb_iter++) - { - if ((*tb_iter).second.first == 1) continue; - - CalpontSystemCatalog::TableAliasName tan = (*tb_iter).first; - CalpontSystemCatalog::TableName tn = make_table((*tb_iter).first.schema, (*tb_iter).first.table); - SimpleColumn* sc = getSmallestColumn(csc, tn, tan, (*tb_iter).second.second->table, gwi); - SRCP srcp(sc); - gwi.columnMap.insert(CalpontSelectExecutionPlan::ColumnMap::value_type(sc->columnName(), srcp)); - (*tb_iter).second.first = 1; - } - } - catch (runtime_error& e) - { - setError(gwi.thd, ER_INTERNAL_ERROR, e.what(), gwi); - CalpontSystemCatalog::removeCalpontSystemCatalog(sessionID); - return ER_INTERNAL_ERROR; - } - catch (...) - { - string emsg = IDBErrorInfo::instance()->errorMsg(ERR_LOST_CONN_EXEMGR); - setError(gwi.thd, ER_INTERNAL_ERROR, emsg, gwi); - CalpontSystemCatalog::removeCalpontSystemCatalog(sessionID); - return ER_INTERNAL_ERROR; - } - - if (!gwi.count_asterisk_list.empty() || !gwi.no_parm_func_list.empty() || - gwi.returnedCols.empty()) - { - // get the smallest column from colmap - CalpontSelectExecutionPlan::ColumnMap::const_iterator iter; - int minColWidth = 0; - CalpontSystemCatalog::ColType ct; - - try - { - for (iter = gwi.columnMap.begin(); iter != gwi.columnMap.end(); ++iter) - { - // should always not null - SimpleColumn* sc = dynamic_cast(iter->second.get()); - - if (sc && !(sc->joinInfo() & JOIN_CORRELATED)) - { - ct = csc->colType(sc->oid()); - - if (minColWidth == 0) - { - minColWidth = ct.colWidth; - minSc = iter->second; - } - else if (ct.colWidth < minColWidth) - { - minColWidth = ct.colWidth; - minSc = iter->second; - } - } - } - } - catch (...) - { - string emsg = IDBErrorInfo::instance()->errorMsg(ERR_LOST_CONN_EXEMGR); - setError(gwi.thd, ER_INTERNAL_ERROR, emsg, gwi); - CalpontSystemCatalog::removeCalpontSystemCatalog(sessionID); - return ER_INTERNAL_ERROR; - } - - if (gwi.returnedCols.empty() && gwi.additionalRetCols.empty()) - gwi.returnedCols.push_back(minSc); - } - - if (!isUnion && !gwi.hasWindowFunc && gwi.subSelectType == CalpontSelectExecutionPlan::MAIN_SELECT) - { - std::ostringstream vtb; - vtb << "infinidb_vtable.$vtable_" << gwi.thd->thread_id; - - //vtb << "$vtable_" << gwi.thd->thread_id; - // re-construct the select query and redo phase 1 - if (redo) - { - // select now() from region case. returnedCols should have minSc. - if (sel_cols_in_create.length() == 0) - { - SimpleColumn* sc = dynamic_cast(gwi.returnedCols[0].get()); - - if (sc) - sel_cols_in_create = dynamic_cast(gwi.returnedCols[0].get())->columnName(); - else - sel_cols_in_create = gwi.returnedCols[0]->alias(); - } - - // select * from derived table case - if (gwi.selectCols.empty()) - sel_cols_in_create = " * "; - - create_query = "create temporary table " + vtb.str() + " engine = aria as select " + sel_cols_in_create + " from "; - TABLE_LIST* table_ptr = select_lex.get_table_list(); - - bool firstTb = true; - - // put all tables, derived tables and views on the list - //TABLE_LIST* table_ptr = select_lex.get_table_list(); - set aliasSet; // to avoid duplicate table alias - - for (; table_ptr; table_ptr = table_ptr->next_global) - { - if (string(table_ptr->table_name).find("$vtable") != string::npos) - continue; - - if (table_ptr->derived) - { - if (aliasSet.find(table_ptr->alias) != aliasSet.end()) - continue; - - String str; - (table_ptr->derived->first_select())->print(gwi.thd, &str, QT_INFINIDB_DERIVED); - - if (!firstTb) - create_query += ", "; - - create_query += "(" + string(str.c_ptr()) + ") " + string(table_ptr->alias); - firstTb = false; - aliasSet.insert(table_ptr->alias); - } - else if (table_ptr->view) - { - if (aliasSet.find(table_ptr->alias) != aliasSet.end()) - continue; - - if (!firstTb) - create_query += ", "; - - create_query += string(table_ptr->db) + "." + string(table_ptr->table_name) + - string(" `") + escapeBackTick(table_ptr->alias) + string("`"); - aliasSet.insert(table_ptr->alias); - firstTb = false; - } - else - { - // table referenced by view is represented by viewAlias_tableAlias. - // consistent with item.cc field print. - if (table_ptr->referencing_view) - { - if (aliasSet.find(string(table_ptr->referencing_view->alias) + "_" + - string(table_ptr->alias)) != aliasSet.end()) - continue; - - if (!firstTb) - create_query += ", "; - - create_query += string(table_ptr->db) + "." + string(table_ptr->table_name) + string(" "); - create_query += string(" `") + - escapeBackTick(table_ptr->referencing_view->alias) + "_" + - escapeBackTick(table_ptr->alias) + string("`"); - aliasSet.insert(string(table_ptr->referencing_view->alias) + "_" + - string(table_ptr->alias)); - } - else - { - if (aliasSet.find(table_ptr->alias) != aliasSet.end()) - continue; - - if (!firstTb) - create_query += ", "; - - create_query += string(table_ptr->db) + "." + string(table_ptr->table_name) + string(" "); - create_query += string("`") + escapeBackTick(table_ptr->alias) + string("`"); - aliasSet.insert(table_ptr->alias); - } - - firstTb = false; - } - } - - - gwi.thd->infinidb_vtable.create_vtable_query.free(); - gwi.thd->infinidb_vtable.create_vtable_query.append(create_query.c_str(), create_query.length()); - gwi.thd->infinidb_vtable.vtable_state = THD::INFINIDB_REDO_PHASE1; // redo phase 1 - - // turn off select distinct from post process unless there're post process functions - // on the select list. - string sel_query = "select "; - - if (/*join->select_options*/select_lex.options & SELECT_DISTINCT && redo) - sel_query = "select distinct "; - else - sel_query = "select "; - - // select * from derived table... - if (gwi.selectCols.size() == 0) - sel_query += " * "; - - for (uint32_t i = 0; i < gwi.selectCols.size(); i++) - { - sel_query += gwi.selectCols[i]; - - if ( i + 1 != gwi.selectCols.size()) - sel_query += ", "; - } - - select_query.replace(lower_select_query.find("select *"), string("select *").length(), sel_query); - } - else - { - // remove order by clause in case this phase has been executed before. - // need a better fix later, like skip all the other non-optimized phase. - size_t pos = lower_select_query.find("order by"); - - if (pos != string::npos) - select_query.replace(pos, lower_select_query.length() - pos, ""); - - //select_query = "select * from " + vtb.str(); - if (unionSel) - order_list = select_lex.master_unit()->global_parameters()->order_list; - - ordercol = reinterpret_cast(order_list.first); - ord_cols = ""; - - for (; ordercol; ordercol = ordercol->next) - { - Item* ord_item = *(ordercol->item); - - // @bug 1706. re-construct the order by item one by one, because the ord_cols constucted so far - // is for REDO phase. - if (ord_cols.length() != 0) - ord_cols += ", "; - - if (ordercol->in_field_list && ordercol->counter_used) - { - ostringstream oss; - oss << ordercol->counter; - ord_cols += oss.str(); - } - else if (ord_item->type() == Item::NULL_ITEM) - { - // MCOL-793 Do nothing for an ORDER BY NULL - } - else if (ord_item->type() == Item::SUM_FUNC_ITEM) - { - Item_sum* ifp = (Item_sum*)(*(ordercol->item)); - ReturnedColumn* fc = buildAggregateColumn(ifp, gwi); - - for (uint32_t i = 0; i < gwi.returnedCols.size(); i++) - { - if (fc->operator==(gwi.returnedCols[i].get())) - { - ostringstream oss; - oss << i + 1; - ord_cols += oss.str(); - break; - } - } - - //continue; - } - // @bug 3518. if order by clause = selected column, use position. - else if (ord_item->name && ord_item->type() == Item::FIELD_ITEM) - { - Item_field* field = reinterpret_cast(ord_item); - string fullname; - - if (field->db_name) - fullname += string(field->db_name) + "."; - - if (field->table_name) - fullname += string(field->table_name) + "."; - - if (field->field_name) - fullname += string(field->field_name); - - uint32_t i = 0; - - for (i = 0; i < gwi.returnedCols.size(); i++) - { - SimpleColumn* sc = dynamic_cast(gwi.returnedCols[i].get()); - - if (sc && ((Item_field*)ord_item)->cached_table && - (strcasecmp(getViewName(((Item_field*)ord_item)->cached_table).c_str(), sc->viewName().c_str()) != 0)) - continue; - - if (strcasecmp(fullname.c_str(), gwi.returnedCols[i]->alias().c_str()) == 0 || - strcasecmp(ord_item->name, gwi.returnedCols[i]->alias().c_str()) == 0) - { - ostringstream oss; - oss << i + 1; - ord_cols += oss.str(); - break; - } - } - - if (i == gwi.returnedCols.size()) - ord_cols += string(" `") + escapeBackTick(ord_item->name) + '`'; - } - - else if (ord_item->name) - { - // for union order by 1 case. For unknown reason, it doesn't show in_field_list - if (ord_item->type() == Item::INT_ITEM) - { - ord_cols += ord_item->name; - } - else if (ord_item->type() == Item::SUBSELECT_ITEM) - { - string emsg = IDBErrorInfo::instance()->errorMsg(ERR_NON_SUPPORT_ORDER_BY); - setError(gwi.thd, ER_CHECK_NOT_IMPLEMENTED, emsg, gwi); - return ER_CHECK_NOT_IMPLEMENTED; - } - else - { - ord_cols += string(" `") + escapeBackTick(ord_item->name) + '`'; - } - } - else if (ord_item->type() == Item::FUNC_ITEM) - { - // @bug5636. check if this order by column is on the select list - ReturnedColumn* rc = buildFunctionColumn((Item_func*)(ord_item), gwi, gwi.fatalParseError); - - for (uint32_t i = 0; i < gwi.returnedCols.size(); i++) - { - if (rc && rc->operator==(gwi.returnedCols[i].get())) - { - ostringstream oss; - oss << i + 1; - ord_cols += oss.str(); - break; - } - } - } - else - { - String str; - ord_item->print(&str, QT_INFINIDB_NO_QUOTE); - ord_cols += string(str.c_ptr()); - } - - if (ordercol->direction != ORDER::ORDER_ASC) - ord_cols += " desc"; - } - } - - if (ord_cols.length() > 0) // has order by - { - gwi.thd->infinidb_vtable.has_order_by = true; - csep->hasOrderBy(true); - ord_cols = " order by " + ord_cols; - select_query += ord_cols; - } - } - - if (unionSel || gwi.subSelectType != CalpontSelectExecutionPlan::MAIN_SELECT) - { - if (select_lex.master_unit()->global_parameters()->explicit_limit) - { - if (select_lex.master_unit()->global_parameters()->offset_limit) - { - Item_int* offset = (Item_int*)select_lex.master_unit()->global_parameters()->offset_limit; - csep->limitStart(offset->val_int()); - } - - if (select_lex.master_unit()->global_parameters()->select_limit) - { - Item_int* select = (Item_int*)select_lex.master_unit()->global_parameters()->select_limit; - csep->limitNum(select->val_int()); - } - - if (unionSel && gwi.subSelectType == CalpontSelectExecutionPlan::MAIN_SELECT) - { - ostringstream limit; - limit << " limit "; - limit << csep->limitStart() << ", "; - limit << csep->limitNum(); - select_query += limit.str(); - } - } - } - else if (isUnion && select_lex.explicit_limit) - { - if (select_lex.braces) - { - if (select_lex.offset_limit) - csep->limitStart(((Item_int*)select_lex.offset_limit)->val_int()); - - if (select_lex.select_limit) - csep->limitNum(((Item_int*)select_lex.select_limit)->val_int()); - } - } - else if (select_lex.explicit_limit) - { - uint32_t limitOffset = 0; - uint32_t limitNum = std::numeric_limits::max(); - - if (join) - { -#if MYSQL_VERSION_ID >= 50172 - - // @bug5729. After upgrade, join->unit sometimes is uninitialized pointer - // (not null though) and will cause seg fault. Prefer checking - // select_lex->offset_limit if not null. - if (join->select_lex && - join->select_lex->offset_limit && - join->select_lex->offset_limit->fixed && - join->select_lex->select_limit && - join->select_lex->select_limit->fixed) - { - limitOffset = join->select_lex->offset_limit->val_int(); - limitNum = join->select_lex->select_limit->val_int(); - } - - else if (join->unit) - { - limitOffset = join->unit->offset_limit_cnt; - limitNum = join->unit->select_limit_cnt - limitOffset; - } - -#else - limitOffset = (join->unit)->offset_limit_cnt; - limitNum = (join->unit)->select_limit_cnt - (join->unit)->offset_limit_cnt; -#endif - } - else - { - if (select_lex.master_unit()->global_parameters()->offset_limit) - { - Item_int* offset = (Item_int*)select_lex.master_unit()->global_parameters()->offset_limit; - limitOffset = offset->val_int(); - } - - if (select_lex.master_unit()->global_parameters()->select_limit) - { - Item_int* select = (Item_int*)select_lex.master_unit()->global_parameters()->select_limit; - limitNum = select->val_int(); - } - } - - // relate to bug4848. let mysql drive limit when limit session variable set. - // do not set in csep. @bug5096. ignore session limit setting for dml - if ((gwi.thd->variables.select_limit == (uint64_t) -1 || - (gwi.thd->variables.select_limit != (uint64_t) -1 && - gwi.thd->infinidb_vtable.vtable_state != THD::INFINIDB_CREATE_VTABLE)) && - !csep->hasOrderBy()) - { - csep->limitStart(limitOffset); - csep->limitNum(limitNum); - } - else - { - ostringstream limit; - limit << " limit " << limitOffset << ", " << limitNum; - select_query += limit.str(); - } - } - - gwi.thd->infinidb_vtable.select_vtable_query.free(); - gwi.thd->infinidb_vtable.select_vtable_query.append(select_query.c_str(), select_query.length()); - - // We don't currently support limit with correlated subquery - if (csep->limitNum() != (uint64_t) -1 && - gwi.subQuery && !gwi.correlatedTbNameVec.empty()) - { - gwi.fatalParseError = true; - gwi.parseErrorText = IDBErrorInfo::instance()->errorMsg(ERR_NON_SUPPORT_LIMIT_SUB); - setError(gwi.thd, ER_INTERNAL_ERROR, gwi.parseErrorText, gwi); - return ER_CHECK_NOT_IMPLEMENTED; - } + if ((*(ordercol->item))->type() == Item::WINDOW_FUNC_ITEM) + gwi.hasWindowFunc = true; } - if (/*join->select_options*/select_lex.options & SELECT_DISTINCT) - csep->distinct(true); + // re-visit the first of ordercol list + ordercol = reinterpret_cast(order_list.first); - // add the smallest column to count(*) parm. - // select constant in subquery case - std::vector::iterator coliter; - - if (!minSc) + // for subquery, order+limit by will be supported in infinidb. build order by columns + // @todo union order by and limit support + if (gwi.hasWindowFunc || gwi.subSelectType != CalpontSelectExecutionPlan::MAIN_SELECT) { - if (!gwi.returnedCols.empty()) - minSc = gwi.returnedCols[0]; - else if (!gwi.additionalRetCols.empty()) - minSc = gwi.additionalRetCols[0]; - } - - // @bug3523, count(*) on subquery always pick column[0]. - SimpleColumn* sc = dynamic_cast(minSc.get()); - - if (sc && sc->schemaName().empty()) - { - if (gwi.derivedTbList.size() >= 1) - { - SimpleColumn* sc1 = new SimpleColumn(); - sc1->columnName(sc->columnName()); - sc1->tableName(sc->tableName()); - sc1->tableAlias(sc->tableAlias()); - sc1->viewName(lower(sc->viewName())); - sc1->colPosition(0); - minSc.reset(sc1); - } - } - - for (coliter = gwi.count_asterisk_list.begin(); coliter != gwi.count_asterisk_list.end(); ++coliter) - { - // @bug5977 @note should never throw this, but checking just in case. - // When ExeMgr fix is ready, this should not error out... - if (dynamic_cast(minSc.get())) - { - gwi.fatalParseError = true; - gwi.parseErrorText = "No project column found for aggregate function"; - setError(gwi.thd, ER_INTERNAL_ERROR, gwi.parseErrorText, gwi); - return ER_CHECK_NOT_IMPLEMENTED; - } - - (*coliter)->functionParms(minSc); - } - - std::vector::iterator funciter; - - SPTP sptp(new ParseTree(minSc.get()->clone())); - - for (funciter = gwi.no_parm_func_list.begin(); funciter != gwi.no_parm_func_list.end(); ++funciter) - { - FunctionParm funcParms = (*funciter)->functionParms(); - funcParms.push_back(sptp); - (*funciter)->functionParms(funcParms); - } - - // set sequence# for subquery localCols - for (uint32_t i = 0; i < gwi.localCols.size(); i++) - gwi.localCols[i]->sequence(i); - - // append additionalRetCols to returnedCols - gwi.returnedCols.insert(gwi.returnedCols.begin(), gwi.additionalRetCols.begin(), - gwi.additionalRetCols.end()); - - csep->groupByCols(gwi.groupByCols); - csep->orderByCols(gwi.orderByCols); - csep->returnedCols(gwi.returnedCols); - csep->columnMap(gwi.columnMap); - csep->having(havingFilter); - csep->derivedTableList(gwi.derivedTbList); - csep->selectSubList(selectSubList); - csep->subSelectList(gwi.subselectList); - gwi.thd->infinidb_vtable.duplicate_field_name = false; - clearStacks(gwi); - return 0; - } - - int cp_get_plan(THD * thd, SCSEP & csep) - { - LEX* lex = thd->lex; - idbassert(lex != 0); - - SELECT_LEX select_lex = lex->select_lex; - gp_walk_info gwi; - gwi.thd = thd; - int status = getSelectPlan(gwi, select_lex, csep); - - if (status > 0) - return ER_INTERNAL_ERROR; - else if (status < 0) - return status; - - // Derived table projection and filter optimization. - derivedTableOptimization(csep); - - return 0; - } - - int cp_get_table_plan(THD * thd, SCSEP & csep, cal_table_info & ti) - { - gp_walk_info* gwi = ti.condInfo; - - if (!gwi) - gwi = new gp_walk_info(); - - gwi->thd = thd; - LEX* lex = thd->lex; - idbassert(lex != 0); - uint32_t sessionID = csep->sessionID(); - gwi->sessionid = sessionID; - TABLE* table = ti.msTablePtr; - boost::shared_ptr csc = CalpontSystemCatalog::makeCalpontSystemCatalog(sessionID); - csc->identity(CalpontSystemCatalog::FE); - - // get all columns that mysql needs to fetch - MY_BITMAP* read_set = table->read_set; - Field** f_ptr, *field; - gwi->columnMap.clear(); - - for (f_ptr = table->field ; (field = *f_ptr) ; f_ptr++) - { - if (bitmap_is_set(read_set, field->field_index)) - { - SimpleColumn* sc = new SimpleColumn(table->s->db.str, table->s->table_name.str, field->field_name, sessionID); - string alias(table->alias.c_ptr()); - sc->tableAlias(lower(alias)); - assert (sc); - boost::shared_ptr spsc(sc); - gwi->returnedCols.push_back(spsc); - gwi->columnMap.insert(CalpontSelectExecutionPlan::ColumnMap::value_type(string(field->field_name), spsc)); - } - } - - if (gwi->columnMap.empty()) - { - CalpontSystemCatalog::TableName tn = make_table(table->s->db.str, table->s->table_name.str); - CalpontSystemCatalog::TableAliasName tan = make_aliastable(table->s->db.str, table->s->table_name.str, table->alias.c_ptr()); - SimpleColumn* sc = getSmallestColumn(csc, tn, tan, table, *gwi); - SRCP srcp(sc); - gwi->columnMap.insert(CalpontSelectExecutionPlan::ColumnMap::value_type(sc->columnName(), srcp)); - gwi->returnedCols.push_back(srcp); - } - - // get filter - if (ti.condInfo) - { - gp_walk_info* gwi = ti.condInfo; - ParseTree* filters = 0; - ParseTree* ptp = 0; - ParseTree* rhs = 0; - - while (!gwi->ptWorkStack.empty()) - { - filters = gwi->ptWorkStack.top(); - gwi->ptWorkStack.pop(); - SimpleFilter* sf = dynamic_cast(filters->data()); - - if (sf && sf->op()->data() == "noop") - { - delete filters; - filters = 0; - - if (gwi->ptWorkStack.empty()) - break; - - continue; - } - - if (gwi->ptWorkStack.empty()) - break; - - ptp = new ParseTree(new LogicOperator("and")); - ptp->left(filters); - rhs = gwi->ptWorkStack.top(); - gwi->ptWorkStack.pop(); - ptp->right(rhs); - gwi->ptWorkStack.push(ptp); - } - - csep->filters(filters); - } - - csep->returnedCols(gwi->returnedCols); - csep->columnMap(gwi->columnMap); - CalpontSelectExecutionPlan::TableList tblist; - tblist.push_back(make_aliastable(table->s->db.str, table->s->table_name.str, table->alias.c_ptr())); - csep->tableList(tblist); - - // @bug 3321. Set max number of blocks in a dictionary file to be scanned for filtering - csep->stringScanThreshold(gwi->thd->variables.infinidb_string_scan_threshold); - - return 0; - } - - int cp_get_group_plan(THD * thd, SCSEP & csep, cal_impl_if::cal_group_info & gi) - { - LEX* lex = thd->lex; - idbassert(lex != 0); - - SELECT_LEX select_lex = lex->select_lex; - gp_walk_info gwi; - gwi.thd = thd; - int status = getGroupPlan(gwi, select_lex, csep, gi); - - if (status > 0) - return ER_INTERNAL_ERROR; - else if (status < 0) - return status; - - return 0; - } - - /*@brief buildConstColFromFilter- change SimpleColumn into ConstColumn*/ - /*********************************************************** - * DESCRIPTION: - * Server could optimize out fields from GROUP BY list, when certain - * filter predicate is used, e.g. - * field = 'AIR', field IN ('AIR'). This utility function tries to - * replace such fields with ConstantColumns using cond_pushed filters. - * PARAMETERS: - * originalSC SimpleColumn* removed field - * gwi main strucutre - * gi auxilary group_by handler structure - * RETURNS - * ConstantColumn* if originalSC equals with cond_pushed columns. - * NULL otherwise - ***********************************************************/ - ConstantColumn* buildConstColFromFilter(SimpleColumn * originalSC, - gp_walk_info & gwi, cal_group_info & gi) - { - execplan::SimpleColumn* simpleCol; - execplan::ConstantColumn* constCol; - execplan::SOP op; - execplan::SimpleFilter* simpFilter; - execplan::ConstantColumn* result = NULL; - std::vector::iterator ptIt = gi.pushedPts.begin(); - - for (; ptIt != gi.pushedPts.end(); ptIt++) - { - simpFilter = dynamic_cast((*ptIt)->data()); - - if (simpFilter == NULL) - continue; - - simpleCol = dynamic_cast(simpFilter->lhs()); - constCol = dynamic_cast(simpFilter->rhs()); - - if (simpleCol == NULL || constCol == NULL) - continue; - - op = simpFilter->op(); - - if ( originalSC->sameColumn(dynamic_cast(simpleCol)) - && op.get()->op() == OP_EQ && constCol) - { -#ifdef DEBUG_WALK_COND - cerr << "buildConstColFromFilter() replaced " << endl; - cerr << simpleCol->toString() << endl; - cerr << " with " << endl; - cerr << constCol << endl; -#endif - result = constCol; - } - } - - return result; - } - - int getGroupPlan(gp_walk_info & gwi, SELECT_LEX & select_lex, SCSEP & csep, cal_group_info & gi, bool isUnion) - { -#ifdef DEBUG_WALK_COND - cerr << "getGroupPlan()" << endl; -#endif - - // rollup is currently not supported - if (select_lex.olap == ROLLUP_TYPE) - { - gwi.fatalParseError = true; - gwi.parseErrorText = IDBErrorInfo::instance()->errorMsg(ERR_ROLLUP_NOT_SUPPORT); - setError(gwi.thd, ER_CHECK_NOT_IMPLEMENTED, gwi.parseErrorText, gwi); - return ER_CHECK_NOT_IMPLEMENTED; - } - - gwi.internalDecimalScale = (gwi.thd->variables.infinidb_use_decimal_scale ? gwi.thd->variables.infinidb_decimal_scale : -1); - gwi.subSelectType = csep->subType(); - - JOIN* join = select_lex.join; - Item_cond* icp = 0; - - if ( gi.groupByWhere ) - icp = reinterpret_cast(gi.groupByWhere); - - uint32_t sessionID = csep->sessionID(); - gwi.sessionid = sessionID; - boost::shared_ptr csc = CalpontSystemCatalog::makeCalpontSystemCatalog(sessionID); - csc->identity(CalpontSystemCatalog::FE); - gwi.csc = csc; - - // @bug 2123. Override large table estimate if infinidb_ordered hint was used. - // @bug 2404. Always override if the infinidb_ordered_only variable is turned on. - if (gwi.thd->infinidb_vtable.override_largeside_estimate || gwi.thd->variables.infinidb_ordered_only) - csep->overrideLargeSideEstimate(true); - - // @bug 5741. Set a flag when in Local PM only query mode - csep->localQuery(gwi.thd->variables.infinidb_local_query); - - // @bug 3321. Set max number of blocks in a dictionary file to be scanned for filtering - csep->stringScanThreshold(gwi.thd->variables.infinidb_string_scan_threshold); - - csep->stringTableThreshold(gwi.thd->variables.infinidb_stringtable_threshold); - - csep->djsSmallSideLimit(gwi.thd->variables.infinidb_diskjoin_smallsidelimit * 1024ULL * 1024); - csep->djsLargeSideLimit(gwi.thd->variables.infinidb_diskjoin_largesidelimit * 1024ULL * 1024); - csep->djsPartitionSize(gwi.thd->variables.infinidb_diskjoin_bucketsize * 1024ULL * 1024); - - if (gwi.thd->variables.infinidb_um_mem_limit == 0) - csep->umMemLimit(numeric_limits::max()); - else - csep->umMemLimit(gwi.thd->variables.infinidb_um_mem_limit * 1024ULL * 1024); - - // populate table map and trigger syscolumn cache for all the tables (@bug 1637). - // all tables on FROM list must have at least one col in colmap - TABLE_LIST* table_ptr = gi.groupByTables; - CalpontSelectExecutionPlan::SelectList derivedTbList; - -// DEBUG -#ifdef DEBUG_WALK_COND - List_iterator sj_list_it(select_lex.sj_nests); - TABLE_LIST* sj_nest; - - while ((sj_nest = sj_list_it++)) - { - cerr << sj_nest->db << "." << sj_nest->table_name << endl; - } - -#endif - - // @bug 1796. Remember table order on the FROM list. - gwi.clauseType = FROM; - - try - { - for (; table_ptr; table_ptr = table_ptr->next_local) - { - // mysql put vtable here for from sub. we ignore it - //if (string(table_ptr->table_name).find("$vtable") != string::npos) - // continue; - - // Until we handle recursive cte: - // Checking here ensures we catch all with clauses in the query. - if (table_ptr->is_recursive_with_table()) - { - gwi.fatalParseError = true; - gwi.parseErrorText = "Recursive CTE"; - setError(gwi.thd, ER_CHECK_NOT_IMPLEMENTED, gwi.parseErrorText, gwi); - return ER_CHECK_NOT_IMPLEMENTED; - } - - string viewName = getViewName(table_ptr); - - // @todo process from subquery - if (table_ptr->derived) - { - String str; - (table_ptr->derived->first_select())->print(gwi.thd, &str, QT_INFINIDB_DERIVED); - - SELECT_LEX* select_cursor = table_ptr->derived->first_select(); - FromSubQuery fromSub(gwi, select_cursor); - string alias(table_ptr->alias); - fromSub.alias(lower(alias)); - - CalpontSystemCatalog::TableAliasName tn = make_aliasview("", "", alias, viewName); - // @bug 3852. check return execplan - SCSEP plan = fromSub.transform(); - - if (!plan) - { - setError(gwi.thd, ER_INTERNAL_ERROR, fromSub.gwip().parseErrorText, gwi); - CalpontSystemCatalog::removeCalpontSystemCatalog(sessionID); - return ER_INTERNAL_ERROR; - } - - gwi.derivedTbList.push_back(plan); - gwi.tbList.push_back(tn); - CalpontSystemCatalog::TableAliasName tan = make_aliastable("", alias, alias); - gwi.tableMap[tan] = make_pair(0, table_ptr); -// gwi.thd->infinidb_vtable.isUnion = true; //by-pass the 2nd pass of rnd_init - } - else if (table_ptr->view) - { - View* view = new View(table_ptr->view->select_lex, &gwi); - CalpontSystemCatalog::TableAliasName tn = make_aliastable(table_ptr->db, table_ptr->table_name, table_ptr->alias); - view->viewName(tn); - gwi.viewList.push_back(view); - view->transform(); - } - else - { - // check foreign engine tables - bool infiniDB = (table_ptr->table ? isInfiniDB(table_ptr->table) : true); - - // trigger system catalog cache - if (infiniDB) - csc->columnRIDs(make_table(table_ptr->db, table_ptr->table_name), true); - - string table_name = table_ptr->table_name; - - // @bug5523 - if (table_ptr->db && strcmp(table_ptr->db, "information_schema") == 0) - table_name = (table_ptr->schema_table_name ? table_ptr->schema_table_name : table_ptr->alias); - - CalpontSystemCatalog::TableAliasName tn = make_aliasview(table_ptr->db, table_name, table_ptr->alias, viewName, infiniDB); - gwi.tbList.push_back(tn); - CalpontSystemCatalog::TableAliasName tan = make_aliastable(table_ptr->db, table_name, table_ptr->alias, infiniDB); - gwi.tableMap[tan] = make_pair(0, table_ptr); -#ifdef DEBUG_WALK_COND - cerr << tn << endl; -#endif - } - } - - if (gwi.fatalParseError) - { - setError(gwi.thd, ER_INTERNAL_ERROR, gwi.parseErrorText, gwi); - return ER_INTERNAL_ERROR; - } - } - catch (IDBExcept& ie) - { - setError(gwi.thd, ER_INTERNAL_ERROR, ie.what(), gwi); - CalpontSystemCatalog::removeCalpontSystemCatalog(sessionID); - // @bug 3852. set error status for gwi. - gwi.fatalParseError = true; - gwi.parseErrorText = ie.what(); - return ER_INTERNAL_ERROR; - } - catch (...) - { - string emsg = IDBErrorInfo::instance()->errorMsg(ERR_LOST_CONN_EXEMGR); - // @bug3852 set error status for gwi. - gwi.fatalParseError = true; - gwi.parseErrorText = emsg; - setError(gwi.thd, ER_INTERNAL_ERROR, emsg, gwi); - CalpontSystemCatalog::removeCalpontSystemCatalog(sessionID); - return ER_INTERNAL_ERROR; - } - - csep->tableList(gwi.tbList); - - bool unionSel = false; - - gwi.clauseType = WHERE; - - - if (icp) - { - // MCOL-1052 The condition could be useless. - // MariaDB bug 624 - without the fix_fields call, delete with join may error with "No query step". - //#if MYSQL_VERSION_ID < 50172 - //@bug 3039. fix fields for constants - if (!icp->fixed) - { - icp->fix_fields(gwi.thd, (Item**)&icp); - } - -//#endif - gwi.fatalParseError = false; -#ifdef DEBUG_WALK_COND - cerr << "------------------ WHERE -----------------------" << endl; - icp->traverse_cond(debug_walk, &gwi, Item::POSTFIX); - cerr << "------------------------------------------------\n" << endl; -#endif - - icp->traverse_cond(gp_walk, &gwi, Item::POSTFIX); - - if (gwi.fatalParseError) - { - // if this is dervied table process phase, mysql may have not developed the plan - // completely. Do not error and eventually mysql will call JOIN::exec() again. - // related to bug 2922. Need to find a way to skip calling rnd_init for derived table - // processing. - if (gwi.thd->derived_tables_processing) - { - gwi.thd->infinidb_vtable.isUnion = false; - gwi.thd->infinidb_vtable.isUpdateWithDerive = true; - return -1; - } - - setError(gwi.thd, ER_INTERNAL_ERROR, gwi.parseErrorText, gwi); - return ER_INTERNAL_ERROR; - } - } - else if (join && join->zero_result_cause) - { - gwi.rcWorkStack.push(new ConstantColumn((int64_t)0, ConstantColumn::NUM)); - } - - uint32_t failed = buildOuterJoin(gwi, select_lex); - - if (failed) return failed; - - // @bug5764. build outer join for view, make sure outerjoin filter is appended - // to the end of the filter list. - for (uint i = 0; i < gwi.viewList.size(); i++) - { - failed = gwi.viewList[i]->processOuterJoin(gwi); - - if (failed) - break; - } - - if (failed != 0) - return failed; - - ParseTree* filters = NULL; - ParseTree* ptp = NULL; - ParseTree* lhs = NULL; - - // @bug 2932. for "select * from region where r_name" case. if icp not null and - // ptWorkStack empty, the item is in rcWorkStack. - // MySQL 5.6 (MariaDB?). when icp is null and zero_result_cause is set, a constant 0 - // is pushed to rcWorkStack. - if (/*icp && */gwi.ptWorkStack.empty() && !gwi.rcWorkStack.empty()) - { - filters = new ParseTree(gwi.rcWorkStack.top()); - gwi.rcWorkStack.pop(); - } - - while (!gwi.ptWorkStack.empty()) - { - filters = gwi.ptWorkStack.top(); - gwi.ptWorkStack.pop(); - - if (gwi.ptWorkStack.empty()) - break; - - ptp = new ParseTree(new LogicOperator("and")); - //ptp->left(filters); - ptp->right(filters); - lhs = gwi.ptWorkStack.top(); - gwi.ptWorkStack.pop(); - //ptp->right(rhs); - ptp->left(lhs); - gwi.ptWorkStack.push(ptp); - } - - if (filters) - { - csep->filters(filters); -#ifdef DEBUG_WALK_COND - filters->drawTree("/tmp/filter1.dot"); -#endif - } - - gwi.clauseType = SELECT; -#ifdef DEBUG_WALK_COND - { - cerr << "------------------- SELECT --------------------" << endl; - List_iterator_fast it(*gi.groupByFields); - Item* item; - - while ((item = it++)) - { - debug_walk(item, 0); - } - - cerr << "-----------------------------------------------\n" << endl; - } -#endif - - // populate returnedcolumnlist and columnmap - List_iterator_fast it(*gi.groupByFields); - Item* item; - vector funcFieldVec; - string sel_cols_in_create; - string sel_cols_in_select; - bool redo = false; - - // empty rcWorkStack and ptWorkStack. They should all be empty by now. - clearStacks(gwi); - - // indicate the starting pos of scalar returned column, because some join column - // has been inserted to the returned column list. - if (gwi.subQuery) - { - ScalarSub* scalar = dynamic_cast(gwi.subQuery); - - if (scalar) - scalar->returnedColPos(gwi.additionalRetCols.size()); - } - - CalpontSelectExecutionPlan::SelectList selectSubList; - - while ((item = it++)) - { - string itemAlias = (item->name ? item->name : ""); - - // @bug 5916. Need to keep checking until getting concret item in case - // of nested view. - while (item->type() == Item::REF_ITEM) - { - Item_ref* ref = (Item_ref*)item; - item = (*(ref->ref)); - } - - Item::Type itype = item->type(); - - switch (itype) - { - case Item::FIELD_ITEM: - { - Item_field* ifp = (Item_field*)item; - SimpleColumn* sc = NULL; - ConstantColumn* constCol = NULL; - - if (ifp->field_name && string(ifp->field_name) == "*") - { - collectAllCols(gwi, ifp); - break; - } - - sc = buildSimpleColumn(ifp, gwi); - - if (sc) - { - constCol = buildConstColFromFilter(sc, gwi, gi); - boost::shared_ptr spcc(constCol); - boost::shared_ptr spsc(sc); - - if (sel_cols_in_create.length() != 0) - sel_cols_in_create += ", "; - - string fullname; - String str; - ifp->print(&str, QT_INFINIDB_NO_QUOTE); - fullname = str.c_ptr(); - - //sel_cols_in_create += fullname; - if (ifp->is_autogenerated_name) // no alias - { - sel_cols_in_create += fullname + " `" + escapeBackTick(str.c_ptr()) + "`"; - sc->alias(fullname); - } - else // alias - { - if (!itemAlias.empty()) - sc->alias(itemAlias); - - sel_cols_in_create += fullname + " `" + escapeBackTick(sc->alias().c_str()) + "`"; - } - - if (ifp->is_autogenerated_name) - gwi.selectCols.push_back("`" + escapeBackTick(fullname.c_str()) + "`" + " `" + - escapeBackTick(itemAlias.empty() ? ifp->name : itemAlias.c_str()) + "`"); - else - gwi.selectCols.push_back("`" + escapeBackTick((itemAlias.empty() ? ifp->name : itemAlias.c_str())) + "`"); - - // MCOL-1052 Replace SimpleColumn with ConstantColumn, - // since it must have a single value only. - if (constCol) - { - gwi.returnedCols.push_back(spcc); - gwi.columnMap.insert(CalpontSelectExecutionPlan::ColumnMap::value_type(string(ifp->field_name), spcc)); - } - else - { - gwi.returnedCols.push_back(spsc); - gwi.columnMap.insert(CalpontSelectExecutionPlan::ColumnMap::value_type(string(ifp->field_name), spsc)); - } - - TABLE_LIST* tmp = 0; - - if (ifp->cached_table) - tmp = ifp->cached_table; - - gwi.tableMap[make_aliastable(sc->schemaName(), sc->tableName(), sc->tableAlias(), sc->isInfiniDB())] = - make_pair(1, tmp); - } - else - { - setError(gwi.thd, ER_INTERNAL_ERROR, gwi.parseErrorText, gwi); - delete sc; - return ER_INTERNAL_ERROR; - } - - break; - } - - //aggregate column - case Item::SUM_FUNC_ITEM: - { - ReturnedColumn* ac = buildAggregateColumn(item, gwi); - - if (gwi.fatalParseError) - { - // e.g., non-support ref column - setError(gwi.thd, ER_CHECK_NOT_IMPLEMENTED, gwi.parseErrorText, gwi); - delete ac; - return ER_CHECK_NOT_IMPLEMENTED; - } - - // add this agg col to returnedColumnList - boost::shared_ptr spac(ac); - gwi.returnedCols.push_back(spac); - // This item will be used in HAVING later. - Item_func_or_sum* isfp = reinterpret_cast(item); - - if ( ! isfp->name_length ) - { - gwi.havingAggColsItems.push_back(item); - } - - gwi.selectCols.push_back('`' + escapeBackTick(spac->alias().c_str()) + '`'); - String str(256); - item->print(&str, QT_INFINIDB_NO_QUOTE); - - if (sel_cols_in_create.length() != 0) - sel_cols_in_create += ", "; - - sel_cols_in_create += string(str.c_ptr()) + " `" + escapeBackTick(spac->alias().c_str()) + "`"; - break; - } - - case Item::FUNC_ITEM: - { - Item_func* ifp = reinterpret_cast(item); - - // @bug4383. error out non-support stored function - if (ifp->functype() == Item_func::FUNC_SP) - { - gwi.fatalParseError = true; - gwi.parseErrorText = IDBErrorInfo::instance()->errorMsg(ERR_SP_FUNCTION_NOT_SUPPORT); - setError(gwi.thd, ER_CHECK_NOT_IMPLEMENTED, gwi.parseErrorText, gwi); - return ER_CHECK_NOT_IMPLEMENTED; - } - - if (string(ifp->func_name()) == "xor") - { - gwi.fatalParseError = true; - gwi.parseErrorText = IDBErrorInfo::instance()->errorMsg(ERR_FILTER_COND_EXP); - setError(gwi.thd, ER_CHECK_NOT_IMPLEMENTED, gwi.parseErrorText, gwi); - return ER_CHECK_NOT_IMPLEMENTED; - } - - uint16_t parseInfo = 0; - vector tmpVec; - bool hasNonSupportItem = false; - parse_item(ifp, tmpVec, hasNonSupportItem, parseInfo); - - if (ifp->has_subquery() || - string(ifp->func_name()) == string("") || - ifp->functype() == Item_func::NOT_ALL_FUNC || - parseInfo & SUB_BIT) - { - gwi.fatalParseError = true; - gwi.parseErrorText = IDBErrorInfo::instance()->errorMsg(ERR_NON_SUPPORT_SELECT_SUB); - setError(gwi.thd, ER_CHECK_NOT_IMPLEMENTED, gwi.parseErrorText, gwi); - return ER_CHECK_NOT_IMPLEMENTED; - } - - ReturnedColumn* rc = buildFunctionColumn(ifp, gwi, hasNonSupportItem); - SRCP srcp(rc); - - if (rc) - { - if (!hasNonSupportItem && !nonConstFunc(ifp) && !(parseInfo & AF_BIT) && tmpVec.size() == 0) - { - if (isUnion || unionSel || gwi.subSelectType != CalpontSelectExecutionPlan::MAIN_SELECT || - parseInfo & SUB_BIT ) //|| select_lex.group_list.elements != 0) - { - srcp.reset(buildReturnedColumn(item, gwi, gwi.fatalParseError)); - gwi.returnedCols.push_back(srcp); - - if (ifp->name) - srcp->alias(ifp->name); - - continue; - } - - if ( ((gwi.thd->lex)->sql_command == SQLCOM_UPDATE ) || - ((gwi.thd->lex)->sql_command == SQLCOM_DELETE ) || - ((gwi.thd->lex)->sql_command == SQLCOM_UPDATE_MULTI ) || - ((gwi.thd->lex)->sql_command == SQLCOM_DELETE_MULTI ) ) - { } - else - { - redo = true; - String str; - ifp->print(&str, QT_INFINIDB_NO_QUOTE); - gwi.selectCols.push_back(string(str.c_ptr()) + " " + "`" + escapeBackTick(item->name) + "`"); - } - - break; - } - - //SRCP srcp(rc); - gwi.returnedCols.push_back(srcp); - - if ( ((gwi.thd->lex)->sql_command == SQLCOM_UPDATE ) || ((gwi.thd->lex)->sql_command == SQLCOM_DELETE ) || ((gwi.thd->lex)->sql_command == SQLCOM_UPDATE_MULTI ) || ((gwi.thd->lex)->sql_command == SQLCOM_DELETE_MULTI )) - { } - else - { - String str(256); - ifp->print(&str, QT_INFINIDB_NO_QUOTE); - - if (sel_cols_in_create.length() != 0) - sel_cols_in_create += ", "; - - sel_cols_in_create += string(str.c_ptr()) + " `" + ifp->name + "`"; - gwi.selectCols.push_back("`" + escapeBackTick(ifp->name) + "`"); - } - } - else // InfiniDB Non support functions still go through post process for now - { - hasNonSupportItem = false; - uint32_t before_size = funcFieldVec.size(); - parse_item(ifp, funcFieldVec, hasNonSupportItem, parseInfo); - uint32_t after_size = funcFieldVec.size(); - - // group by func and func in subquery can not be post processed - // @bug3881. set_user_var can not be treated as constant function - // @bug5716. Try to avoid post process function for union query. - if ((gwi.subQuery /*|| select_lex.group_list.elements != 0 */ || - !csep->unionVec().empty() || isUnion) && - !hasNonSupportItem && (after_size - before_size) == 0 && - !(parseInfo & AGG_BIT) && !(parseInfo & SUB_BIT) && - string(ifp->func_name()) != "set_user_var") - { - String val, *str = ifp->val_str(&val); - string valStr; - - if (str) - valStr.assign(str->ptr(), str->length()); - - ConstantColumn* cc = NULL; - - if (!str) - { - cc = new ConstantColumn("", ConstantColumn::NULLDATA); - } - else if (ifp->result_type() == STRING_RESULT) - { - cc = new ConstantColumn(valStr, ConstantColumn::LITERAL); - } - else if (ifp->result_type() == DECIMAL_RESULT) - { - cc = buildDecimalColumn(ifp, gwi); - } - else - { - cc = new ConstantColumn(valStr, ConstantColumn::NUM); - cc->resultType(colType_MysqlToIDB(item)); - } - - SRCP srcp(cc); - - if (ifp->name) - cc->alias(ifp->name); - - gwi.returnedCols.push_back(srcp); - - // clear the error set by buildFunctionColumn - gwi.fatalParseError = false; - gwi.parseErrorText = ""; - break; - } - else if (hasNonSupportItem || parseInfo & AGG_BIT || parseInfo & SUB_BIT || - (gwi.fatalParseError && gwi.subQuery)) - { - if (gwi.parseErrorText.empty()) - { - Message::Args args; - args.add(ifp->func_name()); - gwi.parseErrorText = IDBErrorInfo::instance()->errorMsg(ERR_NON_SUPPORTED_FUNCTION, args); - } - - setError(gwi.thd, ER_CHECK_NOT_IMPLEMENTED, gwi.parseErrorText, gwi); - return ER_CHECK_NOT_IMPLEMENTED; - } - else if ( gwi.subQuery && (isPredicateFunction(ifp, &gwi) || ifp->type() == Item::COND_ITEM )) - { - gwi.fatalParseError = true; - gwi.parseErrorText = IDBErrorInfo::instance()->errorMsg(ERR_FILTER_COND_EXP); - setError(gwi.thd, ER_CHECK_NOT_IMPLEMENTED, gwi.parseErrorText, gwi); - return ER_CHECK_NOT_IMPLEMENTED; - } - - //@Bug 3030 Add error check for dml statement - if ( ((gwi.thd->lex)->sql_command == SQLCOM_UPDATE ) || ((gwi.thd->lex)->sql_command == SQLCOM_DELETE ) || ((gwi.thd->lex)->sql_command == SQLCOM_UPDATE_MULTI ) || ((gwi.thd->lex)->sql_command == SQLCOM_DELETE_MULTI ) ) - { - if ( after_size - before_size != 0 ) - { - gwi.parseErrorText = ifp->func_name(); - return -1; - } - } - - //@Bug 3021. Bypass postprocess for update and delete. - //if ( ((gwi.thd->lex)->sql_command == SQLCOM_UPDATE ) || ((gwi.thd->lex)->sql_command == SQLCOM_DELETE ) || ((gwi.thd->lex)->sql_command == SQLCOM_UPDATE_MULTI ) || ((gwi.thd->lex)->sql_command == SQLCOM_DELETE_MULTI )) - //{} - else - { - // @bug 3881. Here is the real redo part. - redo = true; - // @bug 1706 - String funcStr; - ifp->print(&funcStr, QT_INFINIDB); - gwi.selectCols.push_back(string(funcStr.c_ptr()) + " `" + escapeBackTick(ifp->name) + "`"); - // clear the error set by buildFunctionColumn - gwi.fatalParseError = false; - gwi.parseErrorText = ""; - } - } - - break; - } - - case Item::INT_ITEM: - { - if ( ((gwi.thd->lex)->sql_command == SQLCOM_UPDATE ) || ((gwi.thd->lex)->sql_command == SQLCOM_DELETE ) || ((gwi.thd->lex)->sql_command == SQLCOM_UPDATE_MULTI ) || ((gwi.thd->lex)->sql_command == SQLCOM_DELETE_MULTI )) - { } - else - { - // do not push the dummy column (mysql added) to returnedCol - if (item->name && string(item->name) == "Not_used") - continue; - - // @bug3509. Constant column is sent to ExeMgr now. - SRCP srcp(buildReturnedColumn(item, gwi, gwi.fatalParseError)); - - if (item->name) - srcp->alias(item->name); - - gwi.returnedCols.push_back(srcp); - - Item_int* isp = reinterpret_cast(item); - ostringstream oss; - oss << isp->value << " `" << escapeBackTick(srcp->alias().c_str()) << "`"; - - if (sel_cols_in_create.length() != 0) - sel_cols_in_create += ", "; - - sel_cols_in_create += oss.str(); - gwi.selectCols.push_back(oss.str()); - } - - break; - } - - case Item::STRING_ITEM: - { - if ( ((gwi.thd->lex)->sql_command == SQLCOM_UPDATE ) || ((gwi.thd->lex)->sql_command == SQLCOM_DELETE ) || ((gwi.thd->lex)->sql_command == SQLCOM_UPDATE_MULTI ) || ((gwi.thd->lex)->sql_command == SQLCOM_DELETE_MULTI )) - { } - else - { - SRCP srcp(buildReturnedColumn(item, gwi, gwi.fatalParseError)); - gwi.returnedCols.push_back(srcp); - - if (item->name) - srcp->alias(item->name); - - Item_string* isp = reinterpret_cast(item); - String val, *str = isp->val_str(&val); - string valStr; - valStr.assign(str->ptr(), str->length()); - string name = "'" + valStr + "'" + " " + "`" + escapeBackTick(srcp->alias().c_str()) + "`"; - - if (sel_cols_in_create.length() != 0) - sel_cols_in_create += ", "; - - sel_cols_in_create += name; - gwi.selectCols.push_back(name); - } - - break; - } - - case Item::DECIMAL_ITEM: - { - if ( ((gwi.thd->lex)->sql_command == SQLCOM_UPDATE ) || ((gwi.thd->lex)->sql_command == SQLCOM_DELETE ) || ((gwi.thd->lex)->sql_command == SQLCOM_UPDATE_MULTI ) || ((gwi.thd->lex)->sql_command == SQLCOM_DELETE_MULTI )) - { } - else - { - SRCP srcp(buildReturnedColumn(item, gwi, gwi.fatalParseError)); - gwi.returnedCols.push_back(srcp); - - if (item->name) - srcp->alias(item->name); - - Item_decimal* isp = reinterpret_cast(item); - String val, *str = isp->val_str(&val); - string valStr; - valStr.assign(str->ptr(), str->length()); - ostringstream oss; - oss << valStr.c_str() << " `" << escapeBackTick(srcp->alias().c_str()) << "`"; - - if (sel_cols_in_create.length() != 0) - sel_cols_in_create += ", "; - - sel_cols_in_create += oss.str(); - gwi.selectCols.push_back(oss.str()); - } - - break; - } - - case Item::NULL_ITEM: - { - /*if ( ((gwi.thd->lex)->sql_command == SQLCOM_UPDATE ) || ((gwi.thd->lex)->sql_command == SQLCOM_DELETE ) || ((gwi.thd->lex)->sql_command == SQLCOM_UPDATE_MULTI ) || ((gwi.thd->lex)->sql_command == SQLCOM_DELETE_MULTI ) ) - { } - else - { - SRCP srcp(buildReturnedColumn(item, gwi, gwi.fatalParseError)); - gwi.returnedCols.push_back(srcp); - - if (item->name) - srcp->alias(item->name); - - string name = string("null `") + escapeBackTick(srcp->alias().c_str()) + string("`") ; - - if (sel_cols_in_create.length() != 0) - sel_cols_in_create += ", "; - - sel_cols_in_create += name; - gwi.selectCols.push_back("null"); - }*/ - - break; - } - - case Item::SUBSELECT_ITEM: - { - Item_subselect* sub = (Item_subselect*)item; - - if (sub->substype() != Item_subselect::SINGLEROW_SUBS) - { - gwi.fatalParseError = true; - gwi.parseErrorText = IDBErrorInfo::instance()->errorMsg(ERR_NON_SUPPORT_SELECT_SUB); - setError(gwi.thd, ER_CHECK_NOT_IMPLEMENTED, gwi.parseErrorText, gwi); - return ER_CHECK_NOT_IMPLEMENTED; - } - -#ifdef DEBUG_WALK_COND - cerr << "SELECT clause SUBSELECT Item: " << sub->substype() << endl; - JOIN* join = sub->get_select_lex()->join; - - if (join) - { - Item_cond* cond = reinterpret_cast(join->conds); - - if (cond) - cond->traverse_cond(debug_walk, &gwi, Item::POSTFIX); - } - - cerr << "Finish SELECT clause subselect item traversing" << endl; -#endif - SelectSubQuery* selectSub = new SelectSubQuery(gwi, sub); - //selectSub->gwip(&gwi); - SCSEP ssub = selectSub->transform(); - - if (!ssub || gwi.fatalParseError) - { - if (gwi.parseErrorText.empty()) - gwi.parseErrorText = "Unsupported Item in SELECT subquery."; - - setError(gwi.thd, ER_CHECK_NOT_IMPLEMENTED, gwi.parseErrorText, gwi); - return ER_CHECK_NOT_IMPLEMENTED; - } - - selectSubList.push_back(ssub); - SimpleColumn* rc = new SimpleColumn(); - rc->colSource(rc->colSource() | SELECT_SUB); - - if (sub->get_select_lex()->get_table_list()) - rc->viewName(lower(getViewName(sub->get_select_lex()->get_table_list()))); - - if (sub->name) - rc->alias(sub->name); - - gwi.returnedCols.push_back(SRCP(rc)); - String str; - sub->get_select_lex()->print(gwi.thd, &str, QT_INFINIDB_NO_QUOTE); - sel_cols_in_create += "(" + string(str.c_ptr()) + ")"; - - if (sub->name) - { - sel_cols_in_create += "`" + escapeBackTick(sub->name) + "`"; - gwi.selectCols.push_back(sub->name); - } - else - { - sel_cols_in_create += "`" + escapeBackTick(str.c_ptr()) + "`"; - gwi.selectCols.push_back("`" + escapeBackTick(str.c_ptr()) + "`"); - } - - break; - } - - case Item::COND_ITEM: - { - gwi.fatalParseError = true; - gwi.parseErrorText = IDBErrorInfo::instance()->errorMsg(ERR_FILTER_COND_EXP); - setError(gwi.thd, ER_CHECK_NOT_IMPLEMENTED, gwi.parseErrorText, gwi); - return ER_CHECK_NOT_IMPLEMENTED; - } - - case Item::EXPR_CACHE_ITEM: - { - printf("EXPR_CACHE_ITEM in getSelectPlan\n"); - gwi.fatalParseError = true; - gwi.parseErrorText = IDBErrorInfo::instance()->errorMsg(ERR_UNKNOWN_COL); - setError(gwi.thd, ER_CHECK_NOT_IMPLEMENTED, gwi.parseErrorText, gwi); - return ER_CHECK_NOT_IMPLEMENTED; - } - - case Item::WINDOW_FUNC_ITEM: - { - SRCP srcp(buildWindowFunctionColumn(item, gwi, gwi.fatalParseError)); - - if (!srcp || gwi.fatalParseError) - { - if (gwi.parseErrorText.empty()) - gwi.parseErrorText = "Unsupported Item in SELECT subquery."; - - setError(gwi.thd, ER_CHECK_NOT_IMPLEMENTED, gwi.parseErrorText, gwi); - return ER_CHECK_NOT_IMPLEMENTED; - } - - gwi.returnedCols.push_back(srcp); - break; - } - - default: - { - break; - } - } - } - - // @bug4388 normalize the project coltypes for union main select list - if (!csep->unionVec().empty()) - { - for (uint32_t i = 0; i < gwi.returnedCols.size(); i++) - { - vector coltypes; - - for (uint32_t j = 0; j < csep->unionVec().size(); j++) - { - coltypes.push_back( - dynamic_cast(csep->unionVec()[j].get())->returnedCols()[i]->resultType()); - - // @bug5976. set hasAggregate true for the main column if - // one corresponding union column has aggregate - if (dynamic_cast(csep->unionVec()[j].get())->returnedCols()[i]->hasAggregate()) - gwi.returnedCols[i]->hasAggregate(true); - } - - gwi.returnedCols[i]->resultType(dataconvert::DataConvert::convertUnionColType(coltypes)); - } - } - - // Having clause handling - gwi.clauseType = HAVING; - clearStacks(gwi); - ParseTree* havingFilter = 0; - // clear fatalParseError that may be left from post process functions - gwi.fatalParseError = false; - gwi.parseErrorText = ""; - - if (gi.groupByHaving != 0) - { - Item_cond* having = reinterpret_cast(gi.groupByHaving); -#ifdef DEBUG_WALK_COND - cerr << "------------------- HAVING ---------------------" << endl; - having->traverse_cond(debug_walk, &gwi, Item::POSTFIX); - cerr << "------------------------------------------------\n" << endl; -#endif - having->traverse_cond(gp_walk, &gwi, Item::POSTFIX); - - if (gwi.fatalParseError) - { - setError(gwi.thd, ER_INTERNAL_ERROR, gwi.parseErrorText, gwi); - return ER_INTERNAL_ERROR; - } - - ParseTree* ptp = 0; - ParseTree* rhs = 0; - - // @bug 4215. some function filter will be in the rcWorkStack. - if (gwi.ptWorkStack.empty() && !gwi.rcWorkStack.empty()) - { - havingFilter = new ParseTree(gwi.rcWorkStack.top()); - gwi.rcWorkStack.pop(); - } - - while (!gwi.ptWorkStack.empty()) - { - havingFilter = gwi.ptWorkStack.top(); - gwi.ptWorkStack.pop(); - - if (gwi.ptWorkStack.empty()) - break; - - ptp = new ParseTree(new LogicOperator("and")); - ptp->left(havingFilter); - rhs = gwi.ptWorkStack.top(); - gwi.ptWorkStack.pop(); - ptp->right(rhs); - gwi.ptWorkStack.push(ptp); - } - } - - // for post process expressions on the select list - // error out post process for union and sub select unit - if (isUnion || gwi.subSelectType != CalpontSelectExecutionPlan::MAIN_SELECT) - { - if (funcFieldVec.size() != 0 && !gwi.fatalParseError) - { - string emsg("Fatal parse error in vtable mode: Unsupported Items in union or sub select unit"); - setError(gwi.thd, ER_CHECK_NOT_IMPLEMENTED, emsg); - return ER_CHECK_NOT_IMPLEMENTED; - } - } - - for (uint32_t i = 0; i < funcFieldVec.size(); i++) - { - SimpleColumn* sc = buildSimpleColumn(funcFieldVec[i], gwi); - - if (!sc || gwi.fatalParseError) - { - string emsg; - - if (gwi.parseErrorText.empty()) - { - emsg = "un-recognized column"; - - if (funcFieldVec[i]->name) - emsg += string(funcFieldVec[i]->name); - } - else - { - emsg = gwi.parseErrorText; - } - - setError(gwi.thd, ER_INTERNAL_ERROR, emsg, gwi); - return ER_INTERNAL_ERROR; - } - - String str; - funcFieldVec[i]->print(&str, QT_INFINIDB_NO_QUOTE); - sc->alias(string(str.c_ptr())); - //sc->tableAlias(funcFieldVec[i]->table_name); - sc->tableAlias(sc->tableAlias()); - SRCP srcp(sc); - uint32_t j = 0; - - for (; j < gwi.returnedCols.size(); j++) - { - if (sc->sameColumn(gwi.returnedCols[j].get())) - { - SimpleColumn* field = dynamic_cast(gwi.returnedCols[j].get()); - - if (field && field->alias() == sc->alias()) - break; - } - } - - if (j == gwi.returnedCols.size()) - { - gwi.returnedCols.push_back(srcp); - gwi.columnMap.insert(CalpontSelectExecutionPlan::ColumnMap::value_type(string(funcFieldVec[i]->field_name), srcp)); - - if (sel_cols_in_create.length() != 0) - sel_cols_in_create += ", "; - - string fullname; - fullname = str.c_ptr(); - sel_cols_in_create += fullname + " `" + escapeBackTick(fullname.c_str()) + "`"; - TABLE_LIST* tmp = (funcFieldVec[i]->cached_table ? funcFieldVec[i]->cached_table : 0); - gwi.tableMap[make_aliastable(sc->schemaName(), sc->tableName(), sc->tableAlias(), sc->isInfiniDB())] = - make_pair(1, tmp); - } - } - - // post-process Order by list and expressions on select by redo phase1. only for vtable - // ignore ORDER BY clause for union select unit - string ord_cols = ""; // for normal select phase - SRCP minSc; // min width projected column. for count(*) use - - // Group by list. not valid for union main query - if (gwi.thd->infinidb_vtable.vtable_state == THD::INFINIDB_CREATE_VTABLE && !unionSel) - { - gwi.clauseType = GROUP_BY; - Item* nonSupportItem = NULL; - ORDER* groupcol = reinterpret_cast(gi.groupByGroup); - - // check if window functions are in order by. InfiniDB process order by list if - // window functions are involved, either in order by or projection. - bool hasWindowFunc = gwi.hasWindowFunc; - gwi.hasWindowFunc = false; - - for (; groupcol; groupcol = groupcol->next) - { - if ((*(groupcol->item))->type() == Item::WINDOW_FUNC_ITEM) - gwi.hasWindowFunc = true; - } - - if (gwi.hasWindowFunc) - { - gwi.fatalParseError = true; - gwi.parseErrorText = IDBErrorInfo::instance()->errorMsg(ERR_WF_NOT_ALLOWED, "GROUP BY clause"); - setError(gwi.thd, ER_CHECK_NOT_IMPLEMENTED, gwi.parseErrorText, gwi); - return ER_CHECK_NOT_IMPLEMENTED; - } - - gwi.hasWindowFunc = hasWindowFunc; - groupcol = reinterpret_cast(gi.groupByGroup); - - for (; groupcol; groupcol = groupcol->next) - { - Item* groupItem = *(groupcol->item); - - // @bug5993. Could be nested ref. - while (groupItem->type() == Item::REF_ITEM) - groupItem = (*((Item_ref*)groupItem)->ref); - - if (groupItem->type() == Item::FUNC_ITEM) - { - Item_func* ifp = (Item_func*)groupItem; - - // call buildFunctionColumn here mostly for finding out - // non-support column on GB list. Should be simplified. - ReturnedColumn* fc = buildFunctionColumn(ifp, gwi, gwi.fatalParseError); - - if (!fc || gwi.fatalParseError) - { - nonSupportItem = ifp; - break; - } - - if (groupcol->in_field_list && groupcol->counter_used) - { - delete fc; - fc = gwi.returnedCols[groupcol->counter - 1].get(); - SRCP srcp(fc->clone()); - - // check if no column parm - for (uint32_t i = 0; i < gwi.no_parm_func_list.size(); i++) - { - if (gwi.no_parm_func_list[i]->expressionId() == fc->expressionId()) - { - gwi.no_parm_func_list.push_back(dynamic_cast(srcp.get())); - break; - } - } - - srcp->orderPos(groupcol->counter - 1); - gwi.groupByCols.push_back(srcp); - continue; - } - else if (!groupItem->is_autogenerated_name) // alias - { - uint32_t i = 0; - - for (; i < gwi.returnedCols.size(); i++) - { - if (string(groupItem->name) == gwi.returnedCols[i]->alias()) - { - ReturnedColumn* rc = gwi.returnedCols[i]->clone(); - rc->orderPos(i); - gwi.groupByCols.push_back(SRCP(rc)); - delete fc; - break; - } - } - - if (i == gwi.returnedCols.size()) - { - nonSupportItem = groupItem; - break; - } - } - else - { - uint32_t i = 0; - - for (; i < gwi.returnedCols.size(); i++) - { - if (fc->operator==(gwi.returnedCols[i].get())) - { - ReturnedColumn* rc = gwi.returnedCols[i]->clone(); - rc->orderPos(i); - gwi.groupByCols.push_back(SRCP(rc)); - delete fc; - break; - } - } - - if (i == gwi.returnedCols.size()) - { - gwi.groupByCols.push_back(SRCP(fc)); - break; - } - } - } - else if (groupItem->type() == Item::FIELD_ITEM) - { - Item_field* ifp = (Item_field*)groupItem; - // this GB col could be an alias of F&E on the SELECT clause, not necessarily a field. - ReturnedColumn* rc = buildSimpleColumn(ifp, gwi); - SimpleColumn* sc = dynamic_cast(rc); - - for (uint32_t j = 0; j < gwi.returnedCols.size(); j++) - { - if (sc) - { - if (sc->sameColumn(gwi.returnedCols[j].get())) - { - sc->orderPos(j); - break; - } - else if (strcasecmp(sc->alias().c_str(), gwi.returnedCols[j]->alias().c_str()) == 0) - { - rc = gwi.returnedCols[j].get()->clone(); - rc->orderPos(j); - break; - } - } - else - { - if (ifp->name && string(ifp->name) == gwi.returnedCols[j].get()->alias()) - { - rc = gwi.returnedCols[j].get()->clone(); - rc->orderPos(j); - break; - } - } - } - - if (!rc) - { - nonSupportItem = ifp; - break; - } - - SRCP srcp(rc); - - // bug 3151 - AggregateColumn* ac = dynamic_cast(rc); - - if (ac) - { - nonSupportItem = ifp; - break; - } - - gwi.groupByCols.push_back(srcp); - gwi.columnMap.insert(CalpontSelectExecutionPlan::ColumnMap::value_type(string(ifp->field_name), srcp)); - } - // @bug5638. The group by column is constant but not counter, alias has to match a column - // on the select list - else if (!groupcol->counter_used && - (groupItem->type() == Item::INT_ITEM || - groupItem->type() == Item::STRING_ITEM || - groupItem->type() == Item::REAL_ITEM || - groupItem->type() == Item::DECIMAL_ITEM)) - { - ReturnedColumn* rc = 0; - - for (uint32_t j = 0; j < gwi.returnedCols.size(); j++) - { - if (groupItem->name && string(groupItem->name) == gwi.returnedCols[j].get()->alias()) - { - rc = gwi.returnedCols[j].get()->clone(); - rc->orderPos(j); - break; - } - } - - if (!rc) - { - nonSupportItem = groupItem; - break; - } - - gwi.groupByCols.push_back(SRCP(rc)); - } - else if ((*(groupcol->item))->type() == Item::SUBSELECT_ITEM) - { - if (!groupcol->in_field_list || !groupItem->name) - { - nonSupportItem = groupItem; - } - else - { - uint32_t i = 0; - - for (; i < gwi.returnedCols.size(); i++) - { - if (string(groupItem->name) == gwi.returnedCols[i]->alias()) - { - ReturnedColumn* rc = gwi.returnedCols[i]->clone(); - rc->orderPos(i); - gwi.groupByCols.push_back(SRCP(rc)); - break; - } - } - - if (i == gwi.returnedCols.size()) - { - nonSupportItem = groupItem; - } - } - } - // @bug 3761. - else if (groupcol->counter_used) - { - if (gwi.returnedCols.size() <= (uint32_t)(groupcol->counter - 1)) - { - nonSupportItem = groupItem; - } - else - { - gwi.groupByCols.push_back(SRCP(gwi.returnedCols[groupcol->counter - 1]->clone())); - } - } - else - { - nonSupportItem = groupItem; - } - - } - - // @bug 4756. Add internal groupby column for correlated join to the groupby list - if (gwi.aggOnSelect && !gwi.subGroupByCols.empty()) - gwi.groupByCols.insert(gwi.groupByCols.end(), gwi.subGroupByCols.begin(), gwi.subGroupByCols.end()); - - // this is window func on SELECT becuase ORDER BY has not been processed - if (!gwi.windowFuncList.empty() && !gwi.subGroupByCols.empty()) - { - for (uint32_t i = 0; i < gwi.windowFuncList.size(); i++) - { - if (gwi.windowFuncList[i]->hasWindowFunc()) - { - vector windowFunctions = gwi.windowFuncList[i]->windowfunctionColumnList(); - - for (uint32_t j = 0; j < windowFunctions.size(); j++) - windowFunctions[j]->addToPartition(gwi.subGroupByCols); - } - } - } - - if (nonSupportItem) - { - Message::Args args; - - if (nonSupportItem->name) - args.add("'" + string(nonSupportItem->name) + "'"); - else - args.add(""); - - gwi.parseErrorText = IDBErrorInfo::instance()->errorMsg(ERR_NON_SUPPORT_GROUP_BY, args); - setError(gwi.thd, ER_CHECK_NOT_IMPLEMENTED, gwi.parseErrorText, gwi); - return ER_CHECK_NOT_IMPLEMENTED; - } - } // GROUP processing ends here - - - if (gwi.thd->infinidb_vtable.vtable_state == THD::INFINIDB_CREATE_VTABLE) - { - ORDER* ordercol = reinterpret_cast(gi.groupByOrder); - string create_query(gwi.thd->infinidb_vtable.create_vtable_query.c_ptr()); - string select_query(gwi.thd->infinidb_vtable.select_vtable_query.c_ptr()); - string lower_create_query(gwi.thd->infinidb_vtable.create_vtable_query.c_ptr()); - string lower_select_query(gwi.thd->infinidb_vtable.select_vtable_query.c_ptr()); - algorithm::to_lower(lower_create_query); - algorithm::to_lower(lower_select_query); - - - // check if window functions are in order by. InfiniDB process order by list if - // window functions are involved, either in order by or projection. - for (; ordercol; ordercol = ordercol->next) - { - if ((*(ordercol->item))->type() == Item::WINDOW_FUNC_ITEM) - gwi.hasWindowFunc = true; - } - - // re-visit the first of ordercol list - ordercol = reinterpret_cast(gi.groupByOrder); - - // for subquery, order+limit by will be supported in infinidb. build order by columns - // @todo union order by and limit support - //if (gwi.hasWindowFunc || gwi.subSelectType != CalpontSelectExecutionPlan::MAIN_SELECT) - for (; ordercol; ordercol = ordercol->next) { ReturnedColumn* rc = NULL; @@ -9382,64 +6953,11 @@ int getSelectPlan(gp_walk_info& gwi, SELECT_LEX& select_lex, SCSEP& csep, bool i else rc = buildReturnedColumn(ord_item, gwi, gwi.fatalParseError); - // Looking for a match for this item in GROUP BY list. - if ( rc && ord_item->type() == Item::FIELD_ITEM ) - { - execplan::CalpontSelectExecutionPlan::ReturnedColumnList::iterator iter = gwi.groupByCols.begin(); - - for ( ; iter != gwi.groupByCols.end(); iter++ ) - { - if ( rc->sameColumn((*iter).get()) ) - break; - } - - // MCOL-1052 Find and remove the optimized field - // from ORDER using cond_pushed filters. - if (buildConstColFromFilter( - dynamic_cast(rc), gwi, gi)) - { - break; - } - - // MCOL-1052 GROUP BY items list doesn't contain - // this ORDER BY item. - if ( iter == gwi.groupByCols.end() ) - { - Item_ident* iip = reinterpret_cast(ord_item); - std::ostringstream ostream; - ostream << "'"; - - if (iip->db_name) - ostream << iip->db_name << '.'; - else - ostream << "unknown db" << '.'; - - if (iip->table_name) - ostream << iip->table_name << '.'; - else - ostream << "unknown table" << '.'; - - if (iip->field_name) - ostream << iip->field_name; - else - ostream << "unknown field"; - - ostream << "'"; - Message::Args args; - args.add(ostream.str()); - string emsg = IDBErrorInfo::instance()->errorMsg(ERR_NOT_GROUPBY_EXPRESSION, args); - gwi.parseErrorText = emsg; - setError(gwi.thd, ER_INTERNAL_ERROR, emsg, gwi); - return ERR_NOT_GROUPBY_EXPRESSION; - } - } - // @bug5501 try item_ptr if item can not be fixed. For some // weird dml statement state, item can not be fixed but the // infomation is available in item_ptr. if (!rc || gwi.fatalParseError) { - gwi.fatalParseError = false; Item* item_ptr = ordercol->item_ptr; while (item_ptr->type() == Item::REF_ITEM) @@ -9448,17 +6966,6 @@ int getSelectPlan(gp_walk_info& gwi, SELECT_LEX& select_lex, SCSEP& csep, bool i rc = buildReturnedColumn(item_ptr, gwi, gwi.fatalParseError); } - // This ORDER BY item must be an agg function - - // the ordercol->item_ptr and exteded SELECT list - // must contain the corresponding item. - if (!rc) - { - Item* item_ptr = ordercol->item_ptr; - - if (item_ptr) - rc = buildReturnedColumn(item_ptr, gwi, gwi.fatalParseError); - } - if (!rc) { string emsg = IDBErrorInfo::instance()->errorMsg(ERR_NON_SUPPORT_ORDER_BY); @@ -9475,31 +6982,370 @@ int getSelectPlan(gp_walk_info& gwi, SELECT_LEX& select_lex, SCSEP& csep, bool i gwi.orderByCols.push_back(SRCP(rc)); } + } + else if (!isUnion) + { + vector fieldVec; + bool addToSel; + // the following order by is just for redo phase + if (!unionSel) + { + for (; ordercol; ordercol = ordercol->next) + { + Item* ord_item = *(ordercol->item); - // make sure columnmap, returnedcols and count(*) arg_list are not empty - TableMap::iterator tb_iter = gwi.tableMap.begin(); + // @bug5993. Could be nested ref. + while (ord_item->type() == Item::REF_ITEM) + ord_item = (*((Item_ref*)ord_item)->ref); + + // @bug 1706. re-construct the order by item one by one + //Item* ord_item = *(ordercol->item); + if (ord_cols.length() != 0) + ord_cols += ", "; + + addToSel = true; + string fullname; + + if (ordercol->in_field_list && ordercol->counter_used) + { + ostringstream oss; + oss << ordercol->counter; + ord_cols += oss.str(); + + if (ordercol->direction != ORDER::ORDER_ASC) + ord_cols += " desc"; + + continue; + } + + else if (ord_item->type() == Item::FUNC_ITEM) + { + // @bug 2621. order by alias + if (!ord_item->is_autogenerated_name && ord_item->name) + { + ord_cols += ord_item->name; + continue; + } + + // if there's group by clause or aggregate column, check to see + // if this item or the arguments is on the GB list. + ReturnedColumn* rc = 0; + // check if this order by column is on the select list + Item_func* ifp = (Item_func*)(*(ordercol->item)); + rc = buildFunctionColumn(ifp, gwi, gwi.fatalParseError); + + if (rc) + { + for (uint32_t i = 0; i < gwi.returnedCols.size(); i++) + { + if (rc && rc->operator==(gwi.returnedCols[i].get())) + { + ostringstream oss; + oss << i + 1; + ord_cols += oss.str(); + addToSel = false; + break; + } + } + } + + if (addToSel) + { + FunctionColumn* fc = dynamic_cast(rc); + + if (fc) + { + addToSel = false; + redo = true; + string ord_func = string(ifp->func_name()) + "("; + + for (uint32_t i = 0; i < fc->functionParms().size(); i++) + { + if (i != 0) + ord_func += ","; + + for (uint32_t j = 0; j < gwi.returnedCols.size(); j++) + { + if (fc->functionParms()[i]->data()->operator==(gwi.returnedCols[j].get())) + { + ord_func += "`" + escapeBackTick(gwi.returnedCols[j]->alias().c_str()) + "`"; + continue; + } + + AggregateColumn* ac = dynamic_cast(fc->functionParms()[i]->data()); + + if (ac) + { + gwi.parseErrorText = IDBErrorInfo::instance()->errorMsg(ERR_NON_SUPPORT_ORDER_BY); + setError(gwi.thd, ER_CHECK_NOT_IMPLEMENTED, gwi.parseErrorText, gwi); + return ER_CHECK_NOT_IMPLEMENTED; + } + + addToSel = true; + //continue; + + } + } + + ord_func += ")"; + + if (!addToSel) + ord_cols += ord_func; + } + } + } + else if (ord_item->type() == Item::SUBSELECT_ITEM) + { + string emsg = IDBErrorInfo::instance()->errorMsg(ERR_NON_SUPPORT_ORDER_BY); + setError(gwi.thd, ER_CHECK_NOT_IMPLEMENTED, emsg, gwi); + return ER_CHECK_NOT_IMPLEMENTED; + } + + else if (ord_item->type() == Item::SUM_FUNC_ITEM) + { + ReturnedColumn* ac = 0; + + Item_sum* ifp = (Item_sum*)(*(ordercol->item)); + // @bug3477. add aggregate column to the select list of the create phase. + ac = buildAggregateColumn(ifp, gwi); + + if (!ac) + { + setError(gwi.thd, ER_CHECK_NOT_IMPLEMENTED, + IDBErrorInfo::instance()->errorMsg(ERR_NON_SUPPORT_ORDER_BY), gwi); + return ER_CHECK_NOT_IMPLEMENTED; + } + + // check if this order by column is on the select list + for (uint32_t i = 0; i < gwi.returnedCols.size(); i++) + { + AggregateColumn* ret = dynamic_cast(gwi.returnedCols[i].get()); + + if (!ret) + continue; + + if (ac->operator==(gwi.returnedCols[i].get())) + { + ostringstream oss; + oss << i + 1; + ord_cols += oss.str(); + addToSel = false; + break; + } + } + + if (ac || !gwi.groupByCols.empty()) + { + if (addToSel) + { + redo = true; + // @bug 3076. do not add the argument of aggregate function to the SELECT list, + // instead, add the whole column + String str; + ord_item->print(&str, QT_INFINIDB_NO_QUOTE); + + if (sel_cols_in_create.length() != 0) + sel_cols_in_create += ", "; + + sel_cols_in_create += str.c_ptr(); + //gwi.selectCols.push_back(" `" + string(str.c_ptr()) + "`"); + SRCP srcp(ac); + gwi.returnedCols.push_back(srcp); + ord_cols += " `" + escapeBackTick(str.c_ptr()) + "`"; + } + + if (ordercol->direction != ORDER::ORDER_ASC) + ord_cols += " desc"; + + continue; + } + } + else if (ord_item->name && ord_item->type() == Item::FIELD_ITEM) + { + Item_field* field = reinterpret_cast(ord_item); + ReturnedColumn* rc = buildSimpleColumn(field, gwi); + fullname = field->full_name(); +// if (field->db_name) +// fullname += string(field->db_name) + "."; +// if (field->table_name) +// fullname += string(field->table_name) + "."; +// if (field->field_name) +// fullname += string(field->field_name); + + for (uint32_t i = 0; i < gwi.returnedCols.size(); i++) + { + SimpleColumn* sc = dynamic_cast(gwi.returnedCols[i].get()); + + if (sc && ((Item_field*)ord_item)->cached_table && + (strcasecmp(getViewName(((Item_field*)ord_item)->cached_table).c_str(), sc->viewName().c_str()) != 0)) + { + continue; + } + + if (strcasecmp(fullname.c_str(), gwi.returnedCols[i]->alias().c_str()) == 0 || + strcasecmp(ord_item->name, gwi.returnedCols[i]->alias().c_str()) == 0) + { + ord_cols += string(" `") + escapeBackTick(gwi.returnedCols[i]->alias().c_str()) + '`'; + addToSel = false; + break; + } + + if (sc && sc->sameColumn(rc)) + { + ostringstream oss; + oss << i + 1; + ord_cols += oss.str(); + addToSel = false; + break; + } + } + } + + if (addToSel) + { + // @bug 2719. Error out order by not on the distinct select list. + if (select_lex.options & SELECT_DISTINCT) + { + gwi.parseErrorText = IDBErrorInfo::instance()->errorMsg(ERR_ORDERBY_NOT_IN_DISTINCT); + setError(gwi.thd, ER_CHECK_NOT_IMPLEMENTED, gwi.parseErrorText, gwi); + return ER_CHECK_NOT_IMPLEMENTED; + } + + bool hasNonSupportItem = false; + uint16_t parseInfo = 0; + parse_item(ord_item, fieldVec, hasNonSupportItem, parseInfo); + + if (hasNonSupportItem) + { + string emsg = IDBErrorInfo::instance()->errorMsg(ERR_NON_SUPPORT_ORDER_BY); + setError(gwi.thd, ER_CHECK_NOT_IMPLEMENTED, emsg, gwi); + return ER_CHECK_NOT_IMPLEMENTED; + } + + String str; + ord_item->print(&str, QT_INFINIDB); + ord_cols += str.c_ptr(); + } + + if (ordercol->direction != ORDER::ORDER_ASC) + ord_cols += " desc"; + } + } + + redo = (redo || fieldVec.size() != 0); + + // populate string to be added to the select list for order by + for (uint32_t i = 0; i < fieldVec.size(); i++) + { + SimpleColumn* sc = buildSimpleColumn(fieldVec[i], gwi); + + if (!sc) + { + string emsg = IDBErrorInfo::instance()->errorMsg(ERR_NON_SUPPORT_ORDER_BY); + setError(gwi.thd, ER_CHECK_NOT_IMPLEMENTED, emsg, gwi); + return ER_CHECK_NOT_IMPLEMENTED; + } + + String str; + fieldVec[i]->print(&str, QT_INFINIDB_NO_QUOTE); + sc->alias(string(str.c_ptr())); + SRCP srcp(sc); + uint32_t j = 0; + + for (; j < gwi.returnedCols.size(); j++) + { + if (sc->sameColumn(gwi.returnedCols[j].get())) + { + SimpleColumn* field = dynamic_cast(gwi.returnedCols[j].get()); + + if (field && field->alias() == sc->alias()) + break; + } + } + + if (j == gwi.returnedCols.size()) + { + string fullname; + + if (sel_cols_in_create.length() != 0) + sel_cols_in_create += ", "; + + fullname = str.c_ptr(); + sel_cols_in_create += fullname + " `" + escapeBackTick(fullname.c_str()) + "`"; + + gwi.returnedCols.push_back(srcp); + gwi.columnMap.insert(CalpontSelectExecutionPlan::ColumnMap::value_type(string(fieldVec[i]->field_name), srcp)); + TABLE_LIST* tmp = (fieldVec[i]->cached_table ? fieldVec[i]->cached_table : 0); + gwi.tableMap[make_aliastable(sc->schemaName(), sc->tableName(), sc->tableAlias(), sc->isInfiniDB())] = + make_pair(1, tmp); + } + } + } + + // make sure columnmap, returnedcols and count(*) arg_list are not empty + TableMap::iterator tb_iter = gwi.tableMap.begin(); + + try + { + for (; tb_iter != gwi.tableMap.end(); tb_iter++) + { + if ((*tb_iter).second.first == 1) continue; + + CalpontSystemCatalog::TableAliasName tan = (*tb_iter).first; + CalpontSystemCatalog::TableName tn = make_table((*tb_iter).first.schema, (*tb_iter).first.table); + SimpleColumn* sc = getSmallestColumn(csc, tn, tan, (*tb_iter).second.second->table, gwi); + SRCP srcp(sc); + gwi.columnMap.insert(CalpontSelectExecutionPlan::ColumnMap::value_type(sc->columnName(), srcp)); + (*tb_iter).second.first = 1; + } + } + catch (runtime_error& e) + { + setError(gwi.thd, ER_INTERNAL_ERROR, e.what(), gwi); + CalpontSystemCatalog::removeCalpontSystemCatalog(sessionID); + return ER_INTERNAL_ERROR; + } + catch (...) + { + string emsg = IDBErrorInfo::instance()->errorMsg(ERR_LOST_CONN_EXEMGR); + setError(gwi.thd, ER_INTERNAL_ERROR, emsg, gwi); + CalpontSystemCatalog::removeCalpontSystemCatalog(sessionID); + return ER_INTERNAL_ERROR; + } + + if (!gwi.count_asterisk_list.empty() || !gwi.no_parm_func_list.empty() || + gwi.returnedCols.empty()) + { + // get the smallest column from colmap + CalpontSelectExecutionPlan::ColumnMap::const_iterator iter; + int minColWidth = 0; + CalpontSystemCatalog::ColType ct; try { - for (; tb_iter != gwi.tableMap.end(); tb_iter++) + for (iter = gwi.columnMap.begin(); iter != gwi.columnMap.end(); ++iter) { - if ((*tb_iter).second.first == 1) continue; + // should always not null + SimpleColumn* sc = dynamic_cast(iter->second.get()); - CalpontSystemCatalog::TableAliasName tan = (*tb_iter).first; - CalpontSystemCatalog::TableName tn = make_table((*tb_iter).first.schema, (*tb_iter).first.table); - SimpleColumn* sc = getSmallestColumn(csc, tn, tan, (*tb_iter).second.second->table, gwi); - SRCP srcp(sc); - gwi.columnMap.insert(CalpontSelectExecutionPlan::ColumnMap::value_type(sc->columnName(), srcp)); - (*tb_iter).second.first = 1; + if (sc && !(sc->joinInfo() & JOIN_CORRELATED)) + { + ct = csc->colType(sc->oid()); + + if (minColWidth == 0) + { + minColWidth = ct.colWidth; + minSc = iter->second; + } + else if (ct.colWidth < minColWidth) + { + minColWidth = ct.colWidth; + minSc = iter->second; + } + } } } - catch (runtime_error& e) - { - setError(gwi.thd, ER_INTERNAL_ERROR, e.what(), gwi); - CalpontSystemCatalog::removeCalpontSystemCatalog(sessionID); - return ER_INTERNAL_ERROR; - } catch (...) { string emsg = IDBErrorInfo::instance()->errorMsg(ERR_LOST_CONN_EXEMGR); @@ -9508,412 +7354,2559 @@ int getSelectPlan(gp_walk_info& gwi, SELECT_LEX& select_lex, SCSEP& csep, bool i return ER_INTERNAL_ERROR; } - if (!gwi.count_asterisk_list.empty() || !gwi.no_parm_func_list.empty() || - gwi.returnedCols.empty()) + if (gwi.returnedCols.empty() && gwi.additionalRetCols.empty()) + gwi.returnedCols.push_back(minSc); + } + + if (!isUnion && !gwi.hasWindowFunc && gwi.subSelectType == CalpontSelectExecutionPlan::MAIN_SELECT) + { + std::ostringstream vtb; + vtb << "infinidb_vtable.$vtable_" << gwi.thd->thread_id; + + //vtb << "$vtable_" << gwi.thd->thread_id; + // re-construct the select query and redo phase 1 + if (redo) { - // get the smallest column from colmap - CalpontSelectExecutionPlan::ColumnMap::const_iterator iter; - int minColWidth = 0; - CalpontSystemCatalog::ColType ct; - - try + // select now() from region case. returnedCols should have minSc. + if (sel_cols_in_create.length() == 0) { - for (iter = gwi.columnMap.begin(); iter != gwi.columnMap.end(); ++iter) + SimpleColumn* sc = dynamic_cast(gwi.returnedCols[0].get()); + + if (sc) + sel_cols_in_create = dynamic_cast(gwi.returnedCols[0].get())->columnName(); + else + sel_cols_in_create = gwi.returnedCols[0]->alias(); + } + + // select * from derived table case + if (gwi.selectCols.empty()) + sel_cols_in_create = " * "; + + create_query = "create temporary table " + vtb.str() + " engine = aria as select " + sel_cols_in_create + " from "; + TABLE_LIST* table_ptr = select_lex.get_table_list(); + + bool firstTb = true; + + // put all tables, derived tables and views on the list + //TABLE_LIST* table_ptr = select_lex.get_table_list(); + set aliasSet; // to avoid duplicate table alias + + for (; table_ptr; table_ptr = table_ptr->next_global) + { + if (string(table_ptr->table_name).find("$vtable") != string::npos) + continue; + + if (table_ptr->derived) { - // should always not null - SimpleColumn* sc = dynamic_cast(iter->second.get()); + if (aliasSet.find(table_ptr->alias) != aliasSet.end()) + continue; - if (sc && !(sc->joinInfo() & JOIN_CORRELATED)) + String str; + (table_ptr->derived->first_select())->print(gwi.thd, &str, QT_INFINIDB_DERIVED); + + if (!firstTb) + create_query += ", "; + + create_query += "(" + string(str.c_ptr()) + ") " + string(table_ptr->alias); + firstTb = false; + aliasSet.insert(table_ptr->alias); + } + else if (table_ptr->view) + { + if (aliasSet.find(table_ptr->alias) != aliasSet.end()) + continue; + + if (!firstTb) + create_query += ", "; + + create_query += string(table_ptr->db) + "." + string(table_ptr->table_name) + + string(" `") + escapeBackTick(table_ptr->alias) + string("`"); + aliasSet.insert(table_ptr->alias); + firstTb = false; + } + else + { + // table referenced by view is represented by viewAlias_tableAlias. + // consistent with item.cc field print. + if (table_ptr->referencing_view) { - ct = csc->colType(sc->oid()); + if (aliasSet.find(string(table_ptr->referencing_view->alias) + "_" + + string(table_ptr->alias)) != aliasSet.end()) + continue; - if (minColWidth == 0) + if (!firstTb) + create_query += ", "; + + create_query += string(table_ptr->db) + "." + string(table_ptr->table_name) + string(" "); + create_query += string(" `") + + escapeBackTick(table_ptr->referencing_view->alias) + "_" + + escapeBackTick(table_ptr->alias) + string("`"); + aliasSet.insert(string(table_ptr->referencing_view->alias) + "_" + + string(table_ptr->alias)); + } + else + { + if (aliasSet.find(table_ptr->alias) != aliasSet.end()) + continue; + + if (!firstTb) + create_query += ", "; + + create_query += string(table_ptr->db) + "." + string(table_ptr->table_name) + string(" "); + create_query += string("`") + escapeBackTick(table_ptr->alias) + string("`"); + aliasSet.insert(table_ptr->alias); + } + + firstTb = false; + } + } + + + gwi.thd->infinidb_vtable.create_vtable_query.free(); + gwi.thd->infinidb_vtable.create_vtable_query.append(create_query.c_str(), create_query.length()); + gwi.thd->infinidb_vtable.vtable_state = THD::INFINIDB_REDO_PHASE1; // redo phase 1 + + // turn off select distinct from post process unless there're post process functions + // on the select list. + string sel_query = "select "; + + if (/*join->select_options*/select_lex.options & SELECT_DISTINCT && redo) + sel_query = "select distinct "; + else + sel_query = "select "; + + // select * from derived table... + if (gwi.selectCols.size() == 0) + sel_query += " * "; + + for (uint32_t i = 0; i < gwi.selectCols.size(); i++) + { + sel_query += gwi.selectCols[i]; + + if ( i + 1 != gwi.selectCols.size()) + sel_query += ", "; + } + + select_query.replace(lower_select_query.find("select *"), string("select *").length(), sel_query); + } + else + { + // remove order by clause in case this phase has been executed before. + // need a better fix later, like skip all the other non-optimized phase. + size_t pos = lower_select_query.find("order by"); + + if (pos != string::npos) + select_query.replace(pos, lower_select_query.length() - pos, ""); + + //select_query = "select * from " + vtb.str(); + if (unionSel) + order_list = select_lex.master_unit()->global_parameters()->order_list; + + ordercol = reinterpret_cast(order_list.first); + ord_cols = ""; + + for (; ordercol; ordercol = ordercol->next) + { + Item* ord_item = *(ordercol->item); + + // @bug 1706. re-construct the order by item one by one, because the ord_cols constucted so far + // is for REDO phase. + if (ord_cols.length() != 0) + ord_cols += ", "; + + if (ordercol->in_field_list && ordercol->counter_used) + { + ostringstream oss; + oss << ordercol->counter; + ord_cols += oss.str(); + } + else if (ord_item->type() == Item::NULL_ITEM) + { + // MCOL-793 Do nothing for an ORDER BY NULL + } + else if (ord_item->type() == Item::SUM_FUNC_ITEM) + { + Item_sum* ifp = (Item_sum*)(*(ordercol->item)); + ReturnedColumn* fc = buildAggregateColumn(ifp, gwi); + + for (uint32_t i = 0; i < gwi.returnedCols.size(); i++) + { + if (fc->operator==(gwi.returnedCols[i].get())) { - minColWidth = ct.colWidth; - minSc = iter->second; + ostringstream oss; + oss << i + 1; + ord_cols += oss.str(); + break; } - else if (ct.colWidth < minColWidth) + } + + //continue; + } + // @bug 3518. if order by clause = selected column, use position. + else if (ord_item->name && ord_item->type() == Item::FIELD_ITEM) + { + Item_field* field = reinterpret_cast(ord_item); + string fullname; + + if (field->db_name) + fullname += string(field->db_name) + "."; + + if (field->table_name) + fullname += string(field->table_name) + "."; + + if (field->field_name) + fullname += string(field->field_name); + + uint32_t i = 0; + + for (i = 0; i < gwi.returnedCols.size(); i++) + { + SimpleColumn* sc = dynamic_cast(gwi.returnedCols[i].get()); + + if (sc && ((Item_field*)ord_item)->cached_table && + (strcasecmp(getViewName(((Item_field*)ord_item)->cached_table).c_str(), sc->viewName().c_str()) != 0)) + continue; + + if (strcasecmp(fullname.c_str(), gwi.returnedCols[i]->alias().c_str()) == 0 || + strcasecmp(ord_item->name, gwi.returnedCols[i]->alias().c_str()) == 0) { - minColWidth = ct.colWidth; - minSc = iter->second; + ostringstream oss; + oss << i + 1; + ord_cols += oss.str(); + break; + } + } + + if (i == gwi.returnedCols.size()) + ord_cols += string(" `") + escapeBackTick(ord_item->name) + '`'; + } + + else if (ord_item->name) + { + // for union order by 1 case. For unknown reason, it doesn't show in_field_list + if (ord_item->type() == Item::INT_ITEM) + { + ord_cols += ord_item->name; + } + else if (ord_item->type() == Item::SUBSELECT_ITEM) + { + string emsg = IDBErrorInfo::instance()->errorMsg(ERR_NON_SUPPORT_ORDER_BY); + setError(gwi.thd, ER_CHECK_NOT_IMPLEMENTED, emsg, gwi); + return ER_CHECK_NOT_IMPLEMENTED; + } + else + { + ord_cols += string(" `") + escapeBackTick(ord_item->name) + '`'; + } + } + else if (ord_item->type() == Item::FUNC_ITEM) + { + // @bug5636. check if this order by column is on the select list + ReturnedColumn* rc = buildFunctionColumn((Item_func*)(ord_item), gwi, gwi.fatalParseError); + + for (uint32_t i = 0; i < gwi.returnedCols.size(); i++) + { + if (rc && rc->operator==(gwi.returnedCols[i].get())) + { + ostringstream oss; + oss << i + 1; + ord_cols += oss.str(); + break; } } } + else + { + String str; + ord_item->print(&str, QT_INFINIDB_NO_QUOTE); + ord_cols += string(str.c_ptr()); + } + + if (ordercol->direction != ORDER::ORDER_ASC) + ord_cols += " desc"; } - catch (...) + } + + if (ord_cols.length() > 0) // has order by + { + gwi.thd->infinidb_vtable.has_order_by = true; + csep->hasOrderBy(true); + ord_cols = " order by " + ord_cols; + select_query += ord_cols; + } + } + + if (unionSel || gwi.subSelectType != CalpontSelectExecutionPlan::MAIN_SELECT) + { + if (select_lex.master_unit()->global_parameters()->explicit_limit) + { + if (select_lex.master_unit()->global_parameters()->offset_limit) { - string emsg = IDBErrorInfo::instance()->errorMsg(ERR_LOST_CONN_EXEMGR); - setError(gwi.thd, ER_INTERNAL_ERROR, emsg, gwi); + Item_int* offset = (Item_int*)select_lex.master_unit()->global_parameters()->offset_limit; + csep->limitStart(offset->val_int()); + } + + if (select_lex.master_unit()->global_parameters()->select_limit) + { + Item_int* select = (Item_int*)select_lex.master_unit()->global_parameters()->select_limit; + csep->limitNum(select->val_int()); + } + + if (unionSel && gwi.subSelectType == CalpontSelectExecutionPlan::MAIN_SELECT) + { + ostringstream limit; + limit << " limit "; + limit << csep->limitStart() << ", "; + limit << csep->limitNum(); + select_query += limit.str(); + } + } + } + else if (isUnion && select_lex.explicit_limit) + { + if (select_lex.braces) + { + if (select_lex.offset_limit) + csep->limitStart(((Item_int*)select_lex.offset_limit)->val_int()); + + if (select_lex.select_limit) + csep->limitNum(((Item_int*)select_lex.select_limit)->val_int()); + } + } + else if (select_lex.explicit_limit) + { + uint32_t limitOffset = 0; + uint32_t limitNum = std::numeric_limits::max(); + + if (join) + { +#if MYSQL_VERSION_ID >= 50172 + + // @bug5729. After upgrade, join->unit sometimes is uninitialized pointer + // (not null though) and will cause seg fault. Prefer checking + // select_lex->offset_limit if not null. + if (join->select_lex && + join->select_lex->offset_limit && + join->select_lex->offset_limit->fixed && + join->select_lex->select_limit && + join->select_lex->select_limit->fixed) + { + limitOffset = join->select_lex->offset_limit->val_int(); + limitNum = join->select_lex->select_limit->val_int(); + } + + else if (join->unit) + { + limitOffset = join->unit->offset_limit_cnt; + limitNum = join->unit->select_limit_cnt - limitOffset; + } + +#else + limitOffset = (join->unit)->offset_limit_cnt; + limitNum = (join->unit)->select_limit_cnt - (join->unit)->offset_limit_cnt; +#endif + } + else + { + if (select_lex.master_unit()->global_parameters()->offset_limit) + { + Item_int* offset = (Item_int*)select_lex.master_unit()->global_parameters()->offset_limit; + limitOffset = offset->val_int(); + } + + if (select_lex.master_unit()->global_parameters()->select_limit) + { + Item_int* select = (Item_int*)select_lex.master_unit()->global_parameters()->select_limit; + limitNum = select->val_int(); + } + } + + // relate to bug4848. let mysql drive limit when limit session variable set. + // do not set in csep. @bug5096. ignore session limit setting for dml + if ((gwi.thd->variables.select_limit == (uint64_t) -1 || + (gwi.thd->variables.select_limit != (uint64_t) -1 && + gwi.thd->infinidb_vtable.vtable_state != THD::INFINIDB_CREATE_VTABLE)) && + !csep->hasOrderBy()) + { + csep->limitStart(limitOffset); + csep->limitNum(limitNum); + } + else + { + ostringstream limit; + limit << " limit " << limitOffset << ", " << limitNum; + select_query += limit.str(); + } + } + + gwi.thd->infinidb_vtable.select_vtable_query.free(); + gwi.thd->infinidb_vtable.select_vtable_query.append(select_query.c_str(), select_query.length()); + + // We don't currently support limit with correlated subquery + if (csep->limitNum() != (uint64_t) -1 && + gwi.subQuery && !gwi.correlatedTbNameVec.empty()) + { + gwi.fatalParseError = true; + gwi.parseErrorText = IDBErrorInfo::instance()->errorMsg(ERR_NON_SUPPORT_LIMIT_SUB); + setError(gwi.thd, ER_INTERNAL_ERROR, gwi.parseErrorText, gwi); + return ER_CHECK_NOT_IMPLEMENTED; + } + } + + if (/*join->select_options*/select_lex.options & SELECT_DISTINCT) + csep->distinct(true); + + // add the smallest column to count(*) parm. + // select constant in subquery case + std::vector::iterator coliter; + + if (!minSc) + { + if (!gwi.returnedCols.empty()) + minSc = gwi.returnedCols[0]; + else if (!gwi.additionalRetCols.empty()) + minSc = gwi.additionalRetCols[0]; + } + + // @bug3523, count(*) on subquery always pick column[0]. + SimpleColumn* sc = dynamic_cast(minSc.get()); + + if (sc && sc->schemaName().empty()) + { + if (gwi.derivedTbList.size() >= 1) + { + SimpleColumn* sc1 = new SimpleColumn(); + sc1->columnName(sc->columnName()); + sc1->tableName(sc->tableName()); + sc1->tableAlias(sc->tableAlias()); + sc1->viewName(lower(sc->viewName())); + sc1->colPosition(0); + minSc.reset(sc1); + } + } + + for (coliter = gwi.count_asterisk_list.begin(); coliter != gwi.count_asterisk_list.end(); ++coliter) + { + // @bug5977 @note should never throw this, but checking just in case. + // When ExeMgr fix is ready, this should not error out... + if (dynamic_cast(minSc.get())) + { + gwi.fatalParseError = true; + gwi.parseErrorText = "No project column found for aggregate function"; + setError(gwi.thd, ER_INTERNAL_ERROR, gwi.parseErrorText, gwi); + return ER_CHECK_NOT_IMPLEMENTED; + } + + (*coliter)->functionParms(minSc); + } + + std::vector::iterator funciter; + + SPTP sptp(new ParseTree(minSc.get()->clone())); + + for (funciter = gwi.no_parm_func_list.begin(); funciter != gwi.no_parm_func_list.end(); ++funciter) + { + FunctionParm funcParms = (*funciter)->functionParms(); + funcParms.push_back(sptp); + (*funciter)->functionParms(funcParms); + } + + // set sequence# for subquery localCols + for (uint32_t i = 0; i < gwi.localCols.size(); i++) + gwi.localCols[i]->sequence(i); + + // append additionalRetCols to returnedCols + gwi.returnedCols.insert(gwi.returnedCols.begin(), gwi.additionalRetCols.begin(), + gwi.additionalRetCols.end()); + + csep->groupByCols(gwi.groupByCols); + csep->orderByCols(gwi.orderByCols); + csep->returnedCols(gwi.returnedCols); + csep->columnMap(gwi.columnMap); + csep->having(havingFilter); + csep->derivedTableList(gwi.derivedTbList); + csep->selectSubList(selectSubList); + csep->subSelectList(gwi.subselectList); + gwi.thd->infinidb_vtable.duplicate_field_name = false; + clearStacks(gwi); + return 0; +} + +int cp_get_plan(THD* thd, SCSEP& csep) +{ + LEX* lex = thd->lex; + idbassert(lex != 0); + + SELECT_LEX select_lex = lex->select_lex; + gp_walk_info gwi; + gwi.thd = thd; + int status = getSelectPlan(gwi, select_lex, csep); + + if (status > 0) + return ER_INTERNAL_ERROR; + else if (status < 0) + return status; + + // Derived table projection and filter optimization. + derivedTableOptimization(csep); + + return 0; +} + +int cp_get_table_plan(THD* thd, SCSEP& csep, cal_table_info& ti) +{ + gp_walk_info* gwi = ti.condInfo; + + if (!gwi) + gwi = new gp_walk_info(); + + gwi->thd = thd; + LEX* lex = thd->lex; + idbassert(lex != 0); + uint32_t sessionID = csep->sessionID(); + gwi->sessionid = sessionID; + TABLE* table = ti.msTablePtr; + boost::shared_ptr csc = CalpontSystemCatalog::makeCalpontSystemCatalog(sessionID); + csc->identity(CalpontSystemCatalog::FE); + + // get all columns that mysql needs to fetch + MY_BITMAP* read_set = table->read_set; + Field** f_ptr, *field; + gwi->columnMap.clear(); + + for (f_ptr = table->field ; (field = *f_ptr) ; f_ptr++) + { + if (bitmap_is_set(read_set, field->field_index)) + { + SimpleColumn* sc = new SimpleColumn(table->s->db.str, table->s->table_name.str, field->field_name, sessionID); + string alias(table->alias.c_ptr()); + sc->tableAlias(lower(alias)); + assert (sc); + boost::shared_ptr spsc(sc); + gwi->returnedCols.push_back(spsc); + gwi->columnMap.insert(CalpontSelectExecutionPlan::ColumnMap::value_type(string(field->field_name), spsc)); + } + } + + if (gwi->columnMap.empty()) + { + CalpontSystemCatalog::TableName tn = make_table(table->s->db.str, table->s->table_name.str); + CalpontSystemCatalog::TableAliasName tan = make_aliastable(table->s->db.str, table->s->table_name.str, table->alias.c_ptr()); + SimpleColumn* sc = getSmallestColumn(csc, tn, tan, table, *gwi); + SRCP srcp(sc); + gwi->columnMap.insert(CalpontSelectExecutionPlan::ColumnMap::value_type(sc->columnName(), srcp)); + gwi->returnedCols.push_back(srcp); + } + + // get filter + if (ti.condInfo) + { + gp_walk_info* gwi = ti.condInfo; + ParseTree* filters = 0; + ParseTree* ptp = 0; + ParseTree* rhs = 0; + + while (!gwi->ptWorkStack.empty()) + { + filters = gwi->ptWorkStack.top(); + gwi->ptWorkStack.pop(); + SimpleFilter* sf = dynamic_cast(filters->data()); + + if (sf && sf->op()->data() == "noop") + { + delete filters; + filters = 0; + + if (gwi->ptWorkStack.empty()) + break; + + continue; + } + + if (gwi->ptWorkStack.empty()) + break; + + ptp = new ParseTree(new LogicOperator("and")); + ptp->left(filters); + rhs = gwi->ptWorkStack.top(); + gwi->ptWorkStack.pop(); + ptp->right(rhs); + gwi->ptWorkStack.push(ptp); + } + + csep->filters(filters); + } + + csep->returnedCols(gwi->returnedCols); + csep->columnMap(gwi->columnMap); + CalpontSelectExecutionPlan::TableList tblist; + tblist.push_back(make_aliastable(table->s->db.str, table->s->table_name.str, table->alias.c_ptr())); + csep->tableList(tblist); + + // @bug 3321. Set max number of blocks in a dictionary file to be scanned for filtering + csep->stringScanThreshold(gwi->thd->variables.infinidb_string_scan_threshold); + + return 0; +} + +int cp_get_group_plan(THD* thd, SCSEP& csep, cal_impl_if::cal_group_info& gi) +{ + LEX* lex = thd->lex; + idbassert(lex != 0); + + SELECT_LEX select_lex = lex->select_lex; + gp_walk_info gwi; + gwi.thd = thd; + int status = getGroupPlan(gwi, select_lex, csep, gi); + + if (status > 0) + return ER_INTERNAL_ERROR; + else if (status < 0) + return status; + + return 0; +} + +/*@brief buildConstColFromFilter- change SimpleColumn into ConstColumn*/ +/*********************************************************** + * DESCRIPTION: + * Server could optimize out fields from GROUP BY list, when certain + * filter predicate is used, e.g. + * field = 'AIR', field IN ('AIR'). This utility function tries to + * replace such fields with ConstantColumns using cond_pushed filters. + * PARAMETERS: + * originalSC SimpleColumn* removed field + * gwi main strucutre + * gi auxilary group_by handler structure + * RETURNS + * ConstantColumn* if originalSC equals with cond_pushed columns. + * NULL otherwise + ***********************************************************/ +ConstantColumn* buildConstColFromFilter(SimpleColumn* originalSC, + gp_walk_info& gwi, cal_group_info& gi) +{ + execplan::SimpleColumn* simpleCol; + execplan::ConstantColumn* constCol; + execplan::SOP op; + execplan::SimpleFilter* simpFilter; + execplan::ConstantColumn* result = NULL; + std::vector::iterator ptIt = gi.pushedPts.begin(); + + for (; ptIt != gi.pushedPts.end(); ptIt++) + { + simpFilter = dynamic_cast((*ptIt)->data()); + + if (simpFilter == NULL) + continue; + + simpleCol = dynamic_cast(simpFilter->lhs()); + constCol = dynamic_cast(simpFilter->rhs()); + + if (simpleCol == NULL || constCol == NULL) + continue; + + op = simpFilter->op(); + + if ( originalSC->sameColumn(dynamic_cast(simpleCol)) + && op.get()->op() == OP_EQ && constCol) + { +#ifdef DEBUG_WALK_COND + cerr << "buildConstColFromFilter() replaced " << endl; + cerr << simpleCol->toString() << endl; + cerr << " with " << endl; + cerr << constCol << endl; +#endif + result = constCol; + } + } + + return result; +} + +int getGroupPlan(gp_walk_info& gwi, SELECT_LEX& select_lex, SCSEP& csep, cal_group_info& gi, bool isUnion) +{ +#ifdef DEBUG_WALK_COND + cerr << "getGroupPlan()" << endl; +#endif + + // rollup is currently not supported + if (select_lex.olap == ROLLUP_TYPE) + { + gwi.fatalParseError = true; + gwi.parseErrorText = IDBErrorInfo::instance()->errorMsg(ERR_ROLLUP_NOT_SUPPORT); + setError(gwi.thd, ER_CHECK_NOT_IMPLEMENTED, gwi.parseErrorText, gwi); + return ER_CHECK_NOT_IMPLEMENTED; + } + + gwi.internalDecimalScale = (gwi.thd->variables.infinidb_use_decimal_scale ? gwi.thd->variables.infinidb_decimal_scale : -1); + gwi.subSelectType = csep->subType(); + + JOIN* join = select_lex.join; + Item_cond* icp = 0; + + if ( gi.groupByWhere ) + icp = reinterpret_cast(gi.groupByWhere); + + uint32_t sessionID = csep->sessionID(); + gwi.sessionid = sessionID; + boost::shared_ptr csc = CalpontSystemCatalog::makeCalpontSystemCatalog(sessionID); + csc->identity(CalpontSystemCatalog::FE); + gwi.csc = csc; + + // @bug 2123. Override large table estimate if infinidb_ordered hint was used. + // @bug 2404. Always override if the infinidb_ordered_only variable is turned on. + if (gwi.thd->infinidb_vtable.override_largeside_estimate || gwi.thd->variables.infinidb_ordered_only) + csep->overrideLargeSideEstimate(true); + + // @bug 5741. Set a flag when in Local PM only query mode + csep->localQuery(gwi.thd->variables.infinidb_local_query); + + // @bug 3321. Set max number of blocks in a dictionary file to be scanned for filtering + csep->stringScanThreshold(gwi.thd->variables.infinidb_string_scan_threshold); + + csep->stringTableThreshold(gwi.thd->variables.infinidb_stringtable_threshold); + + csep->djsSmallSideLimit(gwi.thd->variables.infinidb_diskjoin_smallsidelimit * 1024ULL * 1024); + csep->djsLargeSideLimit(gwi.thd->variables.infinidb_diskjoin_largesidelimit * 1024ULL * 1024); + csep->djsPartitionSize(gwi.thd->variables.infinidb_diskjoin_bucketsize * 1024ULL * 1024); + + if (gwi.thd->variables.infinidb_um_mem_limit == 0) + csep->umMemLimit(numeric_limits::max()); + else + csep->umMemLimit(gwi.thd->variables.infinidb_um_mem_limit * 1024ULL * 1024); + + // populate table map and trigger syscolumn cache for all the tables (@bug 1637). + // all tables on FROM list must have at least one col in colmap + TABLE_LIST* table_ptr = gi.groupByTables; + CalpontSelectExecutionPlan::SelectList derivedTbList; + +// DEBUG +#ifdef DEBUG_WALK_COND + List_iterator sj_list_it(select_lex.sj_nests); + TABLE_LIST* sj_nest; + + while ((sj_nest = sj_list_it++)) + { + cerr << sj_nest->db << "." << sj_nest->table_name << endl; + } + +#endif + + // @bug 1796. Remember table order on the FROM list. + gwi.clauseType = FROM; + + try + { + for (; table_ptr; table_ptr = table_ptr->next_local) + { + // mysql put vtable here for from sub. we ignore it + //if (string(table_ptr->table_name).find("$vtable") != string::npos) + // continue; + + // Until we handle recursive cte: + // Checking here ensures we catch all with clauses in the query. + if (table_ptr->is_recursive_with_table()) + { + gwi.fatalParseError = true; + gwi.parseErrorText = "Recursive CTE"; + setError(gwi.thd, ER_CHECK_NOT_IMPLEMENTED, gwi.parseErrorText, gwi); + return ER_CHECK_NOT_IMPLEMENTED; + } + + string viewName = getViewName(table_ptr); + + // @todo process from subquery + if (table_ptr->derived) + { + String str; + (table_ptr->derived->first_select())->print(gwi.thd, &str, QT_INFINIDB_DERIVED); + + SELECT_LEX* select_cursor = table_ptr->derived->first_select(); + FromSubQuery fromSub(gwi, select_cursor); + string alias(table_ptr->alias); + fromSub.alias(lower(alias)); + + CalpontSystemCatalog::TableAliasName tn = make_aliasview("", "", alias, viewName); + // @bug 3852. check return execplan + SCSEP plan = fromSub.transform(); + + if (!plan) + { + setError(gwi.thd, ER_INTERNAL_ERROR, fromSub.gwip().parseErrorText, gwi); CalpontSystemCatalog::removeCalpontSystemCatalog(sessionID); return ER_INTERNAL_ERROR; } - if (gwi.returnedCols.empty() && gwi.additionalRetCols.empty()) - gwi.returnedCols.push_back(minSc); + gwi.derivedTbList.push_back(plan); + gwi.tbList.push_back(tn); + CalpontSystemCatalog::TableAliasName tan = make_aliastable("", alias, alias); + gwi.tableMap[tan] = make_pair(0, table_ptr); +// gwi.thd->infinidb_vtable.isUnion = true; //by-pass the 2nd pass of rnd_init + } + else if (table_ptr->view) + { + View* view = new View(table_ptr->view->select_lex, &gwi); + CalpontSystemCatalog::TableAliasName tn = make_aliastable(table_ptr->db, table_ptr->table_name, table_ptr->alias); + view->viewName(tn); + gwi.viewList.push_back(view); + view->transform(); + } + else + { + // check foreign engine tables + bool infiniDB = (table_ptr->table ? isInfiniDB(table_ptr->table) : true); + + // trigger system catalog cache + if (infiniDB) + csc->columnRIDs(make_table(table_ptr->db, table_ptr->table_name), true); + + string table_name = table_ptr->table_name; + + // @bug5523 + if (table_ptr->db && strcmp(table_ptr->db, "information_schema") == 0) + table_name = (table_ptr->schema_table_name ? table_ptr->schema_table_name : table_ptr->alias); + + CalpontSystemCatalog::TableAliasName tn = make_aliasview(table_ptr->db, table_name, table_ptr->alias, viewName, infiniDB); + gwi.tbList.push_back(tn); + CalpontSystemCatalog::TableAliasName tan = make_aliastable(table_ptr->db, table_name, table_ptr->alias, infiniDB); + gwi.tableMap[tan] = make_pair(0, table_ptr); +#ifdef DEBUG_WALK_COND + cerr << tn << endl; +#endif + } + } + + if (gwi.fatalParseError) + { + setError(gwi.thd, ER_INTERNAL_ERROR, gwi.parseErrorText, gwi); + return ER_INTERNAL_ERROR; + } + } + catch (IDBExcept& ie) + { + setError(gwi.thd, ER_INTERNAL_ERROR, ie.what(), gwi); + CalpontSystemCatalog::removeCalpontSystemCatalog(sessionID); + // @bug 3852. set error status for gwi. + gwi.fatalParseError = true; + gwi.parseErrorText = ie.what(); + return ER_INTERNAL_ERROR; + } + catch (...) + { + string emsg = IDBErrorInfo::instance()->errorMsg(ERR_LOST_CONN_EXEMGR); + // @bug3852 set error status for gwi. + gwi.fatalParseError = true; + gwi.parseErrorText = emsg; + setError(gwi.thd, ER_INTERNAL_ERROR, emsg, gwi); + CalpontSystemCatalog::removeCalpontSystemCatalog(sessionID); + return ER_INTERNAL_ERROR; + } + + csep->tableList(gwi.tbList); + + bool unionSel = false; + + gwi.clauseType = WHERE; + + + if (icp) + { + // MCOL-1052 The condition could be useless. + // MariaDB bug 624 - without the fix_fields call, delete with join may error with "No query step". + //#if MYSQL_VERSION_ID < 50172 + //@bug 3039. fix fields for constants + if (!icp->fixed) + { + icp->fix_fields(gwi.thd, (Item**)&icp); + } + +//#endif + gwi.fatalParseError = false; +#ifdef DEBUG_WALK_COND + cerr << "------------------ WHERE -----------------------" << endl; + icp->traverse_cond(debug_walk, &gwi, Item::POSTFIX); + cerr << "------------------------------------------------\n" << endl; +#endif + + icp->traverse_cond(gp_walk, &gwi, Item::POSTFIX); + + if (gwi.fatalParseError) + { + // if this is dervied table process phase, mysql may have not developed the plan + // completely. Do not error and eventually mysql will call JOIN::exec() again. + // related to bug 2922. Need to find a way to skip calling rnd_init for derived table + // processing. + if (gwi.thd->derived_tables_processing) + { + gwi.thd->infinidb_vtable.isUnion = false; + gwi.thd->infinidb_vtable.isUpdateWithDerive = true; + return -1; } - if (!isUnion && !gwi.hasWindowFunc && gwi.subSelectType == CalpontSelectExecutionPlan::MAIN_SELECT) + setError(gwi.thd, ER_INTERNAL_ERROR, gwi.parseErrorText, gwi); + return ER_INTERNAL_ERROR; + } + } + else if (join && join->zero_result_cause) + { + gwi.rcWorkStack.push(new ConstantColumn((int64_t)0, ConstantColumn::NUM)); + } + + uint32_t failed = buildOuterJoin(gwi, select_lex); + + if (failed) return failed; + + // @bug5764. build outer join for view, make sure outerjoin filter is appended + // to the end of the filter list. + for (uint i = 0; i < gwi.viewList.size(); i++) + { + failed = gwi.viewList[i]->processOuterJoin(gwi); + + if (failed) + break; + } + + if (failed != 0) + return failed; + + ParseTree* filters = NULL; + ParseTree* ptp = NULL; + ParseTree* lhs = NULL; + + // @bug 2932. for "select * from region where r_name" case. if icp not null and + // ptWorkStack empty, the item is in rcWorkStack. + // MySQL 5.6 (MariaDB?). when icp is null and zero_result_cause is set, a constant 0 + // is pushed to rcWorkStack. + if (/*icp && */gwi.ptWorkStack.empty() && !gwi.rcWorkStack.empty()) + { + filters = new ParseTree(gwi.rcWorkStack.top()); + gwi.rcWorkStack.pop(); + } + + while (!gwi.ptWorkStack.empty()) + { + filters = gwi.ptWorkStack.top(); + gwi.ptWorkStack.pop(); + + if (gwi.ptWorkStack.empty()) + break; + + ptp = new ParseTree(new LogicOperator("and")); + //ptp->left(filters); + ptp->right(filters); + lhs = gwi.ptWorkStack.top(); + gwi.ptWorkStack.pop(); + //ptp->right(rhs); + ptp->left(lhs); + gwi.ptWorkStack.push(ptp); + } + + if (filters) + { + csep->filters(filters); +#ifdef DEBUG_WALK_COND + filters->drawTree("/tmp/filter1.dot"); +#endif + } + + gwi.clauseType = SELECT; +#ifdef DEBUG_WALK_COND + { + cerr << "------------------- SELECT --------------------" << endl; + List_iterator_fast it(*gi.groupByFields); + Item* item; + + while ((item = it++)) + { + debug_walk(item, 0); + } + + cerr << "-----------------------------------------------\n" << endl; + } +#endif + + // populate returnedcolumnlist and columnmap + List_iterator_fast it(*gi.groupByFields); + Item* item; + vector funcFieldVec; + string sel_cols_in_create; + string sel_cols_in_select; + bool redo = false; + + // empty rcWorkStack and ptWorkStack. They should all be empty by now. + clearStacks(gwi); + + // indicate the starting pos of scalar returned column, because some join column + // has been inserted to the returned column list. + if (gwi.subQuery) + { + ScalarSub* scalar = dynamic_cast(gwi.subQuery); + + if (scalar) + scalar->returnedColPos(gwi.additionalRetCols.size()); + } + + CalpontSelectExecutionPlan::SelectList selectSubList; + + while ((item = it++)) + { + string itemAlias = (item->name ? item->name : ""); + + // @bug 5916. Need to keep checking until getting concret item in case + // of nested view. + while (item->type() == Item::REF_ITEM) + { + Item_ref* ref = (Item_ref*)item; + item = (*(ref->ref)); + } + + Item::Type itype = item->type(); + + switch (itype) + { + case Item::FIELD_ITEM: { - std::ostringstream vtb; - vtb << "infinidb_vtable.$vtable_" << gwi.thd->thread_id; + Item_field* ifp = (Item_field*)item; + SimpleColumn* sc = NULL; + ConstantColumn* constCol = NULL; - //vtb << "$vtable_" << gwi.thd->thread_id; - // re-construct the select query and redo phase 1 - if (redo) + if (ifp->field_name && string(ifp->field_name) == "*") { - // select now() from region case. returnedCols should have minSc. - if (sel_cols_in_create.length() == 0) - { - SimpleColumn* sc = dynamic_cast(gwi.returnedCols[0].get()); + collectAllCols(gwi, ifp); + break; + } - if (sc) - sel_cols_in_create = dynamic_cast(gwi.returnedCols[0].get())->columnName(); - else - sel_cols_in_create = gwi.returnedCols[0]->alias(); + sc = buildSimpleColumn(ifp, gwi); + + if (sc) + { + constCol = buildConstColFromFilter(sc, gwi, gi); + boost::shared_ptr spcc(constCol); + boost::shared_ptr spsc(sc); + + if (sel_cols_in_create.length() != 0) + sel_cols_in_create += ", "; + + string fullname; + String str; + ifp->print(&str, QT_INFINIDB_NO_QUOTE); + fullname = str.c_ptr(); + + //sel_cols_in_create += fullname; + if (ifp->is_autogenerated_name) // no alias + { + sel_cols_in_create += fullname + " `" + escapeBackTick(str.c_ptr()) + "`"; + sc->alias(fullname); + } + else // alias + { + if (!itemAlias.empty()) + sc->alias(itemAlias); + + sel_cols_in_create += fullname + " `" + escapeBackTick(sc->alias().c_str()) + "`"; } - // select * from derived table case - if (gwi.selectCols.empty()) - sel_cols_in_create = " * "; - - create_query = "create temporary table " + vtb.str() + " engine = aria as select " + sel_cols_in_create + " from "; - TABLE_LIST* table_ptr = gi.groupByTables; - - bool firstTb = true; - - // put all tables, derived tables and views on the list - //TABLE_LIST* table_ptr = select_lex.get_table_list(); - set aliasSet; // to avoid duplicate table alias - - for (; table_ptr; table_ptr = table_ptr->next_local) - { - if (string(table_ptr->table_name).find("$vtable") != string::npos) - continue; - - if (table_ptr->derived) - { - if (aliasSet.find(table_ptr->alias) != aliasSet.end()) - continue; - - String str; - (table_ptr->derived->first_select())->print(gwi.thd, &str, QT_INFINIDB_DERIVED); - - if (!firstTb) - create_query += ", "; - - create_query += "(" + string(str.c_ptr()) + ") " + string(table_ptr->alias); - firstTb = false; - aliasSet.insert(table_ptr->alias); - } - else if (table_ptr->view) - { - if (aliasSet.find(table_ptr->alias) != aliasSet.end()) - continue; - - if (!firstTb) - create_query += ", "; - - create_query += string(table_ptr->db) + "." + string(table_ptr->table_name) + - string(" `") + escapeBackTick(table_ptr->alias) + string("`"); - aliasSet.insert(table_ptr->alias); - firstTb = false; - } - else - { - // table referenced by view is represented by viewAlias_tableAlias. - // consistent with item.cc field print. - if (table_ptr->referencing_view) - { - if (aliasSet.find(string(table_ptr->referencing_view->alias) + "_" + - string(table_ptr->alias)) != aliasSet.end()) - continue; - - if (!firstTb) - create_query += ", "; - - create_query += string(table_ptr->db) + "." + string(table_ptr->table_name) + string(" "); - create_query += string(" `") + - escapeBackTick(table_ptr->referencing_view->alias) + "_" + - escapeBackTick(table_ptr->alias) + string("`"); - aliasSet.insert(string(table_ptr->referencing_view->alias) + "_" + - string(table_ptr->alias)); - } - else - { - if (aliasSet.find(table_ptr->alias) != aliasSet.end()) - continue; - - if (!firstTb) - create_query += ", "; - - create_query += string(table_ptr->db) + "." + string(table_ptr->table_name) + string(" "); - create_query += string("`") + escapeBackTick(table_ptr->alias) + string("`"); - aliasSet.insert(table_ptr->alias); - } - - firstTb = false; - } - } - - - gwi.thd->infinidb_vtable.create_vtable_query.free(); - gwi.thd->infinidb_vtable.create_vtable_query.append(create_query.c_str(), create_query.length()); - gwi.thd->infinidb_vtable.vtable_state = THD::INFINIDB_REDO_PHASE1; // redo phase 1 - - // turn off select distinct from post process unless there're post process functions - // on the select list. - string sel_query = "select "; - - if (gi.groupByDistinct && redo) - sel_query = "select distinct "; + if (ifp->is_autogenerated_name) + gwi.selectCols.push_back("`" + escapeBackTick(fullname.c_str()) + "`" + " `" + + escapeBackTick(itemAlias.empty() ? ifp->name : itemAlias.c_str()) + "`"); else - sel_query = "select "; + gwi.selectCols.push_back("`" + escapeBackTick((itemAlias.empty() ? ifp->name : itemAlias.c_str())) + "`"); - // select * from derived table... - if (gwi.selectCols.size() == 0) - sel_query += " * "; - - for (uint32_t i = 0; i < gwi.selectCols.size(); i++) + // MCOL-1052 Replace SimpleColumn with ConstantColumn, + // since it must have a single value only. + if (constCol) { - sel_query += gwi.selectCols[i]; - - if ( i + 1 != gwi.selectCols.size()) - sel_query += ", "; + gwi.returnedCols.push_back(spcc); + gwi.columnMap.insert(CalpontSelectExecutionPlan::ColumnMap::value_type(string(ifp->field_name), spcc)); + } + else + { + gwi.returnedCols.push_back(spsc); + gwi.columnMap.insert(CalpontSelectExecutionPlan::ColumnMap::value_type(string(ifp->field_name), spsc)); } - select_query.replace(lower_select_query.find("select *"), string("select *").length(), sel_query); + TABLE_LIST* tmp = 0; + + if (ifp->cached_table) + tmp = ifp->cached_table; + + gwi.tableMap[make_aliastable(sc->schemaName(), sc->tableName(), sc->tableAlias(), sc->isInfiniDB())] = + make_pair(1, tmp); } else { - // remove order by clause in case this phase has been executed before. - // need a better fix later, like skip all the other non-optimized phase. - size_t pos = lower_select_query.find("order by"); + setError(gwi.thd, ER_INTERNAL_ERROR, gwi.parseErrorText, gwi); + delete sc; + return ER_INTERNAL_ERROR; + } - if (pos != string::npos) - select_query.replace(pos, lower_select_query.length() - pos, ""); + break; + } - //select_query = "select * from " + vtb.str(); - // MCOL-1052 - if (unionSel) + //aggregate column + case Item::SUM_FUNC_ITEM: + { + ReturnedColumn* ac = buildAggregateColumn(item, gwi); + + if (gwi.fatalParseError) + { + // e.g., non-support ref column + setError(gwi.thd, ER_CHECK_NOT_IMPLEMENTED, gwi.parseErrorText, gwi); + delete ac; + return ER_CHECK_NOT_IMPLEMENTED; + } + + // add this agg col to returnedColumnList + boost::shared_ptr spac(ac); + gwi.returnedCols.push_back(spac); + // This item will be used in HAVING later. + Item_func_or_sum* isfp = reinterpret_cast(item); + + if ( ! isfp->name_length ) + { + gwi.havingAggColsItems.push_back(item); + } + + gwi.selectCols.push_back('`' + escapeBackTick(spac->alias().c_str()) + '`'); + String str(256); + item->print(&str, QT_INFINIDB_NO_QUOTE); + + if (sel_cols_in_create.length() != 0) + sel_cols_in_create += ", "; + + sel_cols_in_create += string(str.c_ptr()) + " `" + escapeBackTick(spac->alias().c_str()) + "`"; + break; + } + + case Item::FUNC_ITEM: + { + Item_func* ifp = reinterpret_cast(item); + + // @bug4383. error out non-support stored function + if (ifp->functype() == Item_func::FUNC_SP) + { + gwi.fatalParseError = true; + gwi.parseErrorText = IDBErrorInfo::instance()->errorMsg(ERR_SP_FUNCTION_NOT_SUPPORT); + setError(gwi.thd, ER_CHECK_NOT_IMPLEMENTED, gwi.parseErrorText, gwi); + return ER_CHECK_NOT_IMPLEMENTED; + } + + if (string(ifp->func_name()) == "xor") + { + gwi.fatalParseError = true; + gwi.parseErrorText = IDBErrorInfo::instance()->errorMsg(ERR_FILTER_COND_EXP); + setError(gwi.thd, ER_CHECK_NOT_IMPLEMENTED, gwi.parseErrorText, gwi); + return ER_CHECK_NOT_IMPLEMENTED; + } + + uint16_t parseInfo = 0; + vector tmpVec; + bool hasNonSupportItem = false; + parse_item(ifp, tmpVec, hasNonSupportItem, parseInfo); + + if (ifp->has_subquery() || + string(ifp->func_name()) == string("") || + ifp->functype() == Item_func::NOT_ALL_FUNC || + parseInfo & SUB_BIT) + { + gwi.fatalParseError = true; + gwi.parseErrorText = IDBErrorInfo::instance()->errorMsg(ERR_NON_SUPPORT_SELECT_SUB); + setError(gwi.thd, ER_CHECK_NOT_IMPLEMENTED, gwi.parseErrorText, gwi); + return ER_CHECK_NOT_IMPLEMENTED; + } + + ReturnedColumn* rc = buildFunctionColumn(ifp, gwi, hasNonSupportItem); + SRCP srcp(rc); + + if (rc) + { + if (!hasNonSupportItem && !nonConstFunc(ifp) && !(parseInfo & AF_BIT) && tmpVec.size() == 0) { - ordercol = reinterpret_cast(gi.groupByOrder); - //order_list = gi.groupByOrder; + if (isUnion || unionSel || gwi.subSelectType != CalpontSelectExecutionPlan::MAIN_SELECT || + parseInfo & SUB_BIT ) //|| select_lex.group_list.elements != 0) + { + srcp.reset(buildReturnedColumn(item, gwi, gwi.fatalParseError)); + gwi.returnedCols.push_back(srcp); + + if (ifp->name) + srcp->alias(ifp->name); + + continue; + } + + if ( ((gwi.thd->lex)->sql_command == SQLCOM_UPDATE ) || + ((gwi.thd->lex)->sql_command == SQLCOM_DELETE ) || + ((gwi.thd->lex)->sql_command == SQLCOM_UPDATE_MULTI ) || + ((gwi.thd->lex)->sql_command == SQLCOM_DELETE_MULTI ) ) + { } + else + { + redo = true; + String str; + ifp->print(&str, QT_INFINIDB_NO_QUOTE); + gwi.selectCols.push_back(string(str.c_ptr()) + " " + "`" + escapeBackTick(item->name) + "`"); + } + + break; } + + //SRCP srcp(rc); + gwi.returnedCols.push_back(srcp); + + if ( ((gwi.thd->lex)->sql_command == SQLCOM_UPDATE ) || ((gwi.thd->lex)->sql_command == SQLCOM_DELETE ) || ((gwi.thd->lex)->sql_command == SQLCOM_UPDATE_MULTI ) || ((gwi.thd->lex)->sql_command == SQLCOM_DELETE_MULTI )) + { } else - ordercol = 0; - - ord_cols = ""; - - for (; ordercol; ordercol = ordercol->next) { - Item* ord_item = *(ordercol->item); + String str(256); + ifp->print(&str, QT_INFINIDB_NO_QUOTE); - // @bug 1706. re-construct the order by item one by one, because the ord_cols constucted so far - // is for REDO phase. - if (ord_cols.length() != 0) - ord_cols += ", "; + if (sel_cols_in_create.length() != 0) + sel_cols_in_create += ", "; - if (ordercol->in_field_list && ordercol->counter_used) + sel_cols_in_create += string(str.c_ptr()) + " `" + ifp->name + "`"; + gwi.selectCols.push_back("`" + escapeBackTick(ifp->name) + "`"); + } + } + else // InfiniDB Non support functions still go through post process for now + { + hasNonSupportItem = false; + uint32_t before_size = funcFieldVec.size(); + parse_item(ifp, funcFieldVec, hasNonSupportItem, parseInfo); + uint32_t after_size = funcFieldVec.size(); + + // group by func and func in subquery can not be post processed + // @bug3881. set_user_var can not be treated as constant function + // @bug5716. Try to avoid post process function for union query. + if ((gwi.subQuery /*|| select_lex.group_list.elements != 0 */ || + !csep->unionVec().empty() || isUnion) && + !hasNonSupportItem && (after_size - before_size) == 0 && + !(parseInfo & AGG_BIT) && !(parseInfo & SUB_BIT) && + string(ifp->func_name()) != "set_user_var") + { + String val, *str = ifp->val_str(&val); + string valStr; + + if (str) + valStr.assign(str->ptr(), str->length()); + + ConstantColumn* cc = NULL; + + if (!str) { - ostringstream oss; - oss << ordercol->counter; - ord_cols += oss.str(); + cc = new ConstantColumn("", ConstantColumn::NULLDATA); } - else if (ord_item->type() == Item::NULL_ITEM) + else if (ifp->result_type() == STRING_RESULT) { - // MCOL-793 Do nothing for an ORDER BY NULL + cc = new ConstantColumn(valStr, ConstantColumn::LITERAL); } - else if (ord_item->type() == Item::SUM_FUNC_ITEM) + else if (ifp->result_type() == DECIMAL_RESULT) { - Item_sum* ifp = (Item_sum*)(*(ordercol->item)); - ReturnedColumn* fc = buildAggregateColumn(ifp, gwi); - - for (uint32_t i = 0; i < gwi.returnedCols.size(); i++) - { - if (fc->operator==(gwi.returnedCols[i].get())) - { - ostringstream oss; - oss << i + 1; - ord_cols += oss.str(); - break; - } - } - - //continue; - } - // @bug 3518. if order by clause = selected column, use position. - else if (ord_item->name && ord_item->type() == Item::FIELD_ITEM) - { - Item_field* field = reinterpret_cast(ord_item); - string fullname; - - if (field->db_name) - fullname += string(field->db_name) + "."; - - if (field->table_name) - fullname += string(field->table_name) + "."; - - if (field->field_name) - fullname += string(field->field_name); - - uint32_t i = 0; - - for (i = 0; i < gwi.returnedCols.size(); i++) - { - SimpleColumn* sc = dynamic_cast(gwi.returnedCols[i].get()); - - if (sc && ((Item_field*)ord_item)->cached_table && - (strcasecmp(getViewName(((Item_field*)ord_item)->cached_table).c_str(), sc->viewName().c_str()) != 0)) - continue; - - if (strcasecmp(fullname.c_str(), gwi.returnedCols[i]->alias().c_str()) == 0 || - strcasecmp(ord_item->name, gwi.returnedCols[i]->alias().c_str()) == 0) - { - ostringstream oss; - oss << i + 1; - ord_cols += oss.str(); - break; - } - } - - if (i == gwi.returnedCols.size()) - ord_cols += string(" `") + escapeBackTick(ord_item->name) + '`'; - } - - else if (ord_item->name) - { - // for union order by 1 case. For unknown reason, it doesn't show in_field_list - if (ord_item->type() == Item::INT_ITEM) - { - ord_cols += ord_item->name; - } - else if (ord_item->type() == Item::SUBSELECT_ITEM) - { - string emsg = IDBErrorInfo::instance()->errorMsg(ERR_NON_SUPPORT_ORDER_BY); - setError(gwi.thd, ER_CHECK_NOT_IMPLEMENTED, emsg, gwi); - return ER_CHECK_NOT_IMPLEMENTED; - } - else - { - ord_cols += string(" `") + escapeBackTick(ord_item->name) + '`'; - } - } - else if (ord_item->type() == Item::FUNC_ITEM) - { - // @bug5636. check if this order by column is on the select list - ReturnedColumn* rc = buildFunctionColumn((Item_func*)(ord_item), gwi, gwi.fatalParseError); - - for (uint32_t i = 0; i < gwi.returnedCols.size(); i++) - { - if (rc && rc->operator==(gwi.returnedCols[i].get())) - { - ostringstream oss; - oss << i + 1; - ord_cols += oss.str(); - break; - } - } + cc = buildDecimalColumn(ifp, gwi); } else { - String str; - ord_item->print(&str, QT_INFINIDB_NO_QUOTE); - ord_cols += string(str.c_ptr()); + cc = new ConstantColumn(valStr, ConstantColumn::NUM); + cc->resultType(colType_MysqlToIDB(item)); } - if (ordercol->direction != ORDER::ORDER_ASC) - ord_cols += " desc"; + SRCP srcp(cc); + + if (ifp->name) + cc->alias(ifp->name); + + gwi.returnedCols.push_back(srcp); + + // clear the error set by buildFunctionColumn + gwi.fatalParseError = false; + gwi.parseErrorText = ""; + break; + } + else if (hasNonSupportItem || parseInfo & AGG_BIT || parseInfo & SUB_BIT || + (gwi.fatalParseError && gwi.subQuery)) + { + if (gwi.parseErrorText.empty()) + { + Message::Args args; + args.add(ifp->func_name()); + gwi.parseErrorText = IDBErrorInfo::instance()->errorMsg(ERR_NON_SUPPORTED_FUNCTION, args); + } + + setError(gwi.thd, ER_CHECK_NOT_IMPLEMENTED, gwi.parseErrorText, gwi); + return ER_CHECK_NOT_IMPLEMENTED; + } + else if ( gwi.subQuery && (isPredicateFunction(ifp, &gwi) || ifp->type() == Item::COND_ITEM )) + { + gwi.fatalParseError = true; + gwi.parseErrorText = IDBErrorInfo::instance()->errorMsg(ERR_FILTER_COND_EXP); + setError(gwi.thd, ER_CHECK_NOT_IMPLEMENTED, gwi.parseErrorText, gwi); + return ER_CHECK_NOT_IMPLEMENTED; + } + + //@Bug 3030 Add error check for dml statement + if ( ((gwi.thd->lex)->sql_command == SQLCOM_UPDATE ) || ((gwi.thd->lex)->sql_command == SQLCOM_DELETE ) || ((gwi.thd->lex)->sql_command == SQLCOM_UPDATE_MULTI ) || ((gwi.thd->lex)->sql_command == SQLCOM_DELETE_MULTI ) ) + { + if ( after_size - before_size != 0 ) + { + gwi.parseErrorText = ifp->func_name(); + return -1; + } + } + + //@Bug 3021. Bypass postprocess for update and delete. + //if ( ((gwi.thd->lex)->sql_command == SQLCOM_UPDATE ) || ((gwi.thd->lex)->sql_command == SQLCOM_DELETE ) || ((gwi.thd->lex)->sql_command == SQLCOM_UPDATE_MULTI ) || ((gwi.thd->lex)->sql_command == SQLCOM_DELETE_MULTI )) + //{} + else + { + // @bug 3881. Here is the real redo part. + redo = true; + // @bug 1706 + String funcStr; + ifp->print(&funcStr, QT_INFINIDB); + gwi.selectCols.push_back(string(funcStr.c_ptr()) + " `" + escapeBackTick(ifp->name) + "`"); + // clear the error set by buildFunctionColumn + gwi.fatalParseError = false; + gwi.parseErrorText = ""; } } - if (ord_cols.length() > 0) // has order by - { - gwi.thd->infinidb_vtable.has_order_by = true; - csep->hasOrderBy(true); - ord_cols = " order by " + ord_cols; - select_query += ord_cols; - } + break; } - } // ORDER BY processing ends here - if ( gi.groupByDistinct ) - csep->distinct(true); - - // add the smallest column to count(*) parm. - // select constant in subquery case - std::vector::iterator coliter; - - if (!minSc) - { - if (!gwi.returnedCols.empty()) - minSc = gwi.returnedCols[0]; - else if (!gwi.additionalRetCols.empty()) - minSc = gwi.additionalRetCols[0]; - } - - // @bug3523, count(*) on subquery always pick column[0]. - SimpleColumn* sc = dynamic_cast(minSc.get()); - - if (sc && sc->schemaName().empty()) - { - if (gwi.derivedTbList.size() >= 1) + case Item::INT_ITEM: { - SimpleColumn* sc1 = new SimpleColumn(); - sc1->columnName(sc->columnName()); - sc1->tableName(sc->tableName()); - sc1->tableAlias(sc->tableAlias()); - sc1->viewName(lower(sc->viewName())); - sc1->colPosition(0); - minSc.reset(sc1); - } - } + if ( ((gwi.thd->lex)->sql_command == SQLCOM_UPDATE ) || ((gwi.thd->lex)->sql_command == SQLCOM_DELETE ) || ((gwi.thd->lex)->sql_command == SQLCOM_UPDATE_MULTI ) || ((gwi.thd->lex)->sql_command == SQLCOM_DELETE_MULTI )) + { } + else + { + // do not push the dummy column (mysql added) to returnedCol + if (item->name && string(item->name) == "Not_used") + continue; - for (coliter = gwi.count_asterisk_list.begin(); coliter != gwi.count_asterisk_list.end(); ++coliter) - { - // @bug5977 @note should never throw this, but checking just in case. - // When ExeMgr fix is ready, this should not error out... - if (dynamic_cast(minSc.get())) + // @bug3509. Constant column is sent to ExeMgr now. + SRCP srcp(buildReturnedColumn(item, gwi, gwi.fatalParseError)); + + if (item->name) + srcp->alias(item->name); + + gwi.returnedCols.push_back(srcp); + + Item_int* isp = reinterpret_cast(item); + ostringstream oss; + oss << isp->value << " `" << escapeBackTick(srcp->alias().c_str()) << "`"; + + if (sel_cols_in_create.length() != 0) + sel_cols_in_create += ", "; + + sel_cols_in_create += oss.str(); + gwi.selectCols.push_back(oss.str()); + } + + break; + } + + case Item::STRING_ITEM: + { + if ( ((gwi.thd->lex)->sql_command == SQLCOM_UPDATE ) || ((gwi.thd->lex)->sql_command == SQLCOM_DELETE ) || ((gwi.thd->lex)->sql_command == SQLCOM_UPDATE_MULTI ) || ((gwi.thd->lex)->sql_command == SQLCOM_DELETE_MULTI )) + { } + else + { + SRCP srcp(buildReturnedColumn(item, gwi, gwi.fatalParseError)); + gwi.returnedCols.push_back(srcp); + + if (item->name) + srcp->alias(item->name); + + Item_string* isp = reinterpret_cast(item); + String val, *str = isp->val_str(&val); + string valStr; + valStr.assign(str->ptr(), str->length()); + string name = "'" + valStr + "'" + " " + "`" + escapeBackTick(srcp->alias().c_str()) + "`"; + + if (sel_cols_in_create.length() != 0) + sel_cols_in_create += ", "; + + sel_cols_in_create += name; + gwi.selectCols.push_back(name); + } + + break; + } + + case Item::DECIMAL_ITEM: + { + if ( ((gwi.thd->lex)->sql_command == SQLCOM_UPDATE ) || ((gwi.thd->lex)->sql_command == SQLCOM_DELETE ) || ((gwi.thd->lex)->sql_command == SQLCOM_UPDATE_MULTI ) || ((gwi.thd->lex)->sql_command == SQLCOM_DELETE_MULTI )) + { } + else + { + SRCP srcp(buildReturnedColumn(item, gwi, gwi.fatalParseError)); + gwi.returnedCols.push_back(srcp); + + if (item->name) + srcp->alias(item->name); + + Item_decimal* isp = reinterpret_cast(item); + String val, *str = isp->val_str(&val); + string valStr; + valStr.assign(str->ptr(), str->length()); + ostringstream oss; + oss << valStr.c_str() << " `" << escapeBackTick(srcp->alias().c_str()) << "`"; + + if (sel_cols_in_create.length() != 0) + sel_cols_in_create += ", "; + + sel_cols_in_create += oss.str(); + gwi.selectCols.push_back(oss.str()); + } + + break; + } + + case Item::NULL_ITEM: + { + /*if ( ((gwi.thd->lex)->sql_command == SQLCOM_UPDATE ) || ((gwi.thd->lex)->sql_command == SQLCOM_DELETE ) || ((gwi.thd->lex)->sql_command == SQLCOM_UPDATE_MULTI ) || ((gwi.thd->lex)->sql_command == SQLCOM_DELETE_MULTI ) ) + { } + else + { + SRCP srcp(buildReturnedColumn(item, gwi, gwi.fatalParseError)); + gwi.returnedCols.push_back(srcp); + + if (item->name) + srcp->alias(item->name); + + string name = string("null `") + escapeBackTick(srcp->alias().c_str()) + string("`") ; + + if (sel_cols_in_create.length() != 0) + sel_cols_in_create += ", "; + + sel_cols_in_create += name; + gwi.selectCols.push_back("null"); + }*/ + + break; + } + + case Item::SUBSELECT_ITEM: + { + Item_subselect* sub = (Item_subselect*)item; + + if (sub->substype() != Item_subselect::SINGLEROW_SUBS) + { + gwi.fatalParseError = true; + gwi.parseErrorText = IDBErrorInfo::instance()->errorMsg(ERR_NON_SUPPORT_SELECT_SUB); + setError(gwi.thd, ER_CHECK_NOT_IMPLEMENTED, gwi.parseErrorText, gwi); + return ER_CHECK_NOT_IMPLEMENTED; + } + +#ifdef DEBUG_WALK_COND + cerr << "SELECT clause SUBSELECT Item: " << sub->substype() << endl; + JOIN* join = sub->get_select_lex()->join; + + if (join) + { + Item_cond* cond = reinterpret_cast(join->conds); + + if (cond) + cond->traverse_cond(debug_walk, &gwi, Item::POSTFIX); + } + + cerr << "Finish SELECT clause subselect item traversing" << endl; +#endif + SelectSubQuery* selectSub = new SelectSubQuery(gwi, sub); + //selectSub->gwip(&gwi); + SCSEP ssub = selectSub->transform(); + + if (!ssub || gwi.fatalParseError) + { + if (gwi.parseErrorText.empty()) + gwi.parseErrorText = "Unsupported Item in SELECT subquery."; + + setError(gwi.thd, ER_CHECK_NOT_IMPLEMENTED, gwi.parseErrorText, gwi); + return ER_CHECK_NOT_IMPLEMENTED; + } + + selectSubList.push_back(ssub); + SimpleColumn* rc = new SimpleColumn(); + rc->colSource(rc->colSource() | SELECT_SUB); + + if (sub->get_select_lex()->get_table_list()) + rc->viewName(lower(getViewName(sub->get_select_lex()->get_table_list()))); + + if (sub->name) + rc->alias(sub->name); + + gwi.returnedCols.push_back(SRCP(rc)); + String str; + sub->get_select_lex()->print(gwi.thd, &str, QT_INFINIDB_NO_QUOTE); + sel_cols_in_create += "(" + string(str.c_ptr()) + ")"; + + if (sub->name) + { + sel_cols_in_create += "`" + escapeBackTick(sub->name) + "`"; + gwi.selectCols.push_back(sub->name); + } + else + { + sel_cols_in_create += "`" + escapeBackTick(str.c_ptr()) + "`"; + gwi.selectCols.push_back("`" + escapeBackTick(str.c_ptr()) + "`"); + } + + break; + } + + case Item::COND_ITEM: { gwi.fatalParseError = true; - gwi.parseErrorText = "No project column found for aggregate function"; - setError(gwi.thd, ER_INTERNAL_ERROR, gwi.parseErrorText, gwi); + gwi.parseErrorText = IDBErrorInfo::instance()->errorMsg(ERR_FILTER_COND_EXP); + setError(gwi.thd, ER_CHECK_NOT_IMPLEMENTED, gwi.parseErrorText, gwi); return ER_CHECK_NOT_IMPLEMENTED; } - (*coliter)->functionParms(minSc); + case Item::EXPR_CACHE_ITEM: + { + printf("EXPR_CACHE_ITEM in getSelectPlan\n"); + gwi.fatalParseError = true; + gwi.parseErrorText = IDBErrorInfo::instance()->errorMsg(ERR_UNKNOWN_COL); + setError(gwi.thd, ER_CHECK_NOT_IMPLEMENTED, gwi.parseErrorText, gwi); + return ER_CHECK_NOT_IMPLEMENTED; + } + + case Item::WINDOW_FUNC_ITEM: + { + SRCP srcp(buildWindowFunctionColumn(item, gwi, gwi.fatalParseError)); + + if (!srcp || gwi.fatalParseError) + { + if (gwi.parseErrorText.empty()) + gwi.parseErrorText = "Unsupported Item in SELECT subquery."; + + setError(gwi.thd, ER_CHECK_NOT_IMPLEMENTED, gwi.parseErrorText, gwi); + return ER_CHECK_NOT_IMPLEMENTED; + } + + gwi.returnedCols.push_back(srcp); + break; + } + + default: + { + break; + } } - - std::vector::iterator funciter; - - SPTP sptp(new ParseTree(minSc.get()->clone())); - - for (funciter = gwi.no_parm_func_list.begin(); funciter != gwi.no_parm_func_list.end(); ++funciter) - { - FunctionParm funcParms = (*funciter)->functionParms(); - funcParms.push_back(sptp); - (*funciter)->functionParms(funcParms); - } - - // set sequence# for subquery localCols - for (uint32_t i = 0; i < gwi.localCols.size(); i++) - gwi.localCols[i]->sequence(i); - - // append additionalRetCols to returnedCols - gwi.returnedCols.insert(gwi.returnedCols.begin(), gwi.additionalRetCols.begin(), - gwi.additionalRetCols.end()); - - csep->groupByCols(gwi.groupByCols); - csep->orderByCols(gwi.orderByCols); - csep->returnedCols(gwi.returnedCols); - csep->columnMap(gwi.columnMap); - csep->having(havingFilter); - csep->derivedTableList(gwi.derivedTbList); - csep->selectSubList(selectSubList); - csep->subSelectList(gwi.subselectList); - gwi.thd->infinidb_vtable.duplicate_field_name = false; - clearStacks(gwi); - return 0; } + // @bug4388 normalize the project coltypes for union main select list + if (!csep->unionVec().empty()) + { + for (uint32_t i = 0; i < gwi.returnedCols.size(); i++) + { + vector coltypes; + + for (uint32_t j = 0; j < csep->unionVec().size(); j++) + { + coltypes.push_back( + dynamic_cast(csep->unionVec()[j].get())->returnedCols()[i]->resultType()); + + // @bug5976. set hasAggregate true for the main column if + // one corresponding union column has aggregate + if (dynamic_cast(csep->unionVec()[j].get())->returnedCols()[i]->hasAggregate()) + gwi.returnedCols[i]->hasAggregate(true); + } + + gwi.returnedCols[i]->resultType(dataconvert::DataConvert::convertUnionColType(coltypes)); + } + } + + // Having clause handling + gwi.clauseType = HAVING; + clearStacks(gwi); + ParseTree* havingFilter = 0; + // clear fatalParseError that may be left from post process functions + gwi.fatalParseError = false; + gwi.parseErrorText = ""; + + if (gi.groupByHaving != 0) + { + Item_cond* having = reinterpret_cast(gi.groupByHaving); +#ifdef DEBUG_WALK_COND + cerr << "------------------- HAVING ---------------------" << endl; + having->traverse_cond(debug_walk, &gwi, Item::POSTFIX); + cerr << "------------------------------------------------\n" << endl; +#endif + having->traverse_cond(gp_walk, &gwi, Item::POSTFIX); + + if (gwi.fatalParseError) + { + setError(gwi.thd, ER_INTERNAL_ERROR, gwi.parseErrorText, gwi); + return ER_INTERNAL_ERROR; + } + + ParseTree* ptp = 0; + ParseTree* rhs = 0; + + // @bug 4215. some function filter will be in the rcWorkStack. + if (gwi.ptWorkStack.empty() && !gwi.rcWorkStack.empty()) + { + havingFilter = new ParseTree(gwi.rcWorkStack.top()); + gwi.rcWorkStack.pop(); + } + + while (!gwi.ptWorkStack.empty()) + { + havingFilter = gwi.ptWorkStack.top(); + gwi.ptWorkStack.pop(); + + if (gwi.ptWorkStack.empty()) + break; + + ptp = new ParseTree(new LogicOperator("and")); + ptp->left(havingFilter); + rhs = gwi.ptWorkStack.top(); + gwi.ptWorkStack.pop(); + ptp->right(rhs); + gwi.ptWorkStack.push(ptp); + } + } + + // for post process expressions on the select list + // error out post process for union and sub select unit + if (isUnion || gwi.subSelectType != CalpontSelectExecutionPlan::MAIN_SELECT) + { + if (funcFieldVec.size() != 0 && !gwi.fatalParseError) + { + string emsg("Fatal parse error in vtable mode: Unsupported Items in union or sub select unit"); + setError(gwi.thd, ER_CHECK_NOT_IMPLEMENTED, emsg); + return ER_CHECK_NOT_IMPLEMENTED; + } + } + + for (uint32_t i = 0; i < funcFieldVec.size(); i++) + { + SimpleColumn* sc = buildSimpleColumn(funcFieldVec[i], gwi); + + if (!sc || gwi.fatalParseError) + { + string emsg; + + if (gwi.parseErrorText.empty()) + { + emsg = "un-recognized column"; + + if (funcFieldVec[i]->name) + emsg += string(funcFieldVec[i]->name); + } + else + { + emsg = gwi.parseErrorText; + } + + setError(gwi.thd, ER_INTERNAL_ERROR, emsg, gwi); + return ER_INTERNAL_ERROR; + } + + String str; + funcFieldVec[i]->print(&str, QT_INFINIDB_NO_QUOTE); + sc->alias(string(str.c_ptr())); + //sc->tableAlias(funcFieldVec[i]->table_name); + sc->tableAlias(sc->tableAlias()); + SRCP srcp(sc); + uint32_t j = 0; + + for (; j < gwi.returnedCols.size(); j++) + { + if (sc->sameColumn(gwi.returnedCols[j].get())) + { + SimpleColumn* field = dynamic_cast(gwi.returnedCols[j].get()); + + if (field && field->alias() == sc->alias()) + break; + } + } + + if (j == gwi.returnedCols.size()) + { + gwi.returnedCols.push_back(srcp); + gwi.columnMap.insert(CalpontSelectExecutionPlan::ColumnMap::value_type(string(funcFieldVec[i]->field_name), srcp)); + + if (sel_cols_in_create.length() != 0) + sel_cols_in_create += ", "; + + string fullname; + fullname = str.c_ptr(); + sel_cols_in_create += fullname + " `" + escapeBackTick(fullname.c_str()) + "`"; + TABLE_LIST* tmp = (funcFieldVec[i]->cached_table ? funcFieldVec[i]->cached_table : 0); + gwi.tableMap[make_aliastable(sc->schemaName(), sc->tableName(), sc->tableAlias(), sc->isInfiniDB())] = + make_pair(1, tmp); + } + } + + // post-process Order by list and expressions on select by redo phase1. only for vtable + // ignore ORDER BY clause for union select unit + string ord_cols = ""; // for normal select phase + SRCP minSc; // min width projected column. for count(*) use + + // Group by list. not valid for union main query + if (gwi.thd->infinidb_vtable.vtable_state == THD::INFINIDB_CREATE_VTABLE && !unionSel) + { + gwi.clauseType = GROUP_BY; + Item* nonSupportItem = NULL; + ORDER* groupcol = reinterpret_cast(gi.groupByGroup); + + // check if window functions are in order by. InfiniDB process order by list if + // window functions are involved, either in order by or projection. + bool hasWindowFunc = gwi.hasWindowFunc; + gwi.hasWindowFunc = false; + + for (; groupcol; groupcol = groupcol->next) + { + if ((*(groupcol->item))->type() == Item::WINDOW_FUNC_ITEM) + gwi.hasWindowFunc = true; + } + + if (gwi.hasWindowFunc) + { + gwi.fatalParseError = true; + gwi.parseErrorText = IDBErrorInfo::instance()->errorMsg(ERR_WF_NOT_ALLOWED, "GROUP BY clause"); + setError(gwi.thd, ER_CHECK_NOT_IMPLEMENTED, gwi.parseErrorText, gwi); + return ER_CHECK_NOT_IMPLEMENTED; + } + + gwi.hasWindowFunc = hasWindowFunc; + groupcol = reinterpret_cast(gi.groupByGroup); + + for (; groupcol; groupcol = groupcol->next) + { + Item* groupItem = *(groupcol->item); + + // @bug5993. Could be nested ref. + while (groupItem->type() == Item::REF_ITEM) + groupItem = (*((Item_ref*)groupItem)->ref); + + if (groupItem->type() == Item::FUNC_ITEM) + { + Item_func* ifp = (Item_func*)groupItem; + + // call buildFunctionColumn here mostly for finding out + // non-support column on GB list. Should be simplified. + ReturnedColumn* fc = buildFunctionColumn(ifp, gwi, gwi.fatalParseError); + + if (!fc || gwi.fatalParseError) + { + nonSupportItem = ifp; + break; + } + + if (groupcol->in_field_list && groupcol->counter_used) + { + delete fc; + fc = gwi.returnedCols[groupcol->counter - 1].get(); + SRCP srcp(fc->clone()); + + // check if no column parm + for (uint32_t i = 0; i < gwi.no_parm_func_list.size(); i++) + { + if (gwi.no_parm_func_list[i]->expressionId() == fc->expressionId()) + { + gwi.no_parm_func_list.push_back(dynamic_cast(srcp.get())); + break; + } + } + + srcp->orderPos(groupcol->counter - 1); + gwi.groupByCols.push_back(srcp); + continue; + } + else if (!groupItem->is_autogenerated_name) // alias + { + uint32_t i = 0; + + for (; i < gwi.returnedCols.size(); i++) + { + if (string(groupItem->name) == gwi.returnedCols[i]->alias()) + { + ReturnedColumn* rc = gwi.returnedCols[i]->clone(); + rc->orderPos(i); + gwi.groupByCols.push_back(SRCP(rc)); + delete fc; + break; + } + } + + if (i == gwi.returnedCols.size()) + { + nonSupportItem = groupItem; + break; + } + } + else + { + uint32_t i = 0; + + for (; i < gwi.returnedCols.size(); i++) + { + if (fc->operator==(gwi.returnedCols[i].get())) + { + ReturnedColumn* rc = gwi.returnedCols[i]->clone(); + rc->orderPos(i); + gwi.groupByCols.push_back(SRCP(rc)); + delete fc; + break; + } + } + + if (i == gwi.returnedCols.size()) + { + gwi.groupByCols.push_back(SRCP(fc)); + break; + } + } + } + else if (groupItem->type() == Item::FIELD_ITEM) + { + Item_field* ifp = (Item_field*)groupItem; + // this GB col could be an alias of F&E on the SELECT clause, not necessarily a field. + ReturnedColumn* rc = buildSimpleColumn(ifp, gwi); + SimpleColumn* sc = dynamic_cast(rc); + + for (uint32_t j = 0; j < gwi.returnedCols.size(); j++) + { + if (sc) + { + if (sc->sameColumn(gwi.returnedCols[j].get())) + { + sc->orderPos(j); + break; + } + else if (strcasecmp(sc->alias().c_str(), gwi.returnedCols[j]->alias().c_str()) == 0) + { + rc = gwi.returnedCols[j].get()->clone(); + rc->orderPos(j); + break; + } + } + else + { + if (ifp->name && string(ifp->name) == gwi.returnedCols[j].get()->alias()) + { + rc = gwi.returnedCols[j].get()->clone(); + rc->orderPos(j); + break; + } + } + } + + if (!rc) + { + nonSupportItem = ifp; + break; + } + + SRCP srcp(rc); + + // bug 3151 + AggregateColumn* ac = dynamic_cast(rc); + + if (ac) + { + nonSupportItem = ifp; + break; + } + + gwi.groupByCols.push_back(srcp); + gwi.columnMap.insert(CalpontSelectExecutionPlan::ColumnMap::value_type(string(ifp->field_name), srcp)); + } + // @bug5638. The group by column is constant but not counter, alias has to match a column + // on the select list + else if (!groupcol->counter_used && + (groupItem->type() == Item::INT_ITEM || + groupItem->type() == Item::STRING_ITEM || + groupItem->type() == Item::REAL_ITEM || + groupItem->type() == Item::DECIMAL_ITEM)) + { + ReturnedColumn* rc = 0; + + for (uint32_t j = 0; j < gwi.returnedCols.size(); j++) + { + if (groupItem->name && string(groupItem->name) == gwi.returnedCols[j].get()->alias()) + { + rc = gwi.returnedCols[j].get()->clone(); + rc->orderPos(j); + break; + } + } + + if (!rc) + { + nonSupportItem = groupItem; + break; + } + + gwi.groupByCols.push_back(SRCP(rc)); + } + else if ((*(groupcol->item))->type() == Item::SUBSELECT_ITEM) + { + if (!groupcol->in_field_list || !groupItem->name) + { + nonSupportItem = groupItem; + } + else + { + uint32_t i = 0; + + for (; i < gwi.returnedCols.size(); i++) + { + if (string(groupItem->name) == gwi.returnedCols[i]->alias()) + { + ReturnedColumn* rc = gwi.returnedCols[i]->clone(); + rc->orderPos(i); + gwi.groupByCols.push_back(SRCP(rc)); + break; + } + } + + if (i == gwi.returnedCols.size()) + { + nonSupportItem = groupItem; + } + } + } + // @bug 3761. + else if (groupcol->counter_used) + { + if (gwi.returnedCols.size() <= (uint32_t)(groupcol->counter - 1)) + { + nonSupportItem = groupItem; + } + else + { + gwi.groupByCols.push_back(SRCP(gwi.returnedCols[groupcol->counter - 1]->clone())); + } + } + else + { + nonSupportItem = groupItem; + } + + } + + // @bug 4756. Add internal groupby column for correlated join to the groupby list + if (gwi.aggOnSelect && !gwi.subGroupByCols.empty()) + gwi.groupByCols.insert(gwi.groupByCols.end(), gwi.subGroupByCols.begin(), gwi.subGroupByCols.end()); + + // this is window func on SELECT becuase ORDER BY has not been processed + if (!gwi.windowFuncList.empty() && !gwi.subGroupByCols.empty()) + { + for (uint32_t i = 0; i < gwi.windowFuncList.size(); i++) + { + if (gwi.windowFuncList[i]->hasWindowFunc()) + { + vector windowFunctions = gwi.windowFuncList[i]->windowfunctionColumnList(); + + for (uint32_t j = 0; j < windowFunctions.size(); j++) + windowFunctions[j]->addToPartition(gwi.subGroupByCols); + } + } + } + + if (nonSupportItem) + { + Message::Args args; + + if (nonSupportItem->name) + args.add("'" + string(nonSupportItem->name) + "'"); + else + args.add(""); + + gwi.parseErrorText = IDBErrorInfo::instance()->errorMsg(ERR_NON_SUPPORT_GROUP_BY, args); + setError(gwi.thd, ER_CHECK_NOT_IMPLEMENTED, gwi.parseErrorText, gwi); + return ER_CHECK_NOT_IMPLEMENTED; + } + } // GROUP processing ends here + + + if (gwi.thd->infinidb_vtable.vtable_state == THD::INFINIDB_CREATE_VTABLE) + { + ORDER* ordercol = reinterpret_cast(gi.groupByOrder); + string create_query(gwi.thd->infinidb_vtable.create_vtable_query.c_ptr()); + string select_query(gwi.thd->infinidb_vtable.select_vtable_query.c_ptr()); + string lower_create_query(gwi.thd->infinidb_vtable.create_vtable_query.c_ptr()); + string lower_select_query(gwi.thd->infinidb_vtable.select_vtable_query.c_ptr()); + algorithm::to_lower(lower_create_query); + algorithm::to_lower(lower_select_query); + + + // check if window functions are in order by. InfiniDB process order by list if + // window functions are involved, either in order by or projection. + for (; ordercol; ordercol = ordercol->next) + { + if ((*(ordercol->item))->type() == Item::WINDOW_FUNC_ITEM) + gwi.hasWindowFunc = true; + } + + // re-visit the first of ordercol list + ordercol = reinterpret_cast(gi.groupByOrder); + + // for subquery, order+limit by will be supported in infinidb. build order by columns + // @todo union order by and limit support + //if (gwi.hasWindowFunc || gwi.subSelectType != CalpontSelectExecutionPlan::MAIN_SELECT) + + for (; ordercol; ordercol = ordercol->next) + { + ReturnedColumn* rc = NULL; + + if (ordercol->in_field_list && ordercol->counter_used) + { + rc = gwi.returnedCols[ordercol->counter - 1]->clone(); + rc->orderPos(ordercol->counter - 1); + // can not be optimized off if used in order by with counter. + // set with self derived table alias if it's derived table + gwi.returnedCols[ordercol->counter - 1]->incRefCount(); + } + else + { + Item* ord_item = *(ordercol->item); + + // ignore not_used column on order by. + if (ord_item->type() == Item::INT_ITEM && ord_item->full_name() && string(ord_item->full_name()) == "Not_used") + continue; + else if (ord_item->type() == Item::INT_ITEM) + rc = gwi.returnedCols[((Item_int*)ord_item)->val_int() - 1]->clone(); + else if (ord_item->type() == Item::SUBSELECT_ITEM) + gwi.fatalParseError = true; + else + rc = buildReturnedColumn(ord_item, gwi, gwi.fatalParseError); + + // Looking for a match for this item in GROUP BY list. + if ( rc && ord_item->type() == Item::FIELD_ITEM ) + { + execplan::CalpontSelectExecutionPlan::ReturnedColumnList::iterator iter = gwi.groupByCols.begin(); + + for ( ; iter != gwi.groupByCols.end(); iter++ ) + { + if ( rc->sameColumn((*iter).get()) ) + break; + } + + // MCOL-1052 Find and remove the optimized field + // from ORDER using cond_pushed filters. + if (buildConstColFromFilter( + dynamic_cast(rc), gwi, gi)) + { + break; + } + + // MCOL-1052 GROUP BY items list doesn't contain + // this ORDER BY item. + if ( iter == gwi.groupByCols.end() ) + { + Item_ident* iip = reinterpret_cast(ord_item); + std::ostringstream ostream; + ostream << "'"; + + if (iip->db_name) + ostream << iip->db_name << '.'; + else + ostream << "unknown db" << '.'; + + if (iip->table_name) + ostream << iip->table_name << '.'; + else + ostream << "unknown table" << '.'; + + if (iip->field_name) + ostream << iip->field_name; + else + ostream << "unknown field"; + + ostream << "'"; + Message::Args args; + args.add(ostream.str()); + string emsg = IDBErrorInfo::instance()->errorMsg(ERR_NOT_GROUPBY_EXPRESSION, args); + gwi.parseErrorText = emsg; + setError(gwi.thd, ER_INTERNAL_ERROR, emsg, gwi); + return ERR_NOT_GROUPBY_EXPRESSION; + } + } + + // @bug5501 try item_ptr if item can not be fixed. For some + // weird dml statement state, item can not be fixed but the + // infomation is available in item_ptr. + if (!rc || gwi.fatalParseError) + { + gwi.fatalParseError = false; + Item* item_ptr = ordercol->item_ptr; + + while (item_ptr->type() == Item::REF_ITEM) + item_ptr = *(((Item_ref*)item_ptr)->ref); + + rc = buildReturnedColumn(item_ptr, gwi, gwi.fatalParseError); + } + + // This ORDER BY item must be an agg function - + // the ordercol->item_ptr and exteded SELECT list + // must contain the corresponding item. + if (!rc) + { + Item* item_ptr = ordercol->item_ptr; + + if (item_ptr) + rc = buildReturnedColumn(item_ptr, gwi, gwi.fatalParseError); + } + + if (!rc) + { + string emsg = IDBErrorInfo::instance()->errorMsg(ERR_NON_SUPPORT_ORDER_BY); + gwi.parseErrorText = emsg; + setError(gwi.thd, ER_CHECK_NOT_IMPLEMENTED, emsg, gwi); + return ER_CHECK_NOT_IMPLEMENTED; + } + } + + if (ordercol->direction == ORDER::ORDER_ASC) + rc->asc(true); + else + rc->asc(false); + + gwi.orderByCols.push_back(SRCP(rc)); + } + + + // make sure columnmap, returnedcols and count(*) arg_list are not empty + TableMap::iterator tb_iter = gwi.tableMap.begin(); + + try + { + for (; tb_iter != gwi.tableMap.end(); tb_iter++) + { + if ((*tb_iter).second.first == 1) continue; + + CalpontSystemCatalog::TableAliasName tan = (*tb_iter).first; + CalpontSystemCatalog::TableName tn = make_table((*tb_iter).first.schema, (*tb_iter).first.table); + SimpleColumn* sc = getSmallestColumn(csc, tn, tan, (*tb_iter).second.second->table, gwi); + SRCP srcp(sc); + gwi.columnMap.insert(CalpontSelectExecutionPlan::ColumnMap::value_type(sc->columnName(), srcp)); + (*tb_iter).second.first = 1; + } + } + catch (runtime_error& e) + { + setError(gwi.thd, ER_INTERNAL_ERROR, e.what(), gwi); + CalpontSystemCatalog::removeCalpontSystemCatalog(sessionID); + return ER_INTERNAL_ERROR; + } + catch (...) + { + string emsg = IDBErrorInfo::instance()->errorMsg(ERR_LOST_CONN_EXEMGR); + setError(gwi.thd, ER_INTERNAL_ERROR, emsg, gwi); + CalpontSystemCatalog::removeCalpontSystemCatalog(sessionID); + return ER_INTERNAL_ERROR; + } + + if (!gwi.count_asterisk_list.empty() || !gwi.no_parm_func_list.empty() || + gwi.returnedCols.empty()) + { + // get the smallest column from colmap + CalpontSelectExecutionPlan::ColumnMap::const_iterator iter; + int minColWidth = 0; + CalpontSystemCatalog::ColType ct; + + try + { + for (iter = gwi.columnMap.begin(); iter != gwi.columnMap.end(); ++iter) + { + // should always not null + SimpleColumn* sc = dynamic_cast(iter->second.get()); + + if (sc && !(sc->joinInfo() & JOIN_CORRELATED)) + { + ct = csc->colType(sc->oid()); + + if (minColWidth == 0) + { + minColWidth = ct.colWidth; + minSc = iter->second; + } + else if (ct.colWidth < minColWidth) + { + minColWidth = ct.colWidth; + minSc = iter->second; + } + } + } + } + catch (...) + { + string emsg = IDBErrorInfo::instance()->errorMsg(ERR_LOST_CONN_EXEMGR); + setError(gwi.thd, ER_INTERNAL_ERROR, emsg, gwi); + CalpontSystemCatalog::removeCalpontSystemCatalog(sessionID); + return ER_INTERNAL_ERROR; + } + + if (gwi.returnedCols.empty() && gwi.additionalRetCols.empty()) + gwi.returnedCols.push_back(minSc); + } + + if (!isUnion && !gwi.hasWindowFunc && gwi.subSelectType == CalpontSelectExecutionPlan::MAIN_SELECT) + { + std::ostringstream vtb; + vtb << "infinidb_vtable.$vtable_" << gwi.thd->thread_id; + + //vtb << "$vtable_" << gwi.thd->thread_id; + // re-construct the select query and redo phase 1 + if (redo) + { + // select now() from region case. returnedCols should have minSc. + if (sel_cols_in_create.length() == 0) + { + SimpleColumn* sc = dynamic_cast(gwi.returnedCols[0].get()); + + if (sc) + sel_cols_in_create = dynamic_cast(gwi.returnedCols[0].get())->columnName(); + else + sel_cols_in_create = gwi.returnedCols[0]->alias(); + } + + // select * from derived table case + if (gwi.selectCols.empty()) + sel_cols_in_create = " * "; + + create_query = "create temporary table " + vtb.str() + " engine = aria as select " + sel_cols_in_create + " from "; + TABLE_LIST* table_ptr = gi.groupByTables; + + bool firstTb = true; + + // put all tables, derived tables and views on the list + //TABLE_LIST* table_ptr = select_lex.get_table_list(); + set aliasSet; // to avoid duplicate table alias + + for (; table_ptr; table_ptr = table_ptr->next_local) + { + if (string(table_ptr->table_name).find("$vtable") != string::npos) + continue; + + if (table_ptr->derived) + { + if (aliasSet.find(table_ptr->alias) != aliasSet.end()) + continue; + + String str; + (table_ptr->derived->first_select())->print(gwi.thd, &str, QT_INFINIDB_DERIVED); + + if (!firstTb) + create_query += ", "; + + create_query += "(" + string(str.c_ptr()) + ") " + string(table_ptr->alias); + firstTb = false; + aliasSet.insert(table_ptr->alias); + } + else if (table_ptr->view) + { + if (aliasSet.find(table_ptr->alias) != aliasSet.end()) + continue; + + if (!firstTb) + create_query += ", "; + + create_query += string(table_ptr->db) + "." + string(table_ptr->table_name) + + string(" `") + escapeBackTick(table_ptr->alias) + string("`"); + aliasSet.insert(table_ptr->alias); + firstTb = false; + } + else + { + // table referenced by view is represented by viewAlias_tableAlias. + // consistent with item.cc field print. + if (table_ptr->referencing_view) + { + if (aliasSet.find(string(table_ptr->referencing_view->alias) + "_" + + string(table_ptr->alias)) != aliasSet.end()) + continue; + + if (!firstTb) + create_query += ", "; + + create_query += string(table_ptr->db) + "." + string(table_ptr->table_name) + string(" "); + create_query += string(" `") + + escapeBackTick(table_ptr->referencing_view->alias) + "_" + + escapeBackTick(table_ptr->alias) + string("`"); + aliasSet.insert(string(table_ptr->referencing_view->alias) + "_" + + string(table_ptr->alias)); + } + else + { + if (aliasSet.find(table_ptr->alias) != aliasSet.end()) + continue; + + if (!firstTb) + create_query += ", "; + + create_query += string(table_ptr->db) + "." + string(table_ptr->table_name) + string(" "); + create_query += string("`") + escapeBackTick(table_ptr->alias) + string("`"); + aliasSet.insert(table_ptr->alias); + } + + firstTb = false; + } + } + + + gwi.thd->infinidb_vtable.create_vtable_query.free(); + gwi.thd->infinidb_vtable.create_vtable_query.append(create_query.c_str(), create_query.length()); + gwi.thd->infinidb_vtable.vtable_state = THD::INFINIDB_REDO_PHASE1; // redo phase 1 + + // turn off select distinct from post process unless there're post process functions + // on the select list. + string sel_query = "select "; + + if (gi.groupByDistinct && redo) + sel_query = "select distinct "; + else + sel_query = "select "; + + // select * from derived table... + if (gwi.selectCols.size() == 0) + sel_query += " * "; + + for (uint32_t i = 0; i < gwi.selectCols.size(); i++) + { + sel_query += gwi.selectCols[i]; + + if ( i + 1 != gwi.selectCols.size()) + sel_query += ", "; + } + + select_query.replace(lower_select_query.find("select *"), string("select *").length(), sel_query); + } + else + { + // remove order by clause in case this phase has been executed before. + // need a better fix later, like skip all the other non-optimized phase. + size_t pos = lower_select_query.find("order by"); + + if (pos != string::npos) + select_query.replace(pos, lower_select_query.length() - pos, ""); + + //select_query = "select * from " + vtb.str(); + // MCOL-1052 + if (unionSel) + { + ordercol = reinterpret_cast(gi.groupByOrder); + //order_list = gi.groupByOrder; + } + else + ordercol = 0; + + ord_cols = ""; + + for (; ordercol; ordercol = ordercol->next) + { + Item* ord_item = *(ordercol->item); + + // @bug 1706. re-construct the order by item one by one, because the ord_cols constucted so far + // is for REDO phase. + if (ord_cols.length() != 0) + ord_cols += ", "; + + if (ordercol->in_field_list && ordercol->counter_used) + { + ostringstream oss; + oss << ordercol->counter; + ord_cols += oss.str(); + } + else if (ord_item->type() == Item::NULL_ITEM) + { + // MCOL-793 Do nothing for an ORDER BY NULL + } + else if (ord_item->type() == Item::SUM_FUNC_ITEM) + { + Item_sum* ifp = (Item_sum*)(*(ordercol->item)); + ReturnedColumn* fc = buildAggregateColumn(ifp, gwi); + + for (uint32_t i = 0; i < gwi.returnedCols.size(); i++) + { + if (fc->operator==(gwi.returnedCols[i].get())) + { + ostringstream oss; + oss << i + 1; + ord_cols += oss.str(); + break; + } + } + + //continue; + } + // @bug 3518. if order by clause = selected column, use position. + else if (ord_item->name && ord_item->type() == Item::FIELD_ITEM) + { + Item_field* field = reinterpret_cast(ord_item); + string fullname; + + if (field->db_name) + fullname += string(field->db_name) + "."; + + if (field->table_name) + fullname += string(field->table_name) + "."; + + if (field->field_name) + fullname += string(field->field_name); + + uint32_t i = 0; + + for (i = 0; i < gwi.returnedCols.size(); i++) + { + SimpleColumn* sc = dynamic_cast(gwi.returnedCols[i].get()); + + if (sc && ((Item_field*)ord_item)->cached_table && + (strcasecmp(getViewName(((Item_field*)ord_item)->cached_table).c_str(), sc->viewName().c_str()) != 0)) + continue; + + if (strcasecmp(fullname.c_str(), gwi.returnedCols[i]->alias().c_str()) == 0 || + strcasecmp(ord_item->name, gwi.returnedCols[i]->alias().c_str()) == 0) + { + ostringstream oss; + oss << i + 1; + ord_cols += oss.str(); + break; + } + } + + if (i == gwi.returnedCols.size()) + ord_cols += string(" `") + escapeBackTick(ord_item->name) + '`'; + } + + else if (ord_item->name) + { + // for union order by 1 case. For unknown reason, it doesn't show in_field_list + if (ord_item->type() == Item::INT_ITEM) + { + ord_cols += ord_item->name; + } + else if (ord_item->type() == Item::SUBSELECT_ITEM) + { + string emsg = IDBErrorInfo::instance()->errorMsg(ERR_NON_SUPPORT_ORDER_BY); + setError(gwi.thd, ER_CHECK_NOT_IMPLEMENTED, emsg, gwi); + return ER_CHECK_NOT_IMPLEMENTED; + } + else + { + ord_cols += string(" `") + escapeBackTick(ord_item->name) + '`'; + } + } + else if (ord_item->type() == Item::FUNC_ITEM) + { + // @bug5636. check if this order by column is on the select list + ReturnedColumn* rc = buildFunctionColumn((Item_func*)(ord_item), gwi, gwi.fatalParseError); + + for (uint32_t i = 0; i < gwi.returnedCols.size(); i++) + { + if (rc && rc->operator==(gwi.returnedCols[i].get())) + { + ostringstream oss; + oss << i + 1; + ord_cols += oss.str(); + break; + } + } + } + else + { + String str; + ord_item->print(&str, QT_INFINIDB_NO_QUOTE); + ord_cols += string(str.c_ptr()); + } + + if (ordercol->direction != ORDER::ORDER_ASC) + ord_cols += " desc"; + } + } + + if (ord_cols.length() > 0) // has order by + { + gwi.thd->infinidb_vtable.has_order_by = true; + csep->hasOrderBy(true); + ord_cols = " order by " + ord_cols; + select_query += ord_cols; + } + } + } // ORDER BY processing ends here + + if ( gi.groupByDistinct ) + csep->distinct(true); + + // add the smallest column to count(*) parm. + // select constant in subquery case + std::vector::iterator coliter; + + if (!minSc) + { + if (!gwi.returnedCols.empty()) + minSc = gwi.returnedCols[0]; + else if (!gwi.additionalRetCols.empty()) + minSc = gwi.additionalRetCols[0]; + } + + // @bug3523, count(*) on subquery always pick column[0]. + SimpleColumn* sc = dynamic_cast(minSc.get()); + + if (sc && sc->schemaName().empty()) + { + if (gwi.derivedTbList.size() >= 1) + { + SimpleColumn* sc1 = new SimpleColumn(); + sc1->columnName(sc->columnName()); + sc1->tableName(sc->tableName()); + sc1->tableAlias(sc->tableAlias()); + sc1->viewName(lower(sc->viewName())); + sc1->colPosition(0); + minSc.reset(sc1); + } + } + + for (coliter = gwi.count_asterisk_list.begin(); coliter != gwi.count_asterisk_list.end(); ++coliter) + { + // @bug5977 @note should never throw this, but checking just in case. + // When ExeMgr fix is ready, this should not error out... + if (dynamic_cast(minSc.get())) + { + gwi.fatalParseError = true; + gwi.parseErrorText = "No project column found for aggregate function"; + setError(gwi.thd, ER_INTERNAL_ERROR, gwi.parseErrorText, gwi); + return ER_CHECK_NOT_IMPLEMENTED; + } + + (*coliter)->functionParms(minSc); + } + + std::vector::iterator funciter; + + SPTP sptp(new ParseTree(minSc.get()->clone())); + + for (funciter = gwi.no_parm_func_list.begin(); funciter != gwi.no_parm_func_list.end(); ++funciter) + { + FunctionParm funcParms = (*funciter)->functionParms(); + funcParms.push_back(sptp); + (*funciter)->functionParms(funcParms); + } + + // set sequence# for subquery localCols + for (uint32_t i = 0; i < gwi.localCols.size(); i++) + gwi.localCols[i]->sequence(i); + + // append additionalRetCols to returnedCols + gwi.returnedCols.insert(gwi.returnedCols.begin(), gwi.additionalRetCols.begin(), + gwi.additionalRetCols.end()); + + csep->groupByCols(gwi.groupByCols); + csep->orderByCols(gwi.orderByCols); + csep->returnedCols(gwi.returnedCols); + csep->columnMap(gwi.columnMap); + csep->having(havingFilter); + csep->derivedTableList(gwi.derivedTbList); + csep->selectSubList(selectSubList); + csep->subSelectList(gwi.subselectList); + gwi.thd->infinidb_vtable.duplicate_field_name = false; + clearStacks(gwi); + return 0; +} + } From bb06ac4de4fef4bf76178242b1959a68fd9f3969 Mon Sep 17 00:00:00 2001 From: Andrew Hutchings Date: Wed, 9 May 2018 09:47:07 +0100 Subject: [PATCH 53/57] Fix another merge error --- dbcon/mysql/ha_calpont_execplan.cpp | 31 +++++++++++++++++++++++++++++ 1 file changed, 31 insertions(+) diff --git a/dbcon/mysql/ha_calpont_execplan.cpp b/dbcon/mysql/ha_calpont_execplan.cpp index fd8ae8922..eab807994 100644 --- a/dbcon/mysql/ha_calpont_execplan.cpp +++ b/dbcon/mysql/ha_calpont_execplan.cpp @@ -5044,8 +5044,14 @@ void gp_walk(const Item* item, void* arg) gwip->clauseType = SELECT; if (col->type() != Item::COND_ITEM) + { rc = buildReturnedColumn(col, *gwip, gwip->fatalParseError); + if ( col->type() == Item::FIELD_ITEM ) + gwip->fatalParseError = false; + } + + SimpleColumn* sc = dynamic_cast(rc); if (sc) @@ -5116,6 +5122,31 @@ void gp_walk(const Item* item, void* arg) Item_func* ifp = (Item_func*)col; gwip->ptWorkStack.push(buildParseTree(ifp, *gwip, gwip->fatalParseError)); } + else if (col->type() == Item::FIELD_ITEM && gwip->clauseType == HAVING) + { + Item_field* ifip = static_cast(col); + std::vector::iterator iter = gwip->havingAggColsItems.begin(); + Item_func_or_sum* isfp = NULL; + + for ( ; iter != gwip->havingAggColsItems.end(); iter++ ) + { + Item* temp_isfp = *iter; + isfp = reinterpret_cast(temp_isfp); + + if ( isfp->type() == Item::SUM_FUNC_ITEM && + isfp->result_field == ifip->field ) + { + ReturnedColumn* rc = buildAggregateColumn(isfp, *gwip); + + if (rc) + gwip->rcWorkStack.push(rc); + + break; + } + } + + break; + } else cando = false; From 6c22502cb921d467dc0ad558c17caf7ff32d1b5d Mon Sep 17 00:00:00 2001 From: Andrew Hutchings Date: Wed, 9 May 2018 09:55:24 +0100 Subject: [PATCH 54/57] More fixes due to bad merging --- dbcon/mysql/ha_calpont_ddl.cpp | 4 + dbcon/mysql/ha_calpont_execplan.cpp | 331 +++++++++++++++------------- 2 files changed, 179 insertions(+), 156 deletions(-) diff --git a/dbcon/mysql/ha_calpont_ddl.cpp b/dbcon/mysql/ha_calpont_ddl.cpp index bdce5e4ce..fa8ef79dd 100644 --- a/dbcon/mysql/ha_calpont_ddl.cpp +++ b/dbcon/mysql/ha_calpont_ddl.cpp @@ -174,6 +174,10 @@ uint32_t convertDataType(int dataType) calpontDataType = CalpontSystemCatalog::DATETIME; break; + case ddlpackage::DDL_TIME: + calpontDataType = CalpontSystemCatalog::TIME; + break; + case ddlpackage::DDL_CLOB: calpontDataType = CalpontSystemCatalog::CLOB; break; diff --git a/dbcon/mysql/ha_calpont_execplan.cpp b/dbcon/mysql/ha_calpont_execplan.cpp index eab807994..fac0cd032 100644 --- a/dbcon/mysql/ha_calpont_execplan.cpp +++ b/dbcon/mysql/ha_calpont_execplan.cpp @@ -221,7 +221,7 @@ void debug_walk(const Item* item, void* arg) case Item::FIELD_ITEM: { Item_field* ifp = (Item_field*)item; - cout << "FIELD_ITEM: " << (ifp->db_name ? ifp->db_name : "") << '.' << bestTableName(ifp) << + cerr << "FIELD_ITEM: " << (ifp->db_name ? ifp->db_name : "") << '.' << bestTableName(ifp) << '.' << ifp->field_name << endl; break; } @@ -229,10 +229,10 @@ void debug_walk(const Item* item, void* arg) case Item::INT_ITEM: { Item_int* iip = (Item_int*)item; - cout << "INT_ITEM: "; + cerr << "INT_ITEM: "; - if (iip->name) cout << iip->name << " (from name string)" << endl; - else cout << iip->val_int() << endl; + if (iip->name) cerr << iip->name << " (from name string)" << endl; + else cerr << iip->val_int() << endl; break; } @@ -243,19 +243,19 @@ void debug_walk(const Item* item, void* arg) String val, *str = isp->val_str(&val); string valStr; valStr.assign(str->ptr(), str->length()); - cout << "STRING_ITEM: >" << valStr << '<' << endl; + cerr << "STRING_ITEM: >" << valStr << '<' << endl; break; } case Item::REAL_ITEM: { - cout << "REAL_ITEM" << endl; + cerr << "REAL_ITEM" << endl; break; } case Item::DECIMAL_ITEM: { - cout << "DECIMAL_ITEM" << endl; + cerr << "DECIMAL_ITEM" << endl; break; } @@ -263,85 +263,85 @@ void debug_walk(const Item* item, void* arg) { Item_func* ifp = (Item_func*)item; Item_func_opt_neg* inp; - cout << "FUNC_ITEM: "; + cerr << "FUNC_ITEM: "; switch (ifp->functype()) { case Item_func::UNKNOWN_FUNC: // 0 - cout << ifp->func_name() << " (" << ifp->functype() << ")" << endl; + cerr << ifp->func_name() << " (" << ifp->functype() << ")" << endl; break; case Item_func::GT_FUNC: // 7 - cout << '>' << " (" << ifp->functype() << ")" << endl; + cerr << '>' << " (" << ifp->functype() << ")" << endl; break; case Item_func::EQ_FUNC: // 1 - cout << '=' << " (" << ifp->functype() << ")" << endl; + cerr << '=' << " (" << ifp->functype() << ")" << endl; break; case Item_func::GE_FUNC: - cout << ">=" << " (" << ifp->functype() << ")" << endl; + cerr << ">=" << " (" << ifp->functype() << ")" << endl; break; case Item_func::LE_FUNC: - cout << "<=" << " (" << ifp->functype() << ")" << endl; + cerr << "<=" << " (" << ifp->functype() << ")" << endl; break; case Item_func::LT_FUNC: - cout << '<' << " (" << ifp->functype() << ")" << endl; + cerr << '<' << " (" << ifp->functype() << ")" << endl; break; case Item_func::NE_FUNC: - cout << "<>" << " (" << ifp->functype() << ")" << endl; + cerr << "<>" << " (" << ifp->functype() << ")" << endl; break; case Item_func::NEG_FUNC: // 45 - cout << "unary minus" << " (" << ifp->functype() << ")" << endl; + cerr << "unary minus" << " (" << ifp->functype() << ")" << endl; break; case Item_func::IN_FUNC: // 16 inp = (Item_func_opt_neg*)ifp; - if (inp->negated) cout << "not "; + if (inp->negated) cerr << "not "; - cout << "in" << " (" << ifp->functype() << ")" << endl; + cerr << "in" << " (" << ifp->functype() << ")" << endl; break; case Item_func::BETWEEN: inp = (Item_func_opt_neg*)ifp; - if (inp->negated) cout << "not "; + if (inp->negated) cerr << "not "; - cout << "between" << " (" << ifp->functype() << ")" << endl; + cerr << "between" << " (" << ifp->functype() << ")" << endl; break; case Item_func::ISNULL_FUNC: // 10 - cout << "is null" << " (" << ifp->functype() << ")" << endl; + cerr << "is null" << " (" << ifp->functype() << ")" << endl; break; case Item_func::ISNOTNULL_FUNC: // 11 - cout << "is not null" << " (" << ifp->functype() << ")" << endl; + cerr << "is not null" << " (" << ifp->functype() << ")" << endl; break; case Item_func::NOT_ALL_FUNC: - cout << "not_all" << " (" << ifp->functype() << ")" << endl; + cerr << "not_all" << " (" << ifp->functype() << ")" << endl; break; case Item_func::NOT_FUNC: - cout << "not_func" << " (" << ifp->functype() << ")" << endl; + cerr << "not_func" << " (" << ifp->functype() << ")" << endl; break; case Item_func::TRIG_COND_FUNC: - cout << "trig_cond_func" << " (" << ifp->functype() << ")" << endl; + cerr << "trig_cond_func" << " (" << ifp->functype() << ")" << endl; break; case Item_func::ISNOTNULLTEST_FUNC: - cout << "isnotnulltest_func" << " (" << ifp->functype() << ")" << endl; + cerr << "isnotnulltest_func" << " (" << ifp->functype() << ")" << endl; break; case Item_func::MULT_EQUAL_FUNC: { - cout << "mult_equal_func:" << " (" << ifp->functype() << ")" << endl; + cerr << "mult_equal_func:" << " (" << ifp->functype() << ")" << endl; Item_equal* item_eq = (Item_equal*)ifp; Item_equal_fields_iterator it(*item_eq); Item* item; @@ -349,142 +349,142 @@ void debug_walk(const Item* item, void* arg) while ((item = it++)) { Field* equal_field = it.get_curr_field(); - cout << equal_field->field_name << endl; + cerr << equal_field->field_name << endl; } break; } case Item_func::EQUAL_FUNC: - cout << "equal func" << " (" << ifp->functype() << ")" << endl; + cerr << "equal func" << " (" << ifp->functype() << ")" << endl; break; case Item_func::FT_FUNC: - cout << "ft func" << " (" << ifp->functype() << ")" << endl; + cerr << "ft func" << " (" << ifp->functype() << ")" << endl; break; case Item_func::LIKE_FUNC: - cout << "like func" << " (" << ifp->functype() << ")" << endl; + cerr << "like func" << " (" << ifp->functype() << ")" << endl; break; case Item_func::COND_AND_FUNC: - cout << "cond and func" << " (" << ifp->functype() << ")" << endl; + cerr << "cond and func" << " (" << ifp->functype() << ")" << endl; break; case Item_func::COND_OR_FUNC: - cout << "cond or func" << " (" << ifp->functype() << ")" << endl; + cerr << "cond or func" << " (" << ifp->functype() << ")" << endl; break; case Item_func::XOR_FUNC: - cout << "xor func" << " (" << ifp->functype() << ")" << endl; + cerr << "xor func" << " (" << ifp->functype() << ")" << endl; break; case Item_func::INTERVAL_FUNC: - cout << "interval func" << " (" << ifp->functype() << ")" << endl; + cerr << "interval func" << " (" << ifp->functype() << ")" << endl; break; case Item_func::SP_EQUALS_FUNC: - cout << "sp equals func" << " (" << ifp->functype() << ")" << endl; + cerr << "sp equals func" << " (" << ifp->functype() << ")" << endl; break; case Item_func::SP_DISJOINT_FUNC: - cout << "sp disjoint func" << " (" << ifp->functype() << ")" << endl; + cerr << "sp disjoint func" << " (" << ifp->functype() << ")" << endl; break; case Item_func::SP_INTERSECTS_FUNC: - cout << "sp intersects func" << " (" << ifp->functype() << ")" << endl; + cerr << "sp intersects func" << " (" << ifp->functype() << ")" << endl; break; case Item_func::SP_TOUCHES_FUNC: - cout << "sp touches func" << " (" << ifp->functype() << ")" << endl; + cerr << "sp touches func" << " (" << ifp->functype() << ")" << endl; break; case Item_func::SP_CROSSES_FUNC: - cout << "sp crosses func" << " (" << ifp->functype() << ")" << endl; + cerr << "sp crosses func" << " (" << ifp->functype() << ")" << endl; break; case Item_func::SP_WITHIN_FUNC: - cout << "sp within func" << " (" << ifp->functype() << ")" << endl; + cerr << "sp within func" << " (" << ifp->functype() << ")" << endl; break; case Item_func::SP_CONTAINS_FUNC: - cout << "sp contains func" << " (" << ifp->functype() << ")" << endl; + cerr << "sp contains func" << " (" << ifp->functype() << ")" << endl; break; case Item_func::SP_OVERLAPS_FUNC: - cout << "sp overlaps func" << " (" << ifp->functype() << ")" << endl; + cerr << "sp overlaps func" << " (" << ifp->functype() << ")" << endl; break; case Item_func::SP_STARTPOINT: - cout << "sp startpoint func" << " (" << ifp->functype() << ")" << endl; + cerr << "sp startpoint func" << " (" << ifp->functype() << ")" << endl; break; case Item_func::SP_ENDPOINT: - cout << "sp endpoint func" << " (" << ifp->functype() << ")" << endl; + cerr << "sp endpoint func" << " (" << ifp->functype() << ")" << endl; break; case Item_func::SP_EXTERIORRING: - cout << "sp exteriorring func" << " (" << ifp->functype() << ")" << endl; + cerr << "sp exteriorring func" << " (" << ifp->functype() << ")" << endl; break; case Item_func::SP_POINTN: - cout << "sp pointn func" << " (" << ifp->functype() << ")" << endl; + cerr << "sp pointn func" << " (" << ifp->functype() << ")" << endl; break; case Item_func::SP_GEOMETRYN: - cout << "sp geometryn func" << " (" << ifp->functype() << ")" << endl; + cerr << "sp geometryn func" << " (" << ifp->functype() << ")" << endl; break; case Item_func::SP_INTERIORRINGN: - cout << "sp exteriorringn func" << " (" << ifp->functype() << ")" << endl; + cerr << "sp exteriorringn func" << " (" << ifp->functype() << ")" << endl; break; case Item_func::SP_RELATE_FUNC: - cout << "sp relate func" << " (" << ifp->functype() << ")" << endl; + cerr << "sp relate func" << " (" << ifp->functype() << ")" << endl; break; case Item_func::NOW_FUNC: - cout << "now func" << " (" << ifp->functype() << ")" << endl; + cerr << "now func" << " (" << ifp->functype() << ")" << endl; break; case Item_func::SUSERVAR_FUNC: - cout << "suservar func" << " (" << ifp->functype() << ")" << endl; + cerr << "suservar func" << " (" << ifp->functype() << ")" << endl; break; case Item_func::GUSERVAR_FUNC: - cout << "guservar func" << " (" << ifp->functype() << ")" << endl; + cerr << "guservar func" << " (" << ifp->functype() << ")" << endl; break; case Item_func::COLLATE_FUNC: - cout << "collate func" << " (" << ifp->functype() << ")" << endl; + cerr << "collate func" << " (" << ifp->functype() << ")" << endl; break; case Item_func::EXTRACT_FUNC: - cout << "extract func" << " (" << ifp->functype() << ")" << endl; + cerr << "extract func" << " (" << ifp->functype() << ")" << endl; break; case Item_func::CHAR_TYPECAST_FUNC: - cout << "char typecast func" << " (" << ifp->functype() << ")" << endl; + cerr << "char typecast func" << " (" << ifp->functype() << ")" << endl; break; case Item_func::FUNC_SP: - cout << "func sp func" << " (" << ifp->functype() << ")" << endl; + cerr << "func sp func" << " (" << ifp->functype() << ")" << endl; break; case Item_func::UDF_FUNC: - cout << "udf func" << " (" << ifp->functype() << ")" << endl; + cerr << "udf func" << " (" << ifp->functype() << ")" << endl; break; case Item_func::GSYSVAR_FUNC: - cout << "gsysvar func" << " (" << ifp->functype() << ")" << endl; + cerr << "gsysvar func" << " (" << ifp->functype() << ")" << endl; break; case Item_func::DYNCOL_FUNC: - cout << "dyncol func" << " (" << ifp->functype() << ")" << endl; + cerr << "dyncol func" << " (" << ifp->functype() << ")" << endl; break; default: - cout << "type=" << ifp->functype() << endl; + cerr << "type=" << ifp->functype() << endl; break; } @@ -494,7 +494,7 @@ void debug_walk(const Item* item, void* arg) case Item::COND_ITEM: { Item_cond* icp = (Item_cond*)item; - cout << "COND_ITEM: " << icp->func_name() << endl; + cerr << "COND_ITEM: " << icp->func_name() << endl; break; } @@ -503,7 +503,17 @@ void debug_walk(const Item* item, void* arg) Item_sum* isp = (Item_sum*)item; char* item_name = item->name; - if (!item_name) + // MCOL-1052 This is an extended SELECT list item + if (!item_name && isp->get_arg_count() && isp->get_arg(0)->name) + { + item_name = isp->get_arg(0)->name; + } + else if (!item_name && isp->get_arg_count() + && isp->get_arg(0)->type() == Item::INT_ITEM) + { + item_name = (char*)"INT||*"; + } + else if (!item_name) { item_name = (char*)""; } @@ -511,39 +521,39 @@ void debug_walk(const Item* item, void* arg) switch (isp->sum_func()) { case Item_sum::SUM_FUNC: - cout << "SUM_FUNC: " << item_name << endl; + cerr << "SUM_FUNC: " << item_name << endl; break; case Item_sum::SUM_DISTINCT_FUNC: - cout << "SUM_DISTINCT_FUNC: " << item_name << endl; + cerr << "SUM_DISTINCT_FUNC: " << item_name << endl; break; case Item_sum::AVG_FUNC: - cout << "AVG_FUNC: " << item_name << endl; + cerr << "AVG_FUNC: " << item_name << endl; break; case Item_sum::COUNT_FUNC: - cout << "COUNT_FUNC: " << item_name << endl; + cerr << "COUNT_FUNC: " << item_name << endl; break; case Item_sum::COUNT_DISTINCT_FUNC: - cout << "COUNT_DISTINCT_FUNC: " << item_name << endl; + cerr << "COUNT_DISTINCT_FUNC: " << item_name << endl; break; case Item_sum::MIN_FUNC: - cout << "MIN_FUNC: " << item_name << endl; + cerr << "MIN_FUNC: " << item_name << endl; break; case Item_sum::MAX_FUNC: - cout << "MAX_FUNC: " << item_name << endl; + cerr << "MAX_FUNC: " << item_name << endl; break; case Item_sum::UDF_SUM_FUNC: - cout << "UDAF_FUNC: " << item_name << endl; + cerr << "UDAF_FUNC: " << item_name << endl; break; default: - cout << "SUM_FUNC_ITEM type=" << isp->sum_func() << endl; + cerr << "SUM_FUNC_ITEM type=" << isp->sum_func() << endl; break; } @@ -553,24 +563,24 @@ void debug_walk(const Item* item, void* arg) case Item::SUBSELECT_ITEM: { Item_subselect* sub = (Item_subselect*)item; - cout << "SUBSELECT Item: "; + cerr << "SUBSELECT Item: "; switch (sub->substype()) { case Item_subselect::EXISTS_SUBS: - cout << "EXISTS"; + cerr << "EXISTS"; break; case Item_subselect::IN_SUBS: - cout << "IN"; + cerr << "IN"; break; default: - cout << sub->substype(); + cerr << sub->substype(); break; } - cout << endl; + cerr << endl; JOIN* join = sub->get_select_lex()->join; if (join) @@ -581,7 +591,7 @@ void debug_walk(const Item* item, void* arg) cond->traverse_cond(debug_walk, arg, Item::POSTFIX); } - cout << "Finish subselect item traversing" << endl; + cerr << "Finish subselect item traversing" << endl; break; } @@ -599,14 +609,14 @@ void debug_walk(const Item* item, void* arg) //ifp->cached_table->select_lex->select_number gives the select level. // could be used on alias. // could also be used to tell correlated join (equal level). - cout << "CACHED REF FIELD_ITEM: " << ifp->db_name << '.' << bestTableName(ifp) << + cerr << "CACHED REF FIELD_ITEM: " << ifp->db_name << '.' << bestTableName(ifp) << '.' << ifp->field_name << endl; break; } else if (field->type() == Item::FUNC_ITEM) { Item_func* ifp = (Item_func*)field; - cout << "CACHED REF FUNC_ITEM " << ifp->func_name() << endl; + cerr << "CACHED REF FUNC_ITEM " << ifp->func_name() << endl; } else if (field->type() == Item::REF_ITEM) { @@ -686,34 +696,45 @@ void debug_walk(const Item* item, void* arg) } } - cout << "CACHED REF_ITEM: ref type " << refType.c_str() << " real type " << realType.c_str() << endl; + cerr << "CACHED REF_ITEM: ref type " << refType.c_str() << " real type " << realType.c_str() << endl; break; } else { - cout << "REF_ITEM with CACHE_ITEM type unknown " << field->type() << endl; + cerr << "REF_ITEM with CACHE_ITEM type unknown " << field->type() << endl; } } else if (ref->real_item()->type() == Item::FIELD_ITEM) { Item_field* ifp = (Item_field*)ref->real_item(); - cout << "REF FIELD_ITEM: " << ifp->db_name << '.' << bestTableName(ifp) << '.' << - ifp->field_name << endl; + + // MCOL-1052 The field referenced presumable came from + // extended SELECT list. + if ( !ifp->field_name ) + { + cerr << "REF extra FIELD_ITEM: " << ifp->name << endl; + } + else + { + cerr << "REF FIELD_ITEM: " << ifp->db_name << '.' << bestTableName(ifp) << '.' << + ifp->field_name << endl; + } + break; } else if (ref->real_item()->type() == Item::FUNC_ITEM) { Item_func* ifp = (Item_func*)ref->real_item(); - cout << "REF FUNC_ITEM " << ifp->func_name() << endl; + cerr << "REF FUNC_ITEM " << ifp->func_name() << endl; } else if (ref->real_item()->type() == Item::WINDOW_FUNC_ITEM) { Item_window_func* ifp = (Item_window_func*)ref->real_item(); - cout << "REF WINDOW_FUNC_ITEM " << ifp->window_func()->func_name() << endl; + cerr << "REF WINDOW_FUNC_ITEM " << ifp->window_func()->func_name() << endl; } else { - cout << "UNKNOWN REF ITEM type " << ref->real_item()->type() << endl; + cerr << "UNKNOWN REF ITEM type " << ref->real_item()->type() << endl; } break; @@ -722,7 +743,7 @@ void debug_walk(const Item* item, void* arg) case Item::ROW_ITEM: { Item_row* row = (Item_row*)item; - cout << "ROW_ITEM: " << endl; + cerr << "ROW_ITEM: " << endl; for (uint32_t i = 0; i < row->cols(); i++) debug_walk(row->element_index(i), 0); @@ -732,7 +753,7 @@ void debug_walk(const Item* item, void* arg) case Item::EXPR_CACHE_ITEM: { - cout << "Expr Cache Item" << endl; + cerr << "Expr Cache Item" << endl; ((Item_cache_wrapper*)item)->get_orig_item()->traverse_cond(debug_walk, arg, Item::POSTFIX); break; } @@ -747,27 +768,27 @@ void debug_walk(const Item* item, void* arg) switch (item->result_type()) { case STRING_RESULT: - cout << "CACHE_STRING_ITEM" << endl; + cerr << "CACHE_STRING_ITEM" << endl; break; case REAL_RESULT: - cout << "CACHE_REAL_ITEM " << isp->val_real() << endl; + cerr << "CACHE_REAL_ITEM " << isp->val_real() << endl; break; case INT_RESULT: - cout << "CACHE_INT_ITEM " << isp->val_int() << endl; + cerr << "CACHE_INT_ITEM " << isp->val_int() << endl; break; case ROW_RESULT: - cout << "CACHE_ROW_ITEM" << endl; + cerr << "CACHE_ROW_ITEM" << endl; break; case DECIMAL_RESULT: - cout << "CACHE_DECIMAL_ITEM " << isp->val_decimal() << endl; + cerr << "CACHE_DECIMAL_ITEM " << isp->val_decimal() << endl; break; default: - cout << "CACHE_UNKNOWN_ITEM" << endl; + cerr << "CACHE_UNKNOWN_ITEM" << endl; break; } @@ -780,7 +801,7 @@ void debug_walk(const Item* item, void* arg) //ifp->cached_table->select_lex->select_number gives the select level. // could be used on alias. // could also be used to tell correlated join (equal level). - cout << "CACHED FIELD_ITEM: " << ifp->db_name << '.' << bestTableName(ifp) << + cerr << "CACHED FIELD_ITEM: " << ifp->db_name << '.' << bestTableName(ifp) << '.' << ifp->field_name << endl; break; } @@ -862,18 +883,18 @@ void debug_walk(const Item* item, void* arg) } } - cout << "CACHE_ITEM ref type " << refType.c_str() << " real type " << realType.c_str() << endl; + cerr << "CACHE_ITEM ref type " << refType.c_str() << " real type " << realType.c_str() << endl; break; } else if (field->type() == Item::FUNC_ITEM) { Item_func* ifp = (Item_func*)field; - cout << "CACHE_ITEM FUNC_ITEM " << ifp->func_name() << endl; + cerr << "CACHE_ITEM FUNC_ITEM " << ifp->func_name() << endl; break; } else { - cout << "CACHE_ITEM type unknown " << field->type() << endl; + cerr << "CACHE_ITEM type unknown " << field->type() << endl; } break; @@ -884,12 +905,12 @@ void debug_walk(const Item* item, void* arg) String val, *str = NULL; Item_temporal_literal* itp = (Item_temporal_literal*)item; str = itp->val_str(&val); - cout << "DATE ITEM: "; + cerr << "DATE ITEM: "; if (str) - cout << ": (" << str->ptr() << ')' << endl; + cerr << ": (" << str->ptr() << ')' << endl; else - cout << ": " << endl; + cerr << ": " << endl; break; } @@ -897,13 +918,19 @@ void debug_walk(const Item* item, void* arg) case Item::WINDOW_FUNC_ITEM: { Item_window_func* ifp = (Item_window_func*)item; - cout << "Window Function Item " << ifp->window_func()->func_name() << endl; + cerr << "Window Function Item " << ifp->window_func()->func_name() << endl; + break; + } + + case Item::NULL_ITEM: + { + cerr << "NULL item" << endl; break; } default: { - cout << "UNKNOWN_ITEM type " << item->type() << endl; + cerr << "UNKNOWN_ITEM type " << item->type() << endl; break; } } @@ -1015,11 +1042,11 @@ uint32_t buildOuterJoin(gp_walk_info& gwi, SELECT_LEX& select_lex) #ifdef DEBUG_WALK_COND if (table_ptr->alias) - cout << table_ptr->alias ; + cerr << table_ptr->alias ; else if (table_ptr->alias) - cout << table_ptr->alias; + cerr << table_ptr->alias; - cout << " outer table expression: " << endl; + cerr << " outer table expression: " << endl; expr->traverse_cond(debug_walk, &gwi_outer, Item::POSTFIX); #endif expr->traverse_cond(gp_walk, &gwi_outer, Item::POSTFIX); @@ -1053,13 +1080,13 @@ uint32_t buildOuterJoin(gp_walk_info& gwi, SELECT_LEX& select_lex) Item_cond* expr = reinterpret_cast(table_ptr->embedding->on_expr); #ifdef DEBUG_WALK_COND - cout << "inner tables: " << endl; + cerr << "inner tables: " << endl; set::const_iterator it; for (it = gwi_outer.innerTables.begin(); it != gwi_outer.innerTables.end(); ++it) - cout << (*it) << " "; + cerr << (*it) << " "; - cout << endl; + cerr << endl; expr->traverse_cond(debug_walk, &gwi_outer, Item::POSTFIX); #endif expr->traverse_cond(gp_walk, &gwi_outer, Item::POSTFIX); @@ -1485,7 +1512,7 @@ bool buildPredicateItem(Item_func* ifp, gp_walk_info* gwip) // @bug5811. This filter string is for cross engine to use. // Use real table name. ifp->print(&str, QT_INFINIDB_DERIVED); - IDEBUG(cout << str.ptr() << endl); + IDEBUG(cerr << str.ptr() << endl); if (str.ptr()) cf->data(str.c_ptr()); @@ -1811,7 +1838,7 @@ bool buildPredicateItem(Item_func* ifp, gp_walk_info* gwip) (current_thd->lex->sql_command == SQLCOM_DELETE) || (current_thd->lex->sql_command == SQLCOM_DELETE_MULTI))) { - IDEBUG( cout << "deleted func with 2 const columns" << endl ); + IDEBUG( cerr << "deleted func with 2 const columns" << endl ); delete rhs; delete lhs; return false; @@ -1865,7 +1892,7 @@ bool buildPredicateItem(Item_func* ifp, gp_walk_info* gwip) if (notInner) { lhs->returnAll(true); - IDEBUG( cout << "setting returnAll on " << tan_lhs << endl); + IDEBUG( cerr << "setting returnAll on " << tan_lhs << endl); } } @@ -1882,7 +1909,7 @@ bool buildPredicateItem(Item_func* ifp, gp_walk_info* gwip) if (notInner) { rhs->returnAll(true); - IDEBUG( cout << "setting returnAll on " << tan_rhs << endl ); + IDEBUG( cerr << "setting returnAll on " << tan_rhs << endl ); } } @@ -2591,7 +2618,7 @@ CalpontSystemCatalog::ColType fieldType_MysqlToIDB (const Field* field) break; default: - IDEBUG( cout << "fieldType_MysqlToIDB:: Unknown result type of MySQL " + IDEBUG( cerr << "fieldType_MysqlToIDB:: Unknown result type of MySQL " << field->result_type() << endl ); break; } @@ -2718,7 +2745,7 @@ ReturnedColumn* buildReturnedColumn(Item* item, gp_walk_info& gwi, bool& nonSupp if ( gwi.thd) { - if ( ((gwi.thd->lex)->sql_command == SQLCOM_UPDATE ) || ((gwi.thd->lex)->sql_command == SQLCOM_UPDATE_MULTI )) + //if ( ((gwi.thd->lex)->sql_command == SQLCOM_UPDATE ) || ((gwi.thd->lex)->sql_command == SQLCOM_UPDATE_MULTI )) { if ( !item->fixed) { @@ -2908,7 +2935,7 @@ ReturnedColumn* buildReturnedColumn(Item* item, gp_walk_info& gwi, bool& nonSupp { // TODO: item is a Item_cache_wrapper printf("EXPR_CACHE_ITEM in buildReturnedColumn\n"); - cout << "EXPR_CACHE_ITEM in buildReturnedColumn" << endl; + cerr << "EXPR_CACHE_ITEM in buildReturnedColumn" << endl; break; } @@ -3513,18 +3540,6 @@ ReturnedColumn* buildFunctionColumn(Item_func* ifp, gp_walk_info& gwi, bool& non fc->functionParms(funcParms); fc->resultType(colType_MysqlToIDB(ifp)); - // add the sign for addtime function - if (funcName == "add_time") - { - Item_func_add_time* addtime = (Item_func_add_time*)ifp; - sptp.reset(new ParseTree(new ConstantColumn((int64_t)addtime->get_sign()))); - funcParms.push_back(sptp); - } - - fc->functionName(funcName); - fc->functionParms(funcParms); - fc->resultType(colType_MysqlToIDB(ifp)); - // MySQL give string result type for date function, but has the flag set. // we should set the result type to be datetime for comparision. if (ifp->field_type() == MYSQL_TYPE_DATETIME || @@ -3994,7 +4009,7 @@ ParseTree* buildParseTree(Item_func* item, gp_walk_info& gwi, bool& nonSupport) Item_cond* icp = (Item_cond*)item; #ifdef DEBUG_WALK_COND // debug - cout << "Build Parsetree: " << endl; + cerr << "Build Parsetree: " << endl; icp->traverse_cond(debug_walk, &gwi, Item::POSTFIX); #endif //@bug5044. PPSTFIX walking should always be treated as WHERE clause filter @@ -4841,7 +4856,7 @@ void gp_walk(const Item* item, void* arg) gwip->rcWorkStack.push(cc); if (str) - IDEBUG( cout << "Const F&E " << item->full_name() << " evaluate: " << valStr << endl ); + IDEBUG( cerr << "Const F&E " << item->full_name() << " evaluate: " << valStr << endl ); break; } @@ -5051,7 +5066,6 @@ void gp_walk(const Item* item, void* arg) gwip->fatalParseError = false; } - SimpleColumn* sc = dynamic_cast(rc); if (sc) @@ -5432,7 +5446,7 @@ void parse_item (Item* item, vector& field_vec, bool& hasNonSupport } else { - cout << "UNKNOWN REF Item" << endl; + cerr << "UNKNOWN REF Item" << endl; break; } } @@ -5504,7 +5518,7 @@ bool isInfiniDB(TABLE* table_ptr) int getSelectPlan(gp_walk_info& gwi, SELECT_LEX& select_lex, SCSEP& csep, bool isUnion) { #ifdef DEBUG_WALK_COND - cout << "getSelectPlan()" << endl; + cerr << "getSelectPlan()" << endl; #endif // by pass the derived table resolve phase of mysql @@ -5591,7 +5605,7 @@ int getSelectPlan(gp_walk_info& gwi, SELECT_LEX& select_lex, SCSEP& csep, bool i while ((sj_nest = sj_list_it++)) { - cout << sj_nest->db << "." << sj_nest->table_name << endl; + cerr << sj_nest->db << "." << sj_nest->table_name << endl; } #endif @@ -5675,7 +5689,7 @@ int getSelectPlan(gp_walk_info& gwi, SELECT_LEX& select_lex, SCSEP& csep, bool i CalpontSystemCatalog::TableAliasName tan = make_aliastable(table_ptr->db, table_name, table_ptr->alias, infiniDB); gwi.tableMap[tan] = make_pair(0, table_ptr); #ifdef DEBUG_WALK_COND - cout << tn << endl; + cerr << tn << endl; #endif } } @@ -5755,15 +5769,15 @@ int getSelectPlan(gp_walk_info& gwi, SELECT_LEX& select_lex, SCSEP& csep, bool i distUnionNum = unionVec.size(); /*#ifdef DEBUG_WALK_COND - IDEBUG( cout << ">>>> UNION DEBUG" << endl ); + IDEBUG( cerr << ">>>> UNION DEBUG" << endl ); JOIN* join = sl->join; Item_cond* icp = 0; if (join != 0) icp = reinterpret_cast(join->conds); if (icp) icp->traverse_cond(debug_walk, &gwi, Item::POSTFIX); - IDEBUG ( cout << *plan << endl ); - IDEBUG ( cout << "<<<traverse_cond(debug_walk, &gwi, Item::POSTFIX); - cout << "------------------------------------------------\n" << endl; + cerr << "------------------------------------------------\n" << endl; #endif icp->traverse_cond(gp_walk, &gwi, Item::POSTFIX); @@ -5840,16 +5854,16 @@ int getSelectPlan(gp_walk_info& gwi, SELECT_LEX& select_lex, SCSEP& csep, bool i TABLE_LIST* curr = *tbl; if (curr->table_name) - cout << curr->table_name << " "; + cerr << curr->table_name << " "; else - cout << curr->alias << endl; + cerr << curr->alias << endl; if (curr->outer_join) - cout << " is inner table" << endl; + cerr << " is inner table" << endl; else if (curr->straight) - cout << "straight_join" << endl; + cerr << "straight_join" << endl; else - cout << "join" << endl; + cerr << "join" << endl; if (curr->nested_join) { @@ -5865,7 +5879,7 @@ int getSelectPlan(gp_walk_info& gwi, SELECT_LEX& select_lex, SCSEP& csep, bool i for (TABLE_LIST** tb = inner; tb < end1; tb++) { TABLE_LIST* curr1 = *tb; - cout << curr1->alias << endl; + cerr << curr1->alias << endl; if (curr1->sj_on_expr) { @@ -5940,7 +5954,7 @@ int getSelectPlan(gp_walk_info& gwi, SELECT_LEX& select_lex, SCSEP& csep, bool i gwi.clauseType = SELECT; #ifdef DEBUG_WALK_COND { - cout << "------------------- SELECT --------------------" << endl; + cerr << "------------------- SELECT --------------------" << endl; List_iterator_fast it(select_lex.item_list); Item* item; @@ -5949,7 +5963,7 @@ int getSelectPlan(gp_walk_info& gwi, SELECT_LEX& select_lex, SCSEP& csep, bool i debug_walk(item, 0); } - cout << "-----------------------------------------------\n" << endl; + cerr << "-----------------------------------------------\n" << endl; } #endif @@ -6407,7 +6421,7 @@ int getSelectPlan(gp_walk_info& gwi, SELECT_LEX& select_lex, SCSEP& csep, bool i } #ifdef DEBUG_WALK_COND - cout << "SELECT clause SUBSELECT Item: " << sub->substype() << endl; + cerr << "SELECT clause SUBSELECT Item: " << sub->substype() << endl; JOIN* join = sub->get_select_lex()->join; if (join) @@ -6418,7 +6432,7 @@ int getSelectPlan(gp_walk_info& gwi, SELECT_LEX& select_lex, SCSEP& csep, bool i cond->traverse_cond(debug_walk, &gwi, Item::POSTFIX); } - cout << "Finish SELECT clause subselect item traversing" << endl; + cerr << "Finish SELECT clause subselect item traversing" << endl; #endif SelectSubQuery* selectSub = new SelectSubQuery(gwi, sub); //selectSub->gwip(&gwi); @@ -6537,9 +6551,9 @@ int getSelectPlan(gp_walk_info& gwi, SELECT_LEX& select_lex, SCSEP& csep, bool i { Item_cond* having = reinterpret_cast(select_lex.having); #ifdef DEBUG_WALK_COND - cout << "------------------- HAVING ---------------------" << endl; + cerr << "------------------- HAVING ---------------------" << endl; having->traverse_cond(debug_walk, &gwi, Item::POSTFIX); - cout << "------------------------------------------------\n" << endl; + cerr << "------------------------------------------------\n" << endl; #endif having->traverse_cond(gp_walk, &gwi, Item::POSTFIX); @@ -6943,6 +6957,7 @@ int getSelectPlan(gp_walk_info& gwi, SELECT_LEX& select_lex, SCSEP& csep, bool i algorithm::to_lower(lower_create_query); algorithm::to_lower(lower_select_query); + // check if window functions are in order by. InfiniDB process order by list if // window functions are involved, either in order by or projection. for (; ordercol; ordercol = ordercol->next) @@ -7754,8 +7769,8 @@ int getSelectPlan(gp_walk_info& gwi, SELECT_LEX& select_lex, SCSEP& csep, bool i // relate to bug4848. let mysql drive limit when limit session variable set. // do not set in csep. @bug5096. ignore session limit setting for dml - if ((gwi.thd->variables.select_limit == (uint64_t) -1 || - (gwi.thd->variables.select_limit != (uint64_t) -1 && + if ((gwi.thd->variables.select_limit == (uint64_t) - 1 || + (gwi.thd->variables.select_limit != (uint64_t) - 1 && gwi.thd->infinidb_vtable.vtable_state != THD::INFINIDB_CREATE_VTABLE)) && !csep->hasOrderBy()) { @@ -7774,7 +7789,7 @@ int getSelectPlan(gp_walk_info& gwi, SELECT_LEX& select_lex, SCSEP& csep, bool i gwi.thd->infinidb_vtable.select_vtable_query.append(select_query.c_str(), select_query.length()); // We don't currently support limit with correlated subquery - if (csep->limitNum() != (uint64_t) -1 && + if (csep->limitNum() != (uint64_t) - 1 && gwi.subQuery && !gwi.correlatedTbNameVec.empty()) { gwi.fatalParseError = true; @@ -7878,6 +7893,10 @@ int cp_get_plan(THD* thd, SCSEP& csep) else if (status < 0) return status; + cerr << "---------------- cp_get_plan EXECUTION PLAN ----------------" << endl; + cerr << *csep << endl ; + cerr << "-------------- EXECUTION PLAN END --------------\n" << endl; + // Derived table projection and filter optimization. derivedTableOptimization(csep); From 305bae1bcb02612085aa486ee25aea1c72542b45 Mon Sep 17 00:00:00 2001 From: Andrew Hutchings Date: Wed, 9 May 2018 20:34:56 +0100 Subject: [PATCH 55/57] MCOL-1402 Fix addtime/subtime The changes to addtime/subtime for TIME datatype broke the handling of the adding and subtracting of time. This fixes that. --- utils/dataconvert/dataconvert.cpp | 7 +++-- utils/funcexp/func_add_time.cpp | 51 ++++++++++++++----------------- 2 files changed, 27 insertions(+), 31 deletions(-) diff --git a/utils/dataconvert/dataconvert.cpp b/utils/dataconvert/dataconvert.cpp index b1cf8bcb2..657a8d0b1 100644 --- a/utils/dataconvert/dataconvert.cpp +++ b/utils/dataconvert/dataconvert.cpp @@ -2769,7 +2769,8 @@ int64_t DataConvert::stringToTime(const string& data) if (*end != '\0') return -1; - + hour = day * 24; + day = -1; time = data.substr(pos + 1, data.length() - pos - 1); } else @@ -2795,11 +2796,11 @@ int64_t DataConvert::stringToTime(const string& data) if (pos == string::npos) { - hour = atoi(hms.c_str()); + hour += atoi(hms.c_str()); } else { - hour = atoi(hms.substr(0, pos).c_str()); + hour += atoi(hms.substr(0, pos).c_str()); ms = hms.substr(pos + 1, hms.length() - pos - 1); } diff --git a/utils/funcexp/func_add_time.cpp b/utils/funcexp/func_add_time.cpp index ced9beaf2..61eb9ec6c 100644 --- a/utils/funcexp/func_add_time.cpp +++ b/utils/funcexp/func_add_time.cpp @@ -83,8 +83,17 @@ int64_t addTime(DateTime& dt1, Time& dt2) if ((hour < 0) || (hour > 23)) { - dt.hour = hour % 24; dt2.day = hour / 24; + hour = hour % 24; + } + if (hour < 0) + { + dt.hour = hour + 24; + dt2.day--; + } + else + { + dt.hour = hour; } day = (signed)(dt1.day + dt2.day); @@ -141,6 +150,7 @@ int64_t addTime(DateTime& dt1, Time& dt2) int64_t addTime(Time& dt1, Time& dt2) { Time dt; + dt.is_neg = false; dt.hour = 0; dt.minute = 0; dt.second = 0; @@ -174,18 +184,19 @@ int64_t addTime(Time& dt1, Time& dt2) dt2.hour--; } - dt.hour = (signed)(dt1.hour + dt2.hour + min / 60); + dt.hour = tmp = (signed)(dt1.hour + dt2.hour + min / 60); // Saturation - if (dt.hour > 838) + if (tmp > 838) { dt.hour = 838; dt.minute = 59; dt.second = 59; dt.msecond = 999999; } - else if (dt.hour < -838) + else if (tmp < -838) { + dt.is_neg = true; dt.hour = -838; dt.minute = 59; dt.second = 59; @@ -277,29 +288,18 @@ int64_t Func_add_time::getDatetimeIntVal(rowgroup::Row& row, int val_sign = 1; - if (t2.day != 0 && t2.hour < 0) - { - isNull = true; - return -1; - } - else if (t2.day < 0 || t2.hour < 0) + if (t2.hour < 0) { val_sign = -1; } - if ((abs(t2.day) * 24 + abs(t2.hour)) > 838) + if (abs(t2.hour) > 838) { t2.hour = 838; t2.minute = 59; t2.second = 59; t2.msecond = 999999; } - else - { - t2.hour = abs(t2.day) * 24 + t2.hour; - } - - t2.day = 0; if (val_sign * sign < 0) { @@ -316,6 +316,8 @@ int64_t Func_add_time::getDatetimeIntVal(rowgroup::Row& row, t2.msecond = abs(t2.msecond); } + t2.day = 0; + return addTime(dt1, t2); } @@ -332,6 +334,8 @@ int64_t Func_add_time::getTimeIntVal(rowgroup::Row& row, const string& val2 = parm[1]->data()->getStrVal(row, isNull); int sign = parm[2]->data()->getIntVal(row, isNull); Time dt1; + dt1.day = 0; + dt1.is_neg = val1 >> 63; dt1.hour = (val1 >> 40) & 0xfff; dt1.minute = (val1 >> 32) & 0xff; dt1.second = (val1 >> 24) & 0xff; @@ -356,27 +360,18 @@ int64_t Func_add_time::getTimeIntVal(rowgroup::Row& row, int val_sign = 1; - if (t2.day != 0 && t2.hour < 0) - { - isNull = true; - return -1; - } - else if (t2.day < 0 || t2.hour < 0) + if (t2.hour < 0) { val_sign = -1; } - if ((abs(t2.day) * 24 + abs(t2.hour)) > 838) + if (abs(t2.hour) > 838) { t2.hour = 838; t2.minute = 59; t2.second = 59; t2.msecond = 999999; } - else - { - t2.hour = abs(t2.day) * 24 + t2.hour; - } t2.day = 0; From f67c9fd50fccb0e66a62f1dafbe459affa5091eb Mon Sep 17 00:00:00 2001 From: Roman Nozdrin Date: Thu, 10 May 2018 22:04:33 +0300 Subject: [PATCH 56/57] MCOL-1386 Add c-style comments support for DDL statements. --- dbcon/ddlpackage/ddl.l | 6 ++++++ 1 file changed, 6 insertions(+) diff --git a/dbcon/ddlpackage/ddl.l b/dbcon/ddlpackage/ddl.l index 7e7400451..35a3694a0 100644 --- a/dbcon/ddlpackage/ddl.l +++ b/dbcon/ddlpackage/ddl.l @@ -49,6 +49,7 @@ static char* scanner_copy(char *str, yyscan_t yyscanner, copy_action_t action = %x check2 %x inquote %x endquote +%x c_comment space [ \t\n\r\f] horiz_space [ \t\f] @@ -81,6 +82,7 @@ realfail1 ({integer}|{decimal})[Ee] realfail2 ({integer}|{decimal})[Ee][-+] + %% @@ -196,6 +198,10 @@ LONGTEXT {return LONGTEXT;} /* ignore */ } +"/*" { BEGIN(c_comment); } +"*/" { BEGIN(0); } +\n { } +. { } %% void ddlerror(struct pass_to_bison* x, char const *s) From 8a3c5e08e20df71ac63c62b315d6b6bcd0f6290e Mon Sep 17 00:00:00 2001 From: Roman Nozdrin Date: Sun, 13 May 2018 22:02:08 +0300 Subject: [PATCH 57/57] MCOL-1073 DecomSvr service was removed. --- .gitignore | 1 - CMakeLists.txt | 1 - cpackEngineRPM.cmake | 1 - decomsvr/CMakeLists.txt | 14 - decomsvr/DecomSvr.rc | 102 --- decomsvr/DecomSvr.vcxproj | 290 -------- decomsvr/DecomSvr.vcxproj.filters | 35 - decomsvr/cli.cpp | 112 --- decomsvr/quicklz.c | 848 --------------------- decomsvr/quicklz.h | 150 ---- decomsvr/resource.h | 14 - decomsvr/server.cpp | 978 ------------------------- oam/etc/ProcessConfig.xml | 10 - oam/etc/ProcessConfig.xml.singleserver | 10 - procmon/processmonitor.cpp | 22 - 15 files changed, 2588 deletions(-) delete mode 100644 decomsvr/CMakeLists.txt delete mode 100644 decomsvr/DecomSvr.rc delete mode 100644 decomsvr/DecomSvr.vcxproj delete mode 100644 decomsvr/DecomSvr.vcxproj.filters delete mode 100644 decomsvr/cli.cpp delete mode 100644 decomsvr/quicklz.c delete mode 100644 decomsvr/quicklz.h delete mode 100644 decomsvr/resource.h delete mode 100644 decomsvr/server.cpp diff --git a/.gitignore b/.gitignore index f8246dc9f..c7aa228d3 100644 --- a/.gitignore +++ b/.gitignore @@ -39,7 +39,6 @@ dbcon/ddlpackage/ddl-scan.h dbcon/dmlpackage/dml-scan.cpp dbcon/dmlpackage/dml-scan.h ddlproc/DDLProc -decomsvr/DecomSvr dmlproc/DMLProc exemgr/ExeMgr oamapps/calpontDB/calpontDBWrite diff --git a/CMakeLists.txt b/CMakeLists.txt index 44aed9bc8..4dd1f6ede 100755 --- a/CMakeLists.txt +++ b/CMakeLists.txt @@ -240,7 +240,6 @@ ADD_SUBDIRECTORY(dmlproc) ADD_SUBDIRECTORY(procmon) ADD_SUBDIRECTORY(procmgr) ADD_SUBDIRECTORY(oamapps) -ADD_SUBDIRECTORY(decomsvr) ADD_SUBDIRECTORY(primitives) ADD_SUBDIRECTORY(tools) ADD_SUBDIRECTORY(writeengine/server) diff --git a/cpackEngineRPM.cmake b/cpackEngineRPM.cmake index be5df3462..71d5e1f26 100644 --- a/cpackEngineRPM.cmake +++ b/cpackEngineRPM.cmake @@ -132,7 +132,6 @@ SET(CPACK_RPM_platform_USER_FILELIST "/usr/local/mariadb/columnstore/bin/post-mysqld-install" "/usr/local/mariadb/columnstore/bin/pre-uninstall" "/usr/local/mariadb/columnstore/bin/PrimProc" -"/usr/local/mariadb/columnstore/bin/DecomSvr" "/usr/local/mariadb/columnstore/bin/upgrade-columnstore.sh" "/usr/local/mariadb/columnstore/bin/run.sh" "/usr/local/mariadb/columnstore/bin/columnstore" diff --git a/decomsvr/CMakeLists.txt b/decomsvr/CMakeLists.txt deleted file mode 100644 index 5c4e5e6a7..000000000 --- a/decomsvr/CMakeLists.txt +++ /dev/null @@ -1,14 +0,0 @@ - -include_directories( ${ENGINE_COMMON_INCLUDES} ) - - -########### next target ############### - -set(DecomSvr_SRCS quicklz.c server.cpp) - -add_executable(DecomSvr ${DecomSvr_SRCS}) - -target_link_libraries(DecomSvr ${ENGINE_LDFLAGS} ${Boost_LIBRARIES} pthread rt) - -install(TARGETS DecomSvr DESTINATION ${ENGINE_BINDIR} COMPONENT platform) - diff --git a/decomsvr/DecomSvr.rc b/decomsvr/DecomSvr.rc deleted file mode 100644 index 38e974faa..000000000 --- a/decomsvr/DecomSvr.rc +++ /dev/null @@ -1,102 +0,0 @@ -// Microsoft Visual C++ generated resource script. -// -#include "resource.h" - -#define APSTUDIO_READONLY_SYMBOLS -///////////////////////////////////////////////////////////////////////////// -// -// Generated from the TEXTINCLUDE 2 resource. -// -#include "afxres.h" - -///////////////////////////////////////////////////////////////////////////// -#undef APSTUDIO_READONLY_SYMBOLS - -///////////////////////////////////////////////////////////////////////////// -// English (U.S.) resources - -#if !defined(AFX_RESOURCE_DLL) || defined(AFX_TARG_ENU) -#ifdef _WIN32 -LANGUAGE LANG_ENGLISH, SUBLANG_ENGLISH_US -#pragma code_page(1252) -#endif //_WIN32 - -#ifdef APSTUDIO_INVOKED -///////////////////////////////////////////////////////////////////////////// -// -// TEXTINCLUDE -// - -1 TEXTINCLUDE -BEGIN - "resource.h\0" -END - -2 TEXTINCLUDE -BEGIN - "#include ""afxres.h""\r\n" - "\0" -END - -3 TEXTINCLUDE -BEGIN - "\r\n" - "\0" -END - -#endif // APSTUDIO_INVOKED - - -///////////////////////////////////////////////////////////////////////////// -// -// Version -// - -VS_VERSION_INFO VERSIONINFO - FILEVERSION 4,6,0,0 - PRODUCTVERSION 4,6,0,0 - FILEFLAGSMASK 0x17L -#ifdef _DEBUG - FILEFLAGS 0x1L -#else - FILEFLAGS 0x0L -#endif - FILEOS 0x4L - FILETYPE 0x1L - FILESUBTYPE 0x0L -BEGIN - BLOCK "StringFileInfo" - BEGIN - BLOCK "040904b0" - BEGIN - VALUE "CompanyName", "InfiniDB, Inc." - VALUE "FileDescription", "InfiniDB V1 Decompression Server" - VALUE "FileVersion", "4.6.0-0" - VALUE "InternalName", "DecomSvr" - VALUE "LegalCopyright", "Copyright (C) 2014" - VALUE "OriginalFilename", "DecomSvr.exe" - VALUE "ProductName", "InfiniDB" - VALUE "ProductVersion", "4.6.0.0 Beta" - END - END - BLOCK "VarFileInfo" - BEGIN - VALUE "Translation", 0x409, 1200 - END -END - -#endif // English (U.S.) resources -///////////////////////////////////////////////////////////////////////////// - - - -#ifndef APSTUDIO_INVOKED -///////////////////////////////////////////////////////////////////////////// -// -// Generated from the TEXTINCLUDE 3 resource. -// - - -///////////////////////////////////////////////////////////////////////////// -#endif // not APSTUDIO_INVOKED - diff --git a/decomsvr/DecomSvr.vcxproj b/decomsvr/DecomSvr.vcxproj deleted file mode 100644 index 8e3bb55a4..000000000 --- a/decomsvr/DecomSvr.vcxproj +++ /dev/null @@ -1,290 +0,0 @@ - - - - - Debug - Win32 - - - Debug - x64 - - - EnterpriseRelease - Win32 - - - EnterpriseRelease - x64 - - - Release - Win32 - - - Release - x64 - - - - {E7F6F8A8-9DDE-4FC4-8F80-25AB98922E6F} - DecomSvr - Win32Proj - - - - Application - v110 - MultiByte - true - - - Application - v110 - MultiByte - true - - - Application - v110 - MultiByte - - - Application - v110 - MultiByte - true - - - Application - v110 - MultiByte - true - - - Application - v110 - MultiByte - - - - - - - - - - - - - - - - - - - - - - - - - <_ProjectFileVersion>11.0.50727.1 - - - $(SolutionDir)..\..\$(PlatformName)\$(ConfigurationName)\ - $(SolutionDir)..\..\obj\$(ProjectName)\$(PlatformName)\$(ConfigurationName)\ - true - - - $(SolutionDir)..\..\$(Platform)\$(Configuration)\ - $(SolutionDir)..\..\obj\$(ProjectName)\$(Platform)\$(Configuration)\ - true - - - $(SolutionDir)..\..\$(PlatformName)\$(ConfigurationName)\ - $(SolutionDir)..\..\obj\$(ProjectName)\$(PlatformName)\$(ConfigurationName)\ - false - - - $(SolutionDir)..\..\$(Platform)\$(Configuration)\ - $(SolutionDir)..\..\obj\$(ProjectName)\$(Platform)\$(Configuration)\ - false - - - $(SolutionDir)..\..\$(PlatformName)\$(ConfigurationName)\ - $(SolutionDir)..\..\obj\$(ProjectName)\$(PlatformName)\$(ConfigurationName)\ - false - - - $(SolutionDir)..\..\$(Platform)\$(Configuration)\ - $(SolutionDir)..\..\obj\$(ProjectName)\$(Platform)\$(Configuration)\ - false - - - - Disabled - $(SolutionDir)..\..\boost_1_54_0;$(SolutionDir)..\utils\winport;%(AdditionalIncludeDirectories) - WIN32;_DEBUG;_CONSOLE;%(PreprocessorDefinitions) - true - EnableFastChecks - MultiThreadedDebugDLL - - Level3 - EditAndContinue - - - ws2_32.lib;%(AdditionalDependencies) - $(SolutionDir)..\..\boost_1_54_0;%(AdditionalLibraryDirectories) - true - Console - MachineX86 - - - - - X64 - - - Disabled - $(SolutionDir)..\..\boost_1_54_0;$(SolutionDir)..\utils\winport;%(AdditionalIncludeDirectories) - WIN32;_DEBUG;_CONSOLE;%(PreprocessorDefinitions) - true - EnableFastChecks - MultiThreadedDebugDLL - - Level3 - ProgramDatabase - - - ws2_32.lib;%(AdditionalDependencies) - $(SolutionDir)..\..\boost_1_54_0;$(SolutionDir)..\..\x64\Debug;%(AdditionalLibraryDirectories) - true - Console - MachineX64 - - - - - MaxSpeed - true - $(SolutionDir)..\..\boost_1_54_0;$(SolutionDir)..\utils\winport;%(AdditionalIncludeDirectories) - WIN32;NDEBUG;_CONSOLE;%(PreprocessorDefinitions) - MultiThreadedDLL - true - - Level3 - ProgramDatabase - 4996;4244;4267;%(DisableSpecificWarnings) - - - ws2_32.lib;%(AdditionalDependencies) - $(SolutionDir)..\..\boost_1_54_0;%(AdditionalLibraryDirectories) - true - Console - true - true - MachineX86 - - - - - X64 - - - MaxSpeed - true - $(SolutionDir)..\..\boost_1_54_0;$(SolutionDir)..\utils\winport;%(AdditionalIncludeDirectories) - WIN32;NDEBUG;_CONSOLE;%(PreprocessorDefinitions) - MultiThreadedDLL - true - - Level3 - ProgramDatabase - 4996;4244;4267;%(DisableSpecificWarnings) - - - ws2_32.lib;%(AdditionalDependencies) - $(SolutionDir)..\..\boost_1_54_0;C:$(SolutionDir)..\..x64\Release;%(AdditionalLibraryDirectories) - true - Console - true - true - MachineX64 - - - $(SolutionDir)..\..\signit "InfiniDB V1 Decompression Server" $(SolutionDir)..\..x64\Release\DecomSvr.exe - - - - - MaxSpeed - true - $(SolutionDir)..\..\boost_1_54_0;$(SolutionDir)..\utils\winport;%(AdditionalIncludeDirectories) - WIN32;NDEBUG;_CONSOLE;%(PreprocessorDefinitions) - MultiThreadedDLL - true - - Level3 - ProgramDatabase - 4996;4244;4267;%(DisableSpecificWarnings) - - - ws2_32.lib;%(AdditionalDependencies) - $(SolutionDir)..\..\boost_1_54_0;%(AdditionalLibraryDirectories) - true - Console - true - true - MachineX86 - - - - - X64 - - - MaxSpeed - true - $(SolutionDir)..\..\boost_1_54_0;$(SolutionDir)..\utils\winport;%(AdditionalIncludeDirectories) - WIN32;NDEBUG;_CONSOLE;%(PreprocessorDefinitions) - MultiThreadedDLL - true - - Level3 - ProgramDatabase - 4996;4244;4267;%(DisableSpecificWarnings) - false - - - ws2_32.lib;%(AdditionalDependencies) - $(SolutionDir)..\..\boost_1_54_0;$(SolutionDir)..\..\x64\EnterpriseRelease;%(AdditionalLibraryDirectories) - false - Console - true - true - MachineX64 - - - $(SolutionDir)..\..\signit "InfiniDB V1 Decompression Server" $(SolutionDir)..\..\x64\EnterpriseRelease\DecomSvr.exe - - - - - - - - - - - - - - - {4f0851d3-b782-4f12-b748-73efa2da586b} - - - - - - \ No newline at end of file diff --git a/decomsvr/DecomSvr.vcxproj.filters b/decomsvr/DecomSvr.vcxproj.filters deleted file mode 100644 index 3166cce9d..000000000 --- a/decomsvr/DecomSvr.vcxproj.filters +++ /dev/null @@ -1,35 +0,0 @@ - - - - - {4FC737F1-C7A5-4376-A066-2A32D752A2FF} - cpp;c;cc;cxx;def;odl;idl;hpj;bat;asm;asmx - - - {93995380-89BD-4b04-88EB-625FBE52EBFB} - h;hpp;hxx;hm;inl;inc;xsd - - - {67DA6AB6-F800-4c08-8B7A-83BB121AAD01} - rc;ico;cur;bmp;dlg;rc2;rct;bin;rgs;gif;jpg;jpeg;jpe;resx;tiff;tif;png;wav - - - - - Source Files - - - Source Files - - - - - Header Files - - - - - Resource Files - - - \ No newline at end of file diff --git a/decomsvr/cli.cpp b/decomsvr/cli.cpp deleted file mode 100644 index e7b257a1c..000000000 --- a/decomsvr/cli.cpp +++ /dev/null @@ -1,112 +0,0 @@ -#include -#include -#include -#include -#include -#include -#include -#include -//#define NDEBUG -#include -#include -using namespace std; - -/* - Protocol definition: - On the control fifo: - This server waits for the other to send it: - 1. The name of the data fifo to open and read/write (string) - 2. The number of bytes of compressed data to read (number) - - On the data fifo: - The server then reads the compressed data from the data fifo: - 1. The compressed data - then it decompresses it, and sends back to the client: - 1. The number of bytes in the uncompressed stream (number) - 2. The uncompressed data - - strings are sent like this: - uint32_t string len - len bytes of the string - numbers are sent like this: - uint64_t the number - - This server expects numeric values to be in its native byte order, so - the sender needs to do it that way. -*/ - -namespace -{ -const string MessageFifo("/tmp/idbdsfifo"); -} - -int main(int argc, char** argv) -{ -again: - int fd = open(MessageFifo.c_str(), O_WRONLY | O_NONBLOCK); - - if (fd < 0) - { - if (errno == ENXIO) - { - cerr << "waiting for DS to startup..." << endl; - sleep(1); - goto again; - } - - throw runtime_error("while opening fifo for write"); - } - - uint32_t u32; - uint64_t u64; - string s; - ssize_t wrc; - - s = "/tmp/cdatafifo"; - mknod(s.c_str(), S_IFIFO | 0666, 0); - u32 = s.length(); - wrc = write(fd, &u32, 4); - assert(wrc == 4); - wrc = write(fd, s.c_str(), u32); - assert(wrc == u32); - - u64 = 707070; - write(fd, &u64, 8); - - close(fd); - - fd = open(s.c_str(), O_WRONLY); - assert(fd >= 0); - - char* b = new char[u64]; - assert(b); - - wrc = write (fd, b, u64); - assert(wrc == u64); - - delete [] b; - - close(fd); - fd = open(s.c_str(), O_RDONLY); - assert(fd >= 0); - - wrc = read(fd, &u64, 8); - assert(wrc == 8); - - b = new char[u64]; - assert(b); - - cout << "going to read " << u64 << " bytes of uncompressed data" << endl << flush; - wrc = read(fd, b, u64); - assert(wrc == u64); - cout << "read " << u64 << " bytes of uncompressed data" << endl; - - delete [] b; - - close(fd); - - unlink(s.c_str()); - - return 0; -} - diff --git a/decomsvr/quicklz.c b/decomsvr/quicklz.c deleted file mode 100644 index 374212902..000000000 --- a/decomsvr/quicklz.c +++ /dev/null @@ -1,848 +0,0 @@ -// Fast data compression library -// Copyright (C) 2006-2011 Lasse Mikkel Reinhold -// lar@quicklz.com -// -// QuickLZ can be used for free under the GPL 1, 2 or 3 license (where anything -// released into public must be open source) or under a commercial license if such -// has been acquired (see http://www.quicklz.com/order.html). The commercial license -// does not cover derived or ported versions created by third parties under GPL. - -// 1.5.0 final - -#include "quicklz.h" - -#if QLZ_VERSION_MAJOR != 1 || QLZ_VERSION_MINOR != 5 || QLZ_VERSION_REVISION != 0 - #error quicklz.c and quicklz.h have different versions -#endif - -#if (defined(__X86__) || defined(__i386__) || defined(i386) || defined(_M_IX86) || defined(__386__) || defined(__x86_64__) || defined(_M_X64)) - #define X86X64 -#endif - -#define MINOFFSET 2 -#define UNCONDITIONAL_MATCHLEN 6 -#define UNCOMPRESSED_END 4 -#define CWORD_LEN 4 - -#if QLZ_COMPRESSION_LEVEL == 1 && defined QLZ_PTR_64 && QLZ_STREAMING_BUFFER == 0 - #define OFFSET_BASE source - #define CAST (ui32)(size_t) -#else - #define OFFSET_BASE 0 - #define CAST -#endif - -int qlz_get_setting(int setting) -{ - switch (setting) - { - case 0: return QLZ_COMPRESSION_LEVEL; - case 1: return sizeof(qlz_state_compress); - case 2: return sizeof(qlz_state_decompress); - case 3: return QLZ_STREAMING_BUFFER; -#ifdef QLZ_MEMORY_SAFE - case 6: return 1; -#else - case 6: return 0; -#endif - case 7: return QLZ_VERSION_MAJOR; - case 8: return QLZ_VERSION_MINOR; - case 9: return QLZ_VERSION_REVISION; - } - return -1; -} - -#if QLZ_COMPRESSION_LEVEL == 1 -static int same(const unsigned char *src, size_t n) -{ - while(n > 0 && *(src + n) == *src) - n--; - return n == 0 ? 1 : 0; -} -#endif - -static void reset_table_compress(qlz_state_compress *state) -{ - int i; - for(i = 0; i < QLZ_HASH_VALUES; i++) - { -#if QLZ_COMPRESSION_LEVEL == 1 - state->hash[i].offset = 0; -#else - state->hash_counter[i] = 0; -#endif - } -} - -static void reset_table_decompress(qlz_state_decompress *state) -{ - int i; - (void)state; - (void)i; -#if QLZ_COMPRESSION_LEVEL == 2 - for(i = 0; i < QLZ_HASH_VALUES; i++) - { - state->hash_counter[i] = 0; - } -#endif -} - -static __inline ui32 hash_func(ui32 i) -{ -#if QLZ_COMPRESSION_LEVEL == 2 - return ((i >> 9) ^ (i >> 13) ^ i) & (QLZ_HASH_VALUES - 1); -#else - return ((i >> 12) ^ i) & (QLZ_HASH_VALUES - 1); -#endif -} - -static __inline ui32 fast_read(void const *src, ui32 bytes) -{ -#ifndef X86X64 - unsigned char *p = (unsigned char*)src; - switch (bytes) - { - case 4: - return(*p | *(p + 1) << 8 | *(p + 2) << 16 | *(p + 3) << 24); - case 3: - return(*p | *(p + 1) << 8 | *(p + 2) << 16); - case 2: - return(*p | *(p + 1) << 8); - case 1: - return(*p); - } - return 0; -#else - if (bytes >= 1 && bytes <= 4) - return *((ui32*)src); - else - return 0; -#endif -} - -static __inline ui32 hashat(const unsigned char *src) -{ - ui32 fetch, hash; - fetch = fast_read(src, 3); - hash = hash_func(fetch); - return hash; -} - -static __inline void fast_write(ui32 f, void *dst, size_t bytes) -{ -#ifndef X86X64 - unsigned char *p = (unsigned char*)dst; - - switch (bytes) - { - case 4: - *p = (unsigned char)f; - *(p + 1) = (unsigned char)(f >> 8); - *(p + 2) = (unsigned char)(f >> 16); - *(p + 3) = (unsigned char)(f >> 24); - return; - case 3: - *p = (unsigned char)f; - *(p + 1) = (unsigned char)(f >> 8); - *(p + 2) = (unsigned char)(f >> 16); - return; - case 2: - *p = (unsigned char)f; - *(p + 1) = (unsigned char)(f >> 8); - return; - case 1: - *p = (unsigned char)f; - return; - } -#else - switch (bytes) - { - case 4: - *((ui32*)dst) = f; - return; - case 3: - *((ui32*)dst) = f; - return; - case 2: - *((ui16 *)dst) = (ui16)f; - return; - case 1: - *((unsigned char*)dst) = (unsigned char)f; - return; - } -#endif -} - - -size_t qlz_size_decompressed(const char *source) -{ - ui32 n, r; - n = (((*source) & 2) == 2) ? 4 : 1; - r = fast_read(source + 1 + n, n); - r = r & (0xffffffff >> ((4 - n)*8)); - return r; -} - -size_t qlz_size_compressed(const char *source) -{ - ui32 n, r; - n = (((*source) & 2) == 2) ? 4 : 1; - r = fast_read(source + 1, n); - r = r & (0xffffffff >> ((4 - n)*8)); - return r; -} - -size_t qlz_size_header(const char *source) -{ - size_t n = 2*((((*source) & 2) == 2) ? 4 : 1) + 1; - return n; -} - - -static __inline void memcpy_up(unsigned char *dst, const unsigned char *src, ui32 n) -{ - // Caution if modifying memcpy_up! Overlap of dst and src must be special handled. -#ifndef X86X64 - unsigned char *end = dst + n; - while(dst < end) - { - *dst = *src; - dst++; - src++; - } -#else - ui32 f = 0; - do - { - *(ui32 *)(dst + f) = *(ui32 *)(src + f); - f += MINOFFSET + 1; - } - while (f < n); -#endif -} - -static __inline void update_hash(qlz_state_decompress *state, const unsigned char *s) -{ -#if QLZ_COMPRESSION_LEVEL == 1 - ui32 hash; - hash = hashat(s); - state->hash[hash].offset = s; - state->hash_counter[hash] = 1; -#elif QLZ_COMPRESSION_LEVEL == 2 - ui32 hash; - unsigned char c; - hash = hashat(s); - c = state->hash_counter[hash]; - state->hash[hash].offset[c & (QLZ_POINTERS - 1)] = s; - c++; - state->hash_counter[hash] = c; -#endif - (void)state; - (void)s; -} - -#if QLZ_COMPRESSION_LEVEL <= 2 -static void update_hash_upto(qlz_state_decompress *state, unsigned char **lh, const unsigned char *max) -{ - while(*lh < max) - { - (*lh)++; - update_hash(state, *lh); - } -} -#endif - -static size_t qlz_compress_core(const unsigned char *source, unsigned char *destination, size_t size, qlz_state_compress *state) -{ - const unsigned char *last_byte = source + size - 1; - const unsigned char *src = source; - unsigned char *cword_ptr = destination; - unsigned char *dst = destination + CWORD_LEN; - ui32 cword_val = 1U << 31; - const unsigned char *last_matchstart = last_byte - UNCONDITIONAL_MATCHLEN - UNCOMPRESSED_END; - ui32 fetch = 0; - unsigned int lits = 0; - - (void) lits; - - if(src <= last_matchstart) - fetch = fast_read(src, 3); - - while(src <= last_matchstart) - { - if ((cword_val & 1) == 1) - { - // store uncompressed if compression ratio is too low - if (src > source + (size >> 1) && dst - destination > src - source - ((src - source) >> 5)) - return 0; - - fast_write((cword_val >> 1) | (1U << 31), cword_ptr, CWORD_LEN); - - cword_ptr = dst; - dst += CWORD_LEN; - cword_val = 1U << 31; - fetch = fast_read(src, 3); - } -#if QLZ_COMPRESSION_LEVEL == 1 - { - const unsigned char *o; - ui32 hash, cached; - - hash = hash_func(fetch); - cached = fetch ^ state->hash[hash].cache; - state->hash[hash].cache = fetch; - - o = state->hash[hash].offset + OFFSET_BASE; - state->hash[hash].offset = CAST(src - OFFSET_BASE); - -#ifdef X86X64 - if ((cached & 0xffffff) == 0 && o != OFFSET_BASE && (src - o > MINOFFSET || (src == o + 1 && lits >= 3 && src > source + 3 && same(src - 3, 6)))) - { - if(cached != 0) - { -#else - if (cached == 0 && o != OFFSET_BASE && (src - o > MINOFFSET || (src == o + 1 && lits >= 3 && src > source + 3 && same(src - 3, 6)))) - { - if (*(o + 3) != *(src + 3)) - { -#endif - hash <<= 4; - cword_val = (cword_val >> 1) | (1U << 31); - fast_write((3 - 2) | hash, dst, 2); - src += 3; - dst += 2; - } - else - { - const unsigned char *old_src = src; - size_t matchlen; - hash <<= 4; - - cword_val = (cword_val >> 1) | (1U << 31); - src += 4; - - if(*(o + (src - old_src)) == *src) - { - src++; - if(*(o + (src - old_src)) == *src) - { - size_t q = last_byte - UNCOMPRESSED_END - (src - 5) + 1; - size_t remaining = q > 255 ? 255 : q; - src++; - while(*(o + (src - old_src)) == *src && (size_t)(src - old_src) < remaining) - src++; - } - } - - matchlen = src - old_src; - if (matchlen < 18) - { - fast_write((ui32)(matchlen - 2) | hash, dst, 2); - dst += 2; - } - else - { - fast_write((ui32)(matchlen << 16) | hash, dst, 3); - dst += 3; - } - } - fetch = fast_read(src, 3); - lits = 0; - } - else - { - lits++; - *dst = *src; - src++; - dst++; - cword_val = (cword_val >> 1); -#ifdef X86X64 - fetch = fast_read(src, 3); -#else - fetch = (fetch >> 8 & 0xffff) | (*(src + 2) << 16); -#endif - } - } -#elif QLZ_COMPRESSION_LEVEL >= 2 - { - const unsigned char *o, *offset2; - ui32 hash, matchlen, k, m, best_k = 0; - unsigned char c; - size_t remaining = (last_byte - UNCOMPRESSED_END - src + 1) > 255 ? 255 : (last_byte - UNCOMPRESSED_END - src + 1); - (void)best_k; - - - //hash = hashat(src); - fetch = fast_read(src, 3); - hash = hash_func(fetch); - - c = state->hash_counter[hash]; - - offset2 = state->hash[hash].offset[0]; - if(offset2 < src - MINOFFSET && c > 0 && ((fast_read(offset2, 3) ^ fetch) & 0xffffff) == 0) - { - matchlen = 3; - if(*(offset2 + matchlen) == *(src + matchlen)) - { - matchlen = 4; - while(*(offset2 + matchlen) == *(src + matchlen) && matchlen < remaining) - matchlen++; - } - } - else - matchlen = 0; - for(k = 1; k < QLZ_POINTERS && c > k; k++) - { - o = state->hash[hash].offset[k]; -#if QLZ_COMPRESSION_LEVEL == 3 - if(((fast_read(o, 3) ^ fetch) & 0xffffff) == 0 && o < src - MINOFFSET) -#elif QLZ_COMPRESSION_LEVEL == 2 - if(*(src + matchlen) == *(o + matchlen) && ((fast_read(o, 3) ^ fetch) & 0xffffff) == 0 && o < src - MINOFFSET) -#endif - { - m = 3; - while(*(o + m) == *(src + m) && m < remaining) - m++; -#if QLZ_COMPRESSION_LEVEL == 3 - if ((m > matchlen) || (m == matchlen && o > offset2)) -#elif QLZ_COMPRESSION_LEVEL == 2 - if (m > matchlen) -#endif - { - offset2 = o; - matchlen = m; - best_k = k; - } - } - } - o = offset2; - state->hash[hash].offset[c & (QLZ_POINTERS - 1)] = src; - c++; - state->hash_counter[hash] = c; - -#if QLZ_COMPRESSION_LEVEL == 3 - if(matchlen > 2 && src - o < 131071) - { - ui32 u; - size_t offset = src - o; - - for(u = 1; u < matchlen; u++) - { - hash = hashat(src + u); - c = state->hash_counter[hash]++; - state->hash[hash].offset[c & (QLZ_POINTERS - 1)] = src + u; - } - - cword_val = (cword_val >> 1) | (1U << 31); - src += matchlen; - - if(matchlen == 3 && offset <= 63) - { - *dst = (unsigned char)(offset << 2); - dst++; - } - else if (matchlen == 3 && offset <= 16383) - { - ui32 f = (ui32)((offset << 2) | 1); - fast_write(f, dst, 2); - dst += 2; - } - else if (matchlen <= 18 && offset <= 1023) - { - ui32 f = ((matchlen - 3) << 2) | ((ui32)offset << 6) | 2; - fast_write(f, dst, 2); - dst += 2; - } - - else if(matchlen <= 33) - { - ui32 f = ((matchlen - 2) << 2) | ((ui32)offset << 7) | 3; - fast_write(f, dst, 3); - dst += 3; - } - else - { - ui32 f = ((matchlen - 3) << 7) | ((ui32)offset << 15) | 3; - fast_write(f, dst, 4); - dst += 4; - } - } - else - { - *dst = *src; - src++; - dst++; - cword_val = (cword_val >> 1); - } -#elif QLZ_COMPRESSION_LEVEL == 2 - - if(matchlen > 2) - { - cword_val = (cword_val >> 1) | (1U << 31); - src += matchlen; - - if (matchlen < 10) - { - ui32 f = best_k | ((matchlen - 2) << 2) | (hash << 5); - fast_write(f, dst, 2); - dst += 2; - } - else - { - ui32 f = best_k | (matchlen << 16) | (hash << 5); - fast_write(f, dst, 3); - dst += 3; - } - } - else - { - *dst = *src; - src++; - dst++; - cword_val = (cword_val >> 1); - } -#endif - } -#endif - } - while (src <= last_byte) - { - if ((cword_val & 1) == 1) - { - fast_write((cword_val >> 1) | (1U << 31), cword_ptr, CWORD_LEN); - cword_ptr = dst; - dst += CWORD_LEN; - cword_val = 1U << 31; - } -#if QLZ_COMPRESSION_LEVEL < 3 - if (src <= last_byte - 3) - { -#if QLZ_COMPRESSION_LEVEL == 1 - ui32 hash, fetch; - fetch = fast_read(src, 3); - hash = hash_func(fetch); - state->hash[hash].offset = CAST(src - OFFSET_BASE); - state->hash[hash].cache = fetch; -#elif QLZ_COMPRESSION_LEVEL == 2 - ui32 hash; - unsigned char c; - hash = hashat(src); - c = state->hash_counter[hash]; - state->hash[hash].offset[c & (QLZ_POINTERS - 1)] = src; - c++; - state->hash_counter[hash] = c; -#endif - } -#endif - *dst = *src; - src++; - dst++; - cword_val = (cword_val >> 1); - } - - while((cword_val & 1) != 1) - cword_val = (cword_val >> 1); - - fast_write((cword_val >> 1) | (1U << 31), cword_ptr, CWORD_LEN); - - // min. size must be 9 bytes so that the qlz_size functions can take 9 bytes as argument - return dst - destination < 9 ? 9 : dst - destination; -} - -static size_t qlz_decompress_core(const unsigned char *source, unsigned char *destination, size_t size, qlz_state_decompress *state, const unsigned char *history) -{ - const unsigned char *src = source + qlz_size_header((const char *)source); - unsigned char *dst = destination; - const unsigned char *last_destination_byte = destination + size - 1; - ui32 cword_val = 1; - const unsigned char *last_matchstart = last_destination_byte - UNCONDITIONAL_MATCHLEN - UNCOMPRESSED_END; - unsigned char *last_hashed = destination - 1; - const unsigned char *last_source_byte = source + qlz_size_compressed((const char *)source) - 1; - static const ui32 bitlut[16] = {4, 0, 1, 0, 2, 0, 1, 0, 3, 0, 1, 0, 2, 0, 1, 0}; - - (void) last_source_byte; - (void) last_hashed; - (void) state; - (void) history; - - for(;;) - { - ui32 fetch; - - if (cword_val == 1) - { -#ifdef QLZ_MEMORY_SAFE - if(src + CWORD_LEN - 1 > last_source_byte) - return 0; -#endif - cword_val = fast_read(src, CWORD_LEN); - src += CWORD_LEN; - } - -#ifdef QLZ_MEMORY_SAFE - if(src + 4 - 1 > last_source_byte) - return 0; -#endif - - fetch = fast_read(src, 4); - - if ((cword_val & 1) == 1) - { - ui32 matchlen; - const unsigned char *offset2; - -#if QLZ_COMPRESSION_LEVEL == 1 - ui32 hash; - cword_val = cword_val >> 1; - hash = (fetch >> 4) & 0xfff; - offset2 = (const unsigned char *)(size_t)state->hash[hash].offset; - - if((fetch & 0xf) != 0) - { - matchlen = (fetch & 0xf) + 2; - src += 2; - } - else - { - matchlen = *(src + 2); - src += 3; - } - -#elif QLZ_COMPRESSION_LEVEL == 2 - ui32 hash; - unsigned char c; - cword_val = cword_val >> 1; - hash = (fetch >> 5) & 0x7ff; - c = (unsigned char)(fetch & 0x3); - offset2 = state->hash[hash].offset[c]; - - if((fetch & (28)) != 0) - { - matchlen = ((fetch >> 2) & 0x7) + 2; - src += 2; - } - else - { - matchlen = *(src + 2); - src += 3; - } - -#elif QLZ_COMPRESSION_LEVEL == 3 - ui32 offset; - cword_val = cword_val >> 1; - if ((fetch & 3) == 0) - { - offset = (fetch & 0xff) >> 2; - matchlen = 3; - src++; - } - else if ((fetch & 2) == 0) - { - offset = (fetch & 0xffff) >> 2; - matchlen = 3; - src += 2; - } - else if ((fetch & 1) == 0) - { - offset = (fetch & 0xffff) >> 6; - matchlen = ((fetch >> 2) & 15) + 3; - src += 2; - } - else if ((fetch & 127) != 3) - { - offset = (fetch >> 7) & 0x1ffff; - matchlen = ((fetch >> 2) & 0x1f) + 2; - src += 3; - } - else - { - offset = (fetch >> 15); - matchlen = ((fetch >> 7) & 255) + 3; - src += 4; - } - - offset2 = dst - offset; -#endif - -#ifdef QLZ_MEMORY_SAFE - if(offset2 < history || offset2 > dst - MINOFFSET - 1) - return 0; - - if(matchlen > (ui32)(last_destination_byte - dst - UNCOMPRESSED_END + 1)) - return 0; -#endif - - memcpy_up(dst, offset2, matchlen); - dst += matchlen; - -#if QLZ_COMPRESSION_LEVEL <= 2 - update_hash_upto(state, &last_hashed, dst - matchlen); - last_hashed = dst - 1; -#endif - } - else - { - if (dst < last_matchstart) - { - unsigned int n = bitlut[cword_val & 0xf]; -#ifdef X86X64 - *(ui32 *)dst = *(ui32 *)src; -#else - memcpy_up(dst, src, 4); -#endif - cword_val = cword_val >> n; - dst += n; - src += n; -#if QLZ_COMPRESSION_LEVEL <= 2 - update_hash_upto(state, &last_hashed, dst - 3); -#endif - } - else - { - while(dst <= last_destination_byte) - { - if (cword_val == 1) - { - src += CWORD_LEN; - cword_val = 1U << 31; - } -#ifdef QLZ_MEMORY_SAFE - if(src >= last_source_byte + 1) - return 0; -#endif - *dst = *src; - dst++; - src++; - cword_val = cword_val >> 1; - } - -#if QLZ_COMPRESSION_LEVEL <= 2 - update_hash_upto(state, &last_hashed, last_destination_byte - 3); // todo, use constant -#endif - return size; - } - - } - } -} - -size_t qlz_compress(const void *source, char *destination, size_t size, qlz_state_compress *state) -{ - size_t r; - ui32 compressed; - size_t base; - - if(size == 0 || size > 0xffffffff - 400) - return 0; - - if(size < 216) - base = 3; - else - base = 9; - -#if QLZ_STREAMING_BUFFER > 0 - if (state->stream_counter + size - 1 >= QLZ_STREAMING_BUFFER) -#endif - { - reset_table_compress(state); - r = base + qlz_compress_core((const unsigned char *)source, (unsigned char*)destination + base, size, state); -#if QLZ_STREAMING_BUFFER > 0 - reset_table_compress(state); -#endif - if(r == base) - { - memcpy(destination + base, source, size); - r = size + base; - compressed = 0; - } - else - { - compressed = 1; - } - state->stream_counter = 0; - } -#if QLZ_STREAMING_BUFFER > 0 - else - { - unsigned char *src = state->stream_buffer + state->stream_counter; - - memcpy(src, source, size); - r = base + qlz_compress_core(src, (unsigned char*)destination + base, size, state); - - if(r == base) - { - memcpy(destination + base, src, size); - r = size + base; - compressed = 0; - reset_table_compress(state); - } - else - { - compressed = 1; - } - state->stream_counter += size; - } -#endif - if(base == 3) - { - *destination = (unsigned char)(0 | compressed); - *(destination + 1) = (unsigned char)r; - *(destination + 2) = (unsigned char)size; - } - else - { - *destination = (unsigned char)(2 | compressed); - fast_write((ui32)r, destination + 1, 4); - fast_write((ui32)size, destination + 5, 4); - } - - *destination |= (QLZ_COMPRESSION_LEVEL << 2); - *destination |= (1 << 6); - *destination |= ((QLZ_STREAMING_BUFFER == 0 ? 0 : (QLZ_STREAMING_BUFFER == 100000 ? 1 : (QLZ_STREAMING_BUFFER == 1000000 ? 2 : 3))) << 4); - -// 76543210 -// 01SSLLHC - - return r; -} - -size_t qlz_decompress(const char *source, void *destination, qlz_state_decompress *state) -{ - size_t dsiz = qlz_size_decompressed(source); - -#if QLZ_STREAMING_BUFFER > 0 - if (state->stream_counter + qlz_size_decompressed(source) - 1 >= QLZ_STREAMING_BUFFER) -#endif - { - if((*source & 1) == 1) - { - reset_table_decompress(state); - dsiz = qlz_decompress_core((const unsigned char *)source, (unsigned char *)destination, dsiz, state, (const unsigned char *)destination); - } - else - { - memcpy(destination, source + qlz_size_header(source), dsiz); - } - state->stream_counter = 0; - reset_table_decompress(state); - } -#if QLZ_STREAMING_BUFFER > 0 - else - { - unsigned char *dst = state->stream_buffer + state->stream_counter; - if((*source & 1) == 1) - { - dsiz = qlz_decompress_core((const unsigned char *)source, dst, dsiz, state, (const unsigned char *)state->stream_buffer); - } - else - { - memcpy(dst, source + qlz_size_header(source), dsiz); - reset_table_decompress(state); - } - memcpy(destination, dst, dsiz); - state->stream_counter += dsiz; - } -#endif - return dsiz; -} - diff --git a/decomsvr/quicklz.h b/decomsvr/quicklz.h deleted file mode 100644 index 56915b56d..000000000 --- a/decomsvr/quicklz.h +++ /dev/null @@ -1,150 +0,0 @@ -#ifndef QLZ_HEADER -#define QLZ_HEADER - -// Fast data compression library -// Copyright (C) 2006-2011 Lasse Mikkel Reinhold -// lar@quicklz.com -// -// QuickLZ can be used for free under the GPL 1, 2 or 3 license (where anything -// released into public must be open source) or under a commercial license if such -// has been acquired (see http://www.quicklz.com/order.html). The commercial license -// does not cover derived or ported versions created by third parties under GPL. - -// You can edit following user settings. Data must be decompressed with the same -// setting of QLZ_COMPRESSION_LEVEL and QLZ_STREAMING_BUFFER as it was compressed -// (see manual). If QLZ_STREAMING_BUFFER > 0, scratch buffers must be initially -// zeroed out (see manual). First #ifndef makes it possible to define settings from -// the outside like the compiler command line. - -// 1.5.0 final - -#ifndef QLZ_COMPRESSION_LEVEL - -// 1 gives fastest compression speed. 3 gives fastest decompression speed and best -// compression ratio. -//#define QLZ_COMPRESSION_LEVEL 1 -//#define QLZ_COMPRESSION_LEVEL 2 -#define QLZ_COMPRESSION_LEVEL 3 - -// If > 0, zero out both states prior to first call to qlz_compress() or qlz_decompress() -// and decompress packets in the same order as they were compressed -#define QLZ_STREAMING_BUFFER 0 -//#define QLZ_STREAMING_BUFFER 100000 -//#define QLZ_STREAMING_BUFFER 1000000 - -// Guarantees that decompression of corrupted data cannot crash. Decreases decompression -// speed 10-20%. Compression speed not affected. -#define QLZ_MEMORY_SAFE -#endif - -#define QLZ_VERSION_MAJOR 1 -#define QLZ_VERSION_MINOR 5 -#define QLZ_VERSION_REVISION 0 - -// Using size_t, memset() and memcpy() -#include - -// Verify compression level -#if QLZ_COMPRESSION_LEVEL != 1 && QLZ_COMPRESSION_LEVEL != 2 && QLZ_COMPRESSION_LEVEL != 3 -#error QLZ_COMPRESSION_LEVEL must be 1, 2 or 3 -#endif - -typedef unsigned int ui32; -typedef unsigned short int ui16; - -// Decrease QLZ_POINTERS for level 3 to increase compression speed. Do not touch any other values! -#if QLZ_COMPRESSION_LEVEL == 1 -#define QLZ_POINTERS 1 -#define QLZ_HASH_VALUES 4096 -#elif QLZ_COMPRESSION_LEVEL == 2 -#define QLZ_POINTERS 4 -#define QLZ_HASH_VALUES 2048 -#elif QLZ_COMPRESSION_LEVEL == 3 -#define QLZ_POINTERS 16 -#define QLZ_HASH_VALUES 4096 -#endif - -// Detect if pointer size is 64-bit. It's not fatal if some 64-bit target is not detected because this is only for adding an optional 64-bit optimization. -#if defined _LP64 || defined __LP64__ || defined __64BIT__ || _ADDR64 || defined _WIN64 || defined __arch64__ || __WORDSIZE == 64 || (defined __sparc && defined __sparcv9) || defined __x86_64 || defined __amd64 || defined __x86_64__ || defined _M_X64 || defined _M_IA64 || defined __ia64 || defined __IA64__ -#define QLZ_PTR_64 -#endif - -// hash entry -typedef struct -{ -#if QLZ_COMPRESSION_LEVEL == 1 - ui32 cache; -#if defined QLZ_PTR_64 && QLZ_STREAMING_BUFFER == 0 - unsigned int offset; -#else - const unsigned char* offset; -#endif -#else - const unsigned char* offset[QLZ_POINTERS]; -#endif - -} qlz_hash_compress; - -typedef struct -{ -#if QLZ_COMPRESSION_LEVEL == 1 - const unsigned char* offset; -#else - const unsigned char* offset[QLZ_POINTERS]; -#endif -} qlz_hash_decompress; - - -// states -typedef struct -{ -#if QLZ_STREAMING_BUFFER > 0 - unsigned char stream_buffer[QLZ_STREAMING_BUFFER]; -#endif - size_t stream_counter; - qlz_hash_compress hash[QLZ_HASH_VALUES]; - unsigned char hash_counter[QLZ_HASH_VALUES]; -} qlz_state_compress; - - -#if QLZ_COMPRESSION_LEVEL == 1 || QLZ_COMPRESSION_LEVEL == 2 -typedef struct -{ -#if QLZ_STREAMING_BUFFER > 0 - unsigned char stream_buffer[QLZ_STREAMING_BUFFER]; -#endif - qlz_hash_decompress hash[QLZ_HASH_VALUES]; - unsigned char hash_counter[QLZ_HASH_VALUES]; - size_t stream_counter; -} qlz_state_decompress; -#elif QLZ_COMPRESSION_LEVEL == 3 -typedef struct -{ -#if QLZ_STREAMING_BUFFER > 0 - unsigned char stream_buffer[QLZ_STREAMING_BUFFER]; -#endif -#if QLZ_COMPRESSION_LEVEL <= 2 - qlz_hash_decompress hash[QLZ_HASH_VALUES]; -#endif - size_t stream_counter; -} qlz_state_decompress; -#endif - - -#if defined (__cplusplus) -extern "C" { -#endif - -// Public functions of QuickLZ -size_t qlz_size_decompressed(const char* source); -size_t qlz_size_compressed(const char* source); -size_t qlz_compress(const void* source, char* destination, size_t size, qlz_state_compress* state); -size_t qlz_decompress(const char* source, void* destination, qlz_state_decompress* state); -int qlz_get_setting(int setting); - -#if defined (__cplusplus) -} -#endif - -#endif - diff --git a/decomsvr/resource.h b/decomsvr/resource.h deleted file mode 100644 index 17a793ca1..000000000 --- a/decomsvr/resource.h +++ /dev/null @@ -1,14 +0,0 @@ -//{{NO_DEPENDENCIES}} -// Microsoft Visual C++ generated include file. -// Used by DecomSvr.rc - -// Next default values for new objects -// -#ifdef APSTUDIO_INVOKED -#ifndef APSTUDIO_READONLY_SYMBOLS -#define _APS_NEXT_RESOURCE_VALUE 101 -#define _APS_NEXT_COMMAND_VALUE 40001 -#define _APS_NEXT_CONTROL_VALUE 1001 -#define _APS_NEXT_SYMED_VALUE 101 -#endif -#endif diff --git a/decomsvr/server.cpp b/decomsvr/server.cpp deleted file mode 100644 index f49dc35bc..000000000 --- a/decomsvr/server.cpp +++ /dev/null @@ -1,978 +0,0 @@ -/* Copyright (C) 2014 InfiniDB, Inc. - - This program is free software; you can redistribute it and/or - modify it under the terms of the GNU General Public License - as published by the Free Software Foundation; version 2 of - the License. - - This program is distributed in the hope that it will be useful, - but WITHOUT ANY WARRANTY; without even the implied warranty of - MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the - GNU General Public License for more details. - - You should have received a copy of the GNU General Public License - along with this program; if not, write to the Free Software - Foundation, Inc., 51 Franklin Street, Fifth Floor, Boston, - MA 02110-1301, USA. */ - -/* - Protocol definition: - On the control socket: - This server waits for the other end to send it: - 1. The prefix name of the data fifos to open and read/write (string). The incoming, - compressed data is on "name".c and this server will write uncompressed - data on "name".u. - - On the data fifos: - The server then reads the compressed data from the cdata fifo: - 1. The number of bytes of compressed data to read (number) - 2. The compressed data - then it decompresses it, and writes to the udata fifo: - 1. The number of bytes in the uncompressed stream (number) - 2. The uncompressed data - - strings are sent like this: - uint32_t string len - len bytes of the string - numbers are sent like this: - uint64_t the number - - This server expects numeric values to be in its native byte order, so - the sender needs to send them that way. -*/ - -//#define _FILE_OFFSET_BITS 64 -//#define _LARGEFILE64_SOURCE - -#ifdef _MSC_VER -#define WIN32_LEAN_AND_MEAN -#define NOMINMAX -#include -#include -#include -#include -#include -#include -#else -#include -#include -#include -#include -#endif -#include -#include -#include -#include -#include -#include -#include -#include -#include -#include -#include -#ifndef _MSC_VER -#include -#endif -using namespace std; - -#include -#include -#include -using namespace boost; - -#include "quicklz.h" -#ifndef _MSC_VER -#include "config.h" -#endif - -//#define SINGLE_THREADED - -namespace -{ -#ifdef _MSC_VER -typedef SOCKET SockType; -#define SockReadFcn reads -#else -typedef int SockType; -#define SockReadFcn readn -#endif - -short PortNo; -SockType ListenSock; - -// version 1.1 of the chunk data has a short header -const uint8_t CHUNK_MAGIC1 = 0xff; -const int SIG_OFFSET = 0; -const int CHECKSUM_OFFSET = 1; -const int LEN_OFFSET = 5; -const uint32_t HEADER_SIZE = 9; - -const int ERR_OK = 0; -const int ERR_CHECKSUM = -1; -const int ERR_DECOMPRESS = -2; -const int ERR_BADINPUT = -3; - -/* version 1.2 of the chunk data changes the hash function used to calculate - * checksums. We can no longer use the algorithm used in ver 1.1. Everything - * else is the same - */ -const uint8_t CHUNK_MAGIC2 = 0xfe; - -// A version of idbassert_s & log() that doesn't require linking the logging lib. -// Things printed to stderr go to /tmp/decomsvr.err -#ifndef __STRING -#define __STRING(x) #x -#endif -#define idbassert_s(x, s) do { \ - if (!(x)) { \ - std::ostringstream os; \ -\ - os << __FILE__ << "@" << __LINE__ << ": assertion \'" << __STRING(x) << "\' failed. Error msg \'" << s << "\'"; \ - std::cerr << os.str() << std::endl; \ - throw runtime_error(os.str()); \ - } \ -} while (0) - -void log(const string& s) -{ - cerr << s << endl; -} - -struct DecomMessage -{ - DecomMessage() : isValid(false) { } - ~DecomMessage() { } - - string toString() const; - - bool isValid; - string pipeName; -}; - -string DecomMessage::toString() const -{ - ostringstream oss; - oss << "valid: " << boolalpha << isValid << ", " << - "pipepfx: " << pipeName; - return oss.str(); -} - -ostream& operator<<(ostream& os, const DecomMessage& rhs) -{ - os << rhs.toString(); - return os; -} - -class ThreadFunc -{ -public: - ThreadFunc(const DecomMessage& dm) : fDm(dm) { } - ~ThreadFunc() { } - - void operator()(); - -private: - //Defaults okay - //ThreadFunc(const ThreadFunc& rhs); - //ThreadFunc& operator=(const ThreadFunc& rhs); - - DecomMessage fDm; -}; - -bool serverInit() -{ -#ifndef _MSC_VER - - //Set parent PID to init - setsid(); - - //Handle certain signals (we want these to return EINTR so we can throw) - //SIGPIPE - //I don't think we'll get any of these from init (except possibly HUP, but that's an indication - // of bad things anyway) - //SIGHUP? - //SIGUSR1? - //SIGUSR2? - //SIGPOLL? - struct sigaction sa; - memset(&sa, 0, sizeof(struct sigaction)); - sa.sa_handler = SIG_IGN; - sigaction(SIGPIPE, &sa, 0); - sigaction(SIGHUP, &sa, 0); - sigaction(SIGUSR1, &sa, 0); - sigaction(SIGUSR2, &sa, 0); -#ifndef __FreeBSD__ - sigaction(SIGPOLL, &sa, 0); -#endif - int fd; - close(2); - fd = open("/tmp/decomsrv.err", O_CREAT | O_TRUNC | O_WRONLY, 0644); - - if (fd >= 0 && fd != 2) - { - dup2(fd, 2); - close(fd); - } - -#endif - return true; -} - -bool initCtlFifo() -{ -#ifdef _MSC_VER - WSAData wsadata; - const WORD minVersion = MAKEWORD(2, 2); - WSAStartup(minVersion, &wsadata); -#endif - ListenSock = socket(PF_INET, SOCK_STREAM, IPPROTO_TCP); - idbassert_s(ListenSock >= 0, string("socket create error: ") + strerror(errno)); - //if (ListenSock < 0) throw runtime_error(string("socket create error: ") + strerror(errno)); -#ifndef _MSC_VER - int optval = 1; - setsockopt(ListenSock, SOL_SOCKET, SO_REUSEADDR, reinterpret_cast(&optval), sizeof(optval)); -#endif - int rc = 0; - struct sockaddr_in serv_addr; - struct in_addr la; - inet_aton("127.0.0.1", &la); - memset(&serv_addr, 0, sizeof(serv_addr)); - serv_addr.sin_family = AF_INET; - serv_addr.sin_addr.s_addr = la.s_addr; - serv_addr.sin_port = htons(PortNo); - const int MaxTries = 5 * 60 / 10; - int tries = 0; -again: - rc = ::bind(ListenSock, (sockaddr*)&serv_addr, sizeof(serv_addr)); - - if (rc < 0) - { -#ifdef _MSC_VER - int x = WSAGetLastError(); - - if (x == WSAEADDRINUSE) -#else - if (errno == EADDRINUSE) -#endif - { - //cerr << "Addr in use..." << endl; - if (++tries >= MaxTries) - { - log("Waited too long for socket to bind...giving up"); - //cerr << "Waited too long for socket to bind...giving up" << endl; - exit(1); - } - - sleep(10); - goto again; - } - - idbassert_s(0, string("socket bind error: ") + strerror(errno)); - //throw runtime_error(string("socket bind error: ") + strerror(errno)); - } - - rc = listen(ListenSock, 16); - idbassert_s(rc >= 0, string("socket listen error") + strerror(errno)); - //if (rc < 0) throw runtime_error(string("socket listen error") + strerror(errno)); - - return true; -} - -#ifndef _MSC_VER -void readn(int fd, void* buf, const size_t wanted) -{ - size_t needed = wanted; - size_t sofar = 0; - char* p = reinterpret_cast(buf); - ssize_t rrc = -1; - pollfd fds[1]; - int en = 0; - int prc = 0; - ostringstream oss; - unsigned zerocount = 0; - - fds[0].fd = fd; - fds[0].events = POLLIN; - - while (wanted > sofar) - { - fds[0].revents = 0; - errno = 0; - prc = poll(fds, 1, -1); - en = errno; - - if (prc <= 0) - { - if (en == EAGAIN || en == EINTR || en == 512) - continue; - - oss << "decomsvr::readn: poll() returned " << prc << " (" << strerror(en) << ")"; - idbassert_s(0, oss.str()); - } - - //no data on fd - if ((fds[0].revents & POLLIN) == 0) - { - oss << "decomsvr::readn: revents for fd " << fds[0].fd << " was " << fds[0].revents; - idbassert_s(0, oss.str()); - } - - errno = 0; - rrc = read(fd, (p + sofar), needed); - en = errno; - - if (rrc < 0) - { - if (en == EAGAIN || en == EINTR || en == 512) - continue; - - oss << "decomsvr::readn: read() returned " << rrc << " (" << strerror(en) << ")"; - idbassert_s(0, oss.str()); - } - - if (rrc == 0) - { - ostringstream os; - zerocount++; - - if (zerocount >= 10) - { - os << "decomsvr::readn(): too many zero-length reads!"; - idbassert_s(0, oss.str()); - } - - os << "decomsvr::readn(): zero-length read on fd " << fd; - log(os.str()); - sleep(1); - } - else - zerocount = 0; - - needed -= rrc; - sofar += rrc; - } -} - -size_t writen(int fd, const void* data, size_t nbytes) -{ - size_t nleft; - ssize_t nwritten; - const char* bufp = (const char*) data; - nleft = nbytes; - - while (nleft > 0) - { - // the O_NONBLOCK flag is not set, this is a blocking I/O. - if ((nwritten = ::write(fd, bufp, nleft)) < 0) - { - if (errno == EINTR) - nwritten = 0; - else - { - // save the error no first - int e = errno; - string errorMsg = "decomsvr: write() error: "; - scoped_array buf(new char[80]); -#if STRERROR_R_CHAR_P - const char* p; - - if ((p = strerror_r(e, buf.get(), 80)) != 0) - errorMsg += p; - -#else - int p; - - if ((p = strerror_r(e, buf.get(), 80)) == 0) - errorMsg += buf.get(); - -#endif - idbassert_s(0, errorMsg); - } - } - - nleft -= nwritten; - bufp += nwritten; - } - - return nbytes; -} -#else -void reads(SOCKET fd, void* buf, const size_t wanted) -{ - size_t needed = wanted; - size_t sofar = 0; - char* p = reinterpret_cast(buf); - ssize_t rrc = -1; - pollfd fds[1]; - int en = 0; - - fds[0].fd = fd; - fds[0].events = POLLIN; - - while (wanted > sofar) - { - fds[0].revents = 0; - poll(fds, 1, -1); - errno = 0; - rrc = recv(fd, (p + sofar), (int)needed, 0); - en = errno; - - if (rrc < 0) - { - if (en == EAGAIN || en == EINTR) - continue; - - ostringstream oss; - oss << "read() returned " << rrc << " (" << strerror(en) << ")"; - idbassert_s(0, oss.str()); - } - - needed -= rrc; - sofar += rrc; - } -} -#endif - -uint32_t readNumber32(SockType fd) -{ - uint32_t np; - SockReadFcn(fd, &np, 4); - return np; -} - -string readString(SockType fd) -{ - string s; - uint32_t len = readNumber32(fd); - idbassert_s(len <= 64, "while reading a string len (>64)"); - //if (len > 64) - // throw runtime_error("while reading a string len (>64)"); - char* buf = (char*)alloca(len + 1); //this should be at most 65 bytes and should always succeed - SockReadFcn(fd, buf, len); - buf[len] = 0; - s = buf; - return s; -} - -DecomMessage getNextMsg(SockType fd) -{ - DecomMessage dm; - - try - { - dm.pipeName = readString(fd); - dm.isValid = true; - } - catch (runtime_error& rex) - { - cerr << "re reading ctl msg: " << rex.what() << endl; - dm.pipeName = ""; - } - catch (...) - { - cerr << "ex reading ctl msg" << endl; - dm.pipeName = ""; - } - - return dm; -} - -// Murmur3 from code.google.com - -uint64_t fmix(uint64_t k) -{ - k ^= k >> 33; - k *= 0xff51afd7ed558ccdULL; - k ^= k >> 33; - k *= 0xc4ceb9fe1a85ec53ULL; - k ^= k >> 33; - - return k; -} - -uint64_t rotl64(uint64_t x, int8_t r) -{ - return (x << r) | (x >> (64 - r)); -} - -class Hasher128 -{ -public: - inline uint64_t operator()(const char* data, uint64_t len) const - { - const int nblocks = len / 16; - - uint64_t h1 = 0; - uint64_t h2 = 0; - - const uint64_t c1 = 0x87c37b91114253d5ULL; - const uint64_t c2 = 0x4cf5ad432745937fULL; - - //---------- - // body - - const uint64_t* blocks = (const uint64_t*) (data); - - for (int i = 0; i < nblocks; i++) - { - uint64_t k1 = blocks[i * 2 + 0]; - uint64_t k2 = blocks[i * 2 + 1]; - - k1 *= c1; - k1 = rotl64(k1, 31); - k1 *= c2; - h1 ^= k1; - - h1 = rotl64(h1, 27); - h1 += h2; - h1 = h1 * 5 + 0x52dce729; - - k2 *= c2; - k2 = rotl64(k2, 33); - k2 *= c1; - h2 ^= k2; - - h2 = rotl64(h2, 31); - h2 += h1; - h2 = h2 * 5 + 0x38495ab5; - } - - //---------- - // tail - - const uint8_t* tail = (const uint8_t*) (data + nblocks * 16); - - uint64_t k1 = 0; - uint64_t k2 = 0; - - switch (len & 15) - { - case 15: - k2 ^= uint64_t(tail[14]) << 48; - - case 14: - k2 ^= uint64_t(tail[13]) << 40; - - case 13: - k2 ^= uint64_t(tail[12]) << 32; - - case 12: - k2 ^= uint64_t(tail[11]) << 24; - - case 11: - k2 ^= uint64_t(tail[10]) << 16; - - case 10: - k2 ^= uint64_t(tail[9]) << 8; - - case 9: - k2 ^= uint64_t(tail[8]) << 0; - k2 *= c2; - k2 = rotl64(k2, 33); - k2 *= c1; - h2 ^= k2; - - case 8: - k1 ^= uint64_t(tail[7]) << 56; - - case 7: - k1 ^= uint64_t(tail[6]) << 48; - - case 6: - k1 ^= uint64_t(tail[5]) << 40; - - case 5: - k1 ^= uint64_t(tail[4]) << 32; - - case 4: - k1 ^= uint64_t(tail[3]) << 24; - - case 3: - k1 ^= uint64_t(tail[2]) << 16; - - case 2: - k1 ^= uint64_t(tail[1]) << 8; - - case 1: - k1 ^= uint64_t(tail[0]) << 0; - k1 *= c1; - k1 = rotl64(k1, 31); - k1 *= c2; - h1 ^= k1; - }; - - //---------- - // finalization - - h1 ^= len; - - h2 ^= len; - - h1 += h2; - - h2 += h1; - - h1 = fmix(h1); - - h2 = fmix(h2); - - h1 += h2; - - h2 += h1; - - return h1; - } -}; - -int uncompressBlock(const char* in, const size_t inLen, unsigned char* out, - unsigned int& outLen) -{ - int rc = ERR_OK; - int qlzrc = 0; - - boost::scoped_ptr scratch(new qlz_state_decompress()); - - uint32_t realChecksum; - uint32_t storedChecksum; - uint32_t storedLen; - uint8_t storedMagic; - Hasher128 hasher; - - outLen = 0; - - if (inLen < 1) - { - return ERR_BADINPUT; - } - - storedMagic = *((uint8_t*) &in[SIG_OFFSET]); - - if (storedMagic == CHUNK_MAGIC1 || storedMagic == CHUNK_MAGIC2) - { - if (inLen < HEADER_SIZE) - { - return ERR_BADINPUT; - } - - storedChecksum = *((uint32_t*) &in[CHECKSUM_OFFSET]); - storedLen = *((uint32_t*) (&in[LEN_OFFSET])); - - if (inLen < storedLen + HEADER_SIZE) - { - return ERR_BADINPUT; - } - - /* We can no longer verify the checksum on ver 1.1 */ - if (storedMagic == CHUNK_MAGIC2) - { - realChecksum = hasher(&in[HEADER_SIZE], storedLen); - - if (storedChecksum != realChecksum) - { - return ERR_CHECKSUM; - } - } - - qlzrc = qlz_decompress(&in[HEADER_SIZE], out, scratch.get()); - } - else - qlzrc = qlz_decompress(in, out, scratch.get()); - - if (qlzrc == 0) - rc = ERR_DECOMPRESS; - else - outLen = qlzrc; - - return rc; -} - -struct ScopedCleaner -{ -#ifdef _MSC_VER - ScopedCleaner() : handle(INVALID_HANDLE_VALUE) { } - ~ScopedCleaner() - { - if (handle != INVALID_HANDLE_VALUE) CloseHandle(handle); - } - - HANDLE handle; -#else - ScopedCleaner() : fd(-1) { } - ~ScopedCleaner() - { - if (fd >= 0) close(fd); - } - - int fd; -#endif -}; - -void ThreadFunc::operator()() -{ - string cfifo = fDm.pipeName + ".c"; - string ufifo = fDm.pipeName + ".u"; - uint64_t ccount = 0; - ssize_t rrc = -1; - ScopedCleaner cleaner; -#ifdef _MSC_VER - HANDLE h; - h = CreateFile(cfifo.c_str(), GENERIC_READ, 0, 0, OPEN_EXISTING, FILE_ATTRIBUTE_NORMAL, 0); - idbassert_s(h != INVALID_HANDLE_VALUE, "while opening cpipe"); - //if (!(h != INVALID_HANDLE_VALUE)) - // throw runtime_error("while opening cpipe"); - cleaner.handle = h; - - DWORD nread; - BOOL drrc; - drrc = ReadFile(h, &ccount, 8, &nread, 0); - idbassert_s(drrc != 0 && nread == 8 && ccount < 8 * 1024 * 1024, "while reading from cpipe"); - //if (!(drrc != 0 && nread == 8 && ccount < 8 * 1024 * 1024)) - // throw runtime_error("while reading from cpipe"); - - scoped_array in(new char[ccount]); - - drrc = ReadFile(h, in.get(), (DWORD)ccount, &nread, 0); - idbassert_s(drrc != 0 && nread == ccount, "while reading from cpipe"); - //if (!(drrc != 0 && nread == ccount)) - // throw runtime_error("while reading from cpipe"); - - CloseHandle(h); - cleaner.handle = INVALID_HANDLE_VALUE; -#else - scoped_array in; - int fd = -1; - - try - { - fd = open(cfifo.c_str(), O_RDONLY); - idbassert_s(fd >= 0, "when opening data fifo for input"); - - cleaner.fd = fd; - - readn(fd, &ccount, 8); - - in.reset(new char[ccount]); - - readn(fd, in.get(), ccount); - - close(fd); - cleaner.fd = -1; - } - catch (std::exception& ) - { - //This is a protocol error and returning here will clean up resources on - //the stack unwind. - return; - } - -#endif -#ifdef _MSC_VER - h = CreateFile(ufifo.c_str(), GENERIC_WRITE, 0, 0, OPEN_EXISTING, FILE_ATTRIBUTE_NORMAL, 0); - idbassert_s(h != INVALID_HANDLE_VALUE, "while opening upipe"); - //if (!(h != INVALID_HANDLE_VALUE)) - // throw runtime_error("while opening upipe"); - cleaner.handle = h; - - uint64_t outlen = 512 * 1024 * 8; - unsigned int ol = static_cast(outlen); - - scoped_array out(new char[outlen]); - - int crc = uncompressBlock(in.get(), ccount, reinterpret_cast(out.get()), ol); - - if (crc != ERR_OK) - outlen = 0; - else - outlen = ol; - - BOOL dwrc; - DWORD nwritten; - dwrc = WriteFile(h, &outlen, 8, &nwritten, 0); - idbassert_s(dwrc != 0 && nwritten == 8, "while writing to upipe"); - //if (!(dwrc != 0 && nwritten == 8)) - // throw runtime_error("while writing to upipe"); - - dwrc = WriteFile(h, out.get(), (DWORD)outlen, &nwritten, 0); - idbassert_s(dwrc != 0 && nwritten == outlen, "while writing to upipe"); - //if (!(dwrc != 0 && nwritten == outlen)) - // throw runtime_error("while writing to upipe"); - - FlushFileBuffers(h); - CloseHandle(h); - cleaner.handle = INVALID_HANDLE_VALUE; -#else - scoped_array out; - - try - { - fd = open(ufifo.c_str(), O_WRONLY); - idbassert_s(fd >= 0, "when opening data fifo for output"); - //if (fd < 0) - // throw runtime_error("when opening data fifo for output"); - - cleaner.fd = fd; - - uint64_t outlen = 512 * 1024 * 8; - unsigned int ol = outlen; - - out.reset(new char[outlen]); - - int crc = uncompressBlock(in.get(), ccount, reinterpret_cast(out.get()), ol); - - if (crc != ERR_OK) - outlen = 0; - else - outlen = ol; - - rrc = writen(fd, &outlen, 8); - idbassert_s(rrc == 8, "when writing len to data fifo"); - - rrc = writen(fd, out.get(), outlen); - idbassert_s(rrc == (ssize_t)outlen, "when writing data to data fifo"); - - close(fd); - cleaner.fd = -1; - } - catch (std::exception& ) - { - //There was an error writing the uncompressed data back to PrimProc. Cleanup by - //unwinding the stack - return; - } - -#endif -} - -#ifndef _MSC_VER -void cleanupFifos() -{ - //find all existing fifos and try get rid of them.... - DIR* dirp = 0; - struct dirent* direntp = 0; - char fifoname[PATH_MAX]; - int fd = -1; - - dirp = opendir("/tmp"); - strcpy(fifoname, "/tmp/"); - - direntp = readdir(dirp); - - while (direntp != 0) - { - if (memcmp(direntp->d_name, "cdatafifo", 9) == 0) - { - strcpy(&fifoname[5], direntp->d_name); - fd = open(fifoname, O_RDONLY | O_NONBLOCK); - - //opening and closing this fifo will cause PP to unblock and retry - if (fd >= 0) - { - close(fd); - } - else - { - fd = open(fifoname, O_WRONLY | O_NONBLOCK); - - if (fd >= 0) - close(fd); - } - } - - direntp = readdir(dirp); - } - - closedir(dirp); -} -#endif - -} - -int main(int argc, char** argv) -{ - int c; - - PortNo = 0; - char* p = getenv("IDB_DECOMSVR_PORT"); - - if (p && *p) - PortNo = atoi(p); - - if (PortNo <= 0) - PortNo = 9199; - -#ifdef _MSC_VER - ListenSock = INVALID_SOCKET; -#else - ListenSock = -1; -#endif - opterr = 0; - - while ((c = getopt(argc, argv, "p:")) != -1) - switch (c) - { - case 'p': - PortNo = atoi(optarg); - break; - - case '?': - default: - break; - } - - if (!serverInit()) - { - log("Could not initialize the Decompression Server!"); - //cerr << "Could not initialize the Decompression Server!" << endl; - return 1; - } - - initCtlFifo(); - -#ifndef _MSC_VER - cleanupFifos(); -#endif - - DecomMessage m; - - for (;;) - { -#ifdef _MSC_VER - SOCKET dataSock = INVALID_SOCKET; -#else - int dataSock = -1; -#endif - dataSock = accept(ListenSock, 0, 0); -#ifdef _MSC_VER - idbassert_s(dataSock != INVALID_SOCKET, string("socket accept error: ") + strerror(errno)); - //if (dataSock == INVALID_SOCKET) - // throw runtime_error(string("socket accept error: ") + strerror(errno)); -#else - //if (dataSock < 0) - idbassert_s(dataSock >= 0, string("socket accept error: ") + strerror(errno)); -#endif - m = getNextMsg(dataSock); - shutdown(dataSock, SHUT_RDWR); -#ifdef _MSC_VER - closesocket(dataSock); -#else - close(dataSock); -#endif - - if (m.isValid) - { - ThreadFunc tf(m); -#ifdef SINGLE_THREADED - tf(); -#else - thread t(tf); -#endif - } - else - idbassert_s(0, "Invalid msg"); - - //cerr << "Invalid msg" << endl; - } - - return 0; -} - diff --git a/oam/etc/ProcessConfig.xml b/oam/etc/ProcessConfig.xml index 8a0c3618f..8088135bb 100644 --- a/oam/etc/ProcessConfig.xml +++ b/oam/etc/ProcessConfig.xml @@ -58,16 +58,6 @@ LOADSHARE off - - DecomSvr - pm - /$INSTALLDIR/bin/DecomSvr - 2 - 15 - - LOADSHARE - off - PrimProc pm diff --git a/oam/etc/ProcessConfig.xml.singleserver b/oam/etc/ProcessConfig.xml.singleserver index b01ca5585..9c543ae38 100644 --- a/oam/etc/ProcessConfig.xml.singleserver +++ b/oam/etc/ProcessConfig.xml.singleserver @@ -58,16 +58,6 @@ LOADSHARE off - - DecomSvr - pm - /usr/local/mariadb/columnstore/bin/DecomSvr - 2 - 15 - - LOADSHARE - off - PrimProc pm diff --git a/procmon/processmonitor.cpp b/procmon/processmonitor.cpp index 30b239ec1..823060ad2 100644 --- a/procmon/processmonitor.cpp +++ b/procmon/processmonitor.cpp @@ -2730,20 +2730,6 @@ pid_t ProcessMonitor::startProcess(string processModuleType, string processName, return oam::API_MINOR_FAILURE; } - if (processLocation.find("DecomSvr") != string::npos) - { - // DecomSvr app is special - - sleep(1); - //record the process information into processList - config.buildList(processModuleType, processName, processLocation, arg_list, - launchID, newProcessID, oam::ACTIVE, BootLaunch, RunType, - DepProcessName, DepModuleName, LogFile); - - //Update Process Status: Mark Process oam::ACTIVE state - updateProcessInfo(processName, oam::ACTIVE, newProcessID); - } - //FYI - NEEDS TO STAY HERE TO HAVE newProcessID //record the process information into processList @@ -2797,14 +2783,6 @@ pid_t ProcessMonitor::startProcess(string processModuleType, string processName, close(i); } - // open STDIN, STDOUT & STDERR for trapDaemon and DecomSvr - if (processName == "DecomSvr" ) - { - open("/dev/null", O_RDONLY); //Should be fd 0 - open("/dev/null", O_WRONLY); //Should be fd 1 - open("/dev/null", O_WRONLY); //Should be fd 2 - } - else { int fd; fd = open("/dev/null", O_RDONLY);