From 8f1136474f68fc818d0077112be658a950bb6f2c Mon Sep 17 00:00:00 2001 From: Andrew Hutchings Date: Sat, 24 Feb 2018 08:04:17 -0500 Subject: [PATCH 01/49] MCOL-1228 Allow alter table for TEXT/BLOB CHANGE COLUMN was blocked for TEXT and BLOB types. This fix applies to things like TINYTEXT as well as the only difference internally is the column width. --- dbcon/ddlpackageproc/altertableprocessor.cpp | 3 +++ 1 file changed, 3 insertions(+) diff --git a/dbcon/ddlpackageproc/altertableprocessor.cpp b/dbcon/ddlpackageproc/altertableprocessor.cpp index 7c2cdd249..ba8d7ad7b 100644 --- a/dbcon/ddlpackageproc/altertableprocessor.cpp +++ b/dbcon/ddlpackageproc/altertableprocessor.cpp @@ -174,7 +174,10 @@ bool typesAreSame(const CalpontSystemCatalog::ColType& colType, const ColumnType case (CalpontSystemCatalog::CLOB): break; case (CalpontSystemCatalog::BLOB): + if (newType.fType == DDL_BLOB && colType.colWidth == newType.fLength) return true; + break; case (CalpontSystemCatalog::TEXT): + if (newType.fType == DDL_TEXT && colType.colWidth == newType.fLength) return true; break; default: break; From bd71dd1d7a728e9002e52953b43cc1f278ff1e2d Mon Sep 17 00:00:00 2001 From: Andrew Hutchings Date: Sat, 24 Feb 2018 18:34:59 +0000 Subject: [PATCH 02/49] MCOL-1084 Fix TABLE_USAGE() when dict is 0 bytes Dict size of 0 bytes returns NULL which means that the total count is 0. This fix uses COALESCE to stop this happening. --- dbcon/mysql/columnstore_info.sql | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) diff --git a/dbcon/mysql/columnstore_info.sql b/dbcon/mysql/columnstore_info.sql index 80b0f12b1..563052a11 100644 --- a/dbcon/mysql/columnstore_info.sql +++ b/dbcon/mysql/columnstore_info.sql @@ -52,21 +52,21 @@ CREATE PROCEDURE table_usage (IN t_schema char(64), IN t_name char(64)) CREATE TABLE columnstore_info.columnstore_files engine=myisam as (select * from information_schema.columnstore_files); ALTER TABLE columnstore_info.columnstore_files ADD INDEX `object_id` (`object_id`); IF t_name IS NOT NULL THEN -SELECT TABLE_SCHEMA, TABLE_NAME, columnstore_info.format_filesize(data) as DATA_DISK_USAGE, columnstore_info.format_filesize(dict) as DICT_DISK_USAGE, columnstore_info.format_filesize(data + dict) as TOTAL_USAGE FROM ( +SELECT TABLE_SCHEMA, TABLE_NAME, columnstore_info.format_filesize(data) as DATA_DISK_USAGE, columnstore_info.format_filesize(dict) as DICT_DISK_USAGE, columnstore_info.format_filesize(data + COALESCE(dict, 0)) as TOTAL_USAGE FROM ( SELECT TABLE_SCHEMA, TABLE_NAME, (SELECT sum(cf.file_size) as data FROM columnstore_info.columnstore_columns cc JOIN columnstore_info.columnstore_files cf ON cc.object_id = cf.object_id WHERE table_name = ics.table_name and table_schema = ics.table_schema) as data, (SELECT sum(cf.file_size) as dict FROM columnstore_info.columnstore_columns cc JOIN columnstore_info.columnstore_files cf ON cc.dictionary_object_id = cf.object_id WHERE table_name = ics.table_name and table_schema = ics.table_schema GROUP BY table_schema, table_name) as dict FROM columnstore_info.columnstore_columns ics where table_name = t_name and (table_schema = t_schema or t_schema IS NULL) group by table_schema, table_name ) q; ELSEIF t_schema IS NOT NULL THEN -SELECT TABLE_SCHEMA, TABLE_NAME, columnstore_info.format_filesize(data) as DATA_DISK_USAGE, columnstore_info.format_filesize(dict) as DICT_DISK_USAGE, columnstore_info.format_filesize(data + dict) as TOTAL_USAGE FROM ( +SELECT TABLE_SCHEMA, TABLE_NAME, columnstore_info.format_filesize(data) as DATA_DISK_USAGE, columnstore_info.format_filesize(dict) as DICT_DISK_USAGE, columnstore_info.format_filesize(data + COALESCE(dict, 0)) as TOTAL_USAGE FROM ( SELECT TABLE_SCHEMA, TABLE_NAME, (SELECT sum(cf.file_size) as data FROM columnstore_info.columnstore_columns cc JOIN columnstore_info.columnstore_files cf ON cc.object_id = cf.object_id WHERE table_name = ics.table_name and table_schema = ics.table_schema) as data, (SELECT sum(cf.file_size) as dict FROM columnstore_info.columnstore_columns cc JOIN columnstore_info.columnstore_files cf ON cc.dictionary_object_id = cf.object_id WHERE table_name = ics.table_name and table_schema = ics.table_schema GROUP BY table_schema, table_name) as dict FROM columnstore_info.columnstore_columns ics where table_schema = t_schema group by table_schema, table_name ) q; ELSE -SELECT TABLE_SCHEMA, TABLE_NAME, columnstore_info.format_filesize(data) as DATA_DISK_USAGE, columnstore_info.format_filesize(dict) as DICT_DISK_USAGE, columnstore_info.format_filesize(data + dict) as TOTAL_USAGE FROM ( +SELECT TABLE_SCHEMA, TABLE_NAME, columnstore_info.format_filesize(data) as DATA_DISK_USAGE, columnstore_info.format_filesize(dict) as DICT_DISK_USAGE, columnstore_info.format_filesize(data + COALESCE(dict, 0)) as TOTAL_USAGE FROM ( SELECT TABLE_SCHEMA, TABLE_NAME, (SELECT sum(cf.file_size) as data FROM columnstore_info.columnstore_columns cc JOIN columnstore_info.columnstore_files cf ON cc.object_id = cf.object_id WHERE table_name = ics.table_name and table_schema = ics.table_schema) as data, (SELECT sum(cf.file_size) as dict FROM columnstore_info.columnstore_columns cc JOIN columnstore_info.columnstore_files cf ON cc.dictionary_object_id = cf.object_id WHERE table_name = ics.table_name and table_schema = ics.table_schema GROUP BY table_schema, table_name) as dict FROM columnstore_info.columnstore_columns ics From 9e2d8b8e71c2b626b39a145fe82c915a7b11b4f2 Mon Sep 17 00:00:00 2001 From: david hill Date: Wed, 28 Feb 2018 11:12:02 -0500 Subject: [PATCH 03/49] mcol-1235 - fix issue with too many files open --- procmgr/main.cpp | 9 +++++++-- 1 file changed, 7 insertions(+), 2 deletions(-) diff --git a/procmgr/main.cpp b/procmgr/main.cpp index 9fed1540f..3dd9766e0 100644 --- a/procmgr/main.cpp +++ b/procmgr/main.cpp @@ -553,8 +553,11 @@ static void alarmMessageThread(Configuration config) msg = fIos.read(); if (msg.length() <= 0) - continue; - + { + fIos.close(); + continue; + } + //log.writeLog(__LINE__, "MSG RECEIVED: Process Alarm Message"); ByteStream::byte alarmID; @@ -585,6 +588,8 @@ static void alarmMessageThread(Configuration config) ALARMManager aManager; aManager.processAlarmReport(calAlarm); + + fIos.close(); } catch (exception& ex) { From 1ad112a44d3d7770d102a6596b59532c4ac107cc Mon Sep 17 00:00:00 2001 From: david hill Date: Mon, 5 Mar 2018 10:48:46 -0600 Subject: [PATCH 04/49] mcol-1235 - add close in exceptions --- procmgr/main.cpp | 2 ++ 1 file changed, 2 insertions(+) diff --git a/procmgr/main.cpp b/procmgr/main.cpp index 3dd9766e0..434902138 100644 --- a/procmgr/main.cpp +++ b/procmgr/main.cpp @@ -595,11 +595,13 @@ static void alarmMessageThread(Configuration config) { string error = ex.what(); log.writeLog(__LINE__, "EXCEPTION ERROR on read for ProcMgr_Alarm:" + error, LOG_TYPE_ERROR); + fIos.close(); continue; } catch(...) { log.writeLog(__LINE__, "EXCEPTION ERROR on read for ProcMgr_Alarm: Caught unknown exception!", LOG_TYPE_ERROR); + fIos.close(); continue; } } From 8d0312950bfb57a46947e3e6cbe7f83855c830de Mon Sep 17 00:00:00 2001 From: david hill Date: Mon, 5 Mar 2018 11:36:10 -0600 Subject: [PATCH 05/49] update version 1.1.3-2 --- VERSION | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/VERSION b/VERSION index 445a70a7a..21aec268d 100644 --- a/VERSION +++ b/VERSION @@ -1,4 +1,4 @@ COLUMNSTORE_VERSION_MAJOR=1 COLUMNSTORE_VERSION_MINOR=1 -COLUMNSTORE_VERSION_PATCH=4 -COLUMNSTORE_VERSION_RELEASE=1 +COLUMNSTORE_VERSION_PATCH=3 +COLUMNSTORE_VERSION_RELEASE=2 From f96410a3dbf4589c76e98dd4e2879471a53664d8 Mon Sep 17 00:00:00 2001 From: Ben Thompson Date: Mon, 5 Mar 2018 15:27:17 -0600 Subject: [PATCH 06/49] Update ubuntu package dependencies to require libsnappy1v5 and not snappy the media player --- cpackEngineDEB.cmake | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/cpackEngineDEB.cmake b/cpackEngineDEB.cmake index 42c1f6974..5f8a83c42 100644 --- a/cpackEngineDEB.cmake +++ b/cpackEngineDEB.cmake @@ -69,7 +69,7 @@ if ("${DEBIAN_VERSION_NUMBER}" EQUAL "8") elseif ("${DEBIAN_VERSION_NUMBER}" EQUAL "9") SET(CPACK_DEBIAN_PLATFORM_PACKAGE_DEPENDS "expect, perl, openssl, file, sudo, libdbi-perl, libreadline-dev, rsync, net-tools, libboost-all-dev, mariadb-columnstore-libs, mariadb-columnstore-server, libsnappy1v5, libreadline5") else() - SET(CPACK_DEBIAN_PLATFORM_PACKAGE_DEPENDS "expect, perl, openssl, file, sudo, libdbi-perl, libboost-all-dev, libreadline-dev, rsync, snappy, net-tools") + SET(CPACK_DEBIAN_PLATFORM_PACKAGE_DEPENDS "expect, perl, openssl, file, sudo, libdbi-perl, libboost-all-dev, libreadline-dev, rsync, libsnappy1v5, net-tools") endif () SET(CPACK_DEBIAN_STORAGE-ENGINE_PACKAGE_DEPENDS "mariadb-columnstore-libs") From 208bb8b6b93ebc7efc8e691cb61228e0744be53e Mon Sep 17 00:00:00 2001 From: david hill Date: Mon, 5 Mar 2018 16:12:25 -0600 Subject: [PATCH 07/49] MCOL-1224 move touch inside of root check and added SUDO for non-root --- oam/install_scripts/post-install | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/oam/install_scripts/post-install b/oam/install_scripts/post-install index a295a088f..2c11ff12c 100755 --- a/oam/install_scripts/post-install +++ b/oam/install_scripts/post-install @@ -183,11 +183,11 @@ else RCFILE=/etc/rc.local fi -touch $RCFILE - if [ $user = "root" ]; then + touch $RCFILE chmod +x $RCFILE else + $SUDO touch $RCFILE $SUDO chmod 777 $RCFILE printf '%s\n' '#!/bin/bash' "#" | $SUDO tee -a $RCFILEl > /dev/null 2>&1 From 278b1f33640e4f23b53c35e8dbe11d75bbc57f2c Mon Sep 17 00:00:00 2001 From: david hill Date: Mon, 5 Mar 2018 16:15:57 -0600 Subject: [PATCH 08/49] Update post-install --- oam/install_scripts/post-install | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/oam/install_scripts/post-install b/oam/install_scripts/post-install index 2c11ff12c..fba043cef 100755 --- a/oam/install_scripts/post-install +++ b/oam/install_scripts/post-install @@ -189,7 +189,7 @@ if [ $user = "root" ]; then else $SUDO touch $RCFILE $SUDO chmod 777 $RCFILE - printf '%s\n' '#!/bin/bash' "#" | $SUDO tee -a $RCFILEl > /dev/null 2>&1 + $SUDO printf '%s\n' '#!/bin/bash' "#" | $SUDO tee -a $RCFILEl > /dev/null 2>&1 if [ -n "$systemctl" ]; then $SUDO systemctl start rc-local >/dev/null 2>&1 From ad67f17c546dbe19788f43919d3de66d96650b7c Mon Sep 17 00:00:00 2001 From: david hill Date: Mon, 5 Mar 2018 16:19:25 -0600 Subject: [PATCH 09/49] Update post-install --- oam/install_scripts/post-install | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/oam/install_scripts/post-install b/oam/install_scripts/post-install index fba043cef..5f0feee8f 100755 --- a/oam/install_scripts/post-install +++ b/oam/install_scripts/post-install @@ -189,7 +189,7 @@ if [ $user = "root" ]; then else $SUDO touch $RCFILE $SUDO chmod 777 $RCFILE - $SUDO printf '%s\n' '#!/bin/bash' "#" | $SUDO tee -a $RCFILEl > /dev/null 2>&1 + $SUDO printf '%s\n' '#!/bin/bash' "#" | $SUDO tee -a $RCFILE > /dev/null 2>&1 if [ -n "$systemctl" ]; then $SUDO systemctl start rc-local >/dev/null 2>&1 From 75ebfc91425ee248bead67e2788c0785378561d9 Mon Sep 17 00:00:00 2001 From: david hill Date: Wed, 7 Mar 2018 10:02:08 -0600 Subject: [PATCH 10/49] update version bump to 1.1.4 --- VERSION | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/VERSION b/VERSION index 21aec268d..445a70a7a 100644 --- a/VERSION +++ b/VERSION @@ -1,4 +1,4 @@ COLUMNSTORE_VERSION_MAJOR=1 COLUMNSTORE_VERSION_MINOR=1 -COLUMNSTORE_VERSION_PATCH=3 -COLUMNSTORE_VERSION_RELEASE=2 +COLUMNSTORE_VERSION_PATCH=4 +COLUMNSTORE_VERSION_RELEASE=1 From 77a9e49f1028676bf139bf7b6ffc92fcab98a405 Mon Sep 17 00:00:00 2001 From: david hill Date: Wed, 7 Mar 2018 10:48:16 -0600 Subject: [PATCH 11/49] MCOL-1225 - change mysql lib path --- oam/install_scripts/columnstore | 2 +- oam/install_scripts/module_installer.sh | 2 +- oam/install_scripts/post-install | 2 +- oam/install_scripts/post-mysql-install | 2 +- oamapps/calpontSupport/findStranded.sh | 2 +- oamapps/calpontSupport/sqlLogs.sh | 2 +- oamapps/columnstoreSupport/findStranded.sh | 2 +- oamapps/columnstoreSupport/sqlLogs.sh | 2 +- tools/setConfig/configxml.sh | 2 +- 9 files changed, 9 insertions(+), 9 deletions(-) diff --git a/oam/install_scripts/columnstore b/oam/install_scripts/columnstore index f4fc48157..038ad0c43 100644 --- a/oam/install_scripts/columnstore +++ b/oam/install_scripts/columnstore @@ -36,7 +36,7 @@ InstallDir=$COLUMNSTORE_INSTALL_DIR if [ $InstallDir != "/usr/local/mariadb/columnstore" ]; then export PATH=$InstallDir/bin:$InstallDir/mysql/bin:/bin:/usr/bin - export LD_LIBRARY_PATH=$LD_LIBRARY_PATH:$InstallDir/lib:$InstallDir/mysql/lib/mysql + export LD_LIBRARY_PATH=$LD_LIBRARY_PATH:$InstallDir/lib:$InstallDir/mysql/lib fi #hadoop diff --git a/oam/install_scripts/module_installer.sh b/oam/install_scripts/module_installer.sh index 10cefaf5f..6ca39b70f 100755 --- a/oam/install_scripts/module_installer.sh +++ b/oam/install_scripts/module_installer.sh @@ -47,7 +47,7 @@ shift $shiftcnt if [ $installdir != "/usr/local/mariadb/columnstore" ]; then export COLUMNSTORE_INSTALL_DIR=$installdir export PATH=$COLUMNSTORE_INSTALL_DIR/bin:$COLUMNSTORE_INSTALL_DIR/mysql/bin:/bin:/usr/bin - export LD_LIBRARY_PATH=$LD_LIBRARY_PATH:$COLUMNSTORE_INSTALL_DIR/lib:$COLUMNSTORE_INSTALL_DIR/mysql/lib/mysql + export LD_LIBRARY_PATH=$LD_LIBRARY_PATH:$COLUMNSTORE_INSTALL_DIR/lib:$COLUMNSTORE_INSTALL_DIR/mysql/lib else export COLUMNSTORE_INSTALL_DIR=$installdir fi diff --git a/oam/install_scripts/post-install b/oam/install_scripts/post-install index 5f0feee8f..781f3a5ee 100755 --- a/oam/install_scripts/post-install +++ b/oam/install_scripts/post-install @@ -112,7 +112,7 @@ if [ $user != "root" ]; then sudo echo " " >> ${profileFileEnv} sudo echo "# MariaDB Columnstore Non-Root Environment Variables" >> ${profileFileEnv} sudo echo "export COLUMNSTORE_INSTALL_DIR=$COLUMNSTORE_INSTALL_DIR" >> ${profileFileEnv} - sudo echo "export LD_LIBRARY_PATH=$LD_LIBRARY_PATH:$COLUMNSTORE_INSTALL_DIR/lib:$COLUMNSTORE_INSTALL_DIR/mysql/lib/mysql" >> ${profileFileEnv} + sudo echo "export LD_LIBRARY_PATH=$LD_LIBRARY_PATH:$COLUMNSTORE_INSTALL_DIR/lib:$COLUMNSTORE_INSTALL_DIR/mysql/lib" >> ${profileFileEnv} . ${profileFileEnv} fi diff --git a/oam/install_scripts/post-mysql-install b/oam/install_scripts/post-mysql-install index 08c38b84e..24711b048 100755 --- a/oam/install_scripts/post-mysql-install +++ b/oam/install_scripts/post-mysql-install @@ -64,7 +64,7 @@ USER=`whoami 2>/dev/null` if [ $USER != "root" ]; then sudo ldconfig >/dev/null 2>&1 export COLUMNSTORE_INSTALL_DIR=$installdir - export LD_LIBRARY_PATH=$LD_LIBRARY_PATH:$COLUMNSTORE_INSTALL_DIR/lib:$COLUMNSTORE_INSTALL_DIR/mysql/lib/mysql + export LD_LIBRARY_PATH=$LD_LIBRARY_PATH:$COLUMNSTORE_INSTALL_DIR/lib:$COLUMNSTORE_INSTALL_DIR/mysql/lib else ldconfig fi diff --git a/oamapps/calpontSupport/findStranded.sh b/oamapps/calpontSupport/findStranded.sh index 6d6ca28a3..1263aace9 100755 --- a/oamapps/calpontSupport/findStranded.sh +++ b/oamapps/calpontSupport/findStranded.sh @@ -18,7 +18,7 @@ export COLUMNSTORE_INSTALL_DIR=$COLUMNSTORE_INSTALL_DIR if [ $COLUMNSTORE_INSTALL_DIR != "/usr/local/mariadb/columnstore" ]; then export PATH=$COLUMNSTORE_INSTALL_DIR/bin:$COLUMNSTORE_INSTALL_DIR/mysql/bin:/bin:/usr/bin - export LD_LIBRARY_PATH=$COLUMNSTORE_INSTALL_DIR/lib:$COLUMNSTORE_INSTALL_DIR/mysql/lib/mysql + export LD_LIBRARY_PATH=$COLUMNSTORE_INSTALL_DIR/lib:$COLUMNSTORE_INSTALL_DIR/mysql/lib fi cd $COLUMNSTORE_INSTALL_DIR diff --git a/oamapps/calpontSupport/sqlLogs.sh b/oamapps/calpontSupport/sqlLogs.sh index a1a9a403c..818ab546c 100755 --- a/oamapps/calpontSupport/sqlLogs.sh +++ b/oamapps/calpontSupport/sqlLogs.sh @@ -14,7 +14,7 @@ export COLUMNSTORE_INSTALL_DIR=$COLUMNSTORE_INSTALL_DIR if [ $COLUMNSTORE_INSTALL_DIR != "/usr/local/mariadb/columnstore" ]; then export PATH=$COLUMNSTORE_INSTALL_DIR/bin:$COLUMNSTORE_INSTALL_DIR/mysql/bin:/bin:/usr/bin - export LD_LIBRARY_PATH=$COLUMNSTORE_INSTALL_DIR/lib:$COLUMNSTORE_INSTALL_DIR/mysql/lib/mysql + export LD_LIBRARY_PATH=$COLUMNSTORE_INSTALL_DIR/lib:$COLUMNSTORE_INSTALL_DIR/mysql/lib fi diff --git a/oamapps/columnstoreSupport/findStranded.sh b/oamapps/columnstoreSupport/findStranded.sh index 8577e06e3..12e160172 100755 --- a/oamapps/columnstoreSupport/findStranded.sh +++ b/oamapps/columnstoreSupport/findStranded.sh @@ -18,7 +18,7 @@ export COLUMNSTORE_INSTALL_DIR=$COLUMNSTORE_INSTALL_DIR if [ $COLUMNSTORE_INSTALL_DIR != "/usr/local/mariadb/columnstore" ]; then export PATH=$COLUMNSTORE_INSTALL_DIR/bin:$COLUMNSTORE_INSTALL_DIR/mysql/bin:/bin:/usr/bin - export LD_LIBRARY_PATH=$COLUMNSTORE_INSTALL_DIR/lib:$COLUMNSTORE_INSTALL_DIR/mysql/lib/mysql + export LD_LIBRARY_PATH=$COLUMNSTORE_INSTALL_DIR/lib:$COLUMNSTORE_INSTALL_DIR/mysql/lib fi cd $COLUMNSTORE_INSTALL_DIR diff --git a/oamapps/columnstoreSupport/sqlLogs.sh b/oamapps/columnstoreSupport/sqlLogs.sh index 385b2d912..564973e7e 100755 --- a/oamapps/columnstoreSupport/sqlLogs.sh +++ b/oamapps/columnstoreSupport/sqlLogs.sh @@ -14,7 +14,7 @@ export COLUMNSTORE_INSTALL_DIR=$COLUMNSTORE_INSTALL_DIR if [ $COLUMNSTORE_INSTALL_DIR != "/usr/local/mariadb/columnstore" ]; then export PATH=$COLUMNSTORE_INSTALL_DIR/bin:$COLUMNSTORE_INSTALL_DIR/mysql/bin:/bin:/usr/bin - export LD_LIBRARY_PATH=$COLUMNSTORE_INSTALL_DIR/lib:$COLUMNSTORE_INSTALL_DIR/mysql/lib/mysql + export LD_LIBRARY_PATH=$COLUMNSTORE_INSTALL_DIR/lib:$COLUMNSTORE_INSTALL_DIR/mysql/lib fi diff --git a/tools/setConfig/configxml.sh b/tools/setConfig/configxml.sh index 2c3568f00..d78953660 100755 --- a/tools/setConfig/configxml.sh +++ b/tools/setConfig/configxml.sh @@ -18,7 +18,7 @@ InstallDir=$COLUMNSTORE_INSTALL_DIR if [ $InstallDir != "/usr/local/mariadb/columnstore" ]; then export PATH=$InstallDir/bin:$InstallDir/mysql/bin:/bin:/usr/bin - export LD_LIBRARY_PATH=$InstallDir/lib:$InstallDir/mysql/lib/mysql + export LD_LIBRARY_PATH=$InstallDir/lib:$InstallDir/mysql/lib fi case "$1" in From 17e954db7de3a8cc2c68241a2a653af48ca9e516 Mon Sep 17 00:00:00 2001 From: Andrew Hutchings Date: Wed, 7 Mar 2018 16:56:42 +0000 Subject: [PATCH 12/49] MCOL-1246 Fix string matching for whitespace For equality string matches other engines ignore trailing whitespace (this does not apply to LIKE matches). So we should do the same. This patch trims whitespace for MIN/MAX extent elimination checks, fixed width columns and dictionary columns during equality matches against constants (SELECT * FROM t1 WHERE b = 'ABC'). --- dbcon/joblist/lbidlist.cpp | 10 +++++++++- primitives/linux-port/column.cpp | 7 +++++++ primitives/linux-port/dictionary.cpp | 6 +++++- utils/dataconvert/dataconvert.h | 13 +++++++++++++ 4 files changed, 34 insertions(+), 2 deletions(-) diff --git a/dbcon/joblist/lbidlist.cpp b/dbcon/joblist/lbidlist.cpp index 69fbbb7c2..b7cbe9e57 100644 --- a/dbcon/joblist/lbidlist.cpp +++ b/dbcon/joblist/lbidlist.cpp @@ -27,6 +27,7 @@ #include "calpontsystemcatalog.h" #include "brm.h" #include "brmtypes.h" +#include "dataconvert.h" #define IS_VERBOSE (fDebug >= 4) #define IS_DETAIL (fDebug >= 3) @@ -653,7 +654,14 @@ bool LBIDList::CasualPartitionPredicate(const int64_t Min, if (bIsChar && 1 < ct.colWidth) { - scan = compareVal(order_swap(Min), order_swap(Max), order_swap(value), + // MCOL-1246 Trim trailing whitespace for matching so that we have + // the same as InnoDB behaviour + int64_t tMin = Min; + int64_t tMax = Max; + dataconvert::DataConvert::trimWhitespace(tMin); + dataconvert::DataConvert::trimWhitespace(tMax); + + scan = compareVal(order_swap(tMin), order_swap(tMax), order_swap(value), op, lcf); // cout << "scan=" << (uint32_t) scan << endl; } diff --git a/primitives/linux-port/column.cpp b/primitives/linux-port/column.cpp index 6804c8b21..cdb70ef67 100644 --- a/primitives/linux-port/column.cpp +++ b/primitives/linux-port/column.cpp @@ -39,6 +39,7 @@ using namespace boost; #include "we_type.h" #include "stats.h" #include "primproc.h" +#include "dataconvert.h" using namespace logging; using namespace dbbc; using namespace primitives; @@ -527,7 +528,13 @@ inline bool colCompare(int64_t val1, int64_t val2, uint8_t COP, uint8_t rf, int type == CalpontSystemCatalog::TEXT) && !isNull ) { if (!regex.used && !rf) + { + // MCOL-1246 Trim trailing whitespace for matching, but not for + // regex + dataconvert::DataConvert::trimWhitespace(val1); + dataconvert::DataConvert::trimWhitespace(val2); return colCompare_(order_swap(val1), order_swap(val2), COP); + } else return colStrCompare_(order_swap(val1), order_swap(val2), COP, rf, ®ex); } diff --git a/primitives/linux-port/dictionary.cpp b/primitives/linux-port/dictionary.cpp index 0cc56df34..88d71dc3d 100644 --- a/primitives/linux-port/dictionary.cpp +++ b/primitives/linux-port/dictionary.cpp @@ -21,6 +21,7 @@ #include #include +#include #include using namespace std; @@ -164,7 +165,10 @@ void PrimitiveProcessor::p_TokenByScan(const TokenByScanRequestHeader *h, string arg_utf8; if (eqFilter) { - bool gotIt = eqFilter->find(string(sig, siglen)) != eqFilter->end(); + // MCOL-1246 Trim whitespace before match + string strData(sig, siglen); + boost::trim_right(strData); + bool gotIt = eqFilter->find(strData) != eqFilter->end(); if ((h->COP1 == COMPARE_EQ && gotIt) || (h->COP1 == COMPARE_NE && !gotIt)) goto store; diff --git a/utils/dataconvert/dataconvert.h b/utils/dataconvert/dataconvert.h index 019ebcd77..aba5a8bd8 100644 --- a/utils/dataconvert/dataconvert.h +++ b/utils/dataconvert/dataconvert.h @@ -408,6 +408,7 @@ public: static inline std::string decimalToString(int64_t value, uint8_t scale, execplan::CalpontSystemCatalog::ColDataType colDataType); static inline void decimalToString(int64_t value, uint8_t scale, char* buf, unsigned int buflen, execplan::CalpontSystemCatalog::ColDataType colDataType); static inline std::string constructRegexp(const std::string& str); + static inline void trimWhitespace(int64_t &charData); static inline bool isEscapedChar(char c) { return ('%' == c || '_' == c); } // convert string to date @@ -552,6 +553,18 @@ inline void DataConvert::decimalToString(int64_t int_val, uint8_t scale, char* b *(ptr + l1) = '.'; } +inline void DataConvert::trimWhitespace(int64_t &charData) +{ + // Trims whitespace characters off non-dict character data + char *ch_data = (char*) &charData; + for (int8_t i = 7; i > 0; i--) + { + if (isspace(ch_data[i]) || ch_data[i] == '\0') + ch_data[i] = '\0'; + else + break; + } +} //FIXME: copy/pasted from dictionary.cpp: refactor inline std::string DataConvert::constructRegexp(const std::string& str) From b800fc7a2d1702dc2f2c4732803cd81738288a64 Mon Sep 17 00:00:00 2001 From: david hill Date: Wed, 7 Mar 2018 14:30:57 -0600 Subject: [PATCH 13/49] MCOL-1225 - add to post-install to remove older profile files --- oam/install_scripts/post-install | 3 +++ 1 file changed, 3 insertions(+) diff --git a/oam/install_scripts/post-install b/oam/install_scripts/post-install index 781f3a5ee..f7aeeb2ca 100755 --- a/oam/install_scripts/post-install +++ b/oam/install_scripts/post-install @@ -104,6 +104,9 @@ if [ $installdir != "/usr/local/mariadb/columnstore" ]; then fi if [ $user != "root" ]; then + sudo rm -f $profileFileEnv + sudo rm -f $profileFileAlias + sudo touch $profileFileEnv sudo chmod 666 $profileFileEnv egrep -qs 'MariaDB Columnstore Non-Root' ${profileFileEnv} From 905ce2ce8b6d6a41fa129bb0dc8e6c94e9021515 Mon Sep 17 00:00:00 2001 From: Andrew Hutchings Date: Fri, 9 Mar 2018 11:14:33 +0000 Subject: [PATCH 14/49] MCOL-1246 Fix for non-DSS step For TEXT columns (and some other scenarios) we don't do a DSS step to scan dictionaries and do it directly in the BPS step instead. This patch applies the previous fix to this case too. --- primitives/linux-port/dictionary.cpp | 6 ++++-- 1 file changed, 4 insertions(+), 2 deletions(-) diff --git a/primitives/linux-port/dictionary.cpp b/primitives/linux-port/dictionary.cpp index 88d71dc3d..0e6083505 100644 --- a/primitives/linux-port/dictionary.cpp +++ b/primitives/linux-port/dictionary.cpp @@ -773,8 +773,10 @@ void PrimitiveProcessor::p_Dictionary(const DictInput *in, vector *out, filterOffset = sizeof(DictInput) + (in->NVALS * sizeof(PrimToken)); if (eqFilter) { - bool gotIt = (eqFilter->find(string((char *) sigptr.data, sigptr.len)) - != eqFilter->end()); + // MCOL-1246 Trim whitespace before match + string strData((char*)sigptr.data, sigptr.len); + boost::trim_right(strData); + bool gotIt = eqFilter->find(strData) != eqFilter->end(); if ((gotIt && eqOp == COMPARE_EQ) || (!gotIt && eqOp == COMPARE_NE)) goto store; goto no_store; From fa3574b6b1ef5ab753484c1858075b204e622554 Mon Sep 17 00:00:00 2001 From: Andrew Hutchings Date: Fri, 9 Mar 2018 13:08:09 +0000 Subject: [PATCH 15/49] MCOL-1246 Make matching SQL-92 compliant(ish) SQL-92 basically specifies for a NOPAD collation that only space should be ignored for matches. Tabs and other whitespace are handled differently. We don't fully support collations yet so we assume the defaults. --- primitives/linux-port/dictionary.cpp | 4 ++-- utils/dataconvert/dataconvert.h | 2 +- 2 files changed, 3 insertions(+), 3 deletions(-) diff --git a/primitives/linux-port/dictionary.cpp b/primitives/linux-port/dictionary.cpp index 0e6083505..8aba1bf8a 100644 --- a/primitives/linux-port/dictionary.cpp +++ b/primitives/linux-port/dictionary.cpp @@ -167,7 +167,7 @@ void PrimitiveProcessor::p_TokenByScan(const TokenByScanRequestHeader *h, if (eqFilter) { // MCOL-1246 Trim whitespace before match string strData(sig, siglen); - boost::trim_right(strData); + boost::trim_right_if(strData, boost::is_any_if(" ")); bool gotIt = eqFilter->find(strData) != eqFilter->end(); if ((h->COP1 == COMPARE_EQ && gotIt) || (h->COP1 == COMPARE_NE && !gotIt)) @@ -775,7 +775,7 @@ void PrimitiveProcessor::p_Dictionary(const DictInput *in, vector *out, if (eqFilter) { // MCOL-1246 Trim whitespace before match string strData((char*)sigptr.data, sigptr.len); - boost::trim_right(strData); + boost::trim_right_if(strData, boost::is_any_of(" ")); bool gotIt = eqFilter->find(strData) != eqFilter->end(); if ((gotIt && eqOp == COMPARE_EQ) || (!gotIt && eqOp == COMPARE_NE)) goto store; diff --git a/utils/dataconvert/dataconvert.h b/utils/dataconvert/dataconvert.h index aba5a8bd8..d424a52f7 100644 --- a/utils/dataconvert/dataconvert.h +++ b/utils/dataconvert/dataconvert.h @@ -559,7 +559,7 @@ inline void DataConvert::trimWhitespace(int64_t &charData) char *ch_data = (char*) &charData; for (int8_t i = 7; i > 0; i--) { - if (isspace(ch_data[i]) || ch_data[i] == '\0') + if (ch_data[i] == ' ' || ch_data[i] == '\0') ch_data[i] = '\0'; else break; From 44c7693d484e4dd87458dfa6d270f2911c0ed93d Mon Sep 17 00:00:00 2001 From: Andrew Hutchings Date: Fri, 9 Mar 2018 15:21:53 +0000 Subject: [PATCH 16/49] MCOL-1246 Fix typo in boost call --- primitives/linux-port/dictionary.cpp | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/primitives/linux-port/dictionary.cpp b/primitives/linux-port/dictionary.cpp index 8aba1bf8a..81a46e103 100644 --- a/primitives/linux-port/dictionary.cpp +++ b/primitives/linux-port/dictionary.cpp @@ -167,7 +167,7 @@ void PrimitiveProcessor::p_TokenByScan(const TokenByScanRequestHeader *h, if (eqFilter) { // MCOL-1246 Trim whitespace before match string strData(sig, siglen); - boost::trim_right_if(strData, boost::is_any_if(" ")); + boost::trim_right_if(strData, boost::is_any_of(" ")); bool gotIt = eqFilter->find(strData) != eqFilter->end(); if ((h->COP1 == COMPARE_EQ && gotIt) || (h->COP1 == COMPARE_NE && !gotIt)) From 5b694773432745e7116d37c22b013988b0df6138 Mon Sep 17 00:00:00 2001 From: david hill Date: Thu, 15 Mar 2018 09:41:44 -0500 Subject: [PATCH 17/49] MCOL-1222 - improved the waitforsystemactive function --- oam/oamcpp/liboamcpp.cpp | 86 +++++++++++++++++++++++++++++++ oam/oamcpp/liboamcpp.h | 4 ++ oamapps/mcsadmin/mcsadmin.cpp | 59 ++------------------- oamapps/postConfigure/helpers.cpp | 27 ++++------ 4 files changed, 105 insertions(+), 71 deletions(-) diff --git a/oam/oamcpp/liboamcpp.cpp b/oam/oamcpp/liboamcpp.cpp index dfaa71609..aab98c5b8 100644 --- a/oam/oamcpp/liboamcpp.cpp +++ b/oam/oamcpp/liboamcpp.cpp @@ -8832,6 +8832,92 @@ namespace oam return entry; } + + /****************************************************************************************** + * @brief waitForActive + * + * purpose: wait for system to be active + * + ******************************************************************************************/ + void Oam::waitForActive() + { + SystemStatus systemstatus; + SystemProcessStatus systemprocessstatus; + bool bfirst = true; + + for (int i = 0 ; i < 18 ; i ++) + { + sleep (10); + try + { + getSystemStatus(systemstatus); + if (systemstatus.SystemOpState == ACTIVE) + { + BRM::DBRM dbrm; + try { + int rc = dbrm.getSystemQueryReady(); + if (rc == -1 ) { + writeLog("waitForActive: getSystemQueryReady error return: startSystem failed", LOG_TYPE_ERROR); + exceptionControl("waitForActive", API_FAILURE); + } + + if ( rc != 0 ) + return; + + writeLog("waitForActive: getSystemQueryReady not ready", LOG_TYPE_DEBUG); + } + catch(...) + {} + } + + if (systemstatus.SystemOpState == FAILED) + { + exceptionControl("waitForActive", API_FAILURE); + } + + if (systemstatus.SystemOpState == MAN_OFFLINE) + { + exceptionControl("waitForActive", API_FAILURE); + } + + cout << "." << flush; + + // Check DMLProc for a switch to BUSY_INIT. + // In such a case, we need to print a message that rollbacks + // are occurring and will take some time. + if (bfirst) // Once we've printed our message, no need to waste cpu looking + { + getProcessStatus(systemprocessstatus); + for (unsigned int i = 0 ; i < systemprocessstatus.processstatus.size(); i++) + { + if (systemprocessstatus.processstatus[i].ProcessName == "DMLProc") + { + if (systemprocessstatus.processstatus[i].ProcessOpState == oam::ROLLBACK_INIT) + { + cout << endl << endl <<" System Not Ready, DMLProc is checking/processing rollback of abandoned transactions. Processing could take some time, please wait..." << flush; + bfirst = false; + } + // At this point, we've found our DMLProc, so there's no need to spin the for loop + // any further. + break; + } + } + } + } + catch (...) + { + // At some point, we need to give up, ProcMon just isn't going to respond. + if (i > 18) // 3 minutes + { + cout << endl << endl << "TIMEOUT: ProcMon not responding to getSystemStatus"; + break; + } + } + } + + exceptionControl("waitForActive", API_FAILURE); + } + /*************************************************************************** * PRIVATE FUNCTIONS diff --git a/oam/oamcpp/liboamcpp.h b/oam/oamcpp/liboamcpp.h index f23f73dbd..51c1f773c 100644 --- a/oam/oamcpp/liboamcpp.h +++ b/oam/oamcpp/liboamcpp.h @@ -2481,6 +2481,10 @@ namespace oam bool checkSystemRunning(); + /** @brief wait for system to be active + */ + EXPORT void waitForActive(); + private: int sendMsgToProcMgr3(messageqcpp::ByteStream::byte requestType, alarmmanager::AlarmList& alarmlist, const std::string date); diff --git a/oamapps/mcsadmin/mcsadmin.cpp b/oamapps/mcsadmin/mcsadmin.cpp index f7a6e2a38..935d081cc 100644 --- a/oamapps/mcsadmin/mcsadmin.cpp +++ b/oamapps/mcsadmin/mcsadmin.cpp @@ -77,62 +77,13 @@ bool SendToWES(Oam& oam, ByteStream bs); bool waitForActive() { Oam oam; - SystemStatus systemstatus; - SystemProcessStatus systemprocessstatus; - bool bfirst = true; - - for (int i = 0 ; i < 1200 ; i ++) + try { - sleep (3); - try - { - oam.getSystemStatus(systemstatus); - if (systemstatus.SystemOpState == ACTIVE) - { - return true; - } - if (systemstatus.SystemOpState == FAILED) - { - return false; - } - if (systemstatus.SystemOpState == MAN_OFFLINE) - { - return false; - } - cout << "." << flush; - - // Check DMLProc for a switch to BUSY_INIT. - // In such a case, we need to print a message that rollbacks - // are occurring and will take some time. - if (bfirst) // Once we've printed our message, no need to waste cpu looking - { - oam.getProcessStatus(systemprocessstatus); - for (unsigned int i = 0 ; i < systemprocessstatus.processstatus.size(); i++) - { - if (systemprocessstatus.processstatus[i].ProcessName == "DMLProc") - { - if (systemprocessstatus.processstatus[i].ProcessOpState == oam::ROLLBACK_INIT) - { - cout << endl << endl <<" System Not Ready, DMLProc is checking/processing rollback of abandoned transactions. Processing could take some time, please wait..." << flush; - bfirst = false; - } - // At this point, we've found our DMLProc, so there's no need to spin the for loop - // any further. - break; - } - } - } - } - catch (...) - { - // At some point, we need to give up, ProcMgr just isn't going to respond. - if (i > 60) // 3 minutes - { - cout << "ProcMgr not responding while waiting for system to start"; - break; - } - } + oam.waitForActive(); + return true; } + catch (...) + {} return false; } diff --git a/oamapps/postConfigure/helpers.cpp b/oamapps/postConfigure/helpers.cpp index b53a17302..f8cc46990 100644 --- a/oamapps/postConfigure/helpers.cpp +++ b/oamapps/postConfigure/helpers.cpp @@ -74,28 +74,21 @@ void callFree(const char* ) } - bool waitForActive() { - Oam oam; + Oam oam; + try + { + oam.waitForActive(); + return true; + } + catch (...) + {} - const string cmd = installDir + "/bin/mcsadmin getsystemstatus > /tmp/wait.log"; - system(cmd.c_str()); - - for ( int i = 0 ; i < 120 ; i ++ ) - { - if (oam.checkLogStatus("/tmp/wait.log", "System ACTIVE") ) - return true; - if ( oam.checkLogStatus("/tmp/wait.log", "System FAILED") ) - return false; - cout << "."; - cout.flush(); - sleep (10); - system(cmd.c_str()); - } - return false; + return false; } + void dbrmDirCheck() { From 3c6443053e1909f2b8e8a09309d3d828a18a4b8c Mon Sep 17 00:00:00 2001 From: david hill Date: Thu, 15 Mar 2018 16:24:26 -0500 Subject: [PATCH 18/49] MCOL-1217 - setup msqyl rep on addmodule --- oamapps/postConfigure/helpers.cpp | 1 - procmgr/main.cpp | 2 +- procmgr/processmanager.cpp | 181 +++++++++++++----------------- procmgr/processmanager.h | 2 +- procmon/processmonitor.cpp | 48 +++++++- 5 files changed, 122 insertions(+), 112 deletions(-) diff --git a/oamapps/postConfigure/helpers.cpp b/oamapps/postConfigure/helpers.cpp index b53a17302..ba97915a9 100644 --- a/oamapps/postConfigure/helpers.cpp +++ b/oamapps/postConfigure/helpers.cpp @@ -489,7 +489,6 @@ int sendReplicationRequest(int IserverTypeInstall, std::string password, bool pm } else { - cout << endl << "ERROR: Module not Active, replication not done on " << (*pt).DeviceName << endl; pt++; } } diff --git a/procmgr/main.cpp b/procmgr/main.cpp index 434902138..118ac0d73 100644 --- a/procmgr/main.cpp +++ b/procmgr/main.cpp @@ -1803,7 +1803,7 @@ void pingDeviceThread() DeviceNetworkConfig devicenetworkconfig; devicenetworkconfig.DeviceName = moduleName; devicenetworklist.push_back(devicenetworkconfig); - processManager.setMySQLReplication(devicenetworklist, oam::UnassignedName, false, true); + processManager.setMySQLReplication(devicenetworklist, oam::UnassignedName, true); } } diff --git a/procmgr/processmanager.cpp b/procmgr/processmanager.cpp index fdb3f76b1..7c18d6ed8 100755 --- a/procmgr/processmanager.cpp +++ b/procmgr/processmanager.cpp @@ -2546,7 +2546,7 @@ void processMSG(messageqcpp::IOSocket* cfIos) // target = root password oam::DeviceNetworkList devicenetworklist; - status = processManager.setMySQLReplication(devicenetworklist, oam::UnassignedName, false, true, target); + status = processManager.setMySQLReplication(devicenetworklist, oam::UnassignedName, true, target); log.writeLog(__LINE__, "Enable MySQL Replication status: " + oam.itoa(status) ); @@ -2568,7 +2568,7 @@ void processMSG(messageqcpp::IOSocket* cfIos) // target = root password oam::DeviceNetworkList devicenetworklist; - status = processManager.setMySQLReplication(devicenetworklist, oam::UnassignedName, false, false, target, false); + status = processManager.setMySQLReplication(devicenetworklist, oam::UnassignedName, false, target, false); log.writeLog(__LINE__, "Disable MySQL Replication status: " + oam.itoa(status) ); @@ -3621,18 +3621,21 @@ int ProcessManager::startProcess(string moduleName, string processName, { Oam oam; - //skip if module is DISABLED - int opState; - bool degraded; - try{ - oam.getModuleStatus(moduleName, opState, degraded); - } - catch(...) - {} + if ( actionIndicator != oam::STATUS_UPDATE ) + { + //skip if module is DISABLED + int opState; + bool degraded; + try{ + oam.getModuleStatus(moduleName, opState, degraded); + } + catch(...) + {} - //check if disabled - if (opState == oam::MAN_DISABLED || opState == oam::AUTO_DISABLED) - return API_SUCCESS; + //check if disabled + if (opState == oam::MAN_DISABLED || opState == oam::AUTO_DISABLED) + return API_SUCCESS; + } ByteStream msg; ByteStream::byte requestID = START; @@ -4190,8 +4193,8 @@ int ProcessManager::startProcessType( std::string processName ) if ( systemprocessstatus.processstatus[i].ProcessName == processName) { // found one, request restart of it int retStatus = processManager.startProcess(systemprocessstatus.processstatus[i].Module, - processName, - FORCEFUL); + processName, + FORCEFUL); log.writeLog(__LINE__, "StartProcessType: Start ACK received from Process-Monitor, return status = " + oam.itoa(retStatus), LOG_TYPE_DEBUG); } } @@ -5426,9 +5429,23 @@ int ProcessManager::addModule(oam::DeviceNetworkList devicenetworklist, std::str log.writeLog(__LINE__, "addModule - sleep 60 - give ProcMon time to CONFIGURE and restart", LOG_TYPE_DEBUG); sleep(60); + //start mysqld on the new modules so mysql replication can be setup + listPT = devicenetworklist.begin(); + for( ; listPT != devicenetworklist.end() ; listPT++) + { + processManager.startProcess((*listPT).DeviceName, "mysqld", oam::STATUS_UPDATE); + } + log.writeLog(__LINE__, "Setup MySQL Replication for new Modules being Added", LOG_TYPE_DEBUG); - processManager.setMySQLReplication(devicenetworklist, oam::UnassignedName, false, true, password ); + processManager.setMySQLReplication(devicenetworklist, oam::UnassignedName, true, password, true, true ); + //stop mysqld + listPT = devicenetworklist.begin(); + for( ; listPT != devicenetworklist.end() ; listPT++) + { + processManager.stopProcess((*listPT).DeviceName, "mysqld", oam::FORCEFUL, true ); + } + return API_SUCCESS; } @@ -8728,7 +8745,7 @@ int ProcessManager::switchParentOAMModule(std::string newActiveModuleName) //change master MySQL Replication setup log.writeLog(__LINE__, "Setup MySQL Replication for new Parent Module during switch-over", LOG_TYPE_DEBUG); oam::DeviceNetworkList devicenetworklist; - processManager.setMySQLReplication(devicenetworklist, newActiveModuleName, false, false, oam::UnassignedName); + processManager.setMySQLReplication(devicenetworklist, newActiveModuleName, false, oam::UnassignedName); } catch (exception& ex) @@ -9530,7 +9547,7 @@ int ProcessManager::OAMParentModuleChange() //change master MySQL Replication setup log.writeLog(__LINE__, "Setup this node as MySQL Replication Master", LOG_TYPE_DEBUG); oam::DeviceNetworkList devicenetworklist; - processManager.setMySQLReplication(devicenetworklist, config.moduleName(), true); + processManager.setMySQLReplication(devicenetworklist, config.moduleName()); } //set query system state not ready @@ -10207,7 +10224,7 @@ void ProcessManager::flushInodeCache() * * ******************************************************************************************/ -int ProcessManager::setMySQLReplication(oam::DeviceNetworkList devicenetworklist, std::string masterModule, bool failover, bool distributeDB, std::string password, bool enable) +int ProcessManager::setMySQLReplication(oam::DeviceNetworkList devicenetworklist, std::string masterModule, bool distributeDB, std::string password, bool enable, bool addModule) { Oam oam; @@ -10226,57 +10243,6 @@ int ProcessManager::setMySQLReplication(oam::DeviceNetworkList devicenetworklist return oam::API_SUCCESS; } - //also skip if single-server, multi-node seperate with 1 UM, multi-node combo with 1 PM - -/* string SingleServerInstall = "n"; - try { - oam.getSystemConfig("SingleServerInstall", SingleServerInstall); - } - catch(...) { - SingleServerInstall = "n"; - } - - //single server check - if ( SingleServerInstall == "y" ) - { - log.writeLog(__LINE__, "setMySQLReplication: Single-Server, exiting", LOG_TYPE_DEBUG); - return oam::API_SUCCESS; - } - - //combined system check - if ( config.ServerInstallType() == oam::INSTALL_COMBINE_DM_UM_PM && !failover ) { - try { - Config* sysConfig = Config::makeConfig(); - if ( sysConfig->getConfig("DBRM_Controller", "NumWorkers") == "1" ) { - log.writeLog(__LINE__, "setMySQLReplication: 1 configured module, exiting", LOG_TYPE_DEBUG); - return oam::API_SUCCESS; - } - } - catch(...) - { - log.writeLog(__LINE__, "setMySQLReplication: makeConfig exception, exiting", LOG_TYPE_DEBUG); - return oam::API_SUCCESS; - } - } - - //seperate system check - if ( ( config.ServerInstallType() != oam::INSTALL_COMBINE_DM_UM_PM ) && - (PMwithUM == "n" ) ) - { - ModuleTypeConfig moduletypeconfig; - try{ - oam.getSystemConfig("um", moduletypeconfig); - } - catch(...) - {} - - if ( moduletypeconfig.ModuleCount < 2 ) - { - log.writeLog(__LINE__, "setMySQLReplication: moduleCount = 1, exiting", LOG_TYPE_DEBUG); - return oam::API_SUCCESS; - } - } -*/ log.writeLog(__LINE__, "Setup MySQL Replication", LOG_TYPE_DEBUG); //get master info @@ -10329,18 +10295,21 @@ int ProcessManager::setMySQLReplication(oam::DeviceNetworkList devicenetworklist if ( remoteModuleName == masterModule ) continue; - // skip disabled modules - int opState = oam::ACTIVE; - bool degraded; - try { - oam.getModuleStatus(remoteModuleName, opState, degraded); + if ( !addModule ) + { + // skip disabled modules + int opState = oam::ACTIVE; + bool degraded; + try { + oam.getModuleStatus(remoteModuleName, opState, degraded); + } + catch(...) + {} + + if (opState == oam::MAN_DISABLED || opState == oam::AUTO_DISABLED) + continue; } - catch(...) - {} - - if (opState == oam::MAN_DISABLED || opState == oam::AUTO_DISABLED) - continue; - + // don't do PMs unless PMwithUM flag is set if ( config.ServerInstallType() != oam::INSTALL_COMBINE_DM_UM_PM ) { string moduleType = remoteModuleName.substr(0,MAX_MODULE_TYPE_SIZE); @@ -10418,18 +10387,21 @@ int ProcessManager::setMySQLReplication(oam::DeviceNetworkList devicenetworklist if ( remoteModuleName == masterModule ) continue; - // skip disabled modules - int opState = oam::ACTIVE; - bool degraded; - try { - oam.getModuleStatus(remoteModuleName, opState, degraded); + if ( !addModule ) + { + // skip disabled modules + int opState = oam::ACTIVE; + bool degraded; + try { + oam.getModuleStatus(remoteModuleName, opState, degraded); + } + catch(...) + {} + + if (opState == oam::MAN_DISABLED || opState == oam::AUTO_DISABLED) + continue; } - catch(...) - {} - - if (opState == oam::MAN_DISABLED || opState == oam::AUTO_DISABLED) - continue; - + // don't do PMs unless PMwithUM flag is set if ( config.ServerInstallType() != oam::INSTALL_COMBINE_DM_UM_PM ) { string moduleType = remoteModuleName.substr(0,MAX_MODULE_TYPE_SIZE); @@ -10477,18 +10449,21 @@ int ProcessManager::setMySQLReplication(oam::DeviceNetworkList devicenetworklist if ( remoteModuleName == masterModule ) continue; - // skip disabled modules - int opState = oam::ACTIVE; - bool degraded; - try { - oam.getModuleStatus(remoteModuleName, opState, degraded); + if ( !addModule ) + { + // skip disabled modules + int opState = oam::ACTIVE; + bool degraded; + try { + oam.getModuleStatus(remoteModuleName, opState, degraded); + } + catch(...) + {} + + if (opState == oam::MAN_DISABLED || opState == oam::AUTO_DISABLED) + continue; } - catch(...) - {} - - if (opState == oam::MAN_DISABLED || opState == oam::AUTO_DISABLED) - continue; - + log.writeLog(__LINE__, "Setup Slave MySQL Replication on " + remoteModuleName, LOG_TYPE_DEBUG); ByteStream msg1; diff --git a/procmgr/processmanager.h b/procmgr/processmanager.h index d60a9eaf2..55d291b23 100644 --- a/procmgr/processmanager.h +++ b/procmgr/processmanager.h @@ -545,7 +545,7 @@ public: /** @brief Set MySQL Replication */ - int setMySQLReplication(oam::DeviceNetworkList devicenetworklist, std::string masterModule = oam::UnassignedName, bool failover = false, bool distributeDB = false, std::string password = oam::UnassignedName, bool enable = true); + int setMySQLReplication(oam::DeviceNetworkList devicenetworklist, std::string masterModule = oam::UnassignedName, bool distributeDB = false, std::string password = oam::UnassignedName, bool enable = true, bool addModule = false); /** @brief Gluster Assign dbroot to a module */ diff --git a/procmon/processmonitor.cpp b/procmon/processmonitor.cpp index 8f51cccf1..aa10f2666 100644 --- a/procmon/processmonitor.cpp +++ b/procmon/processmonitor.cpp @@ -457,6 +457,24 @@ void ProcessMonitor::processMessage(messageqcpp::ByteStream msg, messageqcpp::IO log.writeLog(__LINE__, "MSG RECEIVED: Stop process request on " + processName); int requestStatus = API_SUCCESS; + // check for mysql + if ( processName == "mysqld" ) { + try { + oam.actionMysqlCalpont(MYSQL_STOP); + } + catch(...) + {} + + ackMsg << (ByteStream::byte) ACK; + ackMsg << (ByteStream::byte) STOP; + ackMsg << (ByteStream::byte) API_SUCCESS; + mq.write(ackMsg); + + log.writeLog(__LINE__, "STOP: ACK back to ProcMgr, return status = " + oam.itoa((int) API_SUCCESS)); + + break; + } + processList::iterator listPtr; processList* aPtr = config.monitoredListPtr(); listPtr = aPtr->begin(); @@ -502,6 +520,24 @@ void ProcessMonitor::processMessage(messageqcpp::ByteStream msg, messageqcpp::IO msg >> manualFlag; log.writeLog(__LINE__, "MSG RECEIVED: Start process request on: " + processName); + // check for mysql + if ( processName == "mysqld" ) { + try { + oam.actionMysqlCalpont(MYSQL_START); + } + catch(...) + {} + + ackMsg << (ByteStream::byte) ACK; + ackMsg << (ByteStream::byte) START; + ackMsg << (ByteStream::byte) API_SUCCESS; + mq.write(ackMsg); + + log.writeLog(__LINE__, "START: ACK back to ProcMgr, return status = " + oam.itoa((int) API_SUCCESS)); + + break; + } + ProcessConfig processconfig; ProcessStatus processstatus; try { @@ -605,7 +641,7 @@ void ProcessMonitor::processMessage(messageqcpp::ByteStream msg, messageqcpp::IO int requestStatus = API_SUCCESS; // check for mysql restart - if ( processName == "mysql" ) { + if ( processName == "mysqld" ) { try { oam.actionMysqlCalpont(MYSQL_RESTART); } @@ -4952,11 +4988,11 @@ int ProcessMonitor::runMasterRep(std::string& masterLogFile, std::string& master //skip if module is not ACTIVE - int opState = oam::ACTIVE; - bool degraded; - oam.getModuleStatus(moduleName, opState, degraded); - if (opState != oam::ACTIVE) - continue; +// int opState = oam::ACTIVE; +// bool degraded; +// oam.getModuleStatus(moduleName, opState, degraded); +// if (opState != oam::ACTIVE) +// continue; bool passwordError = false; From 38684501c912458a008a1365a39b095a3b2327af Mon Sep 17 00:00:00 2001 From: david hill Date: Thu, 22 Mar 2018 13:46:40 -0500 Subject: [PATCH 19/49] MCOL-1222 - changed the delay from 10 to 3 seconds --- oam/oamcpp/liboamcpp.cpp | 15 ++++++++++----- 1 file changed, 10 insertions(+), 5 deletions(-) diff --git a/oam/oamcpp/liboamcpp.cpp b/oam/oamcpp/liboamcpp.cpp index aab98c5b8..649d86f13 100644 --- a/oam/oamcpp/liboamcpp.cpp +++ b/oam/oamcpp/liboamcpp.cpp @@ -8844,10 +8844,11 @@ namespace oam SystemStatus systemstatus; SystemProcessStatus systemprocessstatus; bool bfirst = true; - - for (int i = 0 ; i < 18 ; i ++) + int dot = 0; + + for (int i = 0 ; i < 120 ; i ++, dot ++) { - sleep (10); + sleep (3); try { getSystemStatus(systemstatus); @@ -8880,7 +8881,11 @@ namespace oam exceptionControl("waitForActive", API_FAILURE); } - cout << "." << flush; + if (dot >= 3 ) + { + cout << "." << flush; + dot=0; + } // Check DMLProc for a switch to BUSY_INIT. // In such a case, we need to print a message that rollbacks @@ -8907,7 +8912,7 @@ namespace oam catch (...) { // At some point, we need to give up, ProcMon just isn't going to respond. - if (i > 18) // 3 minutes + if (i > 60) // 3 minutes { cout << endl << endl << "TIMEOUT: ProcMon not responding to getSystemStatus"; break; From 73b1ac68faa9cbf05a6ca312a35df61c6e4f60a0 Mon Sep 17 00:00:00 2001 From: David Hall Date: Tue, 27 Mar 2018 12:43:43 -0500 Subject: [PATCH 20/49] MCOL-1196 Error when using OR in case THEN portion In ha_calpont_execplan, Allow OR to be parsed; in searched_case parsing, reverse the order of processing arguments so that ptWorkStack.pop() is executed in the same order as the arguments being processed. In func_case, modify to pass left and right to getBoolVal, if they exist. --- dbcon/mysql/ha_calpont_execplan.cpp | 12 ++++++++---- utils/funcexp/func_case.cpp | 7 +++++++ 2 files changed, 15 insertions(+), 4 deletions(-) diff --git a/dbcon/mysql/ha_calpont_execplan.cpp b/dbcon/mysql/ha_calpont_execplan.cpp index 6cb25c269..690c8b3b5 100755 --- a/dbcon/mysql/ha_calpont_execplan.cpp +++ b/dbcon/mysql/ha_calpont_execplan.cpp @@ -2553,6 +2553,10 @@ ReturnedColumn* buildReturnedColumn(Item* item, gp_walk_info& gwi, bool& nonSupp break; } + case Item::COND_ITEM: + { + break; + } default: { gwi.fatalParseError = true; @@ -3183,6 +3187,7 @@ FunctionColumn* buildCaseFunction(Item_func* item, gp_walk_info& gwi, bool& nonS if (((Item_func_case*)item)->get_first_expr_num() == -1) funcName = "case_searched"; + funcParms.reserve(item->argument_count()); if (gwi.clauseType == SELECT || gwi.clauseType == HAVING || gwi.clauseType == GROUP_BY) // select clause { // the first argument @@ -3233,13 +3238,12 @@ FunctionColumn* buildCaseFunction(Item_func* item, gp_walk_info& gwi, bool& nonS gwi.clauseType = SELECT; if (funcName == "case_searched") { - for (uint32_t i = 0; i < item->argument_count(); i++) + for (int32_t i = item->argument_count()-1; i >=0; i--) { - if (i % 2 == 0 && i != item->argument_count()-1) + if (i % 2 == 0 && uint(i) != item->argument_count()-1) { // build item from arguments to avoid parm sequence complexity sptp.reset(buildParseTree((Item_func*)(item->arguments()[i]), gwi, nonSupport)); - funcParms.push_back(sptp); if (!gwi.ptWorkStack.empty()) gwi.ptWorkStack.pop(); } @@ -3258,8 +3262,8 @@ FunctionColumn* buildCaseFunction(Item_func* item, gp_walk_info& gwi, bool& nonS if (!gwi.ptWorkStack.empty()) gwi.ptWorkStack.pop(); } - funcParms.push_back(sptp); } + funcParms.insert(funcParms.begin(), sptp); } } else // simple_case diff --git a/utils/funcexp/func_case.cpp b/utils/funcexp/func_case.cpp index 7cc033f85..3787947ab 100644 --- a/utils/funcexp/func_case.cpp +++ b/utils/funcexp/func_case.cpp @@ -470,6 +470,13 @@ bool Func_searched_case::getBoolVal(Row& row, if (isNull) return joblist::BIGINTNULL; + ParseTree* lop = parm[i+1]->left(); + ParseTree* rop = parm[i+1]->right(); + if (lop && rop) + { + return (reinterpret_cast(parm[i+1]->data()))->getBoolVal(row, isNull, lop, rop); + } + return parm[i+1]->data()->getBoolVal(row, isNull); } From c83896609dd76ba5476b3cc1b0edda08ae185288 Mon Sep 17 00:00:00 2001 From: David Hall Date: Tue, 27 Mar 2018 15:23:47 -0500 Subject: [PATCH 21/49] MCOL-1196 add a comment --- dbcon/mysql/ha_calpont_execplan.cpp | 2 ++ 1 file changed, 2 insertions(+) diff --git a/dbcon/mysql/ha_calpont_execplan.cpp b/dbcon/mysql/ha_calpont_execplan.cpp index 690c8b3b5..7e138792b 100755 --- a/dbcon/mysql/ha_calpont_execplan.cpp +++ b/dbcon/mysql/ha_calpont_execplan.cpp @@ -2555,6 +2555,8 @@ ReturnedColumn* buildReturnedColumn(Item* item, gp_walk_info& gwi, bool& nonSupp case Item::COND_ITEM: { + // MCOL-1196: Allow COND_ITEM thru. They will be picked up + // by further logic. It may become desirable to add code here. break; } default: From aa581b5dc3d88b4d0d07a224de4da5c96cf23aa3 Mon Sep 17 00:00:00 2001 From: David Hall Date: Mon, 2 Apr 2018 17:02:54 -0500 Subject: [PATCH 22/49] MCOL-1234 Nested CASE filters not processed Change the way buildCaseFunction() handles the ptWorkStack and the rcWorkStack. --- dbcon/execplan/arithmeticoperator.cpp | 2 +- dbcon/execplan/filter.cpp | 2 +- dbcon/execplan/logicoperator.cpp | 2 +- dbcon/execplan/predicateoperator.cpp | 2 +- dbcon/execplan/simplecolumn.cpp | 10 +- dbcon/execplan/treenodeimpl.cpp | 2 +- dbcon/mysql/ha_calpont_execplan.cpp | 146 +++++++++++--------------- 7 files changed, 73 insertions(+), 93 deletions(-) diff --git a/dbcon/execplan/arithmeticoperator.cpp b/dbcon/execplan/arithmeticoperator.cpp index b16b0dbeb..122f481ae 100644 --- a/dbcon/execplan/arithmeticoperator.cpp +++ b/dbcon/execplan/arithmeticoperator.cpp @@ -102,7 +102,7 @@ void ArithmeticOperator::unserialize(messageqcpp::ByteStream& b) bool ArithmeticOperator::operator==(const ArithmeticOperator& t) const { - if (fData == t.fData) + if (data() == t.data()) return true; return false; } diff --git a/dbcon/execplan/filter.cpp b/dbcon/execplan/filter.cpp index 386acd256..7368d4379 100644 --- a/dbcon/execplan/filter.cpp +++ b/dbcon/execplan/filter.cpp @@ -72,7 +72,7 @@ const string Filter::toString() const bool Filter::operator==(const Filter& t) const { - if (fData == t.fData) + if (data() == t.data()) return true; return false; } diff --git a/dbcon/execplan/logicoperator.cpp b/dbcon/execplan/logicoperator.cpp index 9c12d27ae..99a7d0656 100644 --- a/dbcon/execplan/logicoperator.cpp +++ b/dbcon/execplan/logicoperator.cpp @@ -106,7 +106,7 @@ void LogicOperator::unserialize(messageqcpp::ByteStream& b) bool LogicOperator::operator==(const LogicOperator& t) const { - if (fData == t.fData) + if (data() == t.data()) return true; return false; } diff --git a/dbcon/execplan/predicateoperator.cpp b/dbcon/execplan/predicateoperator.cpp index 38e7a2f54..2b47ec058 100644 --- a/dbcon/execplan/predicateoperator.cpp +++ b/dbcon/execplan/predicateoperator.cpp @@ -152,7 +152,7 @@ void PredicateOperator::unserialize(messageqcpp::ByteStream& b) bool PredicateOperator::operator==(const PredicateOperator& t) const { - if (fData == t.fData) + if (data() == t.data()) return true; return false; } diff --git a/dbcon/execplan/simplecolumn.cpp b/dbcon/execplan/simplecolumn.cpp index a3a4e0dac..895334fc9 100644 --- a/dbcon/execplan/simplecolumn.cpp +++ b/dbcon/execplan/simplecolumn.cpp @@ -370,16 +370,16 @@ bool SimpleColumn::operator==(const SimpleColumn& t) const return false; if (fColumnName != t.fColumnName) return false; - if (fIndexName != t.fIndexName) - return false; +// if (fIndexName != t.fIndexName) +// return false; if (fViewName != t.fViewName) return false; if (fOid != t.fOid) return false; - if (fData != t.fData) - return false; - if (fAlias != t.fAlias) + if (data() != t.data()) return false; +// if (fAlias != t.fAlias) +// return false; if (fTableAlias != t.fTableAlias) return false; if (fAsc != t.fAsc) diff --git a/dbcon/execplan/treenodeimpl.cpp b/dbcon/execplan/treenodeimpl.cpp index 6ffe2787a..50fa7d209 100644 --- a/dbcon/execplan/treenodeimpl.cpp +++ b/dbcon/execplan/treenodeimpl.cpp @@ -63,7 +63,7 @@ const string TreeNodeImpl::toString() const bool TreeNodeImpl::operator==(const TreeNodeImpl& t) const { - if (fData == t.fData) + if (data() == t.data()) return true; return false; } diff --git a/dbcon/mysql/ha_calpont_execplan.cpp b/dbcon/mysql/ha_calpont_execplan.cpp index 7e138792b..da80366b8 100755 --- a/dbcon/mysql/ha_calpont_execplan.cpp +++ b/dbcon/mysql/ha_calpont_execplan.cpp @@ -3190,107 +3190,85 @@ FunctionColumn* buildCaseFunction(Item_func* item, gp_walk_info& gwi, bool& nonS funcName = "case_searched"; funcParms.reserve(item->argument_count()); - if (gwi.clauseType == SELECT || gwi.clauseType == HAVING || gwi.clauseType == GROUP_BY) // select clause + // so buildXXXcolumn function will not pop stack. + ClauseType realClauseType = gwi.clauseType; + gwi.clauseType = SELECT; + + // We ought to be able to just build from the stack, and would + // be able to if there were any way to know which stack had the + // next case item. Unfortunately, parameters may have been pushed + // onto the ptWorkStack or rcWorkStack or neither, depending on type + // and position. We can't tell which at this point, so we + // rebuild the item from the arguments directly and then try to + // figure what to pop, if anything, in order to sync the stacks. + for (int32_t i = item->argument_count()-1; i >=0; i--) { - // the first argument - if (funcName == "case_searched") + // For case_searched, we know the items for the WHEN clause will + // not be ReturnedColumns. We do this separately just to save + // some cpu cycles trying to build a ReturnedColumn as below. + // Every even numbered arg is a WHEN. In between are the THEN. + // An odd number of args indicates an ELSE residing in the last spot. + if (funcName == "case_searched" && + i % 2 == 0 && uint(i) != item->argument_count()-1) { - for (uint32_t i = 0; i < item->argument_count(); i++) + sptp.reset(buildParseTree((Item_func*)(item->arguments()[i]), gwi, nonSupport)); + if (!gwi.ptWorkStack.empty() && *gwi.ptWorkStack.top()->data() == sptp->data()) { - if (i % 2 == 0 && i != 1 && i != item->argument_count()-1) - { - sptp.reset(buildParseTree((Item_func*)(item->arguments()[i]), gwi, nonSupport)); - funcParms.push_back(sptp); - } - else - { - ReturnedColumn* parm = buildReturnedColumn(item->arguments()[i], gwi, nonSupport); - if (parm) - { - sptp.reset(new ParseTree(parm)); - } - else - { - sptp.reset(buildParseTree((Item_func*)(item->arguments()[i]), gwi, nonSupport)); - } - funcParms.push_back(sptp); - } + gwi.ptWorkStack.pop(); } } else { - for (uint32_t i = 0; i < item->argument_count(); i++) + // First try building a ReturnedColumn. It may or may not succeed + // depending on the types involved. There's also little correlation + // between buildReturnedColumn and the existance of the item on + // rwWorkStack or ptWorkStack. + // For example, simple predicates, such as 1=1 or 1=0, land in the + // ptWorkStack but other stuff might land in the rwWorkStack + ReturnedColumn* parm = buildReturnedColumn(item->arguments()[i], gwi, nonSupport); + if (parm) { - ReturnedColumn* parm = buildReturnedColumn(item->arguments()[i], gwi, nonSupport); - if (parm) + sptp.reset(new ParseTree(parm)); + // We need to pop whichever stack is holding it, if any. + if ((!gwi.rcWorkStack.empty()) && + *gwi.rcWorkStack.top() == parm) { - sptp.reset(new ParseTree(parm)); + gwi.rcWorkStack.pop(); } else + if (!gwi.ptWorkStack.empty()) { - sptp.reset(buildParseTree((Item_func*)(item->arguments()[i]), gwi, nonSupport)); - } - funcParms.push_back(sptp); - } - } - } - else // where clause - { - // so buildXXXcolumn function will not pop stack. - gwi.clauseType = SELECT; - if (funcName == "case_searched") - { - for (int32_t i = item->argument_count()-1; i >=0; i--) - { - if (i % 2 == 0 && uint(i) != item->argument_count()-1) - { - // build item from arguments to avoid parm sequence complexity - sptp.reset(buildParseTree((Item_func*)(item->arguments()[i]), gwi, nonSupport)); - if (!gwi.ptWorkStack.empty()) + ReturnedColumn* ptrc = dynamic_cast(gwi.ptWorkStack.top()->data()); + if (ptrc && *ptrc == *parm) gwi.ptWorkStack.pop(); } - else - { - ReturnedColumn* parm = buildReturnedColumn(item->arguments()[i], gwi, nonSupport); - if (parm) - { - sptp.reset(new ParseTree(parm)); - if (!gwi.rcWorkStack.empty()) - gwi.rcWorkStack.pop(); - } - else - { - sptp.reset(buildParseTree((Item_func*)(item->arguments()[i]), gwi, nonSupport)); - if (!gwi.ptWorkStack.empty()) - gwi.ptWorkStack.pop(); - } - } - funcParms.insert(funcParms.begin(), sptp); } - } - else // simple_case - { - for (uint32_t i = 0; i < item->argument_count(); i++) + else { - ReturnedColumn* parm = buildReturnedColumn(item->arguments()[i], gwi, nonSupport); - if (parm) + sptp.reset(buildParseTree((Item_func*)(item->arguments()[i]), gwi, nonSupport)); + // We need to pop whichever stack is holding it, if any. + if ((!gwi.ptWorkStack.empty()) && + *gwi.ptWorkStack.top()->data() == sptp->data()) { - sptp.reset(new ParseTree(parm)); - if (!gwi.rcWorkStack.empty()) + gwi.ptWorkStack.pop(); + } + else + if (!gwi.rcWorkStack.empty()) + { + // Probably won't happen, but it might have been on the + // rcWorkStack all along. + ReturnedColumn* ptrc = dynamic_cast(sptp->data()); + if (ptrc && *ptrc == *gwi.rcWorkStack.top()) + { gwi.rcWorkStack.pop(); + } } - else - { - sptp.reset(buildParseTree((Item_func*)(item->arguments()[i]), gwi, nonSupport)); - if (!gwi.ptWorkStack.empty()) - gwi.ptWorkStack.pop(); - } - funcParms.push_back(sptp); } } - // recover clause type - gwi.clauseType = WHERE; + funcParms.insert(funcParms.begin(), sptp); } + // recover clause type + gwi.clauseType = realClauseType; if (gwi.fatalParseError) { @@ -4259,9 +4237,9 @@ void gp_walk(const Item *item, void *arg) } // bug 3137. If filter constant like 1=0, put it to ptWorkStack // MariaDB bug 750. Breaks if compare is an argument to a function. - if ((int32_t)gwip->rcWorkStack.size() <= (gwip->rcBookMarkStack.empty() ? 0 : gwip->rcBookMarkStack.top()) - && isPredicateFunction(ifp, gwip)) -// if (isPredicateFunction(ifp, gwip)) +// if ((int32_t)gwip->rcWorkStack.size() <= (gwip->rcBookMarkStack.empty() ? 0 : gwip->rcBookMarkStack.top()) +// && isPredicateFunction(ifp, gwip)) + if (isPredicateFunction(ifp, gwip)) gwip->ptWorkStack.push(new ParseTree(cc)); else gwip->rcWorkStack.push(cc); @@ -5505,7 +5483,9 @@ int getSelectPlan(gp_walk_info& gwi, SELECT_LEX& select_lex, SCSEP& csep, bool i // @bug 1706 String funcStr; ifp->print(&funcStr, QT_INFINIDB); - gwi.selectCols.push_back(string(funcStr.c_ptr()) + " `" + escapeBackTick(ifp->name) + "`"); + string valStr; + valStr.assign(funcStr.ptr(), funcStr.length()); + gwi.selectCols.push_back(valStr + " `" + escapeBackTick(ifp->name) + "`"); // clear the error set by buildFunctionColumn gwi.fatalParseError = false; gwi.parseErrorText = ""; From dbcbd6c7e74f0bf4e3d4f757f0b9bff7f2666e57 Mon Sep 17 00:00:00 2001 From: David Hall Date: Wed, 4 Apr 2018 12:34:17 -0500 Subject: [PATCH 23/49] MCOL-1323 cpimport Splitter has incorrect SIGPIPE mapping --- writeengine/splitter/we_splitterapp.cpp | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/writeengine/splitter/we_splitterapp.cpp b/writeengine/splitter/we_splitterapp.cpp index 7ecbca854..0077ebd60 100644 --- a/writeengine/splitter/we_splitterapp.cpp +++ b/writeengine/splitter/we_splitterapp.cpp @@ -159,7 +159,7 @@ void WESplitterApp::setupSignalHandlers() sa.sa_handler = SIG_IGN; sigaction(SIGPIPE, &sa, 0); sa.sa_handler = WESplitterApp::onSigHup; - sigaction(SIGPIPE, &sa, 0); + sigaction(SIGHUP, &sa, 0); sa.sa_handler = WESplitterApp::onSigInterrupt; sigaction(SIGUSR1, &sa, 0); /* From fa43a962a8b19b4bd851055c6793663b705dec2a Mon Sep 17 00:00:00 2001 From: David Hall Date: Wed, 4 Apr 2018 15:56:53 -0500 Subject: [PATCH 24/49] MCOL-1325 rename table fails when database different than current database --- dbcon/mysql/ha_calpont_ddl.cpp | 10 ++++------ 1 file changed, 4 insertions(+), 6 deletions(-) diff --git a/dbcon/mysql/ha_calpont_ddl.cpp b/dbcon/mysql/ha_calpont_ddl.cpp index 68f963825..611f1da3b 100755 --- a/dbcon/mysql/ha_calpont_ddl.cpp +++ b/dbcon/mysql/ha_calpont_ddl.cpp @@ -2074,16 +2074,14 @@ int ha_calpont_impl_rename_table_(const char* from, const char* to, cal_connecti stmt = stmt1.str(); string db; - if ( thd->db ) - db = thd->db; - else if ( fromPair.first.length() !=0 ) + if ( fromPair.first.length() !=0 ) db = fromPair.first; - else - db = toPair.first; + else if ( thd->db ) + db = thd->db; int rc = ProcessDDLStatement(stmt, db, "", tid2sid(thd->thread_id), emsg); if (rc != 0) - push_warning(thd, Sql_condition::WARN_LEVEL_ERROR, 9999, emsg.c_str()); + push_warning(thd, Sql_condition::WARN_LEVEL_WARN, 9999, emsg.c_str()); return rc; } From 4067d6927803f5317cbee9812054de8824913c56 Mon Sep 17 00:00:00 2001 From: david hill Date: Thu, 5 Apr 2018 13:23:06 -0500 Subject: [PATCH 25/49] MCOL-1317 Mcol-1318 --- .../clusterTester/columnstoreClusterTester.sh | 68 ++++++------------- 1 file changed, 20 insertions(+), 48 deletions(-) diff --git a/utils/clusterTester/columnstoreClusterTester.sh b/utils/clusterTester/columnstoreClusterTester.sh index b512eeb20..56d78c49f 100755 --- a/utils/clusterTester/columnstoreClusterTester.sh +++ b/utils/clusterTester/columnstoreClusterTester.sh @@ -494,7 +494,7 @@ checkSELINUX() # SELINUX check # echo "" - echo "** Run SELINUX check - Setting should to be disabled on all nodes" + echo "** Run SELINUX check" echo "" pass=true @@ -502,9 +502,8 @@ checkSELINUX() if [ -f /etc/selinux/config ]; then `cat /etc/selinux/config | grep SELINUX | grep enforcing > /tmp/selinux_check 2>&1` if [ "$?" -eq 0 ]; then - echo "${bold}Failed${normal}, Local Node SELINUX setting is Enabled, please disable" + echo "${bold}Warning${normal}, Local Node SELINUX setting is Enabled, check port test results" pass=false - REPORTPASS=false else echo "Local Node SELINUX setting is Not Enabled" fi @@ -519,19 +518,14 @@ checkSELINUX() else `cat config | grep SELINUX | grep enforcing > /tmp/selinux_check 2>&1` if [ "$?" -eq 0 ]; then - echo "${bold}Failed${normal}, $ipadd SELINUX setting is Enabled, please disable" + echo "${bold}Warning${normal}, $ipadd SELINUX setting is Enabled, check port test results" pass=false - REPORTPASS=false else echo "$ipadd Node SELINUX setting is Not Enabled" fi `rm -f config` fi done - - if ! $pass; then - checkContinue - fi } checkFirewalls() @@ -539,28 +533,23 @@ checkFirewalls() # FIREWALL checks # echo "" - echo "** Run Firewall Services check - Firewall Services should to be Inactive on all nodes" + echo "** Run Firewall Services check" echo "" declare -a FIREWALL_LIST=("iptables" "ufw" "firewalld" "firewall") - fpass=true #check local FIREWALLS for firewall in "${FIREWALL_LIST[@]}"; do pass=true `service $firewall status > /tmp/firewall1_check 2>&1` if [ "$?" -eq 0 ]; then - echo "${bold}Failed${normal}, Local Node $firewall service is Active, please disable" + echo "${bold}Warning${normal}, Local Node $firewall service is Active, check port test results" pass=false - fpass=false - REPORTPASS=false else `systemctl status $firewall > /tmp/firewall1_check 2>&1` if [ "$?" -eq 0 ]; then - echo "${bold}Failed${normal}, Local Node $firewall service is Active, please disable" + echo "${bold}Warning${normal}, Local Node $firewall service is Active, check port test results" pass=false - fpass=false - REPORTPASS=false fi fi @@ -569,10 +558,6 @@ checkFirewalls() fi done - if ! $fpass; then - checkContinue - fi - echo "" fpass=true for ipadd in "${NODE_IPADDRESS[@]}"; do @@ -581,17 +566,13 @@ checkFirewalls() pass=true `$COLUMNSTORE_INSTALL_DIR/bin/remote_command.sh $ipadd $PASSWORD "service '$firewall' status > /tmp/firewall_check 2>&1" 1 > /tmp/remote_command_check` if [ "$?" -eq 0 ]; then - echo "${bold}Failed${normal}, $ipadd Node $firewall service is Active, please disable" + echo "${bold}Warning${normal}, $ipadd Node $firewall service is Active, check port test results" pass=false - fpass=false - REPORTPASS=false else `$COLUMNSTORE_INSTALL_DIR/bin/remote_command.sh $ipadd $PASSWORD "systemctl status '$firewall' > /tmp/firewall_check 2>&1" 1 > /tmp/remote_command_check` if [ "$?" -eq 0 ]; then - echo "${bold}Failed${normal}, $ipadd Node $firewall service is Active, please disable" + echo "${bold}Warning${normal}, $ipadd Node $firewall service is Active, check port test results" pass=false - fpass=false - REPORTPASS=false fi fi @@ -602,25 +583,20 @@ checkFirewalls() echo "" done - - if ! $fpass; then - checkContinue - fi if [ $OS == "suse12" ]; then # rcSuSEfirewall2 check # echo "" - echo "** Run rcSuSEfirewall2 check - Service should to be disabled on all nodes" + echo "** Run rcSuSEfirewall2 check" echo "" pass=true #check local IPTABLES `/sbin/rcSuSEfirewall2 status > /tmp/rcSuSEfirewall2_check 2>&1` if [ "$?" -eq 0 ]; then - echo "${bold}Failed${normal}, Local Node rcSuSEfirewall2 service is Enabled, please disable" + echo "${bold}Failed${normal}, Local Node rcSuSEfirewall2 service is Enabled, check port test results" pass=false - REPORTPASS=false else echo "Local Node rcSuSEfirewall2 service is Not Enabled" fi @@ -629,17 +605,12 @@ checkFirewalls() `$COLUMNSTORE_INSTALL_DIR/bin/remote_command.sh $ipadd $PASSWORD '/sbin/rcSuSEfirewall2 status > /tmp/rcSuSEfirewall2_check 2>&1' 1 > /tmp/remote_command_check` rc="$?" if [ $rc -eq 0 ] ; then - echo "${bold}Failed${normal}, $ipadd Node rcSuSEfirewall2 service is Enabled, please disable" + echo "${bold}Failed${normal}, $ipadd Node rcSuSEfirewall2 service is Enabled, check port test results" pass=false - REPORTPASS=false else echo "$ipadd Node rcSuSEfirewall2 service is Not Enabled" fi done - - if ! $pass; then - checkContinue - fi fi } @@ -648,17 +619,18 @@ checkPorts() # port test # echo "" - echo "** Run MariaDB ColumnStore Port (8600-8620) availibility test" + echo "** Run MariaDB ColumnStore Port (8600-8630,8700,8800,3306) availibility test" echo "" pass=true for ipadd in "${NODE_IPADDRESS[@]}"; do - `nmap $ipadd -p 8600-8620 | grep 'closed unknown' > /dev/null` - if [ "$?" -eq 0 ]; then + `sudo nmap $ipadd -p 8600-8630,8700,8800,3306 | grep 'filtered' > /tmp/port_test` + if [ "$?" -ne 0 ]; then echo $ipadd " Node Passed port test" else - echo $ipadd " Node ${bold}Failed${normal} port test, check and disable any firwalls that were reported enabled" + echo $ipadd " Node ${bold}Failed${normal} port test, check and disable any firewalls or open ports in firewall" + cat /tmp/port_test pass=false REPORTPASS=false fi @@ -764,7 +736,7 @@ checkPackages() echo "** Run MariaDB ColumnStore Dependent Package Check" echo "" - declare -a CENTOS_PKG=("expect" "perl" "perl-DBI" "openssl" "zlib" "file" "sudo" "libaio" "rsync" "snappy" "net-tools" "perl-DBD-MySQL") + declare -a CENTOS_PKG=("expect" "perl" "perl-DBI" "openssl" "zlib" "file" "sudo" "libaio" "rsync" "snappy" "net-tools") declare -a CENTOS_PKG_NOT=("mariadb-libs") if [ "$OS" == "centos6" ] || [ "$OS" == "centos7" ]; then @@ -883,7 +855,7 @@ checkPackages() fi fi - declare -a SUSE_PKG=("boost-devel" "expect" "perl" "perl-DBI" "openssl" "file" "sudo" "libaio1" "rsync" "libsnappy1" "net-tools" "perl-DBD-mysql") + declare -a SUSE_PKG=("boost-devel" "expect" "perl" "perl-DBI" "openssl" "file" "sudo" "libaio1" "rsync" "libsnappy1" "net-tools") declare -a SUSE_PKG_NOT=("mariadb" , "libmariadb18") if [ "$OS" == "suse12" ]; then @@ -974,7 +946,7 @@ checkPackages() fi fi - declare -a UBUNTU_PKG=("libboost-all-dev" "expect" "libdbi-perl" "perl" "openssl" "file" "sudo" "libreadline-dev" "rsync" "libsnappy1V5" "net-tools" "libdbd-mysql-perl") + declare -a UBUNTU_PKG=("libboost-all-dev" "expect" "libdbi-perl" "perl" "openssl" "file" "sudo" "libreadline-dev" "rsync" "libsnappy1V5" "net-tools") declare -a UBUNTU_PKG_NOT=("mariadb-server" "libmariadb18") if [ "$OS" == "ubuntu16" ] ; then @@ -1091,7 +1063,7 @@ checkPackages() fi fi - declare -a DEBIAN_PKG=("libboost-all-dev" "expect" "libdbi-perl" "perl" "openssl" "file" "sudo" "libreadline-dev" "rsync" "libsnappy1" "net-tools" "libdbd-mysql-perl") + declare -a DEBIAN_PKG=("libboost-all-dev" "expect" "libdbi-perl" "perl" "openssl" "file" "sudo" "libreadline-dev" "rsync" "libsnappy1" "net-tools") declare -a DEBIAN_PKG_NOT=("libmariadb18" "mariadb-server") if [ "$OS" == "debian8" ]; then From 333f848a2fb82d9bc1b0af4ad3ab71d8b022afa8 Mon Sep 17 00:00:00 2001 From: david hill Date: Thu, 5 Apr 2018 13:24:55 -0500 Subject: [PATCH 26/49] MCOL-1317 Mcol-1318 --- utils/clusterTester/columnstoreClusterTester.sh | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/utils/clusterTester/columnstoreClusterTester.sh b/utils/clusterTester/columnstoreClusterTester.sh index 56d78c49f..341e37748 100755 --- a/utils/clusterTester/columnstoreClusterTester.sh +++ b/utils/clusterTester/columnstoreClusterTester.sh @@ -619,7 +619,7 @@ checkPorts() # port test # echo "" - echo "** Run MariaDB ColumnStore Port (8600-8630,8700,8800,3306) availibility test" + echo "** Run MariaDB ColumnStore Port (8600-8630,8700,8800,3306) availability test" echo "" pass=true From bd8387202fccaf90dcdd706b52bc56e58e296120 Mon Sep 17 00:00:00 2001 From: david hill Date: Mon, 9 Apr 2018 08:55:45 -0500 Subject: [PATCH 27/49] MCOL-912 - added code to restart dbrm/ddml process when enable module is done --- procmgr/processmanager.cpp | 19 +++++++++++++++++-- procmgr/processmanager.h | 2 +- 2 files changed, 18 insertions(+), 3 deletions(-) diff --git a/procmgr/processmanager.cpp b/procmgr/processmanager.cpp index fdb3f76b1..2172b6153 100755 --- a/procmgr/processmanager.cpp +++ b/procmgr/processmanager.cpp @@ -3385,12 +3385,12 @@ int ProcessManager::disableModule(string target, bool manualFlag) * purpose: recyle process, generally after some disable module is run * ******************************************************************************************/ -void ProcessManager::recycleProcess(string module) +void ProcessManager::recycleProcess(string module, bool enableModule) { Oam oam; ModuleConfig moduleconfig; - log.writeLog(__LINE__, "recycleProcess request after module was disabled: " + module, LOG_TYPE_DEBUG); + log.writeLog(__LINE__, "recycleProcess request after module statsu update: " + module, LOG_TYPE_DEBUG); string moduleType = module.substr(0,MAX_MODULE_TYPE_SIZE); @@ -3399,11 +3399,23 @@ void ProcessManager::recycleProcess(string module) oam.getSystemConfig("PrimaryUMModuleName", PrimaryUMModuleName); } catch(...) {} + + // restart DMLProc and return if enable module is being done + if (enableModule) + { + //recycle DBRM processes in all cases + restartProcessType("DBRMControllerNode"); + restartProcessType("DBRMWorkerNode"); + + restartProcessType("DMLProc"); + return; + } //recycle DBRM processes in all cases restartProcessType("DBRMControllerNode", module); restartProcessType("DBRMWorkerNode"); + // only recycle dmlproc, if down/up module is non-parent UM if ( ( moduleType == "um" ) && ( PrimaryUMModuleName != module) ) @@ -3483,6 +3495,9 @@ int ProcessManager::enableModule(string target, int state) if ( newStandbyModule == target) setStandbyModule(newStandbyModule); + //set recycle process + recycleProcess(target); + log.writeLog(__LINE__, "enableModule request for " + target + " completed", LOG_TYPE_DEBUG); return API_SUCCESS; diff --git a/procmgr/processmanager.h b/procmgr/processmanager.h index d60a9eaf2..a9ba826eb 100644 --- a/procmgr/processmanager.h +++ b/procmgr/processmanager.h @@ -302,7 +302,7 @@ public: /** *@brief recycle Processes */ - void recycleProcess(std::string module); + void recycleProcess(std::string module, bool enableModule = false); /** *@brief Enable a specified module From afb8e4706d58fffbae0631add8c90cf21343d121 Mon Sep 17 00:00:00 2001 From: david hill Date: Mon, 9 Apr 2018 11:35:37 -0500 Subject: [PATCH 28/49] MCOL-912 - fix spelling error --- procmgr/processmanager.cpp | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/procmgr/processmanager.cpp b/procmgr/processmanager.cpp index 2172b6153..6db1a1660 100755 --- a/procmgr/processmanager.cpp +++ b/procmgr/processmanager.cpp @@ -3390,7 +3390,7 @@ void ProcessManager::recycleProcess(string module, bool enableModule) Oam oam; ModuleConfig moduleconfig; - log.writeLog(__LINE__, "recycleProcess request after module statsu update: " + module, LOG_TYPE_DEBUG); + log.writeLog(__LINE__, "recycleProcess request after module status update: " + module, LOG_TYPE_DEBUG); string moduleType = module.substr(0,MAX_MODULE_TYPE_SIZE); @@ -3400,7 +3400,7 @@ void ProcessManager::recycleProcess(string module, bool enableModule) } catch(...) {} - // restart DMLProc and return if enable module is being done + // restart DBRM Process and DMLProc and return if enable module is being done if (enableModule) { //recycle DBRM processes in all cases From a80854c11f55be1882aaf798229f36e63353a966 Mon Sep 17 00:00:00 2001 From: Andrew Hutchings Date: Tue, 10 Apr 2018 14:14:22 +0100 Subject: [PATCH 29/49] MCOL-1182 Keep filters for cross engine step Cross engine step needs the filters for joining a constant with the cross engine table. Deleting them early causes a crash. --- dbcon/joblist/jlf_execplantojoblist.cpp | 21 ++++++++++++++++++--- 1 file changed, 18 insertions(+), 3 deletions(-) diff --git a/dbcon/joblist/jlf_execplantojoblist.cpp b/dbcon/joblist/jlf_execplantojoblist.cpp index 1ac770701..b6242ba77 100644 --- a/dbcon/joblist/jlf_execplantojoblist.cpp +++ b/dbcon/joblist/jlf_execplantojoblist.cpp @@ -1954,6 +1954,7 @@ const JobStepVector doOuterJoinOnFilter(OuterJoinOnFilter* oj, JobInfo& jobInfo) set doneNodes; // solved joins and simple filters map cpMap; // link for node removal JobStepVector join; // join step with its projection steps + bool keepFilters = false; // keep filters for cross engine step // To compromise the front end difficulty on setting outer attributes. set tablesInOuter; @@ -2162,6 +2163,14 @@ const JobStepVector doOuterJoinOnFilter(OuterJoinOnFilter* oj, JobInfo& jobInfo) jsv.insert(jsv.end(), sfv.begin(), sfv.end()); + // MCOL-1182 if we are doing a join between a cross engine + // step and a constant then keep the filter for the cross + // engine step instead of deleting it further down. + if (!sc->isInfiniDB()) + { + keepFilters = true; + } + doneNodes.insert(cn); } } @@ -2198,7 +2207,10 @@ const JobStepVector doOuterJoinOnFilter(OuterJoinOnFilter* oj, JobInfo& jobInfo) if (p == NULL) { filters = NULL; - delete c; + if (!keepFilters) + { + delete c; + } } else { @@ -2240,8 +2252,11 @@ const JobStepVector doOuterJoinOnFilter(OuterJoinOnFilter* oj, JobInfo& jobInfo) p->left(nullTree); p->right(nullTree); - delete p; - delete c; + if (!keepFilters) + { + delete p; + delete c; + } } } From 8220a5103fd77f9f89a94bbb84411292ab6ba1bb Mon Sep 17 00:00:00 2001 From: david hill Date: Fri, 13 Apr 2018 10:29:17 -0500 Subject: [PATCH 30/49] MCOL-1333 - changed addmodule to look for rpm/deb, not tar.gz packages --- procmgr/processmanager.cpp | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/procmgr/processmanager.cpp b/procmgr/processmanager.cpp index fdb3f76b1..7343e6c7a 100755 --- a/procmgr/processmanager.cpp +++ b/procmgr/processmanager.cpp @@ -4530,10 +4530,10 @@ int ProcessManager::addModule(oam::DeviceNetworkList devicenetworklist, std::str } if ( packageType == "rpm") - calpontPackage = homedir + "/mariadb-columnstore*" + columnstore_version + "-" + columnstore_release + "*.rpm.tar.gz"; + calpontPackage = homedir + "/mariadb-columnstore*" + columnstore_version + "-" + columnstore_release + "*.rpm"; else if ( packageType == "deb") - calpontPackage = homedir + "/mariadb-columnstore*" + columnstore_version + "-" + columnstore_release + "*.deb.tar.gz"; + calpontPackage = homedir + "/mariadb-columnstore*" + columnstore_version + "-" + columnstore_release + "*.deb"; else calpontPackage = homedir + "/mariadb-columnstore*" + columnstore_version + "-" + columnstore_release + "*.bin.tar.gz"; From 4013c8b9b259903ba073b7353aa345e077b268a3 Mon Sep 17 00:00:00 2001 From: Roman Nozdrin Date: Fri, 13 Apr 2018 22:16:44 +0300 Subject: [PATCH 31/49] MCOL-1327 CS supports visible special ASCII symbols. --- dbcon/ddlpackage/ddl.l | 6 ++++-- 1 file changed, 4 insertions(+), 2 deletions(-) diff --git a/dbcon/ddlpackage/ddl.l b/dbcon/ddlpackage/ddl.l index 4812bbd0d..e922c08ca 100644 --- a/dbcon/ddlpackage/ddl.l +++ b/dbcon/ddlpackage/ddl.l @@ -60,6 +60,7 @@ double_quote \" grave_accent ` comment ("--"{non_newline}*) +extended_ident_cont [A-Za-z\200-\377_0-9\$#,()\[\].;\:\+\-\*\/\%\^\<\>\=!&|@\\] self [,()\[\].;\:\+\-\*\/\%\^\<\>\=] whitespace ({space}+|{comment}) @@ -67,10 +68,11 @@ digit [0-9] ident_start [A-Za-z\200-\377_0-9] ident_cont [A-Za-z\200-\377_0-9\$] identifier {ident_start}{ident_cont}* +extended_identifier {ident_start}{extended_ident_cont}* /* fully qualified names regexes */ fq_identifier {identifier}\.{identifier} -identifer_quoted {grave_accent}{identifier}{grave_accent} -identifer_double_quoted {double_quote}{identifier}{double_quote} +identifer_quoted {grave_accent}{extended_identifier}{grave_accent} +identifer_double_quoted {double_quote}{extended_identifier}{double_quote} integer [-+]?{digit}+ decimal ([-+]?({digit}*\.{digit}+)|({digit}+\.{digit}*)) From 0c6dc5e15c97061d72df291a8c33100d3b345cfb Mon Sep 17 00:00:00 2001 From: Andrew Hutchings Date: Mon, 16 Apr 2018 19:21:28 +0100 Subject: [PATCH 32/49] MCOL-1341 Fix CASE handling with 10.2.14 MariaDB Server 10.2.14 changed the order that CASE items are processed. This broke the engine's CASE handling. This patch uses the new order instead since this is what will be used in 10.2 and 10.3 going forward. --- dbcon/mysql/ha_calpont_execplan.cpp | 16 +++- utils/funcexp/func_case.cpp | 136 ++++++++++++++++++---------- 2 files changed, 105 insertions(+), 47 deletions(-) diff --git a/dbcon/mysql/ha_calpont_execplan.cpp b/dbcon/mysql/ha_calpont_execplan.cpp index da80366b8..0f54d8bdb 100755 --- a/dbcon/mysql/ha_calpont_execplan.cpp +++ b/dbcon/mysql/ha_calpont_execplan.cpp @@ -3201,6 +3201,20 @@ FunctionColumn* buildCaseFunction(Item_func* item, gp_walk_info& gwi, bool& nonS // and position. We can't tell which at this point, so we // rebuild the item from the arguments directly and then try to // figure what to pop, if anything, in order to sync the stacks. + // + // MCOL-1341 - With MariaDB 10.2.14 onwards CASE is now in the order: + // [case,]when1,when2,...,then1,then2,...[,else] + // See server commit bf1ca14ff3f3faa9f7a018097b25aa0f66d068cd for more + // information. + int32_t arg_offset = 0; + if ((item->argument_count() - 1) % 2) + { + arg_offset = (item->argument_count() - 1) / 2; + } + else + { + arg_offset = item->argument_count() / 2; + } for (int32_t i = item->argument_count()-1; i >=0; i--) { // For case_searched, we know the items for the WHEN clause will @@ -3209,7 +3223,7 @@ FunctionColumn* buildCaseFunction(Item_func* item, gp_walk_info& gwi, bool& nonS // Every even numbered arg is a WHEN. In between are the THEN. // An odd number of args indicates an ELSE residing in the last spot. if (funcName == "case_searched" && - i % 2 == 0 && uint(i) != item->argument_count()-1) + (i < arg_offset)) { sptp.reset(buildParseTree((Item_func*)(item->arguments()[i]), gwi, nonSupport)); if (!gwi.ptWorkStack.empty() && *gwi.ptWorkStack.top()->data() == sptp->data()) diff --git a/utils/funcexp/func_case.cpp b/utils/funcexp/func_case.cpp index 3787947ab..2b5ae58c2 100644 --- a/utils/funcexp/func_case.cpp +++ b/utils/funcexp/func_case.cpp @@ -52,9 +52,10 @@ inline uint64_t simple_case_cmp(Row& row, CalpontSystemCatalog::ColType& operationColType) { uint64_t i = 0; // index to the parm list - uint64_t n = parm.size() - 1; // remove expression from count of expression_i + result_i - uint64_t hasElse = n % 2; // if 1, then ELSE exist - n -= hasElse; // index to expression + uint64_t n = 0; // remove expression from count of expression_i + result_i + uint64_t hasElse = (parm.size()-1) % 2; // if 1, then ELSE exist + uint64_t whereCount = hasElse ? (parm.size() - 2) / 2 : (parm.size() - 1) / 2; + bool foundIt = false; switch (operationColType.colDataType) { @@ -70,10 +71,13 @@ inline uint64_t simple_case_cmp(Row& row, if (isNull) break; - for (; i < n; i += 2) + for (i = 1; i <= whereCount; i++) { if (ev == parm[i]->data()->getIntVal(row, isNull) && !isNull) + { + foundIt = true; break; + } else isNull = false; } @@ -90,10 +94,13 @@ inline uint64_t simple_case_cmp(Row& row, if (isNull) break; - for (; i < n; i += 2) + for (i = 1; i <= whereCount; i++) { if (ev == parm[i]->data()->getUintVal(row, isNull) && !isNull) + { + foundIt = true; break; + } else isNull = false; } @@ -108,11 +115,14 @@ inline uint64_t simple_case_cmp(Row& row, if (isNull) break; - for (; i < n; i += 2) + for (i = 1; i <= whereCount; i++) { //BUG 5362 if (utf8::idb_strcoll(ev.c_str(), parm[i]->data()->getStrVal(row, isNull).c_str()) == 0 && !isNull) + { + foundIt = true; break; + } else isNull = false; } @@ -126,10 +136,13 @@ inline uint64_t simple_case_cmp(Row& row, if (isNull) break; - for (; i < n; i += 2) + for (i = 1; i <= whereCount; i++) { if (ev == parm[i]->data()->getDecimalVal(row, isNull) && !isNull) + { + foundIt = true; break; + } else isNull = false; } @@ -143,10 +156,13 @@ inline uint64_t simple_case_cmp(Row& row, if (isNull) break; - for (; i < n; i += 2) + for (i = 1; i <= whereCount; i++) { if (ev == parm[i]->data()->getDoubleVal(row, isNull) && !isNull) + { + foundIt = true; break; + } else isNull = false; } @@ -160,10 +176,13 @@ inline uint64_t simple_case_cmp(Row& row, if (isNull) break; - for (; i < n; i += 2) + for (i = 1; i <= whereCount; i++) { if (ev == parm[i]->data()->getFloatVal(row, isNull) && !isNull) + { + foundIt = true; break; + } else isNull = false; } @@ -178,16 +197,24 @@ inline uint64_t simple_case_cmp(Row& row, } } - if (i == n && !hasElse) + if (!foundIt && !hasElse) isNull = true; + else if (!foundIt && hasElse && !isNull) + { + i = parm.size() - 1; + } else if (isNull && hasElse) // BUG 5110. Only way we can exit above with isNull == true is when ev is NULL - // if so and we have else condition we need to use it by setting i = n + // if so and we have else condition we need to use it by setting i = else { - i = n; + i = parm.size() - 1; isNull = false; } + if (foundIt) + { + i += whereCount; + } return i; } @@ -198,21 +225,33 @@ inline uint64_t searched_case_cmp(Row& row, bool& isNull) { uint64_t i = 0; // index to the parm list - uint64_t n = parm.size(); // count of boolean_expression_i + result_i - uint64_t hasElse = n % 2; // if 1, then ELSE exist - n -= hasElse; // index to expression + uint64_t hasElse = parm.size() % 2; // if 1, then ELSE exist + uint64_t whereCount = hasElse ? (parm.size() - 1) / 2 : parm.size() / 2; + bool foundIt = false; - for (; i < n; i += 2) + + for (i = 0; i < whereCount; i++) { if (parm[i]->getBoolVal(row, isNull)) + { + foundIt = true; break; + } } isNull = false; - if (i == n && !hasElse) + if (!foundIt && !hasElse) isNull = true; + else if (!foundIt && hasElse) + { + i = parm.size() - 1; + } + if (foundIt) + { + i += whereCount; + } - return (i == n ? i-1 : i); + return i; } @@ -220,7 +259,6 @@ CalpontSystemCatalog::ColType caseOperationType(FunctionParm& fp, CalpontSystemCatalog::ColType& resultType, bool simpleCase) { - // ... expression_i + result_i + ... [[expression] + result_N] FunctionParm::size_type n = fp.size(); if (simpleCase) // simple case has an expression @@ -228,6 +266,9 @@ CalpontSystemCatalog::ColType caseOperationType(FunctionParm& fp, bool hasElse = ((n % 2) != 0); // if 1, then ELSE exist if (hasElse) --n; // n now is an even number + uint64_t parmCount = hasElse ? (fp.size() - 2) : (fp.size() - 1); + uint64_t whereCount = hasElse ? (fp.size() - 2 + simpleCase) / 2 : (fp.size() - 1) / 2 + simpleCase; + idbassert((n % 2) == 0); bool allStringO = true; @@ -238,10 +279,10 @@ CalpontSystemCatalog::ColType caseOperationType(FunctionParm& fp, CalpontSystemCatalog::ColType oct = fp[l]->data()->resultType(); CalpontSystemCatalog::ColType rct = resultType; bool operation = true; - for (uint64_t i = 0; i <= n; i++) + for (uint64_t i = 0; i <= parmCount; i++) { // operation or result type - operation = ((i % 2) == 0); + operation = ((i > 0) && (i <= whereCount)); // the result type of ELSE, if exists. if (i == n) @@ -334,8 +375,9 @@ namespace funcexp // END // // simple CASE parm order: -// expression1 result1 expression2 result2 ... expression [resultN] +// expression condition1 condition2 ... result1 result2 ... [resultN] // +// Note that this order changed in 10.2.14, see MCOL-1341 CalpontSystemCatalog::ColType Func_simple_case::operationType(FunctionParm& fp, CalpontSystemCatalog::ColType& resultType) { @@ -353,7 +395,7 @@ bool Func_simple_case::getBoolVal(Row& row, if (isNull) return joblist::BIGINTNULL; - return parm[i+1]->data()->getBoolVal(row, isNull); + return parm[i]->data()->getBoolVal(row, isNull); } @@ -367,7 +409,7 @@ int64_t Func_simple_case::getIntVal(Row& row, if (isNull) return joblist::BIGINTNULL; - return parm[i+1]->data()->getIntVal(row, isNull); + return parm[i]->data()->getIntVal(row, isNull); } @@ -381,7 +423,7 @@ string Func_simple_case::getStrVal(Row& row, if (isNull) return string(""); - return parm[i+1]->data()->getStrVal(row, isNull); + return parm[i]->data()->getStrVal(row, isNull); } @@ -395,7 +437,7 @@ IDB_Decimal Func_simple_case::getDecimalVal(Row& row, if (isNull) return IDB_Decimal(); // need a null value for IDB_Decimal?? - return parm[i+1]->data()->getDecimalVal(row, isNull); + return parm[i]->data()->getDecimalVal(row, isNull); } @@ -409,7 +451,7 @@ double Func_simple_case::getDoubleVal(Row& row, if (isNull) return doubleNullVal(); - return parm[i+1]->data()->getDoubleVal(row, isNull); + return parm[i]->data()->getDoubleVal(row, isNull); } @@ -423,9 +465,9 @@ int32_t Func_simple_case::getDateIntVal(rowgroup::Row& row, if (isNull) return joblist::DATENULL; - return parm[i+1]->data()->getDateIntVal(row, isNull); -} - + return parm[i]->data()->getDateIntVal(row, isNull); +} + int64_t Func_simple_case::getDatetimeIntVal(rowgroup::Row& row, FunctionParm& parm, @@ -437,8 +479,8 @@ int64_t Func_simple_case::getDatetimeIntVal(rowgroup::Row& row, if (isNull) return joblist::DATETIMENULL; - return parm[i+1]->data()->getDatetimeIntVal(row, isNull); -} + return parm[i]->data()->getDatetimeIntVal(row, isNull); +} @@ -451,8 +493,10 @@ int64_t Func_simple_case::getDatetimeIntVal(rowgroup::Row& row, // END // // searched CASE parm order: -// boolean_expression1 result1 boolean_expression2 result2 ... [resultN] +// boolean_expression1 boolean_expression2 ... result1 result2 ... [resultN] // +// Note that this order changed in 10.2.14, see MCOL-1341 + CalpontSystemCatalog::ColType Func_searched_case::operationType(FunctionParm& fp, CalpontSystemCatalog::ColType& resultType) { // operation type not used by this functor. @@ -470,14 +514,14 @@ bool Func_searched_case::getBoolVal(Row& row, if (isNull) return joblist::BIGINTNULL; - ParseTree* lop = parm[i+1]->left(); - ParseTree* rop = parm[i+1]->right(); + ParseTree* lop = parm[i]->left(); + ParseTree* rop = parm[i]->right(); if (lop && rop) { - return (reinterpret_cast(parm[i+1]->data()))->getBoolVal(row, isNull, lop, rop); + return (reinterpret_cast(parm[i]->data()))->getBoolVal(row, isNull, lop, rop); } - return parm[i+1]->data()->getBoolVal(row, isNull); + return parm[i]->data()->getBoolVal(row, isNull); } int64_t Func_searched_case::getIntVal(Row& row, @@ -490,7 +534,7 @@ int64_t Func_searched_case::getIntVal(Row& row, if (isNull) return joblist::BIGINTNULL; - return parm[i+1]->data()->getIntVal(row, isNull); + return parm[i]->data()->getIntVal(row, isNull); } @@ -504,7 +548,7 @@ string Func_searched_case::getStrVal(Row& row, if (isNull) return string(""); - return parm[i+1]->data()->getStrVal(row, isNull); + return parm[i]->data()->getStrVal(row, isNull); } @@ -518,7 +562,7 @@ IDB_Decimal Func_searched_case::getDecimalVal(Row& row, if (isNull) return IDB_Decimal(); // need a null value for IDB_Decimal?? - return parm[i+1]->data()->getDecimalVal(row, isNull); + return parm[i]->data()->getDecimalVal(row, isNull); } @@ -532,7 +576,7 @@ double Func_searched_case::getDoubleVal(Row& row, if (isNull) return doubleNullVal(); - return parm[i+1]->data()->getDoubleVal(row, isNull); + return parm[i]->data()->getDoubleVal(row, isNull); } @@ -546,10 +590,10 @@ int32_t Func_searched_case::getDateIntVal(rowgroup::Row& row, if (isNull) return joblist::DATENULL; - return parm[i+1]->data()->getDateIntVal(row, isNull); -} + return parm[i]->data()->getDateIntVal(row, isNull); +} + - int64_t Func_searched_case::getDatetimeIntVal(rowgroup::Row& row, FunctionParm& parm, bool& isNull, @@ -560,8 +604,8 @@ int64_t Func_searched_case::getDatetimeIntVal(rowgroup::Row& row, if (isNull) return joblist::DATETIMENULL; - return parm[i+1]->data()->getDatetimeIntVal(row, isNull); -} + return parm[i]->data()->getDatetimeIntVal(row, isNull); +} } // namespace funcexp From f2d748cff197656c7ae5d490d0298eabbabccc7d Mon Sep 17 00:00:00 2001 From: david hill Date: Mon, 16 Apr 2018 16:22:03 -0500 Subject: [PATCH 33/49] MCOL-1293 --- utils/clusterTester/columnstoreClusterTester.sh | 10 +++++----- 1 file changed, 5 insertions(+), 5 deletions(-) diff --git a/utils/clusterTester/columnstoreClusterTester.sh b/utils/clusterTester/columnstoreClusterTester.sh index 341e37748..9f8f3e9a3 100755 --- a/utils/clusterTester/columnstoreClusterTester.sh +++ b/utils/clusterTester/columnstoreClusterTester.sh @@ -736,7 +736,7 @@ checkPackages() echo "** Run MariaDB ColumnStore Dependent Package Check" echo "" - declare -a CENTOS_PKG=("expect" "perl" "perl-DBI" "openssl" "zlib" "file" "sudo" "libaio" "rsync" "snappy" "net-tools") + declare -a CENTOS_PKG=("expect" "perl" "perl-DBI" "openssl" "zlib" "file" "sudo" "libaio" "rsync" "snappy" "net-tools" "numactl-libs") declare -a CENTOS_PKG_NOT=("mariadb-libs") if [ "$OS" == "centos6" ] || [ "$OS" == "centos7" ]; then @@ -855,7 +855,7 @@ checkPackages() fi fi - declare -a SUSE_PKG=("boost-devel" "expect" "perl" "perl-DBI" "openssl" "file" "sudo" "libaio1" "rsync" "libsnappy1" "net-tools") + declare -a SUSE_PKG=("boost-devel" "expect" "perl" "perl-DBI" "openssl" "file" "sudo" "libaio1" "rsync" "libsnappy1" "net-tools" "libnuma1") declare -a SUSE_PKG_NOT=("mariadb" , "libmariadb18") if [ "$OS" == "suse12" ]; then @@ -946,7 +946,7 @@ checkPackages() fi fi - declare -a UBUNTU_PKG=("libboost-all-dev" "expect" "libdbi-perl" "perl" "openssl" "file" "sudo" "libreadline-dev" "rsync" "libsnappy1V5" "net-tools") + declare -a UBUNTU_PKG=("libboost-all-dev" "expect" "libdbi-perl" "perl" "openssl" "file" "sudo" "libreadline-dev" "rsync" "libsnappy1V5" "net-tools" "libnuma1" ) declare -a UBUNTU_PKG_NOT=("mariadb-server" "libmariadb18") if [ "$OS" == "ubuntu16" ] ; then @@ -1063,7 +1063,7 @@ checkPackages() fi fi - declare -a DEBIAN_PKG=("libboost-all-dev" "expect" "libdbi-perl" "perl" "openssl" "file" "sudo" "libreadline-dev" "rsync" "libsnappy1" "net-tools") + declare -a DEBIAN_PKG=("libboost-all-dev" "expect" "libdbi-perl" "perl" "openssl" "file" "sudo" "libreadline-dev" "rsync" "libsnappy1" "net-tools" "libnuma1") declare -a DEBIAN_PKG_NOT=("libmariadb18" "mariadb-server") if [ "$OS" == "debian8" ]; then @@ -1180,7 +1180,7 @@ checkPackages() fi fi - declare -a DEBIAN9_PKG=("libboost-all-dev" "expect" "libdbi-perl" "perl" "openssl" "file" "sudo" "libreadline5" "rsync" "libsnappy1V5" "net-tools" "libaio1") + declare -a DEBIAN9_PKG=("libboost-all-dev" "expect" "libdbi-perl" "perl" "openssl" "file" "sudo" "libreadline5" "rsync" "libsnappy1V5" "net-tools" "libaio1" "libnuma1") declare -a DEBIAN9_PKG_NOT=("libmariadb18" "mariadb-server") if [ "$OS" == "debian9" ]; then From 51715c76eee3992d59970e1b755f64c1cc7ff82d Mon Sep 17 00:00:00 2001 From: Roman Nozdrin Date: Tue, 24 Apr 2018 15:14:08 +0300 Subject: [PATCH 34/49] MCOL-1348 Add multiply UDAF calls support. --- dbcon/joblist/tupleaggregatestep.cpp | 134 +++++++++++++++++---------- 1 file changed, 85 insertions(+), 49 deletions(-) diff --git a/dbcon/joblist/tupleaggregatestep.cpp b/dbcon/joblist/tupleaggregatestep.cpp index b5cde967b..e68bc9f18 100755 --- a/dbcon/joblist/tupleaggregatestep.cpp +++ b/dbcon/joblist/tupleaggregatestep.cpp @@ -958,6 +958,7 @@ void TupleAggregateStep::prep1PhaseAggregate( vector functionVec; uint32_t bigIntWidth = sizeof(int64_t); uint32_t bigUintWidth = sizeof(uint64_t); + uint32_t projColsUDAFIndex = 0; mcsv1sdk::mcsv1_UDAF* pUDAFFunc = NULL; // for count column of average function @@ -1135,18 +1136,27 @@ void TupleAggregateStep::prep1PhaseAggregate( SP_ROWAGG_FUNC_t funct; if (aggOp == ROWAGG_UDAF) - { - UDAFColumn* udafc = dynamic_cast(jobInfo.projectionCols[i].get()); - if (udafc) - { - // Create a RowAggFunctionCol (UDAF subtype) with the context. - funct.reset(new RowUDAFFunctionCol(udafc->getContext(), colProj, i)); - } - else - { - throw logic_error("prep1PhasesAggregate: A UDAF function is called but there's no UDAFColumn"); - } - } + { + std::vector::iterator it = jobInfo.projectionCols.begin() + projColsUDAFIndex; + for (; it != jobInfo.projectionCols.end(); it++) + { + UDAFColumn* udafc = dynamic_cast((*it).get()); + projColsUDAFIndex++; + if (udafc) + { + pUDAFFunc = udafc->getContext().getFunction(); + // Create a RowAggFunctionCol (UDAF subtype) with the context. + funct.reset(new RowUDAFFunctionCol(udafc->getContext(), colProj, i)); + break; + } + + } + + if (it == jobInfo.projectionCols.end()) + { + throw logic_error("prep1PhaseAggregate: A UDAF function is called but there's no/not enough UDAFColumn/-s"); + } + } else { funct.reset(new RowAggFunctionCol(aggOp, stats, colProj, i)); @@ -1484,6 +1494,7 @@ void TupleAggregateStep::prep1PhaseDistinctAggregate( AGG_MAP aggFuncMap; mcsv1sdk::mcsv1_UDAF* pUDAFFunc = NULL; set avgSet; + uint32_t projColsUDAFIndex = 0; // for count column of average function map avgFuncMap, avgDistFuncMap; @@ -1636,19 +1647,27 @@ void TupleAggregateStep::prep1PhaseDistinctAggregate( SP_ROWAGG_FUNC_t funct; if (aggOp == ROWAGG_UDAF) - { - UDAFColumn* udafc = dynamic_cast(jobInfo.projectionCols[i].get()); - if (udafc) - { - pUDAFFunc = udafc->getContext().getFunction(); - // Create a RowAggFunctionCol (UDAF subtype) with the context. - funct.reset(new RowUDAFFunctionCol(udafc->getContext(), colProj, colAgg)); - } - else - { - throw logic_error("prep1PhaseDistinctAggregate: A UDAF function is called but there's no UDAFColumn"); - } - } + { + std::vector::iterator it = jobInfo.projectionCols.begin() + projColsUDAFIndex; + for (; it != jobInfo.projectionCols.end(); it++) + { + UDAFColumn* udafc = dynamic_cast((*it).get()); + projColsUDAFIndex++; + if (udafc) + { + pUDAFFunc = udafc->getContext().getFunction(); + // Create a RowAggFunctionCol (UDAF subtype) with the context. + funct.reset(new RowUDAFFunctionCol(udafc->getContext(), colProj, colAgg)); + break; + } + + } + + if (it == jobInfo.projectionCols.end()) + { + throw logic_error("prep1PhaseDistinctAggregate: A UDAF function is called but there's no/not enough UDAFColumn/-s"); + } + } else { funct.reset(new RowAggFunctionCol(aggOp, stats, colProj, colAgg)); @@ -2579,6 +2598,7 @@ void TupleAggregateStep::prep2PhasesAggregate( vector > aggColVec; set avgSet; vector >& returnedColVec = jobInfo.returnedColVec; + uint32_t projColsUDAFIndex = 0; for (uint64_t i = 0; i < returnedColVec.size(); i++) { // skip if not an aggregation column @@ -2746,18 +2766,26 @@ void TupleAggregateStep::prep2PhasesAggregate( SP_ROWAGG_FUNC_t funct; if (aggOp == ROWAGG_UDAF) { - UDAFColumn* udafc = dynamic_cast(jobInfo.projectionCols[i].get()); - if (udafc) - { - pUDAFFunc = udafc->getContext().getFunction(); - // Create a RowAggFunctionCol (UDAF subtype) with the context. - funct.reset(new RowUDAFFunctionCol(udafc->getContext(), colProj, colAggPm)); - } - else - { - throw logic_error("prep2PhasesAggregate: A UDAF function is called but there's no UDAFColumn"); - } - } + std::vector::iterator it = jobInfo.projectionCols.begin() + projColsUDAFIndex; + for (; it != jobInfo.projectionCols.end(); it++) + { + UDAFColumn* udafc = dynamic_cast((*it).get()); + projColsUDAFIndex++; + if (udafc) + { + pUDAFFunc = udafc->getContext().getFunction(); + // Create a RowAggFunctionCol (UDAF subtype) with the context. + funct.reset(new RowUDAFFunctionCol(udafc->getContext(), colProj, colAggPm)); + break; + } + + } + + if (it == jobInfo.projectionCols.end()) + { + throw logic_error("prep2PhasesAggregate: A UDAF function is called but there's no/not enough UDAFColumn/-s"); + } + } else { funct.reset(new RowAggFunctionCol(aggOp, stats, colProj, colAggPm)); @@ -3301,6 +3329,7 @@ void TupleAggregateStep::prep2PhasesDistinctAggregate( vector > aggColVec, aggNoDistColVec; set avgSet, avgDistSet; vector >& returnedColVec = jobInfo.returnedColVec; + uint32_t projColsUDAFIndex = 0; for (uint64_t i = 0; i < returnedColVec.size(); i++) { // col should be an aggregate or groupBy or window function @@ -3498,18 +3527,25 @@ void TupleAggregateStep::prep2PhasesDistinctAggregate( SP_ROWAGG_FUNC_t funct; if (aggOp == ROWAGG_UDAF) { - UDAFColumn* udafc = dynamic_cast(jobInfo.projectionCols[i].get()); - if (udafc) - { - pUDAFFunc = udafc->getContext().getFunction(); - // Create a RowAggFunctionCol (UDAF subtype) with the context. - funct.reset(new RowUDAFFunctionCol(udafc->getContext(), colProj, colAggPm)); - } - else - { - throw logic_error("prep2PhasesDistinctAggregate: A UDAF function is called but there's no UDAFColumn"); - } - } + std::vector::iterator it = jobInfo.projectionCols.begin() + projColsUDAFIndex; + for (; it != jobInfo.projectionCols.end(); it++) + { + UDAFColumn* udafc = dynamic_cast((*it).get()); + projColsUDAFIndex++; + if (udafc) + { + pUDAFFunc = udafc->getContext().getFunction(); + // Create a RowAggFunctionCol (UDAF subtype) with the context. + funct.reset(new RowUDAFFunctionCol(udafc->getContext(), colProj, colAggPm)); + break; + } + } + + if (it == jobInfo.projectionCols.end()) + { + throw logic_error("prep2PhasesDistinctAggregate: A UDAF function is called but there's no/not enough UDAFColumn/-s"); + } + } else { funct.reset(new RowAggFunctionCol(aggOp, stats, colProj, colAggPm)); From 863e18744bb2da10c881be21667e5bfef23b1dee Mon Sep 17 00:00:00 2001 From: david hill Date: Wed, 25 Apr 2018 14:07:18 -0500 Subject: [PATCH 35/49] update to 1.1.5 --- VERSION | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/VERSION b/VERSION index 445a70a7a..aef96eefc 100644 --- a/VERSION +++ b/VERSION @@ -1,4 +1,4 @@ COLUMNSTORE_VERSION_MAJOR=1 COLUMNSTORE_VERSION_MINOR=1 -COLUMNSTORE_VERSION_PATCH=4 +COLUMNSTORE_VERSION_PATCH=5 COLUMNSTORE_VERSION_RELEASE=1 From b584a7f555a2f40fd5e41fc2c547896e77643910 Mon Sep 17 00:00:00 2001 From: Andrew Hutchings Date: Thu, 19 Apr 2018 18:51:18 +0100 Subject: [PATCH 36/49] MCOL-392 Add DATETIME microscond support Add initial microsecond support for DATETIME --- dbcon/ddlpackage/ddl.y | 3 ++- dbcon/mysql/ha_calpont_dml.cpp | 18 ++++++++++--- dbcon/mysql/ha_calpont_impl.cpp | 4 +-- utils/dataconvert/dataconvert.cpp | 18 +++++++++++-- utils/dataconvert/dataconvert.h | 33 +++++++++++++++++------- writeengine/server/we_dmlcommandproc.cpp | 4 +-- 6 files changed, 60 insertions(+), 20 deletions(-) diff --git a/dbcon/ddlpackage/ddl.y b/dbcon/ddlpackage/ddl.y index 8f6e713fc..c084f44fe 100644 --- a/dbcon/ddlpackage/ddl.y +++ b/dbcon/ddlpackage/ddl.y @@ -1114,10 +1114,11 @@ literal: ; datetime_type: - DATETIME + DATETIME opt_time_precision { $$ = new ColumnType(DDL_DATETIME); $$->fLength = DDLDatatypeLength[DDL_DATETIME]; + $$->fPrecision = $2; } | DATE diff --git a/dbcon/mysql/ha_calpont_dml.cpp b/dbcon/mysql/ha_calpont_dml.cpp index 1cef4d4da..c4ca59862 100644 --- a/dbcon/mysql/ha_calpont_dml.cpp +++ b/dbcon/mysql/ha_calpont_dml.cpp @@ -838,11 +838,21 @@ int ha_calpont_impl_write_batch_row_(uchar* buf, TABLE* table, cal_impl_if::cal_ // mariadb 10.1 compatibility -- MYSQL_TYPE_DATETIME2 introduced in mysql 5.6 MYSQL_TIME ltime; const uchar* pos = buf; - longlong tmp = my_datetime_packed_from_binary(pos, 0); + longlong tmp = my_datetime_packed_from_binary(pos, table->field[colpos]->decimals()); TIME_from_longlong_datetime_packed(<ime, tmp); - fprintf(ci.filePtr, "%04d-%02d-%02d %02d:%02d:%02d%c", - ltime.year, ltime.month, ltime.day, - ltime.hour, ltime.minute, ltime.second, ci.delimiter); + if (!ltime.second_part) + { + fprintf(ci.filePtr, "%04d-%02d-%02d %02d:%02d:%02d%c", + ltime.year, ltime.month, ltime.day, + ltime.hour, ltime.minute, ltime.second, ci.delimiter); + } + else + { + fprintf(ci.filePtr, "%04d-%02d-%02d %02d:%02d:%02d.%ld%c", + ltime.year, ltime.month, ltime.day, + ltime.hour, ltime.minute, ltime.second, + ltime.second_part, ci.delimiter); + } buf += table->field[colpos]->pack_length(); } else diff --git a/dbcon/mysql/ha_calpont_impl.cpp b/dbcon/mysql/ha_calpont_impl.cpp index 8078d452e..6bd887332 100644 --- a/dbcon/mysql/ha_calpont_impl.cpp +++ b/dbcon/mysql/ha_calpont_impl.cpp @@ -486,8 +486,8 @@ int fetchNextRow(uchar* buf, cal_table_info& ti, cal_connection_info* ci) * based on the result set. */ /* MCOL-683: UTF-8 datetime no msecs is 57, this sometimes happens! */ - if (((*f)->field_length > 19) && ((*f)->field_length != 57)) - (*f)->field_length = strlen(tmp); +// if (((*f)->field_length > 19) && ((*f)->field_length != 57)) +// (*f)->field_length = strlen(tmp); Field_varstring* f2 = (Field_varstring*)*f; f2->store(tmp, strlen(tmp), f2->charset()); diff --git a/utils/dataconvert/dataconvert.cpp b/utils/dataconvert/dataconvert.cpp index d0689fb0d..6e8a33dc1 100644 --- a/utils/dataconvert/dataconvert.cpp +++ b/utils/dataconvert/dataconvert.cpp @@ -1717,14 +1717,28 @@ std::string DataConvert::dateToString( int datevalue ) return buf; } -std::string DataConvert::datetimeToString( long long datetimevalue ) +std::string DataConvert::datetimeToString( long long datetimevalue, long decimals ) { + // 10 is default which means we don't need microseconds + if (decimals > 6 || decimals < 0) + { + decimals = 0; + } // @bug 4703 abandon multiple ostringstream's for conversion DateTime dt(datetimevalue); - const int DATETIMETOSTRING_LEN = 21; // YYYY-MM-DD HH:MM:SS\0 + const int DATETIMETOSTRING_LEN = 28; // YYYY-MM-DD HH:MM:SS.mmmmmm\0 char buf[DATETIMETOSTRING_LEN]; sprintf(buf, "%04d-%02d-%02d %02d:%02d:%02d", dt.year, dt.month, dt.day, dt.hour, dt.minute, dt.second); + if (dt.msecond && decimals) + { + snprintf(buf + strlen(buf), 21 + decimals, ".%d", dt.msecond); + // Pad end with zeros + if (strlen(buf) < (21 + decimals)) + { + sprintf(buf + strlen(buf), "%0*d", 21 + decimals - strlen(buf), 0); + } + } return buf; } diff --git a/utils/dataconvert/dataconvert.h b/utils/dataconvert/dataconvert.h index 1debe1bcc..d2d9b834f 100644 --- a/utils/dataconvert/dataconvert.h +++ b/utils/dataconvert/dataconvert.h @@ -361,7 +361,7 @@ public: * @param type the columns database type * @param data the columns string representation of it's data */ - EXPORT static std::string datetimeToString( long long datetimevalue ); + EXPORT static std::string datetimeToString( long long datetimevalue, long decimals = 0 ); static inline void datetimeToString( long long datetimevalue, char* buf, unsigned int buflen ); /** @@ -459,14 +459,29 @@ inline void DataConvert::dateToString( int datevalue, char* buf, unsigned int bu inline void DataConvert::datetimeToString( long long datetimevalue, char* buf, unsigned int buflen ) { - snprintf( buf, buflen, "%04d-%02d-%02d %02d:%02d:%02d", - (unsigned)((datetimevalue >> 48) & 0xffff), - (unsigned)((datetimevalue >> 44) & 0xf), - (unsigned)((datetimevalue >> 38) & 0x3f), - (unsigned)((datetimevalue >> 32) & 0x3f), - (unsigned)((datetimevalue >> 26) & 0x3f), - (unsigned)((datetimevalue >> 20) & 0x3f) - ); + if ((datetimevalue & 0xfffff) > 0) + { + snprintf( buf, buflen, "%04d-%02d-%02d %02d:%02d:%02d.%d", + (unsigned)((datetimevalue >> 48) & 0xffff), + (unsigned)((datetimevalue >> 44) & 0xf), + (unsigned)((datetimevalue >> 38) & 0x3f), + (unsigned)((datetimevalue >> 32) & 0x3f), + (unsigned)((datetimevalue >> 26) & 0x3f), + (unsigned)((datetimevalue >> 20) & 0x3f), + (unsigned)((datetimevalue) & 0xfffff) + ); + } + else + { + snprintf( buf, buflen, "%04d-%02d-%02d %02d:%02d:%02d", + (unsigned)((datetimevalue >> 48) & 0xffff), + (unsigned)((datetimevalue >> 44) & 0xf), + (unsigned)((datetimevalue >> 38) & 0x3f), + (unsigned)((datetimevalue >> 32) & 0x3f), + (unsigned)((datetimevalue >> 26) & 0x3f), + (unsigned)((datetimevalue >> 20) & 0x3f) + ); + } } inline void DataConvert::dateToString1( int datevalue, char* buf, unsigned int buflen) diff --git a/writeengine/server/we_dmlcommandproc.cpp b/writeengine/server/we_dmlcommandproc.cpp index bf5adf906..07037a1e3 100644 --- a/writeengine/server/we_dmlcommandproc.cpp +++ b/writeengine/server/we_dmlcommandproc.cpp @@ -2977,7 +2977,7 @@ uint8_t WE_DMLCommandProc::processUpdate(messageqcpp::ByteStream& bs, case CalpontSystemCatalog::DATETIME: { intColVal = row.getUintField<8>(fetchColPos); - value = DataConvert::datetimeToString(intColVal); + value = DataConvert::datetimeToString(intColVal, colType.precision); break; } @@ -3305,7 +3305,7 @@ uint8_t WE_DMLCommandProc::processUpdate(messageqcpp::ByteStream& bs, case CalpontSystemCatalog::DATETIME: { intColVal = row.getUintField<8>(fetchColPos); - value = DataConvert::datetimeToString(intColVal); + value = DataConvert::datetimeToString(intColVal, colType.precision); break; } From 3c1ebd8b94c7e87c0d7f8956a693e971b4c2142d Mon Sep 17 00:00:00 2001 From: Andrew Hutchings Date: Mon, 23 Apr 2018 19:20:31 +0100 Subject: [PATCH 37/49] MCOL-392 Add initial TIME datatype support --- dbcon/ddlpackage/ddl.l | 1 + dbcon/ddlpackage/ddl.y | 13 +- dbcon/ddlpackage/ddlpkg.h | 3 + dbcon/ddlpackageproc/altertableprocessor.cpp | 5 + dbcon/ddlpackageproc/ddlindexpopulator.cpp | 2 + dbcon/ddlpackageproc/ddlpackageprocessor.cpp | 11 + dbcon/execplan/aggregatecolumn.cpp | 8 + dbcon/execplan/aggregatecolumn.h | 4 +- dbcon/execplan/arithmeticcolumn.h | 5 + dbcon/execplan/arithmeticoperator.h | 5 + dbcon/execplan/calpontsystemcatalog.cpp | 4 + dbcon/execplan/calpontsystemcatalog.h | 1 + dbcon/execplan/constantcolumn.h | 15 + dbcon/execplan/functioncolumn.h | 5 +- dbcon/execplan/operator.h | 4 + dbcon/execplan/parsetree.h | 8 + dbcon/execplan/predicateoperator.cpp | 12 + dbcon/execplan/predicateoperator.h | 31 ++ dbcon/execplan/simplecolumn.cpp | 1 + dbcon/execplan/simplecolumn.h | 6 + dbcon/execplan/simplefilter.cpp | 31 +- dbcon/execplan/treenode.h | 50 +++ dbcon/execplan/windowfunctioncolumn.cpp | 10 + dbcon/execplan/windowfunctioncolumn.h | 6 +- dbcon/joblist/crossenginestep.cpp | 4 + dbcon/joblist/expressionstep.cpp | 5 +- dbcon/joblist/groupconcat.cpp | 24 ++ dbcon/joblist/jlf_common.cpp | 5 + dbcon/joblist/jlf_execplantojoblist.cpp | 8 + dbcon/joblist/jlf_subquery.cpp | 5 + dbcon/joblist/joblisttypes.h | 2 + dbcon/joblist/lbidlist.cpp | 1 + dbcon/joblist/pcolscan.cpp | 1 + dbcon/joblist/subquerytransformer.cpp | 5 +- dbcon/joblist/tupleaggregatestep.cpp | 33 +- dbcon/joblist/tupleconstantstep.cpp | 1 + dbcon/joblist/tupleunion.cpp | 37 +- dbcon/joblist/windowfunctionstep.cpp | 2 + dbcon/mysql/ha_calpont_ddl.cpp | 4 + dbcon/mysql/ha_calpont_dml.cpp | 37 +- dbcon/mysql/ha_calpont_execplan.cpp | 12 + dbcon/mysql/ha_calpont_impl.cpp | 13 + dbcon/mysql/ha_calpont_partition.cpp | 11 + dbcon/mysql/ha_window_function.cpp | 2 + primitives/linux-port/column.cpp | 9 + primitives/primproc/columncommand.cpp | 1 + tools/pingproc/pingproc.cpp | 1 + utils/common/nullvaluemanip.cpp | 6 + utils/dataconvert/dataconvert.cpp | 378 ++++++++++++++++++ utils/dataconvert/dataconvert.h | 116 +++++- utils/funcexp/func_add_time.cpp | 124 ++++++ utils/funcexp/func_between.cpp | 18 + utils/funcexp/func_bitand.cpp | 25 +- utils/funcexp/func_bitwise.cpp | 9 + utils/funcexp/func_case.cpp | 27 ++ utils/funcexp/func_cast.cpp | 116 ++++++ utils/funcexp/func_ceil.cpp | 18 + utils/funcexp/func_char_length.cpp | 6 + utils/funcexp/func_coalesce.cpp | 24 ++ utils/funcexp/func_extract.cpp | 1 + utils/funcexp/func_floor.cpp | 26 ++ utils/funcexp/func_from_unixtime.cpp | 16 + utils/funcexp/func_greatest.cpp | 20 + utils/funcexp/func_hour.cpp | 6 + utils/funcexp/func_if.cpp | 16 +- utils/funcexp/func_ifnull.cpp | 19 + utils/funcexp/func_in.cpp | 21 + utils/funcexp/func_inet_aton.cpp | 20 + utils/funcexp/func_inet_ntoa.cpp | 14 + utils/funcexp/func_least.cpp | 19 + utils/funcexp/func_math.cpp | 225 +++++++++++ utils/funcexp/func_minute.cpp | 1 + utils/funcexp/func_nullif.cpp | 111 +++++ utils/funcexp/func_regexp.cpp | 16 + utils/funcexp/func_round.cpp | 1 + utils/funcexp/func_second.cpp | 6 + utils/funcexp/func_sysdate.cpp | 9 + utils/funcexp/func_time.cpp | 1 + utils/funcexp/func_timediff.cpp | 10 + utils/funcexp/func_timestampdiff.cpp | 7 + utils/funcexp/func_truncate.cpp | 45 +++ utils/funcexp/funcexp.cpp | 12 + utils/funcexp/functor.cpp | 20 + utils/funcexp/functor.h | 10 + utils/funcexp/functor_all.h | 44 +- utils/funcexp/functor_bool.h | 9 + utils/funcexp/functor_dtm.h | 31 ++ utils/funcexp/functor_real.h | 6 + utils/funcexp/functor_str.h | 12 + utils/rowgroup/rowaggregation.cpp | 59 ++- utils/rowgroup/rowgroup.cpp | 7 + .../docs/source/reference/ColumnDatum.rst | 2 + utils/udfsdk/mcsv1_udaf.cpp | 1 + utils/udfsdk/udfsdk.cpp | 4 +- utils/windowfunction/idborderby.cpp | 2 + utils/windowfunction/wf_lead_lag.cpp | 1 + utils/windowfunction/wf_min_max.cpp | 1 + utils/windowfunction/wf_nth_value.cpp | 1 + utils/windowfunction/wf_percentile.cpp | 1 + utils/windowfunction/wf_udaf.cpp | 1 + utils/windowfunction/windowfunctiontype.cpp | 6 + writeengine/bulk/we_bulkloadbuffer.cpp | 61 ++- writeengine/bulk/we_tableinfo.cpp | 5 + writeengine/server/we_ddlcommon.h | 11 + writeengine/server/we_dmlcommandproc.cpp | 15 + writeengine/shared/we_convertor.cpp | 3 + writeengine/splitter/we_sdhandler.cpp | 4 + writeengine/wrapper/writeengine.cpp | 2 + writeengine/xml/we_xmljob.cpp | 16 + 109 files changed, 2241 insertions(+), 47 deletions(-) diff --git a/dbcon/ddlpackage/ddl.l b/dbcon/ddlpackage/ddl.l index e922c08ca..7e7400451 100644 --- a/dbcon/ddlpackage/ddl.l +++ b/dbcon/ddlpackage/ddl.l @@ -118,6 +118,7 @@ CREATE {return CREATE;} CURRENT_USER {return CURRENT_USER;} DATE {ddlget_lval(yyscanner)->str=strdup("date"); return DATE;} DATETIME {return DATETIME;} +TIME {return TIME;} DECIMAL {return DECIMAL;} DEC {return DECIMAL;} DEFAULT {return DEFAULT;} diff --git a/dbcon/ddlpackage/ddl.y b/dbcon/ddlpackage/ddl.y index c084f44fe..68747dfc9 100644 --- a/dbcon/ddlpackage/ddl.y +++ b/dbcon/ddlpackage/ddl.y @@ -177,7 +177,6 @@ VARYING WITH ZONE DOUBLE IDB_FLOAT REAL CHARSET IDB_IF EXISTS CHANGE TRUNCATE %type opt_precision_scale %type opt_referential_triggered_action %type opt_time_precision -%type opt_with_time_zone %type qualified_name %type referential_action %type referential_triggered_action @@ -1127,12 +1126,11 @@ datetime_type: $$->fLength = DDLDatatypeLength[DDL_DATE]; } | - TIME opt_time_precision opt_with_time_zone + TIME opt_time_precision { - $$ = new ColumnType(DDL_DATETIME); - $$->fLength = DDLDatatypeLength[DDL_DATETIME]; + $$ = new ColumnType(DDL_TIME); + $$->fLength = DDLDatatypeLength[DDL_TIME]; $$->fPrecision = $2; - $$->fWithTimezone = $3; } opt_time_precision: @@ -1140,11 +1138,6 @@ opt_time_precision: | {$$ = -1;} ; -opt_with_time_zone: - WITH TIME ZONE {$$ = true;} - | {$$ = false;} - ; - drop_column_def: DROP column_name drop_behavior {$$ = new AtaDropColumn($2, $3);} | DROP COLUMN column_name drop_behavior {$$ = new AtaDropColumn($3, $4);} diff --git a/dbcon/ddlpackage/ddlpkg.h b/dbcon/ddlpackage/ddlpkg.h index 97e971b6b..779577f83 100644 --- a/dbcon/ddlpackage/ddlpkg.h +++ b/dbcon/ddlpackage/ddlpkg.h @@ -236,6 +236,7 @@ enum DDL_DATATYPES DDL_UNSIGNED_DOUBLE, DDL_UNSIGNED_NUMERIC, DDL_TEXT, + DDL_TIME, DDL_INVALID_DATATYPE }; @@ -273,6 +274,7 @@ const std::string DDLDatatypeString[] = "unsigned-double", "unsigned-numeric", "text", + "time" "" }; @@ -328,6 +330,7 @@ const int DDLDatatypeLength[] = 8, // UNSIGNED_DOUBLE, 2, // UNSIGNED_NUMERIC, 8, // TEXT + 8, // TIME -1 // INVALID LENGTH }; diff --git a/dbcon/ddlpackageproc/altertableprocessor.cpp b/dbcon/ddlpackageproc/altertableprocessor.cpp index c6edff71c..a398e656f 100644 --- a/dbcon/ddlpackageproc/altertableprocessor.cpp +++ b/dbcon/ddlpackageproc/altertableprocessor.cpp @@ -209,6 +209,11 @@ bool typesAreSame(const CalpontSystemCatalog::ColType& colType, const ColumnType break; + case (CalpontSystemCatalog::TIME): + if (newType.fType == DDL_TIME) return true; + + break; + case (CalpontSystemCatalog::VARCHAR): if (newType.fType == DDL_VARCHAR && colType.colWidth == newType.fLength) return true; diff --git a/dbcon/ddlpackageproc/ddlindexpopulator.cpp b/dbcon/ddlpackageproc/ddlindexpopulator.cpp index 12d6c9854..cb58df5bc 100644 --- a/dbcon/ddlpackageproc/ddlindexpopulator.cpp +++ b/dbcon/ddlpackageproc/ddlindexpopulator.cpp @@ -330,6 +330,7 @@ boost::any DDLIndexPopulator::convertData(const CalpontSystemCatalog::ColType& return *reinterpret_cast(&data); case execplan::CalpontSystemCatalog::DATETIME: // @bug 375 + case execplan::CalpontSystemCatalog::TIME: case execplan::CalpontSystemCatalog::BIGINT: return *reinterpret_cast(&data); @@ -524,6 +525,7 @@ bool DDLIndexPopulator::checkNotNull(const IdxTuple& data, const CalpontSystemCa break; case execplan::CalpontSystemCatalog::DATETIME: + case execplan::CalpontSystemCatalog::TIME: isNull = any_cast(data.data) == any_cast(nullvalue); break; diff --git a/dbcon/ddlpackageproc/ddlpackageprocessor.cpp b/dbcon/ddlpackageproc/ddlpackageprocessor.cpp index 9b8c66658..2a64dd1f7 100644 --- a/dbcon/ddlpackageproc/ddlpackageprocessor.cpp +++ b/dbcon/ddlpackageproc/ddlpackageprocessor.cpp @@ -232,6 +232,10 @@ execplan::CalpontSystemCatalog::ColDataType DDLPackageProcessor::convertDataType colDataType = CalpontSystemCatalog::DATETIME; break; + case ddlpackage::DDL_TIME: + colDataType = CalpontSystemCatalog::TIME; + break; + case ddlpackage::DDL_CLOB: colDataType = CalpontSystemCatalog::CLOB; break; @@ -476,6 +480,13 @@ DDLPackageProcessor::getNullValueForType(const execplan::CalpontSystemCatalog::C } break; + case execplan::CalpontSystemCatalog::TIME: + { + long long d = joblist::TIMENULL; + value = d; + } + break; + case execplan::CalpontSystemCatalog::CHAR: { std::string charnull; diff --git a/dbcon/execplan/aggregatecolumn.cpp b/dbcon/execplan/aggregatecolumn.cpp index e0914beb3..18cba2607 100644 --- a/dbcon/execplan/aggregatecolumn.cpp +++ b/dbcon/execplan/aggregatecolumn.cpp @@ -353,6 +353,14 @@ void AggregateColumn::evaluate(Row& row, bool& isNull) break; + case CalpontSystemCatalog::TIME: + if (row.equals<8>(TIMENULL, fInputIndex)) + isNull = true; + else + fResult.intVal = row.getIntField<8>(fInputIndex); + + break; + case CalpontSystemCatalog::CHAR: case CalpontSystemCatalog::VARCHAR: case CalpontSystemCatalog::STRINT: diff --git a/dbcon/execplan/aggregatecolumn.h b/dbcon/execplan/aggregatecolumn.h index a70189e50..028f1ee54 100644 --- a/dbcon/execplan/aggregatecolumn.h +++ b/dbcon/execplan/aggregatecolumn.h @@ -411,10 +411,10 @@ public: /** * F&E */ - virtual int64_t getDatetimeIntVal(rowgroup::Row& row, bool& isNull) + virtual int64_t getTimeIntVal(rowgroup::Row& row, bool& isNull) { evaluate(row, isNull); - return TreeNode::getDatetimeIntVal(); + return TreeNode::getTimeIntVal(); } private: diff --git a/dbcon/execplan/arithmeticcolumn.h b/dbcon/execplan/arithmeticcolumn.h index b6ca823c4..191416fbf 100644 --- a/dbcon/execplan/arithmeticcolumn.h +++ b/dbcon/execplan/arithmeticcolumn.h @@ -248,6 +248,11 @@ public: return fExpression->getDatetimeIntVal(row, isNull); } + virtual int64_t getTimeIntVal(rowgroup::Row& row, bool& isNull) + { + return fExpression->getTimeIntVal(row, isNull); + } + virtual bool getBoolVal(rowgroup::Row& row, bool& isNull) { return fExpression->getBoolVal(row, isNull); diff --git a/dbcon/execplan/arithmeticoperator.h b/dbcon/execplan/arithmeticoperator.h index ef7b538fa..ca254075b 100644 --- a/dbcon/execplan/arithmeticoperator.h +++ b/dbcon/execplan/arithmeticoperator.h @@ -151,6 +151,11 @@ public: evaluate(row, isNull, lop, rop); return TreeNode::getDatetimeIntVal(); } + virtual int64_t getTimeIntVal(rowgroup::Row& row, bool& isNull, ParseTree* lop, ParseTree* rop) + { + evaluate(row, isNull, lop, rop); + return TreeNode::getTimeIntVal(); + } virtual bool getBoolVal(rowgroup::Row& row, bool& isNull, ParseTree* lop, ParseTree* rop) { evaluate(row, isNull, lop, rop); diff --git a/dbcon/execplan/calpontsystemcatalog.cpp b/dbcon/execplan/calpontsystemcatalog.cpp index 501f488ce..15ccd73aa 100644 --- a/dbcon/execplan/calpontsystemcatalog.cpp +++ b/dbcon/execplan/calpontsystemcatalog.cpp @@ -156,6 +156,10 @@ const string colDataTypeToString(CalpontSystemCatalog::ColDataType cdt) return "datetime"; break; + case CalpontSystemCatalog::TIME: + return "time"; + break; + case CalpontSystemCatalog::VARCHAR: return "varchar"; break; diff --git a/dbcon/execplan/calpontsystemcatalog.h b/dbcon/execplan/calpontsystemcatalog.h index 5918f7f8c..7b828a297 100644 --- a/dbcon/execplan/calpontsystemcatalog.h +++ b/dbcon/execplan/calpontsystemcatalog.h @@ -154,6 +154,7 @@ public: UBIGINT, /*!< Unsigned BIGINT type */ UDOUBLE, /*!< Unsigned DOUBLE type */ TEXT, /*!< TEXT type */ + TIME, /*!< TIME type */ NUM_OF_COL_DATA_TYPE, /* NEW TYPES ABOVE HERE */ LONGDOUBLE, /* @bug3241, dev and variance calculation only */ STRINT, /* @bug3532, string as int for fast comparison */ diff --git a/dbcon/execplan/constantcolumn.h b/dbcon/execplan/constantcolumn.h index 8e7f178aa..04098faae 100644 --- a/dbcon/execplan/constantcolumn.h +++ b/dbcon/execplan/constantcolumn.h @@ -308,6 +308,21 @@ public: return fResult.intVal; } + /** + * F&E + */ + virtual int64_t getTimeIntVal(rowgroup::Row& row, bool& isNull) + { + isNull = isNull || (fType == NULLDATA); + + if (!fResult.valueConverted) + { + fResult.intVal = dataconvert::DataConvert::stringToTime(fResult.strVal); + fResult.valueConverted = true; + } + + return fResult.intVal; + } /** * F&E */ diff --git a/dbcon/execplan/functioncolumn.h b/dbcon/execplan/functioncolumn.h index 8f27cad75..5a099d1d2 100644 --- a/dbcon/execplan/functioncolumn.h +++ b/dbcon/execplan/functioncolumn.h @@ -255,7 +255,10 @@ public: { return fFunctor->getDatetimeIntVal(row, fFunctionParms, isNull, fOperationType); } - + virtual int64_t getTimeIntVal(rowgroup::Row& row, bool& isNull) + { + return fFunctor->getTimeIntVal(row, fFunctionParms, isNull, fOperationType); + } private: funcexp::FunctionParm fFunctionParms; diff --git a/dbcon/execplan/operator.h b/dbcon/execplan/operator.h index e0e16be2f..1a33dcb94 100644 --- a/dbcon/execplan/operator.h +++ b/dbcon/execplan/operator.h @@ -190,6 +190,10 @@ public: { return fResult.intVal; } + virtual int64_t getTimeIntVal(rowgroup::Row& row, bool& isNull, ParseTree* lop, ParseTree* rop) + { + return fResult.intVal; + } virtual bool getBoolVal(rowgroup::Row& row, bool& isNull, ParseTree* lop, ParseTree* rop) { return fResult.boolVal; diff --git a/dbcon/execplan/parsetree.h b/dbcon/execplan/parsetree.h index 313f2906a..2ad41b6bb 100644 --- a/dbcon/execplan/parsetree.h +++ b/dbcon/execplan/parsetree.h @@ -282,6 +282,14 @@ public: return fData->getDatetimeIntVal(row, isNull); } + inline int64_t getTimeIntVal(rowgroup::Row& row, bool& isNull) + { + if (fLeft && fRight) + return (reinterpret_cast(fData))->getTimeIntVal(row, isNull, fLeft, fRight); + else + return fData->getTimeIntVal(row, isNull); + } + private: /** draw the tree * diff --git a/dbcon/execplan/predicateoperator.cpp b/dbcon/execplan/predicateoperator.cpp index 84579df22..166a8edc0 100644 --- a/dbcon/execplan/predicateoperator.cpp +++ b/dbcon/execplan/predicateoperator.cpp @@ -196,6 +196,7 @@ bool PredicateOperator::operator!=(const TreeNode* t) const void PredicateOperator::setOpType(Type& l, Type& r) { if ( l.colDataType == execplan::CalpontSystemCatalog::DATETIME || + l.colDataType == execplan::CalpontSystemCatalog::TIME || l.colDataType == execplan::CalpontSystemCatalog::DATE ) { switch (r.colDataType) @@ -210,6 +211,11 @@ void PredicateOperator::setOpType(Type& l, Type& r) fOperationType.colWidth = 8; break; + case execplan::CalpontSystemCatalog::TIME: + fOperationType.colDataType = execplan::CalpontSystemCatalog::TIME; + fOperationType.colWidth = 8; + break; + case execplan::CalpontSystemCatalog::DATE: fOperationType = l; break; @@ -221,6 +227,7 @@ void PredicateOperator::setOpType(Type& l, Type& r) } } else if ( r.colDataType == execplan::CalpontSystemCatalog::DATETIME || + r.colDataType == execplan::CalpontSystemCatalog::TIME || r.colDataType == execplan::CalpontSystemCatalog::DATE ) { switch (l.colDataType) @@ -236,6 +243,11 @@ void PredicateOperator::setOpType(Type& l, Type& r) fOperationType.colWidth = 8; break; + case execplan::CalpontSystemCatalog::TIME: + fOperationType.colDataType = execplan::CalpontSystemCatalog::TIME; + fOperationType.colWidth = 8; + break; + case execplan::CalpontSystemCatalog::DATE: fOperationType = r; break; diff --git a/dbcon/execplan/predicateoperator.h b/dbcon/execplan/predicateoperator.h index 2b63548d5..6253d9389 100644 --- a/dbcon/execplan/predicateoperator.h +++ b/dbcon/execplan/predicateoperator.h @@ -344,6 +344,37 @@ inline bool PredicateOperator::getBoolVal(rowgroup::Row& row, bool& isNull, Retu return numericCompare(val1, rop->getDatetimeIntVal(row, isNull)) && !isNull; } + case execplan::CalpontSystemCatalog::TIME: + { + if (fOp == OP_ISNULL) + { + lop->getTimeIntVal(row, isNull); + bool ret = isNull; + isNull = false; + return ret; + } + + if (fOp == OP_ISNOTNULL) + { + lop->getTimeIntVal(row, isNull); + bool ret = isNull; + isNull = false; + return !ret; + } + + if (isNull) + return false; + + int64_t val1 = lop->getTimeIntVal(row, isNull); + + if (isNull) + return false; + + return numericCompare(val1, rop->getTimeIntVal(row, isNull)) && !isNull; + } + + + case execplan::CalpontSystemCatalog::VARCHAR: case execplan::CalpontSystemCatalog::CHAR: case execplan::CalpontSystemCatalog::TEXT: diff --git a/dbcon/execplan/simplecolumn.cpp b/dbcon/execplan/simplecolumn.cpp index 429e4f883..64955401e 100644 --- a/dbcon/execplan/simplecolumn.cpp +++ b/dbcon/execplan/simplecolumn.cpp @@ -502,6 +502,7 @@ void SimpleColumn::evaluate(Row& row, bool& isNull) } case CalpontSystemCatalog::DATETIME: + case CalpontSystemCatalog::TIME: { fResult.intVal = row.getUintField<8>(fInputIndex); break; diff --git a/dbcon/execplan/simplecolumn.h b/dbcon/execplan/simplecolumn.h index 81571e057..60eff939b 100644 --- a/dbcon/execplan/simplecolumn.h +++ b/dbcon/execplan/simplecolumn.h @@ -331,6 +331,12 @@ public: return TreeNode::getDatetimeIntVal(); } + inline int64_t getTimeIntVal(rowgroup::Row& row, bool& isNull) + { + evaluate(row, isNull); + return TreeNode::getTimeIntVal(); + } + }; typedef boost::shared_ptr SSC; diff --git a/dbcon/execplan/simplefilter.cpp b/dbcon/execplan/simplefilter.cpp index 1df673dcd..1da14449c 100644 --- a/dbcon/execplan/simplefilter.cpp +++ b/dbcon/execplan/simplefilter.cpp @@ -209,7 +209,8 @@ const string SimpleFilter::data() const fRhs->resultType().colDataType == CalpontSystemCatalog::TEXT || fRhs->resultType().colDataType == CalpontSystemCatalog::VARBINARY || fRhs->resultType().colDataType == CalpontSystemCatalog::DATE || - fRhs->resultType().colDataType == CalpontSystemCatalog::DATETIME)) + fRhs->resultType().colDataType == CalpontSystemCatalog::DATETIME || + fRhs->resultType().colDataType == CalpontSystemCatalog::TIME)) rhs = "'" + SimpleFilter::escapeString(fRhs->data()) + "'"; else rhs = fRhs->data(); @@ -221,6 +222,7 @@ const string SimpleFilter::data() const fLhs->resultType().colDataType == CalpontSystemCatalog::TEXT || fLhs->resultType().colDataType == CalpontSystemCatalog::VARBINARY || fLhs->resultType().colDataType == CalpontSystemCatalog::DATE || + fLhs->resultType().colDataType == CalpontSystemCatalog::TIME || fLhs->resultType().colDataType == CalpontSystemCatalog::DATETIME)) lhs = "'" + SimpleFilter::escapeString(fLhs->data()) + "'"; else @@ -544,6 +546,19 @@ void SimpleFilter::convertConstant() result.intVal = dataconvert::DataConvert::datetimeToInt(result.strVal); } } + else if (fRhs->resultType().colDataType == CalpontSystemCatalog::TIME) + { + if (lcc->constval().empty()) + { + lcc->constval("00:00:00"); + result.intVal = 0; + result.strVal = lcc->constval(); + } + else + { + result.intVal = dataconvert::DataConvert::timeToInt(result.strVal); + } + } lcc->result(result); } @@ -578,7 +593,19 @@ void SimpleFilter::convertConstant() result.intVal = dataconvert::DataConvert::datetimeToInt(result.strVal); } } - + else if (fLhs->resultType().colDataType == CalpontSystemCatalog::TIME) + { + if (rcc->constval().empty()) + { + rcc->constval("00:00:00"); + result.intVal = 0; + result.strVal = rcc->constval(); + } + else + { + result.intVal = dataconvert::DataConvert::timeToInt(result.strVal); + } + } rcc->result(result); } } diff --git a/dbcon/execplan/treenode.h b/dbcon/execplan/treenode.h index 92420da3f..27795ea9d 100644 --- a/dbcon/execplan/treenode.h +++ b/dbcon/execplan/treenode.h @@ -388,6 +388,10 @@ public: { return fResult.intVal; } + virtual int64_t getTimeIntVal(rowgroup::Row& row, bool& isNull) + { + return fResult.intVal; + } virtual void evaluate(rowgroup::Row& row, bool& isNull) {} inline bool getBoolVal(); @@ -399,6 +403,7 @@ public: inline IDB_Decimal getDecimalVal(); inline int32_t getDateIntVal(); inline int64_t getDatetimeIntVal(); + inline int64_t getTimeIntVal(); virtual const execplan::CalpontSystemCatalog::ColType& resultType() const { @@ -490,6 +495,7 @@ inline bool TreeNode::getBoolVal() case CalpontSystemCatalog::INT: case CalpontSystemCatalog::DATE: case CalpontSystemCatalog::DATETIME: + case CalpontSystemCatalog::TIME: return (fResult.intVal != 0); case CalpontSystemCatalog::UBIGINT: @@ -665,6 +671,13 @@ inline const std::string& TreeNode::getStrVal() break; } + case CalpontSystemCatalog::TIME: + { + dataconvert::DataConvert::timeToString(fResult.intVal, tmp, 255); + fResult.strVal = std::string(tmp); + break; + } + default: throw logging::InvalidConversionExcept("TreeNode::getStrVal: Invalid conversion."); } @@ -727,6 +740,7 @@ inline int64_t TreeNode::getIntVal() case CalpontSystemCatalog::DATE: case CalpontSystemCatalog::DATETIME: + case CalpontSystemCatalog::TIME: return fResult.intVal; default: @@ -769,6 +783,7 @@ inline uint64_t TreeNode::getUintVal() case CalpontSystemCatalog::DATE: case CalpontSystemCatalog::DATETIME: + case CalpontSystemCatalog::TIME: return fResult.intVal; default: @@ -831,6 +846,7 @@ inline float TreeNode::getFloatVal() case CalpontSystemCatalog::DATE: case CalpontSystemCatalog::DATETIME: + case CalpontSystemCatalog::TIME: return (float)fResult.intVal; default: @@ -895,6 +911,7 @@ inline double TreeNode::getDoubleVal() case CalpontSystemCatalog::DATE: case CalpontSystemCatalog::DATETIME: + case CalpontSystemCatalog::TIME: return (double)fResult.intVal; default: @@ -937,9 +954,14 @@ inline IDB_Decimal TreeNode::getDecimalVal() break; case CalpontSystemCatalog::DATE: + throw logging::InvalidConversionExcept("TreeNode::getDecimalVal: Invalid conversion from date."); + case CalpontSystemCatalog::DATETIME: throw logging::InvalidConversionExcept("TreeNode::getDecimalVal: Invalid conversion from datetime."); + case CalpontSystemCatalog::TIME: + throw logging::InvalidConversionExcept("TreeNode::getDecimalVal: Invalid conversion from time."); + case CalpontSystemCatalog::FLOAT: throw logging::InvalidConversionExcept("TreeNode::getDecimalVal: non-support conversion from float"); @@ -967,6 +989,17 @@ inline int64_t TreeNode::getDatetimeIntVal() { if (fResultType.colDataType == execplan::CalpontSystemCatalog::DATE) return (fResult.intVal & 0x00000000FFFFFFC0LL) << 32; + else if (fResultType.colDataType == execplan::CalpontSystemCatalog::TIME) + { + dataconvert::Time tt; + + memcpy(&tt, &fResult.intVal, 8); + if (tt.hour > 23 || tt.hour < 0) + throw logging::InvalidConversionExcept("TreeNode::getDecimalVal: Invalid conversion from time (out of range)."); + dataconvert::DateTime dt(0, 0, 0, tt.hour, tt.minute, tt.second, tt.msecond); + memcpy(&fResult.intVal, &dt, 8); + return fResult.intVal; + } else if (fResultType.colDataType == execplan::CalpontSystemCatalog::DATETIME) //return (fResult.intVal & 0xFFFFFFFFFFF00000LL); return (fResult.intVal); @@ -974,6 +1007,23 @@ inline int64_t TreeNode::getDatetimeIntVal() return getIntVal(); } +inline int64_t TreeNode::getTimeIntVal() +{ + if (fResultType.colDataType == execplan::CalpontSystemCatalog::DATETIME) + { + dataconvert::DateTime dt; + + memcpy(&dt, &fResult.intVal, 8); + dataconvert::Time tt(0, dt.hour, dt.minute, dt.second, dt.msecond); + memcpy(&fResult.intVal, &tt, 8); + return fResult.intVal; + } + else if (fResultType.colDataType == execplan::CalpontSystemCatalog::TIME) + return (fResult.intVal); + else + return getIntVal(); +} + inline int32_t TreeNode::getDateIntVal() { if (fResultType.colDataType == execplan::CalpontSystemCatalog::DATETIME) diff --git a/dbcon/execplan/windowfunctioncolumn.cpp b/dbcon/execplan/windowfunctioncolumn.cpp index f81a6a5d4..5c84ff2d1 100644 --- a/dbcon/execplan/windowfunctioncolumn.cpp +++ b/dbcon/execplan/windowfunctioncolumn.cpp @@ -403,6 +403,16 @@ void WindowFunctionColumn::evaluate(Row& row, bool& isNull) break; } + case CalpontSystemCatalog::TIME: + { + if (row.equals<8>(TIMENULL, fInputIndex)) + isNull = true; + else + fResult.intVal = row.getIntField<8>(fInputIndex); + + break; + } + case CalpontSystemCatalog::CHAR: case CalpontSystemCatalog::VARCHAR: case CalpontSystemCatalog::STRINT: diff --git a/dbcon/execplan/windowfunctioncolumn.h b/dbcon/execplan/windowfunctioncolumn.h index 8b427947c..3fc983e55 100644 --- a/dbcon/execplan/windowfunctioncolumn.h +++ b/dbcon/execplan/windowfunctioncolumn.h @@ -213,7 +213,11 @@ public: evaluate(row, isNull); return TreeNode::getDatetimeIntVal(); } - + virtual int64_t getTimeIntVal(rowgroup::Row& row, bool& isNull) + { + evaluate(row, isNull); + return TreeNode::getTimeIntVal(); + } private: void evaluate(rowgroup::Row& row, bool& isNull); }; diff --git a/dbcon/joblist/crossenginestep.cpp b/dbcon/joblist/crossenginestep.cpp index 483249067..789e58cd4 100644 --- a/dbcon/joblist/crossenginestep.cpp +++ b/dbcon/joblist/crossenginestep.cpp @@ -351,6 +351,10 @@ int64_t CrossEngineStep::convertValueNum( rv = boost::any_cast(anyVal); break; + case CalpontSystemCatalog::TIME: + rv = boost::any_cast(anyVal); + break; + case CalpontSystemCatalog::DECIMAL: case CalpontSystemCatalog::UDECIMAL: if (ct.colWidth == CalpontSystemCatalog::ONE_BYTE) diff --git a/dbcon/joblist/expressionstep.cpp b/dbcon/joblist/expressionstep.cpp index 4a169e081..0e064c359 100644 --- a/dbcon/joblist/expressionstep.cpp +++ b/dbcon/joblist/expressionstep.cpp @@ -403,8 +403,9 @@ void ExpressionStep::populateColumnInfo(SimpleColumn* sc, JobInfo& jobInfo) TupleInfo ti(setTupleInfo(ct, sc->oid(), jobInfo, tblOid, sc, alias)); fColumnKeys.push_back(ti.key); - // @bug 2990, MySQL date/datetime type is different from IDB type - if (ti.dtype == CalpontSystemCatalog::DATE || ti.dtype == CalpontSystemCatalog::DATETIME) + // @bug 2990, MySQL time/date/datetime type is different from IDB type + if (ti.dtype == CalpontSystemCatalog::DATE || ti.dtype == CalpontSystemCatalog::DATETIME || + ti.dtype == CalpontSystemCatalog::TIME) { if (ti.dtype != ct.colDataType) { diff --git a/dbcon/joblist/groupconcat.cpp b/dbcon/joblist/groupconcat.cpp index fd122fe8d..1e84585b3 100644 --- a/dbcon/joblist/groupconcat.cpp +++ b/dbcon/joblist/groupconcat.cpp @@ -507,6 +507,12 @@ void GroupConcator::outputRow(std::ostringstream& oss, const rowgroup::Row& row) break; } + case CalpontSystemCatalog::TIME: + { + oss << DataConvert::timeToString(row.getUintField(*i)); + break; + } + default: { break; @@ -640,6 +646,24 @@ int64_t GroupConcator::lengthEstimate(const rowgroup::Row& row) case CalpontSystemCatalog::DATETIME: { fieldLen = 19; // YYYY-MM-DD HH24:MI:SS + // Decimal point and milliseconds + uint64_t colPrecision = row.getPrecision(*i); + if (colPrecision > 0 && colPrecision < 7) + { + fieldLen += colPrecision + 1; + } + break; + } + + case CalpontSystemCatalog::TIME: + { + fieldLen = 10; // -HHH:MI:SS + // Decimal point and milliseconds + uint64_t colPrecision = row.getPrecision(*i); + if (colPrecision > 0 && colPrecision < 7) + { + fieldLen += colPrecision + 1; + } break; } diff --git a/dbcon/joblist/jlf_common.cpp b/dbcon/joblist/jlf_common.cpp index 063e24dfc..f5dbeee17 100644 --- a/dbcon/joblist/jlf_common.cpp +++ b/dbcon/joblist/jlf_common.cpp @@ -792,6 +792,11 @@ bool compatibleColumnTypes(const CalpontSystemCatalog::ColDataType& dt1, uint32_ break; + case CalpontSystemCatalog::TIME: + if (dt2 != CalpontSystemCatalog::TIME) return false; + + break; + case CalpontSystemCatalog::CHAR: case CalpontSystemCatalog::VARCHAR: case CalpontSystemCatalog::TEXT: diff --git a/dbcon/joblist/jlf_execplantojoblist.cpp b/dbcon/joblist/jlf_execplantojoblist.cpp index 0ff2f7c93..b81a2ec8d 100644 --- a/dbcon/joblist/jlf_execplantojoblist.cpp +++ b/dbcon/joblist/jlf_execplantojoblist.cpp @@ -282,6 +282,10 @@ int64_t valueNullNum(const CalpontSystemCatalog::ColType& ct) n = boost::any_cast(anyVal); break; + case CalpontSystemCatalog::TIME: + n = boost::any_cast(anyVal); + break; + case CalpontSystemCatalog::DECIMAL: case CalpontSystemCatalog::UDECIMAL: if (ct.colWidth == execplan::CalpontSystemCatalog::ONE_BYTE) @@ -421,6 +425,10 @@ int64_t convertValueNum(const string& str, const CalpontSystemCatalog::ColType& v = boost::any_cast(anyVal); break; + case CalpontSystemCatalog::TIME: + v = boost::any_cast(anyVal); + break; + case CalpontSystemCatalog::DECIMAL: case CalpontSystemCatalog::UDECIMAL: if (ct.colWidth == execplan::CalpontSystemCatalog::ONE_BYTE) diff --git a/dbcon/joblist/jlf_subquery.cpp b/dbcon/joblist/jlf_subquery.cpp index e1537d577..1e4eaeeec 100644 --- a/dbcon/joblist/jlf_subquery.cpp +++ b/dbcon/joblist/jlf_subquery.cpp @@ -135,6 +135,11 @@ void getColumnValue(ConstantColumn** cc, uint64_t i, const Row& row) *cc = new ConstantColumn(oss.str()); break; + case CalpontSystemCatalog::TIME: + oss << dataconvert::DataConvert::timeToString(row.getUintField<8>(i)); + *cc = new ConstantColumn(oss.str()); + break; + case CalpontSystemCatalog::CHAR: case CalpontSystemCatalog::VARCHAR: case CalpontSystemCatalog::TEXT: diff --git a/dbcon/joblist/joblisttypes.h b/dbcon/joblist/joblisttypes.h index 3f3755a71..37d52f9f4 100644 --- a/dbcon/joblist/joblisttypes.h +++ b/dbcon/joblist/joblisttypes.h @@ -56,6 +56,8 @@ const uint32_t DATENULL = 0xFFFFFFFE; const uint32_t DATEEMPTYROW = 0xFFFFFFFF; const uint64_t DATETIMENULL = 0xFFFFFFFFFFFFFFFEULL; const uint64_t DATETIMEEMPTYROW = 0xFFFFFFFFFFFFFFFFULL; +const uint64_t TIMENULL = 0xFFFFFFFFFFFFFFFEULL; +const uint64_t TIMEEMPTYROW = 0xFFFFFFFFFFFFFFFFULL; const uint8_t CHAR1NULL = 0xFE; const uint8_t CHAR1EMPTYROW = 0xFF; diff --git a/dbcon/joblist/lbidlist.cpp b/dbcon/joblist/lbidlist.cpp index bd012941e..c317defc9 100644 --- a/dbcon/joblist/lbidlist.cpp +++ b/dbcon/joblist/lbidlist.cpp @@ -510,6 +510,7 @@ bool LBIDList::CasualPartitionDataType(const CalpontSystemCatalog::ColDataType t case CalpontSystemCatalog::DECIMAL: case CalpontSystemCatalog::DATE: case CalpontSystemCatalog::DATETIME: + case CalpontSystemCatalog::TIME: case CalpontSystemCatalog::UTINYINT: case CalpontSystemCatalog::USMALLINT: case CalpontSystemCatalog::UDECIMAL: diff --git a/dbcon/joblist/pcolscan.cpp b/dbcon/joblist/pcolscan.cpp index cd19ce34c..6cea4fc03 100644 --- a/dbcon/joblist/pcolscan.cpp +++ b/dbcon/joblist/pcolscan.cpp @@ -1167,6 +1167,7 @@ bool pColScanStep::isEmptyVal(const uint8_t* val8) const case CalpontSystemCatalog::VARCHAR: case CalpontSystemCatalog::DATE: case CalpontSystemCatalog::DATETIME: + case CalpontSystemCatalog::TIME: if (width == 1) { return (*val8 == joblist::CHAR1EMPTYROW); diff --git a/dbcon/joblist/subquerytransformer.cpp b/dbcon/joblist/subquerytransformer.cpp index a5e0759fa..c212c701d 100644 --- a/dbcon/joblist/subquerytransformer.cpp +++ b/dbcon/joblist/subquerytransformer.cpp @@ -237,9 +237,10 @@ SJSTEP& SubQueryTransformer::makeSubQueryStep(execplan::CalpontSelectExecutionPl fVtable.columnType(ct, i); } } - // MySQL date/datetime type is different from IDB type + // MySQL time/date/datetime type is different from IDB type else if (colDataTypeInRg == CalpontSystemCatalog::DATE || - colDataTypeInRg == CalpontSystemCatalog::DATETIME) + colDataTypeInRg == CalpontSystemCatalog::DATETIME || + colDataTypeInRg == CalpontSystemCatalog::TIME) { ct.colWidth = row.getColumnWidth(i); ct.colDataType = row.getColTypes()[i]; diff --git a/dbcon/joblist/tupleaggregatestep.cpp b/dbcon/joblist/tupleaggregatestep.cpp index ca44cebbc..9e23ac17b 100644 --- a/dbcon/joblist/tupleaggregatestep.cpp +++ b/dbcon/joblist/tupleaggregatestep.cpp @@ -232,6 +232,9 @@ inline string colTypeIdString(CalpontSystemCatalog::ColDataType type) case CalpontSystemCatalog::DATETIME: return string("DATETIME"); + case CalpontSystemCatalog::TIME: + return string("TIME"); + case CalpontSystemCatalog::VARCHAR: return string("VARCHAR"); @@ -1333,7 +1336,8 @@ void TupleAggregateStep::prep1PhaseAggregate( typeProj[colProj] == CalpontSystemCatalog::BLOB || typeProj[colProj] == CalpontSystemCatalog::TEXT || typeProj[colProj] == CalpontSystemCatalog::DATE || - typeProj[colProj] == CalpontSystemCatalog::DATETIME) + typeProj[colProj] == CalpontSystemCatalog::DATETIME || + typeProj[colProj] == CalpontSystemCatalog::TIME) { Message::Args args; args.add("sum/average"); @@ -1415,7 +1419,8 @@ void TupleAggregateStep::prep1PhaseAggregate( typeProj[colProj] == CalpontSystemCatalog::TEXT || typeProj[colProj] == CalpontSystemCatalog::BLOB || typeProj[colProj] == CalpontSystemCatalog::DATE || - typeProj[colProj] == CalpontSystemCatalog::DATETIME) + typeProj[colProj] == CalpontSystemCatalog::DATETIME || + typeProj[colProj] == CalpontSystemCatalog::TIME) { Message::Args args; args.add("variance/standard deviation"); @@ -1881,7 +1886,8 @@ void TupleAggregateStep::prep1PhaseDistinctAggregate( typeProj[colProj] == CalpontSystemCatalog::BLOB || typeProj[colProj] == CalpontSystemCatalog::TEXT || typeProj[colProj] == CalpontSystemCatalog::DATE || - typeProj[colProj] == CalpontSystemCatalog::DATETIME) + typeProj[colProj] == CalpontSystemCatalog::DATETIME || + typeProj[colProj] == CalpontSystemCatalog::TIME) { Message::Args args; args.add("sum/average"); @@ -1966,7 +1972,8 @@ void TupleAggregateStep::prep1PhaseDistinctAggregate( typeProj[colProj] == CalpontSystemCatalog::BLOB || typeProj[colProj] == CalpontSystemCatalog::TEXT || typeProj[colProj] == CalpontSystemCatalog::DATE || - typeProj[colProj] == CalpontSystemCatalog::DATETIME) + typeProj[colProj] == CalpontSystemCatalog::DATETIME || + typeProj[colProj] == CalpontSystemCatalog::TIME) { Message::Args args; args.add("variance/standard deviation"); @@ -2149,7 +2156,8 @@ void TupleAggregateStep::prep1PhaseDistinctAggregate( typeAgg[colAgg] == CalpontSystemCatalog::BLOB || typeAgg[colAgg] == CalpontSystemCatalog::TEXT || typeAgg[colAgg] == CalpontSystemCatalog::DATE || - typeAgg[colAgg] == CalpontSystemCatalog::DATETIME) + typeAgg[colAgg] == CalpontSystemCatalog::DATETIME || + typeAgg[colAgg] == CalpontSystemCatalog::TIME) { Message::Args args; args.add("sum/average"); @@ -3071,7 +3079,8 @@ void TupleAggregateStep::prep2PhasesAggregate( typeProj[colProj] == CalpontSystemCatalog::BLOB || typeProj[colProj] == CalpontSystemCatalog::TEXT || typeProj[colProj] == CalpontSystemCatalog::DATE || - typeProj[colProj] == CalpontSystemCatalog::DATETIME) + typeProj[colProj] == CalpontSystemCatalog::DATETIME || + typeProj[colProj] == CalpontSystemCatalog::TIME) { Message::Args args; args.add("sum/average"); @@ -3160,7 +3169,8 @@ void TupleAggregateStep::prep2PhasesAggregate( typeProj[colProj] == CalpontSystemCatalog::BLOB || typeProj[colProj] == CalpontSystemCatalog::TEXT || typeProj[colProj] == CalpontSystemCatalog::DATE || - typeProj[colProj] == CalpontSystemCatalog::DATETIME) + typeProj[colProj] == CalpontSystemCatalog::DATETIME || + typeProj[colProj] == CalpontSystemCatalog::TIME) { Message::Args args; args.add("variance/standard deviation"); @@ -3884,7 +3894,8 @@ void TupleAggregateStep::prep2PhasesDistinctAggregate( typeProj[colProj] == CalpontSystemCatalog::BLOB || typeProj[colProj] == CalpontSystemCatalog::TEXT || typeProj[colProj] == CalpontSystemCatalog::DATE || - typeProj[colProj] == CalpontSystemCatalog::DATETIME) + typeProj[colProj] == CalpontSystemCatalog::DATETIME || + typeProj[colProj] == CalpontSystemCatalog::TIME) { Message::Args args; args.add("sum/average"); @@ -3969,7 +3980,8 @@ void TupleAggregateStep::prep2PhasesDistinctAggregate( typeProj[colProj] == CalpontSystemCatalog::BLOB || typeProj[colProj] == CalpontSystemCatalog::TEXT || typeProj[colProj] == CalpontSystemCatalog::DATE || - typeProj[colProj] == CalpontSystemCatalog::DATETIME) + typeProj[colProj] == CalpontSystemCatalog::DATETIME || + typeProj[colProj] == CalpontSystemCatalog::TIME) { Message::Args args; args.add("variance/standard deviation"); @@ -4184,7 +4196,8 @@ void TupleAggregateStep::prep2PhasesDistinctAggregate( typeAggUm[colUm] == CalpontSystemCatalog::BLOB || typeAggUm[colUm] == CalpontSystemCatalog::TEXT || typeAggUm[colUm] == CalpontSystemCatalog::DATE || - typeAggUm[colUm] == CalpontSystemCatalog::DATETIME) + typeAggUm[colUm] == CalpontSystemCatalog::DATETIME || + typeAggUm[colUm] == CalpontSystemCatalog::TIME) { Message::Args args; args.add("sum/average"); diff --git a/dbcon/joblist/tupleconstantstep.cpp b/dbcon/joblist/tupleconstantstep.cpp index 6e75ad350..309bb0058 100644 --- a/dbcon/joblist/tupleconstantstep.cpp +++ b/dbcon/joblist/tupleconstantstep.cpp @@ -223,6 +223,7 @@ void TupleConstantStep::constructContanstRow(const JobInfo& jobInfo) case CalpontSystemCatalog::BIGINT: case CalpontSystemCatalog::DATE: case CalpontSystemCatalog::DATETIME: + case CalpontSystemCatalog::TIME: { fRowConst.setIntField(c.intVal, *i); break; diff --git a/dbcon/joblist/tupleunion.cpp b/dbcon/joblist/tupleunion.cpp index 51b5cde97..77665ae90 100644 --- a/dbcon/joblist/tupleunion.cpp +++ b/dbcon/joblist/tupleunion.cpp @@ -473,7 +473,8 @@ void TupleUnion::normalize(const Row& in, Row* out) case CalpontSystemCatalog::DATE: case CalpontSystemCatalog::DATETIME: - throw logic_error("TupleUnion::normalize(): tried to normalize an int to a date or datetime"); + case CalpontSystemCatalog::TIME: + throw logic_error("TupleUnion::normalize(): tried to normalize an int to a time, date or datetime"); case CalpontSystemCatalog::FLOAT: case CalpontSystemCatalog::UFLOAT: @@ -582,7 +583,8 @@ dec1: case CalpontSystemCatalog::DATE: case CalpontSystemCatalog::DATETIME: - throw logic_error("TupleUnion::normalize(): tried to normalize an int to a date or datetime"); + case CalpontSystemCatalog::TIME: + throw logic_error("TupleUnion::normalize(): tried to normalize an int to a time, date or datetime"); case CalpontSystemCatalog::FLOAT: case CalpontSystemCatalog::UFLOAT: @@ -736,6 +738,33 @@ dec2: break; + case CalpontSystemCatalog::TIME: + switch (out->getColTypes()[i]) + { + case CalpontSystemCatalog::TIME: + out->setIntField(in.getIntField(i), i); + break; + + case CalpontSystemCatalog::CHAR: + case CalpontSystemCatalog::TEXT: + case CalpontSystemCatalog::VARCHAR: + { + string d = DataConvert::timeToString(in.getIntField(i)); + out->setStringField(d, i); + break; + } + + default: + { + ostringstream os; + os << "TupleUnion::normalize(): tried an illegal conversion: time to " + << out->getColTypes()[i]; + throw logic_error(os.str()); + } + } + + break; + case CalpontSystemCatalog::FLOAT: case CalpontSystemCatalog::UFLOAT: case CalpontSystemCatalog::DOUBLE: @@ -1069,6 +1098,10 @@ void TupleUnion::writeNull(Row* out, uint32_t col) out->setUintField<8>(joblist::DATETIMENULL, col); break; + case CalpontSystemCatalog::TIME: + out->setUintField<8>(joblist::TIMENULL, col); + break; + case CalpontSystemCatalog::CHAR: case CalpontSystemCatalog::TEXT: case CalpontSystemCatalog::VARCHAR: diff --git a/dbcon/joblist/windowfunctionstep.cpp b/dbcon/joblist/windowfunctionstep.cpp index ed382ee4a..4d24f0b4b 100644 --- a/dbcon/joblist/windowfunctionstep.cpp +++ b/dbcon/joblist/windowfunctionstep.cpp @@ -1201,6 +1201,7 @@ boost::shared_ptr WindowFunctionStep::parseFrameBoundRows( case execplan::CalpontSystemCatalog::UDECIMAL: case execplan::CalpontSystemCatalog::DATE: case execplan::CalpontSystemCatalog::DATETIME: + case execplan::CalpontSystemCatalog::TIME: { fb.reset(new FrameBoundExpressionRow(type, id, idx)); break; @@ -1351,6 +1352,7 @@ boost::shared_ptr WindowFunctionStep::parseFrameBoundRange(const exe case execplan::CalpontSystemCatalog::UDECIMAL: case execplan::CalpontSystemCatalog::DATE: case execplan::CalpontSystemCatalog::DATETIME: + case execplan::CalpontSystemCatalog::TIME: { if (isConstant) { diff --git a/dbcon/mysql/ha_calpont_ddl.cpp b/dbcon/mysql/ha_calpont_ddl.cpp index 221155592..ba9b2c299 100644 --- a/dbcon/mysql/ha_calpont_ddl.cpp +++ b/dbcon/mysql/ha_calpont_ddl.cpp @@ -174,6 +174,10 @@ uint32_t convertDataType(int dataType) calpontDataType = CalpontSystemCatalog::DATETIME; break; + case ddlpackage::DDL_TIME: + calpontDataType = CalpontSystemCatalog::TIME; + break; + case ddlpackage::DDL_CLOB: calpontDataType = CalpontSystemCatalog::CLOB; break; diff --git a/dbcon/mysql/ha_calpont_dml.cpp b/dbcon/mysql/ha_calpont_dml.cpp index c4ca59862..57a91bd89 100644 --- a/dbcon/mysql/ha_calpont_dml.cpp +++ b/dbcon/mysql/ha_calpont_dml.cpp @@ -136,7 +136,8 @@ int buildBuffer(uchar* buf, string& buffer, int& columns, TABLE* table) (*field)->type() == MYSQL_TYPE_STRING || (*field)->type() == MYSQL_TYPE_DATE || (*field)->type() == MYSQL_TYPE_DATETIME || - (*field)->type() == MYSQL_TYPE_DATETIME2 ) + (*field)->type() == MYSQL_TYPE_DATETIME2 || + (*field)->type() == MYSQL_TYPE_TIME ) vals.append("'"); while (ptr < end_ptr) @@ -166,7 +167,8 @@ int buildBuffer(uchar* buf, string& buffer, int& columns, TABLE* table) (*field)->type() == MYSQL_TYPE_STRING || (*field)->type() == MYSQL_TYPE_DATE || (*field)->type() == MYSQL_TYPE_DATETIME || - (*field)->type() == MYSQL_TYPE_DATETIME2 ) + (*field)->type() == MYSQL_TYPE_DATETIME2 || + (*field)->type() == MYSQL_TYPE_TIME ) vals.append("'"); } } @@ -876,6 +878,37 @@ int ha_calpont_impl_write_batch_row_(uchar* buf, TABLE* table, cal_impl_if::cal_ break; } + case CalpontSystemCatalog::TIME: + { + if (nullVal && (ci.columnTypes[colpos].constraintType != CalpontSystemCatalog::NOTNULL_CONSTRAINT)) + { + fprintf(ci.filePtr, "%c", ci.delimiter); + + buf += table->field[colpos]->pack_length(); + } + else + { + MYSQL_TIME ltime; + const uchar* pos = buf; + longlong tmp = my_time_packed_from_binary(pos, table->field[colpos]->decimals()); + TIME_from_longlong_time_packed(<ime, tmp); + if (!ltime.second_part) + { + fprintf(ci.filePtr, "%02d:%02d:%02d%c", + ltime.hour, ltime.minute, ltime.second, ci.delimiter); + } + else + { + fprintf(ci.filePtr, "%02d:%02d:%02d.%ld%c", + ltime.hour, ltime.minute, ltime.second, + ltime.second_part, ci.delimiter); + } + buf += table->field[colpos]->pack_length(); + } + + break; + } + case CalpontSystemCatalog::CHAR: { if (nullVal && (ci.columnTypes[colpos].constraintType != CalpontSystemCatalog::NOTNULL_CONSTRAINT)) diff --git a/dbcon/mysql/ha_calpont_execplan.cpp b/dbcon/mysql/ha_calpont_execplan.cpp index 1079efe02..fe2f96e52 100644 --- a/dbcon/mysql/ha_calpont_execplan.cpp +++ b/dbcon/mysql/ha_calpont_execplan.cpp @@ -2657,6 +2657,11 @@ CalpontSystemCatalog::ColType colType_MysqlToIDB (const Item* item) ct.colDataType = CalpontSystemCatalog::DATETIME; ct.colWidth = 8; } + else if (item->field_type() == MYSQL_TYPE_TIME) + { + ct.colDataType = CalpontSystemCatalog::TIME; + ct.colWidth = 8; + } if (item->field_type() == MYSQL_TYPE_BLOB) { @@ -3528,6 +3533,13 @@ ReturnedColumn* buildFunctionColumn(Item_func* ifp, gp_walk_info& gwi, bool& non ct.colWidth = 4; fc->resultType(ct); } + else if (ifp->field_type() == MYSQL_TYPE_TIME) + { + CalpontSystemCatalog::ColType ct; + ct.colDataType = CalpontSystemCatalog::TIME; + ct.colWidth = 8; + fc->resultType(ct); + } #if 0 diff --git a/dbcon/mysql/ha_calpont_impl.cpp b/dbcon/mysql/ha_calpont_impl.cpp index 6bd887332..e59f7c7d5 100644 --- a/dbcon/mysql/ha_calpont_impl.cpp +++ b/dbcon/mysql/ha_calpont_impl.cpp @@ -494,6 +494,19 @@ int fetchNextRow(uchar* buf, cal_table_info& ti, cal_connection_info* ci) break; } + case CalpontSystemCatalog::TIME: + { + if ((*f)->null_ptr) + *(*f)->null_ptr &= ~(*f)->null_bit; + + intColVal = row.getUintField<8>(s); + DataConvert::timeToString(intColVal, tmp, 255); + + Field_varstring* f2 = (Field_varstring*)*f; + f2->store(tmp, strlen(tmp), f2->charset()); + break; + } + case CalpontSystemCatalog::CHAR: case CalpontSystemCatalog::VARCHAR: { diff --git a/dbcon/mysql/ha_calpont_partition.cpp b/dbcon/mysql/ha_calpont_partition.cpp index 84ea65d82..99940262b 100644 --- a/dbcon/mysql/ha_calpont_partition.cpp +++ b/dbcon/mysql/ha_calpont_partition.cpp @@ -125,6 +125,9 @@ string name(CalpontSystemCatalog::ColType& ct) case CalpontSystemCatalog::DATETIME: return "DATETIME"; + case CalpontSystemCatalog::TIME: + return "TIME"; + case CalpontSystemCatalog::DECIMAL: return "DECIMAL"; @@ -201,6 +204,7 @@ bool CP_type(CalpontSystemCatalog::ColType& ct) ct.colDataType == CalpontSystemCatalog::BIGINT || ct.colDataType == CalpontSystemCatalog::DATE || ct.colDataType == CalpontSystemCatalog::DATETIME || + ct.colDataType == CalpontSystemCatalog::TIME || ct.colDataType == CalpontSystemCatalog::DECIMAL || ct.colDataType == CalpontSystemCatalog::UTINYINT || ct.colDataType == CalpontSystemCatalog::USMALLINT || @@ -261,6 +265,9 @@ const string format(int64_t v, CalpontSystemCatalog::ColType& ct) oss << DataConvert::datetimeToString(v); break; + case CalpontSystemCatalog::TIME: + oss << DataConvert::timeToString(v); + case CalpontSystemCatalog::CHAR: case CalpontSystemCatalog::VARCHAR: { @@ -387,6 +394,10 @@ const int64_t IDB_format(char* str, CalpontSystemCatalog::ColType& ct, uint8_t& v = boost::any_cast(anyVal); break; + case CalpontSystemCatalog::TIME: + v = boost::any_cast(anyVal); + break; + case CalpontSystemCatalog::DECIMAL: case CalpontSystemCatalog::UDECIMAL: if (ct.colWidth == execplan::CalpontSystemCatalog::ONE_BYTE) diff --git a/dbcon/mysql/ha_window_function.cpp b/dbcon/mysql/ha_window_function.cpp index 7e30389c4..5724b231c 100644 --- a/dbcon/mysql/ha_window_function.cpp +++ b/dbcon/mysql/ha_window_function.cpp @@ -590,6 +590,7 @@ ReturnedColumn* buildWindowFunctionColumn(Item* item, gp_walk_info& gwi, bool& n case CalpontSystemCatalog::DATE: case CalpontSystemCatalog::DATETIME: + case CalpontSystemCatalog::TIME: if (!frm.fIsRange) boundTypeErr = true; else if (dynamic_cast(frm.fStart.fVal.get()) == NULL) @@ -641,6 +642,7 @@ ReturnedColumn* buildWindowFunctionColumn(Item* item, gp_walk_info& gwi, bool& n case CalpontSystemCatalog::DATE: case CalpontSystemCatalog::DATETIME: + case CalpontSystemCatalog::TIME: if (!frm.fIsRange) boundTypeErr = true; else if (dynamic_cast(frm.fEnd.fVal.get()) == NULL) diff --git a/primitives/linux-port/column.cpp b/primitives/linux-port/column.cpp index 879859384..f31273b1c 100644 --- a/primitives/linux-port/column.cpp +++ b/primitives/linux-port/column.cpp @@ -290,6 +290,7 @@ inline bool isEmptyVal<8>(uint8_t type, const uint8_t* ival) case CalpontSystemCatalog::VARCHAR: case CalpontSystemCatalog::DATE: case CalpontSystemCatalog::DATETIME: + case CalpontSystemCatalog::TIME: case CalpontSystemCatalog::VARBINARY: case CalpontSystemCatalog::BLOB: case CalpontSystemCatalog::TEXT: @@ -322,6 +323,7 @@ inline bool isEmptyVal<4>(uint8_t type, const uint8_t* ival) case CalpontSystemCatalog::TEXT: case CalpontSystemCatalog::DATE: case CalpontSystemCatalog::DATETIME: + case CalpontSystemCatalog::TIME: return (joblist::CHAR4EMPTYROW == *val); case CalpontSystemCatalog::UINT: @@ -347,6 +349,7 @@ inline bool isEmptyVal<2>(uint8_t type, const uint8_t* ival) case CalpontSystemCatalog::TEXT: case CalpontSystemCatalog::DATE: case CalpontSystemCatalog::DATETIME: + case CalpontSystemCatalog::TIME: return (joblist::CHAR2EMPTYROW == *val); case CalpontSystemCatalog::USMALLINT: @@ -372,6 +375,7 @@ inline bool isEmptyVal<1>(uint8_t type, const uint8_t* ival) case CalpontSystemCatalog::TEXT: case CalpontSystemCatalog::DATE: case CalpontSystemCatalog::DATETIME: + case CalpontSystemCatalog::TIME: return (*val == joblist::CHAR1EMPTYROW); case CalpontSystemCatalog::UTINYINT: @@ -402,6 +406,7 @@ inline bool isNullVal<8>(uint8_t type, const uint8_t* ival) case CalpontSystemCatalog::VARCHAR: case CalpontSystemCatalog::DATE: case CalpontSystemCatalog::DATETIME: + case CalpontSystemCatalog::TIME: case CalpontSystemCatalog::VARBINARY: case CalpontSystemCatalog::BLOB: case CalpontSystemCatalog::TEXT: @@ -438,6 +443,7 @@ inline bool isNullVal<4>(uint8_t type, const uint8_t* ival) case CalpontSystemCatalog::DATE: case CalpontSystemCatalog::DATETIME: + case CalpontSystemCatalog::TIME: return (joblist::DATENULL == *val); case CalpontSystemCatalog::UINT: @@ -463,6 +469,7 @@ inline bool isNullVal<2>(uint8_t type, const uint8_t* ival) case CalpontSystemCatalog::TEXT: case CalpontSystemCatalog::DATE: case CalpontSystemCatalog::DATETIME: + case CalpontSystemCatalog::TIME: return (joblist::CHAR2NULL == *val); case CalpontSystemCatalog::USMALLINT: @@ -488,6 +495,7 @@ inline bool isNullVal<1>(uint8_t type, const uint8_t* ival) case CalpontSystemCatalog::TEXT: case CalpontSystemCatalog::DATE: case CalpontSystemCatalog::DATETIME: + case CalpontSystemCatalog::TIME: return (*val == joblist::CHAR1NULL); case CalpontSystemCatalog::UTINYINT: @@ -548,6 +556,7 @@ inline bool isMinMaxValid(const NewColRequestHeader* in) case CalpontSystemCatalog::DATE: case CalpontSystemCatalog::BIGINT: case CalpontSystemCatalog::DATETIME: + case CalpontSystemCatalog::TIME: case CalpontSystemCatalog::UTINYINT: case CalpontSystemCatalog::USMALLINT: case CalpontSystemCatalog::UINT: diff --git a/primitives/primproc/columncommand.cpp b/primitives/primproc/columncommand.cpp index 19e5e9a1f..ed4b26a44 100644 --- a/primitives/primproc/columncommand.cpp +++ b/primitives/primproc/columncommand.cpp @@ -980,6 +980,7 @@ const uint64_t ColumnCommand::getEmptyRowValue( const execplan::CalpontSystemCat case execplan::CalpontSystemCatalog::VARCHAR : case execplan::CalpontSystemCatalog::DATE : case execplan::CalpontSystemCatalog::DATETIME : + case execplan::CalpontSystemCatalog::TIME : case execplan::CalpontSystemCatalog::VARBINARY : case execplan::CalpontSystemCatalog::BLOB : case execplan::CalpontSystemCatalog::TEXT : diff --git a/tools/pingproc/pingproc.cpp b/tools/pingproc/pingproc.cpp index cb1a7ab11..b40eec905 100644 --- a/tools/pingproc/pingproc.cpp +++ b/tools/pingproc/pingproc.cpp @@ -308,6 +308,7 @@ bool OidOperation::isIntegralDataType() DataType() == CalpontSystemCatalog::DATE || DataType() == CalpontSystemCatalog::BIGINT || DataType() == CalpontSystemCatalog::DATETIME || + DataType() == CalpontSystemCatalog::TIME || DataType() == CalpontSystemCatalog::UTINYINT || DataType() == CalpontSystemCatalog::USMALLINT || DataType() == CalpontSystemCatalog::UMEDINT || diff --git a/utils/common/nullvaluemanip.cpp b/utils/common/nullvaluemanip.cpp index 4eb96f77c..475f495cc 100644 --- a/utils/common/nullvaluemanip.cpp +++ b/utils/common/nullvaluemanip.cpp @@ -56,6 +56,9 @@ uint64_t getNullValue(CalpontSystemCatalog::ColDataType t, uint32_t colWidth) case CalpontSystemCatalog::DATETIME: return joblist::DATETIMENULL; + case CalpontSystemCatalog::TIME: + return joblist::TIMENULL; + case CalpontSystemCatalog::CHAR: case CalpontSystemCatalog::VARCHAR: case CalpontSystemCatalog::STRINT: @@ -163,6 +166,9 @@ int64_t getSignedNullValue(CalpontSystemCatalog::ColDataType t, uint32_t colWidt case CalpontSystemCatalog::DATETIME: return joblist::DATETIMENULL; + case CalpontSystemCatalog::TIME: + return joblist::TIMENULL; + case CalpontSystemCatalog::CHAR: case CalpontSystemCatalog::VARCHAR: case CalpontSystemCatalog::STRINT: diff --git a/utils/dataconvert/dataconvert.cpp b/utils/dataconvert/dataconvert.cpp index 6e8a33dc1..b5b1ae619 100644 --- a/utils/dataconvert/dataconvert.cpp +++ b/utils/dataconvert/dataconvert.cpp @@ -859,6 +859,153 @@ bool mysql_str_to_datetime( const string& input, DateTime& output, bool& isDate return true; } +bool mysql_str_to_time( const string& input, Time& output ) +{ + int32_t datesepct = 0; + uint32_t dtend = 0; + + /** + * We need to deal with the time portion. + * The rules are: + * - Time portion may be empty + * - Time portion may start with 'T' + * - Time portion always ends with '\0' + * - Time portion always starts with hour + * - Without time separators (':'): + * HHMMSS + * - All Times can end with option .[microseconds] + * - With time separators there are no specific field length + * requirements + */ + while ( input[dtend] == ' ' && dtend < input.length() ) + { + ++dtend; + } + + if ( dtend == input.length() ) + { + return false; + } + + uint32_t timesep_ct = 0; + bool has_usec = false; + uint32_t len_before_msec = 0; + uint32_t tmstart = ( input[dtend] == ' ' || input[dtend] == 'T' ) ? dtend + 1 : dtend; + uint32_t tmend = tmstart; + + for ( ; tmend < input.length(); ++tmend ) + { + char c = input[tmend]; + + if ( isdigit( c ) ) + { + // digits always ok + continue; + } +// else if( c == ':' ) +// { +// timesep_ct++; +// } +// else if( c == '.' ) +// { +// len_before_msec = ( tmend - tmstart ); +// has_usec = true; +// } + else if ( ispunct(c) ) + { + if ( c == '.' && timesep_ct == 2 ) + { + len_before_msec = ( tmend - tmstart ); + has_usec = true; + } + else + { + timesep_ct++; + } + } + else + { + // some other character showed up + output.reset(); + return false; + } + } + + if ( !len_before_msec ) + len_before_msec = ( tmend - tmstart ); + + int32_t hour = -1; + int32_t min = 0; + int32_t sec = 0; + int32_t usec = 0; + const char* tstart = input.c_str() + tmstart; + + if ( timesep_ct == 2 ) + { + readDecimal(tstart, hour); + ++tstart; // skip one separator + readDecimal(tstart, min); + ++tstart; // skip one separator + readDecimal(tstart, sec); + } + else if ( timesep_ct == 1 ) + { + readDecimal(tstart, hour); + ++tstart; // skip one separator + readDecimal(tstart, min); + } + else if ( timesep_ct == 0 && len_before_msec == 6 ) + { + readDecimal(tstart, hour, 2); + readDecimal(tstart, min, 2); + readDecimal(tstart, sec, 2); + } + else if ( timesep_ct == 0 && len_before_msec == 4 ) + { + readDecimal(tstart, hour, 2); + readDecimal(tstart, min, 2); + } + else if ( timesep_ct == 0 && len_before_msec == 2 ) + { + readDecimal(tstart, hour, 2); + } + else + { + output.reset(); + return false; + } + + if ( has_usec ) + { + ++tstart; // skip '.' character. We could error check if we wanted to + uint32_t numread = readDecimal(tstart, usec); + + if ( numread > 6 || numread < 1 ) + { + // don't allow more than 6 digits when specifying microseconds + output.reset(); + return false; + } + + // usec have to be scaled up so that it always represents microseconds + for ( int i = numread; i < 6; i++ ) + usec *= 10; + } + + if ( !isTimeValid( hour, min, sec, usec ) ) + { + output.reset(); + return false; + } + + output.hour = hour; + output.minute = min; + output.second = sec; + output.msecond = usec; + return true; +} + + bool stringToDateStruct( const string& data, Date& date ) { bool isDate; @@ -894,6 +1041,14 @@ bool stringToDatetimeStruct(const string& data, DateTime& dtime, bool* date) return true; } +bool stringToTimeStruct(const string& data, Time& dtime) +{ + if ( !mysql_str_to_time( data, dtime ) ) + return false; + + return true; +} + boost::any DataConvert::convertColumnData(const CalpontSystemCatalog::ColType& colType, const std::string& dataOrig, bool& pushWarning, bool nulFlag, bool noRoundup, bool isUpdate ) @@ -1229,6 +1384,23 @@ DataConvert::convertColumnData(const CalpontSystemCatalog::ColType& colType, } break; + case CalpontSystemCatalog::TIME: + { + Time aTime; + + if (stringToTimeStruct(data, aTime)) + { + value = (int64_t) *(reinterpret_cast(&aTime)); + } + else + { + value = (int64_t) 0; + pushWarning = true; + } + } + break; + + case CalpontSystemCatalog::BLOB: case CalpontSystemCatalog::CLOB: value = data; @@ -1345,6 +1517,13 @@ DataConvert::convertColumnData(const CalpontSystemCatalog::ColType& colType, } break; + case CalpontSystemCatalog::TIME: + { + uint64_t d = joblist::TIMENULL; + value = d; + } + break; + case CalpontSystemCatalog::CHAR: { std::string charnull; @@ -1692,6 +1871,91 @@ int64_t DataConvert::convertColumnDatetime( return value; } +//------------------------------------------------------------------------------ +// Convert time string to binary time. Used by BulkLoad. +// Most of this is taken from str_to_time in sql-common/my_time.c +//------------------------------------------------------------------------------ +int64_t DataConvert::convertColumnTime( + const char* dataOrg, + CalpontDateTimeFormat datetimeFormat, + int& status, + unsigned int dataOrgLen ) +{ + status = 0; + const char* p; + p = dataOrg; + char fld[10]; + int16_t value = 0; + int inYear, inMonth, inDay, inHour, inMinute, inSecond, inMicrosecond; + inHour = 0; + inMinute = 0; + inSecond = 0; + inMicrosecond = 0; + if ( datetimeFormat != CALPONTTIME_ENUM ) + { + status = -1; + return value; + } + + memcpy( fld, p, 2); + fld[2] = '\0'; + + inHour = strtol(fld, 0, 10); + + if (!isdigit(p[2]) || !isdigit(p[3])) + { + status = -1; + return value; + } + + memcpy( fld, p + 2, 2); + fld[2] = '\0'; + + inMinute = strtol(fld, 0, 10); + + if (!isdigit(p[4]) || !isdigit(p[5])) + { + status = -1; + return value; + } + + memcpy( fld, p + 4, 2); + fld[2] = '\0'; + + inSecond = strtol(fld, 0, 10); + + if (dataOrgLen > 9) + { + unsigned int microFldLen = dataOrgLen - 9; + + if (microFldLen > (sizeof(fld) - 1)) + microFldLen = sizeof(fld) - 1; + + memcpy( fld, p + 9, microFldLen); + fld[microFldLen] = '\0'; + inMicrosecond = strtol(fld, 0, 10); + } + + if ( isTimeValid (inHour, inMinute, inSecond, inMicrosecond) ) + { + Time atime; + atime.hour = inHour; + atime.minute = inMinute; + atime.second = inSecond; + atime.msecond = inMicrosecond; + + memcpy( &value, &atime, 8); + } + else + { + status = -1; + } + + return value; + +} + + //------------------------------------------------------------------------------ // Verify that specified datetime is valid //------------------------------------------------------------------------------ @@ -1706,6 +1970,14 @@ bool DataConvert::isColumnDateTimeValid( int64_t dateTime ) return false; } +bool DataConvert::isColumnTimeValid( int64_t time ) +{ + Time dt; + memcpy(&dt, &time, sizeof(uint64_t)); + + return isTimeValid(dt.hour, dt.minute, dt.second, dt.msecond); +} + std::string DataConvert::dateToString( int datevalue ) { // @bug 4703 abandon multiple ostringstream's for conversion @@ -1742,6 +2014,32 @@ std::string DataConvert::datetimeToString( long long datetimevalue, long decima return buf; } +std::string DataConvert::timeToString( long long timevalue, long decimals ) +{ + // 10 is default which means we don't need microseconds + if (decimals > 6 || decimals < 0) + { + decimals = 0; + } + // @bug 4703 abandon multiple ostringstream's for conversion + Time dt(timevalue); + const int TIMETOSTRING_LEN = 19; // (-H)HH:MM:SS.mmmmmm\0 + char buf[TIMETOSTRING_LEN]; + + sprintf(buf, "%02d:%02d:%02d", dt.hour, dt.minute, dt.second); + if (dt.msecond && decimals) + { + size_t start = strlen(buf); + snprintf(buf + strlen(buf), 12 + decimals, ".%d", dt.msecond); + // Pad end with zeros + if (strlen(buf) - start < decimals) + { + sprintf(buf + strlen(buf), "%0*d", decimals - (strlen(buf) - start), 0); + } + } + return buf; +} + std::string DataConvert::dateToString1( int datevalue ) { // @bug 4703 abandon multiple ostringstream's for conversion @@ -1763,6 +2061,18 @@ std::string DataConvert::datetimeToString1( long long datetimevalue ) sprintf(buf, "%04d%02d%02d%02d%02d%02d%06d", dt.year, dt.month, dt.day, dt.hour, dt.minute, dt.second, dt.msecond); return buf; } + +std::string DataConvert::timeToString1( long long datetimevalue ) +{ + // @bug 4703 abandon multiple ostringstream's for conversion + DateTime dt(datetimevalue); + const int TIMETOSTRING1_LEN = 14; // HHMMSSmmmmmm\0 + char buf[TIMETOSTRING1_LEN]; + + sprintf(buf, "%02d%02d%02d%06d", dt.hour, dt.minute, dt.second, dt.msecond); + return buf; +} + #if 0 bool DataConvert::isNullData(ColumnResult* cr, int rownum, CalpontSystemCatalog::ColType colType) { @@ -1983,6 +2293,11 @@ int64_t DataConvert::datetimeToInt(const string& datetime) return stringToDatetime(datetime); } +int64_t DataConvert::timeToInt(const string& time) +{ + return stringToTime(time); +} + int64_t DataConvert::stringToDate(const string& data) { Date aDay; @@ -2256,6 +2571,66 @@ int64_t DataConvert::intToDatetime(int64_t data, bool* date) return *(reinterpret_cast(&adaytime)); } +int64_t DataConvert::intToTime(int64_t data) +{ + char buf[21] = {0}; + Time atime; + + if (data == 0) + { + atime.hour = 0; + atime.minute = 0; + atime.second = 0; + atime.msecond = 0; + + return *(reinterpret_cast(&atime)); + } + + snprintf( buf, 15, "%llu", (long long unsigned int)data); + //string date = buf; + string hour, min, sec, msec; + int64_t h = 0, minute = 0, s = 0, ms = 0; + + switch (strlen(buf)) + { + case 6: + hour = string(buf, 2); + min = string(buf + 2, 2); + sec = string(buf + 4, 2); + msec = string(buf + 6, 6); + break; + + case 4: + min = string(buf, 2); + sec = string(buf + 2, 2); + msec = string(buf + 4, 6); + break; + + case 2: + sec = string(buf, 2); + msec = string(buf + 2, 6); + break; + + default: + return -1; + } + + h = atoi(hour.c_str()); + minute = atoi(min.c_str()); + s = atoi(sec.c_str()); + ms = atoi(msec.c_str()); + + if (!isTimeValid(h, minute, s, ms)) + return -1; + + atime.hour = h; + atime.minute = minute; + atime.second = s; + atime.msecond = ms; + + return *(reinterpret_cast(&atime)); +} + int64_t DataConvert::stringToTime(const string& data) { // MySQL supported time value format 'D HHH:MM:SS.fraction' @@ -2402,6 +2777,7 @@ CalpontSystemCatalog::ColType DataConvert::convertUnionColType(vector> 40) & 0xfff), day((val >> 52) & 0xfff) {} + + Time(signed d, signed h, signed min, signed sec, signed msec) : + msecond(sec), second(sec), minute(min), hour(h), day(d) {} + + int64_t convertToMySQLint() const; + void reset(); }; +inline +void Time::reset() +{ + msecond = 0xFFFFFE; + second = 0xFF; + minute = 0xFF; + hour = 0xFFF; + day = 0xFFF; +} + +inline +int64_t Time::convertToMySQLint() const +{ + return (int64_t) (hour * 10000000000) + (minute * 100000000) + (second * 1000000) + msecond; +} + static uint32_t daysInMonth[13] = {31, 28, 31, 30, 31, 30, 31, 31, 30, 31, 30, 31, 0}; inline uint32_t getDaysInMonth(uint32_t month) @@ -281,6 +304,28 @@ bool isDateTimeValid ( int hour, int minute, int second, int microSecond) return valid; } +inline +bool isTimeValid ( int hour, int minute, int second, int microSecond) +{ + bool valid = false; + + if ( hour >= -838 && hour <= 838 ) + { + if ( minute >= 0 && minute < 60 ) + { + if ( second >= 0 && second < 60 ) + { + if ( microSecond >= 0 && microSecond <= 999999 ) + { + valid = true; + } + } + } + } + + return valid; +} + inline int64_t string_to_ll( const std::string& data, bool& bSaturate ) { @@ -364,6 +409,15 @@ public: EXPORT static std::string datetimeToString( long long datetimevalue, long decimals = 0 ); static inline void datetimeToString( long long datetimevalue, char* buf, unsigned int buflen ); + /** + * @brief convert a columns data from native format to a string + * + * @param type the columns database type + * @param data the columns string representation of it's data + */ + EXPORT static std::string timeToString( long long timevalue, long decimals = 0 ); + static inline void timeToString( long long timevalue, char* buf, unsigned int buflen ); + /** * @brief convert a columns data from native format to a string * @@ -382,6 +436,15 @@ public: EXPORT static std::string datetimeToString1( long long datetimevalue ); static inline void datetimeToString1( long long datetimevalue, char* buf, unsigned int buflen ); + /** + * @brief convert a columns data from native format to a string + * + * @param type the columns database type + * @param data the columns string representation of it's data + */ + EXPORT static std::string timeToString1( long long timevalue ); + static inline void timeToString1( long long timevalue, char* buf, unsigned int buflen ); + /** * @brief convert a date column data, represnted as a string, to it's native * format. This function is for bulkload to use. @@ -415,10 +478,25 @@ public: CalpontDateTimeFormat datetimeFormat, int& status, unsigned int dataOrgLen ); + /** + * @brief convert a time column data, represented as a string, + * to it's native format. This function is for bulkload to use. + * + * @param type the columns data type + * @param dataOrig the columns string representation of it's data + * @param timeFormat the format the time value in + * @param status 0 - success, -1 - fail + * @param dataOrgLen length specification of dataOrg + */ + EXPORT static int64_t convertColumnTime( const char* dataOrg, + CalpontDateTimeFormat datetimeFormat, + int& status, unsigned int dataOrgLen ); + /** * @brief Is specified datetime valid; used by binary bulk load */ EXPORT static bool isColumnDateTimeValid( int64_t dateTime ); + EXPORT static bool isColumnTimeValid( int64_t time ); EXPORT static bool isNullData(execplan::ColumnResult* cr, int rownum, execplan::CalpontSystemCatalog::ColType colType); static inline std::string decimalToString(int64_t value, uint8_t scale, execplan::CalpontSystemCatalog::ColDataType colDataType); @@ -438,11 +516,13 @@ public: EXPORT static int64_t intToDate(int64_t data); // convert integer to datetime EXPORT static int64_t intToDatetime(int64_t data, bool* isDate = NULL); - + // convert integer to date + EXPORT static int64_t intToTime(int64_t data); // convert string to date. alias to stringToDate EXPORT static int64_t dateToInt(const std::string& date); // convert string to datetime. alias to datetimeToInt EXPORT static int64_t datetimeToInt(const std::string& datetime); + EXPORT static int64_t timeToInt(const std::string& time); EXPORT static int64_t stringToTime (const std::string& data); // bug4388, union type conversion EXPORT static execplan::CalpontSystemCatalog::ColType convertUnionColType(std::vector&); @@ -484,6 +564,27 @@ inline void DataConvert::datetimeToString( long long datetimevalue, char* buf, u } } +inline void DataConvert::timeToString( long long timevalue, char* buf, unsigned int buflen ) +{ + if ((timevalue & 0xffffff) > 0) + { + snprintf( buf, buflen, "%02d:%02d:%02d.%d", + (unsigned)((timevalue >> 40) & 0xfff), + (unsigned)((timevalue >> 32) & 0xff), + (unsigned)((timevalue >> 24) & 0xff), + (unsigned)((timevalue) & 0xffffff) + ); + } + else + { + snprintf( buf, buflen, "%02d:%02d:%02d", + (unsigned)((timevalue >> 40) & 0xfff), + (unsigned)((timevalue >> 32) & 0xff), + (unsigned)((timevalue >> 24) & 0xff) + ); + } +} + inline void DataConvert::dateToString1( int datevalue, char* buf, unsigned int buflen) { snprintf( buf, buflen, "%04d%02d%02d", @@ -505,6 +606,15 @@ inline void DataConvert::datetimeToString1( long long datetimevalue, char* buf, ); } +inline void DataConvert::timeToString1( long long timevalue, char* buf, unsigned int buflen ) +{ + snprintf( buf, buflen, "%02d%02d%02d", + (unsigned)((timevalue >> 40) & 0xfff), + (unsigned)((timevalue >> 32) & 0xff), + (unsigned)((timevalue >> 14) & 0xff) + ); +} + inline std::string DataConvert::decimalToString(int64_t value, uint8_t scale, execplan::CalpontSystemCatalog::ColDataType colDataType) { char buf[80]; diff --git a/utils/funcexp/func_add_time.cpp b/utils/funcexp/func_add_time.cpp index fb4469eec..6fa7a5c19 100644 --- a/utils/funcexp/func_add_time.cpp +++ b/utils/funcexp/func_add_time.cpp @@ -138,6 +138,50 @@ int64_t addTime(DateTime& dt1, Time& dt2) return *(reinterpret_cast(&dt)); } + +int64_t addTime(Time& dt1, Time& dt2) +{ + Time dt; + dt.hour = 0; + dt.minute = 0; + dt.second = 0; + dt.msecond = 0; + + int64_t hour, min, sec, msec, tmp; + msec = (signed)(dt1.msecond + dt2.msecond); + dt.msecond = tmp = msec % 1000000; + + if (tmp < 0) + { + dt.msecond = tmp + 1000000; + dt2.second--; + } + + sec = (signed)(dt1.second + dt2.second + msec / 1000000); + dt.second = tmp = sec % 60; + + if (tmp < 0) + { + dt.second = tmp + 60; + dt2.minute--; + } + + min = (signed)(dt1.minute + dt2.minute + sec / 60); + dt.minute = tmp = min % 60; + + if (tmp < 0) + { + dt.minute = tmp + 60; + dt2.hour--; + } + + hour = (signed)(dt1.hour + dt2.hour + min / 60); + dt.hour = tmp = hour % 838; + + return *(reinterpret_cast(&dt)); +} + + } namespace funcexp @@ -254,6 +298,86 @@ int64_t Func_add_time::getDatetimeIntVal(rowgroup::Row& row, return addTime(dt1, t2); } +int64_t Func_add_time::getTimeIntVal(rowgroup::Row& row, + FunctionParm& parm, + bool& isNull, + CalpontSystemCatalog::ColType& ct) +{ + int64_t val1 = parm[0]->data()->getTimeIntVal(row, isNull); + + if (isNull) + return -1; + + const string& val2 = parm[1]->data()->getStrVal(row, isNull); + int sign = parm[2]->data()->getIntVal(row, isNull); + Time dt1; + dt1.hour = (val1 >> 40) & 0xff; + dt1.minute = (val1 >> 32) & 0xff; + dt1.second = (val1 >> 24) & 0xff; + dt1.msecond = val1 & 0xffffff; + + int64_t time = DataConvert::stringToTime(val2); + + if (time == -1) + { + isNull = true; + return -1; + } + + Time t2 = *(reinterpret_cast(&time)); + + // MySQL TIME type range '-838:59:59' and '838:59:59' + if (t2.minute > 59 || t2.second > 59 || t2.msecond > 999999) + { + isNull = true; + return -1; + } + + int val_sign = 1; + + if (t2.day != 0 && t2.hour < 0) + { + isNull = true; + return -1; + } + else if (t2.day < 0 || t2.hour < 0) + { + val_sign = -1; + } + + if ((abs(t2.day) * 24 + abs(t2.hour)) > 838) + { + t2.hour = 838; + t2.minute = 59; + t2.second = 59; + t2.msecond = 999999; + } + else + { + t2.hour = abs(t2.day) * 24 + t2.hour; + } + + t2.day = 0; + + if (val_sign * sign < 0) + { + t2.hour = -abs(t2.hour); + t2.minute = -abs(t2.minute); + t2.second = -abs(t2.second); + t2.msecond = -abs(t2.msecond); + } + else + { + t2.hour = abs(t2.hour); + t2.minute = abs(t2.minute); + t2.second = abs(t2.second); + t2.msecond = abs(t2.msecond); + } + + return addTime(dt1, t2); +} + + } // namespace funcexp // vim:ts=4 sw=4: diff --git a/utils/funcexp/func_between.cpp b/utils/funcexp/func_between.cpp index af950cc95..8398593c4 100644 --- a/utils/funcexp/func_between.cpp +++ b/utils/funcexp/func_between.cpp @@ -156,6 +156,24 @@ inline bool getBool(rowgroup::Row& row, numericLE(val, pm[2]->data()->getDatetimeIntVal(row, isNull)); } + case execplan::CalpontSystemCatalog::TIME: + { + int64_t val = pm[0]->data()->getTimeIntVal(row, isNull); + + if (notBetween) + { + if (!numericGE(val, pm[1]->data()->getTimeIntVal(row, isNull)) && !isNull) + return true; + + isNull = false; + return (!numericLE(val, pm[2]->data()->getTimeIntVal(row, isNull)) && !isNull); + } + + return !isNull && + numericGE(val, pm[1]->data()->getTimeIntVal(row, isNull)) && + numericLE(val, pm[2]->data()->getTimeIntVal(row, isNull)); + } + case execplan::CalpontSystemCatalog::DOUBLE: case execplan::CalpontSystemCatalog::UDOUBLE: case execplan::CalpontSystemCatalog::FLOAT: diff --git a/utils/funcexp/func_bitand.cpp b/utils/funcexp/func_bitand.cpp index 89aadf39b..eceb4e4c0 100644 --- a/utils/funcexp/func_bitand.cpp +++ b/utils/funcexp/func_bitand.cpp @@ -141,7 +141,8 @@ int64_t Func_bitand::getIntVal(Row& row, day = 0, hour = 0, min = 0, - sec = 0; + sec = 0, + msec = 0; year = (uint32_t)((time >> 48) & 0xffff); month = (uint32_t)((time >> 44) & 0xf); @@ -149,9 +150,29 @@ int64_t Func_bitand::getIntVal(Row& row, hour = (uint32_t)((time >> 32) & 0x3f); min = (uint32_t)((time >> 26) & 0x3f); sec = (uint32_t)((time >> 20) & 0x3f); + msec = (uint32_t)(time & 0xfffff); // return (int64_t) (year*1000000000000)+(month*100000000)+(day*1000000)+(hour*10000)+(min*100)+sec; - values.push_back((month * 100000000) + (day * 1000000) + (hour * 10000) + (min * 100) + sec); + values.push_back((month * 100000000000000) + (day * 1000000000000) + (hour * 10000000000) + (min * 100000000) + (sec * 1000000) + msec); + } + break; + + case execplan::CalpontSystemCatalog::TIME: + { + int64_t time = parm[i]->data()->getTimeIntVal(row, isNull); + + int32_t hour = 0, + min = 0, + sec = 0, + msec = 0; + + hour = (uint32_t)((time >> 40) & 0xfff); + min = (uint32_t)((time >> 32) & 0xff); + sec = (uint32_t)((time >> 24) & 0xff); + msec = (uint32_t)(time & 0xffffff); + + // return (int64_t) (year*1000000000000)+(month*100000000)+(day*1000000)+(hour*10000)+(min*100)+sec; + values.push_back((hour * 10000000000) + (min * 100000000) + (sec * 1000000) + msec); } break; diff --git a/utils/funcexp/func_bitwise.cpp b/utils/funcexp/func_bitwise.cpp index 84c83f231..752c28561 100644 --- a/utils/funcexp/func_bitwise.cpp +++ b/utils/funcexp/func_bitwise.cpp @@ -143,6 +143,15 @@ bool getUIntValFromParm( } break; + case execplan::CalpontSystemCatalog::TIME: + { + int64_t time = parm->data()->getTimeIntVal(row, isNull); + + Time dt(time); + value = dt.convertToMySQLint(); + } + break; + default: { return false; diff --git a/utils/funcexp/func_case.cpp b/utils/funcexp/func_case.cpp index 4eec004ca..46a479491 100644 --- a/utils/funcexp/func_case.cpp +++ b/utils/funcexp/func_case.cpp @@ -66,6 +66,7 @@ inline uint64_t simple_case_cmp(Row& row, case execplan::CalpontSystemCatalog::BIGINT: case execplan::CalpontSystemCatalog::DATE: case execplan::CalpontSystemCatalog::DATETIME: + case execplan::CalpontSystemCatalog::TIME: { int64_t ev = parm[n]->data()->getIntVal(row, isNull); @@ -503,6 +504,19 @@ int64_t Func_simple_case::getDatetimeIntVal(rowgroup::Row& row, } +int64_t Func_simple_case::getTimeIntVal(rowgroup::Row& row, + FunctionParm& parm, + bool& isNull, + execplan::CalpontSystemCatalog::ColType& op_ct) +{ + uint64_t i = simple_case_cmp(row, parm, isNull, op_ct); + + if (isNull) + return joblist::TIMENULL; + + return parm[i + 1]->data()->getTimeIntVal(row, isNull); +} + // searched CASE: // SELECT CASE @@ -629,5 +643,18 @@ int64_t Func_searched_case::getDatetimeIntVal(rowgroup::Row& row, } +int64_t Func_searched_case::getTimeIntVal(rowgroup::Row& row, + FunctionParm& parm, + bool& isNull, + execplan::CalpontSystemCatalog::ColType& op_ct) +{ + uint64_t i = simple_case_cmp(row, parm, isNull, op_ct); + + if (isNull) + return joblist::TIMENULL; + + return parm[i + 1]->data()->getTimeIntVal(row, isNull); +} + } // namespace funcexp // vim:ts=4 sw=4: diff --git a/utils/funcexp/func_cast.cpp b/utils/funcexp/func_cast.cpp index 9d2423293..3542e341d 100644 --- a/utils/funcexp/func_cast.cpp +++ b/utils/funcexp/func_cast.cpp @@ -191,6 +191,15 @@ int64_t Func_cast_signed::getIntVal(Row& row, } break; + case execplan::CalpontSystemCatalog::TIME: + { + int64_t time = parm[0]->data()->getTimeIntVal(row, isNull); + + Time dt(time); + return dt.convertToMySQLint(); + } + break; + default: { std::ostringstream oss; @@ -313,6 +322,15 @@ uint64_t Func_cast_unsigned::getUintVal(Row& row, } break; + case execplan::CalpontSystemCatalog::TIME: + { + int64_t time = parm[0]->data()->getTimeIntVal(row, isNull); + + Time dt(time); + return dt.convertToMySQLint(); + } + break; + default: { std::ostringstream oss; @@ -805,6 +823,85 @@ int64_t Func_cast_datetime::getDatetimeIntVal(rowgroup::Row& row, return -1; } + +int64_t Func_cast_datetime::getTimeIntVal(rowgroup::Row& row, + FunctionParm& parm, + bool& isNull, + execplan::CalpontSystemCatalog::ColType& operationColType) +{ + int64_t val; + + switch (parm[0]->data()->resultType().colDataType) + { + case execplan::CalpontSystemCatalog::BIGINT: + case execplan::CalpontSystemCatalog::INT: + case execplan::CalpontSystemCatalog::MEDINT: + case execplan::CalpontSystemCatalog::TINYINT: + case execplan::CalpontSystemCatalog::SMALLINT: + case execplan::CalpontSystemCatalog::UBIGINT: + case execplan::CalpontSystemCatalog::UINT: + case execplan::CalpontSystemCatalog::UMEDINT: + case execplan::CalpontSystemCatalog::UTINYINT: + case execplan::CalpontSystemCatalog::USMALLINT: + { + val = dataconvert::DataConvert::intToTime(parm[0]->data()->getIntVal(row, isNull)); + + if (val == -1) + isNull = true; + else + return val; + + break; + } + + case execplan::CalpontSystemCatalog::DECIMAL: + case execplan::CalpontSystemCatalog::UDECIMAL: + { + val = dataconvert::DataConvert::intToTime(parm[0]->data()->getIntVal(row, isNull)); + + if (val == -1) + isNull = true; + else + return val; + + break; + } + + case execplan::CalpontSystemCatalog::VARCHAR: + case execplan::CalpontSystemCatalog::CHAR: + case execplan::CalpontSystemCatalog::TEXT: + { + val = dataconvert::DataConvert::stringToTime(parm[0]->data()->getStrVal(row, isNull)); + + if (val == -1) + isNull = true; + else + return val; + + break; + } + + case execplan::CalpontSystemCatalog::DATE: + { + return parm[0]->data()->getTimeIntVal(row, isNull); + } + + case execplan::CalpontSystemCatalog::DATETIME: + { + return parm[0]->data()->getTimeIntVal(row, isNull); + } + + default: + { + isNull = true; + } + } + + return -1; +} + + + // // Func_cast_decimal // @@ -1138,6 +1235,25 @@ IDB_Decimal Func_cast_decimal::getDecimalVal(Row& row, } break; + case execplan::CalpontSystemCatalog::TIME: + { + int32_t s = 0; + + string value = dataconvert::DataConvert::timeToString1(parm[0]->data()->getTimeIntVal(row, isNull)); + + //strip off micro seconds + string date = value.substr(0, 14); + + int64_t x = atoll(date.c_str()); + + if (!isNull) + { + decimal.value = x; + decimal.scale = s; + } + } + break; + default: { std::ostringstream oss; diff --git a/utils/funcexp/func_ceil.cpp b/utils/funcexp/func_ceil.cpp index 7c1cafdfb..7c2f7f571 100644 --- a/utils/funcexp/func_ceil.cpp +++ b/utils/funcexp/func_ceil.cpp @@ -152,6 +152,15 @@ int64_t Func_ceil::getIntVal(Row& row, } break; + case CalpontSystemCatalog::TIME: + { + Time dt(parm[0]->data()->getTimeIntVal(row, isNull)); + + if (!isNull) + ret = dt.convertToMySQLint(); + } + break; + default: { std::ostringstream oss; @@ -233,6 +242,15 @@ uint64_t Func_ceil::getUintVal(Row& row, } break; + case CalpontSystemCatalog::TIME: + { + Time dt(parm[0]->data()->getTimeIntVal(row, isNull)); + + if (!isNull) + ret = dt.convertToMySQLint(); + } + break; + default: { std::ostringstream oss; diff --git a/utils/funcexp/func_char_length.cpp b/utils/funcexp/func_char_length.cpp index 9421554f8..b47afd58a 100644 --- a/utils/funcexp/func_char_length.cpp +++ b/utils/funcexp/func_char_length.cpp @@ -101,6 +101,12 @@ int64_t Func_char_length::getIntVal(rowgroup::Row& row, return (int64_t)date.size(); } + case execplan::CalpontSystemCatalog::TIME: + { + string date = dataconvert::DataConvert::timeToString(parm[0]->data()->getTimeIntVal(row, isNull)); + return (int64_t)date.size(); + } + default: { std::ostringstream oss; diff --git a/utils/funcexp/func_coalesce.cpp b/utils/funcexp/func_coalesce.cpp index 33a1312bc..30091a81e 100644 --- a/utils/funcexp/func_coalesce.cpp +++ b/utils/funcexp/func_coalesce.cpp @@ -139,6 +139,30 @@ int64_t Func_coalesce::getDatetimeIntVal(rowgroup::Row& row, return val; } +int64_t Func_coalesce::getTimeIntVal(rowgroup::Row& row, + FunctionParm& parm, + bool& isNull, + CalpontSystemCatalog::ColType& ct) +{ + int64_t val = 0; + + for (uint32_t i = 0; i < parm.size(); i++) + { + val = parm[i]->data()->getTimeIntVal(row, isNull); + + if (isNull) + { + isNull = false; + continue; + } + + return val; + } + + isNull = true; + return val; +} + double Func_coalesce::getDoubleVal(rowgroup::Row& row, FunctionParm& parm, bool& isNull, diff --git a/utils/funcexp/func_extract.cpp b/utils/funcexp/func_extract.cpp index 08a545fde..d5a6cc5b5 100644 --- a/utils/funcexp/func_extract.cpp +++ b/utils/funcexp/func_extract.cpp @@ -154,6 +154,7 @@ int64_t Func_extract::getIntVal(rowgroup::Row& row, { case CalpontSystemCatalog::DATE: case CalpontSystemCatalog::DATETIME: + case CalpontSystemCatalog::TIME: time = parm[0]->data()->getDatetimeIntVal(row, isNull); break; diff --git a/utils/funcexp/func_floor.cpp b/utils/funcexp/func_floor.cpp index ad47dc988..eb05b405f 100644 --- a/utils/funcexp/func_floor.cpp +++ b/utils/funcexp/func_floor.cpp @@ -151,6 +151,19 @@ int64_t Func_floor::getIntVal(Row& row, } break; + case execplan::CalpontSystemCatalog::TIME: + { + string str = + DataConvert::timeToString1(parm[0]->data()->getTimeIntVal(row, isNull)); + + // strip off micro seconds + str = str.substr(0, 14); + + if (!isNull) + ret = atoll(str.c_str()); + } + break; + default: { std::ostringstream oss; @@ -236,6 +249,19 @@ uint64_t Func_floor::getUintVal(Row& row, } break; + case execplan::CalpontSystemCatalog::TIME: + { + string str = + DataConvert::timeToString1(parm[0]->data()->getTimeIntVal(row, isNull)); + + // strip off micro seconds + str = str.substr(0, 14); + + if (!isNull) + ret = strtoull(str.c_str(), NULL, 10); + } + break; + default: { std::ostringstream oss; diff --git a/utils/funcexp/func_from_unixtime.cpp b/utils/funcexp/func_from_unixtime.cpp index 183a75a56..9968ba1f0 100644 --- a/utils/funcexp/func_from_unixtime.cpp +++ b/utils/funcexp/func_from_unixtime.cpp @@ -148,6 +148,22 @@ int64_t Func_from_unixtime::getDatetimeIntVal(rowgroup::Row& row, return *reinterpret_cast(&dt); } +int64_t Func_from_unixtime::getTimeIntVal(rowgroup::Row& row, + FunctionParm& parm, + bool& isNull, + CalpontSystemCatalog::ColType& ct) +{ + DateTime dt = getDateTime(row, parm, isNull); + + if (*reinterpret_cast(&dt) == 0) + { + isNull = true; + return 0; + } + + return *reinterpret_cast(&dt); +} + int64_t Func_from_unixtime::getIntVal(rowgroup::Row& row, FunctionParm& parm, bool& isNull, diff --git a/utils/funcexp/func_greatest.cpp b/utils/funcexp/func_greatest.cpp index 8309456ae..50ff4204a 100644 --- a/utils/funcexp/func_greatest.cpp +++ b/utils/funcexp/func_greatest.cpp @@ -206,6 +206,26 @@ int64_t Func_greatest::getDatetimeIntVal(rowgroup::Row& row, return greatestStr; } +int64_t Func_greatest::getTimeIntVal(rowgroup::Row& row, + FunctionParm& fp, + bool& isNull, + execplan::CalpontSystemCatalog::ColType& ct) +{ + int64_t str = fp[0]->data()->getTimeIntVal(row, isNull); + + int64_t greatestStr = str; + + for (uint32_t i = 1; i < fp.size(); i++) + { + int64_t str1 = fp[i]->data()->getTimeIntVal(row, isNull); + + if ( greatestStr < str1 ) + greatestStr = str1; + } + + return greatestStr; +} + } // namespace funcexp // vim:ts=4 sw=4: diff --git a/utils/funcexp/func_hour.cpp b/utils/funcexp/func_hour.cpp index f5058a084..b9e3da9ec 100644 --- a/utils/funcexp/func_hour.cpp +++ b/utils/funcexp/func_hour.cpp @@ -108,6 +108,12 @@ int64_t Func_hour::getIntVal(rowgroup::Row& row, break; } + case execplan::CalpontSystemCatalog::TIME: + { + val = parm[0]->data()->getDatetimeIntVal(row, isNull); + break; + } + default: { isNull = true; diff --git a/utils/funcexp/func_if.cpp b/utils/funcexp/func_if.cpp index 378c6d74a..4cc8cd164 100644 --- a/utils/funcexp/func_if.cpp +++ b/utils/funcexp/func_if.cpp @@ -75,6 +75,7 @@ bool boolVal(SPTP& parm, Row& row) case CalpontSystemCatalog::UINT: case CalpontSystemCatalog::DATE: case CalpontSystemCatalog::DATETIME: + case CalpontSystemCatalog::TIME: default: ret = (parm->data()->getIntVal(row, isNull) != 0); } @@ -219,6 +220,19 @@ int64_t Func_if::getDatetimeIntVal(Row& row, } } - +int64_t Func_if::getTimeIntVal(Row& row, + FunctionParm& parm, + bool& isNull, + CalpontSystemCatalog::ColType&) +{ + if (boolVal(parm[0], row)) + { + return parm[1]->data()->getTimeIntVal(row, isNull); + } + else + { + return parm[2]->data()->getTimeIntVal(row, isNull); + } +} } // namespace funcexp // vim:ts=4 sw=4: diff --git a/utils/funcexp/func_ifnull.cpp b/utils/funcexp/func_ifnull.cpp index 41db7e3b4..18887f00e 100644 --- a/utils/funcexp/func_ifnull.cpp +++ b/utils/funcexp/func_ifnull.cpp @@ -168,6 +168,25 @@ int64_t Func_ifnull::getDatetimeIntVal(Row& row, return r; } +int64_t Func_ifnull::getTimeIntVal(Row& row, + FunctionParm& parm, + bool& isNull, + CalpontSystemCatalog::ColType&) +{ + if (isNull) + return 0; + + int64_t r = parm[0]->data()->getTimeIntVal(row, isNull); + + if (isNull) + { + isNull = false; + return parm[1]->data()->getTimeIntVal(row, isNull); + } + + return r; +} + bool Func_ifnull::getBoolVal(Row& row, FunctionParm& parm, bool& isNull, diff --git a/utils/funcexp/func_in.cpp b/utils/funcexp/func_in.cpp index 404c0f39f..e271fa0f7 100644 --- a/utils/funcexp/func_in.cpp +++ b/utils/funcexp/func_in.cpp @@ -158,6 +158,27 @@ inline bool getBoolForIn(rowgroup::Row& row, return false; } + case execplan::CalpontSystemCatalog::TIME: + { + int64_t val = pm[0]->data()->getTimeIntVal(row, isNull); + + if (isNull) + return false; + + for (uint32_t i = 1; i < pm.size(); i++) + { + isNull = false; + + if ( val == pm[i]->data()->getTimeIntVal(row, isNull) && !isNull ) + return true; + + if (isNull && isNotIn) + return true; // will be reversed to false by the caller + } + + return false; + } + case execplan::CalpontSystemCatalog::DOUBLE: case execplan::CalpontSystemCatalog::UDOUBLE: case execplan::CalpontSystemCatalog::FLOAT: diff --git a/utils/funcexp/func_inet_aton.cpp b/utils/funcexp/func_inet_aton.cpp index bd2d2a2d7..d6e3465cf 100644 --- a/utils/funcexp/func_inet_aton.cpp +++ b/utils/funcexp/func_inet_aton.cpp @@ -222,6 +222,26 @@ int64_t Func_inet_aton::getDatetimeIntVal(rowgroup::Row& row, return iValue; } +int64_t Func_inet_aton::getTimeIntVal(rowgroup::Row& row, + FunctionParm& fp, + bool& isNull, + execplan::CalpontSystemCatalog::ColType& op_ct) +{ + int64_t iValue = joblist::TIMENULL; + + const std::string& sValue = fp[0]->data()->getStrVal(row, isNull); + + if (!isNull) + { + int64_t iVal = convertAton( sValue, isNull ); + + if (!isNull) + iValue = iVal; + } + + return iValue; +} + //------------------------------------------------------------------------------ // Convert an ascii IP address string to it's integer equivalent. // isNull is set to true if the IP address string has invalid content. diff --git a/utils/funcexp/func_inet_ntoa.cpp b/utils/funcexp/func_inet_ntoa.cpp index 84c476eb6..e25f56097 100644 --- a/utils/funcexp/func_inet_ntoa.cpp +++ b/utils/funcexp/func_inet_ntoa.cpp @@ -256,6 +256,20 @@ int64_t Func_inet_ntoa::getDatetimeIntVal(rowgroup::Row& row, return iValue; } +int64_t Func_inet_ntoa::getTimeIntVal(rowgroup::Row& row, + FunctionParm& fp, + bool& isNull, + execplan::CalpontSystemCatalog::ColType& op_ct) +{ +// std::cout << "In Func_inet_ntoa::getTimeVal" << std::endl; + +// int64t iValue = fp[0]->data()->getTimeIntVal(row, isNull); + int64_t iValue = joblist::TIMENULL; + isNull = true; + + return iValue; +} + //------------------------------------------------------------------------------ // Convert an integer IP address to its equivalent IP address string. // Source code based on MySQL source (Item_func_inet_ntoa() in item_strfunc.cc). diff --git a/utils/funcexp/func_least.cpp b/utils/funcexp/func_least.cpp index e273a4618..ef71e44e6 100644 --- a/utils/funcexp/func_least.cpp +++ b/utils/funcexp/func_least.cpp @@ -182,6 +182,25 @@ int64_t Func_least::getDatetimeIntVal(rowgroup::Row& row, return leastStr; } +int64_t Func_least::getTimeIntVal(rowgroup::Row& row, + FunctionParm& fp, + bool& isNull, + execplan::CalpontSystemCatalog::ColType& op_ct) +{ + int64_t str = fp[0]->data()->getTimeIntVal(row, isNull); + + int64_t leastStr = str; + + for (uint32_t i = 1; i < fp.size(); i++) + { + int64_t str1 = fp[i]->data()->getTimeIntVal(row, isNull); + + if ( leastStr > str1 ) + leastStr = str1; + } + + return leastStr; +} } // namespace funcexp diff --git a/utils/funcexp/func_math.cpp b/utils/funcexp/func_math.cpp index 8e5b84def..8592ec8b0 100644 --- a/utils/funcexp/func_math.cpp +++ b/utils/funcexp/func_math.cpp @@ -154,6 +154,20 @@ double Func_acos::getDoubleVal(Row& row, } break; + case execplan::CalpontSystemCatalog::TIME: + { + int64_t value = parm[0]->data()->getTimeIntVal(row, isNull); + + if (isNull || (value < -1.0 || value > 1.0)) + { + isNull = true; + return doubleNullVal(); + } + + return acos((double)value); + } + break; + default: { std::ostringstream oss; @@ -244,6 +258,20 @@ double Func_asin::getDoubleVal(Row& row, } break; + case execplan::CalpontSystemCatalog::TIME: + { + int64_t value = parm[0]->data()->getTimeIntVal(row, isNull); + + if (isNull || (value < -1.0 || value > 1.0)) + { + isNull = true; + return doubleNullVal(); + } + + return asin((double)value); + } + break; + default: { std::ostringstream oss; @@ -373,6 +401,34 @@ double Func_atan::getDoubleVal(Row& row, } break; + + case execplan::CalpontSystemCatalog::TIME: + { + int64_t value = parm[0]->data()->getTimeIntVal(row, isNull); + + if (isNull) + { + isNull = true; + return doubleNullVal(); + } + + if (parm.size() > 1 ) + { + double value2 = parm[1]->data()->getDoubleVal(row, isNull); + + if (isNull) + { + isNull = true; + return doubleNullVal(); + } + + return atan2(value, value2); + } + + return atan((double)value); + } + break; + default: { std::ostringstream oss; @@ -462,6 +518,20 @@ double Func_cos::getDoubleVal(Row& row, } break; + case execplan::CalpontSystemCatalog::TIME: + { + int64_t value = parm[0]->data()->getTimeIntVal(row, isNull); + + if (isNull) + { + isNull = true; + return doubleNullVal(); + } + + return cos((double)value); + } + break; + default: { std::ostringstream oss; @@ -578,6 +648,30 @@ double Func_cot::getDoubleVal(Row& row, } break; + case execplan::CalpontSystemCatalog::TIME: + { + int64_t value = parm[0]->data()->getTimeIntVal(row, isNull); + + if (value == 0) + { + Message::Args args; + args.add("cot"); + args.add((uint64_t)value); + unsigned errcode = ERR_FUNC_OUT_OF_RANGE_RESULT; + throw IDBExcept(IDBErrorInfo::instance()->errorMsg(errcode, args), errcode); + } + + if (isNull) + { + isNull = true; + return doubleNullVal(); + } + + return 1.0 / tan((double)value); + } + break; + + default: { std::ostringstream oss; @@ -703,6 +797,33 @@ double Func_log::getDoubleVal(Row& row, } break; + case execplan::CalpontSystemCatalog::TIME: + { + int64_t value = parm[0]->data()->getTimeIntVal(row, isNull); + + if (isNull || value <= 0.0) + { + isNull = true; + return doubleNullVal(); + } + + if (parm.size() > 1 ) + { + double value2 = parm[1]->data()->getDoubleVal(row, isNull); + + if (isNull || (value2 <= 0.0 || value == 1.0) ) + { + isNull = true; + return doubleNullVal(); + } + + return log(value2) / log((double)value); + } + + return log((double)value); + } + break; + case execplan::CalpontSystemCatalog::VARCHAR: case execplan::CalpontSystemCatalog::CHAR: case execplan::CalpontSystemCatalog::TEXT: @@ -797,6 +918,20 @@ double Func_log2::getDoubleVal(Row& row, } break; + case execplan::CalpontSystemCatalog::TIME: + { + int64_t value = parm[0]->data()->getTimeIntVal(row, isNull); + + if (isNull || value <= 0.0) + { + isNull = true; + return doubleNullVal(); + } + + return log2(value); + } + break; + case execplan::CalpontSystemCatalog::VARCHAR: case execplan::CalpontSystemCatalog::CHAR: case execplan::CalpontSystemCatalog::TEXT: @@ -891,6 +1026,20 @@ double Func_log10::getDoubleVal(Row& row, } break; + case execplan::CalpontSystemCatalog::TIME: + { + int64_t value = parm[0]->data()->getTimeIntVal(row, isNull); + + if (isNull || value <= 0.0) + { + isNull = true; + return doubleNullVal(); + } + + return log10((double)value); + } + break; + case execplan::CalpontSystemCatalog::VARCHAR: case execplan::CalpontSystemCatalog::CHAR: case execplan::CalpontSystemCatalog::TEXT: @@ -988,6 +1137,20 @@ double Func_sin::getDoubleVal(Row& row, } break; + case execplan::CalpontSystemCatalog::TIME: + { + int64_t value = parm[0]->data()->getTimeIntVal(row, isNull); + + if (isNull) + { + isNull = true; + return doubleNullVal(); + } + + return sin((double)value); + } + break; + default: { std::ostringstream oss; @@ -1077,6 +1240,20 @@ double Func_sqrt::getDoubleVal(Row& row, } break; + case execplan::CalpontSystemCatalog::TIME: + { + int64_t value = parm[0]->data()->getTimeIntVal(row, isNull); + + if (isNull || value < 0) + { + isNull = true; + return doubleNullVal(); + } + + return sqrt((double)value); + } + break; + default: { std::ostringstream oss; @@ -1166,6 +1343,20 @@ double Func_tan::getDoubleVal(Row& row, } break; + case execplan::CalpontSystemCatalog::TIME: + { + int64_t value = parm[0]->data()->getTimeIntVal(row, isNull); + + if (isNull) + { + isNull = true; + return doubleNullVal(); + } + + return tan((double)value); + } + break; + default: { std::ostringstream oss; @@ -1253,6 +1444,12 @@ string Func_format::getStrVal(Row& row, } break; + case execplan::CalpontSystemCatalog::TIME: + { + value = dataconvert::DataConvert::timeToString1(parm[0]->data()->getTimeIntVal(row, isNull)); + } + break; + case execplan::CalpontSystemCatalog::DECIMAL: case execplan::CalpontSystemCatalog::UDECIMAL: { @@ -1474,6 +1671,20 @@ double Func_radians::getDoubleVal(Row& row, } break; + case execplan::CalpontSystemCatalog::TIME: + { + int64_t value = parm[0]->data()->getTimeIntVal(row, isNull); + + if (isNull) + { + isNull = true; + return doubleNullVal(); + } + + return radians((double)value); + } + break; + default: { std::ostringstream oss; @@ -1563,6 +1774,20 @@ double Func_degrees::getDoubleVal(Row& row, } break; + case execplan::CalpontSystemCatalog::TIME: + { + int64_t value = parm[0]->data()->getTimeIntVal(row, isNull); + + if (isNull) + { + isNull = true; + return doubleNullVal(); + } + + return degrees((double)value); + } + break; + default: { std::ostringstream oss; diff --git a/utils/funcexp/func_minute.cpp b/utils/funcexp/func_minute.cpp index f70af29e3..299dd6ae2 100644 --- a/utils/funcexp/func_minute.cpp +++ b/utils/funcexp/func_minute.cpp @@ -102,6 +102,7 @@ int64_t Func_minute::getIntVal(rowgroup::Row& row, break; } + case execplan::CalpontSystemCatalog::TIME: case execplan::CalpontSystemCatalog::DATETIME: { val = parm[0]->data()->getDatetimeIntVal(row, isNull); diff --git a/utils/funcexp/func_nullif.cpp b/utils/funcexp/func_nullif.cpp index 7d1647d1d..23362f4ef 100644 --- a/utils/funcexp/func_nullif.cpp +++ b/utils/funcexp/func_nullif.cpp @@ -155,6 +155,38 @@ int64_t Func_nullif::getIntVal(rowgroup::Row& row, break; } + case execplan::CalpontSystemCatalog::TIME: + { + exp2 = parm[1]->data()->getTimeIntVal(row, isNull); + + if (parm[0]->data()->resultType().colDataType == + execplan::CalpontSystemCatalog::DATETIME) + { + // NULLIF arg0 is DATETIME, arg1 is TIME, + // Upgrade arg1 to time + // When comparing exp1 as a Date, we can't simply promote. We have + // to be careful of the return value in case of not null return. + int64_t exp1 = parm[0]->data()->getTimeIntVal(row, isNull); + + if ( exp1 == exp2 ) + { + isNull = true; + return 0; + } + + // since exp1 here is inside the block, when we leave the block, the + // original (Date) value is restored. + } + + if (isNull) + { + isNull = false; + return exp1; + } + + break; + } + default: { isNull = true; @@ -244,6 +276,19 @@ uint64_t Func_nullif::getUintVal(rowgroup::Row& row, break; } + case execplan::CalpontSystemCatalog::TIME: + { + exp2 = parm[1]->data()->getTimeIntVal(row, isNull); + + if (isNull) + { + isNull = false; + return exp1; + } + + break; + } + default: { isNull = true; @@ -427,6 +472,7 @@ int64_t Func_nullif::getDatetimeIntVal(rowgroup::Row& row, } break; + case execplan::CalpontSystemCatalog::TIME: case execplan::CalpontSystemCatalog::DATETIME: { exp2 = parm[1]->data()->getDatetimeIntVal(row, isNull); @@ -455,6 +501,69 @@ int64_t Func_nullif::getDatetimeIntVal(rowgroup::Row& row, return exp1; } +int64_t Func_nullif::getTimeIntVal(rowgroup::Row& row, + FunctionParm& parm, + bool& isNull, + CalpontSystemCatalog::ColType& ct) +{ + int64_t exp1 = parm[0]->data()->getTimeIntVal(row, isNull); + int64_t exp2 = 0; + + switch (parm[1]->data()->resultType().colDataType) + { + case execplan::CalpontSystemCatalog::BIGINT: + case execplan::CalpontSystemCatalog::INT: + case execplan::CalpontSystemCatalog::MEDINT: + case execplan::CalpontSystemCatalog::TINYINT: + case execplan::CalpontSystemCatalog::SMALLINT: + case execplan::CalpontSystemCatalog::DOUBLE: + case execplan::CalpontSystemCatalog::FLOAT: + case execplan::CalpontSystemCatalog::DECIMAL: + case execplan::CalpontSystemCatalog::VARCHAR: + case execplan::CalpontSystemCatalog::CHAR: + case execplan::CalpontSystemCatalog::TEXT: + { + exp2 = parm[1]->data()->getIntVal(row, isNull); + + if (isNull) + { + isNull = false; + return exp1; + } + + break; + } + + case execplan::CalpontSystemCatalog::TIME: + case execplan::CalpontSystemCatalog::DATETIME: + { + exp2 = parm[1]->data()->getTimeIntVal(row, isNull); + + if (isNull) + { + isNull = false; + return exp1; + } + + break; + } + + default: + { + isNull = true; + } + } + + if ( exp1 == exp2 ) + { + isNull = true; + return 0; + } + + return exp1; +} + + double Func_nullif::getDoubleVal(rowgroup::Row& row, FunctionParm& parm, bool& isNull, @@ -502,6 +611,7 @@ double Func_nullif::getDoubleVal(rowgroup::Row& row, } break; + case execplan::CalpontSystemCatalog::TIME: case execplan::CalpontSystemCatalog::DATETIME: { exp2 = parm[1]->data()->getDatetimeIntVal(row, isNull); @@ -644,6 +754,7 @@ execplan::IDB_Decimal Func_nullif::getDecimalVal(rowgroup::Row& row, } break; + case execplan::CalpontSystemCatalog::TIME: case execplan::CalpontSystemCatalog::DATETIME: { int32_t s = 0; diff --git a/utils/funcexp/func_regexp.cpp b/utils/funcexp/func_regexp.cpp index 11f702a60..420c792cf 100644 --- a/utils/funcexp/func_regexp.cpp +++ b/utils/funcexp/func_regexp.cpp @@ -94,6 +94,14 @@ inline bool getBool(rowgroup::Row& row, break; } + case execplan::CalpontSystemCatalog::TIME: + { + expr = dataconvert::DataConvert::timeToString(pm[0]->data()->getTimeIntVal(row, isNull)); + //strip off micro seconds + expr = expr.substr(0, 19); + break; + } + case execplan::CalpontSystemCatalog::DECIMAL: case execplan::CalpontSystemCatalog::UDECIMAL: { @@ -153,6 +161,14 @@ inline bool getBool(rowgroup::Row& row, break; } + case execplan::CalpontSystemCatalog::TIME: + { + pattern = dataconvert::DataConvert::timeToString(pm[1]->data()->getTimeIntVal(row, isNull)); + //strip off micro seconds + pattern = pattern.substr(0, 19); + break; + } + case execplan::CalpontSystemCatalog::DECIMAL: case execplan::CalpontSystemCatalog::UDECIMAL: { diff --git a/utils/funcexp/func_round.cpp b/utils/funcexp/func_round.cpp index 18b888b2c..c107fc0fc 100644 --- a/utils/funcexp/func_round.cpp +++ b/utils/funcexp/func_round.cpp @@ -390,6 +390,7 @@ IDB_Decimal Func_round::getDecimalVal(Row& row, } break; + case execplan::CalpontSystemCatalog::TIME: case execplan::CalpontSystemCatalog::DATETIME: { int32_t s = 0; diff --git a/utils/funcexp/func_second.cpp b/utils/funcexp/func_second.cpp index 509633c46..678a1fa61 100644 --- a/utils/funcexp/func_second.cpp +++ b/utils/funcexp/func_second.cpp @@ -108,6 +108,12 @@ int64_t Func_second::getIntVal(rowgroup::Row& row, break; } + case execplan::CalpontSystemCatalog::TIME: + { + val = parm[0]->data()->getTimeIntVal(row, isNull); + break; + } + default: { isNull = true; diff --git a/utils/funcexp/func_sysdate.cpp b/utils/funcexp/func_sysdate.cpp index f8a5a98bd..d8e2580bb 100644 --- a/utils/funcexp/func_sysdate.cpp +++ b/utils/funcexp/func_sysdate.cpp @@ -100,6 +100,15 @@ int64_t Func_sysdate::getDatetimeIntVal(rowgroup::Row& row, return getIntVal(row, parm, isNull, operationColType); } +int64_t Func_sysdate::getTimeIntVal(rowgroup::Row& row, + FunctionParm& parm, + bool& isNull, + CalpontSystemCatalog::ColType& operationColType) +{ + return getIntVal(row, parm, isNull, operationColType); +} + + } // namespace funcexp // vim:ts=4 sw=4: diff --git a/utils/funcexp/func_time.cpp b/utils/funcexp/func_time.cpp index 509caa777..8aab1389b 100644 --- a/utils/funcexp/func_time.cpp +++ b/utils/funcexp/func_time.cpp @@ -112,6 +112,7 @@ string Func_time::getStrVal(rowgroup::Row& row, break; } + case execplan::CalpontSystemCatalog::TIME: case execplan::CalpontSystemCatalog::DATETIME: { val = parm[0]->data()->getDatetimeIntVal(row, isNull); diff --git a/utils/funcexp/func_timediff.cpp b/utils/funcexp/func_timediff.cpp index c87e99b64..3c9ce96a8 100644 --- a/utils/funcexp/func_timediff.cpp +++ b/utils/funcexp/func_timediff.cpp @@ -116,6 +116,7 @@ string Func_timediff::getStrVal(rowgroup::Row& row, isDate1 = true; break; + case execplan::CalpontSystemCatalog::TIME: case execplan::CalpontSystemCatalog::DATETIME: val1 = parm[0]->data()->getDatetimeIntVal(row, isNull); break; @@ -157,6 +158,7 @@ string Func_timediff::getStrVal(rowgroup::Row& row, isDate2 = true; break; + case execplan::CalpontSystemCatalog::TIME: case execplan::CalpontSystemCatalog::DATETIME: val2 = parm[1]->data()->getDatetimeIntVal(row, isNull); break; @@ -213,6 +215,14 @@ int64_t Func_timediff::getDatetimeIntVal(rowgroup::Row& row, return dataconvert::DataConvert::datetimeToInt(getStrVal(row, parm, isNull, ct)); } +int64_t Func_timediff::getTimeIntVal(rowgroup::Row& row, + FunctionParm& parm, + bool& isNull, + CalpontSystemCatalog::ColType& ct) +{ + return dataconvert::DataConvert::timeToInt(getStrVal(row, parm, isNull, ct)); +} + int64_t Func_timediff::getIntVal(rowgroup::Row& row, FunctionParm& parm, bool& isNull, diff --git a/utils/funcexp/func_timestampdiff.cpp b/utils/funcexp/func_timestampdiff.cpp index f8d1fc7ac..9394203d7 100644 --- a/utils/funcexp/func_timestampdiff.cpp +++ b/utils/funcexp/func_timestampdiff.cpp @@ -152,6 +152,13 @@ int64_t Func_timestampdiff::getDatetimeIntVal(rowgroup::Row& row, return getIntVal(row, parm, isNull, ct); } +int64_t Func_timestampdiff::getTimeIntVal(rowgroup::Row& row, + FunctionParm& parm, + bool& isNull, + CalpontSystemCatalog::ColType& ct) +{ + return getIntVal(row, parm, isNull, ct); +} } // namespace funcexp // vim:ts=4 sw=4: diff --git a/utils/funcexp/func_truncate.cpp b/utils/funcexp/func_truncate.cpp index c8be27de0..b822caacd 100644 --- a/utils/funcexp/func_truncate.cpp +++ b/utils/funcexp/func_truncate.cpp @@ -387,6 +387,51 @@ IDB_Decimal Func_truncate::getDecimalVal(Row& row, } break; + case execplan::CalpontSystemCatalog::TIME: + { + int32_t s = 0; + int64_t x = 0; + + string value = + DataConvert::timeToString1(parm[0]->data()->getTimeIntVal(row, isNull)); + + s = parm[1]->data()->getIntVal(row, isNull); + + if (!isNull) + { + //strip off micro seconds + value = value.substr(0, 14); + int64_t x = atoll(value.c_str()); + + if ( s > 5 ) + s = 0; + + if ( s > 0 ) + { + x *= helpers::powerOf10_c[s]; + } + else if (s < 0) + { + s = -s; + + if ( s >= (int32_t) value.size() ) + { + x = 0; + } + else + { + x /= helpers::powerOf10_c[s]; + x *= helpers::powerOf10_c[s]; + } + + s = 0; + } + } + + decimal.value = x; + decimal.scale = s; + } + break; default: { diff --git a/utils/funcexp/funcexp.cpp b/utils/funcexp/funcexp.cpp index a59be9a7c..3c530f381 100644 --- a/utils/funcexp/funcexp.cpp +++ b/utils/funcexp/funcexp.cpp @@ -280,6 +280,18 @@ void FuncExp::evaluate(rowgroup::Row& row, std::vector& expressi break; } + case CalpontSystemCatalog::TIME: + { + int64_t val = expression[i]->getTimeIntVal(row, isNull); + + if (isNull) + row.setIntField<8>(TIMENULL, expression[i]->outputIndex()); + else + row.setIntField<8>(val, expression[i]->outputIndex()); + + break; + } + case CalpontSystemCatalog::CHAR: case CalpontSystemCatalog::VARCHAR: diff --git a/utils/funcexp/functor.cpp b/utils/funcexp/functor.cpp index 662db6448..b74812ee9 100644 --- a/utils/funcexp/functor.cpp +++ b/utils/funcexp/functor.cpp @@ -106,6 +106,21 @@ uint64_t Func::stringToDatetime(const string str) return ret; } +int64_t Func::stringToTime(const string str) +{ + int64_t ret = DataConvert::stringToTime(str); + + if (ret == -1) + { + Message::Args args; + args.add("time"); + args.add(str); + unsigned errcode = ERR_INCORRECT_VALUE; + throw IDBExcept(IDBErrorInfo::instance()->errorMsg(errcode, args), errcode); + } + + return ret; +} uint32_t Func::intToDate(int64_t i) { @@ -124,6 +139,11 @@ uint64_t Func::intToDatetime(int64_t i) return i; } +int64_t Func::intToTime(int64_t i) +{ + // Don't think we need to do anything here? + return i; +} string Func::intToString(int64_t i) { diff --git a/utils/funcexp/functor.h b/utils/funcexp/functor.h index 9e03e4af2..065eb7865 100644 --- a/utils/funcexp/functor.h +++ b/utils/funcexp/functor.h @@ -116,6 +116,14 @@ public: return intToDatetime(getIntVal(row, fp, isNull, op_ct)); } + virtual int64_t getTimeIntVal(rowgroup::Row& row, + FunctionParm& fp, + bool& isNull, + execplan::CalpontSystemCatalog::ColType& op_ct) + { + return intToTime(getIntVal(row, fp, isNull, op_ct)); + } + virtual bool getBoolVal(rowgroup::Row& row, FunctionParm& fp, bool& isNull, @@ -145,9 +153,11 @@ public: protected: virtual uint32_t stringToDate(std::string); virtual uint64_t stringToDatetime(std::string); + virtual int64_t stringToTime(std::string); virtual uint32_t intToDate(int64_t); virtual uint64_t intToDatetime(int64_t); + virtual int64_t intToTime(int64_t); virtual std::string intToString(int64_t); virtual std::string doubleToString(double); diff --git a/utils/funcexp/functor_all.h b/utils/funcexp/functor_all.h index 6300084cd..0eb0df845 100644 --- a/utils/funcexp/functor_all.h +++ b/utils/funcexp/functor_all.h @@ -104,6 +104,11 @@ public: FunctionParm& fp, bool& isNull, execplan::CalpontSystemCatalog::ColType& op_ct); + + int64_t getTimeIntVal(rowgroup::Row& row, + FunctionParm& fp, + bool& isNull, + execplan::CalpontSystemCatalog::ColType& op_ct); }; @@ -151,6 +156,12 @@ public: FunctionParm& fp, bool& isNull, execplan::CalpontSystemCatalog::ColType& op_ct); + + int64_t getTimeIntVal(rowgroup::Row& row, + FunctionParm& fp, + bool& isNull, + execplan::CalpontSystemCatalog::ColType& op_ct); + }; @@ -193,7 +204,11 @@ public: FunctionParm& fp, bool& isNull, execplan::CalpontSystemCatalog::ColType& op_ct); -}; + + int64_t getTimeIntVal(rowgroup::Row& row, + FunctionParm& fp, + bool& isNull, + execplan::CalpontSystemCatalog::ColType& op_ct);}; /** @brief Func_ifnull class @@ -236,6 +251,11 @@ public: bool& isNull, execplan::CalpontSystemCatalog::ColType& op_ct); + int64_t getTimeIntVal(rowgroup::Row& row, + FunctionParm& fp, + bool& isNull, + execplan::CalpontSystemCatalog::ColType& op_ct); + bool getBoolVal(rowgroup::Row& row, FunctionParm& fp, bool& isNull, @@ -287,6 +307,12 @@ public: FunctionParm& fp, bool& isNull, execplan::CalpontSystemCatalog::ColType& op_ct); + + int64_t getTimeIntVal(rowgroup::Row& row, + FunctionParm& fp, + bool& isNull, + execplan::CalpontSystemCatalog::ColType& op_ct); + }; @@ -329,7 +355,11 @@ public: FunctionParm& fp, bool& isNull, execplan::CalpontSystemCatalog::ColType& op_ct); -}; + + int64_t getTimeIntVal(rowgroup::Row& row, + FunctionParm& fp, + bool& isNull, + execplan::CalpontSystemCatalog::ColType& op_ct);}; /** @brief Func_coalesce class @@ -371,6 +401,11 @@ public: FunctionParm& fp, bool& isNull, execplan::CalpontSystemCatalog::ColType& op_ct); + + int64_t getTimeIntVal(rowgroup::Row& row, + FunctionParm& fp, + bool& isNull, + execplan::CalpontSystemCatalog::ColType& op_ct); }; @@ -399,6 +434,11 @@ public: bool& isNull, execplan::CalpontSystemCatalog::ColType& op_ct); + int64_t getTimeIntVal(rowgroup::Row& row, + FunctionParm& fp, + bool& isNull, + execplan::CalpontSystemCatalog::ColType& op_ct); + int64_t getIntVal(rowgroup::Row& row, FunctionParm& fp, bool& isNull, diff --git a/utils/funcexp/functor_bool.h b/utils/funcexp/functor_bool.h index a20f925b3..f67f405f1 100644 --- a/utils/funcexp/functor_bool.h +++ b/utils/funcexp/functor_bool.h @@ -97,6 +97,15 @@ public: isNull = true; return 0; } + + int64_t getTimeIntVal(rowgroup::Row& row, + FunctionParm& fp, + bool& isNull, + execplan::CalpontSystemCatalog::ColType& op_ct) + { + isNull = true; + return 0; + } }; diff --git a/utils/funcexp/functor_dtm.h b/utils/funcexp/functor_dtm.h index dc662a176..74aa41e38 100644 --- a/utils/funcexp/functor_dtm.h +++ b/utils/funcexp/functor_dtm.h @@ -137,6 +137,11 @@ public: bool& isNull, execplan::CalpontSystemCatalog::ColType& op_ct); + int64_t getTimeIntVal(rowgroup::Row& row, + FunctionParm& fp, + bool& isNull, + execplan::CalpontSystemCatalog::ColType& op_ct); + int64_t getIntVal(rowgroup::Row& row, FunctionParm& fp, bool& isNull, @@ -247,6 +252,12 @@ public: FunctionParm& fp, bool& isNull, execplan::CalpontSystemCatalog::ColType& op_ct); + + int64_t getTimeIntVal(rowgroup::Row& row, + FunctionParm& fp, + bool& isNull, + execplan::CalpontSystemCatalog::ColType& op_ct); + }; @@ -311,6 +322,11 @@ public: FunctionParm& fp, bool& isNull, execplan::CalpontSystemCatalog::ColType& op_ct); + + int64_t getTimeIntVal(rowgroup::Row& row, + FunctionParm& fp, + bool& isNull, + execplan::CalpontSystemCatalog::ColType& op_ct); }; @@ -343,6 +359,11 @@ public: FunctionParm& fp, bool& isNull, execplan::CalpontSystemCatalog::ColType& op_ct); + + int64_t getTimeIntVal(rowgroup::Row& row, + FunctionParm& fp, + bool& isNull, + execplan::CalpontSystemCatalog::ColType& op_ct); }; @@ -371,6 +392,11 @@ public: bool& isNull, execplan::CalpontSystemCatalog::ColType& op_ct); + int64_t getTimeIntVal(rowgroup::Row& row, + FunctionParm& fp, + bool& isNull, + execplan::CalpontSystemCatalog::ColType& op_ct); + int64_t getIntVal(rowgroup::Row& row, FunctionParm& fp, bool& isNull, @@ -411,6 +437,11 @@ public: FunctionParm& fp, bool& isNull, execplan::CalpontSystemCatalog::ColType& op_ct); + + int64_t getTimeIntVal(rowgroup::Row& row, + FunctionParm& fp, + bool& isNull, + execplan::CalpontSystemCatalog::ColType& op_ct); }; /** @brief Func_str_to_date class diff --git a/utils/funcexp/functor_real.h b/utils/funcexp/functor_real.h index 936401f80..779f4d108 100644 --- a/utils/funcexp/functor_real.h +++ b/utils/funcexp/functor_real.h @@ -654,6 +654,12 @@ public: bool& isNull, execplan::CalpontSystemCatalog::ColType& op_ct); + int64_t getTimeIntVal(rowgroup::Row& row, + FunctionParm& fp, + bool& isNull, + execplan::CalpontSystemCatalog::ColType& op_ct); + + private: int64_t convertAton(const std::string& ipString, bool& isNull); }; diff --git a/utils/funcexp/functor_str.h b/utils/funcexp/functor_str.h index 0b40685f6..895101211 100644 --- a/utils/funcexp/functor_str.h +++ b/utils/funcexp/functor_str.h @@ -89,6 +89,14 @@ public: return (isNull ? 0 : stringToDatetime(str)); } + int64_t getTimeIntVal(rowgroup::Row& row, + FunctionParm& fp, + bool& isNull, + execplan::CalpontSystemCatalog::ColType& op_ct) + { + std::string str = getStrVal(row, fp, isNull, op_ct); + return (isNull ? 0 : stringToTime(str)); + } protected: const std::string& stringValue(execplan::SPTP& fp, rowgroup::Row& row, bool& isNull) @@ -676,6 +684,10 @@ public: bool& isNull, execplan::CalpontSystemCatalog::ColType& op_ct); + int64_t getTimeIntVal(rowgroup::Row& row, + FunctionParm& fp, + bool& isNull, + execplan::CalpontSystemCatalog::ColType& op_ct); private: void convertNtoa(int64_t ipNum, std::string& ipString); }; diff --git a/utils/rowgroup/rowaggregation.cpp b/utils/rowgroup/rowaggregation.cpp index 0a98f6870..a7edd9248 100644 --- a/utils/rowgroup/rowaggregation.cpp +++ b/utils/rowgroup/rowaggregation.cpp @@ -125,6 +125,11 @@ inline uint64_t getUintNullValue(int colType, int colWidth = 0) return joblist::DATETIMENULL; } + case execplan::CalpontSystemCatalog::TIME: + { + return joblist::TIMENULL; + } + case execplan::CalpontSystemCatalog::DECIMAL: case execplan::CalpontSystemCatalog::UDECIMAL: { @@ -640,6 +645,12 @@ inline bool RowAggregation::isNull(const RowGroup* pRowGroup, const Row& row, in break; } + case execplan::CalpontSystemCatalog::TIME: + { + ret = ((uint64_t)row.getUintField(col) == joblist::TIMENULL); + break; + } + case execplan::CalpontSystemCatalog::VARBINARY: case execplan::CalpontSystemCatalog::BLOB: { @@ -1117,6 +1128,7 @@ void RowAggregation::initMapData(const Row& rowIn) case execplan::CalpontSystemCatalog::DATE: case execplan::CalpontSystemCatalog::DATETIME: + case execplan::CalpontSystemCatalog::TIME: { fRow.setUintField(rowIn.getUintField(colIn), colOut); break; @@ -1247,6 +1259,7 @@ void RowAggregation::makeAggFieldsNull(Row& row) case execplan::CalpontSystemCatalog::DATE: case execplan::CalpontSystemCatalog::DATETIME: + case execplan::CalpontSystemCatalog::TIME: { row.setUintField(getUintNullValue(colDataType), colOut); break; @@ -1375,6 +1388,7 @@ void RowAggregation::doMinMaxSum(const Row& rowIn, int64_t colIn, int64_t colOut case execplan::CalpontSystemCatalog::DATE: case execplan::CalpontSystemCatalog::DATETIME: + case execplan::CalpontSystemCatalog::TIME: { if (funcType == ROWAGG_SUM) { @@ -1518,9 +1532,17 @@ void RowAggregation::doBitOp(const Row& rowIn, int64_t colIn, int64_t colOut, in case execplan::CalpontSystemCatalog::DATETIME: { uint64_t dtm = rowIn.getUintField(colIn); - valIn = ((dtm >> 48) * 10000000000LL) + (((dtm >> 44) & 0xF) * 100000000) + - (((dtm >> 38) & 077) * 1000000) + (((dtm >> 32) & 077) * 10000) + - (((dtm >> 26) & 077) * 100) + ((dtm >> 20) & 077); + valIn = ((dtm >> 48) * 10000000000000000LL) + (((dtm >> 44) & 0xF) * 100000000000000) + + (((dtm >> 38) & 077) * 1000000000000) + (((dtm >> 32) & 077) * 10000000000) + + (((dtm >> 26) & 077) * 100000000) + (((dtm >> 20) & 077) * 1000000) + (dtm & 0xfffff); + break; + } + + case execplan::CalpontSystemCatalog::TIME: + { + int64_t dtm = rowIn.getUintField(colIn); + valIn = (((dtm >> 40) & 0xfff) * 10000000000) + + (((dtm >> 32) & 0xff) * 100000000) + (((dtm >> 24) & 0xff) * 1000000) + (dtm & 0xffffff); break; } @@ -2047,6 +2069,12 @@ void RowAggregation::doUDAF(const Row& rowIn, int64_t colIn, int64_t colOut, int datum.columnData = rowIn.getUintField(colIn); break; } + case execplan::CalpontSystemCatalog::TIME: + { + datum.dataType = execplan::CalpontSystemCatalog::BIGINT; + datum.columnData = rowIn.getIntField(colIn); + break; + } case execplan::CalpontSystemCatalog::CHAR: case execplan::CalpontSystemCatalog::VARCHAR: @@ -2723,6 +2751,11 @@ void RowAggregationUM::SetUDAFValue(static_any::any& valOut, int64_t colOut) fRow.setUintField<8>(uintOut, colOut); break; + case execplan::CalpontSystemCatalog::TIME: + + fRow.setIntField<8>(intOut, colOut); + break; + case execplan::CalpontSystemCatalog::FLOAT: case execplan::CalpontSystemCatalog::UFLOAT: fRow.setFloatField(floatOut, colOut); @@ -3031,6 +3064,12 @@ void RowAggregationUM::doNullConstantAggregate(const ConstantAggData& aggData, u } break; + case execplan::CalpontSystemCatalog::TIME: + { + fRow.setIntField(getIntNullValue(colDataType), colOut); + } + break; + case execplan::CalpontSystemCatalog::CHAR: case execplan::CalpontSystemCatalog::VARCHAR: case execplan::CalpontSystemCatalog::TEXT: @@ -3207,6 +3246,12 @@ void RowAggregationUM::doNotNullConstantAggregate(const ConstantAggData& aggData } break; + case execplan::CalpontSystemCatalog::TIME: + { + fRow.setIntField(DataConvert::stringToTime(aggData.fConstValue), colOut); + } + break; + case execplan::CalpontSystemCatalog::CHAR: case execplan::CalpontSystemCatalog::VARCHAR: case execplan::CalpontSystemCatalog::TEXT: @@ -3295,6 +3340,7 @@ void RowAggregationUM::doNotNullConstantAggregate(const ConstantAggData& aggData case execplan::CalpontSystemCatalog::DATE: case execplan::CalpontSystemCatalog::DATETIME: + case execplan::CalpontSystemCatalog::TIME: case execplan::CalpontSystemCatalog::CHAR: case execplan::CalpontSystemCatalog::VARCHAR: case execplan::CalpontSystemCatalog::TEXT: @@ -3355,6 +3401,7 @@ void RowAggregationUM::doNotNullConstantAggregate(const ConstantAggData& aggData break; case execplan::CalpontSystemCatalog::DATETIME: + case execplan::CalpontSystemCatalog::TIME: { fRow.setUintField(0, colOut); } @@ -3492,6 +3539,12 @@ void RowAggregationUM::doNotNullConstantAggregate(const ConstantAggData& aggData } break; + case execplan::CalpontSystemCatalog::TIME: + { + datum.columnData = DataConvert::stringToTime(aggData.fConstValue); + } + break; + case execplan::CalpontSystemCatalog::CHAR: case execplan::CalpontSystemCatalog::VARCHAR: case execplan::CalpontSystemCatalog::TEXT: diff --git a/utils/rowgroup/rowgroup.cpp b/utils/rowgroup/rowgroup.cpp index c35129d03..185d3de67 100644 --- a/utils/rowgroup/rowgroup.cpp +++ b/utils/rowgroup/rowgroup.cpp @@ -699,6 +699,10 @@ void Row::initToNull() *((uint64_t*) &data[offsets[i]]) = joblist::DATETIMENULL; break; + case CalpontSystemCatalog::TIME: + *((uint64_t*) &data[offsets[i]]) = joblist::TIMENULL; + break; + case CalpontSystemCatalog::CHAR: case CalpontSystemCatalog::VARCHAR: case CalpontSystemCatalog::TEXT: @@ -841,6 +845,9 @@ bool Row::isNullValue(uint32_t colIndex) const case CalpontSystemCatalog::DATETIME: return (*((uint64_t*) &data[offsets[colIndex]]) == joblist::DATETIMENULL); + case CalpontSystemCatalog::TIME: + return (*((uint64_t*) &data[offsets[colIndex]]) == joblist::TIMENULL); + case CalpontSystemCatalog::CHAR: case CalpontSystemCatalog::VARCHAR: case CalpontSystemCatalog::STRINT: diff --git a/utils/udfsdk/docs/source/reference/ColumnDatum.rst b/utils/udfsdk/docs/source/reference/ColumnDatum.rst index 564b24e44..dd1006363 100644 --- a/utils/udfsdk/docs/source/reference/ColumnDatum.rst +++ b/utils/udfsdk/docs/source/reference/ColumnDatum.rst @@ -74,6 +74,8 @@ The provided values are: - A floating point number. Represented as a C++ double type. * - DATETIME - A Columnstore date-time stored as an eight byte unsigned integer. + * - TIME + - A Columnstore time stored as an eight byte unsigned integer. * - VARCHAR - A mariadb variable length string. Represented a std::string * - VARBINARY diff --git a/utils/udfsdk/mcsv1_udaf.cpp b/utils/udfsdk/mcsv1_udaf.cpp index a31ae71c0..349a642ec 100644 --- a/utils/udfsdk/mcsv1_udaf.cpp +++ b/utils/udfsdk/mcsv1_udaf.cpp @@ -95,6 +95,7 @@ int32_t mcsv1Context::getColWidth() case CalpontSystemCatalog::DOUBLE: case CalpontSystemCatalog::UDOUBLE: case CalpontSystemCatalog::DATETIME: + case CalpontSystemCatalog::TIME: case CalpontSystemCatalog::STRINT: fColWidth = 8; break; diff --git a/utils/udfsdk/udfsdk.cpp b/utils/udfsdk/udfsdk.cpp index ea206a167..9a8973232 100644 --- a/utils/udfsdk/udfsdk.cpp +++ b/utils/udfsdk/udfsdk.cpp @@ -101,7 +101,9 @@ CalpontSystemCatalog::ColType MCS_add::operationType (FunctionParm& fp, else if (fp[0]->data()->resultType().colDataType == CalpontSystemCatalog::DATE || fp[1]->data()->resultType().colDataType == CalpontSystemCatalog::DATE || fp[0]->data()->resultType().colDataType == CalpontSystemCatalog::DATETIME || - fp[1]->data()->resultType().colDataType == CalpontSystemCatalog::DATETIME) + fp[1]->data()->resultType().colDataType == CalpontSystemCatalog::DATETIME || + fp[0]->data()->resultType().colDataType == CalpontSystemCatalog::TIME || + fp[1]->data()->resultType().colDataType == CalpontSystemCatalog::TIME) { rt.colDataType = CalpontSystemCatalog::BIGINT; rt.colWidth = 8; diff --git a/utils/windowfunction/idborderby.cpp b/utils/windowfunction/idborderby.cpp index 5764afba5..8a021f8d8 100644 --- a/utils/windowfunction/idborderby.cpp +++ b/utils/windowfunction/idborderby.cpp @@ -281,6 +281,7 @@ void CompareRule::compileRules(const std::vector& spec, const rowgr case CalpontSystemCatalog::DATE: case CalpontSystemCatalog::DATETIME: + case CalpontSystemCatalog::TIME: { Compare* c = new UintCompare(*i); fCompares.push_back(c); @@ -413,6 +414,7 @@ bool EqualCompData::operator()(Row::Pointer a, Row::Pointer b) case CalpontSystemCatalog::UBIGINT: case CalpontSystemCatalog::DATE: case CalpontSystemCatalog::DATETIME: + case CalpontSystemCatalog::TIME: { // equal compare. ignore sign and null eq = (fRow1.getUintField(*i) == fRow2.getUintField(*i)); diff --git a/utils/windowfunction/wf_lead_lag.cpp b/utils/windowfunction/wf_lead_lag.cpp index dddfb91c6..5160b75bf 100644 --- a/utils/windowfunction/wf_lead_lag.cpp +++ b/utils/windowfunction/wf_lead_lag.cpp @@ -77,6 +77,7 @@ boost::shared_ptr WF_lead_lag::makeFunction(int id, const case CalpontSystemCatalog::UDECIMAL: case CalpontSystemCatalog::DATE: case CalpontSystemCatalog::DATETIME: + case CalpontSystemCatalog::TIME: { func.reset(new WF_lead_lag(id, name)); break; diff --git a/utils/windowfunction/wf_min_max.cpp b/utils/windowfunction/wf_min_max.cpp index 99a8fef84..b0c1fe033 100644 --- a/utils/windowfunction/wf_min_max.cpp +++ b/utils/windowfunction/wf_min_max.cpp @@ -76,6 +76,7 @@ boost::shared_ptr WF_min_max::makeFunction(int id, const case CalpontSystemCatalog::UDECIMAL: case CalpontSystemCatalog::DATE: case CalpontSystemCatalog::DATETIME: + case CalpontSystemCatalog::TIME: { func.reset(new WF_min_max(id, name)); break; diff --git a/utils/windowfunction/wf_nth_value.cpp b/utils/windowfunction/wf_nth_value.cpp index 5362ee8ec..eacd1202a 100644 --- a/utils/windowfunction/wf_nth_value.cpp +++ b/utils/windowfunction/wf_nth_value.cpp @@ -77,6 +77,7 @@ boost::shared_ptr WF_nth_value::makeFunction(int id, cons case CalpontSystemCatalog::UDECIMAL: case CalpontSystemCatalog::DATE: case CalpontSystemCatalog::DATETIME: + case CalpontSystemCatalog::TIME: { func.reset(new WF_nth_value(id, name)); break; diff --git a/utils/windowfunction/wf_percentile.cpp b/utils/windowfunction/wf_percentile.cpp index 9ccc07b9c..83d6f57d8 100644 --- a/utils/windowfunction/wf_percentile.cpp +++ b/utils/windowfunction/wf_percentile.cpp @@ -82,6 +82,7 @@ boost::shared_ptr WF_percentile::makeFunction(int id, con case CalpontSystemCatalog::UDECIMAL: case CalpontSystemCatalog::DATE: case CalpontSystemCatalog::DATETIME: + case CalpontSystemCatalog::TIME: { func.reset(new WF_percentile(id, name)); break; diff --git a/utils/windowfunction/wf_udaf.cpp b/utils/windowfunction/wf_udaf.cpp index fe394c028..f302c49cd 100644 --- a/utils/windowfunction/wf_udaf.cpp +++ b/utils/windowfunction/wf_udaf.cpp @@ -392,6 +392,7 @@ void WF_udaf::SetUDAFValue(static_any::any& valOut, int64_t colOut, case execplan::CalpontSystemCatalog::UBIGINT: case execplan::CalpontSystemCatalog::DATE: case execplan::CalpontSystemCatalog::DATETIME: + case execplan::CalpontSystemCatalog::TIME: setValue(colDataType, b, e, c, &uintOut); break; diff --git a/utils/windowfunction/windowfunctiontype.cpp b/utils/windowfunction/windowfunctiontype.cpp index 5a101f870..950045899 100644 --- a/utils/windowfunction/windowfunctiontype.cpp +++ b/utils/windowfunction/windowfunctiontype.cpp @@ -91,6 +91,7 @@ map colType2String = assign::map_list_of (CalpontSystemCatalog::LONGDOUBLE, "INTERNAL LONG DOUBLE") (CalpontSystemCatalog::STRINT, "INTERNAL SHORT STRING") (CalpontSystemCatalog::TEXT, "TEXT") + (CalpontSystemCatalog::TIME, "TIME") ; @@ -490,6 +491,7 @@ void* WindowFunctionType::getNullValueByType(int ct, int pos) static uint64_t doubleNull = joblist::DOUBLENULL; static uint64_t dateNull = joblist::DATENULL; static uint64_t datetimeNull = joblist::DATETIMENULL; + static uint64_t timeNull = joblist::TIMENULL; static uint64_t char1Null = joblist::CHAR1NULL; static uint64_t char2Null = joblist::CHAR2NULL; static uint64_t char4Null = joblist::CHAR4NULL; @@ -525,6 +527,10 @@ void* WindowFunctionType::getNullValueByType(int ct, int pos) v = &datetimeNull; break; + case CalpontSystemCatalog::TIME: + v = &timeNull; + break; + case CalpontSystemCatalog::FLOAT: case CalpontSystemCatalog::UFLOAT: v = &floatNull; diff --git a/writeengine/bulk/we_bulkloadbuffer.cpp b/writeengine/bulk/we_bulkloadbuffer.cpp index 68bf71027..79ea553f1 100644 --- a/writeengine/bulk/we_bulkloadbuffer.cpp +++ b/writeengine/bulk/we_bulkloadbuffer.cpp @@ -902,7 +902,8 @@ void BulkLoadBuffer::convert(char* field, int fieldLength, { bool bSatVal = false; - if ( column.dataType != CalpontSystemCatalog::DATETIME ) + if ( column.dataType != CalpontSystemCatalog::DATETIME && + column.dataType != CalpontSystemCatalog::TIME ) { if (nullFlag) { @@ -976,6 +977,59 @@ void BulkLoadBuffer::convert(char* field, int fieldLength, pVal = &llVal; } + else if (column.dataType == CalpontSystemCatalog::TIME) + { + // time conversion + int rc = 0; + + if (nullFlag) + { + if (column.fWithDefault) + { + llDate = column.fDefaultInt; + // fall through to update saturation and min/max + } + else + { + llDate = joblist::TIMENULL; + pVal = &llDate; + break; + } + } + else + { + if (fImportDataMode != IMPORT_DATA_TEXT) + { + memcpy(&llDate, field, sizeof(llDate)); + + if (!dataconvert::DataConvert::isColumnTimeValid( + llDate)) + rc = -1; + } + else + { + llDate = dataconvert::DataConvert::convertColumnTime( + field, dataconvert::CALPONTTIME_ENUM, + rc, fieldLength ); + } + } + + if (rc == 0) + { + if (llDate < bufStats.minBufferVal) + bufStats.minBufferVal = llDate; + + if (llDate > bufStats.maxBufferVal) + bufStats.maxBufferVal = llDate; + } + else + { + llDate = 0; + bufStats.satCount++; + } + + pVal = &llDate; + } else { // datetime conversion @@ -2973,6 +3027,11 @@ bool BulkLoadBuffer::isBinaryFieldNull(void* val, if ((*(uint64_t*)val) == joblist::DATETIMENULL) isNullFlag = true; } + else if (dt == execplan::CalpontSystemCatalog::TIME) + { + if ((*(uint64_t*)val) == joblist::TIMENULL) + isNullFlag = true; + } else { if ((*(uint64_t*)val) == joblist::BIGINTNULL) diff --git a/writeengine/bulk/we_tableinfo.cpp b/writeengine/bulk/we_tableinfo.cpp index 8da9ce968..da28204d2 100644 --- a/writeengine/bulk/we_tableinfo.cpp +++ b/writeengine/bulk/we_tableinfo.cpp @@ -978,6 +978,11 @@ void TableInfo::reportTotals(double elapsedTime) ossSatCnt << "invalid date/times replaced with zero value : "; } + else if (fColumns[i].column.dataType == CalpontSystemCatalog::TIME) + { + ossSatCnt << + "invalid times replaced with zero value : "; + } else if (fColumns[i].column.dataType == CalpontSystemCatalog::CHAR) ossSatCnt << "character strings truncated: "; diff --git a/writeengine/server/we_ddlcommon.h b/writeengine/server/we_ddlcommon.h index aca25dc39..1d482b442 100644 --- a/writeengine/server/we_ddlcommon.h +++ b/writeengine/server/we_ddlcommon.h @@ -274,6 +274,13 @@ inline boost::any getNullValueForType(const execplan::CalpontSystemCatalog::ColT } break; + case execplan::CalpontSystemCatalog::TIME: + { + long long d = joblist::TIMENULL; + value = d; + } + break; + case execplan::CalpontSystemCatalog::CHAR: { std::string charnull; @@ -433,6 +440,10 @@ inline int convertDataType(int dataType) calpontDataType = CalpontSystemCatalog::DATETIME; break; + case ddlpackage::DDL_TIME: + calpontDataType = CalpontSystemCatalog::TIME; + break; + case ddlpackage::DDL_CLOB: calpontDataType = CalpontSystemCatalog::CLOB; break; diff --git a/writeengine/server/we_dmlcommandproc.cpp b/writeengine/server/we_dmlcommandproc.cpp index 07037a1e3..86625d013 100644 --- a/writeengine/server/we_dmlcommandproc.cpp +++ b/writeengine/server/we_dmlcommandproc.cpp @@ -1781,6 +1781,7 @@ uint8_t WE_DMLCommandProc::processBatchInsertBinary(messageqcpp::ByteStream& bs, case execplan::CalpontSystemCatalog::BIGINT: case execplan::CalpontSystemCatalog::DATETIME: + case execplan::CalpontSystemCatalog::TIME: case execplan::CalpontSystemCatalog::UBIGINT: bs >> val64; @@ -2981,6 +2982,13 @@ uint8_t WE_DMLCommandProc::processUpdate(messageqcpp::ByteStream& bs, break; } + case CalpontSystemCatalog::TIME: + { + intColVal = row.getIntField<8>(fetchColPos); + value = DataConvert::timeToString(intColVal, colType.precision); + break; + } + case CalpontSystemCatalog::CHAR: case CalpontSystemCatalog::VARCHAR: { @@ -3309,6 +3317,13 @@ uint8_t WE_DMLCommandProc::processUpdate(messageqcpp::ByteStream& bs, break; } + case CalpontSystemCatalog::TIME: + { + intColVal = row.getIntField<8>(fetchColPos); + value = DataConvert::timeToString(intColVal, colType.precision); + break; + } + case CalpontSystemCatalog::CHAR: case CalpontSystemCatalog::VARCHAR: { diff --git a/writeengine/shared/we_convertor.cpp b/writeengine/shared/we_convertor.cpp index dc9754355..8050426f3 100644 --- a/writeengine/shared/we_convertor.cpp +++ b/writeengine/shared/we_convertor.cpp @@ -420,6 +420,7 @@ void Convertor::convertColType(CalpontSystemCatalog::ColDataType dataType, // Map BIGINT and DATETIME to WR_LONGLONG case CalpontSystemCatalog::BIGINT : case CalpontSystemCatalog::DATETIME : + case CalpontSystemCatalog::TIME : internalType = WriteEngine::WR_LONGLONG; break; @@ -629,6 +630,7 @@ void Convertor::convertColType(ColStruct* curStruct) // Map BIGINT and DATETIME to WR_LONGLONG case CalpontSystemCatalog::BIGINT : case CalpontSystemCatalog::DATETIME : + case CalpontSystemCatalog::TIME : *internalType = WriteEngine::WR_LONGLONG; break; @@ -792,6 +794,7 @@ int Convertor::getCorrectRowWidth(CalpontSystemCatalog::ColDataType dataType, in break; case CalpontSystemCatalog::DATETIME: + case CalpontSystemCatalog::TIME: newWidth = 8; break; diff --git a/writeengine/splitter/we_sdhandler.cpp b/writeengine/splitter/we_sdhandler.cpp index 68d5bf784..802fd108b 100644 --- a/writeengine/splitter/we_sdhandler.cpp +++ b/writeengine/splitter/we_sdhandler.cpp @@ -1907,6 +1907,10 @@ void WESDHandler::onCleanupResult(int PmId, messageqcpp::SBS& Sbs) ossSatCnt << "invalid date/times replaced with zero value: "; break; + case CalpontSystemCatalog::TIME: + ossSatCnt << "invalid times replaced with zero value: "; + break; + case CalpontSystemCatalog::CHAR: ossSatCnt << "character strings truncated: "; break; diff --git a/writeengine/wrapper/writeengine.cpp b/writeengine/wrapper/writeengine.cpp index cb6eb3745..5d3dfec85 100644 --- a/writeengine/wrapper/writeengine.cpp +++ b/writeengine/wrapper/writeengine.cpp @@ -443,6 +443,8 @@ void WriteEngineWrapper::convertValue(const ColType colType, void* valArray, con case WriteEngine::WR_LONGLONG: if (data.type() == typeid(long long)) ((long long*)valArray)[pos] = boost::any_cast(data); + else if (data.type() == typeid(long)) + ((long long*)valArray)[pos] = (long long)boost::any_cast(data); else ((long long*)valArray)[pos] = boost::any_cast(data); diff --git a/writeengine/xml/we_xmljob.cpp b/writeengine/xml/we_xmljob.cpp index 135754ac1..8d755b824 100644 --- a/writeengine/xml/we_xmljob.cpp +++ b/writeengine/xml/we_xmljob.cpp @@ -1116,6 +1116,22 @@ void XMLJob::fillInXMLDataNotNullDefault( break; } + case execplan::CalpontSystemCatalog::TIME: + { + int convertStatus; + int64_t dt = + dataconvert::DataConvert::convertColumnTime( + col_defaultValue.c_str(), + dataconvert::CALPONTTIME_ENUM, convertStatus, + col_defaultValue.length() ); + + if (convertStatus != 0) + bDefaultConvertError = true; + + col.fDefaultInt = dt; + break; + } + case execplan::CalpontSystemCatalog::FLOAT: case execplan::CalpontSystemCatalog::DOUBLE: case execplan::CalpontSystemCatalog::UFLOAT: From edb2e2f36d2f9ff40e7b4c6413cdc1b5b29e151f Mon Sep 17 00:00:00 2001 From: Andrew Hutchings Date: Thu, 26 Apr 2018 14:28:17 +0100 Subject: [PATCH 38/49] MCOL-392 Fix negative time handling --- utils/dataconvert/dataconvert.cpp | 12 ++++++++---- utils/dataconvert/dataconvert.h | 24 +++++++++++++++++++++--- utils/funcexp/func_bitand.cpp | 7 ++++++- utils/rowgroup/rowaggregation.cpp | 10 +++++++++- 4 files changed, 44 insertions(+), 9 deletions(-) diff --git a/utils/dataconvert/dataconvert.cpp b/utils/dataconvert/dataconvert.cpp index b5b1ae619..013e3f05c 100644 --- a/utils/dataconvert/dataconvert.cpp +++ b/utils/dataconvert/dataconvert.cpp @@ -863,12 +863,11 @@ bool mysql_str_to_time( const string& input, Time& output ) { int32_t datesepct = 0; uint32_t dtend = 0; + bool isNeg = false; /** * We need to deal with the time portion. * The rules are: - * - Time portion may be empty - * - Time portion may start with 'T' * - Time portion always ends with '\0' * - Time portion always starts with hour * - Without time separators (':'): @@ -890,7 +889,7 @@ bool mysql_str_to_time( const string& input, Time& output ) uint32_t timesep_ct = 0; bool has_usec = false; uint32_t len_before_msec = 0; - uint32_t tmstart = ( input[dtend] == ' ' || input[dtend] == 'T' ) ? dtend + 1 : dtend; + uint32_t tmstart = dtend; uint32_t tmend = tmstart; for ( ; tmend < input.length(); ++tmend ) @@ -918,6 +917,11 @@ bool mysql_str_to_time( const string& input, Time& output ) len_before_msec = ( tmend - tmstart ); has_usec = true; } + else if (c == '-' && (tmend == tmstart)) + { + isNeg = true; + ++tmstart; + } else { timesep_ct++; @@ -998,7 +1002,7 @@ bool mysql_str_to_time( const string& input, Time& output ) return false; } - output.hour = hour; + output.hour = isNeg ? 0-hour : hour; output.minute = min; output.second = sec; output.msecond = usec; diff --git a/utils/dataconvert/dataconvert.h b/utils/dataconvert/dataconvert.h index 7f148daf8..1c7b7f663 100644 --- a/utils/dataconvert/dataconvert.h +++ b/utils/dataconvert/dataconvert.h @@ -566,10 +566,19 @@ inline void DataConvert::datetimeToString( long long datetimevalue, char* buf, u inline void DataConvert::timeToString( long long timevalue, char* buf, unsigned int buflen ) { + // Handle negative correctly + int hour = 0; + if ((timevalue >> 40) & 0xf00) + { + hour = 0xfffff000; + } + + hour |= ((timevalue >> 40) & 0xfff); + if ((timevalue & 0xffffff) > 0) { snprintf( buf, buflen, "%02d:%02d:%02d.%d", - (unsigned)((timevalue >> 40) & 0xfff), + hour, (unsigned)((timevalue >> 32) & 0xff), (unsigned)((timevalue >> 24) & 0xff), (unsigned)((timevalue) & 0xffffff) @@ -578,7 +587,7 @@ inline void DataConvert::timeToString( long long timevalue, char* buf, unsigned else { snprintf( buf, buflen, "%02d:%02d:%02d", - (unsigned)((timevalue >> 40) & 0xfff), + hour, (unsigned)((timevalue >> 32) & 0xff), (unsigned)((timevalue >> 24) & 0xff) ); @@ -608,8 +617,17 @@ inline void DataConvert::datetimeToString1( long long datetimevalue, char* buf, inline void DataConvert::timeToString1( long long timevalue, char* buf, unsigned int buflen ) { + // Handle negative correctly + int hour = 0; + if ((timevalue >> 40) & 0xf00) + { + hour = 0xfffff000; + } + + hour |= ((timevalue >> 40) & 0xfff); + snprintf( buf, buflen, "%02d%02d%02d", - (unsigned)((timevalue >> 40) & 0xfff), + hour, (unsigned)((timevalue >> 32) & 0xff), (unsigned)((timevalue >> 14) & 0xff) ); diff --git a/utils/funcexp/func_bitand.cpp b/utils/funcexp/func_bitand.cpp index eceb4e4c0..3966d7a42 100644 --- a/utils/funcexp/func_bitand.cpp +++ b/utils/funcexp/func_bitand.cpp @@ -165,8 +165,13 @@ int64_t Func_bitand::getIntVal(Row& row, min = 0, sec = 0, msec = 0; + // Handle negative correctly + if ((time >> 40) & 0xf00) + { + hour = 0xfffff000; + } - hour = (uint32_t)((time >> 40) & 0xfff); + hour |= ((time >> 40) & 0xfff); min = (uint32_t)((time >> 32) & 0xff); sec = (uint32_t)((time >> 24) & 0xff); msec = (uint32_t)(time & 0xffffff); diff --git a/utils/rowgroup/rowaggregation.cpp b/utils/rowgroup/rowaggregation.cpp index a7edd9248..b6d72fede 100644 --- a/utils/rowgroup/rowaggregation.cpp +++ b/utils/rowgroup/rowaggregation.cpp @@ -1541,7 +1541,15 @@ void RowAggregation::doBitOp(const Row& rowIn, int64_t colIn, int64_t colOut, in case execplan::CalpontSystemCatalog::TIME: { int64_t dtm = rowIn.getUintField(colIn); - valIn = (((dtm >> 40) & 0xfff) * 10000000000) + + // Handle negative correctly + int hour = 0; + if ((dtm >> 40) & 0xf00) + { + hour = 0xfffff000; + } + + hour |= ((dtm >> 40) & 0xfff); + valIn = (hour * 10000000000) + (((dtm >> 32) & 0xff) * 100000000) + (((dtm >> 24) & 0xff) * 1000000) + (dtm & 0xffffff); break; } From dba04e8b72d31c955d4371f41d48d982832654c4 Mon Sep 17 00:00:00 2001 From: Andrew Hutchings Date: Thu, 26 Apr 2018 17:13:24 +0100 Subject: [PATCH 39/49] MCOL-392 Fix cpimport and >8bit positive hour --- utils/dataconvert/dataconvert.cpp | 81 ++++++++++++++++++------------- utils/dataconvert/dataconvert.h | 2 +- utils/funcexp/func_bitand.cpp | 2 +- utils/rowgroup/rowaggregation.cpp | 2 +- writeengine/shared/we_type.h | 3 +- 5 files changed, 53 insertions(+), 37 deletions(-) diff --git a/utils/dataconvert/dataconvert.cpp b/utils/dataconvert/dataconvert.cpp index 013e3f05c..ea0a4f2c9 100644 --- a/utils/dataconvert/dataconvert.cpp +++ b/utils/dataconvert/dataconvert.cpp @@ -1886,11 +1886,11 @@ int64_t DataConvert::convertColumnTime( unsigned int dataOrgLen ) { status = 0; - const char* p; - p = dataOrg; - char fld[10]; - int16_t value = 0; - int inYear, inMonth, inDay, inHour, inMinute, inSecond, inMicrosecond; + char* p; + char* savePoint = NULL; + p = const_cast(dataOrg); + int64_t value = 0; + int inHour, inMinute, inSecond, inMicrosecond; inHour = 0; inMinute = 0; inSecond = 0; @@ -1901,44 +1901,59 @@ int64_t DataConvert::convertColumnTime( return value; } - memcpy( fld, p, 2); - fld[2] = '\0'; + errno = 0; - inHour = strtol(fld, 0, 10); + p = strtok_r(p, ":.", &savePoint); + inHour = strtol(p, 0, 10); - if (!isdigit(p[2]) || !isdigit(p[3])) - { - status = -1; - return value; - } + if (errno) + { + status = -1; + return value; + } - memcpy( fld, p + 2, 2); - fld[2] = '\0'; + p = strtok_r(NULL, ":.", &savePoint); + if (p == NULL) + { + status = -1; + return value; + } - inMinute = strtol(fld, 0, 10); + inMinute = strtol(p, 0, 10); - if (!isdigit(p[4]) || !isdigit(p[5])) - { - status = -1; - return value; - } + if (errno) + { + status = -1; + return value; + } - memcpy( fld, p + 4, 2); - fld[2] = '\0'; + p = strtok_r(NULL, ":.", &savePoint); - inSecond = strtol(fld, 0, 10); + if (p == NULL) + { + status = -1; + return value; + } - if (dataOrgLen > 9) - { - unsigned int microFldLen = dataOrgLen - 9; + inSecond = strtol(p, 0, 10); - if (microFldLen > (sizeof(fld) - 1)) - microFldLen = sizeof(fld) - 1; + if (errno) + { + status = -1; + return value; + } - memcpy( fld, p + 9, microFldLen); - fld[microFldLen] = '\0'; - inMicrosecond = strtol(fld, 0, 10); - } + p = strtok_r(NULL, ":.", &savePoint); + + if (p != NULL) + { + inMicrosecond = strtol(p, 0, 10); + if (errno) + { + status = -1; + return value; + } + } if ( isTimeValid (inHour, inMinute, inSecond, inMicrosecond) ) { diff --git a/utils/dataconvert/dataconvert.h b/utils/dataconvert/dataconvert.h index 1c7b7f663..057d617ef 100644 --- a/utils/dataconvert/dataconvert.h +++ b/utils/dataconvert/dataconvert.h @@ -568,7 +568,7 @@ inline void DataConvert::timeToString( long long timevalue, char* buf, unsigned { // Handle negative correctly int hour = 0; - if ((timevalue >> 40) & 0xf00) + if ((timevalue >> 40) & 0x800) { hour = 0xfffff000; } diff --git a/utils/funcexp/func_bitand.cpp b/utils/funcexp/func_bitand.cpp index 3966d7a42..32cb846fe 100644 --- a/utils/funcexp/func_bitand.cpp +++ b/utils/funcexp/func_bitand.cpp @@ -166,7 +166,7 @@ int64_t Func_bitand::getIntVal(Row& row, sec = 0, msec = 0; // Handle negative correctly - if ((time >> 40) & 0xf00) + if ((time >> 40) & 0x800) { hour = 0xfffff000; } diff --git a/utils/rowgroup/rowaggregation.cpp b/utils/rowgroup/rowaggregation.cpp index b6d72fede..c496d91e6 100644 --- a/utils/rowgroup/rowaggregation.cpp +++ b/utils/rowgroup/rowaggregation.cpp @@ -1543,7 +1543,7 @@ void RowAggregation::doBitOp(const Row& rowIn, int64_t colIn, int64_t colOut, in int64_t dtm = rowIn.getUintField(colIn); // Handle negative correctly int hour = 0; - if ((dtm >> 40) & 0xf00) + if ((dtm >> 40) & 0x800) { hour = 0xfffff000; } diff --git a/writeengine/shared/we_type.h b/writeengine/shared/we_type.h index 21fad3e08..06a367ecb 100644 --- a/writeengine/shared/we_type.h +++ b/writeengine/shared/we_type.h @@ -161,7 +161,8 @@ const char ColDataTypeStr[execplan::CalpontSystemCatalog::NUM_OF_COL_DATA_TYPE] "unsigned-float", "unsigned-bigint", "unsigned-double", - "text" + "text", + "time" }; enum FuncType { FUNC_WRITE_ENGINE, FUNC_INDEX, FUNC_DICTIONARY }; From bd50bbb8bb37f3d2d9e15f601c56b9126c67d273 Mon Sep 17 00:00:00 2001 From: Andrew Hutchings Date: Thu, 26 Apr 2018 17:48:16 +0100 Subject: [PATCH 40/49] MCOL-392 Fix saturation handling --- utils/dataconvert/dataconvert.cpp | 42 +++++++++++++++++++++++++- writeengine/bulk/we_bulkloadbuffer.cpp | 1 - 2 files changed, 41 insertions(+), 2 deletions(-) diff --git a/utils/dataconvert/dataconvert.cpp b/utils/dataconvert/dataconvert.cpp index ea0a4f2c9..bfdb25239 100644 --- a/utils/dataconvert/dataconvert.cpp +++ b/utils/dataconvert/dataconvert.cpp @@ -998,7 +998,26 @@ bool mysql_str_to_time( const string& input, Time& output ) if ( !isTimeValid( hour, min, sec, usec ) ) { - output.reset(); + // Emulate MariaDB's time saturation + if (hour > 838) + { + output.hour = 838; + output.minute = 59; + output.second = 59; + output.msecond = 999999; + } + else if (hour < -838) + { + output.hour = -838; + output.minute = 59; + output.second = 59; + output.msecond = 999999; + } + // If neither of the above match then we return a 0 time + else + { + output.reset(); + } return false; } @@ -1967,6 +1986,27 @@ int64_t DataConvert::convertColumnTime( } else { + // Emulate MariaDB's time saturation + if (inHour > 838) + { + Time atime; + atime.hour = 838; + atime.minute = 59; + atime.second = 59; + atime.msecond = 999999; + memcpy( &value, &atime, 8); + } + else if (inHour < -838) + { + Time atime; + atime.hour = -838; + atime.minute = 59; + atime.second = 59; + atime.msecond = 999999; + memcpy( &value, &atime, 8); + } + // If neither of the above match then we return a 0 time + status = -1; } diff --git a/writeengine/bulk/we_bulkloadbuffer.cpp b/writeengine/bulk/we_bulkloadbuffer.cpp index 79ea553f1..ccf73363f 100644 --- a/writeengine/bulk/we_bulkloadbuffer.cpp +++ b/writeengine/bulk/we_bulkloadbuffer.cpp @@ -1024,7 +1024,6 @@ void BulkLoadBuffer::convert(char* field, int fieldLength, } else { - llDate = 0; bufStats.satCount++; } From 957dc446151d340abe783cebf9a415a2d03256de Mon Sep 17 00:00:00 2001 From: Andrew Hutchings Date: Fri, 27 Apr 2018 21:18:14 +0100 Subject: [PATCH 41/49] MCOL-392 Function fixes Fixes most of the functions in funcexp so that time and datetime's microseconds are handled correctly --- dbcon/execplan/treenode.h | 19 +++-- dbcon/mysql/ha_calpont_execplan.cpp | 3 +- utils/dataconvert/dataconvert.cpp | 12 +-- utils/dataconvert/dataconvert.h | 111 ++++++++++++++++------------ utils/funcexp/func_add_time.cpp | 49 ++++++++---- utils/funcexp/func_between.cpp | 31 ++++++-- utils/funcexp/func_extract.cpp | 97 +++++++++++++++++++++++- utils/funcexp/func_greatest.cpp | 5 +- utils/funcexp/func_hour.cpp | 20 ++++- utils/funcexp/func_least.cpp | 5 +- utils/funcexp/func_time.cpp | 20 +++-- 11 files changed, 271 insertions(+), 101 deletions(-) diff --git a/dbcon/execplan/treenode.h b/dbcon/execplan/treenode.h index 27795ea9d..08b8b43c8 100644 --- a/dbcon/execplan/treenode.h +++ b/dbcon/execplan/treenode.h @@ -666,14 +666,14 @@ inline const std::string& TreeNode::getStrVal() case CalpontSystemCatalog::DATETIME: { - dataconvert::DataConvert::datetimeToString(fResult.intVal, tmp, 255); + dataconvert::DataConvert::datetimeToString(fResult.intVal, tmp, 255, fResultType.precision); fResult.strVal = std::string(tmp); break; } case CalpontSystemCatalog::TIME: { - dataconvert::DataConvert::timeToString(fResult.intVal, tmp, 255); + dataconvert::DataConvert::timeToString(fResult.intVal, tmp, 255, fResultType.precision); fResult.strVal = std::string(tmp); break; } @@ -992,11 +992,20 @@ inline int64_t TreeNode::getDatetimeIntVal() else if (fResultType.colDataType == execplan::CalpontSystemCatalog::TIME) { dataconvert::Time tt; + int day = 0; memcpy(&tt, &fResult.intVal, 8); - if (tt.hour > 23 || tt.hour < 0) - throw logging::InvalidConversionExcept("TreeNode::getDecimalVal: Invalid conversion from time (out of range)."); - dataconvert::DateTime dt(0, 0, 0, tt.hour, tt.minute, tt.second, tt.msecond); + // Note, this should probably be current date +/- time + if (tt.hour > 23) + { + day = tt.hour / 24; + tt.hour = tt.hour % 24; + } + else if (tt.hour < 0) + { + tt.hour = 0; + } + dataconvert::DateTime dt(0, 0, day, tt.hour, tt.minute, tt.second, tt.msecond); memcpy(&fResult.intVal, &dt, 8); return fResult.intVal; } diff --git a/dbcon/mysql/ha_calpont_execplan.cpp b/dbcon/mysql/ha_calpont_execplan.cpp index fe2f96e52..210da9992 100644 --- a/dbcon/mysql/ha_calpont_execplan.cpp +++ b/dbcon/mysql/ha_calpont_execplan.cpp @@ -3518,8 +3518,7 @@ ReturnedColumn* buildFunctionColumn(Item_func* ifp, gp_walk_info& gwi, bool& non if (ifp->field_type() == MYSQL_TYPE_DATETIME || ifp->field_type() == MYSQL_TYPE_DATETIME2 || ifp->field_type() == MYSQL_TYPE_TIMESTAMP || - ifp->field_type() == MYSQL_TYPE_TIMESTAMP2 || - funcName == "add_time") + ifp->field_type() == MYSQL_TYPE_TIMESTAMP2) { CalpontSystemCatalog::ColType ct; ct.colDataType = CalpontSystemCatalog::DATETIME; diff --git a/utils/dataconvert/dataconvert.cpp b/utils/dataconvert/dataconvert.cpp index bfdb25239..41dff9aa9 100644 --- a/utils/dataconvert/dataconvert.cpp +++ b/utils/dataconvert/dataconvert.cpp @@ -2065,9 +2065,9 @@ std::string DataConvert::datetimeToString( long long datetimevalue, long decima { snprintf(buf + strlen(buf), 21 + decimals, ".%d", dt.msecond); // Pad end with zeros - if (strlen(buf) < (21 + decimals)) + if (strlen(buf) < (size_t)(21 + decimals)) { - sprintf(buf + strlen(buf), "%0*d", 21 + decimals - strlen(buf), 0); + sprintf(buf + strlen(buf), "%0*d", (int)(21 + decimals - strlen(buf)), 0); } } return buf; @@ -2091,9 +2091,9 @@ std::string DataConvert::timeToString( long long timevalue, long decimals ) size_t start = strlen(buf); snprintf(buf + strlen(buf), 12 + decimals, ".%d", dt.msecond); // Pad end with zeros - if (strlen(buf) - start < decimals) + if (strlen(buf) - start < (size_t)decimals) { - sprintf(buf + strlen(buf), "%0*d", decimals - (strlen(buf) - start), 0); + sprintf(buf + strlen(buf), "%0*d", (int)(decimals - (strlen(buf) - start) + 1), 0); } } return buf; @@ -2642,7 +2642,7 @@ int64_t DataConvert::intToTime(int64_t data) atime.second = 0; atime.msecond = 0; - return *(reinterpret_cast(&atime)); + return *(reinterpret_cast(&atime)); } snprintf( buf, 15, "%llu", (long long unsigned int)data); @@ -2696,7 +2696,7 @@ int64_t DataConvert::stringToTime(const string& data) // -34 <= D <= 34 // -838 <= H <= 838 uint64_t min = 0, sec = 0, msec = 0; - int64_t day = 0, hour = 0; + int64_t day = -1, hour = 0; string time, hms, ms; char* end = NULL; diff --git a/utils/dataconvert/dataconvert.h b/utils/dataconvert/dataconvert.h index 057d617ef..8c35bec4b 100644 --- a/utils/dataconvert/dataconvert.h +++ b/utils/dataconvert/dataconvert.h @@ -173,7 +173,7 @@ struct DateTime inline int64_t DateTime::convertToMySQLint() const { - return (int64_t) (year * 10000000000000000LL) + (month * 100000000000000) + (day * 1000000000000) + (hour * 10000000000) + (minute * 100000000) + (second * 1000000) + msecond; + return (int64_t) (year * 10000000000LL) + (month * 100000000) + (day * 1000000) + (hour * 10000) + (minute * 100) + second; } inline @@ -235,16 +235,11 @@ void Time::reset() inline int64_t Time::convertToMySQLint() const { - return (int64_t) (hour * 10000000000) + (minute * 100000000) + (second * 1000000) + msecond; + return (int64_t) (hour * 10000) + (minute * 100) + second; } static uint32_t daysInMonth[13] = {31, 28, 31, 30, 31, 30, 31, 31, 30, 31, 30, 31, 0}; -inline uint32_t getDaysInMonth(uint32_t month) -{ - return ( (month < 1 || month > 12) ? 0 : daysInMonth[month - 1]); -} - inline bool isLeapYear ( int year) { if ( year % 400 == 0 ) @@ -256,6 +251,18 @@ inline bool isLeapYear ( int year) return false; } +inline uint32_t getDaysInMonth(uint32_t month, int year) +{ + if (month < 1 || month > 12) + return 0; + + uint32_t days = daysInMonth[month - 1]; + if ((month == 2) && isLeapYear(year)) + days++; + + return days; +} + inline bool isDateValid ( int day, int month, int year) { @@ -266,11 +273,7 @@ bool isDateValid ( int day, int month, int year) return true; } - int daycheck = getDaysInMonth( month ); - - if ( month == 2 && isLeapYear( year ) ) - // 29 days in February in a leap year - daycheck = 29; + int daycheck = getDaysInMonth( month, year ); if ( ( year < 1000 ) || ( year > 9999 ) ) valid = false; @@ -407,7 +410,7 @@ public: * @param data the columns string representation of it's data */ EXPORT static std::string datetimeToString( long long datetimevalue, long decimals = 0 ); - static inline void datetimeToString( long long datetimevalue, char* buf, unsigned int buflen ); + static inline void datetimeToString( long long datetimevalue, char* buf, unsigned int buflen, long decimals = 0 ); /** * @brief convert a columns data from native format to a string @@ -416,7 +419,7 @@ public: * @param data the columns string representation of it's data */ EXPORT static std::string timeToString( long long timevalue, long decimals = 0 ); - static inline void timeToString( long long timevalue, char* buf, unsigned int buflen ); + static inline void timeToString( long long timevalue, char* buf, unsigned int buflen, long decimals = 0); /** * @brief convert a columns data from native format to a string @@ -537,37 +540,48 @@ inline void DataConvert::dateToString( int datevalue, char* buf, unsigned int bu ); } -inline void DataConvert::datetimeToString( long long datetimevalue, char* buf, unsigned int buflen ) +inline void DataConvert::datetimeToString( long long datetimevalue, char* buf, unsigned int buflen, long decimals ) { + // 10 is default which means we don't need microseconds + if (decimals > 6 || decimals < 0) + { + decimals = 0; + } + int msec = 0; if ((datetimevalue & 0xfffff) > 0) { - snprintf( buf, buflen, "%04d-%02d-%02d %02d:%02d:%02d.%d", - (unsigned)((datetimevalue >> 48) & 0xffff), - (unsigned)((datetimevalue >> 44) & 0xf), - (unsigned)((datetimevalue >> 38) & 0x3f), - (unsigned)((datetimevalue >> 32) & 0x3f), - (unsigned)((datetimevalue >> 26) & 0x3f), - (unsigned)((datetimevalue >> 20) & 0x3f), - (unsigned)((datetimevalue) & 0xfffff) - ); + msec = (unsigned)((datetimevalue) & 0xfffff); } - else + snprintf( buf, buflen, "%04d-%02d-%02d %02d:%02d:%02d", + (unsigned)((datetimevalue >> 48) & 0xffff), + (unsigned)((datetimevalue >> 44) & 0xf), + (unsigned)((datetimevalue >> 38) & 0x3f), + (unsigned)((datetimevalue >> 32) & 0x3f), + (unsigned)((datetimevalue >> 26) & 0x3f), + (unsigned)((datetimevalue >> 20) & 0x3f) + ); + + if (msec || decimals) { - snprintf( buf, buflen, "%04d-%02d-%02d %02d:%02d:%02d", - (unsigned)((datetimevalue >> 48) & 0xffff), - (unsigned)((datetimevalue >> 44) & 0xf), - (unsigned)((datetimevalue >> 38) & 0x3f), - (unsigned)((datetimevalue >> 32) & 0x3f), - (unsigned)((datetimevalue >> 26) & 0x3f), - (unsigned)((datetimevalue >> 20) & 0x3f) - ); + size_t start = strlen(buf); + snprintf(buf + strlen(buf), buflen - start, ".%d", msec); + // Pad end with zeros + if (strlen(buf) - start < (size_t)decimals) + { + snprintf(buf + strlen(buf), buflen - strlen(buf), "%0*d", (int)(decimals - (strlen(buf) - start) + 1), 0); + } } } -inline void DataConvert::timeToString( long long timevalue, char* buf, unsigned int buflen ) +inline void DataConvert::timeToString( long long timevalue, char* buf, unsigned int buflen, long decimals ) { + // 10 is default which means we don't need microseconds + if (decimals > 6 || decimals < 0) + { + decimals = 0; + } // Handle negative correctly - int hour = 0; + int hour = 0, msec = 0; if ((timevalue >> 40) & 0x800) { hour = 0xfffff000; @@ -577,20 +591,23 @@ inline void DataConvert::timeToString( long long timevalue, char* buf, unsigned if ((timevalue & 0xffffff) > 0) { - snprintf( buf, buflen, "%02d:%02d:%02d.%d", - hour, - (unsigned)((timevalue >> 32) & 0xff), - (unsigned)((timevalue >> 24) & 0xff), - (unsigned)((timevalue) & 0xffffff) - ); + msec = (unsigned)((timevalue) & 0xffffff); } - else + snprintf( buf, buflen, "%02d:%02d:%02d", + hour, + (unsigned)((timevalue >> 32) & 0xff), + (unsigned)((timevalue >> 24) & 0xff) + ); + + if (msec || decimals) { - snprintf( buf, buflen, "%02d:%02d:%02d", - hour, - (unsigned)((timevalue >> 32) & 0xff), - (unsigned)((timevalue >> 24) & 0xff) - ); + size_t start = strlen(buf); + snprintf(buf + strlen(buf), buflen - start, ".%d", msec); + // Pad end with zeros + if (strlen(buf) - start < (size_t)decimals) + { + snprintf(buf + strlen(buf), buflen - strlen(buf), "%0*d", (int)(decimals - (strlen(buf) - start) + 1), 0); + } } } diff --git a/utils/funcexp/func_add_time.cpp b/utils/funcexp/func_add_time.cpp index 6fa7a5c19..0fa379bfd 100644 --- a/utils/funcexp/func_add_time.cpp +++ b/utils/funcexp/func_add_time.cpp @@ -80,16 +80,15 @@ int64_t addTime(DateTime& dt1, Time& dt2) } hour = (signed)(dt1.hour + dt2.hour + min / 60); - dt.hour = tmp = hour % 24; -// if (tmp < -1) - if (tmp < 0) // fix for subtime dlh + if ((hour < 0) || (hour > 23)) { - dt.hour = tmp + 24; - dt2.day--; + dt.hour = hour % 24; + dt2.day = hour / 24; } - day = (signed)(dt1.day + dt2.day + hour / 24); + day = (signed)(dt1.day + dt2.day); + if (isLeapYear(dt1.year) && dt1.month == 2) day--; @@ -97,7 +96,7 @@ int64_t addTime(DateTime& dt1, Time& dt2) month = dt1.month; int addyear = 0; - if (dt2.day < 0 || dt2.hour < 0) + if (day < 0) { int monthSave = month; @@ -106,7 +105,7 @@ int64_t addTime(DateTime& dt1, Time& dt2) month = (month == 1 ? 12 : month - 1); for (; day <= 0 && month > 0; month--) - day += getDaysInMonth(month); + day += getDaysInMonth(month, dt1.year); month++; // month=12; @@ -119,10 +118,10 @@ int64_t addTime(DateTime& dt1, Time& dt2) { int monthSave = month; - while (day > getDaysInMonth(month)) + while (day > getDaysInMonth(month, dt1.year)) { - for (; day > getDaysInMonth(month) && month <= 12; month++) - day -= getDaysInMonth(month); + for (; day > getDaysInMonth(month, dt1.year) && month <= 12; month++) + day -= getDaysInMonth(month, dt1.year); if (month > 12) month = 1; @@ -175,8 +174,23 @@ int64_t addTime(Time& dt1, Time& dt2) dt2.hour--; } - hour = (signed)(dt1.hour + dt2.hour + min / 60); - dt.hour = tmp = hour % 838; + dt.hour = (signed)(dt1.hour + dt2.hour + min / 60); + + // Saturation + if (dt.hour > 838) + { + dt.hour = 838; + dt.minute = 59; + dt.second = 59; + dt.msecond = 999999; + } + else if (dt.hour < -838) + { + dt.hour = -838; + dt.minute = 59; + dt.second = 59; + dt.msecond = 999999; + } return *(reinterpret_cast(&dt)); } @@ -226,6 +240,13 @@ int64_t Func_add_time::getDatetimeIntVal(rowgroup::Row& row, if (isNull) return -1; + // Adding a zero date to a time is always NULL + if (val1 == 0) + { + isNull = true; + return -1; + } + const string& val2 = parm[1]->data()->getStrVal(row, isNull); int sign = parm[2]->data()->getIntVal(row, isNull); DateTime dt1; @@ -311,7 +332,7 @@ int64_t Func_add_time::getTimeIntVal(rowgroup::Row& row, const string& val2 = parm[1]->data()->getStrVal(row, isNull); int sign = parm[2]->data()->getIntVal(row, isNull); Time dt1; - dt1.hour = (val1 >> 40) & 0xff; + dt1.hour = (val1 >> 40) & 0xfff; dt1.minute = (val1 >> 32) & 0xff; dt1.second = (val1 >> 24) & 0xff; dt1.msecond = val1 & 0xffffff; diff --git a/utils/funcexp/func_between.cpp b/utils/funcexp/func_between.cpp index 8398593c4..f23d67826 100644 --- a/utils/funcexp/func_between.cpp +++ b/utils/funcexp/func_between.cpp @@ -158,20 +158,21 @@ inline bool getBool(rowgroup::Row& row, case execplan::CalpontSystemCatalog::TIME: { - int64_t val = pm[0]->data()->getTimeIntVal(row, isNull); + // Shift out unused day for compare + int64_t val = pm[0]->data()->getTimeIntVal(row, isNull) << 12; if (notBetween) { - if (!numericGE(val, pm[1]->data()->getTimeIntVal(row, isNull)) && !isNull) + if (!numericGE(val, pm[1]->data()->getTimeIntVal(row, isNull) << 12) && !isNull) return true; isNull = false; - return (!numericLE(val, pm[2]->data()->getTimeIntVal(row, isNull)) && !isNull); + return (!numericLE(val, pm[2]->data()->getTimeIntVal(row, isNull) << 12) && !isNull); } return !isNull && - numericGE(val, pm[1]->data()->getTimeIntVal(row, isNull)) && - numericLE(val, pm[2]->data()->getTimeIntVal(row, isNull)); + numericGE(val, pm[1]->data()->getTimeIntVal(row, isNull) << 12) && + numericLE(val, pm[2]->data()->getTimeIntVal(row, isNull) << 12); } case execplan::CalpontSystemCatalog::DOUBLE: @@ -265,7 +266,8 @@ CalpontSystemCatalog::ColType Func_between::operationType( FunctionParm& fp, Cal fp[i]->data()->resultType().colDataType != CalpontSystemCatalog::TEXT && fp[i]->data()->resultType().colDataType != CalpontSystemCatalog::VARCHAR) || ct.colDataType == CalpontSystemCatalog::DATE || - ct.colDataType == CalpontSystemCatalog::DATETIME) + ct.colDataType == CalpontSystemCatalog::DATETIME || + ct.colDataType == CalpontSystemCatalog::TIME) { allString = false; } @@ -293,6 +295,23 @@ CalpontSystemCatalog::ColType Func_between::operationType( FunctionParm& fp, Cal } } } + else if (op.operationType().colDataType == CalpontSystemCatalog::TIME) + { + ConstantColumn* cc = NULL; + + for (uint32_t i = 1; i < fp.size(); i++) + { + cc = dynamic_cast(fp[i]->data()); + + if (cc) + { + Result result = cc->result(); + result.intVal = dataconvert::DataConvert::timeToInt(result.strVal); + cc->result(result); + } + } + } + return ct; } diff --git a/utils/funcexp/func_extract.cpp b/utils/funcexp/func_extract.cpp index d5a6cc5b5..67fc70a4d 100644 --- a/utils/funcexp/func_extract.cpp +++ b/utils/funcexp/func_extract.cpp @@ -131,6 +131,90 @@ long long dateGet( uint64_t time, IntervalColumn::interval_type unit, bool dateT throw runtime_error("unit type is not supported: " + unit); }; } + +long long timeGet( uint64_t time, IntervalColumn::interval_type unit ) +{ + int32_t hour = 0, + min = 0, + sec = 0, + msec = 0, + day = 0; + + min = (int32_t)((time >> 32) & 0xff); + sec = (int32_t)((time >> 24) & 0xff); + msec = (int32_t)((time & 0xfffff)); + + // If negative, mask so it doesn't turn positive + int64_t mask = 0; + if ((time >> 40) & 0x800) + mask = 0xfffffffffffff000; + hour = mask | ((time >> 40) & 0xfff); + + // Always positive! + day = abs(hour / 24); + + switch ( unit ) + { + case IntervalColumn::INTERVAL_YEAR: + case IntervalColumn::INTERVAL_MONTH: + return 0; + + case IntervalColumn::INTERVAL_DAY: + return day; + + case IntervalColumn::INTERVAL_HOUR: + return hour; + + case IntervalColumn::INTERVAL_MINUTE: + return min; + + case IntervalColumn::INTERVAL_SECOND: + return sec; + + case IntervalColumn::INTERVAL_MICROSECOND: + return msec; + + case IntervalColumn::INTERVAL_QUARTER: + case IntervalColumn::INTERVAL_WEEK: + case IntervalColumn::INTERVAL_YEAR_MONTH: + return 0; + + case IntervalColumn::INTERVAL_DAY_HOUR: + return (day * 100) + hour; + + case IntervalColumn::INTERVAL_DAY_MINUTE: + return (day * 10000) + (hour * 100) + min; + + case IntervalColumn::INTERVAL_DAY_SECOND: + return (day * 1000000) + (hour * 10000) + (min * 100) + sec; + + case IntervalColumn::INTERVAL_HOUR_MINUTE: + return (hour * 100) + min; + + case IntervalColumn::INTERVAL_HOUR_SECOND: + return (hour * 10000) + (min * 100) + sec; + + case IntervalColumn::INTERVAL_MINUTE_SECOND: + return (min * 100) + sec; + + case IntervalColumn::INTERVAL_DAY_MICROSECOND: + return (((day * 1000000) + (hour * 10000) + (min * 100) + sec) * 1000000) + msec; + + case IntervalColumn::INTERVAL_HOUR_MICROSECOND: + return (((hour * 10000) + (min * 100) + sec) * 1000000) + msec; + + case IntervalColumn::INTERVAL_MINUTE_MICROSECOND: + return (((min * 100) + sec) * 1000000) + msec; + + case IntervalColumn::INTERVAL_SECOND_MICROSECOND: + return (sec * 1000000) + msec; + + default: + throw runtime_error("unit type is not supported: " + unit); + }; +} + + } namespace funcexp @@ -148,16 +232,21 @@ int64_t Func_extract::getIntVal(rowgroup::Row& row, { IntervalColumn::interval_type unit = static_cast(parm[1]->data()->getIntVal(row, isNull)); uint64_t time; + bool isTime = false; //@bug4678 handle conversion from non date/datetime datatype switch (parm[0]->data()->resultType().colDataType) { case CalpontSystemCatalog::DATE: case CalpontSystemCatalog::DATETIME: - case CalpontSystemCatalog::TIME: time = parm[0]->data()->getDatetimeIntVal(row, isNull); break; + case CalpontSystemCatalog::TIME: + time = parm[0]->data()->getTimeIntVal(row, isNull); + isTime = true; + break; + case CalpontSystemCatalog::VARCHAR: case CalpontSystemCatalog::CHAR: case CalpontSystemCatalog::TEXT: @@ -182,7 +271,11 @@ int64_t Func_extract::getIntVal(rowgroup::Row& row, time = parm[0]->data()->getIntVal(row, isNull); } - long long value = dateGet( time, unit, false ); + long long value; + if (isTime) + value = timeGet( time, unit ); + else + value = dateGet( time, unit, false ); return value; } diff --git a/utils/funcexp/func_greatest.cpp b/utils/funcexp/func_greatest.cpp index 50ff4204a..11650ddca 100644 --- a/utils/funcexp/func_greatest.cpp +++ b/utils/funcexp/func_greatest.cpp @@ -211,13 +211,14 @@ int64_t Func_greatest::getTimeIntVal(rowgroup::Row& row, bool& isNull, execplan::CalpontSystemCatalog::ColType& ct) { - int64_t str = fp[0]->data()->getTimeIntVal(row, isNull); + // Strip off unused day + int64_t str = fp[0]->data()->getTimeIntVal(row, isNull) << 12; int64_t greatestStr = str; for (uint32_t i = 1; i < fp.size(); i++) { - int64_t str1 = fp[i]->data()->getTimeIntVal(row, isNull); + int64_t str1 = fp[i]->data()->getTimeIntVal(row, isNull) << 12; if ( greatestStr < str1 ) greatestStr = str1; diff --git a/utils/funcexp/func_hour.cpp b/utils/funcexp/func_hour.cpp index b9e3da9ec..353d0cc4e 100644 --- a/utils/funcexp/func_hour.cpp +++ b/utils/funcexp/func_hour.cpp @@ -48,6 +48,7 @@ int64_t Func_hour::getIntVal(rowgroup::Row& row, CalpontSystemCatalog::ColType& op_ct) { int64_t val = 0; + bool isTime = false; switch (parm[0]->data()->resultType().colDataType) { @@ -110,7 +111,8 @@ int64_t Func_hour::getIntVal(rowgroup::Row& row, case execplan::CalpontSystemCatalog::TIME: { - val = parm[0]->data()->getDatetimeIntVal(row, isNull); + isTime = true; + val = parm[0]->data()->getTimeIntVal(row, isNull); break; } @@ -123,10 +125,20 @@ int64_t Func_hour::getIntVal(rowgroup::Row& row, if (isNull) return -1; - if ( val < 1000000000 ) - return 0; + if (isTime) + { + // If negative, mask so it doesn't turn positive + int64_t mask = 0; + if ((val >> 40) & 0x800) + mask = 0xfffffffffffff000; + val = mask | ((val >> 40) & 0xfff); + } + else + { + val = (val >> 32) & 0x3f; + } - return (uint32_t)((val >> 32) & 0x3f); + return val; } diff --git a/utils/funcexp/func_least.cpp b/utils/funcexp/func_least.cpp index ef71e44e6..70a64f267 100644 --- a/utils/funcexp/func_least.cpp +++ b/utils/funcexp/func_least.cpp @@ -187,13 +187,14 @@ int64_t Func_least::getTimeIntVal(rowgroup::Row& row, bool& isNull, execplan::CalpontSystemCatalog::ColType& op_ct) { - int64_t str = fp[0]->data()->getTimeIntVal(row, isNull); + // Strip off unused day + int64_t str = fp[0]->data()->getTimeIntVal(row, isNull) << 12; int64_t leastStr = str; for (uint32_t i = 1; i < fp.size(); i++) { - int64_t str1 = fp[i]->data()->getTimeIntVal(row, isNull); + int64_t str1 = fp[i]->data()->getTimeIntVal(row, isNull) << 12; if ( leastStr > str1 ) leastStr = str1; diff --git a/utils/funcexp/func_time.cpp b/utils/funcexp/func_time.cpp index 8aab1389b..bd28d6e6c 100644 --- a/utils/funcexp/func_time.cpp +++ b/utils/funcexp/func_time.cpp @@ -59,7 +59,7 @@ string Func_time::getStrVal(rowgroup::Row& row, case execplan::CalpontSystemCatalog::TINYINT: case execplan::CalpontSystemCatalog::SMALLINT: { - val = dataconvert::DataConvert::intToDatetime(parm[0]->data()->getIntVal(row, isNull)); + val = dataconvert::DataConvert::intToTime(parm[0]->data()->getIntVal(row, isNull)); if (val == -1) isNull = true; @@ -73,7 +73,7 @@ string Func_time::getStrVal(rowgroup::Row& row, { if (parm[0]->data()->resultType().scale) { - val = dataconvert::DataConvert::intToDatetime(parm[0]->data()->getIntVal(row, isNull)); + val = dataconvert::DataConvert::intToTime(parm[0]->data()->getIntVal(row, isNull)); if (val == -1) isNull = true; @@ -96,7 +96,7 @@ string Func_time::getStrVal(rowgroup::Row& row, case execplan::CalpontSystemCatalog::CHAR: case execplan::CalpontSystemCatalog::TEXT: { - val = dataconvert::DataConvert::stringToDatetime(parm[0]->data()->getStrVal(row, isNull)); + val = dataconvert::DataConvert::stringToTime(parm[0]->data()->getStrVal(row, isNull)); if (val == -1) isNull = true; @@ -106,16 +106,14 @@ string Func_time::getStrVal(rowgroup::Row& row, break; } - case execplan::CalpontSystemCatalog::DATE: + case execplan::CalpontSystemCatalog::TIME: { - val = parm[0]->data()->getDatetimeIntVal(row, isNull); + val = parm[0]->data()->getTimeIntVal(row, isNull); break; } - - case execplan::CalpontSystemCatalog::TIME: case execplan::CalpontSystemCatalog::DATETIME: { - val = parm[0]->data()->getDatetimeIntVal(row, isNull); + val = parm[0]->data()->getTimeIntVal(row, isNull); break; } @@ -129,9 +127,9 @@ string Func_time::getStrVal(rowgroup::Row& row, return ""; char buf[30] = {'\0'}; - dataconvert::DataConvert::datetimeToString(val, buf, sizeof(buf)); + dataconvert::DataConvert::timeToString(val, buf, sizeof(buf)); string time(buf); - return time.substr(11, 80); + return time; } int64_t Func_time::getIntVal(rowgroup::Row& row, @@ -139,7 +137,7 @@ int64_t Func_time::getIntVal(rowgroup::Row& row, bool& isNull, execplan::CalpontSystemCatalog::ColType& op_ct) { - return dataconvert::DataConvert::datetimeToInt(getStrVal(row, fp, isNull, op_ct)); + return dataconvert::DataConvert::timeToInt(getStrVal(row, fp, isNull, op_ct)); } double Func_time::getDoubleVal(rowgroup::Row& row, From aae2810d73099a7d33b674a8df384ab868ca0632 Mon Sep 17 00:00:00 2001 From: Andrew Hutchings Date: Mon, 30 Apr 2018 07:56:47 +0100 Subject: [PATCH 42/49] MCOL-392 Fix time() for a DATETIME value --- utils/dataconvert/dataconvert.h | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/utils/dataconvert/dataconvert.h b/utils/dataconvert/dataconvert.h index 8c35bec4b..0992cd843 100644 --- a/utils/dataconvert/dataconvert.h +++ b/utils/dataconvert/dataconvert.h @@ -216,7 +216,7 @@ struct Time {} Time(signed d, signed h, signed min, signed sec, signed msec) : - msecond(sec), second(sec), minute(min), hour(h), day(d) {} + msecond(msec), second(sec), minute(min), hour(h), day(d) {} int64_t convertToMySQLint() const; void reset(); From fc05a9c6c230fb30b14725e4ec1aae1c73e7388a Mon Sep 17 00:00:00 2001 From: Andrew Hutchings Date: Mon, 30 Apr 2018 08:08:42 +0100 Subject: [PATCH 43/49] MCOL-392 Fix greatest() and least() for time --- utils/funcexp/func_greatest.cpp | 12 ++++++++---- utils/funcexp/func_least.cpp | 12 ++++++++---- 2 files changed, 16 insertions(+), 8 deletions(-) diff --git a/utils/funcexp/func_greatest.cpp b/utils/funcexp/func_greatest.cpp index 11650ddca..2d7dd15d6 100644 --- a/utils/funcexp/func_greatest.cpp +++ b/utils/funcexp/func_greatest.cpp @@ -212,16 +212,20 @@ int64_t Func_greatest::getTimeIntVal(rowgroup::Row& row, execplan::CalpontSystemCatalog::ColType& ct) { // Strip off unused day - int64_t str = fp[0]->data()->getTimeIntVal(row, isNull) << 12; + int64_t greatestStr = fp[0]->data()->getTimeIntVal(row, isNull); - int64_t greatestStr = str; + int64_t str = greatestStr << 12; for (uint32_t i = 1; i < fp.size(); i++) { - int64_t str1 = fp[i]->data()->getTimeIntVal(row, isNull) << 12; + int64_t str1 = fp[i]->data()->getTimeIntVal(row, isNull); + int64_t str2 = str1 << 12; - if ( greatestStr < str1 ) + if ( str < str1 ) + { greatestStr = str1; + str = str2; + } } return greatestStr; diff --git a/utils/funcexp/func_least.cpp b/utils/funcexp/func_least.cpp index 70a64f267..2f645af5f 100644 --- a/utils/funcexp/func_least.cpp +++ b/utils/funcexp/func_least.cpp @@ -188,16 +188,20 @@ int64_t Func_least::getTimeIntVal(rowgroup::Row& row, execplan::CalpontSystemCatalog::ColType& op_ct) { // Strip off unused day - int64_t str = fp[0]->data()->getTimeIntVal(row, isNull) << 12; + int64_t leastStr = fp[0]->data()->getTimeIntVal(row, isNull); - int64_t leastStr = str; + int64_t str = leastStr << 12; for (uint32_t i = 1; i < fp.size(); i++) { - int64_t str1 = fp[i]->data()->getTimeIntVal(row, isNull) << 12; + int64_t str1 = fp[i]->data()->getTimeIntVal(row, isNull); + int64_t str2 = str1 << 12; - if ( leastStr > str1 ) + if ( str > str1 ) + { leastStr = str1; + str = str2; + } } return leastStr; From a5f2f80bed670c9e2f07fa0c56c7899ad417793c Mon Sep 17 00:00:00 2001 From: Andrew Hutchings Date: Mon, 30 Apr 2018 10:41:52 +0100 Subject: [PATCH 44/49] MCOL-392 Fix case --- utils/funcexp/func_add_time.cpp | 2 +- utils/funcexp/func_case.cpp | 49 ++++++++++++++++++++++++++++++--- 2 files changed, 46 insertions(+), 5 deletions(-) diff --git a/utils/funcexp/func_add_time.cpp b/utils/funcexp/func_add_time.cpp index 0fa379bfd..bb32dc79d 100644 --- a/utils/funcexp/func_add_time.cpp +++ b/utils/funcexp/func_add_time.cpp @@ -146,7 +146,7 @@ int64_t addTime(Time& dt1, Time& dt2) dt.second = 0; dt.msecond = 0; - int64_t hour, min, sec, msec, tmp; + int64_t min, sec, msec, tmp; msec = (signed)(dt1.msecond + dt2.msecond); dt.msecond = tmp = msec % 1000000; diff --git a/utils/funcexp/func_case.cpp b/utils/funcexp/func_case.cpp index 46a479491..f9d70cb75 100644 --- a/utils/funcexp/func_case.cpp +++ b/utils/funcexp/func_case.cpp @@ -65,8 +65,6 @@ inline uint64_t simple_case_cmp(Row& row, case execplan::CalpontSystemCatalog::INT: case execplan::CalpontSystemCatalog::BIGINT: case execplan::CalpontSystemCatalog::DATE: - case execplan::CalpontSystemCatalog::DATETIME: - case execplan::CalpontSystemCatalog::TIME: { int64_t ev = parm[n]->data()->getIntVal(row, isNull); @@ -87,6 +85,49 @@ inline uint64_t simple_case_cmp(Row& row, break; } + case execplan::CalpontSystemCatalog::DATETIME: + { + int64_t ev = parm[n]->data()->getDatetimeIntVal(row, isNull); + + if (isNull) + break; + + for (i = 1; i <= whereCount; i++) + { + if (ev == parm[i]->data()->getDatetimeIntVal(row, isNull) && !isNull) + { + foundIt = true; + break; + } + else + isNull = false; + } + + break; + } + + case execplan::CalpontSystemCatalog::TIME: + { + int64_t ev = parm[n]->data()->getTimeIntVal(row, isNull); + + if (isNull) + break; + + for (i = 1; i <= whereCount; i++) + { + if (ev == parm[i]->data()->getTimeIntVal(row, isNull) && !isNull) + { + foundIt = true; + break; + } + else + isNull = false; + } + + break; + } + + case execplan::CalpontSystemCatalog::UBIGINT: case execplan::CalpontSystemCatalog::UINT: case execplan::CalpontSystemCatalog::UMEDINT: @@ -514,7 +555,7 @@ int64_t Func_simple_case::getTimeIntVal(rowgroup::Row& row, if (isNull) return joblist::TIMENULL; - return parm[i + 1]->data()->getTimeIntVal(row, isNull); + return parm[i]->data()->getTimeIntVal(row, isNull); } @@ -653,7 +694,7 @@ int64_t Func_searched_case::getTimeIntVal(rowgroup::Row& row, if (isNull) return joblist::TIMENULL; - return parm[i + 1]->data()->getTimeIntVal(row, isNull); + return parm[i]->data()->getTimeIntVal(row, isNull); } } // namespace funcexp From 3c090647af01dd3c0c5a9de25c79babadf160438 Mon Sep 17 00:00:00 2001 From: Andrew Hutchings Date: Mon, 30 Apr 2018 11:34:58 +0100 Subject: [PATCH 45/49] MCOL-392 Fix microsecond and nullif for TIME --- utils/funcexp/func_microsecond.cpp | 5 ++++ utils/funcexp/func_nullif.cpp | 37 ++++++++---------------------- 2 files changed, 14 insertions(+), 28 deletions(-) diff --git a/utils/funcexp/func_microsecond.cpp b/utils/funcexp/func_microsecond.cpp index e0a8e6d11..f292e26a5 100644 --- a/utils/funcexp/func_microsecond.cpp +++ b/utils/funcexp/func_microsecond.cpp @@ -63,6 +63,11 @@ int64_t Func_microsecond::getIntVal(rowgroup::Row& row, microSecond = (uint32_t)((val & 0xfffff)); break; + case CalpontSystemCatalog::TIME: + val = parm[0]->data()->getIntVal(row, isNull); + microSecond = (uint32_t)((val & 0xffffff)); + break; + case CalpontSystemCatalog::CHAR: case CalpontSystemCatalog::TEXT: case CalpontSystemCatalog::VARCHAR: diff --git a/utils/funcexp/func_nullif.cpp b/utils/funcexp/func_nullif.cpp index 23362f4ef..30d4682e5 100644 --- a/utils/funcexp/func_nullif.cpp +++ b/utils/funcexp/func_nullif.cpp @@ -127,8 +127,11 @@ int64_t Func_nullif::getIntVal(rowgroup::Row& row, { exp2 = parm[1]->data()->getDatetimeIntVal(row, isNull); - if (parm[0]->data()->resultType().colDataType == - execplan::CalpontSystemCatalog::DATE) + if ((parm[0]->data()->resultType().colDataType == + execplan::CalpontSystemCatalog::DATE) || + (parm[0]->data()->resultType().colDataType == + execplan::CalpontSystemCatalog::DATETIME)) + { // NULLIF arg0 is DATE, arg1 is DATETIME, // Upgrade arg1 to datetime @@ -159,8 +162,10 @@ int64_t Func_nullif::getIntVal(rowgroup::Row& row, { exp2 = parm[1]->data()->getTimeIntVal(row, isNull); - if (parm[0]->data()->resultType().colDataType == - execplan::CalpontSystemCatalog::DATETIME) + if ((parm[0]->data()->resultType().colDataType == + execplan::CalpontSystemCatalog::DATETIME) || + (parm[0]->data()->resultType().colDataType == + execplan::CalpontSystemCatalog::TIME)) { // NULLIF arg0 is DATETIME, arg1 is TIME, // Upgrade arg1 to time @@ -445,18 +450,6 @@ int64_t Func_nullif::getDatetimeIntVal(rowgroup::Row& row, case execplan::CalpontSystemCatalog::VARCHAR: case execplan::CalpontSystemCatalog::CHAR: case execplan::CalpontSystemCatalog::TEXT: - { - exp2 = parm[1]->data()->getIntVal(row, isNull); - - if (isNull) - { - isNull = false; - return exp1; - } - - break; - } - case execplan::CalpontSystemCatalog::DATE: { // Upgrade to datetime for proper comparison @@ -522,18 +515,6 @@ int64_t Func_nullif::getTimeIntVal(rowgroup::Row& row, case execplan::CalpontSystemCatalog::VARCHAR: case execplan::CalpontSystemCatalog::CHAR: case execplan::CalpontSystemCatalog::TEXT: - { - exp2 = parm[1]->data()->getIntVal(row, isNull); - - if (isNull) - { - isNull = false; - return exp1; - } - - break; - } - case execplan::CalpontSystemCatalog::TIME: case execplan::CalpontSystemCatalog::DATETIME: { From dfc351b73068b5b7018ceeca4fe05436cf788016 Mon Sep 17 00:00:00 2001 From: Andrew Hutchings Date: Mon, 30 Apr 2018 22:08:10 +0100 Subject: [PATCH 46/49] MCOL-392 fix negative zero hours Also fix some functions that were not behaving correctly --- dbcon/execplan/treenode.h | 6 +-- utils/dataconvert/dataconvert.cpp | 74 ++++++++++++++++++++++++------ utils/dataconvert/dataconvert.h | 49 ++++++++++++++++---- utils/funcexp/func_bitand.cpp | 2 + utils/funcexp/func_extract.cpp | 3 ++ utils/funcexp/func_greatest.cpp | 2 +- utils/funcexp/func_hour.cpp | 8 ++++ utils/funcexp/func_least.cpp | 2 +- utils/funcexp/func_second.cpp | 1 + utils/funcexp/func_time_to_sec.cpp | 37 +++++++++++---- 10 files changed, 148 insertions(+), 36 deletions(-) diff --git a/dbcon/execplan/treenode.h b/dbcon/execplan/treenode.h index 08b8b43c8..3b2dd7200 100644 --- a/dbcon/execplan/treenode.h +++ b/dbcon/execplan/treenode.h @@ -996,12 +996,12 @@ inline int64_t TreeNode::getDatetimeIntVal() memcpy(&tt, &fResult.intVal, 8); // Note, this should probably be current date +/- time - if (tt.hour > 23) + if ((tt.hour > 23) && (!tt.is_neg)) { day = tt.hour / 24; tt.hour = tt.hour % 24; } - else if (tt.hour < 0) + else if ((tt.hour < 0) || (tt.is_neg)) { tt.hour = 0; } @@ -1023,7 +1023,7 @@ inline int64_t TreeNode::getTimeIntVal() dataconvert::DateTime dt; memcpy(&dt, &fResult.intVal, 8); - dataconvert::Time tt(0, dt.hour, dt.minute, dt.second, dt.msecond); + dataconvert::Time tt(0, dt.hour, dt.minute, dt.second, dt.msecond, false); memcpy(&fResult.intVal, &tt, 8); return fResult.intVal; } diff --git a/utils/dataconvert/dataconvert.cpp b/utils/dataconvert/dataconvert.cpp index 41dff9aa9..0ba1e4a87 100644 --- a/utils/dataconvert/dataconvert.cpp +++ b/utils/dataconvert/dataconvert.cpp @@ -1005,6 +1005,7 @@ bool mysql_str_to_time( const string& input, Time& output ) output.minute = 59; output.second = 59; output.msecond = 999999; + output.is_neg = 0; } else if (hour < -838) { @@ -1012,6 +1013,7 @@ bool mysql_str_to_time( const string& input, Time& output ) output.minute = 59; output.second = 59; output.msecond = 999999; + output.is_neg = 1; } // If neither of the above match then we return a 0 time else @@ -1025,6 +1027,7 @@ bool mysql_str_to_time( const string& input, Time& output ) output.minute = min; output.second = sec; output.msecond = usec; + output.is_neg = isNeg; return true; } @@ -1914,12 +1917,18 @@ int64_t DataConvert::convertColumnTime( inMinute = 0; inSecond = 0; inMicrosecond = 0; + bool isNeg = false; if ( datetimeFormat != CALPONTTIME_ENUM ) { status = -1; return value; } + if (p[0] == '-') + { + isNeg = true; + } + errno = 0; p = strtok_r(p, ":.", &savePoint); @@ -1981,6 +1990,7 @@ int64_t DataConvert::convertColumnTime( atime.minute = inMinute; atime.second = inSecond; atime.msecond = inMicrosecond; + atime.is_neg = isNeg; memcpy( &value, &atime, 8); } @@ -1994,6 +2004,7 @@ int64_t DataConvert::convertColumnTime( atime.minute = 59; atime.second = 59; atime.msecond = 999999; + atime.is_neg = false; memcpy( &value, &atime, 8); } else if (inHour < -838) @@ -2003,6 +2014,7 @@ int64_t DataConvert::convertColumnTime( atime.minute = 59; atime.second = 59; atime.msecond = 999999; + atime.is_neg = false; memcpy( &value, &atime, 8); } // If neither of the above match then we return a 0 time @@ -2084,8 +2096,15 @@ std::string DataConvert::timeToString( long long timevalue, long decimals ) Time dt(timevalue); const int TIMETOSTRING_LEN = 19; // (-H)HH:MM:SS.mmmmmm\0 char buf[TIMETOSTRING_LEN]; + char* outbuf = buf; - sprintf(buf, "%02d:%02d:%02d", dt.hour, dt.minute, dt.second); + if ((dt.hour >= 0) && dt.is_neg) + { + outbuf[0] = '-'; + outbuf++; + } + + sprintf(outbuf, "%02d:%02d:%02d", dt.hour, dt.minute, dt.second); if (dt.msecond && decimals) { size_t start = strlen(buf); @@ -2128,7 +2147,9 @@ std::string DataConvert::timeToString1( long long datetimevalue ) const int TIMETOSTRING1_LEN = 14; // HHMMSSmmmmmm\0 char buf[TIMETOSTRING1_LEN]; - sprintf(buf, "%02d%02d%02d%06d", dt.hour, dt.minute, dt.second, dt.msecond); + char* outbuf = buf; + + sprintf(outbuf, "%02d%02d%02d%06d", dt.hour, dt.minute, dt.second, dt.msecond); return buf; } @@ -2633,7 +2654,9 @@ int64_t DataConvert::intToDatetime(int64_t data, bool* date) int64_t DataConvert::intToTime(int64_t data) { char buf[21] = {0}; + char* bufread = buf; Time atime; + bool isNeg = false; if (data == 0) { @@ -2641,6 +2664,7 @@ int64_t DataConvert::intToTime(int64_t data) atime.minute = 0; atime.second = 0; atime.msecond = 0; + atime.is_neg = 0; return *(reinterpret_cast(&atime)); } @@ -2650,24 +2674,37 @@ int64_t DataConvert::intToTime(int64_t data) string hour, min, sec, msec; int64_t h = 0, minute = 0, s = 0, ms = 0; - switch (strlen(buf)) + if (bufread[0] == '-') { + isNeg = true; + bufread++; + } + + switch (strlen(bufread)) + { + case 7: + hour = string(bufread, 3); + min = string(bufread + 2, 2); + sec = string(bufread + 4, 2); + msec = string(bufread + 6, 6); + break; + case 6: - hour = string(buf, 2); - min = string(buf + 2, 2); - sec = string(buf + 4, 2); - msec = string(buf + 6, 6); + hour = string(bufread, 2); + min = string(bufread + 2, 2); + sec = string(bufread + 4, 2); + msec = string(bufread + 6, 6); break; case 4: - min = string(buf, 2); - sec = string(buf + 2, 2); - msec = string(buf + 4, 6); + min = string(bufread, 2); + sec = string(bufread + 2, 2); + msec = string(bufread + 4, 6); break; case 2: - sec = string(buf, 2); - msec = string(buf + 2, 6); + sec = string(bufread, 2); + msec = string(bufread + 2, 6); break; default: @@ -2686,6 +2723,7 @@ int64_t DataConvert::intToTime(int64_t data) atime.minute = minute; atime.second = s; atime.msecond = ms; + atime.is_neg = isNeg; return *(reinterpret_cast(&atime)); } @@ -2697,11 +2735,20 @@ int64_t DataConvert::stringToTime(const string& data) // -838 <= H <= 838 uint64_t min = 0, sec = 0, msec = 0; int64_t day = -1, hour = 0; + bool isNeg = false; string time, hms, ms; char* end = NULL; + + size_t pos = data.find("-"); + + if (pos != string::npos) + { + isNeg = true; + } + // Day - size_t pos = data.find(" "); + pos = data.find(" "); if (pos != string::npos) { @@ -2762,6 +2809,7 @@ int64_t DataConvert::stringToTime(const string& data) atime.minute = min; atime.second = sec; atime.msecond = msec; + atime.is_neg = isNeg; return *(reinterpret_cast(&atime)); } diff --git a/utils/dataconvert/dataconvert.h b/utils/dataconvert/dataconvert.h index 0992cd843..01e530709 100644 --- a/utils/dataconvert/dataconvert.h +++ b/utils/dataconvert/dataconvert.h @@ -197,14 +197,17 @@ struct Time signed second : 8; signed minute : 8; signed hour : 12; - signed day : 12; + signed day : 11; + signed is_neg : 1; // NULL column value = 0xFFFFFFFFFFFFFFFE Time() : msecond (0xFFFFFE), second (0xFF), minute (0xFF), hour (0xFFF), - day (0xFFF) {} + day (0x7FF), + is_neg (0b1) + {} // Construct a Time from a 64 bit integer InfiniDB time. Time(int64_t val) : @@ -212,11 +215,16 @@ struct Time second((val >> 24) & 0xff), minute((val >> 32) & 0xff), hour((val >> 40) & 0xfff), - day((val >> 52) & 0xfff) + day((val >> 52) & 0x7ff), + is_neg(val >> 63) {} - Time(signed d, signed h, signed min, signed sec, signed msec) : - msecond(msec), second(sec), minute(min), hour(h), day(d) {} + Time(signed d, signed h, signed min, signed sec, signed msec, bool neg) : + msecond(msec), second(sec), minute(min), hour(h), day(d), is_neg(neg) + { + if (h < 0) + is_neg = 0b1; + } int64_t convertToMySQLint() const; void reset(); @@ -229,13 +237,25 @@ void Time::reset() second = 0xFF; minute = 0xFF; hour = 0xFFF; - day = 0xFFF; + is_neg = 0b1; + day = 0x7FF; } inline int64_t Time::convertToMySQLint() const { - return (int64_t) (hour * 10000) + (minute * 100) + second; + if ((hour >= 0) && is_neg) + { + return (int64_t) ((hour * 10000) + (minute * 100) + second) * -1; + } + else if (hour >= 0) + { + return (int64_t) (hour * 10000) + (minute * 100) + second; + } + else + { + return (int64_t) (hour * 10000) - (minute * 100) - second; + } } static uint32_t daysInMonth[13] = {31, 28, 31, 30, 31, 30, 31, 31, 30, 31, 30, 31, 0}; @@ -593,6 +613,12 @@ inline void DataConvert::timeToString( long long timevalue, char* buf, unsigned { msec = (unsigned)((timevalue) & 0xffffff); } + if ((hour >= 0) && (timevalue >> 63)) + { + buf[0] = '-'; + buf++; + buflen--; + } snprintf( buf, buflen, "%02d:%02d:%02d", hour, (unsigned)((timevalue >> 32) & 0xff), @@ -636,13 +662,20 @@ inline void DataConvert::timeToString1( long long timevalue, char* buf, unsigned { // Handle negative correctly int hour = 0; - if ((timevalue >> 40) & 0xf00) + if ((timevalue >> 40) & 0x800) { hour = 0xfffff000; } hour |= ((timevalue >> 40) & 0xfff); + if ((hour >= 0) && (timevalue >> 63)) + { + buf[0] = '-'; + buf++; + buflen--; + } + snprintf( buf, buflen, "%02d%02d%02d", hour, (unsigned)((timevalue >> 32) & 0xff), diff --git a/utils/funcexp/func_bitand.cpp b/utils/funcexp/func_bitand.cpp index 32cb846fe..6906a928a 100644 --- a/utils/funcexp/func_bitand.cpp +++ b/utils/funcexp/func_bitand.cpp @@ -172,6 +172,8 @@ int64_t Func_bitand::getIntVal(Row& row, } hour |= ((time >> 40) & 0xfff); + if ((hour >= 0) && (time >> 63)) + hour*= -1; min = (uint32_t)((time >> 32) & 0xff); sec = (uint32_t)((time >> 24) & 0xff); msec = (uint32_t)(time & 0xffffff); diff --git a/utils/funcexp/func_extract.cpp b/utils/funcexp/func_extract.cpp index 67fc70a4d..c77a23d70 100644 --- a/utils/funcexp/func_extract.cpp +++ b/utils/funcexp/func_extract.cpp @@ -150,6 +150,9 @@ long long timeGet( uint64_t time, IntervalColumn::interval_type unit ) mask = 0xfffffffffffff000; hour = mask | ((time >> 40) & 0xfff); + if ((hour >= 0) && (time >> 63)) + hour*= -1; + // Always positive! day = abs(hour / 24); diff --git a/utils/funcexp/func_greatest.cpp b/utils/funcexp/func_greatest.cpp index 2d7dd15d6..80dbd251f 100644 --- a/utils/funcexp/func_greatest.cpp +++ b/utils/funcexp/func_greatest.cpp @@ -221,7 +221,7 @@ int64_t Func_greatest::getTimeIntVal(rowgroup::Row& row, int64_t str1 = fp[i]->data()->getTimeIntVal(row, isNull); int64_t str2 = str1 << 12; - if ( str < str1 ) + if ( str < str2 ) { greatestStr = str1; str = str2; diff --git a/utils/funcexp/func_hour.cpp b/utils/funcexp/func_hour.cpp index 353d0cc4e..631b070c8 100644 --- a/utils/funcexp/func_hour.cpp +++ b/utils/funcexp/func_hour.cpp @@ -128,10 +128,18 @@ int64_t Func_hour::getIntVal(rowgroup::Row& row, if (isTime) { // If negative, mask so it doesn't turn positive + bool isNeg = false; int64_t mask = 0; if ((val >> 40) & 0x800) mask = 0xfffffffffffff000; + + if (!mask && (val >> 63)) + { + isNeg = true; + } val = mask | ((val >> 40) & 0xfff); + if (isNeg) + val*= -1; } else { diff --git a/utils/funcexp/func_least.cpp b/utils/funcexp/func_least.cpp index 2f645af5f..ee5b2afc0 100644 --- a/utils/funcexp/func_least.cpp +++ b/utils/funcexp/func_least.cpp @@ -197,7 +197,7 @@ int64_t Func_least::getTimeIntVal(rowgroup::Row& row, int64_t str1 = fp[i]->data()->getTimeIntVal(row, isNull); int64_t str2 = str1 << 12; - if ( str > str1 ) + if ( str > str2 ) { leastStr = str1; str = str2; diff --git a/utils/funcexp/func_second.cpp b/utils/funcexp/func_second.cpp index 678a1fa61..ae12b157c 100644 --- a/utils/funcexp/func_second.cpp +++ b/utils/funcexp/func_second.cpp @@ -111,6 +111,7 @@ int64_t Func_second::getIntVal(rowgroup::Row& row, case execplan::CalpontSystemCatalog::TIME: { val = parm[0]->data()->getTimeIntVal(row, isNull); + return (uint32_t)((val >> 24) & 0xff); break; } diff --git a/utils/funcexp/func_time_to_sec.cpp b/utils/funcexp/func_time_to_sec.cpp index b87c2995a..2b8d6e3cc 100644 --- a/utils/funcexp/func_time_to_sec.cpp +++ b/utils/funcexp/func_time_to_sec.cpp @@ -49,12 +49,13 @@ int64_t Func_time_to_sec::getIntVal(rowgroup::Row& row, CalpontSystemCatalog::ColType& op_ct) { // assume 256 is enough. assume not allowing incomplete date - uint32_t hour = 0, + int32_t hour = 0, min = 0, sec = 0; bool bIsNegative = false; // Only set to true if CHAR or VARCHAR with a '-' int64_t val = 0; + int64_t mask = 0; dataconvert::Time tval; switch (parm[0]->data()->resultType().colDataType) @@ -64,9 +65,25 @@ int64_t Func_time_to_sec::getIntVal(rowgroup::Row& row, case CalpontSystemCatalog::DATETIME: val = parm[0]->data()->getIntVal(row, isNull); - hour = (uint32_t)((val >> 32) & 0x3f); - min = (uint32_t)((val >> 26) & 0x3f); - sec = (uint32_t)((val >> 20) & 0x3f); + hour = (int32_t)((val >> 32) & 0x3f); + min = (int32_t)((val >> 26) & 0x3f); + sec = (int32_t)((val >> 20) & 0x3f); + break; + + case CalpontSystemCatalog::TIME: + val = parm[0]->data()->getTimeIntVal(row, isNull); + // If negative, mask so it doesn't turn positive + if ((val >> 40) & 0x800) + mask = 0xfffffffffffff000; + + bIsNegative = val >> 63; + hour = (int32_t)(mask | ((val >> 40) & 0xfff)); + if ((hour >= 0) && bIsNegative) + hour*= -1; + else + bIsNegative = false; + min = (int32_t)((val >> 32) & 0xff); + sec = (int32_t)((val >> 24) & 0xff); break; case CalpontSystemCatalog::CHAR: @@ -112,9 +129,9 @@ int64_t Func_time_to_sec::getIntVal(rowgroup::Row& row, } else { - hour = (uint32_t)((val >> 32) & 0x3f); - min = (uint32_t)((val >> 26) & 0x3f); - sec = (uint32_t)((val >> 20) & 0x3f); + hour = (int32_t)((val >> 32) & 0x3f); + min = (int32_t)((val >> 26) & 0x3f); + sec = (int32_t)((val >> 20) & 0x3f); } break; @@ -131,9 +148,9 @@ int64_t Func_time_to_sec::getIntVal(rowgroup::Row& row, } else { - hour = (uint32_t)((val >> 32) & 0x3f); - min = (uint32_t)((val >> 26) & 0x3f); - sec = (uint32_t)((val >> 20) & 0x3f); + hour = (int32_t)((val >> 32) & 0x3f); + min = (int32_t)((val >> 26) & 0x3f); + sec = (int32_t)((val >> 20) & 0x3f); } } From 4ef42860220231e3d6adbba7581b352705ed2939 Mon Sep 17 00:00:00 2001 From: Andrew Hutchings Date: Tue, 1 May 2018 07:25:25 +0100 Subject: [PATCH 47/49] MCOL-392 Fix time_to_sec() for negative time --- utils/funcexp/func_time_to_sec.cpp | 10 +++++++++- 1 file changed, 9 insertions(+), 1 deletion(-) diff --git a/utils/funcexp/func_time_to_sec.cpp b/utils/funcexp/func_time_to_sec.cpp index 2b8d6e3cc..0d804242c 100644 --- a/utils/funcexp/func_time_to_sec.cpp +++ b/utils/funcexp/func_time_to_sec.cpp @@ -161,7 +161,15 @@ int64_t Func_time_to_sec::getIntVal(rowgroup::Row& row, return -1; } - int64_t rtn = (int64_t)(hour * 60 * 60) + (min * 60) + sec; + int64_t rtn; + if (hour < 0) + { + rtn = (int64_t)(hour * 60 * 60) - (min * 60) - sec; + } + else + { + rtn = (int64_t)(hour * 60 * 60) + (min * 60) + sec; + } if (bIsNegative) { From c40903de9bf421c006714543a4927d708ecf07c6 Mon Sep 17 00:00:00 2001 From: Andrew Hutchings Date: Tue, 1 May 2018 09:52:26 +0100 Subject: [PATCH 48/49] MCOL-392 Apply astyle Make this branch apply our style guidelines --- dbcon/execplan/simplefilter.cpp | 1 + dbcon/execplan/treenode.h | 2 + dbcon/joblist/groupconcat.cpp | 4 ++ dbcon/mysql/ha_calpont_dml.cpp | 4 ++ utils/dataconvert/dataconvert.cpp | 17 ++++++++- utils/dataconvert/dataconvert.h | 11 ++++++ utils/funcexp/func_add_time.cpp | 6 +-- utils/funcexp/func_bitand.cpp | 5 ++- utils/funcexp/func_case.cpp | 6 +-- utils/funcexp/func_coalesce.cpp | 6 +-- utils/funcexp/func_extract.cpp | 35 +++++++++-------- utils/funcexp/func_greatest.cpp | 6 +-- utils/funcexp/func_hour.cpp | 5 ++- utils/funcexp/func_if.cpp | 6 +-- utils/funcexp/func_ifnull.cpp | 6 +-- utils/funcexp/func_inet_aton.cpp | 6 +-- utils/funcexp/func_inet_ntoa.cpp | 6 +-- utils/funcexp/func_least.cpp | 6 +-- utils/funcexp/func_nullif.cpp | 14 +++---- utils/funcexp/func_sysdate.cpp | 6 +-- utils/funcexp/func_time.cpp | 1 + utils/funcexp/func_time_to_sec.cpp | 10 +++-- utils/funcexp/func_timediff.cpp | 6 +-- utils/funcexp/functor.h | 6 +-- utils/funcexp/functor_all.h | 52 +++++++++++++------------- utils/funcexp/functor_bool.h | 6 +-- utils/funcexp/functor_dtm.h | 36 +++++++++--------- utils/funcexp/functor_real.h | 6 +-- utils/funcexp/functor_str.h | 12 +++--- utils/rowgroup/rowaggregation.cpp | 2 + writeengine/bulk/we_bulkloadbuffer.cpp | 4 +- 31 files changed, 176 insertions(+), 123 deletions(-) diff --git a/dbcon/execplan/simplefilter.cpp b/dbcon/execplan/simplefilter.cpp index 1da14449c..639a04945 100644 --- a/dbcon/execplan/simplefilter.cpp +++ b/dbcon/execplan/simplefilter.cpp @@ -606,6 +606,7 @@ void SimpleFilter::convertConstant() result.intVal = dataconvert::DataConvert::timeToInt(result.strVal); } } + rcc->result(result); } } diff --git a/dbcon/execplan/treenode.h b/dbcon/execplan/treenode.h index 3b2dd7200..cef9579e9 100644 --- a/dbcon/execplan/treenode.h +++ b/dbcon/execplan/treenode.h @@ -995,6 +995,7 @@ inline int64_t TreeNode::getDatetimeIntVal() int day = 0; memcpy(&tt, &fResult.intVal, 8); + // Note, this should probably be current date +/- time if ((tt.hour > 23) && (!tt.is_neg)) { @@ -1005,6 +1006,7 @@ inline int64_t TreeNode::getDatetimeIntVal() { tt.hour = 0; } + dataconvert::DateTime dt(0, 0, day, tt.hour, tt.minute, tt.second, tt.msecond); memcpy(&fResult.intVal, &dt, 8); return fResult.intVal; diff --git a/dbcon/joblist/groupconcat.cpp b/dbcon/joblist/groupconcat.cpp index 1e84585b3..234fc0a8e 100644 --- a/dbcon/joblist/groupconcat.cpp +++ b/dbcon/joblist/groupconcat.cpp @@ -648,10 +648,12 @@ int64_t GroupConcator::lengthEstimate(const rowgroup::Row& row) fieldLen = 19; // YYYY-MM-DD HH24:MI:SS // Decimal point and milliseconds uint64_t colPrecision = row.getPrecision(*i); + if (colPrecision > 0 && colPrecision < 7) { fieldLen += colPrecision + 1; } + break; } @@ -660,10 +662,12 @@ int64_t GroupConcator::lengthEstimate(const rowgroup::Row& row) fieldLen = 10; // -HHH:MI:SS // Decimal point and milliseconds uint64_t colPrecision = row.getPrecision(*i); + if (colPrecision > 0 && colPrecision < 7) { fieldLen += colPrecision + 1; } + break; } diff --git a/dbcon/mysql/ha_calpont_dml.cpp b/dbcon/mysql/ha_calpont_dml.cpp index 57a91bd89..cf103a801 100644 --- a/dbcon/mysql/ha_calpont_dml.cpp +++ b/dbcon/mysql/ha_calpont_dml.cpp @@ -842,6 +842,7 @@ int ha_calpont_impl_write_batch_row_(uchar* buf, TABLE* table, cal_impl_if::cal_ const uchar* pos = buf; longlong tmp = my_datetime_packed_from_binary(pos, table->field[colpos]->decimals()); TIME_from_longlong_datetime_packed(<ime, tmp); + if (!ltime.second_part) { fprintf(ci.filePtr, "%04d-%02d-%02d %02d:%02d:%02d%c", @@ -855,6 +856,7 @@ int ha_calpont_impl_write_batch_row_(uchar* buf, TABLE* table, cal_impl_if::cal_ ltime.hour, ltime.minute, ltime.second, ltime.second_part, ci.delimiter); } + buf += table->field[colpos]->pack_length(); } else @@ -892,6 +894,7 @@ int ha_calpont_impl_write_batch_row_(uchar* buf, TABLE* table, cal_impl_if::cal_ const uchar* pos = buf; longlong tmp = my_time_packed_from_binary(pos, table->field[colpos]->decimals()); TIME_from_longlong_time_packed(<ime, tmp); + if (!ltime.second_part) { fprintf(ci.filePtr, "%02d:%02d:%02d%c", @@ -903,6 +906,7 @@ int ha_calpont_impl_write_batch_row_(uchar* buf, TABLE* table, cal_impl_if::cal_ ltime.hour, ltime.minute, ltime.second, ltime.second_part, ci.delimiter); } + buf += table->field[colpos]->pack_length(); } diff --git a/utils/dataconvert/dataconvert.cpp b/utils/dataconvert/dataconvert.cpp index 0ba1e4a87..b1cf8bcb2 100644 --- a/utils/dataconvert/dataconvert.cpp +++ b/utils/dataconvert/dataconvert.cpp @@ -1020,10 +1020,11 @@ bool mysql_str_to_time( const string& input, Time& output ) { output.reset(); } + return false; } - output.hour = isNeg ? 0-hour : hour; + output.hour = isNeg ? 0 - hour : hour; output.minute = min; output.second = sec; output.msecond = usec; @@ -1416,7 +1417,7 @@ DataConvert::convertColumnData(const CalpontSystemCatalog::ColType& colType, if (stringToTimeStruct(data, aTime)) { - value = (int64_t) *(reinterpret_cast(&aTime)); + value = (int64_t) * (reinterpret_cast(&aTime)); } else { @@ -1918,6 +1919,7 @@ int64_t DataConvert::convertColumnTime( inSecond = 0; inMicrosecond = 0; bool isNeg = false; + if ( datetimeFormat != CALPONTTIME_ENUM ) { status = -1; @@ -1941,6 +1943,7 @@ int64_t DataConvert::convertColumnTime( } p = strtok_r(NULL, ":.", &savePoint); + if (p == NULL) { status = -1; @@ -1976,6 +1979,7 @@ int64_t DataConvert::convertColumnTime( if (p != NULL) { inMicrosecond = strtol(p, 0, 10); + if (errno) { status = -1; @@ -2017,6 +2021,7 @@ int64_t DataConvert::convertColumnTime( atime.is_neg = false; memcpy( &value, &atime, 8); } + // If neither of the above match then we return a 0 time status = -1; @@ -2067,21 +2072,25 @@ std::string DataConvert::datetimeToString( long long datetimevalue, long decima { decimals = 0; } + // @bug 4703 abandon multiple ostringstream's for conversion DateTime dt(datetimevalue); const int DATETIMETOSTRING_LEN = 28; // YYYY-MM-DD HH:MM:SS.mmmmmm\0 char buf[DATETIMETOSTRING_LEN]; sprintf(buf, "%04d-%02d-%02d %02d:%02d:%02d", dt.year, dt.month, dt.day, dt.hour, dt.minute, dt.second); + if (dt.msecond && decimals) { snprintf(buf + strlen(buf), 21 + decimals, ".%d", dt.msecond); + // Pad end with zeros if (strlen(buf) < (size_t)(21 + decimals)) { sprintf(buf + strlen(buf), "%0*d", (int)(21 + decimals - strlen(buf)), 0); } } + return buf; } @@ -2092,6 +2101,7 @@ std::string DataConvert::timeToString( long long timevalue, long decimals ) { decimals = 0; } + // @bug 4703 abandon multiple ostringstream's for conversion Time dt(timevalue); const int TIMETOSTRING_LEN = 19; // (-H)HH:MM:SS.mmmmmm\0 @@ -2105,16 +2115,19 @@ std::string DataConvert::timeToString( long long timevalue, long decimals ) } sprintf(outbuf, "%02d:%02d:%02d", dt.hour, dt.minute, dt.second); + if (dt.msecond && decimals) { size_t start = strlen(buf); snprintf(buf + strlen(buf), 12 + decimals, ".%d", dt.msecond); + // Pad end with zeros if (strlen(buf) - start < (size_t)decimals) { sprintf(buf + strlen(buf), "%0*d", (int)(decimals - (strlen(buf) - start) + 1), 0); } } + return buf; } diff --git a/utils/dataconvert/dataconvert.h b/utils/dataconvert/dataconvert.h index 01e530709..0cf3480c5 100644 --- a/utils/dataconvert/dataconvert.h +++ b/utils/dataconvert/dataconvert.h @@ -277,6 +277,7 @@ inline uint32_t getDaysInMonth(uint32_t month, int year) return 0; uint32_t days = daysInMonth[month - 1]; + if ((month == 2) && isLeapYear(year)) days++; @@ -567,11 +568,14 @@ inline void DataConvert::datetimeToString( long long datetimevalue, char* buf, u { decimals = 0; } + int msec = 0; + if ((datetimevalue & 0xfffff) > 0) { msec = (unsigned)((datetimevalue) & 0xfffff); } + snprintf( buf, buflen, "%04d-%02d-%02d %02d:%02d:%02d", (unsigned)((datetimevalue >> 48) & 0xffff), (unsigned)((datetimevalue >> 44) & 0xf), @@ -585,6 +589,7 @@ inline void DataConvert::datetimeToString( long long datetimevalue, char* buf, u { size_t start = strlen(buf); snprintf(buf + strlen(buf), buflen - start, ".%d", msec); + // Pad end with zeros if (strlen(buf) - start < (size_t)decimals) { @@ -600,8 +605,10 @@ inline void DataConvert::timeToString( long long timevalue, char* buf, unsigned { decimals = 0; } + // Handle negative correctly int hour = 0, msec = 0; + if ((timevalue >> 40) & 0x800) { hour = 0xfffff000; @@ -613,12 +620,14 @@ inline void DataConvert::timeToString( long long timevalue, char* buf, unsigned { msec = (unsigned)((timevalue) & 0xffffff); } + if ((hour >= 0) && (timevalue >> 63)) { buf[0] = '-'; buf++; buflen--; } + snprintf( buf, buflen, "%02d:%02d:%02d", hour, (unsigned)((timevalue >> 32) & 0xff), @@ -629,6 +638,7 @@ inline void DataConvert::timeToString( long long timevalue, char* buf, unsigned { size_t start = strlen(buf); snprintf(buf + strlen(buf), buflen - start, ".%d", msec); + // Pad end with zeros if (strlen(buf) - start < (size_t)decimals) { @@ -662,6 +672,7 @@ inline void DataConvert::timeToString1( long long timevalue, char* buf, unsigned { // Handle negative correctly int hour = 0; + if ((timevalue >> 40) & 0x800) { hour = 0xfffff000; diff --git a/utils/funcexp/func_add_time.cpp b/utils/funcexp/func_add_time.cpp index bb32dc79d..ced9beaf2 100644 --- a/utils/funcexp/func_add_time.cpp +++ b/utils/funcexp/func_add_time.cpp @@ -320,9 +320,9 @@ int64_t Func_add_time::getDatetimeIntVal(rowgroup::Row& row, } int64_t Func_add_time::getTimeIntVal(rowgroup::Row& row, - FunctionParm& parm, - bool& isNull, - CalpontSystemCatalog::ColType& ct) + FunctionParm& parm, + bool& isNull, + CalpontSystemCatalog::ColType& ct) { int64_t val1 = parm[0]->data()->getTimeIntVal(row, isNull); diff --git a/utils/funcexp/func_bitand.cpp b/utils/funcexp/func_bitand.cpp index 6906a928a..69429bfb8 100644 --- a/utils/funcexp/func_bitand.cpp +++ b/utils/funcexp/func_bitand.cpp @@ -165,6 +165,7 @@ int64_t Func_bitand::getIntVal(Row& row, min = 0, sec = 0, msec = 0; + // Handle negative correctly if ((time >> 40) & 0x800) { @@ -172,8 +173,10 @@ int64_t Func_bitand::getIntVal(Row& row, } hour |= ((time >> 40) & 0xfff); + if ((hour >= 0) && (time >> 63)) - hour*= -1; + hour *= -1; + min = (uint32_t)((time >> 32) & 0xff); sec = (uint32_t)((time >> 24) & 0xff); msec = (uint32_t)(time & 0xffffff); diff --git a/utils/funcexp/func_case.cpp b/utils/funcexp/func_case.cpp index f9d70cb75..4c416360c 100644 --- a/utils/funcexp/func_case.cpp +++ b/utils/funcexp/func_case.cpp @@ -546,9 +546,9 @@ int64_t Func_simple_case::getDatetimeIntVal(rowgroup::Row& row, int64_t Func_simple_case::getTimeIntVal(rowgroup::Row& row, - FunctionParm& parm, - bool& isNull, - execplan::CalpontSystemCatalog::ColType& op_ct) + FunctionParm& parm, + bool& isNull, + execplan::CalpontSystemCatalog::ColType& op_ct) { uint64_t i = simple_case_cmp(row, parm, isNull, op_ct); diff --git a/utils/funcexp/func_coalesce.cpp b/utils/funcexp/func_coalesce.cpp index 30091a81e..e03040350 100644 --- a/utils/funcexp/func_coalesce.cpp +++ b/utils/funcexp/func_coalesce.cpp @@ -140,9 +140,9 @@ int64_t Func_coalesce::getDatetimeIntVal(rowgroup::Row& row, } int64_t Func_coalesce::getTimeIntVal(rowgroup::Row& row, - FunctionParm& parm, - bool& isNull, - CalpontSystemCatalog::ColType& ct) + FunctionParm& parm, + bool& isNull, + CalpontSystemCatalog::ColType& ct) { int64_t val = 0; diff --git a/utils/funcexp/func_extract.cpp b/utils/funcexp/func_extract.cpp index c77a23d70..332c05441 100644 --- a/utils/funcexp/func_extract.cpp +++ b/utils/funcexp/func_extract.cpp @@ -135,26 +135,28 @@ long long dateGet( uint64_t time, IntervalColumn::interval_type unit, bool dateT long long timeGet( uint64_t time, IntervalColumn::interval_type unit ) { int32_t hour = 0, - min = 0, - sec = 0, - msec = 0, - day = 0; + min = 0, + sec = 0, + msec = 0, + day = 0; - min = (int32_t)((time >> 32) & 0xff); - sec = (int32_t)((time >> 24) & 0xff); - msec = (int32_t)((time & 0xfffff)); + min = (int32_t)((time >> 32) & 0xff); + sec = (int32_t)((time >> 24) & 0xff); + msec = (int32_t)((time & 0xfffff)); - // If negative, mask so it doesn't turn positive - int64_t mask = 0; - if ((time >> 40) & 0x800) - mask = 0xfffffffffffff000; - hour = mask | ((time >> 40) & 0xfff); + // If negative, mask so it doesn't turn positive + int64_t mask = 0; - if ((hour >= 0) && (time >> 63)) - hour*= -1; + if ((time >> 40) & 0x800) + mask = 0xfffffffffffff000; - // Always positive! - day = abs(hour / 24); + hour = mask | ((time >> 40) & 0xfff); + + if ((hour >= 0) && (time >> 63)) + hour *= -1; + + // Always positive! + day = abs(hour / 24); switch ( unit ) { @@ -275,6 +277,7 @@ int64_t Func_extract::getIntVal(rowgroup::Row& row, } long long value; + if (isTime) value = timeGet( time, unit ); else diff --git a/utils/funcexp/func_greatest.cpp b/utils/funcexp/func_greatest.cpp index 80dbd251f..de63c283e 100644 --- a/utils/funcexp/func_greatest.cpp +++ b/utils/funcexp/func_greatest.cpp @@ -207,9 +207,9 @@ int64_t Func_greatest::getDatetimeIntVal(rowgroup::Row& row, } int64_t Func_greatest::getTimeIntVal(rowgroup::Row& row, - FunctionParm& fp, - bool& isNull, - execplan::CalpontSystemCatalog::ColType& ct) + FunctionParm& fp, + bool& isNull, + execplan::CalpontSystemCatalog::ColType& ct) { // Strip off unused day int64_t greatestStr = fp[0]->data()->getTimeIntVal(row, isNull); diff --git a/utils/funcexp/func_hour.cpp b/utils/funcexp/func_hour.cpp index 631b070c8..685a264db 100644 --- a/utils/funcexp/func_hour.cpp +++ b/utils/funcexp/func_hour.cpp @@ -130,6 +130,7 @@ int64_t Func_hour::getIntVal(rowgroup::Row& row, // If negative, mask so it doesn't turn positive bool isNeg = false; int64_t mask = 0; + if ((val >> 40) & 0x800) mask = 0xfffffffffffff000; @@ -137,9 +138,11 @@ int64_t Func_hour::getIntVal(rowgroup::Row& row, { isNeg = true; } + val = mask | ((val >> 40) & 0xfff); + if (isNeg) - val*= -1; + val *= -1; } else { diff --git a/utils/funcexp/func_if.cpp b/utils/funcexp/func_if.cpp index 4cc8cd164..ad65d10c9 100644 --- a/utils/funcexp/func_if.cpp +++ b/utils/funcexp/func_if.cpp @@ -221,9 +221,9 @@ int64_t Func_if::getDatetimeIntVal(Row& row, } int64_t Func_if::getTimeIntVal(Row& row, - FunctionParm& parm, - bool& isNull, - CalpontSystemCatalog::ColType&) + FunctionParm& parm, + bool& isNull, + CalpontSystemCatalog::ColType&) { if (boolVal(parm[0], row)) { diff --git a/utils/funcexp/func_ifnull.cpp b/utils/funcexp/func_ifnull.cpp index 18887f00e..4b8606d4e 100644 --- a/utils/funcexp/func_ifnull.cpp +++ b/utils/funcexp/func_ifnull.cpp @@ -169,9 +169,9 @@ int64_t Func_ifnull::getDatetimeIntVal(Row& row, } int64_t Func_ifnull::getTimeIntVal(Row& row, - FunctionParm& parm, - bool& isNull, - CalpontSystemCatalog::ColType&) + FunctionParm& parm, + bool& isNull, + CalpontSystemCatalog::ColType&) { if (isNull) return 0; diff --git a/utils/funcexp/func_inet_aton.cpp b/utils/funcexp/func_inet_aton.cpp index d6e3465cf..9692cf850 100644 --- a/utils/funcexp/func_inet_aton.cpp +++ b/utils/funcexp/func_inet_aton.cpp @@ -223,9 +223,9 @@ int64_t Func_inet_aton::getDatetimeIntVal(rowgroup::Row& row, } int64_t Func_inet_aton::getTimeIntVal(rowgroup::Row& row, - FunctionParm& fp, - bool& isNull, - execplan::CalpontSystemCatalog::ColType& op_ct) + FunctionParm& fp, + bool& isNull, + execplan::CalpontSystemCatalog::ColType& op_ct) { int64_t iValue = joblist::TIMENULL; diff --git a/utils/funcexp/func_inet_ntoa.cpp b/utils/funcexp/func_inet_ntoa.cpp index e25f56097..ff8c889ed 100644 --- a/utils/funcexp/func_inet_ntoa.cpp +++ b/utils/funcexp/func_inet_ntoa.cpp @@ -257,9 +257,9 @@ int64_t Func_inet_ntoa::getDatetimeIntVal(rowgroup::Row& row, } int64_t Func_inet_ntoa::getTimeIntVal(rowgroup::Row& row, - FunctionParm& fp, - bool& isNull, - execplan::CalpontSystemCatalog::ColType& op_ct) + FunctionParm& fp, + bool& isNull, + execplan::CalpontSystemCatalog::ColType& op_ct) { // std::cout << "In Func_inet_ntoa::getTimeVal" << std::endl; diff --git a/utils/funcexp/func_least.cpp b/utils/funcexp/func_least.cpp index ee5b2afc0..9ba5c98de 100644 --- a/utils/funcexp/func_least.cpp +++ b/utils/funcexp/func_least.cpp @@ -183,9 +183,9 @@ int64_t Func_least::getDatetimeIntVal(rowgroup::Row& row, } int64_t Func_least::getTimeIntVal(rowgroup::Row& row, - FunctionParm& fp, - bool& isNull, - execplan::CalpontSystemCatalog::ColType& op_ct) + FunctionParm& fp, + bool& isNull, + execplan::CalpontSystemCatalog::ColType& op_ct) { // Strip off unused day int64_t leastStr = fp[0]->data()->getTimeIntVal(row, isNull); diff --git a/utils/funcexp/func_nullif.cpp b/utils/funcexp/func_nullif.cpp index 30d4682e5..a268b0ea1 100644 --- a/utils/funcexp/func_nullif.cpp +++ b/utils/funcexp/func_nullif.cpp @@ -129,8 +129,8 @@ int64_t Func_nullif::getIntVal(rowgroup::Row& row, if ((parm[0]->data()->resultType().colDataType == execplan::CalpontSystemCatalog::DATE) || - (parm[0]->data()->resultType().colDataType == - execplan::CalpontSystemCatalog::DATETIME)) + (parm[0]->data()->resultType().colDataType == + execplan::CalpontSystemCatalog::DATETIME)) { // NULLIF arg0 is DATE, arg1 is DATETIME, @@ -164,8 +164,8 @@ int64_t Func_nullif::getIntVal(rowgroup::Row& row, if ((parm[0]->data()->resultType().colDataType == execplan::CalpontSystemCatalog::DATETIME) || - (parm[0]->data()->resultType().colDataType == - execplan::CalpontSystemCatalog::TIME)) + (parm[0]->data()->resultType().colDataType == + execplan::CalpontSystemCatalog::TIME)) { // NULLIF arg0 is DATETIME, arg1 is TIME, // Upgrade arg1 to time @@ -495,9 +495,9 @@ int64_t Func_nullif::getDatetimeIntVal(rowgroup::Row& row, } int64_t Func_nullif::getTimeIntVal(rowgroup::Row& row, - FunctionParm& parm, - bool& isNull, - CalpontSystemCatalog::ColType& ct) + FunctionParm& parm, + bool& isNull, + CalpontSystemCatalog::ColType& ct) { int64_t exp1 = parm[0]->data()->getTimeIntVal(row, isNull); int64_t exp2 = 0; diff --git a/utils/funcexp/func_sysdate.cpp b/utils/funcexp/func_sysdate.cpp index d8e2580bb..bf71e0415 100644 --- a/utils/funcexp/func_sysdate.cpp +++ b/utils/funcexp/func_sysdate.cpp @@ -101,9 +101,9 @@ int64_t Func_sysdate::getDatetimeIntVal(rowgroup::Row& row, } int64_t Func_sysdate::getTimeIntVal(rowgroup::Row& row, - FunctionParm& parm, - bool& isNull, - CalpontSystemCatalog::ColType& operationColType) + FunctionParm& parm, + bool& isNull, + CalpontSystemCatalog::ColType& operationColType) { return getIntVal(row, parm, isNull, operationColType); } diff --git a/utils/funcexp/func_time.cpp b/utils/funcexp/func_time.cpp index bd28d6e6c..4a90e0fe2 100644 --- a/utils/funcexp/func_time.cpp +++ b/utils/funcexp/func_time.cpp @@ -111,6 +111,7 @@ string Func_time::getStrVal(rowgroup::Row& row, val = parm[0]->data()->getTimeIntVal(row, isNull); break; } + case execplan::CalpontSystemCatalog::DATETIME: { val = parm[0]->data()->getTimeIntVal(row, isNull); diff --git a/utils/funcexp/func_time_to_sec.cpp b/utils/funcexp/func_time_to_sec.cpp index 0d804242c..ee5c4b669 100644 --- a/utils/funcexp/func_time_to_sec.cpp +++ b/utils/funcexp/func_time_to_sec.cpp @@ -50,8 +50,8 @@ int64_t Func_time_to_sec::getIntVal(rowgroup::Row& row, { // assume 256 is enough. assume not allowing incomplete date int32_t hour = 0, - min = 0, - sec = 0; + min = 0, + sec = 0; bool bIsNegative = false; // Only set to true if CHAR or VARCHAR with a '-' int64_t val = 0; @@ -72,16 +72,19 @@ int64_t Func_time_to_sec::getIntVal(rowgroup::Row& row, case CalpontSystemCatalog::TIME: val = parm[0]->data()->getTimeIntVal(row, isNull); + // If negative, mask so it doesn't turn positive if ((val >> 40) & 0x800) mask = 0xfffffffffffff000; bIsNegative = val >> 63; hour = (int32_t)(mask | ((val >> 40) & 0xfff)); + if ((hour >= 0) && bIsNegative) - hour*= -1; + hour *= -1; else bIsNegative = false; + min = (int32_t)((val >> 32) & 0xff); sec = (int32_t)((val >> 24) & 0xff); break; @@ -162,6 +165,7 @@ int64_t Func_time_to_sec::getIntVal(rowgroup::Row& row, } int64_t rtn; + if (hour < 0) { rtn = (int64_t)(hour * 60 * 60) - (min * 60) - sec; diff --git a/utils/funcexp/func_timediff.cpp b/utils/funcexp/func_timediff.cpp index 3c9ce96a8..742e8faf7 100644 --- a/utils/funcexp/func_timediff.cpp +++ b/utils/funcexp/func_timediff.cpp @@ -216,9 +216,9 @@ int64_t Func_timediff::getDatetimeIntVal(rowgroup::Row& row, } int64_t Func_timediff::getTimeIntVal(rowgroup::Row& row, - FunctionParm& parm, - bool& isNull, - CalpontSystemCatalog::ColType& ct) + FunctionParm& parm, + bool& isNull, + CalpontSystemCatalog::ColType& ct) { return dataconvert::DataConvert::timeToInt(getStrVal(row, parm, isNull, ct)); } diff --git a/utils/funcexp/functor.h b/utils/funcexp/functor.h index 065eb7865..9edb9bf62 100644 --- a/utils/funcexp/functor.h +++ b/utils/funcexp/functor.h @@ -117,9 +117,9 @@ public: } virtual int64_t getTimeIntVal(rowgroup::Row& row, - FunctionParm& fp, - bool& isNull, - execplan::CalpontSystemCatalog::ColType& op_ct) + FunctionParm& fp, + bool& isNull, + execplan::CalpontSystemCatalog::ColType& op_ct) { return intToTime(getIntVal(row, fp, isNull, op_ct)); } diff --git a/utils/funcexp/functor_all.h b/utils/funcexp/functor_all.h index 0eb0df845..553abc40f 100644 --- a/utils/funcexp/functor_all.h +++ b/utils/funcexp/functor_all.h @@ -105,10 +105,10 @@ public: bool& isNull, execplan::CalpontSystemCatalog::ColType& op_ct); - int64_t getTimeIntVal(rowgroup::Row& row, - FunctionParm& fp, - bool& isNull, - execplan::CalpontSystemCatalog::ColType& op_ct); + int64_t getTimeIntVal(rowgroup::Row& row, + FunctionParm& fp, + bool& isNull, + execplan::CalpontSystemCatalog::ColType& op_ct); }; @@ -158,9 +158,9 @@ public: execplan::CalpontSystemCatalog::ColType& op_ct); int64_t getTimeIntVal(rowgroup::Row& row, - FunctionParm& fp, - bool& isNull, - execplan::CalpontSystemCatalog::ColType& op_ct); + FunctionParm& fp, + bool& isNull, + execplan::CalpontSystemCatalog::ColType& op_ct); }; @@ -206,9 +206,10 @@ public: execplan::CalpontSystemCatalog::ColType& op_ct); int64_t getTimeIntVal(rowgroup::Row& row, - FunctionParm& fp, - bool& isNull, - execplan::CalpontSystemCatalog::ColType& op_ct);}; + FunctionParm& fp, + bool& isNull, + execplan::CalpontSystemCatalog::ColType& op_ct); +}; /** @brief Func_ifnull class @@ -252,9 +253,9 @@ public: execplan::CalpontSystemCatalog::ColType& op_ct); int64_t getTimeIntVal(rowgroup::Row& row, - FunctionParm& fp, - bool& isNull, - execplan::CalpontSystemCatalog::ColType& op_ct); + FunctionParm& fp, + bool& isNull, + execplan::CalpontSystemCatalog::ColType& op_ct); bool getBoolVal(rowgroup::Row& row, FunctionParm& fp, @@ -309,9 +310,9 @@ public: execplan::CalpontSystemCatalog::ColType& op_ct); int64_t getTimeIntVal(rowgroup::Row& row, - FunctionParm& fp, - bool& isNull, - execplan::CalpontSystemCatalog::ColType& op_ct); + FunctionParm& fp, + bool& isNull, + execplan::CalpontSystemCatalog::ColType& op_ct); }; @@ -357,9 +358,10 @@ public: execplan::CalpontSystemCatalog::ColType& op_ct); int64_t getTimeIntVal(rowgroup::Row& row, - FunctionParm& fp, - bool& isNull, - execplan::CalpontSystemCatalog::ColType& op_ct);}; + FunctionParm& fp, + bool& isNull, + execplan::CalpontSystemCatalog::ColType& op_ct); +}; /** @brief Func_coalesce class @@ -403,9 +405,9 @@ public: execplan::CalpontSystemCatalog::ColType& op_ct); int64_t getTimeIntVal(rowgroup::Row& row, - FunctionParm& fp, - bool& isNull, - execplan::CalpontSystemCatalog::ColType& op_ct); + FunctionParm& fp, + bool& isNull, + execplan::CalpontSystemCatalog::ColType& op_ct); }; @@ -435,9 +437,9 @@ public: execplan::CalpontSystemCatalog::ColType& op_ct); int64_t getTimeIntVal(rowgroup::Row& row, - FunctionParm& fp, - bool& isNull, - execplan::CalpontSystemCatalog::ColType& op_ct); + FunctionParm& fp, + bool& isNull, + execplan::CalpontSystemCatalog::ColType& op_ct); int64_t getIntVal(rowgroup::Row& row, FunctionParm& fp, diff --git a/utils/funcexp/functor_bool.h b/utils/funcexp/functor_bool.h index f67f405f1..b7b85106b 100644 --- a/utils/funcexp/functor_bool.h +++ b/utils/funcexp/functor_bool.h @@ -99,9 +99,9 @@ public: } int64_t getTimeIntVal(rowgroup::Row& row, - FunctionParm& fp, - bool& isNull, - execplan::CalpontSystemCatalog::ColType& op_ct) + FunctionParm& fp, + bool& isNull, + execplan::CalpontSystemCatalog::ColType& op_ct) { isNull = true; return 0; diff --git a/utils/funcexp/functor_dtm.h b/utils/funcexp/functor_dtm.h index 74aa41e38..d7837a4fe 100644 --- a/utils/funcexp/functor_dtm.h +++ b/utils/funcexp/functor_dtm.h @@ -138,9 +138,9 @@ public: execplan::CalpontSystemCatalog::ColType& op_ct); int64_t getTimeIntVal(rowgroup::Row& row, - FunctionParm& fp, - bool& isNull, - execplan::CalpontSystemCatalog::ColType& op_ct); + FunctionParm& fp, + bool& isNull, + execplan::CalpontSystemCatalog::ColType& op_ct); int64_t getIntVal(rowgroup::Row& row, FunctionParm& fp, @@ -254,9 +254,9 @@ public: execplan::CalpontSystemCatalog::ColType& op_ct); int64_t getTimeIntVal(rowgroup::Row& row, - FunctionParm& fp, - bool& isNull, - execplan::CalpontSystemCatalog::ColType& op_ct); + FunctionParm& fp, + bool& isNull, + execplan::CalpontSystemCatalog::ColType& op_ct); }; @@ -324,9 +324,9 @@ public: execplan::CalpontSystemCatalog::ColType& op_ct); int64_t getTimeIntVal(rowgroup::Row& row, - FunctionParm& fp, - bool& isNull, - execplan::CalpontSystemCatalog::ColType& op_ct); + FunctionParm& fp, + bool& isNull, + execplan::CalpontSystemCatalog::ColType& op_ct); }; @@ -361,9 +361,9 @@ public: execplan::CalpontSystemCatalog::ColType& op_ct); int64_t getTimeIntVal(rowgroup::Row& row, - FunctionParm& fp, - bool& isNull, - execplan::CalpontSystemCatalog::ColType& op_ct); + FunctionParm& fp, + bool& isNull, + execplan::CalpontSystemCatalog::ColType& op_ct); }; @@ -393,9 +393,9 @@ public: execplan::CalpontSystemCatalog::ColType& op_ct); int64_t getTimeIntVal(rowgroup::Row& row, - FunctionParm& fp, - bool& isNull, - execplan::CalpontSystemCatalog::ColType& op_ct); + FunctionParm& fp, + bool& isNull, + execplan::CalpontSystemCatalog::ColType& op_ct); int64_t getIntVal(rowgroup::Row& row, FunctionParm& fp, @@ -439,9 +439,9 @@ public: execplan::CalpontSystemCatalog::ColType& op_ct); int64_t getTimeIntVal(rowgroup::Row& row, - FunctionParm& fp, - bool& isNull, - execplan::CalpontSystemCatalog::ColType& op_ct); + FunctionParm& fp, + bool& isNull, + execplan::CalpontSystemCatalog::ColType& op_ct); }; /** @brief Func_str_to_date class diff --git a/utils/funcexp/functor_real.h b/utils/funcexp/functor_real.h index 779f4d108..5d40ce589 100644 --- a/utils/funcexp/functor_real.h +++ b/utils/funcexp/functor_real.h @@ -655,9 +655,9 @@ public: execplan::CalpontSystemCatalog::ColType& op_ct); int64_t getTimeIntVal(rowgroup::Row& row, - FunctionParm& fp, - bool& isNull, - execplan::CalpontSystemCatalog::ColType& op_ct); + FunctionParm& fp, + bool& isNull, + execplan::CalpontSystemCatalog::ColType& op_ct); private: diff --git a/utils/funcexp/functor_str.h b/utils/funcexp/functor_str.h index 895101211..c71cdec91 100644 --- a/utils/funcexp/functor_str.h +++ b/utils/funcexp/functor_str.h @@ -90,9 +90,9 @@ public: } int64_t getTimeIntVal(rowgroup::Row& row, - FunctionParm& fp, - bool& isNull, - execplan::CalpontSystemCatalog::ColType& op_ct) + FunctionParm& fp, + bool& isNull, + execplan::CalpontSystemCatalog::ColType& op_ct) { std::string str = getStrVal(row, fp, isNull, op_ct); return (isNull ? 0 : stringToTime(str)); @@ -685,9 +685,9 @@ public: execplan::CalpontSystemCatalog::ColType& op_ct); int64_t getTimeIntVal(rowgroup::Row& row, - FunctionParm& fp, - bool& isNull, - execplan::CalpontSystemCatalog::ColType& op_ct); + FunctionParm& fp, + bool& isNull, + execplan::CalpontSystemCatalog::ColType& op_ct); private: void convertNtoa(int64_t ipNum, std::string& ipString); }; diff --git a/utils/rowgroup/rowaggregation.cpp b/utils/rowgroup/rowaggregation.cpp index c496d91e6..8d110cfc8 100644 --- a/utils/rowgroup/rowaggregation.cpp +++ b/utils/rowgroup/rowaggregation.cpp @@ -1543,6 +1543,7 @@ void RowAggregation::doBitOp(const Row& rowIn, int64_t colIn, int64_t colOut, in int64_t dtm = rowIn.getUintField(colIn); // Handle negative correctly int hour = 0; + if ((dtm >> 40) & 0x800) { hour = 0xfffff000; @@ -2077,6 +2078,7 @@ void RowAggregation::doUDAF(const Row& rowIn, int64_t colIn, int64_t colOut, int datum.columnData = rowIn.getUintField(colIn); break; } + case execplan::CalpontSystemCatalog::TIME: { datum.dataType = execplan::CalpontSystemCatalog::BIGINT; diff --git a/writeengine/bulk/we_bulkloadbuffer.cpp b/writeengine/bulk/we_bulkloadbuffer.cpp index ccf73363f..506bff2eb 100644 --- a/writeengine/bulk/we_bulkloadbuffer.cpp +++ b/writeengine/bulk/we_bulkloadbuffer.cpp @@ -902,8 +902,8 @@ void BulkLoadBuffer::convert(char* field, int fieldLength, { bool bSatVal = false; - if ( column.dataType != CalpontSystemCatalog::DATETIME && - column.dataType != CalpontSystemCatalog::TIME ) + if ( column.dataType != CalpontSystemCatalog::DATETIME && + column.dataType != CalpontSystemCatalog::TIME ) { if (nullFlag) { From f68f80eb0068890f0a35bdadbf40a496ac922f91 Mon Sep 17 00:00:00 2001 From: Andrew Hutchings Date: Tue, 1 May 2018 20:56:52 +0100 Subject: [PATCH 49/49] MCOL-392 Add back accidental removal of function --- dbcon/execplan/aggregatecolumn.h | 8 ++++++++ 1 file changed, 8 insertions(+) diff --git a/dbcon/execplan/aggregatecolumn.h b/dbcon/execplan/aggregatecolumn.h index 028f1ee54..d1db7e5a4 100644 --- a/dbcon/execplan/aggregatecolumn.h +++ b/dbcon/execplan/aggregatecolumn.h @@ -416,6 +416,14 @@ public: evaluate(row, isNull); return TreeNode::getTimeIntVal(); } + /** + * F&E + */ + virtual int64_t getDatetimeIntVal(rowgroup::Row& row, bool& isNull) + { + evaluate(row, isNull); + return TreeNode::getDatetimeIntVal(); + } private: void evaluate(rowgroup::Row& row, bool& isNull);