From 39a976c39a29463e73fc5cec92c77ed152fa77a6 Mon Sep 17 00:00:00 2001 From: Serguey Zefirov Date: Thu, 21 Nov 2024 14:52:01 +0300 Subject: [PATCH] fix(ubsan): MCOL-5844 - iron out UBSAN reports The most important fix here is the fix of possible buffer overrun in DATEFORMAT() function. A "%W" format, repeated enough times, would overflow the 256-bytes buffer for result. Now we use ostringstream to construct result and we are safe. Changes in date/time projection functions made me fix difference between us and server behavior. The new, better behavior is reflected in changes in tests' results. Also, there was incorrect logic in TRUNCATE() and ROUND() functions in computing the decimal "shift." --- dbcon/execplan/arithmeticoperator.h | 13 ++ dbcon/joblist/jlf_common.h | 1 + dbcon/mysql/ha_mcs_execplan.cpp | 2 +- dmlproc/dmlprocessor.cpp | 5 +- .../basic/r/mcs119_date_formats.result | 4 +- .../basic/r/mcs185_dayname_function.result | 4 +- .../basic/r/mcs186_dayofyear_function.result | 4 +- .../basic/r/mcs236_extract_function.result | 12 +- .../basic/r/mcs275_timediff_function.result | 6 +- .../basic/r/mcs92_date_functions.result | 20 +-- .../basic/t/mcs92_date_functions.test | 4 + utils/funcexp/func_date_format.cpp | 133 +++++++++--------- utils/funcexp/func_dayname.cpp | 2 +- utils/funcexp/func_dayofweek.cpp | 2 +- utils/funcexp/func_dayofyear.cpp | 5 + utils/funcexp/func_extract.cpp | 8 +- utils/funcexp/func_from_unixtime.cpp | 2 +- utils/funcexp/func_round.cpp | 25 +++- utils/funcexp/func_timediff.cpp | 6 +- utils/funcexp/func_truncate.cpp | 43 ++++-- utils/funcexp/func_weekday.cpp | 2 +- utils/funcexp/funchelpers.h | 16 ++- utils/funcexp/tdriver.cpp | 3 +- utils/funcexp/timeextract.h | 3 +- utils/loggingcpp/message.cpp | 7 + utils/loggingcpp/messageobj.h | 8 ++ 26 files changed, 207 insertions(+), 133 deletions(-) diff --git a/dbcon/execplan/arithmeticoperator.h b/dbcon/execplan/arithmeticoperator.h index f0275349b..87be33a0e 100644 --- a/dbcon/execplan/arithmeticoperator.h +++ b/dbcon/execplan/arithmeticoperator.h @@ -323,6 +323,19 @@ inline void ArithmeticOperator::evaluate(rowgroup::Row& row, bool& isNull, Parse template inline result_t ArithmeticOperator::execute(result_t op1, result_t op2, bool& isNull) { + if (isNull) + { + // at least one operand is NULL. + // do nothing, return 0. + if constexpr (std::is_same::value) + { + return datatypes::TSInt128(); // returns 0 + } + else + { + return T{0}; + } + } switch (fOp) { case OP_ADD: return op1 + op2; diff --git a/dbcon/joblist/jlf_common.h b/dbcon/joblist/jlf_common.h index 427b5bfed..a0c03548a 100644 --- a/dbcon/joblist/jlf_common.h +++ b/dbcon/joblist/jlf_common.h @@ -225,6 +225,7 @@ struct JobInfo , stringScanThreshold(1) , wfqLimitStart(0) , wfqLimitCount(-1) + , djsForceRun(false) , timeZone(0) , maxPmJoinResultCount(1048576) { diff --git a/dbcon/mysql/ha_mcs_execplan.cpp b/dbcon/mysql/ha_mcs_execplan.cpp index 69cc63e6f..68b0e4d0e 100644 --- a/dbcon/mysql/ha_mcs_execplan.cpp +++ b/dbcon/mysql/ha_mcs_execplan.cpp @@ -448,7 +448,7 @@ bool sortItemIsInGroupRec(Item* sort_item, Item* group_item) Item_ref* ifp_sort_ref = static_cast(sort_item); found = sortItemIsInGroupRec(*ifp_sort_ref->ref, group_item); } - else if (!found && sort_item->type() == Item::FIELD_ITEM) + else if (sort_item->type() == Item::FIELD_ITEM) { return found; } diff --git a/dmlproc/dmlprocessor.cpp b/dmlproc/dmlprocessor.cpp index 34529212f..833705b68 100644 --- a/dmlproc/dmlprocessor.cpp +++ b/dmlproc/dmlprocessor.cpp @@ -1316,7 +1316,10 @@ int DMLServer::start() } } -DMLProcessor::DMLProcessor(messageqcpp::IOSocket ios, BRM::DBRM* aDbrm) : fIos(ios), fDbrm(aDbrm) +DMLProcessor::DMLProcessor(messageqcpp::IOSocket ios, BRM::DBRM* aDbrm) + : fIos(ios) + , fDbrm(aDbrm) + , fConcurrentSupport(false) { csc = CalpontSystemCatalog::makeCalpontSystemCatalog(); csc->identity(CalpontSystemCatalog::EC); diff --git a/mysql-test/columnstore/basic/r/mcs119_date_formats.result b/mysql-test/columnstore/basic/r/mcs119_date_formats.result index 6a956fcae..b6ef46fd2 100644 --- a/mysql-test/columnstore/basic/r/mcs119_date_formats.result +++ b/mysql-test/columnstore/basic/r/mcs119_date_formats.result @@ -71,14 +71,14 @@ INSERT INTO t1 VALUES('2016-02-01', '11:11:11', '2017-02-01 12:12:12'); SELECT t1_dt, DATE_FORMAT(t1_dt, '%Y-%m-%d') a, DATE_FORMAT(t1_d, '%a %D %b %Y') b, DATE_FORMAT(t1_d, '%W %D %M %Y') c FROM t1; t1_dt a b c NULL NULL NULL NULL -0000-00-00 00:00:00 0000-00-00 Sun 0th NON_VALID 0000 Sunday 0th NON_VALID 0000 +0000-00-00 00:00:00 0000-00-00 NULL NULL 2020-11-11 12:12:12 2020-11-11 Mon 11th Nov 2222 Monday 11th November 2222 2020-12-31 12:34:56 2020-12-31 Wed 1st Jan 2020 Wednesday 1st January 2020 2017-02-01 12:12:12 2017-02-01 Mon 1st Feb 2016 Monday 1st February 2016 SELECT t1_dt, DATE_FORMAT(t1_dt, '%Y/%m/%d %T') a, DATE_FORMAT(t1_dt, '%a %D %b %Y %H:%i') b, DATE_FORMAT(t1_dt, '%W %D %M %Y %T') c FROM t1; t1_dt a b c NULL NULL NULL NULL -0000-00-00 00:00:00 0000/00/00 00:00:00 Sun 0th NON_VALID 0000 00:00 Sunday 0th NON_VALID 0000 00:00:00 +0000-00-00 00:00:00 0000/00/00 00:00:00 NULL NULL 2020-11-11 12:12:12 2020/11/11 12:12:12 Wed 11th Nov 2020 12:12 Wednesday 11th November 2020 12:12:12 2020-12-31 12:34:56 2020/12/31 12:34:56 Thu 31st Dec 2020 12:34 Thursday 31st December 2020 12:34:56 2017-02-01 12:12:12 2017/02/01 12:12:12 Wed 1st Feb 2017 12:12 Wednesday 1st February 2017 12:12:12 diff --git a/mysql-test/columnstore/basic/r/mcs185_dayname_function.result b/mysql-test/columnstore/basic/r/mcs185_dayname_function.result index f5534e802..99e98e92e 100644 --- a/mysql-test/columnstore/basic/r/mcs185_dayname_function.result +++ b/mysql-test/columnstore/basic/r/mcs185_dayname_function.result @@ -18,12 +18,12 @@ DAYNAME('2020-12-22') Tuesday SELECT a, DAYNAME(a) FROM t1 ORDER BY 1; a DAYNAME(a) -0000-00-00 Sunday +0000-00-00 NULL 1212-12-12 Wednesday 3333-03-03 Tuesday SELECT b, DAYNAME(b) FROM t1 ORDER BY 1; b DAYNAME(b) -0000-00-00 00:00:00 Sunday +0000-00-00 00:00:00 NULL 1212-12-11 11:11:11 Tuesday 3333-03-04 03:33:33 Wednesday DROP DATABASE mcs185_db; diff --git a/mysql-test/columnstore/basic/r/mcs186_dayofyear_function.result b/mysql-test/columnstore/basic/r/mcs186_dayofyear_function.result index 43b19bee8..468635df6 100644 --- a/mysql-test/columnstore/basic/r/mcs186_dayofyear_function.result +++ b/mysql-test/columnstore/basic/r/mcs186_dayofyear_function.result @@ -18,12 +18,12 @@ DAYOFYEAR('2020-12-22') 357 SELECT a, DAYOFYEAR(a) FROM t1 ORDER BY 1; a DAYOFYEAR(a) -0000-00-00 2147483647 +0000-00-00 NULL 1212-12-12 347 3333-03-03 62 SELECT b, DAYOFYEAR(b) FROM t1 ORDER BY 1; b DAYOFYEAR(b) -0000-00-00 00:00:00 2147483647 +0000-00-00 00:00:00 NULL 1212-12-11 11:11:11 346 3333-03-04 03:33:33 63 DROP DATABASE mcs186_db; diff --git a/mysql-test/columnstore/basic/r/mcs236_extract_function.result b/mysql-test/columnstore/basic/r/mcs236_extract_function.result index f8f8fe75b..1fa1097ee 100644 --- a/mysql-test/columnstore/basic/r/mcs236_extract_function.result +++ b/mysql-test/columnstore/basic/r/mcs236_extract_function.result @@ -321,9 +321,9 @@ NULL NULL SELECT t1_t, EXTRACT(HOUR_MICROSECOND FROM t1_t) FROM t1 ORDER BY 1; t1_t EXTRACT(HOUR_MICROSECOND FROM t1_t) NULL NULL -11:11:11 -558149696 -11:11:11 -558149696 -12:12:12 952915712 +11:11:11 111111000000 +11:11:11 111111000000 +12:12:12 121212000000 SELECT t1_t, EXTRACT(HOUR_SECOND FROM t1_t) FROM t1 ORDER BY 1; t1_t EXTRACT(HOUR_SECOND FROM t1_t) NULL NULL @@ -339,9 +339,9 @@ NULL NULL SELECT t1_t, EXTRACT(DAY_MICROSECOND FROM t1_t) FROM t1 ORDER BY 1; t1_t EXTRACT(DAY_MICROSECOND FROM t1_t) NULL NULL -11:11:11 -558149696 -11:11:11 -558149696 -12:12:12 952915712 +11:11:11 111111000000 +11:11:11 111111000000 +12:12:12 121212000000 SELECT t1_t, EXTRACT(DAY_SECOND FROM t1_t) FROM t1 ORDER BY 1; t1_t EXTRACT(DAY_SECOND FROM t1_t) NULL NULL diff --git a/mysql-test/columnstore/basic/r/mcs275_timediff_function.result b/mysql-test/columnstore/basic/r/mcs275_timediff_function.result index 36abb50fd..75f61850f 100644 --- a/mysql-test/columnstore/basic/r/mcs275_timediff_function.result +++ b/mysql-test/columnstore/basic/r/mcs275_timediff_function.result @@ -33,9 +33,9 @@ t1_TIME TIMEDIFF(t1_TIME, '2008-02-19 22:22:22') 23:59:59 NULL SELECT t1_TIME, TIMEDIFF(t1_TIME, '23:59:59') FROM t1 ORDER BY 1; t1_TIME TIMEDIFF(t1_TIME, '23:59:59') -01:37:50 -838:59:59 -22:12:02 -838:59:59 -23:59:59 -838:59:59 +01:37:50 -22:22:09 +22:12:02 -01:47:57 +23:59:59 00:00:00 SELECT t1_DATETIME, TIMEDIFF(t1_DATETIME, '2001-02-19 22:22:22') FROM t1 ORDER BY 1; t1_DATETIME TIMEDIFF(t1_DATETIME, '2001-02-19 22:22:22') 1997-12-12 22:12:02 -838:59:59 diff --git a/mysql-test/columnstore/basic/r/mcs92_date_functions.result b/mysql-test/columnstore/basic/r/mcs92_date_functions.result index e7936dcd3..bc563807a 100644 --- a/mysql-test/columnstore/basic/r/mcs92_date_functions.result +++ b/mysql-test/columnstore/basic/r/mcs92_date_functions.result @@ -295,14 +295,14 @@ NULL NULL SELECT t1_dt, DATE_FORMAT(t1_dt, '%Y-%m-%d') a, DATE_FORMAT(t1_d, '%a %D %b %Y') b, DATE_FORMAT(t1_d, '%W %D %M %Y') c FROM t1; t1_dt a b c NULL NULL NULL NULL -0000-00-00 00:00:00 0000-00-00 Sun 0th NON_VALID 0000 Sunday 0th NON_VALID 0000 +0000-00-00 00:00:00 0000-00-00 NULL NULL 2020-11-11 12:12:12 2020-11-11 Mon 11th Nov 2222 Monday 11th November 2222 2020-12-31 12:34:56 2020-12-31 Wed 1st Jan 2020 Wednesday 1st January 2020 2017-02-01 12:12:12 2017-02-01 Mon 1st Feb 2016 Monday 1st February 2016 SELECT t1_dt, DATE_FORMAT(t1_dt, '%Y/%m/%d %T') a, DATE_FORMAT(t1_dt, '%a %D %b %Y %H:%i') b, DATE_FORMAT(t1_dt, '%W %D %M %Y %T') c FROM t1; t1_dt a b c NULL NULL NULL NULL -0000-00-00 00:00:00 0000/00/00 00:00:00 Sun 0th NON_VALID 0000 00:00 Sunday 0th NON_VALID 0000 00:00:00 +0000-00-00 00:00:00 0000/00/00 00:00:00 NULL NULL 2020-11-11 12:12:12 2020/11/11 12:12:12 Wed 11th Nov 2020 12:12 Wednesday 11th November 2020 12:12:12 2020-12-31 12:34:56 2020/12/31 12:34:56 Thu 31st Dec 2020 12:34 Thursday 31st December 2020 12:34:56 2017-02-01 12:12:12 2017/02/01 12:12:12 Wed 1st Feb 2017 12:12 Wednesday 1st February 2017 12:12:12 @@ -312,14 +312,14 @@ en_US SELECT t1_dt, DAYNAME(t1_dt), DAYOFWEEK(t1_dt), WEEKDAY(t1_dt) FROM t1; t1_dt DAYNAME(t1_dt) DAYOFWEEK(t1_dt) WEEKDAY(t1_dt) NULL NULL NULL NULL -0000-00-00 00:00:00 Sunday 1 6 +0000-00-00 00:00:00 NULL NULL NULL 2020-11-11 12:12:12 Wednesday 4 2 2020-12-31 12:34:56 Thursday 5 3 2017-02-01 12:12:12 Wednesday 4 2 SELECT t1_d, DAYNAME(t1_d), DAYOFWEEK(t1_d), WEEKDAY(t1_d) FROM t1; t1_d DAYNAME(t1_d) DAYOFWEEK(t1_d) WEEKDAY(t1_d) NULL NULL NULL NULL -0000-00-00 Sunday 1 6 +0000-00-00 NULL NULL NULL 2222-11-11 Monday 2 0 2020-01-01 Wednesday 4 2 2016-02-01 Monday 2 0 @@ -333,7 +333,7 @@ NULL NULL NULL NULL NULL SELECT t1_dt, EXTRACT(DAY FROM t1_dt) 'DAY', EXTRACT(YEAR FROM t1_dt) 'YEAR', EXTRACT(MONTH FROM t1_dt) 'MONTH', EXTRACT(WEEK FROM t1_dt) 'WEEK' FROM t1; t1_dt DAY YEAR MONTH WEEK NULL NULL NULL NULL NULL -0000-00-00 00:00:00 0 0 0 613566753 +0000-00-00 00:00:00 0 0 0 1 2020-11-11 12:12:12 11 2020 11 46 2020-12-31 12:34:56 31 2020 12 53 2017-02-01 12:12:12 1 2017 2 5 @@ -361,10 +361,10 @@ NULL NULL SELECT t1_t, TIMEDIFF('12:58:11', t1_t) FROM t1; t1_t TIMEDIFF('12:58:11', t1_t) NULL NULL -00:00:00 838:59:59 -12:12:12 838:59:59 -11:11:11 838:59:59 -11:11:11 838:59:59 +00:00:00 12:58:11 +12:12:12 00:45:59 +11:11:11 01:47:00 +11:11:11 01:47:00 SELECT t1_d, t1_dt, TIMESTAMPDIFF(MONTH, t1_d, t1_dt) FROM t1; t1_d t1_dt TIMESTAMPDIFF(MONTH, t1_d, t1_dt) NULL NULL NULL @@ -396,7 +396,7 @@ NULL NULL NULL SELECT t1_d, WEEK(t1_d), t1_dt, WEEK(t1_dt) FROM t1; t1_d WEEK(t1_d) t1_dt WEEK(t1_dt) NULL NULL NULL NULL -0000-00-00 613566753 0000-00-00 00:00:00 613566753 +0000-00-00 1 0000-00-00 00:00:00 1 2222-11-11 45 2020-11-11 12:12:12 45 2020-01-01 0 2020-12-31 12:34:56 52 2016-02-01 5 2017-02-01 12:12:12 5 diff --git a/mysql-test/columnstore/basic/t/mcs92_date_functions.test b/mysql-test/columnstore/basic/t/mcs92_date_functions.test index 8301c7f85..012dec94c 100644 --- a/mysql-test/columnstore/basic/t/mcs92_date_functions.test +++ b/mysql-test/columnstore/basic/t/mcs92_date_functions.test @@ -70,6 +70,8 @@ SELECT t1_dt, DAYNAME(t1_dt), DAYOFWEEK(t1_dt), WEEKDAY(t1_dt) FROM t1; SELECT t1_d, DAYNAME(t1_d), DAYOFWEEK(t1_d), WEEKDAY(t1_d) FROM t1; SELECT t1_dt, EXTRACT(SECOND FROM t1_dt) 'SECOND', EXTRACT(DAY_HOUR FROM t1_dt) 'DAY_HOUR', EXTRACT(HOUR FROM t1_dt) 'HOUR', EXTRACT(MINUTE FROM t1_dt) 'MINUTE' FROM t1; +# please note that server's 613566753 is much more nonsensical than our new 1. +# returning NULL would be even more sensible. SELECT t1_dt, EXTRACT(DAY FROM t1_dt) 'DAY', EXTRACT(YEAR FROM t1_dt) 'YEAR', EXTRACT(MONTH FROM t1_dt) 'MONTH', EXTRACT(WEEK FROM t1_dt) 'WEEK' FROM t1; SELECT t1_dt, MONTH(t1_dt) FROM t1; @@ -82,6 +84,8 @@ SELECT t1_d, t1_dt, TIMESTAMPDIFF(DAY, t1_d, t1_dt) FROM t1; SELECT t1_d, t1_dt, TIMESTAMPDIFF(MINUTE, t1_d, t1_dt) FROM t1; SELECT t1_d, t1_dt, TIMESTAMPDIFF(SECOND, t1_d, t1_dt) FROM t1; +# please note that server's 613566753 is much more nonsensical than our new 1. +# returning NULL would be even more sensible. SELECT t1_d, WEEK(t1_d), t1_dt, WEEK(t1_dt) FROM t1; SELECT t1_d, YEAR(t1_d), t1_dt, YEAR(t1_dt) FROM t1; diff --git a/utils/funcexp/func_date_format.cpp b/utils/funcexp/func_date_format.cpp index a2e996e18..bb595aa5d 100644 --- a/utils/funcexp/func_date_format.cpp +++ b/utils/funcexp/func_date_format.cpp @@ -38,20 +38,21 @@ namespace funcexp { namespace helpers { -const string IDB_date_format(const DateTime& dt, const string& format) +const string IDB_date_format(const DateTime& dt, const string& format, bool& isNull) { // assume 256 is enough. assume not allowing incomplete date + // XXX: imagine %W gets repeated 60 times and day of week is "wednesday".. + std::ostringstream oss; char buf[256]; - char* ptr = buf; uint32_t weekday = 0; uint32_t dayval = 0; uint32_t weekval = 0; uint32_t weekyear = 0; - for (uint32_t i = 0; i < format.length(); i++) + for (uint32_t i = 0; !isNull && i < format.length(); i++) { if (format[i] != '%') - *ptr++ = format[i]; + oss << format[i]; else { i++; @@ -59,176 +60,170 @@ const string IDB_date_format(const DateTime& dt, const string& format) switch (format[i]) { case 'M': - sprintf(ptr, "%s", helpers::monthFullNames[dt.month].c_str()); - ptr += helpers::monthFullNames[dt.month].length(); + oss << helpers::monthFullNames[dt.month]; break; case 'b': - sprintf(ptr, "%s", helpers::monthAbNames[dt.month].c_str()); - ptr += helpers::monthAbNames[dt.month].length(); + oss << helpers::monthAbNames[dt.month].c_str(); break; case 'W': - weekday = helpers::calc_mysql_weekday(dt.year, dt.month, dt.day, false); - sprintf(ptr, "%s", helpers::weekdayFullNames[weekday].c_str()); - ptr += helpers::weekdayFullNames[weekday].length(); + weekday = helpers::calc_mysql_weekday(dt.year, dt.month, dt.day, false, isNull); + oss << helpers::weekdayFullNames[weekday]; break; case 'w': - weekday = helpers::calc_mysql_weekday(dt.year, dt.month, dt.day, true); - sprintf(ptr, "%01d", weekday); - ptr += 1; + weekday = helpers::calc_mysql_weekday(dt.year, dt.month, dt.day, true, isNull); + sprintf(buf, "%01d", weekday); + oss << buf; break; case 'a': - weekday = helpers::calc_mysql_weekday(dt.year, dt.month, dt.day, false); - sprintf(ptr, "%s", helpers::weekdayAbNames[weekday].c_str()); - ptr += helpers::weekdayAbNames[weekday].length(); + weekday = helpers::calc_mysql_weekday(dt.year, dt.month, dt.day, false, isNull); + oss << helpers::weekdayAbNames[weekday]; break; case 'D': - sprintf(ptr, "%s", helpers::dayOfMonth[dt.day].c_str()); - ptr += helpers::dayOfMonth[dt.day].length(); + oss << helpers::dayOfMonth[dt.day].c_str(); break; case 'Y': - sprintf(ptr, "%04d", dt.year); - ptr += 4; + sprintf(buf, "%04d", dt.year); + oss << buf; break; case 'y': - sprintf(ptr, "%02d", dt.year % 100); - ptr += 2; + sprintf(buf, "%02d", dt.year % 100); + oss << buf; break; case 'm': - sprintf(ptr, "%02d", dt.month); - ptr += 2; + sprintf(buf, "%02d", dt.month); + oss << buf; break; case 'c': - sprintf(ptr, "%d", dt.month); - ptr = ptr + (dt.month >= 10 ? 2 : 1); + sprintf(buf, "%d", dt.month); + oss << buf; break; case 'd': - sprintf(ptr, "%02d", dt.day); - ptr += 2; + sprintf(buf, "%02d", dt.day); + oss << buf; break; case 'e': - sprintf(ptr, "%d", dt.day); - ptr = ptr + (dt.day >= 10 ? 2 : 1); + sprintf(buf, "%d", dt.day); + oss << buf; break; case 'f': - sprintf(ptr, "%06d", dt.msecond); - ptr += 6; + sprintf(buf, "%06d", dt.msecond); + oss << buf; break; case 'H': - sprintf(ptr, "%02d", dt.hour); - ptr += 2; + sprintf(buf, "%02d", dt.hour); + oss << buf; break; case 'h': case 'I': - sprintf(ptr, "%02d", (dt.hour % 24 + 11) % 12 + 1); - ptr += 2; + sprintf(buf, "%02d", (dt.hour % 24 + 11) % 12 + 1); + oss << buf; break; case 'i': /* minutes */ - sprintf(ptr, "%02d", dt.minute); - ptr += 2; + sprintf(buf, "%02d", dt.minute); + oss << buf; break; case 'j': dayval = helpers::calc_mysql_daynr(dt.year, dt.month, dt.day) - helpers::calc_mysql_daynr(dt.year, 1, 1) + 1; - sprintf(ptr, "%03d", dayval); - ptr += 3; + sprintf(buf, "%03d", dayval); + oss << buf; break; case 'k': - sprintf(ptr, "%d", dt.hour); - ptr += (dt.hour >= 10 ? 2 : 1); + sprintf(buf, "%d", dt.hour); + oss << buf; break; case 'l': - sprintf(ptr, "%d", (dt.hour % 24 + 11) % 12 + 1); - ptr += ((dt.hour % 24 + 11) % 12 + 1 >= 10 ? 2 : 1); + sprintf(buf, "%d", (dt.hour % 24 + 11) % 12 + 1); + oss << buf; break; case 'p': - sprintf(ptr, "%s", (dt.hour % 24 < 12 ? "AM" : "PM")); - ptr += 2; + sprintf(buf, "%s", (dt.hour % 24 < 12 ? "AM" : "PM")); + oss << buf; break; case 'r': - sprintf(ptr, (dt.hour % 24 < 12 ? "%02d:%02d:%02d AM" : "%02d:%02d:%02d PM"), + sprintf(buf, (dt.hour % 24 < 12 ? "%02d:%02d:%02d AM" : "%02d:%02d:%02d PM"), (dt.hour + 11) % 12 + 1, dt.minute, dt.second); - ptr += 11; + oss << buf; break; case 'S': case 's': - sprintf(ptr, "%02d", dt.second); - ptr += 2; + sprintf(buf, "%02d", dt.second); + oss << buf; break; case 'T': - sprintf(ptr, "%02d:%02d:%02d", dt.hour, dt.minute, dt.second); - ptr += 8; + sprintf(buf, "%02d:%02d:%02d", dt.hour, dt.minute, dt.second); + oss << buf; break; case 'U': weekval = helpers::calc_mysql_week(dt.year, dt.month, dt.day, 0); - sprintf(ptr, "%02d", weekval); - ptr += 2; + sprintf(buf, "%02d", weekval); + oss << buf; break; case 'V': weekval = helpers::calc_mysql_week(dt.year, dt.month, dt.day, helpers::WEEK_NO_ZERO); - sprintf(ptr, "%02d", weekval); - ptr += 2; + sprintf(buf, "%02d", weekval); + oss << buf; break; case 'u': weekval = helpers::calc_mysql_week(dt.year, dt.month, dt.day, helpers::WEEK_MONDAY_FIRST | helpers::WEEK_GT_THREE_DAYS); - sprintf(ptr, "%02d", weekval); - ptr += 2; + sprintf(buf, "%02d", weekval); + oss << buf; break; case 'v': weekval = helpers::calc_mysql_week( dt.year, dt.month, dt.day, helpers::WEEK_NO_ZERO | helpers::WEEK_MONDAY_FIRST | helpers::WEEK_GT_THREE_DAYS); - sprintf(ptr, "%02d", weekval); - ptr += 2; + sprintf(buf, "%02d", weekval); + oss << buf; break; case 'x': helpers::calc_mysql_week( dt.year, dt.month, dt.day, helpers::WEEK_NO_ZERO | helpers::WEEK_MONDAY_FIRST | helpers::WEEK_GT_THREE_DAYS, &weekyear); - sprintf(ptr, "%04d", weekyear); - ptr += 4; + sprintf(buf, "%04d", weekyear); + oss << buf; break; case 'X': helpers::calc_mysql_week(dt.year, dt.month, dt.day, helpers::WEEK_NO_ZERO, &weekyear); - sprintf(ptr, "%04d", weekyear); - ptr += 4; + sprintf(buf, "%04d", weekyear); + oss << buf; break; - default: *ptr++ = format[i]; + default: oss << format[i]; } } } - *ptr = 0; - return string(buf); + return oss.str(); } } // namespace helpers @@ -394,7 +389,7 @@ string Func_date_format::getStrVal(rowgroup::Row& row, FunctionParm& parm, bool& const string& format = parm[1]->data()->getStrVal(row, isNull).safeString(""); - return helpers::IDB_date_format(dt, format); + return helpers::IDB_date_format(dt, format, isNull); } int32_t Func_date_format::getDateIntVal(rowgroup::Row& row, FunctionParm& parm, bool& isNull, diff --git a/utils/funcexp/func_dayname.cpp b/utils/funcexp/func_dayname.cpp index e418bc67d..a5c870207 100644 --- a/utils/funcexp/func_dayname.cpp +++ b/utils/funcexp/func_dayname.cpp @@ -163,7 +163,7 @@ int64_t Func_dayname::getIntVal(rowgroup::Row& row, FunctionParm& parm, bool& is default: isNull = true; return -1; } - dayofweek = helpers::calc_mysql_weekday(year, month, day, false); + dayofweek = helpers::calc_mysql_weekday(year, month, day, false, isNull); return dayofweek; } diff --git a/utils/funcexp/func_dayofweek.cpp b/utils/funcexp/func_dayofweek.cpp index b9700bdab..0f8b07253 100644 --- a/utils/funcexp/func_dayofweek.cpp +++ b/utils/funcexp/func_dayofweek.cpp @@ -170,7 +170,7 @@ int64_t Func_dayofweek::getIntVal(rowgroup::Row& row, FunctionParm& parm, bool& default: isNull = true; return -1; } - return helpers::calc_mysql_weekday(year, month, day, true) + 1; + return helpers::calc_mysql_weekday(year, month, day, true, isNull) + 1; } } // namespace funcexp diff --git a/utils/funcexp/func_dayofyear.cpp b/utils/funcexp/func_dayofyear.cpp index 215ec2150..5cd0dd47a 100644 --- a/utils/funcexp/func_dayofyear.cpp +++ b/utils/funcexp/func_dayofyear.cpp @@ -156,6 +156,11 @@ int64_t Func_dayofyear::getIntVal(rowgroup::Row& row, FunctionParm& parm, bool& default: isNull = true; return -1; } + if (year == 0 && month == 0 && day == 0) + { + isNull = true; + return 0; + } return helpers::calc_mysql_daynr(year, month, day) - helpers::calc_mysql_daynr(year, 1, 1) + 1; } diff --git a/utils/funcexp/func_extract.cpp b/utils/funcexp/func_extract.cpp index 14cb62051..4f04bb220 100644 --- a/utils/funcexp/func_extract.cpp +++ b/utils/funcexp/func_extract.cpp @@ -109,11 +109,11 @@ long long dateGet(uint64_t time, IntervalColumn::interval_type unit, bool dateTy long long timeGet(uint64_t time, IntervalColumn::interval_type unit) { - int32_t hour = 0, min = 0, sec = 0, msec = 0, day = 0; + int64_t hour = 0, min = 0, sec = 0, msec = 0, day = 0; - min = (int32_t)((time >> 32) & 0xff); - sec = (int32_t)((time >> 24) & 0xff); - msec = (int32_t)((time & 0xfffff)); + min = (int64_t)((time >> 32) & 0xff); + sec = (int64_t)((time >> 24) & 0xff); + msec = (int64_t)((time & 0xfffff)); // If negative, mask so it doesn't turn positive int64_t mask = 0; diff --git a/utils/funcexp/func_from_unixtime.cpp b/utils/funcexp/func_from_unixtime.cpp index 73103ad59..6f9279661 100644 --- a/utils/funcexp/func_from_unixtime.cpp +++ b/utils/funcexp/func_from_unixtime.cpp @@ -125,7 +125,7 @@ string Func_from_unixtime::getStrVal(rowgroup::Row& row, FunctionParm& parm, boo if (parm.size() == 2) { const auto& format = parm[1]->data()->getStrVal(row, isNull); - return helpers::IDB_date_format(dt, format.safeString("")); + return helpers::IDB_date_format(dt, format.safeString(""), isNull); } char buf[256] = {0}; diff --git a/utils/funcexp/func_round.cpp b/utils/funcexp/func_round.cpp index 3b765a916..2b7d68b72 100644 --- a/utils/funcexp/func_round.cpp +++ b/utils/funcexp/func_round.cpp @@ -389,19 +389,36 @@ IDB_Decimal Func_round::getDecimalVal(Row& row, FunctionParm& parm, bool& isNull //@Bug 3101 - GCC 4.5.1 optimizes too aggressively here. Mark as volatile. volatile int128_t p = 1; + if (isNull) + break; + if (!isNull && parm.size() > 1) // round(X, D) { int128_t nvp = p; d = parm[1]->data()->getIntVal(row, isNull); - if (!isNull) - helpers::decimalPlaceDec(d, nvp, decimal.scale); + if (isNull) + break; + + int64_t expectedScale = decimal.scale - d; + + // prevent overflow. + if (expectedScale > datatypes::INT128MAXPRECISION) + { + decimal.s128Value = 0; + break; + } + + // also do not allow for incorrect behavior due to underflow. + if (expectedScale < 0) + { + d += expectedScale; + } + helpers::decimalPlaceDec(d, nvp, decimal.scale); p = nvp; } - if (isNull) - break; if (d < -datatypes::INT128MAXPRECISION) { diff --git a/utils/funcexp/func_timediff.cpp b/utils/funcexp/func_timediff.cpp index bed01600a..68d4000bb 100644 --- a/utils/funcexp/func_timediff.cpp +++ b/utils/funcexp/func_timediff.cpp @@ -137,7 +137,8 @@ string Func_timediff::getStrVal(rowgroup::Row& row, FunctionParm& parm, bool& is isNull = true; break; } - val1 = parm[0]->data()->getDatetimeIntVal(row, isNull); + val1 = isTime1 ? parm[0]->data()->getTimeIntVal(row, isNull) + : parm[0]->data()->getDatetimeIntVal(row, isNull); break; case execplan::CalpontSystemCatalog::TIMESTAMP: @@ -225,7 +226,8 @@ string Func_timediff::getStrVal(rowgroup::Row& row, FunctionParm& parm, bool& is isTime2 = true; /* fall through */ case execplan::CalpontSystemCatalog::DATETIME: - val2 = parm[1]->data()->getDatetimeIntVal(row, isNull); + val2 = isTime2 ? parm[1]->data()->getTimeIntVal(row, isNull) + : parm[1]->data()->getDatetimeIntVal(row, isNull); break; case execplan::CalpontSystemCatalog::TIMESTAMP: diff --git a/utils/funcexp/func_truncate.cpp b/utils/funcexp/func_truncate.cpp index c8bafca5c..cccaa3666 100644 --- a/utils/funcexp/func_truncate.cpp +++ b/utils/funcexp/func_truncate.cpp @@ -315,25 +315,25 @@ IDB_Decimal Func_truncate::getDecimalVal(Row& row, FunctionParm& parm, bool& isN int64_t d = 0; decimal = parm[0]->data()->getDecimalVal(row, isNull); + if (isNull) + { + break; + } if (!op_ct.isWideDecimalType()) { //@Bug 3101 - GCC 4.5.1 optimizes too aggressively here. Mark as volatile. volatile int64_t p = 1; - if (!isNull) - { - int64_t nvp = p; - d = parm[1]->data()->getIntVal(row, isNull); - - if (!isNull) - helpers::decimalPlaceDec(d, nvp, decimal.scale); - - p = nvp; - } + int64_t nvp = p; + d = parm[1]->data()->getIntVal(row, isNull); if (isNull) break; + helpers::decimalPlaceDec(d, nvp, decimal.scale); + + p = nvp; + int64_t x = decimal.value; if (d > 0) @@ -371,20 +371,33 @@ IDB_Decimal Func_truncate::getDecimalVal(Row& row, FunctionParm& parm, bool& isN //@Bug 3101 - GCC 4.5.1 optimizes too aggressively here. Mark as volatile. volatile int128_t p = 1; + if (isNull) + break; + if (!isNull) { int128_t nvp = p; d = parm[1]->data()->getIntVal(row, isNull); - if (!isNull) - helpers::decimalPlaceDec(d, nvp, decimal.scale); + int64_t expectedScale = decimal.scale - d; + + // prevent overflow. + if (expectedScale > datatypes::INT128MAXPRECISION) + { + decimal.s128Value = 0; + break; + } + + // also do not allow for incorrect behavior due to underflow. + if (expectedScale < 0) + { + d += expectedScale; + } + helpers::decimalPlaceDec(d, nvp, decimal.scale); p = nvp; } - if (isNull) - break; - if (d < -datatypes::INT128MAXPRECISION) { decimal.s128Value = 0; diff --git a/utils/funcexp/func_weekday.cpp b/utils/funcexp/func_weekday.cpp index e78d12c66..7b7fb9377 100644 --- a/utils/funcexp/func_weekday.cpp +++ b/utils/funcexp/func_weekday.cpp @@ -160,7 +160,7 @@ int64_t Func_weekday::getIntVal(rowgroup::Row& row, FunctionParm& parm, bool& is default: isNull = true; return -1; } - return helpers::calc_mysql_weekday(year, month, day, false); + return helpers::calc_mysql_weekday(year, month, day, false, isNull); } } // namespace funcexp diff --git a/utils/funcexp/funchelpers.h b/utils/funcexp/funchelpers.h index 2738b674d..be65235d0 100644 --- a/utils/funcexp/funchelpers.h +++ b/utils/funcexp/funchelpers.h @@ -125,7 +125,7 @@ inline uint32_t calc_mysql_daynr(uint32_t year, uint32_t month, uint32_t day) int y = year; long delsum; - if (!dataconvert::isDateValid(day, month, year)) + if (!dataconvert::isDateValid(day, month, year) || (day == 0 && month == 0 && year == 0)) return 0; delsum = (long)(365 * y + 31 * ((int)month - 1) + (int)day); @@ -204,10 +204,13 @@ inline void get_date_from_mysql_daynr(long daynr, dataconvert::DateTime& dateTim // else: // 0 = Monday, 1 = Tuesday, ..., 6 = Sunday // This is a mirror of calc_weekday, at a later date we should use sql_time.h -inline uint32_t calc_mysql_weekday(uint32_t year, uint32_t month, uint32_t day, bool sundayFirst) +inline uint32_t calc_mysql_weekday(uint32_t year, uint32_t month, uint32_t day, bool sundayFirst, bool& isNull) { - if (!dataconvert::isDateValid(day, month, year)) + if (!dataconvert::isDateValid(day, month, year) || (day == 0 && month == 0 && year == 0)) + { + isNull = true; return 0; + } uint32_t daynr = calc_mysql_daynr(year, month, day); return ((int)((daynr + 5L + (sundayFirst ? 1L : 0L)) % 7)); @@ -252,7 +255,8 @@ inline uint32_t calc_mysql_week(uint32_t year, uint32_t month, uint32_t day, int bool week_year = modeflags & WEEK_NO_ZERO; bool first_weekday = modeflags & WEEK_GT_THREE_DAYS; - uint32_t weekday = calc_mysql_weekday(year, 1, 1, !monday_first); + bool isNullDummy = false; + uint32_t weekday = calc_mysql_weekday(year, 1, 1, !monday_first, isNullDummy); if (weekyear) { @@ -351,7 +355,7 @@ inline bool calc_time_diff(int64_t time1, int64_t time2, int l_sign, long long* days -= l_sign * calc_mysql_daynr(year2, month2, day2); - microseconds = ((long long)days * (long)(86400) + (long long)(hour1 * 3600L + min1 * 60L + sec1) - + microseconds = (int128_t(days) * (86400) + (long long)(hour1 * 3600L + min1 * 60L + sec1) - l_sign * (long long)(hour2 * 3600L + min2 * 60L + sec2)) * (long long)(1000000) + (long long)msec1 - l_sign * (long long)msec2; @@ -683,7 +687,7 @@ inline string longDoubleToString(long double ld) uint64_t dateAdd(uint64_t time, const std::string& expr, execplan::IntervalColumn::interval_type unit, bool dateType, execplan::OpType funcType); -const std::string IDB_date_format(const dataconvert::DateTime&, const std::string&); +const std::string IDB_date_format(const dataconvert::DateTime&, const std::string&, bool& isNull); const std::string timediff(int64_t, int64_t, bool isDateTime = true); const char* convNumToStr(int64_t, char*, int); diff --git a/utils/funcexp/tdriver.cpp b/utils/funcexp/tdriver.cpp index 8ab2fe614..ebcbfc9ea 100644 --- a/utils/funcexp/tdriver.cpp +++ b/utils/funcexp/tdriver.cpp @@ -275,8 +275,9 @@ class FuncExpTest : public CppUnit::TestFixture for (unsigned i = 0; i < sizeof(date_tests) / sizeof(DateCheck); i++) { boost::gregorian::date d(date_tests[i].date.year, date_tests[i].date.month, date_tests[i].date.day); + bool isNullDummy = false; uint32_t dayofweek = helpers::calc_mysql_weekday(date_tests[i].date.year, date_tests[i].date.month, - date_tests[i].date.day, false); + date_tests[i].date.day, false, isNullDummy); bool check = (strcmp(helpers::weekdayFullNames[dayofweek].c_str(), date_tests[i].dayname) == 0); diff --git a/utils/funcexp/timeextract.h b/utils/funcexp/timeextract.h index 5e0fbc792..181ae5021 100644 --- a/utils/funcexp/timeextract.h +++ b/utils/funcexp/timeextract.h @@ -69,7 +69,8 @@ class TimeExtractor uint32_t yearfirst = helpers::calc_mysql_daynr(dateTime.year, 1, 1); // figure out which day of week Jan-01 is - uint32_t firstweekday = helpers::calc_mysql_weekday(dateTime.year, 1, 1, sundayFirst); + bool isNullDummy = false; + uint32_t firstweekday = helpers::calc_mysql_weekday(dateTime.year, 1, 1, sundayFirst, isNullDummy); // calculate the offset to the first week starting day uint32_t firstoffset = firstweekday ? (7 - firstweekday) : 0; diff --git a/utils/loggingcpp/message.cpp b/utils/loggingcpp/message.cpp index f8edf9703..765c1bd47 100644 --- a/utils/loggingcpp/message.cpp +++ b/utils/loggingcpp/message.cpp @@ -42,6 +42,8 @@ using namespace config; #include "installdir.h" #include "format.h" +#include "mcs_int128.h" + namespace { boost::mutex mx; @@ -127,6 +129,11 @@ void Message::Args::add(uint64_t u64) fArgs.push_back(u64); } +void Message::Args::add(int128_t i128) +{ + fArgs.push_back(datatypes::TSInt128(i128).toString()); +} + void Message::Args::add(const string& s) { fArgs.push_back(s); diff --git a/utils/loggingcpp/messageobj.h b/utils/loggingcpp/messageobj.h index 2d67d0a2d..3d73ea1ab 100644 --- a/utils/loggingcpp/messageobj.h +++ b/utils/loggingcpp/messageobj.h @@ -65,6 +65,14 @@ class Message */ void add(uint64_t i); + /* define types to not to include mcs_numeric_limits.h */ + using int128_t = __int128; + using uint128_t = unsigned __int128; + + /** @brief add an 128 bit int arg to the message + */ + void add(int128_t i128); + /** @brief add a float arg to the message */ void add(double d);