MCOL-1647 Fix TIME regressions

Fixes the following:

* Read past buffer end in intToDatetime / intToTime
* Allow intToTime to convert datetime
* Allow intToTime to convert shortened time values
* Allow stringToTime to convert datetime and int time values
* Fix saturation / bad values in intToTime and stringToTime
* Fix TIME return in STR_TO_DATE()
* Fix NULL return on type inequality for TIMEDIFF()
* Fix zero day calculation error in ADDTIME()/SUBTIME()
* Fix DATETIME to int calculation error in aggregate bit operations
* Make the new harderning flags optional with -DSECURITY_HARDENED_NEW

CMakeLists.txt

@@ -122,6 +122,7 @@ ELSE()
   SET(security_default ON)
 ENDIF()
 OPTION(SECURITY_HARDENED "Use security-enhancing compiler features (stack protector, relro, etc)" ${security_default})
+OPTION(SECURITY_HARDENED_NEW "Use new security-enhancing compilier features" OFF)
 IF(SECURITY_HARDENED)
   # security-enhancing flags
   MY_CHECK_AND_SET_COMPILER_FLAG("-pie -fPIC")
@@ -129,12 +130,14 @@ IF(SECURITY_HARDENED)
   MY_CHECK_AND_SET_COMPILER_FLAG("-fstack-protector --param=ssp-buffer-size=4")
   MY_CHECK_AND_SET_COMPILER_FLAG("-D_FORTIFY_SOURCE=2" RELEASE RELWITHDEBINFO)
   MY_CHECK_AND_SET_COMPILER_FLAG("-fexceptions")
-  MY_CHECK_AND_SET_COMPILER_FLAG("-mcet -fcf-protection")
+  IF(SECURITY_HARDENED_NEW)
-  MY_CHECK_AND_SET_COMPILER_FLAG("-fstack-protector-strong")
+    MY_CHECK_AND_SET_COMPILER_FLAG("-mcet -fcf-protection")
-  MY_CHECK_AND_SET_COMPILER_FLAG("-fstack-clash-protection")
+    MY_CHECK_AND_SET_COMPILER_FLAG("-fstack-protector-strong")
+    MY_CHECK_AND_SET_COMPILER_FLAG("-fstack-clash-protection")
+  ENDIF()
 ENDIF()
 
-SET (ENGINE_LDFLAGS    "-Wl,--no-as-needed -Wl,--add-needed")  
+SET (ENGINE_LDFLAGS    "-Wl,--no-as-needed -Wl,--add-needed")
 
 
 FIND_PACKAGE(Boost 1.53.0 REQUIRED COMPONENTS system filesystem thread regex date_time) 

utils/dataconvert/dataconvert.cpp

@@ -2566,7 +2566,6 @@ int64_t DataConvert::intToDatetime(int64_t data, bool* date)
             hour = string(buf + 8, 2);
             min = string(buf + 10, 2);
             sec = string(buf + 12, 2);
-            msec = string(buf + 14, 6);
             break;
 
         case 12:
@@ -2576,7 +2575,6 @@ int64_t DataConvert::intToDatetime(int64_t data, bool* date)
             hour = string(buf + 6, 2);
             min = string(buf + 8, 2);
             sec = string(buf + 10, 2);
-            msec = string(buf + 12, 6);
             break;
 
         case 10:
@@ -2585,7 +2583,6 @@ int64_t DataConvert::intToDatetime(int64_t data, bool* date)
             hour = string(buf + 4, 2);
             min = string(buf + 6, 2);
             sec = string(buf + 8, 2);
-            msec = string(buf + 10, 6);
             break;
 
         case 9:
@@ -2594,7 +2591,6 @@ int64_t DataConvert::intToDatetime(int64_t data, bool* date)
             hour = string(buf + 3, 2);
             min = string(buf + 5, 2);
             sec = string(buf + 7, 2);
-            msec = string(buf + 9, 6);
             break;
 
         case 8:
@@ -2645,7 +2641,7 @@ int64_t DataConvert::intToDatetime(int64_t data, bool* date)
     h = atoi(hour.c_str());
     minute = atoi(min.c_str());
     s = atoi(sec.c_str());
-    ms = atoi(msec.c_str());
+    ms = 0;
 
     if (!isDateValid(d, m, y) || !isDateTimeValid(h, minute, s, ms))
         return -1;
@@ -2664,7 +2660,7 @@ int64_t DataConvert::intToDatetime(int64_t data, bool* date)
     return *(reinterpret_cast<uint64_t*>(&adaytime));
 }
 
-int64_t DataConvert::intToTime(int64_t data)
+int64_t DataConvert::intToTime(int64_t data, bool fromString)
 {
     char buf[21] = {0};
     char* bufread = buf;
@@ -2693,43 +2689,78 @@ int64_t DataConvert::intToTime(int64_t data)
         bufread++;
     }
 
+    bool zero = false;
+
     switch (strlen(bufread))
     {
+        // A full datetime
+        case 14:
+            hour = string(buf + 8, 2);
+            min = string(buf + 10, 2);
+            sec = string(buf + 12, 2);
+            break;
+
+        // Date so this is all 0
+        case 8:
+            zero = true;
+            break;
+
         case 7:
             hour = string(bufread, 3);
             min = string(bufread + 2, 2);
             sec = string(bufread + 4, 2);
-            msec = string(bufread + 6, 6);
             break;
 
         case 6:
             hour = string(bufread, 2);
             min = string(bufread + 2, 2);
             sec = string(bufread + 4, 2);
-            msec = string(bufread + 6, 6);
+            break;
+
+        case 5:
+            hour = string(bufread, 1);
+            min = string(bufread + 1, 2);
+            sec = string(bufread + 3, 2);
             break;
 
         case 4:
             min = string(bufread, 2);
             sec = string(bufread + 2, 2);
-            msec = string(bufread + 4, 6);
+            break;
+
+        case 3:
+            min = string(bufread, 1);
+            sec = string(bufread + 1, 2);
             break;
 
         case 2:
             sec = string(bufread, 2);
-            msec = string(bufread + 2, 6);
+            break;
+
+        case 1:
+            sec = string(bufread, 1);
             break;
 
         default:
             return -1;
     }
 
-    h = atoi(hour.c_str());
+    if (!zero)
-    minute = atoi(min.c_str());
+    {
-    s = atoi(sec.c_str());
+        h = atoi(hour.c_str());
-    ms = atoi(msec.c_str());
+        minute = atoi(min.c_str());
+        s = atoi(sec.c_str());
+    }
+    else if (fromString)
+    {
+        // Saturate fromString
+        h = 838;
+        minute = 59;
+        s = 59;
+        ms = 999999;
+    }
 
-    if (!isTimeValid(h, minute, s, ms))
+    if (!isTimeValid(h, minute, s, 0))
         return -1;
 
     atime.hour = h;
@@ -2749,6 +2780,7 @@ int64_t DataConvert::stringToTime(const string& data)
     uint64_t min = 0, sec = 0, msec = 0;
     int64_t day = -1, hour = 0;
     bool isNeg = false;
+    bool hasDate = false;
     string time, hms, ms;
     char* end = NULL;
 
@@ -2760,18 +2792,27 @@ int64_t DataConvert::stringToTime(const string& data)
         isNeg = true;
     }
 
+    if (data.substr(pos+1, data.length()-pos-1).find("-") != string::npos)
+    {
+        // A second dash, this has a date
+        hasDate = true;
+        isNeg = false;
+    }
     // Day
     pos = data.find(" ");
 
     if (pos != string::npos)
     {
-        day = strtol(data.substr(0, pos).c_str(), &end, 10);
+        if (!hasDate)
+        {
+            day = strtol(data.substr(0, pos).c_str(), &end, 10);
 
-        if (*end != '\0')
+            if (*end != '\0')
-            return -1;
+                return -1;
 
-        hour = day * 24;
+            hour = day * 24;
-        day = -1;
+            day = -1;
+        }
         time = data.substr(pos + 1, data.length() - pos - 1);
     }
     else
@@ -2779,6 +2820,22 @@ int64_t DataConvert::stringToTime(const string& data)
         time = data;
     }
 
+    if (time.find(":") == string::npos)
+    {
+        if (hasDate)
+        {
+            // Has dashes, no colons. This is just a date!
+            // Or the length < 6 (MariaDB returns NULL)
+            return -1;
+        }
+        else
+        {
+            // This is an int time
+            return intToTime(atoll(time.c_str()), true);
+        }
+    }
+
+
     // Fraction
     pos = time.find(".");
 
@@ -2797,11 +2854,18 @@ int64_t DataConvert::stringToTime(const string& data)
 
     if (pos == string::npos)
     {
-        hour += atoi(hms.c_str());
+        if (hour >= 0)
+            hour += atoi(hms.c_str());
+        else
+            hour -= atoi(hms.c_str());
     }
     else
     {
-        hour += atoi(hms.substr(0, pos).c_str());
+        if (hour >= 0)
+            hour += atoi(hms.substr(0, pos).c_str());
+        else
+            hour -= atoi(hms.substr(0, pos).c_str());
+
         ms = hms.substr(pos + 1, hms.length() - pos - 1);
     }
 

utils/dataconvert/dataconvert.h

@@ -541,7 +541,7 @@ public:
     // convert integer to datetime
     EXPORT static int64_t intToDatetime(int64_t data, bool* isDate = NULL);
     // convert integer to date
-    EXPORT static int64_t intToTime(int64_t data);
+    EXPORT static int64_t intToTime(int64_t data, bool fromString = false);
     // convert string to date. alias to stringToDate
     EXPORT static int64_t dateToInt(const std::string& date);
     // convert string to datetime. alias to datetimeToInt

utils/funcexp/func_str_to_date.cpp

@@ -198,6 +198,24 @@ int64_t Func_str_to_date::getDatetimeIntVal(rowgroup::Row& row,
     return time;
 }
 
+int64_t Func_str_to_date::getTimeIntVal(rowgroup::Row& row,
+                                    FunctionParm& parm,
+                                    bool& isNull,
+                                    CalpontSystemCatalog::ColType& ct)
+{
+    dataconvert::DateTime dateTime;
+    dataconvert::Time retTime;
+    dateTime = getDateTime(row, parm, isNull, ct);
+    retTime.day = 0;
+    retTime.is_neg = false;
+    retTime.hour = dateTime.hour;
+    retTime.minute = dateTime.minute;
+    retTime.second = dateTime.second;
+    retTime.msecond = dateTime.msecond;
+    int64_t time = *(reinterpret_cast<int64_t*>(&retTime));
+    return time;
+}
+
 int64_t Func_str_to_date::getIntVal(rowgroup::Row& row,
                                     FunctionParm& parm,
                                     bool& isNull,

utils/funcexp/func_timediff.cpp

@@ -109,12 +109,6 @@ string Func_timediff::getStrVal(rowgroup::Row& row,
     int64_t val1 = -1, val2 = -1;
     bool isDate1 = false, isDate2 = false;
 
-    if (type1 != type2)
-    {
-        isNull = true;
-        return "";
-    }
-
     switch (type1)
     {
         case execplan::CalpontSystemCatalog::DATE:

utils/funcexp/functor.cpp

@@ -228,7 +228,7 @@ int64_t Func::addTime(DateTime& dt1, Time& dt2)
     month = dt1.month;
     int addyear = 0;
 
-    if (day < 0)
+    if (day <= 0)
     {
         int monthSave = month;
 

utils/funcexp/functor_dtm.h

@@ -473,6 +473,10 @@ public:
                               FunctionParm& fp,
                               bool& isNull,
                               execplan::CalpontSystemCatalog::ColType& op_ct);
+    int64_t getTimeIntVal(rowgroup::Row& row,
+                              FunctionParm& fp,
+                              bool& isNull,
+                              execplan::CalpontSystemCatalog::ColType& op_ct);
 };
 
 

utils/rowgroup/rowaggregation.cpp

@@ -1547,9 +1547,9 @@ void RowAggregation::doBitOp(const Row& rowIn, int64_t colIn, int64_t colOut, in
         case execplan::CalpontSystemCatalog::DATETIME:
         {
             uint64_t dtm = rowIn.getUintField(colIn);
-            valIn = ((dtm >> 48) * 10000000000000000LL) + (((dtm >> 44) & 0xF) * 100000000000000) +
+            valIn = ((dtm >> 48) * 10000000000LL) + (((dtm >> 44) & 0xF) * 100000000) +
-                    (((dtm >> 38) & 077) * 1000000000000) + (((dtm >> 32) & 077) * 10000000000) +
+                    (((dtm >> 38) & 077) * 1000000) + (((dtm >> 32) & 077) * 10000) +
-                    (((dtm >> 26) & 077) * 100000000) + (((dtm >> 20) & 077) * 1000000) + (dtm & 0xfffff);
+                    (((dtm >> 26) & 077) * 100) + ((dtm >> 20) & 077);
             break;
         }
 
@@ -1565,8 +1565,8 @@ void RowAggregation::doBitOp(const Row& rowIn, int64_t colIn, int64_t colOut, in
             }
 
             hour |= ((dtm >> 40) & 0xfff);
-            valIn = (hour * 10000000000) +
+            valIn = (hour * 10000) +
-                    (((dtm >> 32) & 0xff) * 100000000) + (((dtm >> 24) & 0xff) * 1000000) + (dtm & 0xffffff);
+                    (((dtm >> 32) & 0xff) * 100) + ((dtm >> 24) & 0xff);
             break;
         }