Merge branch 'develop-1.2' into develop-merge-up-20190514

utils/cacheutils/cacheutils.cpp

@@ -258,7 +258,8 @@ int flushOIDsFromCache(const vector<BRM::OID_t>& oids)
     ISMPacketHeader ism;
     uint32_t i;
 
-    memset(&ism, 0, sizeof(ISMPacketHeader));
+    void *ismp = static_cast<void*>(&ism);
+    memset(ismp, 0, sizeof(ISMPacketHeader));
     ism.Command = CACHE_FLUSH_BY_OID;
     bs.load((uint8_t*) &ism, sizeof(ISMPacketHeader));
     bs << (uint32_t) oids.size();
@@ -285,7 +286,8 @@ int flushPartition(const std::vector<BRM::OID_t>& oids, set<BRM::LogicalPartitio
     ByteStream bs;
     ISMPacketHeader ism;
 
-    memset(&ism, 0, sizeof(ISMPacketHeader));
+    void *ismp = static_cast<void*>(&ism);
+    memset(ismp, 0, sizeof(ISMPacketHeader));
     ism.Command = CACHE_FLUSH_PARTITION;
     bs.load((uint8_t*) &ism, sizeof(ISMPacketHeader));
     serializeSet<BRM::LogicalPartition>(bs, partitionNums);

utils/compress/version1.cpp

@@ -90,6 +90,8 @@ namespace
 {
 short DSPort = 9199;
 
+// this isn't currently used but could be in the future.
+#if 0
 void log(const string& s)
 {
     logging::MessageLog logger((logging::LoggingID()));
@@ -100,6 +102,7 @@ void log(const string& s)
     message.format(args);
     logger.logErrorMessage(message);
 }
+#endif
 
 struct ScopedCleaner
 {

utils/dataconvert/dataconvert.cpp

@@ -83,19 +83,6 @@ bool from_string(T& t, const std::string& s, std::ios_base & (*f)(std::ios_base&
     return !(iss >> f >> t).fail();
 }
 
-uint64_t pow10_(int32_t scale)
-{
-    if (scale <= 0) return 1;
-
-    idbassert(scale < 20);
-    uint64_t res = 1;
-
-    for (int32_t i = 0; i < scale; i++)
-        res *= 10;
-
-    return res;
-}
-
 bool number_value ( const string& data )
 {
     for (unsigned int i = 0; i < strlen(data.c_str()); i++)
@@ -893,7 +880,6 @@ bool mysql_str_to_datetime( const string& input, DateTime& output, bool& isDate
 
 bool mysql_str_to_time( const string& input, Time& output, long decimals )
 {
-//    int32_t datesepct = 0;
     uint32_t dtend = 0;
     bool isNeg = false;
 
@@ -1779,7 +1765,8 @@ int32_t DataConvert::convertColumnDate(
 bool DataConvert::isColumnDateValid( int32_t date )
 {
     Date d;
-    memcpy(&d, &date, sizeof(int32_t));
+    void* dp = static_cast<void*>(&d);
+    memcpy(dp, &date, sizeof(int32_t));
     return (isDateValid(d.day, d.month, d.year));
 }
 
@@ -2081,7 +2068,8 @@ int64_t DataConvert::convertColumnTime(
 bool DataConvert::isColumnDateTimeValid( int64_t dateTime )
 {
     DateTime dt;
-    memcpy(&dt, &dateTime, sizeof(uint64_t));
+    void* dtp = static_cast<void*>(&dt);
+    memcpy(dtp, &dateTime, sizeof(uint64_t));
 
     if (isDateValid(dt.day, dt.month, dt.year))
         return isDateTimeValid(dt.hour, dt.minute, dt.second, dt.msecond);
@@ -2092,7 +2080,8 @@ bool DataConvert::isColumnDateTimeValid( int64_t dateTime )
 bool DataConvert::isColumnTimeValid( int64_t time )
 {
     Time dt;
-    memcpy(&dt, &time, sizeof(uint64_t));
+    void* dtp = static_cast<void*>(&dt);
+    memcpy(dtp, &time, sizeof(uint64_t));
 
     return isTimeValid(dt.hour, dt.minute, dt.second, dt.msecond);
 }
@@ -2178,7 +2167,8 @@ std::string DataConvert::datetimeToString1( long long  datetimevalue )
 {
     // @bug 4703 abandon multiple ostringstream's for conversion
     DateTime dt(datetimevalue);
-    const int DATETIMETOSTRING1_LEN = 22; // YYYYMMDDHHMMSSmmmmmm\0
+    // Interesting, gcc 7 says the sprintf below generates between 21 and 23 bytes of output.
+    const int DATETIMETOSTRING1_LEN = 23; // YYYYMMDDHHMMSSmmmmmm\0
     char buf[DATETIMETOSTRING1_LEN];
 
     sprintf(buf, "%04d%02d%02d%02d%02d%02d%06d", dt.year, dt.month, dt.day, dt.hour, dt.minute, dt.second, dt.msecond);
@@ -2443,11 +2433,10 @@ int64_t DataConvert::stringToDatetime(const string& data, bool* date)
         return -1;
 }
 
+/* This is really painful and expensive b/c it seems the input is not normalized or
+sanitized.  That should really be done on ingestion. */
 int64_t DataConvert::intToDate(int64_t data)
 {
-    //char buf[10] = {0};
-    //snprintf( buf, 10, "%llu", (long long unsigned int)data);
-    //string date = buf;
     char buf[21] = {0};
     Date aday;
 
@@ -2459,7 +2448,16 @@ int64_t DataConvert::intToDate(int64_t data)
         return *(reinterpret_cast<uint32_t*>(&aday));
     }
 
+    // this snprintf call causes a compiler warning b/c we're potentially copying a 20-digit #
+    // into 15 bytes, however, that appears to be intentional.
+#if defined(__GNUC__) && __GNUC__ >= 6
+#pragma GCC diagnostic push
+#pragma GCC diagnostic ignored "-Wformat-truncation="
     snprintf( buf, 15, "%llu", (long long unsigned int)data);
+#pragma GCC diagnostic pop
+#else
+    snprintf( buf, 15, "%llu", (long long unsigned int)data);
+#endif
 
     string year, month, day, hour, min, sec, msec;
     int64_t y = 0, m = 0, d = 0, h = 0, minute = 0, s = 0, ms = 0;
@@ -2562,6 +2560,8 @@ int64_t DataConvert::intToDate(int64_t data)
     return *(reinterpret_cast<uint32_t*>(&aday));
 }
 
+/* This is really painful and expensive b/c it seems the input is not normalized or
+sanitized.  That should really be done on ingestion. */
 int64_t DataConvert::intToDatetime(int64_t data, bool* date)
 {
     bool isDate = false;
@@ -2584,7 +2584,17 @@ int64_t DataConvert::intToDatetime(int64_t data, bool* date)
         return *(reinterpret_cast<uint64_t*>(&adaytime));
     }
 
+    // this snprintf call causes a compiler warning b/c we're potentially copying a 20-digit #
+    // into 15 bytes, however, that appears to be intentional.
+#if defined(__GNUC__) && __GNUC__ >= 6
+#pragma GCC diagnostic push
+#pragma GCC diagnostic ignored "-Wformat-truncation="
     snprintf( buf, 15, "%llu", (long long unsigned int)data);
+#pragma GCC diagnostic pop
+#else
+    snprintf( buf, 15, "%llu", (long long unsigned int)data);
+#endif
+
     //string date = buf;
     string year, month, day, hour, min, sec, msec;
     int64_t y = 0, m = 0, d = 0, h = 0, minute = 0, s = 0, ms = 0;
@@ -2692,6 +2702,8 @@ int64_t DataConvert::intToDatetime(int64_t data, bool* date)
     return *(reinterpret_cast<uint64_t*>(&adaytime));
 }
 
+/* This is really painful and expensive b/c it seems the input is not normalized or
+sanitized.  That should really be done on ingestion. */
 int64_t DataConvert::intToTime(int64_t data, bool fromString)
 {
     char buf[21] = {0};
@@ -2710,7 +2722,17 @@ int64_t DataConvert::intToTime(int64_t data, bool fromString)
         return *(reinterpret_cast<int64_t*>(&atime));
     }
 
+    // this snprintf call causes a compiler warning b/c we're potentially copying a 20-digit #
+    // into 15 bytes, however, that appears to be intentional.
+#if defined(__GNUC__) && __GNUC__ >= 6
+#pragma GCC diagnostic push
+#pragma GCC diagnostic ignored "-Wformat-truncation="
     snprintf( buf, 15, "%llu", (long long unsigned int)data);
+#pragma GCC diagnostic pop
+#else
+    snprintf( buf, 15, "%llu", (long long unsigned int)data);
+#endif
+
     //string date = buf;
     string hour, min, sec, msec;
     int64_t h = 0, minute = 0, s = 0, ms = 0;

utils/dataconvert/dataconvert.h

@@ -673,12 +673,24 @@ inline void DataConvert::timeToString1( long long timevalue, char* buf, unsigned
         buf++;
         buflen--;
     }
-
+    // this snprintf call causes a compiler warning b/c buffer size is less
+    // then maximum string size.
+#if defined(__GNUC__) && __GNUC__ >= 6
+#pragma GCC diagnostic push
+#pragma GCC diagnostic ignored "-Wformat-truncation="
     snprintf( buf, buflen, "%02d%02d%02d",
               hour,
               (unsigned)((timevalue >> 32) & 0xff),
               (unsigned)((timevalue >> 14) & 0xff)
             );
+#pragma GCC diagnostic pop
+#else
+    snprintf( buf, buflen, "%02d%02d%02d",
+              hour,
+              (unsigned)((timevalue >> 32) & 0xff),
+              (unsigned)((timevalue >> 14) & 0xff)
+            );
+#endif
 }
 
 inline std::string DataConvert::decimalToString(int64_t value, uint8_t scale, execplan::CalpontSystemCatalog::ColDataType colDataType)

utils/funcexp/func_insert.cpp

@@ -89,11 +89,13 @@ std::string Func_insert::getStrVal(rowgroup::Row& row,
                                    execplan::CalpontSystemCatalog::ColType&)
 {
 	string tstr;
+	string tnewstr;
     stringValue(fp[0], row, isNull, tstr);
     if (isNull)
+    {
         return "";
+    }
 
-	string tnewstr;
     stringValue(fp[3], row, isNull, tnewstr);
     if (isNull)
         return "";

utils/funcexp/func_lpad.cpp

@@ -103,8 +103,6 @@ std::string Func_lpad::getStrVal(rowgroup::Row& row,
                 value += 0.5;
             else if (value < 0)
                 value -= 0.5;
-            else if (value < 0)
-                value -= 0.5;
 
             int64_t ret = (int64_t) value;
 

utils/funcexp/func_md5.cpp

@@ -82,7 +82,7 @@ typedef unsigned      char uchar;
 
 char* PrintMD5(uchar md5Digest[16]);
 char* MD5String(const char* szString);
-char* MD5File(char* szFilename);
+//char* MD5File(char* szFilename);
 
 class md5
 {
@@ -236,6 +236,7 @@ char* PrintMD5(uchar md5Digest[16])
     char chBuffer[256];
     char chEach[10];
     int nCount;
+    size_t chEachSize = 0;
 
     memset(chBuffer, 0, 256);
     memset(chEach, 0, 10);
@@ -243,7 +244,8 @@ char* PrintMD5(uchar md5Digest[16])
     for (nCount = 0; nCount < 16; nCount++)
     {
         sprintf(chEach, "%02x", md5Digest[nCount]);
-        strncat(chBuffer, chEach, sizeof(chEach));
+        chEachSize = sizeof(chEach);
+        strncat(chBuffer, chEach, chEachSize);
     }
 
     return strdup(chBuffer);
@@ -263,7 +265,9 @@ char* MD5String(const char* szString)
 
 }
 
-// MD5File: Performs the MD5 algorithm on a file (binar or text),
+// this fcn isn't used, so commenting it
+#if 0
+// MD5File: Performs the MD5 algorithm on a file (binary or text),
 // returning the results as a char*.  Returns NULL if it fails.
 char* MD5File(char* szFilename)
 {
@@ -295,7 +299,7 @@ char* MD5File(char* szFilename)
 
     return NULL; // failed
 }
-
+#endif
 
 // md5::Init
 // Initializes a new context.

utils/funcexp/func_repeat.cpp

@@ -93,7 +93,7 @@ std::string Func_repeat::getStrVal(rowgroup::Row& row,
 
     for ( int i = 0 ; i < count ; i ++ )
     {
-        if (strcat(result, str.c_str()) == NULL)
+        if (strcat(result, str.c_str()) == NULL) //questionable check
             return "";
     }
 

utils/funcexp/func_substring_index.cpp

@@ -76,7 +76,7 @@ std::string Func_substring_index::getStrVal(rowgroup::Row& row,
     if ( count >  end )
         return str;
 
-    if (( count < 0 ) && ((count * -1) > end))
+    if (( count < 0 ) && ((count * -1) > (int64_t) end))
         return str;
 
     string value = str;
@@ -99,8 +99,7 @@ std::string Func_substring_index::getStrVal(rowgroup::Row& row,
     }
     else
     {
-        count = count * -1;
-        size_t end = strlen(str.c_str());
+        count = -count;
         int pointer = end;
         int start = 0;
 

utils/idbdatafile/PosixFileSystem.cpp

@@ -261,7 +261,7 @@ int PosixFileSystem::listDirectory(const char* pathname, std::list<std::string>&
             contents.push_back( itr->path().filename().generic_string() );
         }
     }
-    catch (std::exception)
+    catch (std::exception &)
     {
         ret = -1;
     }

utils/idbhdfs/hdfs-shared/HdfsRdwrFileBuffer.cpp

@@ -151,7 +151,7 @@ HdfsRdwrFileBuffer::HdfsRdwrFileBuffer(const char* fname, const char* mode, unsi
 
 // This constructor is for use by HdfsRdwrMemBuffer to create a file buffer when we
 // run out of memory.
-HdfsRdwrFileBuffer::HdfsRdwrFileBuffer(HdfsRdwrMemBuffer* pMemBuffer) throw (std::exception) :
+HdfsRdwrFileBuffer::HdfsRdwrFileBuffer(HdfsRdwrMemBuffer* pMemBuffer) :
     IDBDataFile(pMemBuffer->name().c_str()),
     m_buffer(NULL),
     m_dirty(false)

utils/idbhdfs/hdfs-shared/HdfsRdwrFileBuffer.h

@@ -50,7 +50,7 @@ class HdfsRdwrFileBuffer: public IDBDataFile, boost::noncopyable
 {
 public:
     HdfsRdwrFileBuffer(const char* fname, const char* mode, unsigned opts);
-    HdfsRdwrFileBuffer(HdfsRdwrMemBuffer* pMemBuffer) throw (std::exception);
+    HdfsRdwrFileBuffer(HdfsRdwrMemBuffer* pMemBuffer);
     /* virtual */ ~HdfsRdwrFileBuffer();
 
     /* virtual */ ssize_t pread(void* ptr, off64_t offset, size_t count);

utils/loggingcpp/messagelog.cpp

@@ -180,21 +180,21 @@ const string MessageLog::format(const Message& msg, const char prefix)
 void MessageLog::logDebugMessage(const Message& msg)
 {
     ::openlog(SubsystemID[fLogData.fSubsysID].c_str(), 0 | LOG_PID, fFacility);
-    ::syslog(LOG_DEBUG, format(msg, 'D').c_str());
+    ::syslog(LOG_DEBUG, "%s", format(msg, 'D').c_str());
     ::closelog();
 }
 
 void MessageLog::logInfoMessage(const Message& msg)
 {
     ::openlog(SubsystemID[fLogData.fSubsysID].c_str(), 0 | LOG_PID, fFacility);
-    ::syslog(LOG_INFO, format(msg, 'I').c_str());
+    ::syslog(LOG_INFO, "%s", format(msg, 'I').c_str());
     ::closelog();
 }
 
 void MessageLog::logWarningMessage(const Message& msg)
 {
     ::openlog(SubsystemID[fLogData.fSubsysID].c_str(), 0 | LOG_PID, fFacility);
-    ::syslog(LOG_WARNING, format(msg, 'W').c_str());
+    ::syslog(LOG_WARNING, "%s", format(msg, 'W').c_str());
     ::closelog();
 }
 
@@ -202,14 +202,14 @@ void MessageLog::logErrorMessage(const Message& msg)
 {
     // @bug 24 use 'E' instead of 'S'
     ::openlog(SubsystemID[fLogData.fSubsysID].c_str(), 0 | LOG_PID, fFacility);
-    ::syslog(LOG_ERR, format(msg, 'E').c_str());
+    ::syslog(LOG_ERR, "%s", format(msg, 'E').c_str());
     ::closelog();
 }
 
 void MessageLog::logCriticalMessage(const Message& msg)
 {
     ::openlog(SubsystemID[fLogData.fSubsysID].c_str(), 0 | LOG_PID, fFacility);
-    ::syslog(LOG_CRIT, format(msg, 'C').c_str());
+    ::syslog(LOG_CRIT, "%s", format(msg, 'C').c_str());
     ::closelog();
 }
 //Bug 5218. comment out the following functions to alleviate issue where dml messages show up in crit.log. This

utils/messageqcpp/inetstreamsocket.cpp

@@ -945,7 +945,6 @@ void InetStreamSocket::connect(const sockaddr* serv_addr)
     /* read a byte to artificially synchronize with accept() on the remote */
     int ret = -1;
     int e = EBADF;
-    char buf = '\0';
     struct pollfd pfd;
 
     long msecs = fConnectionTimeout.tv_sec * 1000 + fConnectionTimeout.tv_nsec / 1000000;
@@ -964,9 +963,19 @@ void InetStreamSocket::connect(const sockaddr* serv_addr)
     if (ret == 1)
     {
 #ifdef _MSC_VER
+        char buf = '\0';
         (void)::recv(socketParms().sd(), &buf, 1, 0);
 #else
-        (void)::read(socketParms().sd(), &buf, 1);
+#if defined(__GNUC__) && __GNUC__ >= 6
+#pragma GCC diagnostic push
+#pragma GCC diagnostic ignored "-Wunused-result"
+        char buf = '\0';
+        ::read(socketParms().sd(), &buf, 1);   // we know 1 byte is in the recv buffer
+#pragma GCC diagnostic pop
+#else
+        char buf = '\0';
+        ::read(socketParms().sd(), &buf, 1);   // we know 1 byte is in the recv buffer
+#endif // pragma
 #endif
         return;
     }

utils/querytele/queryteleprotoimpl.cpp

@@ -80,6 +80,7 @@ struct QStats
 
 QStats fQStats;
 
+#ifdef QUERY_TELE_DEBUG
 string get_trace_file()
 {
     ostringstream oss;
@@ -125,7 +126,9 @@ void log_query(const querytele::QueryTele& qtdata)
     trace << endl;
     trace.close();
 }
+#endif
 
+#ifdef QUERY_TELE_DEBUG
 const string st2str(enum querytele::StepType::type t)
 {
     switch (t)
@@ -172,7 +175,9 @@ const string st2str(enum querytele::StepType::type t)
 
     return "INV";
 }
+#endif
 
+#ifdef QUERY_TELE_DEBUG
 void log_step(const querytele::StepTele& stdata)
 {
     ofstream trace(get_trace_file().c_str(), ios::out | ios::app);
@@ -207,6 +212,7 @@ void log_step(const querytele::StepTele& stdata)
     trace << endl;
     trace.close();
 }
+#endif
 
 void TeleConsumer()
 {

utils/rwlock/rwlock.cpp

@@ -32,7 +32,9 @@
 #include <tr1/unordered_map>
 #endif
 
+#ifndef NDEBUG
 #define NDEBUG
+#endif
 #include <cassert>
 using namespace std;
 

utils/threadpool/prioritythreadpool.cpp

@@ -139,8 +139,8 @@ PriorityThreadPool::Priority PriorityThreadPool::pickAQueue(Priority preference)
 
 void PriorityThreadPool::threadFcn(const Priority preferredQueue) throw()
 {
-    Priority queue;
-    uint32_t weight, i;
+    Priority queue = LOW;
+    uint32_t weight, i = 0;
     vector<Job> runList;
     vector<bool> reschedule;
     uint32_t rescheduleCount;

utils/threadpool/threadpool.h

@@ -44,6 +44,8 @@
 #include <boost/shared_ptr.hpp>
 #include <boost/function.hpp>
 
+#include <memory>
+
 #if defined(_MSC_VER) && defined(xxxTHREADPOOL_DLLEXPORT)
 #define EXPORT __declspec(dllexport)
 #else
@@ -75,8 +77,13 @@ public:
     boost::thread* create_thread(F threadfunc)
     {
         boost::lock_guard<boost::shared_mutex> guard(m);
-        threads.push_back(new boost::thread(threadfunc));
-        return threads.back();
+#if defined(__GNUC__) && __GNUC__ >= 7
+        std::unique_ptr<boost::thread> new_thread(new boost::thread(threadfunc));
+#else
+        std::auto_ptr<boost::thread> new_thread(new boost::thread(threadfunc));
+#endif
+        threads.push_back(new_thread.get());
+        return new_thread.release();
     }
 
     void add_thread(boost::thread* thrd)

utils/udfsdk/mcsv1_udaf.h

@@ -1027,6 +1027,10 @@ inline T mcsv1_UDAF::convertAnyTo(static_any::any& valIn)
     {
         val = valIn.cast<double>();
     }
+    else
+    {
+        throw runtime_error("mcsv1_UDAF::convertAnyTo(): input param has unrecognized type");
+    }
     return val;
 }