MCOL-537, cleanup compiler warnings. Checkpointing a bunch of fixes.

Work in progress...

dbcon/execplan/predicateoperator.cpp

@@ -46,22 +46,6 @@ struct to_lower
     }
 };
 
-//Trim any leading/trailing ws
-const string lrtrim(const string& in)
-{
-    string::size_type p1;
-    p1 = in.find_first_not_of(" \t\n");
-
-    if (p1 == string::npos) p1 = 0;
-
-    string::size_type p2;
-    p2 = in.find_last_not_of(" \t\n");
-
-    if (p2 == string::npos) p2 = in.size() - 1;
-
-    return string(in, p1, (p2 - p1 + 1));
-}
-
 }
 
 namespace execplan

dbcon/joblist/jlf_subquery.cpp

@@ -336,7 +336,8 @@ bool isNotInSubquery(JobStepVector& jsv)
     return notIn;
 }
 
-
+// This fcn is currently unused.  Will keep it in the code for now.
+#if 0
 void alterCsepInExistsFilter(CalpontSelectExecutionPlan* csep, JobInfo& jobInfo)
 {
     // This is for window function in IN/EXISTS sub-query.
@@ -364,7 +365,7 @@ void alterCsepInExistsFilter(CalpontSelectExecutionPlan* csep, JobInfo& jobInfo)
     if (wcs.size() > 1)
         retCols.insert(retCols.end(), wcs.begin() + 1, wcs.end());
 }
-
+#endif
 
 void doCorrelatedExists(const ExistsFilter* ef, JobInfo& jobInfo)
 {

dbcon/joblist/joblistfactory.cpp

@@ -93,38 +93,6 @@ namespace
 {
 using namespace joblist;
 
-//Find the next step downstream from *in. Assumes only the first such step is needed.
-const JobStepVector::iterator getNextStep(JobStepVector::iterator& in, JobStepVector& list)
-{
-    JobStepVector::iterator end = list.end();
-
-    for (unsigned i = 0; i < in->get()->outputAssociation().outSize(); ++i)
-    {
-        JobStepVector::iterator iter = list.begin();
-        AnyDataListSPtr outAdl = in->get()->outputAssociation().outAt(i);
-
-        while (iter != end)
-        {
-            if (iter != in)
-            {
-                AnyDataListSPtr inAdl;
-
-                for (unsigned j = 0; j < iter->get()->inputAssociation().outSize(); j++)
-                {
-                    inAdl = iter->get()->inputAssociation().outAt(j);
-
-                    if (inAdl.get() == outAdl.get())
-                        return iter;
-                }
-            }
-
-            ++iter;
-        }
-    }
-
-    return end;
-}
-
 
 bool checkCombinable(JobStep* jobStepPtr)
 {
@@ -1446,54 +1414,6 @@ void changePcolStepToPcolScan(JobStepVector::iterator& it, JobStepVector::iterat
     }
 }
 
-uint32_t shouldSort(const JobStep* inJobStep, int colWidth)
-{
-    //only pColStep and pColScan have colType
-    const pColStep* inStep = dynamic_cast<const pColStep*>(inJobStep);
-
-    if (inStep && colWidth > inStep->colType().colWidth)
-    {
-        return 1;
-    }
-
-    const pColScanStep* inScan = dynamic_cast<const pColScanStep*>(inJobStep);
-
-    if (inScan && colWidth > inScan->colType().colWidth)
-    {
-        return 1;
-    }
-
-    return 0;
-}
-
-void convertPColStepInProjectToPassThru(JobStepVector& psv, JobInfo& jobInfo)
-{
-    for (JobStepVector::iterator iter = psv.begin(); iter != psv.end(); ++iter)
-    {
-        pColStep* colStep = dynamic_cast<pColStep*>(iter->get());
-
-        if (colStep != NULL)
-        {
-            JobStepAssociation ia = iter->get()->inputAssociation();
-            DataList_t* fifoDlp = ia.outAt(0).get()->dataList();
-
-            if (fifoDlp)
-            {
-                if (iter->get()->oid() >= 3000 && iter->get()->oid() == fifoDlp->OID())
-                {
-                    PassThruStep* pts = 0;
-                    pts = new PassThruStep(*colStep);
-                    pts->alias(colStep->alias());
-                    pts->view(colStep->view());
-                    pts->name(colStep->name());
-                    pts->tupleId(iter->get()->tupleId());
-                    iter->reset(pts);
-                }
-            }
-        }
-    }
-}
-
 // optimize filter order
 //   perform none string filters first because string filter joins the tokens.
 void optimizeFilterOrder(JobStepVector& qsv)
@@ -1818,7 +1738,7 @@ void makeVtableModeSteps(CalpontSelectExecutionPlan* csep, JobInfo& jobInfo,
             jobInfo.limitCount = (uint64_t) - 1;
         }
 
-        // support order by and limit in sub-query/union or 
+        // support order by and limit in sub-query/union or
         // GROUP BY handler processed outer query order
         else if (csep->orderByCols().size() > 0)
         {

dbcon/joblist/windowfunctionstep.cpp

@@ -81,21 +81,6 @@ using namespace joblist;
 namespace
 {
 
-string keyName(uint64_t i, uint32_t key, const joblist::JobInfo& jobInfo)
-{
-    string name = jobInfo.projectionCols[i]->alias();
-
-    if (name.empty())
-    {
-        name = jobInfo.keyInfo->tupleKeyToName[key];
-
-        if (jobInfo.keyInfo->tupleKeyVec[key].fId < 100)
-            name = "Expression/Function";
-    }
-
-    return name = "'" + name + "'";
-}
-
 
 uint64_t getColumnIndex(const SRCP& c, const map<uint64_t, uint64_t>& m, JobInfo& jobInfo)
 {

oam/oamcpp/liboamcpp.cpp

@@ -154,12 +154,12 @@ Oam::Oam()
     char* p = getenv("USER");
 
     if (p && *p)
-	USER = p;
+	    USER = p;
 
     userDir = USER;
 
     if ( USER != "root")
-	userDir = "home/" + USER;
+	    userDir = "home/" + USER;
 
 	tmpdir = startup::StartUp::tmpDir();
 
@@ -8601,7 +8601,7 @@ bool Oam::attachEC2Volume(std::string volumeName, std::string deviceName, std::s
             	writeLog("attachEC2Volume: Attach failed, call detach:" + volumeName + " " + instanceName + " " + deviceName, LOG_TYPE_ERROR );
 
 				detachEC2Volume(volumeName);
-			}	
+			}
 			else
 				return true;
 		}
@@ -10470,7 +10470,6 @@ void Oam::sendStatusUpdate(ByteStream obs, ByteStream::byte returnRequestType)
             if (ibs.length() > 0)
             {
                 ibs >> returnRequestType;
-
                 if ( returnRequestType == returnRequestType )
                 {
                     processor.shutdown();

oamapps/alarmmanager/alarmmanager.cpp

@@ -484,7 +484,6 @@ void ALARMManager::sendAlarmReport (const char* componentID, int alarmID, int st
     else
         processName = repProcessName;
 
-    int returnStatus = API_SUCCESS;           //default
     ByteStream msg1;
 
     // setup message
@@ -621,7 +620,7 @@ void ALARMManager::getActiveAlarm(AlarmList& alarmList) const
 *****************************************************************************************/
 void ALARMManager::getAlarm(std::string date, AlarmList& alarmList) const
 {
-	
+
     string alarmFile = startup::StartUp::tmpDir() + "/alarms";
 
     //make 1 alarm log file made up of archive and current alarm.log

utils/compress/version1.cpp

@@ -90,6 +90,8 @@ namespace
 {
 short DSPort = 9199;
 
+// this isn't currently used but could be in the future.
+#if 0
 void log(const string& s)
 {
     logging::MessageLog logger((logging::LoggingID()));
@@ -100,6 +102,7 @@ void log(const string& s)
     message.format(args);
     logger.logErrorMessage(message);
 }
+#endif
 
 struct ScopedCleaner
 {

utils/dataconvert/dataconvert.cpp

@@ -83,19 +83,6 @@ bool from_string(T& t, const std::string& s, std::ios_base & (*f)(std::ios_base&
     return !(iss >> f >> t).fail();
 }
 
-uint64_t pow10_(int32_t scale)
-{
-    if (scale <= 0) return 1;
-
-    idbassert(scale < 20);
-    uint64_t res = 1;
-
-    for (int32_t i = 0; i < scale; i++)
-        res *= 10;
-
-    return res;
-}
-
 bool number_value ( const string& data )
 {
     for (unsigned int i = 0; i < strlen(data.c_str()); i++)
@@ -872,7 +859,6 @@ bool mysql_str_to_datetime( const string& input, DateTime& output, bool& isDate
 
 bool mysql_str_to_time( const string& input, Time& output, long decimals )
 {
-//    int32_t datesepct = 0;
     uint32_t dtend = 0;
     bool isNeg = false;
 
@@ -2157,7 +2143,8 @@ std::string DataConvert::datetimeToString1( long long  datetimevalue )
 {
     // @bug 4703 abandon multiple ostringstream's for conversion
     DateTime dt(datetimevalue);
-    const int DATETIMETOSTRING1_LEN = 22; // YYYYMMDDHHMMSSmmmmmm\0
+    // Interesting, gcc 7 says the sprintf below generates between 21 and 23 bytes of output.
+    const int DATETIMETOSTRING1_LEN = 23; // YYYYMMDDHHMMSSmmmmmm\0
     char buf[DATETIMETOSTRING1_LEN];
 
     sprintf(buf, "%04d%02d%02d%02d%02d%02d%06d", dt.year, dt.month, dt.day, dt.hour, dt.minute, dt.second, dt.msecond);
@@ -2422,11 +2409,10 @@ int64_t DataConvert::stringToDatetime(const string& data, bool* date)
         return -1;
 }
 
+/* This is really painful and expensive b/c it seems the input is not normalized or
+sanitized.  That should really be done on ingestion. */
 int64_t DataConvert::intToDate(int64_t data)
 {
-    //char buf[10] = {0};
-    //snprintf( buf, 10, "%llu", (long long unsigned int)data);
-    //string date = buf;
     char buf[21] = {0};
     Date aday;
 
@@ -2438,7 +2424,11 @@ int64_t DataConvert::intToDate(int64_t data)
         return *(reinterpret_cast<uint32_t*>(&aday));
     }
 
+    // this snprintf call causes a compiler warning b/c we're potentially copying a 20-digit #
+    // into 15 bytes, however, that appears to be intentional.
+    #pragma GCC diagnostic ignored "-Wformat-truncation="
     snprintf( buf, 15, "%llu", (long long unsigned int)data);
+    #pragma GCC diagnostic pop
 
     string year, month, day, hour, min, sec, msec;
     int64_t y = 0, m = 0, d = 0, h = 0, minute = 0, s = 0, ms = 0;
@@ -2541,6 +2531,8 @@ int64_t DataConvert::intToDate(int64_t data)
     return *(reinterpret_cast<uint32_t*>(&aday));
 }
 
+/* This is really painful and expensive b/c it seems the input is not normalized or
+sanitized.  That should really be done on ingestion. */
 int64_t DataConvert::intToDatetime(int64_t data, bool* date)
 {
     bool isDate = false;
@@ -2563,7 +2555,12 @@ int64_t DataConvert::intToDatetime(int64_t data, bool* date)
         return *(reinterpret_cast<uint64_t*>(&adaytime));
     }
 
+    // this snprintf call causes a compiler warning b/c we're potentially copying a 20-digit #
+    // into 15 bytes, however, that appears to be intentional.
+    #pragma GCC diagnostic ignored "-Wformat-truncation="
     snprintf( buf, 15, "%llu", (long long unsigned int)data);
+    #pragma GCC diagnostic pop
+
     //string date = buf;
     string year, month, day, hour, min, sec, msec;
     int64_t y = 0, m = 0, d = 0, h = 0, minute = 0, s = 0, ms = 0;
@@ -2671,6 +2668,8 @@ int64_t DataConvert::intToDatetime(int64_t data, bool* date)
     return *(reinterpret_cast<uint64_t*>(&adaytime));
 }
 
+/* This is really painful and expensive b/c it seems the input is not normalized or
+sanitized.  That should really be done on ingestion. */
 int64_t DataConvert::intToTime(int64_t data, bool fromString)
 {
     char buf[21] = {0};
@@ -2689,7 +2688,12 @@ int64_t DataConvert::intToTime(int64_t data, bool fromString)
         return *(reinterpret_cast<int64_t*>(&atime));
     }
 
+    // this snprintf call causes a compiler warning b/c we're potentially copying a 20-digit #
+    // into 15 bytes, however, that appears to be intentional.
+    #pragma GCC diagnostic ignored "-Wformat-truncation="
     snprintf( buf, 15, "%llu", (long long unsigned int)data);
+    #pragma GCC diagnostic pop
+
     //string date = buf;
     string hour, min, sec, msec;
     int64_t h = 0, minute = 0, s = 0, ms = 0;

utils/funcexp/func_insert.cpp

@@ -89,11 +89,11 @@ std::string Func_insert::getStrVal(rowgroup::Row& row,
                                    execplan::CalpontSystemCatalog::ColType&)
 {
 	string tstr;
+	string tnewstr;
     stringValue(fp[0], row, isNull, tstr);
     if (isNull)
         return "";
 
-	string tnewstr;
     stringValue(fp[3], row, isNull, tnewstr);
     if (isNull)
         return "";

utils/funcexp/func_md5.cpp

@@ -82,7 +82,7 @@ typedef unsigned      char uchar;
 
 char* PrintMD5(uchar md5Digest[16]);
 char* MD5String(const char* szString);
-char* MD5File(char* szFilename);
+//char* MD5File(char* szFilename);
 
 class md5
 {
@@ -263,7 +263,9 @@ char* MD5String(const char* szString)
 
 }
 
-// MD5File: Performs the MD5 algorithm on a file (binar or text),
+// this fcn isn't used, so commenting it
+#if 0
+// MD5File: Performs the MD5 algorithm on a file (binary or text),
 // returning the results as a char*.  Returns NULL if it fails.
 char* MD5File(char* szFilename)
 {
@@ -295,7 +297,7 @@ char* MD5File(char* szFilename)
 
     return NULL; // failed
 }
-
+#endif
 
 // md5::Init
 // Initializes a new context.

utils/funcexp/func_substring_index.cpp

@@ -99,8 +99,7 @@ std::string Func_substring_index::getStrVal(rowgroup::Row& row,
     }
     else
     {
-        count = count * -1;
+        count = -count;
-        size_t end = strlen(str.c_str());
         int pointer = end;
         int start = 0;
 

utils/idbhdfs/hdfs-shared/HdfsRdwrFileBuffer.cpp

@@ -151,7 +151,7 @@ HdfsRdwrFileBuffer::HdfsRdwrFileBuffer(const char* fname, const char* mode, unsi
 
 // This constructor is for use by HdfsRdwrMemBuffer to create a file buffer when we
 // run out of memory.
-HdfsRdwrFileBuffer::HdfsRdwrFileBuffer(HdfsRdwrMemBuffer* pMemBuffer) throw (std::exception) :
+HdfsRdwrFileBuffer::HdfsRdwrFileBuffer(HdfsRdwrMemBuffer* pMemBuffer) :
     IDBDataFile(pMemBuffer->name().c_str()),
     m_buffer(NULL),
     m_dirty(false)

utils/idbhdfs/hdfs-shared/HdfsRdwrFileBuffer.h

@@ -50,7 +50,7 @@ class HdfsRdwrFileBuffer: public IDBDataFile, boost::noncopyable
 {
 public:
     HdfsRdwrFileBuffer(const char* fname, const char* mode, unsigned opts);
-    HdfsRdwrFileBuffer(HdfsRdwrMemBuffer* pMemBuffer) throw (std::exception);
+    HdfsRdwrFileBuffer(HdfsRdwrMemBuffer* pMemBuffer);
     /* virtual */ ~HdfsRdwrFileBuffer();
 
     /* virtual */ ssize_t pread(void* ptr, off64_t offset, size_t count);

utils/loggingcpp/messagelog.cpp

@@ -180,21 +180,21 @@ const string MessageLog::format(const Message& msg, const char prefix)
 void MessageLog::logDebugMessage(const Message& msg)
 {
     ::openlog(SubsystemID[fLogData.fSubsysID].c_str(), 0 | LOG_PID, fFacility);
-    ::syslog(LOG_DEBUG, format(msg, 'D').c_str());
+    ::syslog(LOG_DEBUG, "%s", format(msg, 'D').c_str());
     ::closelog();
 }
 
 void MessageLog::logInfoMessage(const Message& msg)
 {
     ::openlog(SubsystemID[fLogData.fSubsysID].c_str(), 0 | LOG_PID, fFacility);
-    ::syslog(LOG_INFO, format(msg, 'I').c_str());
+    ::syslog(LOG_INFO, "%s", format(msg, 'I').c_str());
     ::closelog();
 }
 
 void MessageLog::logWarningMessage(const Message& msg)
 {
     ::openlog(SubsystemID[fLogData.fSubsysID].c_str(), 0 | LOG_PID, fFacility);
-    ::syslog(LOG_WARNING, format(msg, 'W').c_str());
+    ::syslog(LOG_WARNING, "%s", format(msg, 'W').c_str());
     ::closelog();
 }
 
@@ -202,14 +202,14 @@ void MessageLog::logErrorMessage(const Message& msg)
 {
     // @bug 24 use 'E' instead of 'S'
     ::openlog(SubsystemID[fLogData.fSubsysID].c_str(), 0 | LOG_PID, fFacility);
-    ::syslog(LOG_ERR, format(msg, 'E').c_str());
+    ::syslog(LOG_ERR, "%s", format(msg, 'E').c_str());
     ::closelog();
 }
 
 void MessageLog::logCriticalMessage(const Message& msg)
 {
     ::openlog(SubsystemID[fLogData.fSubsysID].c_str(), 0 | LOG_PID, fFacility);
-    ::syslog(LOG_CRIT, format(msg, 'C').c_str());
+    ::syslog(LOG_CRIT, "%s", format(msg, 'C').c_str());
     ::closelog();
 }
 //Bug 5218. comment out the following functions to alleviate issue where dml messages show up in crit.log. This

utils/messageqcpp/inetstreamsocket.cpp

@@ -966,7 +966,9 @@ void InetStreamSocket::connect(const sockaddr* serv_addr)
 #ifdef _MSC_VER
         (void)::recv(socketParms().sd(), &buf, 1, 0);
 #else
-        (void)::read(socketParms().sd(), &buf, 1);
+        #pragma GCC diagnostic ignored "-Wunused-result"
+        ::read(socketParms().sd(), &buf, 1);   // we know 1 byte is in the recv buffer
+        #pragma GCC diagnostic pop
 #endif
         return;
     }

utils/querytele/queryteleprotoimpl.cpp

@@ -94,6 +94,7 @@ string get_trace_file()
     return oss.str();
 }
 
+#ifdef QUERY_TELE_DEBUG
 void log_query(const querytele::QueryTele& qtdata)
 {
     ofstream trace(get_trace_file().c_str(), ios::out | ios::app);
@@ -125,7 +126,9 @@ void log_query(const querytele::QueryTele& qtdata)
     trace << endl;
     trace.close();
 }
+#endif
 
+#ifdef QUERY_TELE_DEBUG
 const string st2str(enum querytele::StepType::type t)
 {
     switch (t)
@@ -172,7 +175,9 @@ const string st2str(enum querytele::StepType::type t)
 
     return "INV";
 }
+#endif
 
+#ifdef QUERY_TELE_DEBUG
 void log_step(const querytele::StepTele& stdata)
 {
     ofstream trace(get_trace_file().c_str(), ios::out | ios::app);
@@ -207,6 +212,7 @@ void log_step(const querytele::StepTele& stdata)
     trace << endl;
     trace.close();
 }
+#endif
 
 void TeleConsumer()
 {

utils/rwlock/rwlock.cpp

@@ -32,7 +32,9 @@
 #include <tr1/unordered_map>
 #endif
 
+#ifndef NDEBUG
 #define NDEBUG
+#endif
 #include <cassert>
 using namespace std;
 

utils/threadpool/prioritythreadpool.cpp

@@ -139,8 +139,8 @@ PriorityThreadPool::Priority PriorityThreadPool::pickAQueue(Priority preference)
 
 void PriorityThreadPool::threadFcn(const Priority preferredQueue) throw()
 {
-    Priority queue;
+    Priority queue = LOW;
-    uint32_t weight, i;
+    uint32_t weight, i = 0;
     vector<Job> runList;
     vector<bool> reschedule;
     uint32_t rescheduleCount;

utils/threadpool/threadpool.h

@@ -75,7 +75,9 @@ public:
     boost::thread* create_thread(F threadfunc)
     {
         boost::lock_guard<boost::shared_mutex> guard(m);
-        std::auto_ptr<boost::thread> new_thread(new boost::thread(threadfunc));
+        std::unique_ptr<boost::thread> new_thread(new boost::thread(threadfunc));
+        // auto_ptr was deprecated
+        //std::auto_ptr<boost::thread> new_thread(new boost::thread(threadfunc));
         threads.push_back(new_thread.get());
         return new_thread.release();
     }

utils/udfsdk/mcsv1_udaf.h

@@ -1029,6 +1029,10 @@ inline T mcsv1_UDAF::convertAnyTo(static_any::any& valIn)
     {
         val = valIn.cast<double>();
     }
+    else
+    {
+        throw runtime_error("mcsv1_UDAF::convertAnyTo(): input param has unrecognized type");
+    }
     return val;
 }
 

writeengine/wrapper/we_colop.cpp

@@ -238,7 +238,7 @@ int ColumnOp::allocRowId(const TxnID& txnid, bool useStartingExtent,
 
                     for (i = 0; i < dbRootExtentTrackers.size(); i++)
                     {
-                        if (i != column.colNo)
+                        if (i != (int) column.colNo)
                             dbRootExtentTrackers[i]->nextSegFile(dbRoot, partition, segment, newHwm, startLbid);
 
                         // Round up HWM to the end of the current extent

writeengine/xml/we_xmlgenproc.cpp

@@ -442,6 +442,9 @@ void XMLGenProc::getColumnsForTable(
 //------------------------------------------------------------------------------
 // Generate Job XML File Name
 //------------------------------------------------------------------------------
+
+// This isn't used currently, commenting it out
+#if 0
 std::string XMLGenProc::genJobXMLFileName( ) const
 {
     std::string xmlFileName;
@@ -465,7 +468,10 @@ std::string XMLGenProc::genJobXMLFileName( ) const
     if (!p.has_root_path())
     {
         char cwdPath[4096];
-        getcwd(cwdPath, sizeof(cwdPath));
+        char *buf;
+        buf = getcwd(cwdPath, sizeof(cwdPath));
+        if (buf == NULL)
+            throw runtime_error("Failed to get the current working directory!");
         boost::filesystem::path p2(cwdPath);
         p2 /= p;
         xmlFileName = p2.string();
@@ -479,6 +485,8 @@ std::string XMLGenProc::genJobXMLFileName( ) const
 
     return xmlFileName;
 }
+#endif
+
 
 //------------------------------------------------------------------------------
 // writeXMLFile

writeengine/xml/we_xmlgenproc.h

@@ -90,7 +90,7 @@ public:
 
     /** @brief Generate Job XML file name
      */
-    EXPORT std::string genJobXMLFileName( ) const;
+    //EXPORT std::string genJobXMLFileName( ) const;
 
     /** @brief Write xml file document to the destination Job XML file.
      *

writeengine/xml/we_xmljob.cpp

@@ -1301,7 +1301,13 @@ int XMLJob::genJobXMLFileName(
             // nothing else to do
 #else
             char cwdPath[4096];
-            getcwd(cwdPath, sizeof(cwdPath));
+            char *err;
+            err = getcwd(cwdPath, sizeof(cwdPath));
+            if (err == NULL)
+            {
+                errMsg = "Failed to get the current working directory.";
+                return -1;
+            }
             string trailingPath(xmlFilePath.string());
             xmlFilePath  = cwdPath;
             xmlFilePath /= trailingPath;