Merge branch 'develop-1.1' into 1.1-merge-up-20180531

dbcon/ddlpackage/ddl.l

@@ -32,7 +32,7 @@
 #endif
 
 using namespace ddlpackage;
-typedef enum { NOOP, STRIP_QUOTES, STRIP_QUOTES_FQ } copy_action_t;
+typedef enum { NOOP, STRIP_QUOTES } copy_action_t;
 int lineno = 1;
 void ddlerror(struct pass_to_bison* x, char const *s);
 
@@ -75,8 +75,6 @@ extended_identifier {ident_start}{extended_ident_cont}*
 fq_identifier {identifier}\.{identifier}
 identifier_quoted {grave_accent}{extended_identifier}{grave_accent}
 identifier_double_quoted {double_quote}{extended_identifier}{double_quote}
-fq_quoted ({identifier_quoted}|{extended_identifier})\.({identifier_quoted}|{identifier})
-fq_double_quoted ({identifier_double_quoted}|{extended_identifier})\.({identifier_double_quoted}|{identifier})
 
 integer			[-+]?{digit}+
 decimal			([-+]?({digit}*\.{digit}+)|({digit}+\.{digit}*))
@@ -90,10 +88,7 @@ realfail2		({integer}|{decimal})[Ee][-+]
 
 
 {identifier_quoted} { ddlget_lval(yyscanner)->str = scanner_copy( ddlget_text(yyscanner), yyscanner, STRIP_QUOTES ); return IDENT; }
-{identifier_double_quoted} { ddlget_lval(yyscanner)->str = scanner_copy( ddlget_text(yyscanner), yyscanner, STRIP_QUOTES ); return IDENT; }
+{identifier_double_quoted} { ddlget_lval(yyscanner)->str = scanner_copy( ddlget_text(yyscanner), yyscanner, STRIP_QUOTES ); return DQ_IDENT; }
-{fq_identifier} { ddlget_lval(yyscanner)->str = scanner_copy(ddlget_text(yyscanner), yyscanner); return FQ_IDENT; }
-{fq_quoted} { ddlget_lval(yyscanner)->str = scanner_copy(ddlget_text(yyscanner), yyscanner, STRIP_QUOTES_FQ); return FQ_IDENT; }
-{fq_double_quoted} { ddlget_lval(yyscanner)->str = scanner_copy(ddlget_text(yyscanner), yyscanner, STRIP_QUOTES_FQ); return FQ_IDENT; }
 
 ACTION {return ACTION;}
 ADD {return ADD;}
@@ -193,14 +188,14 @@ LONGTEXT {return LONGTEXT;}
     /* ignore */
 }
 
-{identifier} {ddlget_lval(yyscanner)->str = scanner_copy(ddlget_text(yyscanner), yyscanner); return IDENT;}
+{identifier} { ddlget_lval(yyscanner)->str = scanner_copy(ddlget_text(yyscanner), yyscanner); return DQ_IDENT;}
 
 {self} {
     return ddlget_text(yyscanner)[0];
 }
 
 {grave_accent} {
-    /* ignore */
+    return ddlget_text(yyscanner)[0];
 }
 
 "/*"            { BEGIN(c_comment); }
@@ -282,7 +277,6 @@ char* scanner_copy (char *str, yyscan_t yyscanner, copy_action_t action)
     char* result;
     char* nv = strdup(str);
     result = nv;
-
     // free strduped memory later to prevent possible memory leak
     if(nv)
         ((scan_data*)ddlget_extra(yyscanner))->valbuf.push_back(nv);
@@ -292,57 +286,6 @@ char* scanner_copy (char *str, yyscan_t yyscanner, copy_action_t action)
         nv[strlen(str) - 1] = '\0';
         result = nv + 1;
     }
-    else if (action == STRIP_QUOTES_FQ)
-    {
-        bool move_left = false;
-        bool move_right = false;
-        char* left = nv;
-        char* tmp_first = nv;
-        // MCOL-1384 Loop through all comas in this quoted fq id
-        // looking for $quote_sign.$quote_sign sequence.
-        char* fq_delimiter;
-        int tmp_pos = 0;
-        while((fq_delimiter = strchr(tmp_first, '.')) != NULL)
-        {
-            if( (*(fq_delimiter -1) == '`' && *(fq_delimiter + 1) == '`') ||
-              (*(fq_delimiter -1) == '"' && *(fq_delimiter + 1) == '"') )
-            {
-                tmp_pos += fq_delimiter - tmp_first;
-                break;
-            }
-            tmp_first = fq_delimiter;
-        }
-
-        char* fq_delimiter_orig = str + tmp_pos;
-        char* right = fq_delimiter + 1;
-        char* right_orig = fq_delimiter_orig + 1;
-        // MCOL-1384 Strip quotes from the left part.
-        if(*left == '"' || *left == '`')
-        {
-            result = left + 1;
-            *(fq_delimiter - 1) = '.';
-            move_left = true;
-        }
-        else
-        {
-            fq_delimiter += 1;
-        }
-
-        int right_length = strlen(right);
-        // MCOL-1384 Strip quotes from the right part.
-        if(*right == '`' || *right == '"')
-        {
-            right += 1; right_orig += 1;
-            right_length -= 2;
-            move_right = true;
-            *(fq_delimiter + right_length) = '\0';
-        }
-
-        if(move_left || move_right)
-        {
-            strncpy(fq_delimiter, right_orig, right_length);
-        }
-    }
 
     return result;
 }

dbcon/ddlpackage/ddl.y

@@ -29,26 +29,18 @@
    Understanding the New Sql book
    The postgress and mysql sources.  find x -name \*.y -o -name \*.yy.
 
-   We don't support delimited identifiers.
+   We support quoted identifiers.
 
    All literals are stored as unconverted strings.
 
    You can't say "NOT DEFERRABLE".  See the comment below.
 
-   This is not a reentrant parser.  It uses the original global
+   This is a reentrant parser.
-   variable style method of communication between the parser and
-   scanner.  If we ever needed more than one parser thread per
-   processes, we would use the pure/reentrant options of bison and
-   flex.  In that model, things that are traditionally global live
-   inside a struct that is passed around.  We would need to upgrade to
-   a more recent version of flex.  At the time of this writing, our
-   development systems have: flex version 2.5.4
 
    MCOL-66 Modify to be a reentrant parser
    */
 
 %{
-#include "string.h"
 #include "sqlparser.h"
 
 #ifdef _MSC_VER
@@ -122,7 +114,7 @@ REFERENCES RENAME RESTRICT SET SMALLINT TABLE TEXT TIME TINYBLOB TINYTEXT
 TINYINT TO UNIQUE UNSIGNED UPDATE USER SESSION_USER SYSTEM_USER VARCHAR VARBINARY
 VARYING WITH ZONE DOUBLE IDB_FLOAT REAL CHARSET IDB_IF EXISTS CHANGE TRUNCATE
 
-%token <str> FQ_IDENT IDENT FCONST SCONST CP_SEARCH_CONDITION_TEXT ICONST DATE
+%token <str> DQ_IDENT IDENT FCONST SCONST CP_SEARCH_CONDITION_TEXT ICONST DATE
 
 /* Notes:
  * 1. "ata" stands for alter_table_action
@@ -476,7 +468,7 @@ opt_equal:
 	;
 
 table_option:
- 	ENGINE opt_equal IDENT {$$ = new pair<string,string>("engine", $3);}
+ 	ENGINE opt_equal ident {$$ = new pair<string,string>("engine", $3);}
 	|
  	MAX_ROWS opt_equal ICONST {$$ = new pair<string,string>("max_rows", $3);}
  	|
@@ -491,9 +483,9 @@ table_option:
        $$ = new pair<string,string>("auto_increment", $3);
     }
  	|
- 	DEFAULT CHARSET opt_equal IDENT {$$ = new pair<string,string>("default charset", $4);}
+ 	DEFAULT CHARSET opt_equal ident {$$ = new pair<string,string>("default charset", $4);}
  	|
- 	DEFAULT IDB_CHAR SET opt_equal IDENT {$$ = new pair<string,string>("default charset", $5);}
+ 	DEFAULT IDB_CHAR SET opt_equal ident {$$ = new pair<string,string>("default charset", $5);}
 	;
 
 alter_table_statement:
@@ -623,25 +615,21 @@ table_name:
 	;
 
 qualified_name:
-	FQ_IDENT {
+	| ident {
-                char* delimeterPosition = strchr(const_cast<char*>($1), '.');
-                if( delimeterPosition )
-                {
-                    *delimeterPosition = '\0';
-                    char* schemaName = const_cast<char*>($1);
-                    char* tableName = delimeterPosition + 1;
-                    $$ = new QualifiedName(schemaName, tableName);
-                    *delimeterPosition = '.';
-                }
-                else
-                    $$ = new QualifiedName($1);
-             }
-	| IDENT {
 				if (x->fDBSchema.size())
 					$$ = new QualifiedName((char*)x->fDBSchema.c_str(), $1);
 				else
 				    $$ = new QualifiedName($1);   
 			}
+    | ident '.' ident
+        {
+            $$ = new QualifiedName($1, $3);
+        }
+	;
+
+ident:
+    DQ_IDENT
+    | IDENT
     ;
 
 ata_add_column:
@@ -656,11 +644,11 @@ ata_add_column:
 
 column_name:
 	DATE
-	|IDENT
+	|ident
 	;
 
 constraint_name:
-	IDENT
+	ident
 	;
 
 column_option:
@@ -720,6 +708,10 @@ default_clause:
 	{
 		$$ = new ColumnDefaultValue($2);
 	}
+	| DEFAULT DQ_IDENT /* MCOL-1406 */
+	{
+		$$ = new ColumnDefaultValue($2);
+	}
 	| DEFAULT NULL_TOK {$$ = new ColumnDefaultValue(NULL);}
 	| DEFAULT USER {$$ = new ColumnDefaultValue("$USER");}
 	| DEFAULT CURRENT_USER {$$ = new ColumnDefaultValue("$CURRENT_USER");}

dbcon/joblist/jlf_execplantojoblist.cpp

@@ -1634,11 +1634,8 @@ const JobStepVector doSimpleFilter(SimpleFilter* sf, JobInfo& jobInfo)
             return doExpressionFilter(sf, jobInfo);
         }
 
-        // trim trailing space char in the predicate
         string constval(cc->constval());
-        size_t spos = constval.find_last_not_of(" ");
 
-        if (spos != string::npos) constval = constval.substr(0, spos + 1);
 
         CalpontSystemCatalog::OID dictOid = 0;
         CalpontSystemCatalog::ColType ct = sc->colType();
@@ -2772,11 +2769,8 @@ const JobStepVector doConstantFilter(const ConstantFilter* cf, JobInfo& jobInfo)
                     if (ConstantColumn::NULLDATA == cc->type() && (opeq == *sop || opne == *sop))
                         cop = COMPARE_NIL;
 
-                    // trim trailing space char
                     string value = cc->constval();
-                    size_t spos = value.find_last_not_of(" ");
 
-                    if (spos != string::npos) value = value.substr(0, spos + 1);
 
                     pds->addFilter(cop, value);
                 }
@@ -2858,11 +2852,8 @@ const JobStepVector doConstantFilter(const ConstantFilter* cf, JobInfo& jobInfo)
                     if (ConstantColumn::NULLDATA == cc->type() && (opeq == *sop || opne == *sop))
                         cop = COMPARE_NIL;
 
-                    // trim trailing space char
                     string value = cc->constval();
-                    size_t spos = value.find_last_not_of(" ");
 
-                    if (spos != string::npos) value = value.substr(0, spos + 1);
 
                     pds->addFilter(cop, value);
                 }
@@ -2968,10 +2959,7 @@ const JobStepVector doConstantFilter(const ConstantFilter* cf, JobInfo& jobInfo)
                     int8_t cop = op2num(sop);
                     int64_t value = 0;
                     string constval = cc->constval();
-                    // trim trailing space char
-                    size_t spos = constval.find_last_not_of(" ");
 
-                    if (spos != string::npos) constval = constval.substr(0, spos + 1);
 
                     // @bug 1151 string longer than colwidth of char/varchar.
                     uint8_t rf = 0;

dbcon/mysql/ha_calpont_ddl.cpp

@@ -2224,7 +2224,7 @@ int ha_calpont_impl_rename_table_(const char* from, const char* to, cal_connecti
     int rc = ProcessDDLStatement(stmt, db, "", tid2sid(thd->thread_id), emsg);
 
     if (rc != 0)
-        push_warning(thd, Sql_condition::WARN_LEVEL_ERROR, 9999, emsg.c_str());
+        push_warning(thd, Sql_condition::WARN_LEVEL_WARN, 9999, emsg.c_str());
 
     return rc;
 }
@@ -2266,7 +2266,7 @@ extern "C"
         int rc = ProcessDDLStatement(stmt, db, "", tid2sid(thd->thread_id), emsg, compressiontype);
 
         if (rc != 0)
-            push_warning(thd, Sql_condition::WARN_LEVEL_ERROR, 9999, emsg.c_str());
+		push_warning(thd, Sql_condition::WARN_LEVEL_WARN, 9999, emsg.c_str());
 
         return rc;
     }

dbcon/mysql/ha_calpont_execplan.cpp

@@ -4674,7 +4674,6 @@ void gp_walk(const Item* item, void* arg)
 
             if (isp)
             {
-                // @bug 3669. trim trailing spaces for the compare value
                 if (isp->result_type() == STRING_RESULT)
                 {
                     String val, *str = isp->val_str(&val);
@@ -4685,10 +4684,7 @@ void gp_walk(const Item* item, void* arg)
                         cval.assign(str->ptr(), str->length());
                     }
 
-                    size_t spos = cval.find_last_not_of(" ");
 
-                    if (spos != string::npos)
-                        cval = cval.substr(0, spos + 1);
 
                     gwip->rcWorkStack.push(new ConstantColumn(cval));
                     break;

utils/funcexp/func_substring_index.cpp

@@ -74,6 +74,9 @@ std::string Func_substring_index::getStrVal(rowgroup::Row& row,
     if ( count > (int64_t) end )
         return str;
 
+    if (( count < 0 ) && ((count * -1) > end))
+        return str;
+
     string value = str;
 
     if ( count > 0 )

writeengine/splitter/we_cmdargs.h

@@ -77,7 +77,7 @@ public:
     {
         return fLocFile;
     }
-    int getMode() const
+        int getReadBufSize() { return fReadBufSize; }
     {
         return fMode;
     }

writeengine/splitter/we_filereadthread.cpp

@@ -87,6 +87,15 @@ WEFileReadThread::WEFileReadThread(WESDHandler& aSdh): fSdh(aSdh),
 {
     //TODO batch qty to get from config
     fBatchQty = 10000;
+    if (fSdh.getReadBufSize() < DEFAULTBUFFSIZE)
+    {
+        fBuffSize = DEFAULTBUFFSIZE;
+    }
+    else
+    {
+        fBuffSize = fSdh.getReadBufSize();
+    }
+    fBuff = new char [fBuffSize];
 
 }
 
@@ -109,6 +118,7 @@ WEFileReadThread::~WEFileReadThread()
     }
 
     fpThread = 0;
+    delete []fBuff;
     //cout << "WEFileReadThread destructor called" << endl;
 }
 
@@ -352,17 +362,17 @@ unsigned int WEFileReadThread::readDataFile(messageqcpp::SBS& Sbs)
             if (fEnclEsc)
             {
                 //pStart = aBuff;
-                aLen = getNextRow(fInFile, fBuff, sizeof(fBuff) - 1);
+				aLen = getNextRow(fInFile, fBuff, fBuffSize-1);
             }
             else
             {
-                fInFile.getline(fBuff, sizeof(fBuff) - 1);
+				fInFile.getline(fBuff, fBuffSize-1);
                 aLen = fInFile.gcount();
             }
 
             ////aLen chars incl \n, Therefore aLen-1; '<<' oper won't go past it
             //cout << "Data Length " << aLen <<endl;
-            if ((aLen < (sizeof(fBuff) - 2)) && (aLen > 0))
+			if((aLen < (fBuffSize-2)) && (aLen>0))
             {
                 fBuff[aLen - 1] = '\n';
                 fBuff[aLen] = 0;
@@ -374,7 +384,7 @@ unsigned int WEFileReadThread::readDataFile(messageqcpp::SBS& Sbs)
 
                 if (fSdh.getDebugLvl() > 2) cout << "File data line = " << aIdx << endl;
             }
-            else if (aLen >= sizeof(fBuff) - 2)	//Didn't hit delim; BIG ROW
+			else if(aLen>=fBuffSize-2)	//Didn't hit delim; BIG ROW
             {
                 cout << "Bad Row data " << endl;
                 cout << fBuff << endl;

writeengine/splitter/we_filereadthread.h

@@ -126,7 +126,7 @@ public:
     void add2InputDataFileList(std::string& FileName);
 
 private:
-    enum { MAXBUFFSIZE = 1024 * 1024 };
+    enum { DEFAULTBUFFSIZE=1024*1024 };
 
     // don't allow anyone else to set
     void setTgtPmId(unsigned int fTgtPmId)
@@ -151,7 +151,8 @@ private:
     char fEncl;							// Encl char
     char fEsc;							// Esc char
     char fDelim;						// Column Delimit char
-    char fBuff[MAXBUFFSIZE];			// main data buffer
+    char* fBuff;			// main data buffer
+    int fBuffSize;
 };
 
 } /* namespace WriteEngine */

writeengine/splitter/we_sdhandler.cpp

@@ -2663,6 +2663,13 @@ char WESDHandler::getEscChar()
 
 //------------------------------------------------------------------------------
 
+int WESDHandler::getReadBufSize()
+{
+    return fRef.fCmdArgs.getReadBufSize();
+}
+
+//------------------------------------------------------------------------------
+
 char WESDHandler::getDelimChar()
 {
     return fRef.fCmdArgs.getDelimChar();

writeengine/splitter/we_sdhandler.h

@@ -152,6 +152,7 @@ public:
     char getEscChar();
     char getDelimChar();
     bool getConsoleLog();
+    int getReadBufSize();
     ImportDataMode getImportDataMode() const;
     void sysLog(const logging::Message::Args& msgArgs,
                 logging::LOG_TYPE logType, logging::Message::MessageID msgId);

writeengine/wrapper/we_colop.cpp

@@ -222,7 +222,7 @@ int ColumnOp::allocRowId(const TxnID& txnid, bool useStartingExtent,
                 //Find out where the rest rows go
                 BRM::LBID_t startLbid;
                 //need to put in a loop until newExtent is true
-                newExtent = dbRootExtentTrackers[0]->nextSegFile(dbRoot, partition, segment, newHwm, startLbid);
+				newExtent = dbRootExtentTrackers[column.colNo]->nextSegFile(dbRoot, partition, segment, newHwm, startLbid);
                 TableMetaData* tableMetaData = TableMetaData::makeTableMetaData(tableOid);
 
                 while (!newExtent)
@@ -238,7 +238,7 @@ int ColumnOp::allocRowId(const TxnID& txnid, bool useStartingExtent,
 
                     for (i = 0; i < dbRootExtentTrackers.size(); i++)
                     {
-                        if (i != 0)
+						if (i != column.colNo)
                             dbRootExtentTrackers[i]->nextSegFile(dbRoot, partition, segment, newHwm, startLbid);
 
                         // Round up HWM to the end of the current extent
@@ -302,8 +302,7 @@ int ColumnOp::allocRowId(const TxnID& txnid, bool useStartingExtent,
 
                         tableMetaData->setColExtsInfo(newColStructList[i].dataOid, aColExtsInfo);
                     }
-
+					newExtent = dbRootExtentTrackers[column.colNo]->nextSegFile(dbRoot, partition, segment, newHwm, startLbid);	
-                    newExtent = dbRootExtentTrackers[0]->nextSegFile(dbRoot, partition, segment, newHwm, startLbid);
                 }
             }
 
@@ -324,7 +323,7 @@ int ColumnOp::allocRowId(const TxnID& txnid, bool useStartingExtent,
                 }
 
                 rc = BRMWrapper::getInstance()->allocateStripeColExtents(cols, dbRoot, partition, segment, extents);
-                newHwm = extents[0].startBlkOffset;
+				newHwm = extents[column.colNo].startBlkOffset;
 
                 if (rc != NO_ERROR)
                     return rc;

writeengine/wrapper/writeengine.cpp

@@ -1683,6 +1683,19 @@ int WriteEngineWrapper::insertColumnRecsBinary(const TxnID& txnid,
     for (i = 0; i < colStructList.size(); i++)
         Convertor::convertColType(&colStructList[i]);
 
+   // MCOL-984: find the smallest column width to calculate the RowID from so
+   // that all HWMs will be incremented by this operation
+   int32_t lowColLen = 8192;
+   int32_t colId = 0;
+   for (uint32_t colIt = 0; colIt < colStructList.size(); colIt++)
+   {
+        if (colStructList[colIt].colWidth < lowColLen)
+        {
+            colId = colIt;
+            lowColLen = colStructList[colId].colWidth;
+        }
+   }
+
     // rc = checkValid(txnid, colStructList, colValueList, ridList);
     // if (rc != NO_ERROR)
     //   return rc;
@@ -1709,8 +1722,8 @@ int WriteEngineWrapper::insertColumnRecsBinary(const TxnID& txnid,
     //--------------------------------------------------------------------------
     if (isFirstBatchPm)
     {
-        currentDBrootIdx = dbRootExtentTrackers[0]->getCurrentDBRootIdx();
+        currentDBrootIdx = dbRootExtentTrackers[colId]->getCurrentDBRootIdx();
-        extentInfo = dbRootExtentTrackers[0]->getDBRootExtentList();
+        extentInfo = dbRootExtentTrackers[colId]->getDBRootExtentList();
         dbRoot = extentInfo[currentDBrootIdx].fDbRoot;
         partitionNum = extentInfo[currentDBrootIdx].fPartition;
 
@@ -1895,7 +1908,7 @@ int WriteEngineWrapper::insertColumnRecsBinary(const TxnID& txnid,
     } // if (isFirstBatchPm)
     else //get the extent info from tableMetaData
     {
-        ColExtsInfo aColExtsInfo = tableMetaData->getColExtsInfo(colStructList[0].dataOid);
+        ColExtsInfo aColExtsInfo = tableMetaData->getColExtsInfo(colStructList[colId].dataOid);
         ColExtsInfo::iterator it = aColExtsInfo.begin();
 
         while (it != aColExtsInfo.end())
@@ -1931,20 +1944,7 @@ int WriteEngineWrapper::insertColumnRecsBinary(const TxnID& txnid,
     // allocate row id(s)
     //--------------------------------------------------------------------------
 
-    // MCOL-984: find the smallest column width to calculate the RowID from so
-    // that all HWMs will be incremented by this operation
-    int32_t lowColLen = 8192;
-    int32_t colId = 0;
-
-    for (uint32_t colIt = 0; colIt < colStructList.size(); colIt++)
-    {
-        if (colStructList[colIt].colWidth < lowColLen)
-        {
-            colId = colIt;
-            lowColLen = colStructList[colId].colWidth;
    curColStruct = colStructList[colId];
-        }
-    }
 
     colOp = m_colOp[op(curColStruct.fCompressionType)];
 
@@ -1970,7 +1970,7 @@ int WriteEngineWrapper::insertColumnRecsBinary(const TxnID& txnid,
     if (it != aColExtsInfo.end())
     {
         hwm = it->hwm;
-        //cout << "Got from colextinfo hwm for oid " << colStructList[0].dataOid << " is " << hwm << " and seg is " << colStructList[0].fColSegment << endl;
+        //cout << "Got from colextinfo hwm for oid " << colStructList[colId].dataOid << " is " << hwm << " and seg is " << colStructList[colId].fColSegment << endl;
     }
 
     oldHwm = hwm; //Save this info for rollback