diff --git a/dbcon/ddlpackage/ddl.l b/dbcon/ddlpackage/ddl.l index fd5ee6900..f3f2e7bfb 100644 --- a/dbcon/ddlpackage/ddl.l +++ b/dbcon/ddlpackage/ddl.l @@ -32,7 +32,7 @@ #endif using namespace ddlpackage; -typedef enum { NOOP, STRIP_QUOTES, STRIP_QUOTES_FQ } copy_action_t; +typedef enum { NOOP, STRIP_QUOTES } copy_action_t; int lineno = 1; void ddlerror(struct pass_to_bison* x, char const *s); @@ -75,8 +75,6 @@ extended_identifier {ident_start}{extended_ident_cont}* fq_identifier {identifier}\.{identifier} identifier_quoted {grave_accent}{extended_identifier}{grave_accent} identifier_double_quoted {double_quote}{extended_identifier}{double_quote} -fq_quoted ({identifier_quoted}|{extended_identifier})\.({identifier_quoted}|{identifier}) -fq_double_quoted ({identifier_double_quoted}|{extended_identifier})\.({identifier_double_quoted}|{identifier}) integer [-+]?{digit}+ decimal ([-+]?({digit}*\.{digit}+)|({digit}+\.{digit}*)) @@ -90,10 +88,7 @@ realfail2 ({integer}|{decimal})[Ee][-+] {identifier_quoted} { ddlget_lval(yyscanner)->str = scanner_copy( ddlget_text(yyscanner), yyscanner, STRIP_QUOTES ); return IDENT; } -{identifier_double_quoted} { ddlget_lval(yyscanner)->str = scanner_copy( ddlget_text(yyscanner), yyscanner, STRIP_QUOTES ); return IDENT; } -{fq_identifier} { ddlget_lval(yyscanner)->str = scanner_copy(ddlget_text(yyscanner), yyscanner); return FQ_IDENT; } -{fq_quoted} { ddlget_lval(yyscanner)->str = scanner_copy(ddlget_text(yyscanner), yyscanner, STRIP_QUOTES_FQ); return FQ_IDENT; } -{fq_double_quoted} { ddlget_lval(yyscanner)->str = scanner_copy(ddlget_text(yyscanner), yyscanner, STRIP_QUOTES_FQ); return FQ_IDENT; } +{identifier_double_quoted} { ddlget_lval(yyscanner)->str = scanner_copy( ddlget_text(yyscanner), yyscanner, STRIP_QUOTES ); return DQ_IDENT; } ACTION {return ACTION;} ADD {return ADD;} @@ -193,14 +188,14 @@ LONGTEXT {return LONGTEXT;} /* ignore */ } -{identifier} {ddlget_lval(yyscanner)->str = scanner_copy(ddlget_text(yyscanner), yyscanner); return IDENT;} +{identifier} { ddlget_lval(yyscanner)->str = scanner_copy(ddlget_text(yyscanner), yyscanner); return DQ_IDENT;} {self} { return ddlget_text(yyscanner)[0]; } {grave_accent} { - /* ignore */ + return ddlget_text(yyscanner)[0]; } "/*" { BEGIN(c_comment); } @@ -282,7 +277,6 @@ char* scanner_copy (char *str, yyscan_t yyscanner, copy_action_t action) char* result; char* nv = strdup(str); result = nv; - // free strduped memory later to prevent possible memory leak if(nv) ((scan_data*)ddlget_extra(yyscanner))->valbuf.push_back(nv); @@ -292,57 +286,6 @@ char* scanner_copy (char *str, yyscan_t yyscanner, copy_action_t action) nv[strlen(str) - 1] = '\0'; result = nv + 1; } - else if (action == STRIP_QUOTES_FQ) - { - bool move_left = false; - bool move_right = false; - char* left = nv; - char* tmp_first = nv; - // MCOL-1384 Loop through all comas in this quoted fq id - // looking for $quote_sign.$quote_sign sequence. - char* fq_delimiter; - int tmp_pos = 0; - while((fq_delimiter = strchr(tmp_first, '.')) != NULL) - { - if( (*(fq_delimiter -1) == '`' && *(fq_delimiter + 1) == '`') || - (*(fq_delimiter -1) == '"' && *(fq_delimiter + 1) == '"') ) - { - tmp_pos += fq_delimiter - tmp_first; - break; - } - tmp_first = fq_delimiter; - } - - char* fq_delimiter_orig = str + tmp_pos; - char* right = fq_delimiter + 1; - char* right_orig = fq_delimiter_orig + 1; - // MCOL-1384 Strip quotes from the left part. - if(*left == '"' || *left == '`') - { - result = left + 1; - *(fq_delimiter - 1) = '.'; - move_left = true; - } - else - { - fq_delimiter += 1; - } - - int right_length = strlen(right); - // MCOL-1384 Strip quotes from the right part. - if(*right == '`' || *right == '"') - { - right += 1; right_orig += 1; - right_length -= 2; - move_right = true; - *(fq_delimiter + right_length) = '\0'; - } - - if(move_left || move_right) - { - strncpy(fq_delimiter, right_orig, right_length); - } - } return result; } diff --git a/dbcon/ddlpackage/ddl.y b/dbcon/ddlpackage/ddl.y index 68747dfc9..b143895b9 100644 --- a/dbcon/ddlpackage/ddl.y +++ b/dbcon/ddlpackage/ddl.y @@ -29,26 +29,18 @@ Understanding the New Sql book The postgress and mysql sources. find x -name \*.y -o -name \*.yy. - We don't support delimited identifiers. + We support quoted identifiers. All literals are stored as unconverted strings. You can't say "NOT DEFERRABLE". See the comment below. - This is not a reentrant parser. It uses the original global - variable style method of communication between the parser and - scanner. If we ever needed more than one parser thread per - processes, we would use the pure/reentrant options of bison and - flex. In that model, things that are traditionally global live - inside a struct that is passed around. We would need to upgrade to - a more recent version of flex. At the time of this writing, our - development systems have: flex version 2.5.4 + This is a reentrant parser. MCOL-66 Modify to be a reentrant parser */ %{ -#include "string.h" #include "sqlparser.h" #ifdef _MSC_VER @@ -122,7 +114,7 @@ REFERENCES RENAME RESTRICT SET SMALLINT TABLE TEXT TIME TINYBLOB TINYTEXT TINYINT TO UNIQUE UNSIGNED UPDATE USER SESSION_USER SYSTEM_USER VARCHAR VARBINARY VARYING WITH ZONE DOUBLE IDB_FLOAT REAL CHARSET IDB_IF EXISTS CHANGE TRUNCATE -%token FQ_IDENT IDENT FCONST SCONST CP_SEARCH_CONDITION_TEXT ICONST DATE +%token DQ_IDENT IDENT FCONST SCONST CP_SEARCH_CONDITION_TEXT ICONST DATE /* Notes: * 1. "ata" stands for alter_table_action @@ -476,7 +468,7 @@ opt_equal: ; table_option: - ENGINE opt_equal IDENT {$$ = new pair("engine", $3);} + ENGINE opt_equal ident {$$ = new pair("engine", $3);} | MAX_ROWS opt_equal ICONST {$$ = new pair("max_rows", $3);} | @@ -491,9 +483,9 @@ table_option: $$ = new pair("auto_increment", $3); } | - DEFAULT CHARSET opt_equal IDENT {$$ = new pair("default charset", $4);} + DEFAULT CHARSET opt_equal ident {$$ = new pair("default charset", $4);} | - DEFAULT IDB_CHAR SET opt_equal IDENT {$$ = new pair("default charset", $5);} + DEFAULT IDB_CHAR SET opt_equal ident {$$ = new pair("default charset", $5);} ; alter_table_statement: @@ -623,27 +615,23 @@ table_name: ; qualified_name: - FQ_IDENT { - char* delimeterPosition = strchr(const_cast($1), '.'); - if( delimeterPosition ) - { - *delimeterPosition = '\0'; - char* schemaName = const_cast($1); - char* tableName = delimeterPosition + 1; - $$ = new QualifiedName(schemaName, tableName); - *delimeterPosition = '.'; - } - else - $$ = new QualifiedName($1); - } - | IDENT { + | ident { if (x->fDBSchema.size()) $$ = new QualifiedName((char*)x->fDBSchema.c_str(), $1); else $$ = new QualifiedName($1); } + | ident '.' ident + { + $$ = new QualifiedName($1, $3); + } ; +ident: + DQ_IDENT + | IDENT + ; + ata_add_column: /* See the documentation for SchemaObject for an explanation of why we are using * dynamic_cast here. @@ -656,11 +644,11 @@ ata_add_column: column_name: DATE - |IDENT + |ident ; constraint_name: - IDENT + ident ; column_option: @@ -720,6 +708,10 @@ default_clause: { $$ = new ColumnDefaultValue($2); } + | DEFAULT DQ_IDENT /* MCOL-1406 */ + { + $$ = new ColumnDefaultValue($2); + } | DEFAULT NULL_TOK {$$ = new ColumnDefaultValue(NULL);} | DEFAULT USER {$$ = new ColumnDefaultValue("$USER");} | DEFAULT CURRENT_USER {$$ = new ColumnDefaultValue("$CURRENT_USER");} diff --git a/dbcon/joblist/jlf_execplantojoblist.cpp b/dbcon/joblist/jlf_execplantojoblist.cpp index b81a2ec8d..f3782c9d5 100644 --- a/dbcon/joblist/jlf_execplantojoblist.cpp +++ b/dbcon/joblist/jlf_execplantojoblist.cpp @@ -1634,11 +1634,8 @@ const JobStepVector doSimpleFilter(SimpleFilter* sf, JobInfo& jobInfo) return doExpressionFilter(sf, jobInfo); } - // trim trailing space char in the predicate string constval(cc->constval()); - size_t spos = constval.find_last_not_of(" "); - if (spos != string::npos) constval = constval.substr(0, spos + 1); CalpontSystemCatalog::OID dictOid = 0; CalpontSystemCatalog::ColType ct = sc->colType(); @@ -2772,11 +2769,8 @@ const JobStepVector doConstantFilter(const ConstantFilter* cf, JobInfo& jobInfo) if (ConstantColumn::NULLDATA == cc->type() && (opeq == *sop || opne == *sop)) cop = COMPARE_NIL; - // trim trailing space char string value = cc->constval(); - size_t spos = value.find_last_not_of(" "); - if (spos != string::npos) value = value.substr(0, spos + 1); pds->addFilter(cop, value); } @@ -2858,11 +2852,8 @@ const JobStepVector doConstantFilter(const ConstantFilter* cf, JobInfo& jobInfo) if (ConstantColumn::NULLDATA == cc->type() && (opeq == *sop || opne == *sop)) cop = COMPARE_NIL; - // trim trailing space char string value = cc->constval(); - size_t spos = value.find_last_not_of(" "); - if (spos != string::npos) value = value.substr(0, spos + 1); pds->addFilter(cop, value); } @@ -2968,10 +2959,7 @@ const JobStepVector doConstantFilter(const ConstantFilter* cf, JobInfo& jobInfo) int8_t cop = op2num(sop); int64_t value = 0; string constval = cc->constval(); - // trim trailing space char - size_t spos = constval.find_last_not_of(" "); - if (spos != string::npos) constval = constval.substr(0, spos + 1); // @bug 1151 string longer than colwidth of char/varchar. uint8_t rf = 0; diff --git a/dbcon/joblist/tupleunion.cpp b/dbcon/joblist/tupleunion.cpp index d0892e45f..cf2d20373 100644 --- a/dbcon/joblist/tupleunion.cpp +++ b/dbcon/joblist/tupleunion.cpp @@ -460,7 +460,7 @@ void TupleUnion::normalize(const Row& in, Row* out) if (in.getScale(i)) { double d = in.getIntField(i); - d /= exp10(in.getScale(i)); + d /= exp10(in.getScale(i)); os.precision(15); os << d; } diff --git a/dbcon/mysql/ha_calpont_ddl.cpp b/dbcon/mysql/ha_calpont_ddl.cpp index fa8ef79dd..8628f1238 100644 --- a/dbcon/mysql/ha_calpont_ddl.cpp +++ b/dbcon/mysql/ha_calpont_ddl.cpp @@ -2224,7 +2224,7 @@ int ha_calpont_impl_rename_table_(const char* from, const char* to, cal_connecti int rc = ProcessDDLStatement(stmt, db, "", tid2sid(thd->thread_id), emsg); if (rc != 0) - push_warning(thd, Sql_condition::WARN_LEVEL_ERROR, 9999, emsg.c_str()); + push_warning(thd, Sql_condition::WARN_LEVEL_WARN, 9999, emsg.c_str()); return rc; } @@ -2266,7 +2266,7 @@ extern "C" int rc = ProcessDDLStatement(stmt, db, "", tid2sid(thd->thread_id), emsg, compressiontype); if (rc != 0) - push_warning(thd, Sql_condition::WARN_LEVEL_ERROR, 9999, emsg.c_str()); + push_warning(thd, Sql_condition::WARN_LEVEL_WARN, 9999, emsg.c_str()); return rc; } diff --git a/dbcon/mysql/ha_calpont_execplan.cpp b/dbcon/mysql/ha_calpont_execplan.cpp index 14a7786a1..ca7599e47 100644 --- a/dbcon/mysql/ha_calpont_execplan.cpp +++ b/dbcon/mysql/ha_calpont_execplan.cpp @@ -4674,7 +4674,6 @@ void gp_walk(const Item* item, void* arg) if (isp) { - // @bug 3669. trim trailing spaces for the compare value if (isp->result_type() == STRING_RESULT) { String val, *str = isp->val_str(&val); @@ -4685,10 +4684,7 @@ void gp_walk(const Item* item, void* arg) cval.assign(str->ptr(), str->length()); } - size_t spos = cval.find_last_not_of(" "); - if (spos != string::npos) - cval = cval.substr(0, spos + 1); gwip->rcWorkStack.push(new ConstantColumn(cval)); break; diff --git a/utils/funcexp/func_substring_index.cpp b/utils/funcexp/func_substring_index.cpp index 4948cfc5f..cf1395759 100644 --- a/utils/funcexp/func_substring_index.cpp +++ b/utils/funcexp/func_substring_index.cpp @@ -74,6 +74,9 @@ std::string Func_substring_index::getStrVal(rowgroup::Row& row, if ( count > (int64_t) end ) return str; + if (( count < 0 ) && ((count * -1) > end)) + return str; + string value = str; if ( count > 0 ) diff --git a/writeengine/splitter/we_cmdargs.h b/writeengine/splitter/we_cmdargs.h index 46a7524cc..db5c0c0ac 100644 --- a/writeengine/splitter/we_cmdargs.h +++ b/writeengine/splitter/we_cmdargs.h @@ -77,7 +77,7 @@ public: { return fLocFile; } - int getMode() const + int getReadBufSize() { return fReadBufSize; } { return fMode; } diff --git a/writeengine/splitter/we_filereadthread.cpp b/writeengine/splitter/we_filereadthread.cpp index 3110137dc..fb80189bb 100644 --- a/writeengine/splitter/we_filereadthread.cpp +++ b/writeengine/splitter/we_filereadthread.cpp @@ -87,6 +87,15 @@ WEFileReadThread::WEFileReadThread(WESDHandler& aSdh): fSdh(aSdh), { //TODO batch qty to get from config fBatchQty = 10000; + if (fSdh.getReadBufSize() < DEFAULTBUFFSIZE) + { + fBuffSize = DEFAULTBUFFSIZE; + } + else + { + fBuffSize = fSdh.getReadBufSize(); + } + fBuff = new char [fBuffSize]; } @@ -109,6 +118,7 @@ WEFileReadThread::~WEFileReadThread() } fpThread = 0; + delete []fBuff; //cout << "WEFileReadThread destructor called" << endl; } @@ -352,17 +362,17 @@ unsigned int WEFileReadThread::readDataFile(messageqcpp::SBS& Sbs) if (fEnclEsc) { //pStart = aBuff; - aLen = getNextRow(fInFile, fBuff, sizeof(fBuff) - 1); + aLen = getNextRow(fInFile, fBuff, fBuffSize-1); } else { - fInFile.getline(fBuff, sizeof(fBuff) - 1); + fInFile.getline(fBuff, fBuffSize-1); aLen = fInFile.gcount(); } ////aLen chars incl \n, Therefore aLen-1; '<<' oper won't go past it //cout << "Data Length " << aLen < 0)) + if((aLen < (fBuffSize-2)) && (aLen>0)) { fBuff[aLen - 1] = '\n'; fBuff[aLen] = 0; @@ -374,7 +384,7 @@ unsigned int WEFileReadThread::readDataFile(messageqcpp::SBS& Sbs) if (fSdh.getDebugLvl() > 2) cout << "File data line = " << aIdx << endl; } - else if (aLen >= sizeof(fBuff) - 2) //Didn't hit delim; BIG ROW + else if(aLen>=fBuffSize-2) //Didn't hit delim; BIG ROW { cout << "Bad Row data " << endl; cout << fBuff << endl; diff --git a/writeengine/splitter/we_filereadthread.h b/writeengine/splitter/we_filereadthread.h index c7cd54e82..eaa65037a 100644 --- a/writeengine/splitter/we_filereadthread.h +++ b/writeengine/splitter/we_filereadthread.h @@ -126,7 +126,7 @@ public: void add2InputDataFileList(std::string& FileName); private: - enum { MAXBUFFSIZE = 1024 * 1024 }; + enum { DEFAULTBUFFSIZE=1024*1024 }; // don't allow anyone else to set void setTgtPmId(unsigned int fTgtPmId) @@ -151,7 +151,8 @@ private: char fEncl; // Encl char char fEsc; // Esc char char fDelim; // Column Delimit char - char fBuff[MAXBUFFSIZE]; // main data buffer + char* fBuff; // main data buffer + int fBuffSize; }; } /* namespace WriteEngine */ diff --git a/writeengine/splitter/we_sdhandler.cpp b/writeengine/splitter/we_sdhandler.cpp index 802fd108b..61fe45239 100644 --- a/writeengine/splitter/we_sdhandler.cpp +++ b/writeengine/splitter/we_sdhandler.cpp @@ -2663,6 +2663,13 @@ char WESDHandler::getEscChar() //------------------------------------------------------------------------------ +int WESDHandler::getReadBufSize() +{ + return fRef.fCmdArgs.getReadBufSize(); +} + +//------------------------------------------------------------------------------ + char WESDHandler::getDelimChar() { return fRef.fCmdArgs.getDelimChar(); diff --git a/writeengine/splitter/we_sdhandler.h b/writeengine/splitter/we_sdhandler.h index b593b1420..f31f40c63 100644 --- a/writeengine/splitter/we_sdhandler.h +++ b/writeengine/splitter/we_sdhandler.h @@ -152,6 +152,7 @@ public: char getEscChar(); char getDelimChar(); bool getConsoleLog(); + int getReadBufSize(); ImportDataMode getImportDataMode() const; void sysLog(const logging::Message::Args& msgArgs, logging::LOG_TYPE logType, logging::Message::MessageID msgId); diff --git a/writeengine/wrapper/we_colop.cpp b/writeengine/wrapper/we_colop.cpp index 7aecf726d..df2080af6 100644 --- a/writeengine/wrapper/we_colop.cpp +++ b/writeengine/wrapper/we_colop.cpp @@ -222,7 +222,7 @@ int ColumnOp::allocRowId(const TxnID& txnid, bool useStartingExtent, //Find out where the rest rows go BRM::LBID_t startLbid; //need to put in a loop until newExtent is true - newExtent = dbRootExtentTrackers[0]->nextSegFile(dbRoot, partition, segment, newHwm, startLbid); + newExtent = dbRootExtentTrackers[column.colNo]->nextSegFile(dbRoot, partition, segment, newHwm, startLbid); TableMetaData* tableMetaData = TableMetaData::makeTableMetaData(tableOid); while (!newExtent) @@ -238,7 +238,7 @@ int ColumnOp::allocRowId(const TxnID& txnid, bool useStartingExtent, for (i = 0; i < dbRootExtentTrackers.size(); i++) { - if (i != 0) + if (i != column.colNo) dbRootExtentTrackers[i]->nextSegFile(dbRoot, partition, segment, newHwm, startLbid); // Round up HWM to the end of the current extent @@ -302,8 +302,7 @@ int ColumnOp::allocRowId(const TxnID& txnid, bool useStartingExtent, tableMetaData->setColExtsInfo(newColStructList[i].dataOid, aColExtsInfo); } - - newExtent = dbRootExtentTrackers[0]->nextSegFile(dbRoot, partition, segment, newHwm, startLbid); + newExtent = dbRootExtentTrackers[column.colNo]->nextSegFile(dbRoot, partition, segment, newHwm, startLbid); } } @@ -324,7 +323,7 @@ int ColumnOp::allocRowId(const TxnID& txnid, bool useStartingExtent, } rc = BRMWrapper::getInstance()->allocateStripeColExtents(cols, dbRoot, partition, segment, extents); - newHwm = extents[0].startBlkOffset; + newHwm = extents[column.colNo].startBlkOffset; if (rc != NO_ERROR) return rc; diff --git a/writeengine/wrapper/writeengine.cpp b/writeengine/wrapper/writeengine.cpp index 5d3dfec85..19f66fcbf 100644 --- a/writeengine/wrapper/writeengine.cpp +++ b/writeengine/wrapper/writeengine.cpp @@ -1683,6 +1683,19 @@ int WriteEngineWrapper::insertColumnRecsBinary(const TxnID& txnid, for (i = 0; i < colStructList.size(); i++) Convertor::convertColType(&colStructList[i]); + // MCOL-984: find the smallest column width to calculate the RowID from so + // that all HWMs will be incremented by this operation + int32_t lowColLen = 8192; + int32_t colId = 0; + for (uint32_t colIt = 0; colIt < colStructList.size(); colIt++) + { + if (colStructList[colIt].colWidth < lowColLen) + { + colId = colIt; + lowColLen = colStructList[colId].colWidth; + } + } + // rc = checkValid(txnid, colStructList, colValueList, ridList); // if (rc != NO_ERROR) // return rc; @@ -1709,8 +1722,8 @@ int WriteEngineWrapper::insertColumnRecsBinary(const TxnID& txnid, //-------------------------------------------------------------------------- if (isFirstBatchPm) { - currentDBrootIdx = dbRootExtentTrackers[0]->getCurrentDBRootIdx(); - extentInfo = dbRootExtentTrackers[0]->getDBRootExtentList(); + currentDBrootIdx = dbRootExtentTrackers[colId]->getCurrentDBRootIdx(); + extentInfo = dbRootExtentTrackers[colId]->getDBRootExtentList(); dbRoot = extentInfo[currentDBrootIdx].fDbRoot; partitionNum = extentInfo[currentDBrootIdx].fPartition; @@ -1895,7 +1908,7 @@ int WriteEngineWrapper::insertColumnRecsBinary(const TxnID& txnid, } // if (isFirstBatchPm) else //get the extent info from tableMetaData { - ColExtsInfo aColExtsInfo = tableMetaData->getColExtsInfo(colStructList[0].dataOid); + ColExtsInfo aColExtsInfo = tableMetaData->getColExtsInfo(colStructList[colId].dataOid); ColExtsInfo::iterator it = aColExtsInfo.begin(); while (it != aColExtsInfo.end()) @@ -1931,20 +1944,7 @@ int WriteEngineWrapper::insertColumnRecsBinary(const TxnID& txnid, // allocate row id(s) //-------------------------------------------------------------------------- - // MCOL-984: find the smallest column width to calculate the RowID from so - // that all HWMs will be incremented by this operation - int32_t lowColLen = 8192; - int32_t colId = 0; - - for (uint32_t colIt = 0; colIt < colStructList.size(); colIt++) - { - if (colStructList[colIt].colWidth < lowColLen) - { - colId = colIt; - lowColLen = colStructList[colId].colWidth; - curColStruct = colStructList[colId]; - } - } + curColStruct = colStructList[colId]; colOp = m_colOp[op(curColStruct.fCompressionType)]; @@ -1970,7 +1970,7 @@ int WriteEngineWrapper::insertColumnRecsBinary(const TxnID& txnid, if (it != aColExtsInfo.end()) { hwm = it->hwm; - //cout << "Got from colextinfo hwm for oid " << colStructList[0].dataOid << " is " << hwm << " and seg is " << colStructList[0].fColSegment << endl; + //cout << "Got from colextinfo hwm for oid " << colStructList[colId].dataOid << " is " << hwm << " and seg is " << colStructList[colId].fColSegment << endl; } oldHwm = hwm; //Save this info for rollback