From d010397be647a540189687434c57084bed1e7938 Mon Sep 17 00:00:00 2001 From: Andrew Hutchings Date: Wed, 9 May 2018 14:52:42 +0100 Subject: [PATCH 1/9] MCOL-1197 Make -c work in cpimport It turns out -c wasn't actually connected to anything and now with have BLOB/TEXT it is pretty useful. If -c is set to < 1MB then 1MB is used, otherwise it will use the selected buffer size. --- writeengine/splitter/we_cmdargs.h | 1 + writeengine/splitter/we_filereadthread.cpp | 18 ++++++++++++++---- writeengine/splitter/we_filereadthread.h | 5 +++-- writeengine/splitter/we_sdhandler.cpp | 7 +++++++ writeengine/splitter/we_sdhandler.h | 1 + 5 files changed, 26 insertions(+), 6 deletions(-) diff --git a/writeengine/splitter/we_cmdargs.h b/writeengine/splitter/we_cmdargs.h index 96e06a4bc..803f77c54 100644 --- a/writeengine/splitter/we_cmdargs.h +++ b/writeengine/splitter/we_cmdargs.h @@ -77,6 +77,7 @@ class WECmdArgs char getDelimChar() { return fColDelim; } ImportDataMode getImportDataMode() const { return fImportDataMode; } bool getConsoleLog() { return fConsoleLog; } + int getReadBufSize() { return fReadBufSize; } bool isCpimportInvokeMode(){return (fBlockMode3)? false : fCpiInvoke;} bool isQuiteMode() const { return fQuiteMode; } diff --git a/writeengine/splitter/we_filereadthread.cpp b/writeengine/splitter/we_filereadthread.cpp index 6e4fcb4a7..185557965 100644 --- a/writeengine/splitter/we_filereadthread.cpp +++ b/writeengine/splitter/we_filereadthread.cpp @@ -87,6 +87,15 @@ WEFileReadThread::WEFileReadThread(WESDHandler& aSdh):fSdh(aSdh), { //TODO batch qty to get from config fBatchQty = 10000; + if (fSdh.getReadBufSize() < DEFAULTBUFFSIZE) + { + fBuffSize = DEFAULTBUFFSIZE; + } + else + { + fBuffSize = fSdh.getReadBufSize(); + } + fBuff = new char [fBuffSize]; } @@ -106,6 +115,7 @@ WEFileReadThread::~WEFileReadThread() delete fpThread; } fpThread=0; + delete []fBuff; //cout << "WEFileReadThread destructor called" << endl; } @@ -330,16 +340,16 @@ unsigned int WEFileReadThread::readDataFile(messageqcpp::SBS& Sbs) if(fEnclEsc) { //pStart = aBuff; - aLen = getNextRow(fInFile, fBuff, sizeof(fBuff)-1); + aLen = getNextRow(fInFile, fBuff, fBuffSize-1); } else { - fInFile.getline(fBuff, sizeof(fBuff)-1); + fInFile.getline(fBuff, fBuffSize-1); aLen=fInFile.gcount(); } ////aLen chars incl \n, Therefore aLen-1; '<<' oper won't go past it //cout << "Data Length " << aLen <0)) + if((aLen < (fBuffSize-2)) && (aLen>0)) { fBuff[aLen-1] = '\n'; fBuff[aLen]=0; @@ -348,7 +358,7 @@ unsigned int WEFileReadThread::readDataFile(messageqcpp::SBS& Sbs) aIdx++; if(fSdh.getDebugLvl()>2) cout << "File data line = " << aIdx <=sizeof(fBuff)-2) //Didn't hit delim; BIG ROW + else if(aLen>=fBuffSize-2) //Didn't hit delim; BIG ROW { cout <<"Bad Row data " << endl; cout << fBuff << endl; diff --git a/writeengine/splitter/we_filereadthread.h b/writeengine/splitter/we_filereadthread.h index 623184e8d..f9a486d5c 100644 --- a/writeengine/splitter/we_filereadthread.h +++ b/writeengine/splitter/we_filereadthread.h @@ -98,7 +98,7 @@ public: void add2InputDataFileList(std::string& FileName); private: - enum { MAXBUFFSIZE=1024*1024 }; + enum { DEFAULTBUFFSIZE=1024*1024 }; // don't allow anyone else to set void setTgtPmId(unsigned int fTgtPmId) { this->fTgtPmId = fTgtPmId; } @@ -120,7 +120,8 @@ private: char fEncl; // Encl char char fEsc; // Esc char char fDelim; // Column Delimit char - char fBuff[MAXBUFFSIZE]; // main data buffer + char* fBuff; // main data buffer + int fBuffSize; }; } /* namespace WriteEngine */ diff --git a/writeengine/splitter/we_sdhandler.cpp b/writeengine/splitter/we_sdhandler.cpp index 56ace80dc..78019b338 100644 --- a/writeengine/splitter/we_sdhandler.cpp +++ b/writeengine/splitter/we_sdhandler.cpp @@ -2301,6 +2301,13 @@ char WESDHandler::getEscChar() //------------------------------------------------------------------------------ +int WESDHandler::getReadBufSize() +{ + return fRef.fCmdArgs.getReadBufSize(); +} + +//------------------------------------------------------------------------------ + char WESDHandler::getDelimChar() { return fRef.fCmdArgs.getDelimChar(); diff --git a/writeengine/splitter/we_sdhandler.h b/writeengine/splitter/we_sdhandler.h index 2d4b20cc2..ff6bc829e 100644 --- a/writeengine/splitter/we_sdhandler.h +++ b/writeengine/splitter/we_sdhandler.h @@ -149,6 +149,7 @@ public: char getEscChar(); char getDelimChar(); bool getConsoleLog(); + int getReadBufSize(); ImportDataMode getImportDataMode() const; void sysLog(const logging::Message::Args& msgArgs, logging::LOG_TYPE logType, logging::Message::MessageID msgId); From fbf2f2e979499a1ccf2d892133ff7cda28f41bd1 Mon Sep 17 00:00:00 2001 From: Andrew Hutchings Date: Thu, 10 May 2018 17:35:38 +0100 Subject: [PATCH 2/9] MCOL-1403 Remove whitespace trimming on constants This appears to be to fix equality matches in InfiniDB but at the same time it breaks LIKE processing. Equality matching with trailing whitespace was fixed in MCOL-1246 so the old InfiniDB patch can be removed. --- dbcon/joblist/jlf_execplantojoblist.cpp | 12 ------------ dbcon/mysql/ha_calpont_execplan.cpp | 4 ---- 2 files changed, 16 deletions(-) diff --git a/dbcon/joblist/jlf_execplantojoblist.cpp b/dbcon/joblist/jlf_execplantojoblist.cpp index b6242ba77..cfd694f9b 100644 --- a/dbcon/joblist/jlf_execplantojoblist.cpp +++ b/dbcon/joblist/jlf_execplantojoblist.cpp @@ -1500,10 +1500,7 @@ const JobStepVector doSimpleFilter(SimpleFilter* sf, JobInfo& jobInfo) return doExpressionFilter(sf, jobInfo); } - // trim trailing space char in the predicate string constval(cc->constval()); - size_t spos = constval.find_last_not_of(" "); - if (spos != string::npos) constval = constval.substr(0, spos+1); CalpontSystemCatalog::OID dictOid = 0; CalpontSystemCatalog::ColType ct = sc->colType(); @@ -2569,10 +2566,7 @@ const JobStepVector doConstantFilter(const ConstantFilter* cf, JobInfo& jobInfo) if (ConstantColumn::NULLDATA == cc->type() && (opeq == *sop || opne == *sop)) cop = COMPARE_NIL; - // trim trailing space char string value = cc->constval(); - size_t spos = value.find_last_not_of(" "); - if (spos != string::npos) value = value.substr(0, spos+1); pds->addFilter(cop, value); } @@ -2652,10 +2646,7 @@ const JobStepVector doConstantFilter(const ConstantFilter* cf, JobInfo& jobInfo) if (ConstantColumn::NULLDATA == cc->type() && (opeq == *sop || opne == *sop)) cop = COMPARE_NIL; - // trim trailing space char string value = cc->constval(); - size_t spos = value.find_last_not_of(" "); - if (spos != string::npos) value = value.substr(0, spos+1); pds->addFilter(cop, value); } @@ -2759,9 +2750,6 @@ const JobStepVector doConstantFilter(const ConstantFilter* cf, JobInfo& jobInfo) int8_t cop = op2num(sop); int64_t value = 0; string constval = cc->constval(); - // trim trailing space char - size_t spos = constval.find_last_not_of(" "); - if (spos != string::npos) constval = constval.substr(0, spos+1); // @bug 1151 string longer than colwidth of char/varchar. uint8_t rf = 0; diff --git a/dbcon/mysql/ha_calpont_execplan.cpp b/dbcon/mysql/ha_calpont_execplan.cpp index 9cc3a99c8..a30d71688 100755 --- a/dbcon/mysql/ha_calpont_execplan.cpp +++ b/dbcon/mysql/ha_calpont_execplan.cpp @@ -4099,7 +4099,6 @@ void gp_walk(const Item *item, void *arg) Item_string* isp = (Item_string*)item; if (isp) { - // @bug 3669. trim trailing spaces for the compare value if (isp->result_type() == STRING_RESULT) { String val, *str = isp->val_str(&val); @@ -4108,9 +4107,6 @@ void gp_walk(const Item *item, void *arg) { cval.assign(str->ptr(), str->length()); } - size_t spos = cval.find_last_not_of(" "); - if (spos != string::npos) - cval = cval.substr(0, spos+1); gwip->rcWorkStack.push(new ConstantColumn(cval)); break; } From baf42e7b4a83ce88e89ed7d551f2286e2016de1b Mon Sep 17 00:00:00 2001 From: Andrew Hutchings Date: Thu, 10 May 2018 18:32:22 +0100 Subject: [PATCH 3/9] MCOL-1390 Fix SUBSTRING_INDEX for negative count If negative count number is more than the number of characters in the string then it should always return the string. For example if a table contains SUBSTRING_INDEX('zzz', 'z', -5) should return 'zzz'. Before this patch it would return NULL. --- utils/funcexp/func_substring_index.cpp | 3 +++ 1 file changed, 3 insertions(+) diff --git a/utils/funcexp/func_substring_index.cpp b/utils/funcexp/func_substring_index.cpp index e3ec80b5f..a4b9fb9a6 100644 --- a/utils/funcexp/func_substring_index.cpp +++ b/utils/funcexp/func_substring_index.cpp @@ -71,6 +71,9 @@ std::string Func_substring_index::getStrVal(rowgroup::Row& row, if ( count > (int64_t) end ) return str; + if (( count < 0 ) && ((count * -1) > end)) + return str; + string value = str; if ( count > 0 ) { From 5b1f5d5fe400b00a83a9b38ac66b7d7b34d1640a Mon Sep 17 00:00:00 2001 From: Andrew Hutchings Date: Mon, 14 May 2018 22:03:25 +0100 Subject: [PATCH 4/9] MCOL-1412 Ubuntu 18.04 support Backport Ubuntu 18.04 support to 1.1 --- dbcon/joblist/tupleunion.cpp | 6 +++--- utils/common/any.hpp | 28 +++++++++++++++++++--------- utils/common/cgroupconfigurator.cpp | 1 + utils/threadpool/threadpool.cpp | 1 + 4 files changed, 24 insertions(+), 12 deletions(-) diff --git a/dbcon/joblist/tupleunion.cpp b/dbcon/joblist/tupleunion.cpp index 2fdd4330f..2ae631abb 100644 --- a/dbcon/joblist/tupleunion.cpp +++ b/dbcon/joblist/tupleunion.cpp @@ -47,7 +47,7 @@ using namespace dataconvert; #endif namespace { //returns the value of 10 raised to the power x. -inline double pow10(double x) +inline double exp10(double x) { return exp(x * M_LN10); } @@ -406,7 +406,7 @@ void TupleUnion::normalize(const Row &in, Row *out) ostringstream os; if (in.getScale(i)) { double d = in.getIntField(i); - d /= pow10(in.getScale(i)); + d /= exp10(in.getScale(i)); os.precision(15); os << d; } @@ -488,7 +488,7 @@ dec1: uint64_t val = in.getIntField(i); ostringstream os; if (in.getScale(i)) { double d = in.getUintField(i); - d /= pow10(in.getScale(i)); + d /= exp10(in.getScale(i)); os.precision(15); os << d; } diff --git a/utils/common/any.hpp b/utils/common/any.hpp index 5265015f1..be0ca679b 100755 --- a/utils/common/any.hpp +++ b/utils/common/any.hpp @@ -54,15 +54,25 @@ namespace anyimpl template struct big_any_policy : typed_base_any_policy { - virtual void static_delete(void** x) { if (*x) - delete(*reinterpret_cast(x)); *x = NULL; } - virtual void copy_from_value(void const* src, void** dest) { - *dest = new T(*reinterpret_cast(src)); } - virtual void clone(void* const* src, void** dest) { - *dest = new T(**reinterpret_cast(src)); } - virtual void move(void* const* src, void** dest) { - (*reinterpret_cast(dest))->~T(); - **reinterpret_cast(dest) = **reinterpret_cast(src); } + virtual void static_delete(void** x) + { + if (*x) + delete(*reinterpret_cast(x)); + *x = NULL; + } + virtual void copy_from_value(void const* src, void** dest) + { + *dest = new T(*reinterpret_cast(src)); + } + virtual void clone(void* const* src, void** dest) + { + *dest = new T(**reinterpret_cast(src)); + } + virtual void move(void* const* src, void** dest) + { + (*reinterpret_cast(dest))->~T(); + **reinterpret_cast(dest) = **reinterpret_cast(src); + } virtual void* get_value(void** src) { return *src; } }; diff --git a/utils/common/cgroupconfigurator.cpp b/utils/common/cgroupconfigurator.cpp index a4a67d68e..fcf7ef0e9 100644 --- a/utils/common/cgroupconfigurator.cpp +++ b/utils/common/cgroupconfigurator.cpp @@ -19,6 +19,7 @@ #include "configcpp.h" #include "logger.h" #include +#include #include #ifdef _MSC_VER #include "unistd.h" diff --git a/utils/threadpool/threadpool.cpp b/utils/threadpool/threadpool.cpp index 197bdedd9..d903f9892 100644 --- a/utils/threadpool/threadpool.cpp +++ b/utils/threadpool/threadpool.cpp @@ -21,6 +21,7 @@ * ***********************************************************************/ #include +#include using namespace std; #include "messageobj.h" From 59858aa8962ff857df829b0b65dfc6016eeab8a7 Mon Sep 17 00:00:00 2001 From: Roman Nozdrin Date: Thu, 17 May 2018 10:01:17 +0300 Subject: [PATCH 5/9] MCOL-1415 Fixed regression with extra spaces after dot in qualified identifiers. --- dbcon/ddlpackage/ddl.y | 4 ++++ 1 file changed, 4 insertions(+) diff --git a/dbcon/ddlpackage/ddl.y b/dbcon/ddlpackage/ddl.y index 982167287..398a8612f 100644 --- a/dbcon/ddlpackage/ddl.y +++ b/dbcon/ddlpackage/ddl.y @@ -631,6 +631,10 @@ qualified_name: else $$ = new QualifiedName($1); } + | IDENT '.' IDENT + { + $$ = new QualifiedName($1, $3); + } ; ata_add_column: From 8f3faee25dd94ef95b126791d319da4cb007980d Mon Sep 17 00:00:00 2001 From: Roman Nozdrin Date: Wed, 23 May 2018 23:38:11 +0300 Subject: [PATCH 6/9] MCOL-1406 Fixed the regression. --- dbcon/ddlpackage/ddl.l | 4 ++-- dbcon/ddlpackage/ddl.y | 36 ++++++++++++++++++++---------------- 2 files changed, 22 insertions(+), 18 deletions(-) diff --git a/dbcon/ddlpackage/ddl.l b/dbcon/ddlpackage/ddl.l index f65ef161d..34d80e902 100644 --- a/dbcon/ddlpackage/ddl.l +++ b/dbcon/ddlpackage/ddl.l @@ -86,7 +86,7 @@ realfail2 ({integer}|{decimal})[Ee][-+] {identifier_quoted} { ddlget_lval(yyscanner)->str = scanner_copy( ddlget_text(yyscanner), yyscanner, STRIP_QUOTES ); return IDENT; } -{identifier_double_quoted} { ddlget_lval(yyscanner)->str = scanner_copy( ddlget_text(yyscanner), yyscanner, STRIP_QUOTES ); return IDENT; } +{identifier_double_quoted} { ddlget_lval(yyscanner)->str = scanner_copy( ddlget_text(yyscanner), yyscanner, STRIP_QUOTES ); return DQ_IDENT; } {fq_identifier} { ddlget_lval(yyscanner)->str = scanner_copy(ddlget_text(yyscanner), yyscanner); return FQ_IDENT; } {fq_quoted} { ddlget_lval(yyscanner)->str = scanner_copy(ddlget_text(yyscanner), yyscanner, STRIP_QUOTES_FQ); return FQ_IDENT; } {fq_double_quoted} { ddlget_lval(yyscanner)->str = scanner_copy(ddlget_text(yyscanner), yyscanner, STRIP_QUOTES_FQ); return FQ_IDENT; } @@ -188,7 +188,7 @@ LONGTEXT {return LONGTEXT;} /* ignore */ } -{identifier} {ddlget_lval(yyscanner)->str = scanner_copy(ddlget_text(yyscanner), yyscanner); return IDENT;} +{identifier} { ddlget_lval(yyscanner)->str = scanner_copy(ddlget_text(yyscanner), yyscanner); return DQ_IDENT;} {self} { return ddlget_text(yyscanner)[0]; diff --git a/dbcon/ddlpackage/ddl.y b/dbcon/ddlpackage/ddl.y index 398a8612f..644cb9d9e 100644 --- a/dbcon/ddlpackage/ddl.y +++ b/dbcon/ddlpackage/ddl.y @@ -29,20 +29,13 @@ Understanding the New Sql book The postgress and mysql sources. find x -name \*.y -o -name \*.yy. - We don't support delimited identifiers. + We support quoted identifiers. All literals are stored as unconverted strings. You can't say "NOT DEFERRABLE". See the comment below. - This is not a reentrant parser. It uses the original global - variable style method of communication between the parser and - scanner. If we ever needed more than one parser thread per - processes, we would use the pure/reentrant options of bison and - flex. In that model, things that are traditionally global live - inside a struct that is passed around. We would need to upgrade to - a more recent version of flex. At the time of this writing, our - development systems have: flex version 2.5.4 + This is a reentrant parser. MCOL-66 Modify to be a reentrant parser */ @@ -122,7 +115,7 @@ REFERENCES RENAME RESTRICT SET SMALLINT TABLE TEXT TIME TINYBLOB TINYTEXT TINYINT TO UNIQUE UNSIGNED UPDATE USER SESSION_USER SYSTEM_USER VARCHAR VARBINARY VARYING WITH ZONE DOUBLE IDB_FLOAT REAL CHARSET IDB_IF EXISTS CHANGE TRUNCATE -%token FQ_IDENT IDENT FCONST SCONST CP_SEARCH_CONDITION_TEXT ICONST DATE +%token DQ_IDENT FQ_IDENT IDENT FCONST SCONST CP_SEARCH_CONDITION_TEXT ICONST DATE /* Notes: * 1. "ata" stands for alter_table_action @@ -206,6 +199,8 @@ VARYING WITH ZONE DOUBLE IDB_FLOAT REAL CHARSET IDB_IF EXISTS CHANGE TRUNCATE %type opt_if_exists %type opt_if_not_exists %type trunc_table_statement +%type rename_table_statement +%type ident %% stmtblock: stmtmulti { x->fParseTree = $1; } @@ -465,7 +460,7 @@ opt_equal: ; table_option: - ENGINE opt_equal IDENT {$$ = new pair("engine", $3);} + ENGINE opt_equal ident {$$ = new pair("engine", $3);} | MAX_ROWS opt_equal ICONST {$$ = new pair("max_rows", $3);} | @@ -480,9 +475,9 @@ table_option: $$ = new pair("auto_increment", $3); } | - DEFAULT CHARSET opt_equal IDENT {$$ = new pair("default charset", $4);} + DEFAULT CHARSET opt_equal ident {$$ = new pair("default charset", $4);} | - DEFAULT IDB_CHAR SET opt_equal IDENT {$$ = new pair("default charset", $5);} + DEFAULT IDB_CHAR SET opt_equal ident {$$ = new pair("default charset", $5);} ; alter_table_statement: @@ -625,7 +620,7 @@ qualified_name: else $$ = new QualifiedName($1); } - | IDENT { + | ident { if (x->fDBSchema.size()) $$ = new QualifiedName((char*)x->fDBSchema.c_str(), $1); else @@ -637,6 +632,11 @@ qualified_name: } ; +ident: + DQ_IDENT + | IDENT + ; + ata_add_column: /* See the documentation for SchemaObject for an explanation of why we are using * dynamic_cast here. @@ -649,11 +649,11 @@ ata_add_column: column_name: DATE - |IDENT + |ident ; constraint_name: - IDENT + ident ; column_option: @@ -713,6 +713,10 @@ default_clause: { $$ = new ColumnDefaultValue($2); } + | DEFAULT DQ_IDENT /* MCOL-1406 */ + { + $$ = new ColumnDefaultValue($2); + } | DEFAULT NULL_TOK {$$ = new ColumnDefaultValue(NULL);} | DEFAULT USER {$$ = new ColumnDefaultValue("$USER");} | DEFAULT CURRENT_USER {$$ = new ColumnDefaultValue("$CURRENT_USER");} From 04c87aca31945112769b5adade11ef1e305d4f17 Mon Sep 17 00:00:00 2001 From: Roman Nozdrin Date: Fri, 25 May 2018 12:28:00 +0300 Subject: [PATCH 7/9] MCOL-1406 Removed unused non-terminal token. --- dbcon/ddlpackage/ddl.y | 1 - 1 file changed, 1 deletion(-) diff --git a/dbcon/ddlpackage/ddl.y b/dbcon/ddlpackage/ddl.y index 644cb9d9e..cf5893773 100644 --- a/dbcon/ddlpackage/ddl.y +++ b/dbcon/ddlpackage/ddl.y @@ -199,7 +199,6 @@ VARYING WITH ZONE DOUBLE IDB_FLOAT REAL CHARSET IDB_IF EXISTS CHANGE TRUNCATE %type opt_if_exists %type opt_if_not_exists %type trunc_table_statement -%type rename_table_statement %type ident %% From b6424480c077411f8045b6374334fb196871d43b Mon Sep 17 00:00:00 2001 From: Andrew Hutchings Date: Tue, 29 May 2018 21:21:38 +0100 Subject: [PATCH 8/9] MCOL-1408 Fix HWM calculation for DML & API HWM for DML and API was being calculated using the first column in a table instead of the smallest column. This shifts the calculation to the correct column. --- writeengine/wrapper/we_colop.cpp | 8 +++---- writeengine/wrapper/writeengine.cpp | 36 ++++++++++++++--------------- 2 files changed, 22 insertions(+), 22 deletions(-) diff --git a/writeengine/wrapper/we_colop.cpp b/writeengine/wrapper/we_colop.cpp index ac13a7bf0..fdaceef3e 100644 --- a/writeengine/wrapper/we_colop.cpp +++ b/writeengine/wrapper/we_colop.cpp @@ -208,7 +208,7 @@ int ColumnOp::allocRowId(const TxnID& txnid, bool useStartingExtent, //Find out where the rest rows go BRM::LBID_t startLbid; //need to put in a loop until newExtent is true - newExtent = dbRootExtentTrackers[0]->nextSegFile(dbRoot, partition, segment, newHwm, startLbid); + newExtent = dbRootExtentTrackers[column.colNo]->nextSegFile(dbRoot, partition, segment, newHwm, startLbid); TableMetaData* tableMetaData= TableMetaData::makeTableMetaData(tableOid); while (!newExtent) { @@ -223,7 +223,7 @@ int ColumnOp::allocRowId(const TxnID& txnid, bool useStartingExtent, for (i=0; i < dbRootExtentTrackers.size(); i++) { - if (i != 0) + if (i != column.colNo) dbRootExtentTrackers[i]->nextSegFile(dbRoot, partition, segment, newHwm, startLbid); // Round up HWM to the end of the current extent @@ -278,7 +278,7 @@ int ColumnOp::allocRowId(const TxnID& txnid, bool useStartingExtent, } tableMetaData->setColExtsInfo(newColStructList[i].dataOid, aColExtsInfo); } - newExtent = dbRootExtentTrackers[0]->nextSegFile(dbRoot, partition, segment, newHwm, startLbid); + newExtent = dbRootExtentTrackers[column.colNo]->nextSegFile(dbRoot, partition, segment, newHwm, startLbid); } } @@ -297,7 +297,7 @@ int ColumnOp::allocRowId(const TxnID& txnid, bool useStartingExtent, } rc = BRMWrapper::getInstance()->allocateStripeColExtents(cols, dbRoot, partition, segment, extents); - newHwm = extents[0].startBlkOffset; + newHwm = extents[column.colNo].startBlkOffset; if (rc != NO_ERROR) return rc; diff --git a/writeengine/wrapper/writeengine.cpp b/writeengine/wrapper/writeengine.cpp index 50d846e7c..7cb3ca85e 100644 --- a/writeengine/wrapper/writeengine.cpp +++ b/writeengine/wrapper/writeengine.cpp @@ -1505,6 +1505,19 @@ int WriteEngineWrapper::insertColumnRecsBinary(const TxnID& txnid, for (i = 0; i < colStructList.size(); i++) Convertor::convertColType(&colStructList[i]); + // MCOL-984: find the smallest column width to calculate the RowID from so + // that all HWMs will be incremented by this operation + int32_t lowColLen = 8192; + int32_t colId = 0; + for (uint32_t colIt = 0; colIt < colStructList.size(); colIt++) + { + if (colStructList[colIt].colWidth < lowColLen) + { + colId = colIt; + lowColLen = colStructList[colId].colWidth; + } + } + // rc = checkValid(txnid, colStructList, colValueList, ridList); // if (rc != NO_ERROR) // return rc; @@ -1531,8 +1544,8 @@ int WriteEngineWrapper::insertColumnRecsBinary(const TxnID& txnid, //-------------------------------------------------------------------------- if (isFirstBatchPm) { - currentDBrootIdx = dbRootExtentTrackers[0]->getCurrentDBRootIdx(); - extentInfo = dbRootExtentTrackers[0]->getDBRootExtentList(); + currentDBrootIdx = dbRootExtentTrackers[colId]->getCurrentDBRootIdx(); + extentInfo = dbRootExtentTrackers[colId]->getDBRootExtentList(); dbRoot = extentInfo[currentDBrootIdx].fDbRoot; partitionNum = extentInfo[currentDBrootIdx].fPartition; @@ -1698,7 +1711,7 @@ int WriteEngineWrapper::insertColumnRecsBinary(const TxnID& txnid, } // if (isFirstBatchPm) else //get the extent info from tableMetaData { - ColExtsInfo aColExtsInfo = tableMetaData->getColExtsInfo(colStructList[0].dataOid); + ColExtsInfo aColExtsInfo = tableMetaData->getColExtsInfo(colStructList[colId].dataOid); ColExtsInfo::iterator it = aColExtsInfo.begin(); while (it != aColExtsInfo.end()) { @@ -1730,20 +1743,7 @@ int WriteEngineWrapper::insertColumnRecsBinary(const TxnID& txnid, //-------------------------------------------------------------------------- // allocate row id(s) //-------------------------------------------------------------------------- - - // MCOL-984: find the smallest column width to calculate the RowID from so - // that all HWMs will be incremented by this operation - int32_t lowColLen = 8192; - int32_t colId = 0; - for (uint32_t colIt = 0; colIt < colStructList.size(); colIt++) - { - if (colStructList[colIt].colWidth < lowColLen) - { - colId = colIt; - lowColLen = colStructList[colId].colWidth; - curColStruct = colStructList[colId]; - } - } + curColStruct = colStructList[colId]; colOp = m_colOp[op(curColStruct.fCompressionType)]; colOp->initColumn(curCol); @@ -1765,7 +1765,7 @@ int WriteEngineWrapper::insertColumnRecsBinary(const TxnID& txnid, if (it != aColExtsInfo.end()) { hwm = it->hwm; - //cout << "Got from colextinfo hwm for oid " << colStructList[0].dataOid << " is " << hwm << " and seg is " << colStructList[0].fColSegment << endl; + //cout << "Got from colextinfo hwm for oid " << colStructList[colId].dataOid << " is " << hwm << " and seg is " << colStructList[colId].fColSegment << endl; } oldHwm = hwm; //Save this info for rollback From efbf297eb75fb4b3e9d8edc2e195631744494be8 Mon Sep 17 00:00:00 2001 From: Roman Nozdrin Date: Thu, 31 May 2018 10:45:22 +0300 Subject: [PATCH 9/9] MCOL-1384 Backport the MCOL-573 feature to 1.1. Change msg type to avoid server code assert violation. --- dbcon/ddlpackage/ddl.l | 61 ++-------------------------------- dbcon/ddlpackage/ddl.y | 18 ++-------- dbcon/mysql/ha_calpont_ddl.cpp | 4 +-- 3 files changed, 6 insertions(+), 77 deletions(-) diff --git a/dbcon/ddlpackage/ddl.l b/dbcon/ddlpackage/ddl.l index 34d80e902..6eeaafb0b 100644 --- a/dbcon/ddlpackage/ddl.l +++ b/dbcon/ddlpackage/ddl.l @@ -32,7 +32,7 @@ #endif using namespace ddlpackage; -typedef enum { NOOP, STRIP_QUOTES, STRIP_QUOTES_FQ } copy_action_t; +typedef enum { NOOP, STRIP_QUOTES } copy_action_t; int lineno = 1; void ddlerror(struct pass_to_bison* x, char const *s); @@ -72,8 +72,6 @@ identifier {ident_start}{ident_cont}* fq_identifier {identifier}\.{identifier} identifier_quoted {grave_accent}{identifier}{grave_accent} identifier_double_quoted {double_quote}{identifier}{double_quote} -fq_quoted ({identifier_quoted}|{identifier})\.({identifier_quoted}|{identifier}) -fq_double_quoted ({identifier_double_quoted}|{identifier})\.({identifier_double_quoted}|{identifier}) integer [-+]?{digit}+ decimal ([-+]?({digit}*\.{digit}+)|({digit}+\.{digit}*)) @@ -87,9 +85,6 @@ realfail2 ({integer}|{decimal})[Ee][-+] {identifier_quoted} { ddlget_lval(yyscanner)->str = scanner_copy( ddlget_text(yyscanner), yyscanner, STRIP_QUOTES ); return IDENT; } {identifier_double_quoted} { ddlget_lval(yyscanner)->str = scanner_copy( ddlget_text(yyscanner), yyscanner, STRIP_QUOTES ); return DQ_IDENT; } -{fq_identifier} { ddlget_lval(yyscanner)->str = scanner_copy(ddlget_text(yyscanner), yyscanner); return FQ_IDENT; } -{fq_quoted} { ddlget_lval(yyscanner)->str = scanner_copy(ddlget_text(yyscanner), yyscanner, STRIP_QUOTES_FQ); return FQ_IDENT; } -{fq_double_quoted} { ddlget_lval(yyscanner)->str = scanner_copy(ddlget_text(yyscanner), yyscanner, STRIP_QUOTES_FQ); return FQ_IDENT; } ACTION {return ACTION;} ADD {return ADD;} @@ -195,7 +190,7 @@ LONGTEXT {return LONGTEXT;} } {grave_accent} { - /* ignore */ + return ddlget_text(yyscanner)[0]; } %% @@ -273,7 +268,6 @@ char* scanner_copy (char *str, yyscan_t yyscanner, copy_action_t action) char* result; char* nv = strdup(str); result = nv; - // free strduped memory later to prevent possible memory leak if(nv) ((scan_data*)ddlget_extra(yyscanner))->valbuf.push_back(nv); @@ -283,57 +277,6 @@ char* scanner_copy (char *str, yyscan_t yyscanner, copy_action_t action) nv[strlen(str) - 1] = '\0'; result = nv + 1; } - else if (action == STRIP_QUOTES_FQ) - { - bool move_left = false; - bool move_right = false; - char* left = nv; - char* tmp_first = nv; - // MCOL-1384 Loop through all comas in this quoted fq id - // looking for $quote_sign.$quote_sign sequence. - char* fq_delimiter; - int tmp_pos = 0; - while((fq_delimiter = strchr(tmp_first, '.')) != NULL) - { - if( (*(fq_delimiter -1) == '`' && *(fq_delimiter + 1) == '`') || - (*(fq_delimiter -1) == '"' && *(fq_delimiter + 1) == '"') ) - { - tmp_pos += fq_delimiter - tmp_first; - break; - } - tmp_first = fq_delimiter; - } - - char* fq_delimiter_orig = str + tmp_pos; - char* right = fq_delimiter + 1; - char* right_orig = fq_delimiter_orig + 1; - // MCOL-1384 Strip quotes from the left part. - if(*left == '"' || *left == '`') - { - result = left + 1; - *(fq_delimiter - 1) = '.'; - move_left = true; - } - else - { - fq_delimiter += 1; - } - - int right_length = strlen(right); - // MCOL-1384 Strip quotes from the right part. - if(*right == '`' || *right == '"') - { - right += 1; right_orig += 1; - right_length -= 2; - move_right = true; - *(fq_delimiter + right_length) = '\0'; - } - - if(move_left || move_right) - { - strncpy(fq_delimiter, right_orig, right_length); - } - } return result; } diff --git a/dbcon/ddlpackage/ddl.y b/dbcon/ddlpackage/ddl.y index cf5893773..96867cfb8 100644 --- a/dbcon/ddlpackage/ddl.y +++ b/dbcon/ddlpackage/ddl.y @@ -41,7 +41,6 @@ */ %{ -#include "string.h" #include "sqlparser.h" #ifdef _MSC_VER @@ -115,7 +114,7 @@ REFERENCES RENAME RESTRICT SET SMALLINT TABLE TEXT TIME TINYBLOB TINYTEXT TINYINT TO UNIQUE UNSIGNED UPDATE USER SESSION_USER SYSTEM_USER VARCHAR VARBINARY VARYING WITH ZONE DOUBLE IDB_FLOAT REAL CHARSET IDB_IF EXISTS CHANGE TRUNCATE -%token DQ_IDENT FQ_IDENT IDENT FCONST SCONST CP_SEARCH_CONDITION_TEXT ICONST DATE +%token DQ_IDENT IDENT FCONST SCONST CP_SEARCH_CONDITION_TEXT ICONST DATE /* Notes: * 1. "ata" stands for alter_table_action @@ -606,26 +605,13 @@ table_name: ; qualified_name: - FQ_IDENT { - char* delimeterPosition = strchr(const_cast($1), '.'); - if( delimeterPosition ) - { - *delimeterPosition = '\0'; - char* schemaName = const_cast($1); - char* tableName = delimeterPosition + 1; - $$ = new QualifiedName(schemaName, tableName); - *delimeterPosition = '.'; - } - else - $$ = new QualifiedName($1); - } | ident { if (x->fDBSchema.size()) $$ = new QualifiedName((char*)x->fDBSchema.c_str(), $1); else $$ = new QualifiedName($1); } - | IDENT '.' IDENT + | ident '.' ident { $$ = new QualifiedName($1, $3); } diff --git a/dbcon/mysql/ha_calpont_ddl.cpp b/dbcon/mysql/ha_calpont_ddl.cpp index 74b413667..9e757e426 100755 --- a/dbcon/mysql/ha_calpont_ddl.cpp +++ b/dbcon/mysql/ha_calpont_ddl.cpp @@ -2083,7 +2083,7 @@ int ha_calpont_impl_rename_table_(const char* from, const char* to, cal_connecti int rc = ProcessDDLStatement(stmt, db, "", tid2sid(thd->thread_id), emsg); if (rc != 0) - push_warning(thd, Sql_condition::WARN_LEVEL_ERROR, 9999, emsg.c_str()); + push_warning(thd, Sql_condition::WARN_LEVEL_WARN, 9999, emsg.c_str()); return rc; } @@ -2123,7 +2123,7 @@ long long calonlinealter(UDF_INIT* initid, UDF_ARGS* args, int rc = ProcessDDLStatement(stmt, db, "", tid2sid(thd->thread_id), emsg, compressiontype); if (rc != 0) - push_warning(thd, Sql_condition::WARN_LEVEL_ERROR, 9999, emsg.c_str()); + push_warning(thd, Sql_condition::WARN_LEVEL_WARN, 9999, emsg.c_str()); return rc; }