database/mariadb-columnstore-engine
1
0
Fork 0
mirror of https://github.com/mariadb-corporation/mariadb-columnstore-engine.git synced 2025-07-29 08:21:15 +03:00
Code Activity

Merge branch 'develop-1.1' into 1.1-merge-up-20180531

Browse Source
This commit is contained in:
Andrew Hutchings
2018-05-31 14:43:52 +01:00
parent 29346339ad 3ab634bde6
commit 53f281933a
14 changed files with 80 additions and 140 deletions
Download Patch File Download Diff File Expand all files Collapse all files

65
dbcon/ddlpackage/ddl.l
View File

@ -32,7 +32,7 @@
#endif
using namespace ddlpackage;
typedef enum { NOOP, STRIP_QUOTES, STRIP_QUOTES_FQ } copy_action_t;
typedef enum { NOOP, STRIP_QUOTES } copy_action_t;
int lineno = 1;
void ddlerror(struct pass_to_bison* x, char const *s);
@ -75,8 +75,6 @@ extended_identifier {ident_start}{extended_ident_cont}*
fq_identifier {identifier}\.{identifier}
identifier_quoted {grave_accent}{extended_identifier}{grave_accent}
identifier_double_quoted {double_quote}{extended_identifier}{double_quote}
fq_quoted ({identifier_quoted}|{extended_identifier})\.({identifier_quoted}|{identifier})
fq_double_quoted ({identifier_double_quoted}|{extended_identifier})\.({identifier_double_quoted}|{identifier})
integer [-+]?{digit}+
decimal ([-+]?({digit}*\.{digit}+)|({digit}+\.{digit}*))
@ -90,10 +88,7 @@ realfail2 ({integer}|{decimal})[Ee][-+]
{identifier_quoted} { ddlget_lval(yyscanner)->str = scanner_copy( ddlget_text(yyscanner), yyscanner, STRIP_QUOTES ); return IDENT; }
{identifier_double_quoted} { ddlget_lval(yyscanner)->str = scanner_copy( ddlget_text(yyscanner), yyscanner, STRIP_QUOTES ); return IDENT; }
{fq_identifier} { ddlget_lval(yyscanner)->str = scanner_copy(ddlget_text(yyscanner), yyscanner); return FQ_IDENT; }
{fq_quoted} { ddlget_lval(yyscanner)->str = scanner_copy(ddlget_text(yyscanner), yyscanner, STRIP_QUOTES_FQ); return FQ_IDENT; }
{fq_double_quoted} { ddlget_lval(yyscanner)->str = scanner_copy(ddlget_text(yyscanner), yyscanner, STRIP_QUOTES_FQ); return FQ_IDENT; }
{identifier_double_quoted} { ddlget_lval(yyscanner)->str = scanner_copy( ddlget_text(yyscanner), yyscanner, STRIP_QUOTES ); return DQ_IDENT; }
ACTION {return ACTION;}
ADD {return ADD;}
@ -193,14 +188,14 @@ LONGTEXT {return LONGTEXT;}
/* ignore */
}
{identifier} {ddlget_lval(yyscanner)->str = scanner_copy(ddlget_text(yyscanner), yyscanner); return IDENT;}
{identifier} { ddlget_lval(yyscanner)->str = scanner_copy(ddlget_text(yyscanner), yyscanner); return DQ_IDENT;}
{self} {
return ddlget_text(yyscanner)[0];
}
{grave_accent} {
/* ignore */
return ddlget_text(yyscanner)[0];
}
"/*" { BEGIN(c_comment); }
@ -282,7 +277,6 @@ char* scanner_copy (char *str, yyscan_t yyscanner, copy_action_t action)
char* result;
char* nv = strdup(str);
result = nv;
// free strduped memory later to prevent possible memory leak
if(nv)
((scan_data*)ddlget_extra(yyscanner))->valbuf.push_back(nv);
@ -292,57 +286,6 @@ char* scanner_copy (char *str, yyscan_t yyscanner, copy_action_t action)
nv[strlen(str) - 1] = '\0';
result = nv + 1;
}
else if (action == STRIP_QUOTES_FQ)
{
bool move_left = false;
bool move_right = false;
char* left = nv;
char* tmp_first = nv;
// MCOL-1384 Loop through all comas in this quoted fq id
// looking for $quote_sign.$quote_sign sequence.
char* fq_delimiter;
int tmp_pos = 0;
while((fq_delimiter = strchr(tmp_first, '.')) != NULL)
{
if( (*(fq_delimiter -1) == '`' && *(fq_delimiter + 1) == '`') ||
(*(fq_delimiter -1) == '"' && *(fq_delimiter + 1) == '"') )
{
tmp_pos += fq_delimiter - tmp_first;
break;
}
tmp_first = fq_delimiter;
}
char* fq_delimiter_orig = str + tmp_pos;
char* right = fq_delimiter + 1;
char* right_orig = fq_delimiter_orig + 1;
// MCOL-1384 Strip quotes from the left part.
if(*left == '"' || *left == '`')
{
result = left + 1;
*(fq_delimiter - 1) = '.';
move_left = true;
}
else
{
fq_delimiter += 1;
}
int right_length = strlen(right);
// MCOL-1384 Strip quotes from the right part.
if(*right == '`' || *right == '"')
{
right += 1; right_orig += 1;
right_length -= 2;
move_right = true;
*(fq_delimiter + right_length) = '\0';
}
if(move_left || move_right)
{
strncpy(fq_delimiter, right_orig, right_length);
}
}
return result;
}

52
dbcon/ddlpackage/ddl.y
View File

@ -29,26 +29,18 @@
Understanding the New Sql book
The postgress and mysql sources. find x -name \*.y -o -name \*.yy.
We don't support delimited identifiers.
We support quoted identifiers.
All literals are stored as unconverted strings.
You can't say "NOT DEFERRABLE". See the comment below.
This is not a reentrant parser. It uses the original global
variable style method of communication between the parser and
scanner. If we ever needed more than one parser thread per
processes, we would use the pure/reentrant options of bison and
flex. In that model, things that are traditionally global live
inside a struct that is passed around. We would need to upgrade to
a more recent version of flex. At the time of this writing, our
development systems have: flex version 2.5.4
This is a reentrant parser.
MCOL-66 Modify to be a reentrant parser
*/
%{
#include "string.h"
#include "sqlparser.h"
#ifdef _MSC_VER
@ -122,7 +114,7 @@ REFERENCES RENAME RESTRICT SET SMALLINT TABLE TEXT TIME TINYBLOB TINYTEXT
TINYINT TO UNIQUE UNSIGNED UPDATE USER SESSION_USER SYSTEM_USER VARCHAR VARBINARY
VARYING WITH ZONE DOUBLE IDB_FLOAT REAL CHARSET IDB_IF EXISTS CHANGE TRUNCATE
%token <str> FQ_IDENT IDENT FCONST SCONST CP_SEARCH_CONDITION_TEXT ICONST DATE
%token <str> DQ_IDENT IDENT FCONST SCONST CP_SEARCH_CONDITION_TEXT ICONST DATE
/* Notes:
* 1. "ata" stands for alter_table_action
@ -476,7 +468,7 @@ opt_equal:
;
table_option:
ENGINE opt_equal IDENT {$$ = new pair<string,string>("engine", $3);}
ENGINE opt_equal ident {$$ = new pair<string,string>("engine", $3);}
|
MAX_ROWS opt_equal ICONST {$$ = new pair<string,string>("max_rows", $3);}
|
@ -491,9 +483,9 @@ table_option:
$$ = new pair<string,string>("auto_increment", $3);
}
|
DEFAULT CHARSET opt_equal IDENT {$$ = new pair<string,string>("default charset", $4);}
DEFAULT CHARSET opt_equal ident {$$ = new pair<string,string>("default charset", $4);}
|
DEFAULT IDB_CHAR SET opt_equal IDENT {$$ = new pair<string,string>("default charset", $5);}
DEFAULT IDB_CHAR SET opt_equal ident {$$ = new pair<string,string>("default charset", $5);}
;
alter_table_statement:
@ -623,25 +615,21 @@ table_name:
;
qualified_name:
FQ_IDENT {
char* delimeterPosition = strchr(const_cast<char*>($1), '.');
if( delimeterPosition )
{
*delimeterPosition = '\0';
char* schemaName = const_cast<char*>($1);
char* tableName = delimeterPosition + 1;
$$ = new QualifiedName(schemaName, tableName);
*delimeterPosition = '.';
}
else
$$ = new QualifiedName($1);
}
| IDENT {
| ident {
if (x->fDBSchema.size())
$$ = new QualifiedName((char*)x->fDBSchema.c_str(), $1);
else
$$ = new QualifiedName($1);
}
| ident '.' ident
{
$$ = new QualifiedName($1, $3);
}
;
ident:
DQ_IDENT
| IDENT
;
ata_add_column:
@ -656,11 +644,11 @@ ata_add_column:
column_name:
DATE
|IDENT
|ident
;
constraint_name:
IDENT
ident
;
column_option:
@ -720,6 +708,10 @@ default_clause:
{
$$ = new ColumnDefaultValue($2);
}
| DEFAULT DQ_IDENT /* MCOL-1406 */
{
$$ = new ColumnDefaultValue($2);
}
| DEFAULT NULL_TOK {$$ = new ColumnDefaultValue(NULL);}
| DEFAULT USER {$$ = new ColumnDefaultValue("$USER");}
| DEFAULT CURRENT_USER {$$ = new ColumnDefaultValue("$CURRENT_USER");}

12
dbcon/joblist/jlf_execplantojoblist.cpp
View File

@ -1634,11 +1634,8 @@ const JobStepVector doSimpleFilter(SimpleFilter* sf, JobInfo& jobInfo)
return doExpressionFilter(sf, jobInfo);
}
// trim trailing space char in the predicate
string constval(cc->constval());
size_t spos = constval.find_last_not_of(" ");
if (spos != string::npos) constval = constval.substr(0, spos + 1);
CalpontSystemCatalog::OID dictOid = 0;
CalpontSystemCatalog::ColType ct = sc->colType();
@ -2772,11 +2769,8 @@ const JobStepVector doConstantFilter(const ConstantFilter* cf, JobInfo& jobInfo)
if (ConstantColumn::NULLDATA == cc->type() && (opeq == *sop || opne == *sop))
cop = COMPARE_NIL;
// trim trailing space char
string value = cc->constval();
size_t spos = value.find_last_not_of(" ");
if (spos != string::npos) value = value.substr(0, spos + 1);
pds->addFilter(cop, value);
}
@ -2858,11 +2852,8 @@ const JobStepVector doConstantFilter(const ConstantFilter* cf, JobInfo& jobInfo)
if (ConstantColumn::NULLDATA == cc->type() && (opeq == *sop || opne == *sop))
cop = COMPARE_NIL;
// trim trailing space char
string value = cc->constval();
size_t spos = value.find_last_not_of(" ");
if (spos != string::npos) value = value.substr(0, spos + 1);
pds->addFilter(cop, value);
}
@ -2968,10 +2959,7 @@ const JobStepVector doConstantFilter(const ConstantFilter* cf, JobInfo& jobInfo)
int8_t cop = op2num(sop);
int64_t value = 0;
string constval = cc->constval();
// trim trailing space char
size_t spos = constval.find_last_not_of(" ");
if (spos != string::npos) constval = constval.substr(0, spos + 1);
// @bug 1151 string longer than colwidth of char/varchar.
uint8_t rf = 0;

4
dbcon/mysql/ha_calpont_ddl.cpp
View File

@ -2224,7 +2224,7 @@ int ha_calpont_impl_rename_table_(const char* from, const char* to, cal_connecti
int rc = ProcessDDLStatement(stmt, db, "", tid2sid(thd->thread_id), emsg);
if (rc != 0)
push_warning(thd, Sql_condition::WARN_LEVEL_ERROR, 9999, emsg.c_str());
push_warning(thd, Sql_condition::WARN_LEVEL_WARN, 9999, emsg.c_str());
return rc;
}
@ -2266,7 +2266,7 @@ extern "C"
int rc = ProcessDDLStatement(stmt, db, "", tid2sid(thd->thread_id), emsg, compressiontype);
if (rc != 0)
push_warning(thd, Sql_condition::WARN_LEVEL_ERROR, 9999, emsg.c_str());
push_warning(thd, Sql_condition::WARN_LEVEL_WARN, 9999, emsg.c_str());
return rc;
}

4
dbcon/mysql/ha_calpont_execplan.cpp
View File

@ -4674,7 +4674,6 @@ void gp_walk(const Item* item, void* arg)
if (isp)
{
// @bug 3669. trim trailing spaces for the compare value
if (isp->result_type() == STRING_RESULT)
{
String val, *str = isp->val_str(&val);
@ -4685,10 +4684,7 @@ void gp_walk(const Item* item, void* arg)
cval.assign(str->ptr(), str->length());
}
size_t spos = cval.find_last_not_of(" ");
if (spos != string::npos)
cval = cval.substr(0, spos + 1);
gwip->rcWorkStack.push(new ConstantColumn(cval));
break;

3
utils/funcexp/func_substring_index.cpp
View File

@ -74,6 +74,9 @@ std::string Func_substring_index::getStrVal(rowgroup::Row& row,
if ( count > (int64_t) end )
return str;
if (( count < 0 ) && ((count * -1) > end))
return str;
string value = str;
if ( count > 0 )

2
writeengine/splitter/we_cmdargs.h
View File

@ -77,7 +77,7 @@ public:
{
return fLocFile;
}
int getMode() const
int getReadBufSize() { return fReadBufSize; }
{
return fMode;
}

18
writeengine/splitter/we_filereadthread.cpp
View File

@ -87,6 +87,15 @@ WEFileReadThread::WEFileReadThread(WESDHandler& aSdh): fSdh(aSdh),
{
//TODO batch qty to get from config
fBatchQty = 10000;
if (fSdh.getReadBufSize() < DEFAULTBUFFSIZE)
{
fBuffSize = DEFAULTBUFFSIZE;
}
else
{
fBuffSize = fSdh.getReadBufSize();
}
fBuff = new char [fBuffSize];
}
@ -109,6 +118,7 @@ WEFileReadThread::~WEFileReadThread()
}
fpThread = 0;
delete []fBuff;
//cout << "WEFileReadThread destructor called" << endl;
}
@ -352,17 +362,17 @@ unsigned int WEFileReadThread::readDataFile(messageqcpp::SBS& Sbs)
if (fEnclEsc)
{
//pStart = aBuff;
aLen = getNextRow(fInFile, fBuff, sizeof(fBuff) - 1);
aLen = getNextRow(fInFile, fBuff, fBuffSize-1);
}
else
{
fInFile.getline(fBuff, sizeof(fBuff) - 1);
fInFile.getline(fBuff, fBuffSize-1);
aLen = fInFile.gcount();
}
////aLen chars incl \n, Therefore aLen-1; '<<' oper won't go past it
//cout << "Data Length " << aLen <<endl;
if ((aLen < (sizeof(fBuff) - 2)) && (aLen > 0))
if((aLen < (fBuffSize-2)) && (aLen>0))
{
fBuff[aLen - 1] = '\n';
fBuff[aLen] = 0;
@ -374,7 +384,7 @@ unsigned int WEFileReadThread::readDataFile(messageqcpp::SBS& Sbs)
if (fSdh.getDebugLvl() > 2) cout << "File data line = " << aIdx << endl;
}
else if (aLen >= sizeof(fBuff) - 2) //Didn't hit delim; BIG ROW
else if(aLen>=fBuffSize-2) //Didn't hit delim; BIG ROW
{
cout << "Bad Row data " << endl;
cout << fBuff << endl;

5
writeengine/splitter/we_filereadthread.h
View File

@ -126,7 +126,7 @@ public:
void add2InputDataFileList(std::string& FileName);
private:
enum { MAXBUFFSIZE = 1024 * 1024 };
enum { DEFAULTBUFFSIZE=1024*1024 };
// don't allow anyone else to set
void setTgtPmId(unsigned int fTgtPmId)
@ -151,7 +151,8 @@ private:
char fEncl; // Encl char
char fEsc; // Esc char
char fDelim; // Column Delimit char
char fBuff[MAXBUFFSIZE]; // main data buffer
char* fBuff; // main data buffer
int fBuffSize;
};
} /* namespace WriteEngine */

7
writeengine/splitter/we_sdhandler.cpp
View File

@ -2663,6 +2663,13 @@ char WESDHandler::getEscChar()
//------------------------------------------------------------------------------
int WESDHandler::getReadBufSize()
{
return fRef.fCmdArgs.getReadBufSize();
}
//------------------------------------------------------------------------------
char WESDHandler::getDelimChar()
{
return fRef.fCmdArgs.getDelimChar();

1
writeengine/splitter/we_sdhandler.h
View File

@ -152,6 +152,7 @@ public:
char getEscChar();
char getDelimChar();
bool getConsoleLog();
int getReadBufSize();
ImportDataMode getImportDataMode() const;
void sysLog(const logging::Message::Args& msgArgs,
logging::LOG_TYPE logType, logging::Message::MessageID msgId);

9
writeengine/wrapper/we_colop.cpp
View File

@ -222,7 +222,7 @@ int ColumnOp::allocRowId(const TxnID& txnid, bool useStartingExtent,
//Find out where the rest rows go
BRM::LBID_t startLbid;
//need to put in a loop until newExtent is true
newExtent = dbRootExtentTrackers[0]->nextSegFile(dbRoot, partition, segment, newHwm, startLbid);
newExtent = dbRootExtentTrackers[column.colNo]->nextSegFile(dbRoot, partition, segment, newHwm, startLbid);
TableMetaData* tableMetaData = TableMetaData::makeTableMetaData(tableOid);
while (!newExtent)
@ -238,7 +238,7 @@ int ColumnOp::allocRowId(const TxnID& txnid, bool useStartingExtent,
for (i = 0; i < dbRootExtentTrackers.size(); i++)
{
if (i != 0)
if (i != column.colNo)
dbRootExtentTrackers[i]->nextSegFile(dbRoot, partition, segment, newHwm, startLbid);
// Round up HWM to the end of the current extent
@ -302,8 +302,7 @@ int ColumnOp::allocRowId(const TxnID& txnid, bool useStartingExtent,
tableMetaData->setColExtsInfo(newColStructList[i].dataOid, aColExtsInfo);
}
newExtent = dbRootExtentTrackers[0]->nextSegFile(dbRoot, partition, segment, newHwm, startLbid);
newExtent = dbRootExtentTrackers[column.colNo]->nextSegFile(dbRoot, partition, segment, newHwm, startLbid);
}
}
@ -324,7 +323,7 @@ int ColumnOp::allocRowId(const TxnID& txnid, bool useStartingExtent,
}
rc = BRMWrapper::getInstance()->allocateStripeColExtents(cols, dbRoot, partition, segment, extents);
newHwm = extents[0].startBlkOffset;
newHwm = extents[column.colNo].startBlkOffset;
if (rc != NO_ERROR)
return rc;

34
writeengine/wrapper/writeengine.cpp
View File

@ -1683,6 +1683,19 @@ int WriteEngineWrapper::insertColumnRecsBinary(const TxnID& txnid,
for (i = 0; i < colStructList.size(); i++)
Convertor::convertColType(&colStructList[i]);
// MCOL-984: find the smallest column width to calculate the RowID from so
// that all HWMs will be incremented by this operation
int32_t lowColLen = 8192;
int32_t colId = 0;
for (uint32_t colIt = 0; colIt < colStructList.size(); colIt++)
{
if (colStructList[colIt].colWidth < lowColLen)
{
colId = colIt;
lowColLen = colStructList[colId].colWidth;
}
}
// rc = checkValid(txnid, colStructList, colValueList, ridList);
// if (rc != NO_ERROR)
// return rc;
@ -1709,8 +1722,8 @@ int WriteEngineWrapper::insertColumnRecsBinary(const TxnID& txnid,
//--------------------------------------------------------------------------
if (isFirstBatchPm)
{
currentDBrootIdx = dbRootExtentTrackers[0]->getCurrentDBRootIdx();
extentInfo = dbRootExtentTrackers[0]->getDBRootExtentList();
currentDBrootIdx = dbRootExtentTrackers[colId]->getCurrentDBRootIdx();
extentInfo = dbRootExtentTrackers[colId]->getDBRootExtentList();
dbRoot = extentInfo[currentDBrootIdx].fDbRoot;
partitionNum = extentInfo[currentDBrootIdx].fPartition;
@ -1895,7 +1908,7 @@ int WriteEngineWrapper::insertColumnRecsBinary(const TxnID& txnid,
} // if (isFirstBatchPm)
else //get the extent info from tableMetaData
{
ColExtsInfo aColExtsInfo = tableMetaData->getColExtsInfo(colStructList[0].dataOid);
ColExtsInfo aColExtsInfo = tableMetaData->getColExtsInfo(colStructList[colId].dataOid);
ColExtsInfo::iterator it = aColExtsInfo.begin();
while (it != aColExtsInfo.end())
@ -1931,20 +1944,7 @@ int WriteEngineWrapper::insertColumnRecsBinary(const TxnID& txnid,
// allocate row id(s)
//--------------------------------------------------------------------------
// MCOL-984: find the smallest column width to calculate the RowID from so
// that all HWMs will be incremented by this operation
int32_t lowColLen = 8192;
int32_t colId = 0;
for (uint32_t colIt = 0; colIt < colStructList.size(); colIt++)
{
if (colStructList[colIt].colWidth < lowColLen)
{
colId = colIt;
lowColLen = colStructList[colId].colWidth;
curColStruct = colStructList[colId];
}
}
colOp = m_colOp[op(curColStruct.fCompressionType)];
@ -1970,7 +1970,7 @@ int WriteEngineWrapper::insertColumnRecsBinary(const TxnID& txnid,
if (it != aColExtsInfo.end())
{
hwm = it->hwm;
//cout << "Got from colextinfo hwm for oid " << colStructList[0].dataOid << " is " << hwm << " and seg is " << colStructList[0].fColSegment << endl;
//cout << "Got from colextinfo hwm for oid " << colStructList[colId].dataOid << " is " << hwm << " and seg is " << colStructList[colId].fColSegment << endl;
}
oldHwm = hwm; //Save this info for rollback