database/mariadb-columnstore-engine
1
0
Fork 0
mirror of https://github.com/mariadb-corporation/mariadb-columnstore-engine.git synced 2025-07-29 08:21:15 +03:00
Code Activity

MCOL-642 Separate TEXT from BLOB

Browse Source 
* TEXT and BLOB now have separate identifiers internally
* TEXT columns are identified as such in system catalog
* cpimport only requires hex input for BLOB, not TEXT
This commit is contained in:
Andrew Hutchings
2017-03-27 21:36:27 +01:00
parent 99b62cf97b
commit e9db44424c
43 changed files with 181 additions and 44 deletions
Download Patch File Download Diff File Expand all files Collapse all files

31
writeengine/server/we_ddlcommandproc.cpp
View File

@ -451,7 +451,8 @@ uint8_t WE_DDLCommandProc::writeCreateSyscolumn(ByteStream& bs, std::string &err
if ( (dataType == CalpontSystemCatalog::CHAR && colDefPtr->fType->fLength > 8) ||
(dataType == CalpontSystemCatalog::VARCHAR && colDefPtr->fType->fLength > 7) ||
(dataType == CalpontSystemCatalog::VARBINARY && colDefPtr->fType->fLength > 7) ||
(dataType == CalpontSystemCatalog::BLOB && colDefPtr->fType->fLength > 7) )
(dataType == CalpontSystemCatalog::BLOB && colDefPtr->fType->fLength > 7) ||
(dataType == CalpontSystemCatalog::TEXT && colDefPtr->fType->fLength > 7) )
{
hasDict = true;
dictOID.compressionType = colDefPtr->fType->fCompressiontype;
@ -461,7 +462,8 @@ uint8_t WE_DDLCommandProc::writeCreateSyscolumn(ByteStream& bs, std::string &err
//@Bug 2534. Take away the limit of 255 and set the limit to 8000.
if ((colDefPtr->fType->fLength > 8000) &&
(dataType != CalpontSystemCatalog::BLOB))
(dataType != CalpontSystemCatalog::BLOB) &&
(dataType != CalpontSystemCatalog::TEXT))
{
ostringstream os;
os << "char, varchar and varbinary length may not exceed 8000";
@ -469,7 +471,8 @@ uint8_t WE_DDLCommandProc::writeCreateSyscolumn(ByteStream& bs, std::string &err
}
}
else if ((dataType == CalpontSystemCatalog::VARBINARY
|| dataType == CalpontSystemCatalog::BLOB)
|| dataType == CalpontSystemCatalog::BLOB
|| dataType == CalpontSystemCatalog::TEXT)
&& colDefPtr->fType->fLength <= 7)
{
ostringstream os;
@ -519,7 +522,8 @@ uint8_t WE_DDLCommandProc::writeCreateSyscolumn(ByteStream& bs, std::string &err
if (dataType == CalpontSystemCatalog::CHAR ||
dataType == CalpontSystemCatalog::VARCHAR ||
dataType == CalpontSystemCatalog::VARBINARY ||
dataType == CalpontSystemCatalog::BLOB)
dataType == CalpontSystemCatalog::BLOB ||
dataType == CalpontSystemCatalog::TEXT)
{
if (colDefPtr->fType->fLength <= 0)
{
@ -835,7 +839,8 @@ uint8_t WE_DDLCommandProc::writeSyscolumn(ByteStream& bs, std::string & err)
//@Bug 2534. Take away the limit of 255 and set the limit to 8000.
if ((colDefPtr->fType->fLength > 8000) &&
(dataType != CalpontSystemCatalog::BLOB))
(dataType != CalpontSystemCatalog::BLOB) &&
(dataType != CalpontSystemCatalog::TEXT))
{
ostringstream os;
os << "char, varchar and varbinary length may not exceed 8000";
@ -843,7 +848,8 @@ uint8_t WE_DDLCommandProc::writeSyscolumn(ByteStream& bs, std::string & err)
}
}
else if ((dataType == CalpontSystemCatalog::VARBINARY
|| dataType == CalpontSystemCatalog::BLOB)
|| dataType == CalpontSystemCatalog::BLOB
|| dataType == CalpontSystemCatalog::TEXT)
&& colDefPtr->fType->fLength <= 7)
{
ostringstream os;
@ -894,7 +900,8 @@ uint8_t WE_DDLCommandProc::writeSyscolumn(ByteStream& bs, std::string & err)
if (dataType == CalpontSystemCatalog::CHAR ||
dataType == CalpontSystemCatalog::VARCHAR ||
dataType == CalpontSystemCatalog::VARBINARY ||
dataType == CalpontSystemCatalog::BLOB)
dataType == CalpontSystemCatalog::BLOB ||
dataType == CalpontSystemCatalog::TEXT)
{
if (colDefPtr->fType->fLength <= 0)
{
@ -2253,6 +2260,8 @@ uint8_t WE_DDLCommandProc::updateSyscolumnTablename(ByteStream& bs, std::string
&& column.colType.colWidth > 7)
|| (column.colType.colDataType == CalpontSystemCatalog::BLOB
&& column.colType.colWidth > 7)
|| (column.colType.colDataType == CalpontSystemCatalog::TEXT
&& column.colType.colWidth > 7)
|| (column.colType.colDataType == CalpontSystemCatalog::DECIMAL
&& column.colType.precision > 18)
|| (column.colType.colDataType == CalpontSystemCatalog::UDECIMAL
@ -3011,6 +3020,8 @@ uint8_t WE_DDLCommandProc::updateSystablesTablename(ByteStream& bs, std::string
&& column.colType.colWidth > 7)
|| (column.colType.colDataType == CalpontSystemCatalog::BLOB
&& column.colType.colWidth > 7)
|| (column.colType.colDataType == CalpontSystemCatalog::TEXT
&& column.colType.colWidth > 7)
|| (column.colType.colDataType == CalpontSystemCatalog::DECIMAL
&& column.colType.precision > 18)
|| (column.colType.colDataType == CalpontSystemCatalog::UDECIMAL
@ -3893,6 +3904,8 @@ uint8_t WE_DDLCommandProc::updateSyscolumnSetDefault(messageqcpp::ByteStream& bs
&& column.colType.colWidth > 7)
|| (column.colType.colDataType == CalpontSystemCatalog::BLOB
&& column.colType.colWidth > 7)
|| (column.colType.colDataType == CalpontSystemCatalog::TEXT
&& column.colType.colWidth > 7)
|| (column.colType.colDataType == CalpontSystemCatalog::DECIMAL
&& column.colType.precision > 18)
|| (column.colType.colDataType == CalpontSystemCatalog::UDECIMAL
@ -4163,6 +4176,8 @@ uint8_t WE_DDLCommandProc::updateSyscolumnRenameColumn(messageqcpp::ByteStream&
&& column1.colType.colWidth > 7)
|| (column1.colType.colDataType == CalpontSystemCatalog::BLOB
&& column1.colType.colWidth > 7)
|| (column1.colType.colDataType == CalpontSystemCatalog::TEXT
&& column1.colType.colWidth > 7)
|| (column1.colType.colDataType == CalpontSystemCatalog::DECIMAL
&& column1.colType.precision > 18)
|| (column1.colType.colDataType == CalpontSystemCatalog::UDECIMAL
@ -4355,6 +4370,8 @@ uint8_t WE_DDLCommandProc::updateSyscolumnRenameColumn(messageqcpp::ByteStream&
&& column5.colType.colWidth > 7)
|| (column5.colType.colDataType == CalpontSystemCatalog::BLOB
&& column5.colType.colWidth > 7)
|| (column5.colType.colDataType == CalpontSystemCatalog::TEXT
&& column5.colType.colWidth > 7)
|| (column5.colType.colDataType == CalpontSystemCatalog::DECIMAL
&& column5.colType.precision > 18)
|| (column5.colType.colDataType == CalpontSystemCatalog::UDECIMAL

5
writeengine/server/we_ddlcommon.h
View File

@ -317,6 +317,7 @@ inline boost::any getNullValueForType(const execplan::CalpontSystemCatalog::ColT
}
break;
case execplan::CalpontSystemCatalog::BLOB:
case execplan::CalpontSystemCatalog::TEXT:
case execplan::CalpontSystemCatalog::VARBINARY:
{
std::string charnull;
@ -425,6 +426,10 @@ inline int convertDataType(int dataType)
calpontDataType = CalpontSystemCatalog::BLOB;
break;
case ddlpackage::DDL_TEXT:
calpontDataType = CalpontSystemCatalog::TEXT;
break;
case ddlpackage::DDL_UNSIGNED_TINYINT:
calpontDataType = CalpontSystemCatalog::UTINYINT;
break;

2
writeengine/server/we_dmlcommandproc.cpp
View File

@ -2072,6 +2072,7 @@ uint8_t WE_DMLCommandProc::processUpdate(messageqcpp::ByteStream& bs,
}
case CalpontSystemCatalog::VARBINARY:
case CalpontSystemCatalog::BLOB:
case CalpontSystemCatalog::TEXT:
{
value = row.getVarBinaryStringField(fetchColPos);
break;
@ -2358,6 +2359,7 @@ uint8_t WE_DMLCommandProc::processUpdate(messageqcpp::ByteStream& bs,
}
case CalpontSystemCatalog::VARBINARY:
case CalpontSystemCatalog::BLOB:
case CalpontSystemCatalog::TEXT:
{
value = row.getVarBinaryStringField(fetchColPos);
break;

3
writeengine/server/we_dmlcommandproc.h
View File

@ -111,7 +111,8 @@ class WE_DMLCommandProc
|| ((colType.colDataType == execplan::CalpontSystemCatalog::DECIMAL) && (colType.precision > 18))
|| ((colType.colDataType == execplan::CalpontSystemCatalog::UDECIMAL) && (colType.precision > 18))
|| (colType.colDataType == execplan::CalpontSystemCatalog::VARBINARY)
|| (colType.colDataType == execplan::CalpontSystemCatalog::BLOB))
|| (colType.colDataType == execplan::CalpontSystemCatalog::BLOB)
|| (colType.colDataType == execplan::CalpontSystemCatalog::TEXT))
{
return true;
}

12
writeengine/shared/we_convertor.cpp
View File

@ -414,6 +414,10 @@ void Convertor::convertColType(CalpontSystemCatalog::ColDataType dataType,
case CalpontSystemCatalog::BLOB :
internalType = WriteEngine::WR_BLOB; break;
// Map TEXT to WR_TEXT
case CalpontSystemCatalog::TEXT :
internalType = WriteEngine::WR_TEXT; break;
// Map VARBINARY to WR_VARBINARY
case CalpontSystemCatalog::VARBINARY:
internalType = WriteEngine::WR_VARBINARY; break;
@ -497,6 +501,10 @@ void Convertor::convertWEColType(ColType internalType,
case WriteEngine::WR_BLOB :
dataType = CalpontSystemCatalog::BLOB; break;
// Map TEXT to WR_TEXT
case WriteEngine::WR_TEXT :
dataType = CalpontSystemCatalog::TEXT; break;
// Map VARBINARY to WR_VARBINARY
case WriteEngine::WR_VARBINARY:
dataType = CalpontSystemCatalog::VARBINARY; break;
@ -598,6 +606,10 @@ void Convertor::convertColType(ColStruct* curStruct)
case CalpontSystemCatalog::BLOB :
*internalType = WriteEngine::WR_BLOB; break;
// Map TEXT to WR_TEXT
case CalpontSystemCatalog::TEXT :
*internalType = WriteEngine::WR_TEXT; break;
// Map VARBINARY to WR_VARBINARY
case CalpontSystemCatalog::VARBINARY:
*internalType = WriteEngine::WR_VARBINARY; break;

1
writeengine/shared/we_type.h
View File

@ -104,6 +104,7 @@ namespace WriteEngine
WR_USHORT = 14, /** @brief Unsigned Short */
WR_UINT = 15, /** @brief Unsigned Int */
WR_ULONGLONG = 16, /** @brief Unsigned Long long*/
WR_TEXT = 17 /** @brief TEXT */
};
// Describes relation of field to column for a bulk load

3
writeengine/wrapper/we_colop.cpp
View File

@ -1455,6 +1455,7 @@ int ColumnOp::addExtent(
break;
case WriteEngine::WR_VARBINARY : // treat same as char for now
case WriteEngine::WR_BLOB :
case WriteEngine::WR_TEXT :
case WriteEngine::WR_CHAR :
if (!bDelete)
{
@ -1590,6 +1591,7 @@ int ColumnOp::addExtent(
break;
case WriteEngine::WR_VARBINARY : // treat same as char for now
case WriteEngine::WR_BLOB :
case WriteEngine::WR_TEXT :
case WriteEngine::WR_CHAR :
if (!bDelete)
{
@ -1720,6 +1722,7 @@ int ColumnOp::addExtent(
break;
case WriteEngine::WR_VARBINARY : // treat same as char for now
case WriteEngine::WR_BLOB :
case WriteEngine::WR_TEXT :
case WriteEngine::WR_CHAR :
{
memcpy(charTmpBuf, (char*)valArray+i*8, 8);

13
writeengine/wrapper/writeengine.cpp
View File

@ -244,6 +244,7 @@ void WriteEngineWrapper::convertValue(const ColType colType, void* value, boost:
case WriteEngine::WR_VARBINARY : // treat same as char for now
case WriteEngine::WR_CHAR :
case WriteEngine::WR_BLOB :
case WriteEngine::WR_TEXT :
curStr = boost::any_cast<string>(data);
if ((int) curStr.length() > MAX_COLUMN_BOUNDARY)
curStr = curStr.substr(0, MAX_COLUMN_BOUNDARY);
@ -363,7 +364,8 @@ void WriteEngineWrapper::convertValue(const ColType colType, void* valArray, con
break;
case WriteEngine::WR_VARBINARY : // treat same as char for now
case WriteEngine::WR_CHAR :
case WriteEngine::WR_BLOB : curStr = boost::any_cast<string>(data);
case WriteEngine::WR_BLOB :
case WriteEngine::WR_TEXT : curStr = boost::any_cast<string>(data);
if ((int) curStr.length() > MAX_COLUMN_BOUNDARY)
curStr = curStr.substr(0, MAX_COLUMN_BOUNDARY);
memcpy((char*)valArray + pos * MAX_COLUMN_BOUNDARY, curStr.c_str(), curStr.length());
@ -413,7 +415,8 @@ void WriteEngineWrapper::convertValue(const ColType colType, void* valArray, con
break;
case WriteEngine::WR_VARBINARY : // treat same as char for now
case WriteEngine::WR_CHAR :
case WriteEngine::WR_BLOB : char tmp[10];
case WriteEngine::WR_BLOB :
case WriteEngine::WR_TEXT : char tmp[10];
memcpy(tmp, (char*)valArray + pos*8, 8);
curStr = tmp;
data = curStr;
@ -687,6 +690,7 @@ int WriteEngineWrapper::deleteBadRows(const TxnID& txnid, ColStructList& colStru
case WriteEngine::WR_VARBINARY : // treat same as char for now
case WriteEngine::WR_CHAR:
case WriteEngine::WR_BLOB:
case WriteEngine::WR_TEXT:
valArray = (char*) calloc(sizeof(char), 1 * MAX_COLUMN_BOUNDARY);
break;
case WriteEngine::WR_FLOAT:
@ -3314,6 +3318,7 @@ int WriteEngineWrapper::writeColumnRecords(const TxnID& txnid,
case WriteEngine::WR_VARBINARY : // treat same as char for now
case WriteEngine::WR_CHAR:
case WriteEngine::WR_BLOB:
case WriteEngine::WR_TEXT:
valArray = (char*) calloc(sizeof(char), totalRow * MAX_COLUMN_BOUNDARY);
break;
case WriteEngine::WR_FLOAT:
@ -3519,6 +3524,7 @@ StopWatch timer;
case WriteEngine::WR_VARBINARY : // treat same as char for now
case WriteEngine::WR_CHAR:
case WriteEngine::WR_BLOB:
case WriteEngine::WR_TEXT:
valArray = (char*) calloc(sizeof(char), totalRow1 * MAX_COLUMN_BOUNDARY);
break;
case WriteEngine::WR_FLOAT:
@ -3664,6 +3670,7 @@ timer.stop("writeRow ");
case WriteEngine::WR_VARBINARY : // treat same as char for now
case WriteEngine::WR_CHAR:
case WriteEngine::WR_BLOB:
case WriteEngine::WR_TEXT:
valArray = (char*) calloc(sizeof(char), totalRow2 * MAX_COLUMN_BOUNDARY);
break;
case WriteEngine::WR_FLOAT:
@ -3806,6 +3813,7 @@ timer.stop("writeRow ");
case WriteEngine::WR_VARBINARY : // treat same as char for now
case WriteEngine::WR_CHAR:
case WriteEngine::WR_BLOB:
case WriteEngine::WR_TEXT:
valArray = (char*) calloc(sizeof(char), totalRow1 * MAX_COLUMN_BOUNDARY);
break;
case WriteEngine::WR_FLOAT:
@ -4068,6 +4076,7 @@ StopWatch timer;
case WriteEngine::WR_VARBINARY : // treat same as char for now
case WriteEngine::WR_CHAR:
case WriteEngine::WR_BLOB:
case WriteEngine::WR_TEXT:
valArray = (char*) calloc(sizeof(char), 1 * MAX_COLUMN_BOUNDARY);
break;
case WriteEngine::WR_FLOAT: