database/mariadb-columnstore-engine
1
0
Fork 0
mirror of https://github.com/mariadb-corporation/mariadb-columnstore-engine.git synced 2025-07-27 21:01:50 +03:00
Code Activity

MCOL-267 Make BLOB DDL/DML work

Browse Source 
Currently limited to 8KB inserts.
This commit is contained in:
Andrew Hutchings
2017-03-17 12:10:02 +00:00
parent cf58b8de5c
commit c08d03fba4
11 changed files with 51 additions and 16 deletions
Download Patch File Download Diff File Expand all files Collapse all files

3
dbcon/ddlpackageproc/altertableprocessor.cpp
View File

@ -561,7 +561,8 @@ void AlterTableProcessor::addColumn (uint32_t sessionID, execplan::CalpontSystem
if ((columnDefPtr->fType->fType == CalpontSystemCatalog::CHAR && columnDefPtr->fType->fLength > 8) || if ((columnDefPtr->fType->fType == CalpontSystemCatalog::CHAR && columnDefPtr->fType->fLength > 8) ||
(columnDefPtr->fType->fType == CalpontSystemCatalog::VARCHAR && columnDefPtr->fType->fLength > 7) || (columnDefPtr->fType->fType == CalpontSystemCatalog::VARCHAR && columnDefPtr->fType->fLength > 7) ||
(columnDefPtr->fType->fType == CalpontSystemCatalog::VARBINARY && columnDefPtr->fType->fLength > 7)) (columnDefPtr->fType->fType == CalpontSystemCatalog::VARBINARY && columnDefPtr->fType->fLength > 7) ||
(columnDefPtr->fType->fType == CalpontSystemCatalog::BLOB))
{ {
isDict = true; isDict = true;
} }

6
dbcon/ddlpackageproc/createtableprocessor.cpp
View File

@ -244,7 +244,8 @@ keepGoing:
dataType = convertDataType(tableDef.fColumns[i]->fType->fType); dataType = convertDataType(tableDef.fColumns[i]->fType->fType);
if ( (dataType == CalpontSystemCatalog::CHAR && tableDef.fColumns[i]->fType->fLength > 8) || if ( (dataType == CalpontSystemCatalog::CHAR && tableDef.fColumns[i]->fType->fLength > 8) ||
(dataType == CalpontSystemCatalog::VARCHAR && tableDef.fColumns[i]->fType->fLength > 7) || (dataType == CalpontSystemCatalog::VARCHAR && tableDef.fColumns[i]->fType->fLength > 7) ||
(dataType == CalpontSystemCatalog::VARBINARY && tableDef.fColumns[i]->fType->fLength > 7) ) (dataType == CalpontSystemCatalog::VARBINARY && tableDef.fColumns[i]->fType->fLength > 7) ||
(dataType == CalpontSystemCatalog::BLOB && tableDef.fColumns[i]->fType->fLength > 7) )
numDictCols++; numDictCols++;
} }
fStartingColOID = fObjectIDManager.allocOIDs(numColumns+numDictCols+1); //include column, oids,dictionary oids and tableoid fStartingColOID = fObjectIDManager.allocOIDs(numColumns+numDictCols+1); //include column, oids,dictionary oids and tableoid
@ -533,7 +534,8 @@ cout << fTxnid.id << " Create table WE_SVR_WRITE_CREATE_SYSCOLUMN: " << errorMsg
bytestream << (uint32_t) colDefPtr->fType->fCompressiontype; bytestream << (uint32_t) colDefPtr->fType->fCompressiontype;
if ( (dataType == CalpontSystemCatalog::CHAR && colDefPtr->fType->fLength > 8) || if ( (dataType == CalpontSystemCatalog::CHAR && colDefPtr->fType->fLength > 8) ||
(dataType == CalpontSystemCatalog::VARCHAR && colDefPtr->fType->fLength > 7) || (dataType == CalpontSystemCatalog::VARCHAR && colDefPtr->fType->fLength > 7) ||
(dataType == CalpontSystemCatalog::VARBINARY && colDefPtr->fType->fLength > 7) ) (dataType == CalpontSystemCatalog::VARBINARY && colDefPtr->fType->fLength > 7) ||
(dataType == CalpontSystemCatalog::BLOB && colDefPtr->fType->fLength > 7) )
{ {
bytestream << (uint32_t) (fStartingColOID+numColumns+(dictNum++)+1); bytestream << (uint32_t) (fStartingColOID+numColumns+(dictNum++)+1);
bytestream << (uint8_t) dataType; bytestream << (uint8_t) dataType;

3
dbcon/dmlpackageproc/dmlpackageprocessor.h
View File

@ -443,7 +443,8 @@ protected:
if (((colType.colDataType == execplan::CalpontSystemCatalog::CHAR) && (colType.colWidth > 8)) if (((colType.colDataType == execplan::CalpontSystemCatalog::CHAR) && (colType.colWidth > 8))
|| ((colType.colDataType == execplan::CalpontSystemCatalog::VARCHAR) && (colType.colWidth > 7)) || ((colType.colDataType == execplan::CalpontSystemCatalog::VARCHAR) && (colType.colWidth > 7))
|| ((colType.colDataType == execplan::CalpontSystemCatalog::DECIMAL) && (colType.precision > 18)) || ((colType.colDataType == execplan::CalpontSystemCatalog::DECIMAL) && (colType.precision > 18))
|| (colType.colDataType == execplan::CalpontSystemCatalog::VARBINARY)) || (colType.colDataType == execplan::CalpontSystemCatalog::VARBINARY)
|| (colType.colDataType == execplan::CalpontSystemCatalog::BLOB))
{ {
return true; return true;
} }

3
dbcon/joblist/pcolscan.cpp
View File

@ -171,7 +171,8 @@ pColScanStep::pColScanStep(
} }
//If this is a dictionary column, fudge the numbers... //If this is a dictionary column, fudge the numbers...
if (fColType.colDataType == CalpontSystemCatalog::VARBINARY) if ((fColType.colDataType == CalpontSystemCatalog::VARBINARY)
|| (fColType.colDataType == CalpontSystemCatalog::BLOB))
{ {
fColType.colWidth = 8; fColType.colWidth = 8;
fIsDict = true; fIsDict = true;

3
dbcon/joblist/pcolstep.cpp
View File

@ -169,7 +169,8 @@ pColStep::pColStep(
} }
//If this is a dictionary column, fudge the numbers... //If this is a dictionary column, fudge the numbers...
if (fColType.colDataType == CalpontSystemCatalog::VARBINARY) if ((fColType.colDataType == CalpontSystemCatalog::VARBINARY)
|| (fColType.colDataType == CalpontSystemCatalog::BLOB))
{ {
fColType.colWidth = 8; fColType.colWidth = 8;
fIsDict = true; fIsDict = true;

37
writeengine/server/we_ddlcommandproc.cpp
View File

@ -450,7 +450,8 @@ uint8_t WE_DDLCommandProc::writeCreateSyscolumn(ByteStream& bs, std::string &err
bool hasDict = false; bool hasDict = false;
if ( (dataType == CalpontSystemCatalog::CHAR && colDefPtr->fType->fLength > 8) || if ( (dataType == CalpontSystemCatalog::CHAR && colDefPtr->fType->fLength > 8) ||
(dataType == CalpontSystemCatalog::VARCHAR && colDefPtr->fType->fLength > 7) || (dataType == CalpontSystemCatalog::VARCHAR && colDefPtr->fType->fLength > 7) ||
(dataType == CalpontSystemCatalog::VARBINARY && colDefPtr->fType->fLength > 7) ) (dataType == CalpontSystemCatalog::VARBINARY && colDefPtr->fType->fLength > 7) ||
(dataType == CalpontSystemCatalog::BLOB && colDefPtr->fType->fLength > 7) )
{ {
hasDict = true; hasDict = true;
dictOID.compressionType = colDefPtr->fType->fCompressiontype; dictOID.compressionType = colDefPtr->fType->fCompressiontype;
@ -459,17 +460,20 @@ uint8_t WE_DDLCommandProc::writeCreateSyscolumn(ByteStream& bs, std::string &err
dictcol++; dictcol++;
//@Bug 2534. Take away the limit of 255 and set the limit to 8000. //@Bug 2534. Take away the limit of 255 and set the limit to 8000.
if (colDefPtr->fType->fLength > 8000) if ((colDefPtr->fType->fLength > 8000) &&
(dataType != CalpontSystemCatalog::BLOB))
{ {
ostringstream os; ostringstream os;
os << "char, varchar and varbinary length may not exceed 8000"; os << "char, varchar and varbinary length may not exceed 8000";
throw std::runtime_error(os.str()); throw std::runtime_error(os.str());
} }
} }
else if (dataType == CalpontSystemCatalog::VARBINARY && colDefPtr->fType->fLength <= 7) else if ((dataType == CalpontSystemCatalog::VARBINARY
|| dataType == CalpontSystemCatalog::BLOB)
&& colDefPtr->fType->fLength <= 7)
{ {
ostringstream os; ostringstream os;
os << "varbinary length may not be less than 8"; os << "varbinary and blob length may not be less than 8";
throw std::runtime_error(os.str()); throw std::runtime_error(os.str());
} }
@ -514,7 +518,8 @@ uint8_t WE_DDLCommandProc::writeCreateSyscolumn(ByteStream& bs, std::string &err
//@Bug 2089 Disallow zero length char and varch column to be created //@Bug 2089 Disallow zero length char and varch column to be created
if (dataType == CalpontSystemCatalog::CHAR || if (dataType == CalpontSystemCatalog::CHAR ||
dataType == CalpontSystemCatalog::VARCHAR || dataType == CalpontSystemCatalog::VARCHAR ||
dataType == CalpontSystemCatalog::VARBINARY) dataType == CalpontSystemCatalog::VARBINARY ||
dataType == CalpontSystemCatalog::BLOB)
{ {
if (colDefPtr->fType->fLength <= 0) if (colDefPtr->fType->fLength <= 0)
{ {
@ -829,17 +834,20 @@ uint8_t WE_DDLCommandProc::writeSyscolumn(ByteStream& bs, std::string & err)
dictOID.dictOID = dictoid; dictOID.dictOID = dictoid;
//@Bug 2534. Take away the limit of 255 and set the limit to 8000. //@Bug 2534. Take away the limit of 255 and set the limit to 8000.
if (colDefPtr->fType->fLength > 8000) if ((colDefPtr->fType->fLength > 8000) &&
(dataType != CalpontSystemCatalog::BLOB))
{ {
ostringstream os; ostringstream os;
os << "char, varchar and varbinary length may not exceed 8000"; os << "char, varchar and varbinary length may not exceed 8000";
throw std::runtime_error(os.str()); throw std::runtime_error(os.str());
} }
} }
else if (dataType == CalpontSystemCatalog::VARBINARY && colDefPtr->fType->fLength <= 7) else if ((dataType == CalpontSystemCatalog::VARBINARY
|| dataType == CalpontSystemCatalog::BLOB)
&& colDefPtr->fType->fLength <= 7)
{ {
ostringstream os; ostringstream os;
os << "varbinary length may not be less than 8"; os << "varbinary and blob length may not be less than 8";
throw std::runtime_error(os.str()); throw std::runtime_error(os.str());
} }
@ -885,7 +893,8 @@ uint8_t WE_DDLCommandProc::writeSyscolumn(ByteStream& bs, std::string & err)
//@Bug 2089 Disallow zero length char and varch column to be created //@Bug 2089 Disallow zero length char and varch column to be created
if (dataType == CalpontSystemCatalog::CHAR || if (dataType == CalpontSystemCatalog::CHAR ||
dataType == CalpontSystemCatalog::VARCHAR || dataType == CalpontSystemCatalog::VARCHAR ||
dataType == CalpontSystemCatalog::VARBINARY) dataType == CalpontSystemCatalog::VARBINARY ||
dataType == CalpontSystemCatalog::BLOB)
{ {
if (colDefPtr->fType->fLength <= 0) if (colDefPtr->fType->fLength <= 0)
{ {
@ -2242,6 +2251,8 @@ uint8_t WE_DDLCommandProc::updateSyscolumnTablename(ByteStream& bs, std::string
&& column.colType.colWidth > 7) && column.colType.colWidth > 7)
|| (column.colType.colDataType == CalpontSystemCatalog::VARBINARY || (column.colType.colDataType == CalpontSystemCatalog::VARBINARY
&& column.colType.colWidth > 7) && column.colType.colWidth > 7)
|| (column.colType.colDataType == CalpontSystemCatalog::BLOB
&& column.colType.colWidth > 7)
|| (column.colType.colDataType == CalpontSystemCatalog::DECIMAL || (column.colType.colDataType == CalpontSystemCatalog::DECIMAL
&& column.colType.precision > 18) && column.colType.precision > 18)
|| (column.colType.colDataType == CalpontSystemCatalog::UDECIMAL || (column.colType.colDataType == CalpontSystemCatalog::UDECIMAL
@ -2998,6 +3009,8 @@ uint8_t WE_DDLCommandProc::updateSystablesTablename(ByteStream& bs, std::string
&& column.colType.colWidth > 7) && column.colType.colWidth > 7)
|| (column.colType.colDataType == CalpontSystemCatalog::VARBINARY || (column.colType.colDataType == CalpontSystemCatalog::VARBINARY
&& column.colType.colWidth > 7) && column.colType.colWidth > 7)
|| (column.colType.colDataType == CalpontSystemCatalog::BLOB
&& column.colType.colWidth > 7)
|| (column.colType.colDataType == CalpontSystemCatalog::DECIMAL || (column.colType.colDataType == CalpontSystemCatalog::DECIMAL
&& column.colType.precision > 18) && column.colType.precision > 18)
|| (column.colType.colDataType == CalpontSystemCatalog::UDECIMAL || (column.colType.colDataType == CalpontSystemCatalog::UDECIMAL
@ -3878,6 +3891,8 @@ uint8_t WE_DDLCommandProc::updateSyscolumnSetDefault(messageqcpp::ByteStream& bs
&& column.colType.colWidth > 7) && column.colType.colWidth > 7)
|| (column.colType.colDataType == CalpontSystemCatalog::VARBINARY || (column.colType.colDataType == CalpontSystemCatalog::VARBINARY
&& column.colType.colWidth > 7) && column.colType.colWidth > 7)
|| (column.colType.colDataType == CalpontSystemCatalog::BLOB
&& column.colType.colWidth > 7)
|| (column.colType.colDataType == CalpontSystemCatalog::DECIMAL || (column.colType.colDataType == CalpontSystemCatalog::DECIMAL
&& column.colType.precision > 18) && column.colType.precision > 18)
|| (column.colType.colDataType == CalpontSystemCatalog::UDECIMAL || (column.colType.colDataType == CalpontSystemCatalog::UDECIMAL
@ -4146,6 +4161,8 @@ uint8_t WE_DDLCommandProc::updateSyscolumnRenameColumn(messageqcpp::ByteStream&
&& column1.colType.colWidth > 7) && column1.colType.colWidth > 7)
|| (column1.colType.colDataType == CalpontSystemCatalog::VARBINARY || (column1.colType.colDataType == CalpontSystemCatalog::VARBINARY
&& column1.colType.colWidth > 7) && column1.colType.colWidth > 7)
|| (column1.colType.colDataType == CalpontSystemCatalog::BLOB
&& column1.colType.colWidth > 7)
|| (column1.colType.colDataType == CalpontSystemCatalog::DECIMAL || (column1.colType.colDataType == CalpontSystemCatalog::DECIMAL
&& column1.colType.precision > 18) && column1.colType.precision > 18)
|| (column1.colType.colDataType == CalpontSystemCatalog::UDECIMAL || (column1.colType.colDataType == CalpontSystemCatalog::UDECIMAL
@ -4336,6 +4353,8 @@ uint8_t WE_DDLCommandProc::updateSyscolumnRenameColumn(messageqcpp::ByteStream&
&& column5.colType.colWidth > 7) && column5.colType.colWidth > 7)
|| (column5.colType.colDataType == CalpontSystemCatalog::VARBINARY || (column5.colType.colDataType == CalpontSystemCatalog::VARBINARY
&& column5.colType.colWidth > 7) && column5.colType.colWidth > 7)
|| (column5.colType.colDataType == CalpontSystemCatalog::BLOB
&& column5.colType.colWidth > 7)
|| (column5.colType.colDataType == CalpontSystemCatalog::DECIMAL || (column5.colType.colDataType == CalpontSystemCatalog::DECIMAL
&& column5.colType.precision > 18) && column5.colType.precision > 18)
|| (column5.colType.colDataType == CalpontSystemCatalog::UDECIMAL || (column5.colType.colDataType == CalpontSystemCatalog::UDECIMAL

1
writeengine/server/we_ddlcommon.h
View File

@ -316,6 +316,7 @@ inline boost::any getNullValueForType(const execplan::CalpontSystemCatalog::ColT
} }
break; break;
case execplan::CalpontSystemCatalog::BLOB:
case execplan::CalpontSystemCatalog::VARBINARY: case execplan::CalpontSystemCatalog::VARBINARY:
{ {
std::string charnull; std::string charnull;

2
writeengine/server/we_dmlcommandproc.cpp
View File

@ -2071,6 +2071,7 @@ uint8_t WE_DMLCommandProc::processUpdate(messageqcpp::ByteStream& bs,
break; break;
} }
case CalpontSystemCatalog::VARBINARY: case CalpontSystemCatalog::VARBINARY:
case CalpontSystemCatalog::BLOB:
{ {
value = row.getVarBinaryStringField(fetchColPos); value = row.getVarBinaryStringField(fetchColPos);
break; break;
@ -2356,6 +2357,7 @@ uint8_t WE_DMLCommandProc::processUpdate(messageqcpp::ByteStream& bs,
break; break;
} }
case CalpontSystemCatalog::VARBINARY: case CalpontSystemCatalog::VARBINARY:
case CalpontSystemCatalog::BLOB:
{ {
value = row.getVarBinaryStringField(fetchColPos); value = row.getVarBinaryStringField(fetchColPos);
break; break;

3
writeengine/server/we_dmlcommandproc.h
View File

@ -110,7 +110,8 @@ class WE_DMLCommandProc
|| ((colType.colDataType == execplan::CalpontSystemCatalog::VARCHAR) && (colType.colWidth > 7)) || ((colType.colDataType == execplan::CalpontSystemCatalog::VARCHAR) && (colType.colWidth > 7))
|| ((colType.colDataType == execplan::CalpontSystemCatalog::DECIMAL) && (colType.precision > 18)) || ((colType.colDataType == execplan::CalpontSystemCatalog::DECIMAL) && (colType.precision > 18))
|| ((colType.colDataType == execplan::CalpontSystemCatalog::UDECIMAL) && (colType.precision > 18)) || ((colType.colDataType == execplan::CalpontSystemCatalog::UDECIMAL) && (colType.precision > 18))
|| (colType.colDataType == execplan::CalpontSystemCatalog::VARBINARY)) || (colType.colDataType == execplan::CalpontSystemCatalog::VARBINARY)
|| (colType.colDataType == execplan::CalpontSystemCatalog::BLOB))
{ {
return true; return true;
} }

3
writeengine/shared/we_chunkmanager.cpp
View File

@ -1587,6 +1587,9 @@ int ChunkManager::calculateHeaderSize(int width)
int rowsPerExtent = BRMWrapper::getInstance()->getExtentRows(); int rowsPerExtent = BRMWrapper::getInstance()->getExtentRows();
int rowsPerFile = rowsPerExtent * extentsPerFile; int rowsPerFile = rowsPerExtent * extentsPerFile;
int stringsPerBlock = 8180 / (width + 2); // 8180 = 8192 - 12 int stringsPerBlock = 8180 / (width + 2); // 8180 = 8192 - 12
//TODO: temporary fix for Blob
if (stringsPerBlock == 0)
stringsPerBlock = 1;
int blocksNeeded = rowsPerFile / stringsPerBlock; int blocksNeeded = rowsPerFile / stringsPerBlock;
int blocksPerChunk = UNCOMPRESSED_CHUNK_SIZE / BYTE_PER_BLOCK; int blocksPerChunk = UNCOMPRESSED_CHUNK_SIZE / BYTE_PER_BLOCK;
lldiv_t chunks = lldiv(blocksNeeded, blocksPerChunk); lldiv_t chunks = lldiv(blocksNeeded, blocksPerChunk);

3
writeengine/wrapper/we_colop.cpp
View File

@ -1454,6 +1454,7 @@ int ColumnOp::addExtent(
//pOldVal = &((double *) oldValArray)[i]; //pOldVal = &((double *) oldValArray)[i];
break; break;
case WriteEngine::WR_VARBINARY : // treat same as char for now case WriteEngine::WR_VARBINARY : // treat same as char for now
case WriteEngine::WR_BLOB :
case WriteEngine::WR_CHAR : case WriteEngine::WR_CHAR :
if (!bDelete) if (!bDelete)
{ {
@ -1588,6 +1589,7 @@ int ColumnOp::addExtent(
//pOldVal = &((double *) oldValArray)[i]; //pOldVal = &((double *) oldValArray)[i];
break; break;
case WriteEngine::WR_VARBINARY : // treat same as char for now case WriteEngine::WR_VARBINARY : // treat same as char for now
case WriteEngine::WR_BLOB :
case WriteEngine::WR_CHAR : case WriteEngine::WR_CHAR :
if (!bDelete) if (!bDelete)
{ {
@ -1717,6 +1719,7 @@ int ColumnOp::addExtent(
pVal = &((double *) valArray)[i]; pVal = &((double *) valArray)[i];
break; break;
case WriteEngine::WR_VARBINARY : // treat same as char for now case WriteEngine::WR_VARBINARY : // treat same as char for now
case WriteEngine::WR_BLOB :
case WriteEngine::WR_CHAR : case WriteEngine::WR_CHAR :
{ {
memcpy(charTmpBuf, (char*)valArray+i*8, 8); memcpy(charTmpBuf, (char*)valArray+i*8, 8);