From c08d03fba46c3a89159bc2f5e44f1f1d051d4575 Mon Sep 17 00:00:00 2001 From: Andrew Hutchings Date: Fri, 17 Mar 2017 12:10:02 +0000 Subject: [PATCH] MCOL-267 Make BLOB DDL/DML work Currently limited to 8KB inserts. --- dbcon/ddlpackageproc/altertableprocessor.cpp | 3 +- dbcon/ddlpackageproc/createtableprocessor.cpp | 6 ++- dbcon/dmlpackageproc/dmlpackageprocessor.h | 3 +- dbcon/joblist/pcolscan.cpp | 3 +- dbcon/joblist/pcolstep.cpp | 3 +- writeengine/server/we_ddlcommandproc.cpp | 37 ++++++++++++++----- writeengine/server/we_ddlcommon.h | 1 + writeengine/server/we_dmlcommandproc.cpp | 2 + writeengine/server/we_dmlcommandproc.h | 3 +- writeengine/shared/we_chunkmanager.cpp | 3 ++ writeengine/wrapper/we_colop.cpp | 3 ++ 11 files changed, 51 insertions(+), 16 deletions(-) diff --git a/dbcon/ddlpackageproc/altertableprocessor.cpp b/dbcon/ddlpackageproc/altertableprocessor.cpp index df2d77eb8..d89d648b2 100644 --- a/dbcon/ddlpackageproc/altertableprocessor.cpp +++ b/dbcon/ddlpackageproc/altertableprocessor.cpp @@ -561,7 +561,8 @@ void AlterTableProcessor::addColumn (uint32_t sessionID, execplan::CalpontSystem if ((columnDefPtr->fType->fType == CalpontSystemCatalog::CHAR && columnDefPtr->fType->fLength > 8) || (columnDefPtr->fType->fType == CalpontSystemCatalog::VARCHAR && columnDefPtr->fType->fLength > 7) || - (columnDefPtr->fType->fType == CalpontSystemCatalog::VARBINARY && columnDefPtr->fType->fLength > 7)) + (columnDefPtr->fType->fType == CalpontSystemCatalog::VARBINARY && columnDefPtr->fType->fLength > 7) || + (columnDefPtr->fType->fType == CalpontSystemCatalog::BLOB)) { isDict = true; } diff --git a/dbcon/ddlpackageproc/createtableprocessor.cpp b/dbcon/ddlpackageproc/createtableprocessor.cpp index ba1a780a9..8e9e85039 100644 --- a/dbcon/ddlpackageproc/createtableprocessor.cpp +++ b/dbcon/ddlpackageproc/createtableprocessor.cpp @@ -244,7 +244,8 @@ keepGoing: dataType = convertDataType(tableDef.fColumns[i]->fType->fType); if ( (dataType == CalpontSystemCatalog::CHAR && tableDef.fColumns[i]->fType->fLength > 8) || (dataType == CalpontSystemCatalog::VARCHAR && tableDef.fColumns[i]->fType->fLength > 7) || - (dataType == CalpontSystemCatalog::VARBINARY && tableDef.fColumns[i]->fType->fLength > 7) ) + (dataType == CalpontSystemCatalog::VARBINARY && tableDef.fColumns[i]->fType->fLength > 7) || + (dataType == CalpontSystemCatalog::BLOB && tableDef.fColumns[i]->fType->fLength > 7) ) numDictCols++; } fStartingColOID = fObjectIDManager.allocOIDs(numColumns+numDictCols+1); //include column, oids,dictionary oids and tableoid @@ -533,7 +534,8 @@ cout << fTxnid.id << " Create table WE_SVR_WRITE_CREATE_SYSCOLUMN: " << errorMsg bytestream << (uint32_t) colDefPtr->fType->fCompressiontype; if ( (dataType == CalpontSystemCatalog::CHAR && colDefPtr->fType->fLength > 8) || (dataType == CalpontSystemCatalog::VARCHAR && colDefPtr->fType->fLength > 7) || - (dataType == CalpontSystemCatalog::VARBINARY && colDefPtr->fType->fLength > 7) ) + (dataType == CalpontSystemCatalog::VARBINARY && colDefPtr->fType->fLength > 7) || + (dataType == CalpontSystemCatalog::BLOB && colDefPtr->fType->fLength > 7) ) { bytestream << (uint32_t) (fStartingColOID+numColumns+(dictNum++)+1); bytestream << (uint8_t) dataType; diff --git a/dbcon/dmlpackageproc/dmlpackageprocessor.h b/dbcon/dmlpackageproc/dmlpackageprocessor.h index 2b93c1300..b30285c43 100644 --- a/dbcon/dmlpackageproc/dmlpackageprocessor.h +++ b/dbcon/dmlpackageproc/dmlpackageprocessor.h @@ -443,7 +443,8 @@ protected: if (((colType.colDataType == execplan::CalpontSystemCatalog::CHAR) && (colType.colWidth > 8)) || ((colType.colDataType == execplan::CalpontSystemCatalog::VARCHAR) && (colType.colWidth > 7)) || ((colType.colDataType == execplan::CalpontSystemCatalog::DECIMAL) && (colType.precision > 18)) - || (colType.colDataType == execplan::CalpontSystemCatalog::VARBINARY)) + || (colType.colDataType == execplan::CalpontSystemCatalog::VARBINARY) + || (colType.colDataType == execplan::CalpontSystemCatalog::BLOB)) { return true; } diff --git a/dbcon/joblist/pcolscan.cpp b/dbcon/joblist/pcolscan.cpp index 7a5927b75..455ed97ba 100644 --- a/dbcon/joblist/pcolscan.cpp +++ b/dbcon/joblist/pcolscan.cpp @@ -171,7 +171,8 @@ pColScanStep::pColScanStep( } //If this is a dictionary column, fudge the numbers... - if (fColType.colDataType == CalpontSystemCatalog::VARBINARY) + if ((fColType.colDataType == CalpontSystemCatalog::VARBINARY) + || (fColType.colDataType == CalpontSystemCatalog::BLOB)) { fColType.colWidth = 8; fIsDict = true; diff --git a/dbcon/joblist/pcolstep.cpp b/dbcon/joblist/pcolstep.cpp index 93680cd33..7bb46a182 100644 --- a/dbcon/joblist/pcolstep.cpp +++ b/dbcon/joblist/pcolstep.cpp @@ -169,7 +169,8 @@ pColStep::pColStep( } //If this is a dictionary column, fudge the numbers... - if (fColType.colDataType == CalpontSystemCatalog::VARBINARY) + if ((fColType.colDataType == CalpontSystemCatalog::VARBINARY) + || (fColType.colDataType == CalpontSystemCatalog::BLOB)) { fColType.colWidth = 8; fIsDict = true; diff --git a/writeengine/server/we_ddlcommandproc.cpp b/writeengine/server/we_ddlcommandproc.cpp index 2ccdd08ae..a4344beb8 100644 --- a/writeengine/server/we_ddlcommandproc.cpp +++ b/writeengine/server/we_ddlcommandproc.cpp @@ -450,7 +450,8 @@ uint8_t WE_DDLCommandProc::writeCreateSyscolumn(ByteStream& bs, std::string &err bool hasDict = false; if ( (dataType == CalpontSystemCatalog::CHAR && colDefPtr->fType->fLength > 8) || (dataType == CalpontSystemCatalog::VARCHAR && colDefPtr->fType->fLength > 7) || - (dataType == CalpontSystemCatalog::VARBINARY && colDefPtr->fType->fLength > 7) ) + (dataType == CalpontSystemCatalog::VARBINARY && colDefPtr->fType->fLength > 7) || + (dataType == CalpontSystemCatalog::BLOB && colDefPtr->fType->fLength > 7) ) { hasDict = true; dictOID.compressionType = colDefPtr->fType->fCompressiontype; @@ -459,17 +460,20 @@ uint8_t WE_DDLCommandProc::writeCreateSyscolumn(ByteStream& bs, std::string &err dictcol++; //@Bug 2534. Take away the limit of 255 and set the limit to 8000. - if (colDefPtr->fType->fLength > 8000) + if ((colDefPtr->fType->fLength > 8000) && + (dataType != CalpontSystemCatalog::BLOB)) { ostringstream os; os << "char, varchar and varbinary length may not exceed 8000"; throw std::runtime_error(os.str()); } } - else if (dataType == CalpontSystemCatalog::VARBINARY && colDefPtr->fType->fLength <= 7) + else if ((dataType == CalpontSystemCatalog::VARBINARY + || dataType == CalpontSystemCatalog::BLOB) + && colDefPtr->fType->fLength <= 7) { ostringstream os; - os << "varbinary length may not be less than 8"; + os << "varbinary and blob length may not be less than 8"; throw std::runtime_error(os.str()); } @@ -514,7 +518,8 @@ uint8_t WE_DDLCommandProc::writeCreateSyscolumn(ByteStream& bs, std::string &err //@Bug 2089 Disallow zero length char and varch column to be created if (dataType == CalpontSystemCatalog::CHAR || dataType == CalpontSystemCatalog::VARCHAR || - dataType == CalpontSystemCatalog::VARBINARY) + dataType == CalpontSystemCatalog::VARBINARY || + dataType == CalpontSystemCatalog::BLOB) { if (colDefPtr->fType->fLength <= 0) { @@ -829,17 +834,20 @@ uint8_t WE_DDLCommandProc::writeSyscolumn(ByteStream& bs, std::string & err) dictOID.dictOID = dictoid; //@Bug 2534. Take away the limit of 255 and set the limit to 8000. - if (colDefPtr->fType->fLength > 8000) + if ((colDefPtr->fType->fLength > 8000) && + (dataType != CalpontSystemCatalog::BLOB)) { ostringstream os; os << "char, varchar and varbinary length may not exceed 8000"; throw std::runtime_error(os.str()); } } - else if (dataType == CalpontSystemCatalog::VARBINARY && colDefPtr->fType->fLength <= 7) + else if ((dataType == CalpontSystemCatalog::VARBINARY + || dataType == CalpontSystemCatalog::BLOB) + && colDefPtr->fType->fLength <= 7) { ostringstream os; - os << "varbinary length may not be less than 8"; + os << "varbinary and blob length may not be less than 8"; throw std::runtime_error(os.str()); } @@ -885,7 +893,8 @@ uint8_t WE_DDLCommandProc::writeSyscolumn(ByteStream& bs, std::string & err) //@Bug 2089 Disallow zero length char and varch column to be created if (dataType == CalpontSystemCatalog::CHAR || dataType == CalpontSystemCatalog::VARCHAR || - dataType == CalpontSystemCatalog::VARBINARY) + dataType == CalpontSystemCatalog::VARBINARY || + dataType == CalpontSystemCatalog::BLOB) { if (colDefPtr->fType->fLength <= 0) { @@ -2242,6 +2251,8 @@ uint8_t WE_DDLCommandProc::updateSyscolumnTablename(ByteStream& bs, std::string && column.colType.colWidth > 7) || (column.colType.colDataType == CalpontSystemCatalog::VARBINARY && column.colType.colWidth > 7) + || (column.colType.colDataType == CalpontSystemCatalog::BLOB + && column.colType.colWidth > 7) || (column.colType.colDataType == CalpontSystemCatalog::DECIMAL && column.colType.precision > 18) || (column.colType.colDataType == CalpontSystemCatalog::UDECIMAL @@ -2998,6 +3009,8 @@ uint8_t WE_DDLCommandProc::updateSystablesTablename(ByteStream& bs, std::string && column.colType.colWidth > 7) || (column.colType.colDataType == CalpontSystemCatalog::VARBINARY && column.colType.colWidth > 7) + || (column.colType.colDataType == CalpontSystemCatalog::BLOB + && column.colType.colWidth > 7) || (column.colType.colDataType == CalpontSystemCatalog::DECIMAL && column.colType.precision > 18) || (column.colType.colDataType == CalpontSystemCatalog::UDECIMAL @@ -3878,6 +3891,8 @@ uint8_t WE_DDLCommandProc::updateSyscolumnSetDefault(messageqcpp::ByteStream& bs && column.colType.colWidth > 7) || (column.colType.colDataType == CalpontSystemCatalog::VARBINARY && column.colType.colWidth > 7) + || (column.colType.colDataType == CalpontSystemCatalog::BLOB + && column.colType.colWidth > 7) || (column.colType.colDataType == CalpontSystemCatalog::DECIMAL && column.colType.precision > 18) || (column.colType.colDataType == CalpontSystemCatalog::UDECIMAL @@ -4146,6 +4161,8 @@ uint8_t WE_DDLCommandProc::updateSyscolumnRenameColumn(messageqcpp::ByteStream& && column1.colType.colWidth > 7) || (column1.colType.colDataType == CalpontSystemCatalog::VARBINARY && column1.colType.colWidth > 7) + || (column1.colType.colDataType == CalpontSystemCatalog::BLOB + && column1.colType.colWidth > 7) || (column1.colType.colDataType == CalpontSystemCatalog::DECIMAL && column1.colType.precision > 18) || (column1.colType.colDataType == CalpontSystemCatalog::UDECIMAL @@ -4336,6 +4353,8 @@ uint8_t WE_DDLCommandProc::updateSyscolumnRenameColumn(messageqcpp::ByteStream& && column5.colType.colWidth > 7) || (column5.colType.colDataType == CalpontSystemCatalog::VARBINARY && column5.colType.colWidth > 7) + || (column5.colType.colDataType == CalpontSystemCatalog::BLOB + && column5.colType.colWidth > 7) || (column5.colType.colDataType == CalpontSystemCatalog::DECIMAL && column5.colType.precision > 18) || (column5.colType.colDataType == CalpontSystemCatalog::UDECIMAL diff --git a/writeengine/server/we_ddlcommon.h b/writeengine/server/we_ddlcommon.h index 827511646..e83e2d97f 100644 --- a/writeengine/server/we_ddlcommon.h +++ b/writeengine/server/we_ddlcommon.h @@ -316,6 +316,7 @@ inline boost::any getNullValueForType(const execplan::CalpontSystemCatalog::ColT } break; + case execplan::CalpontSystemCatalog::BLOB: case execplan::CalpontSystemCatalog::VARBINARY: { std::string charnull; diff --git a/writeengine/server/we_dmlcommandproc.cpp b/writeengine/server/we_dmlcommandproc.cpp index 0afb5f342..125a170b9 100644 --- a/writeengine/server/we_dmlcommandproc.cpp +++ b/writeengine/server/we_dmlcommandproc.cpp @@ -2071,6 +2071,7 @@ uint8_t WE_DMLCommandProc::processUpdate(messageqcpp::ByteStream& bs, break; } case CalpontSystemCatalog::VARBINARY: + case CalpontSystemCatalog::BLOB: { value = row.getVarBinaryStringField(fetchColPos); break; @@ -2356,6 +2357,7 @@ uint8_t WE_DMLCommandProc::processUpdate(messageqcpp::ByteStream& bs, break; } case CalpontSystemCatalog::VARBINARY: + case CalpontSystemCatalog::BLOB: { value = row.getVarBinaryStringField(fetchColPos); break; diff --git a/writeengine/server/we_dmlcommandproc.h b/writeengine/server/we_dmlcommandproc.h index b92462bd7..eacdddf5b 100644 --- a/writeengine/server/we_dmlcommandproc.h +++ b/writeengine/server/we_dmlcommandproc.h @@ -110,7 +110,8 @@ class WE_DMLCommandProc || ((colType.colDataType == execplan::CalpontSystemCatalog::VARCHAR) && (colType.colWidth > 7)) || ((colType.colDataType == execplan::CalpontSystemCatalog::DECIMAL) && (colType.precision > 18)) || ((colType.colDataType == execplan::CalpontSystemCatalog::UDECIMAL) && (colType.precision > 18)) - || (colType.colDataType == execplan::CalpontSystemCatalog::VARBINARY)) + || (colType.colDataType == execplan::CalpontSystemCatalog::VARBINARY) + || (colType.colDataType == execplan::CalpontSystemCatalog::BLOB)) { return true; } diff --git a/writeengine/shared/we_chunkmanager.cpp b/writeengine/shared/we_chunkmanager.cpp index b0d40ff3d..043d4eceb 100644 --- a/writeengine/shared/we_chunkmanager.cpp +++ b/writeengine/shared/we_chunkmanager.cpp @@ -1587,6 +1587,9 @@ int ChunkManager::calculateHeaderSize(int width) int rowsPerExtent = BRMWrapper::getInstance()->getExtentRows(); int rowsPerFile = rowsPerExtent * extentsPerFile; int stringsPerBlock = 8180 / (width + 2); // 8180 = 8192 - 12 + //TODO: temporary fix for Blob + if (stringsPerBlock == 0) + stringsPerBlock = 1; int blocksNeeded = rowsPerFile / stringsPerBlock; int blocksPerChunk = UNCOMPRESSED_CHUNK_SIZE / BYTE_PER_BLOCK; lldiv_t chunks = lldiv(blocksNeeded, blocksPerChunk); diff --git a/writeengine/wrapper/we_colop.cpp b/writeengine/wrapper/we_colop.cpp index b14a1c225..d0e936aec 100644 --- a/writeengine/wrapper/we_colop.cpp +++ b/writeengine/wrapper/we_colop.cpp @@ -1454,6 +1454,7 @@ int ColumnOp::addExtent( //pOldVal = &((double *) oldValArray)[i]; break; case WriteEngine::WR_VARBINARY : // treat same as char for now + case WriteEngine::WR_BLOB : case WriteEngine::WR_CHAR : if (!bDelete) { @@ -1588,6 +1589,7 @@ int ColumnOp::addExtent( //pOldVal = &((double *) oldValArray)[i]; break; case WriteEngine::WR_VARBINARY : // treat same as char for now + case WriteEngine::WR_BLOB : case WriteEngine::WR_CHAR : if (!bDelete) { @@ -1717,6 +1719,7 @@ int ColumnOp::addExtent( pVal = &((double *) valArray)[i]; break; case WriteEngine::WR_VARBINARY : // treat same as char for now + case WriteEngine::WR_BLOB : case WriteEngine::WR_CHAR : { memcpy(charTmpBuf, (char*)valArray+i*8, 8);