MCOL-267 multi-block support for PrimProc and bulk

* Adds multi-block bulk write support
* Adds PrimProc multi-block read support
* Allows the functions length() and hex() to work with BLOB columns

dbcon/execplan/calpontsystemcatalog.h

@@ -905,7 +905,8 @@ const CalpontSystemCatalog::TableAliasName make_aliasview(const std::string& s,
 inline bool isCharType(const execplan::CalpontSystemCatalog::ColDataType type)
 {
 	return (execplan::CalpontSystemCatalog::VARCHAR == type ||
-			execplan::CalpontSystemCatalog::CHAR == type);
+			execplan::CalpontSystemCatalog::CHAR == type ||
+            execplan::CalpontSystemCatalog::BLOB == type);
 }
 
 /** convenience function to determine if column type is an

dbcon/execplan/simplecolumn.cpp

@@ -583,6 +583,7 @@ void SimpleColumn::evaluate(Row& row, bool& isNull)
 			break;
 		}
 		case CalpontSystemCatalog::VARBINARY:
+        case CalpontSystemCatalog::BLOB:
 		{
 			fResult.strVal = row.getVarBinaryStringField(fInputIndex);
 			break;

dbcon/execplan/simplefilter.cpp

@@ -176,6 +176,7 @@ const string SimpleFilter::data() const
 	if (dynamic_cast<ConstantColumn*>(fRhs) &&
 		  (fRhs->resultType().colDataType == CalpontSystemCatalog::VARCHAR ||
 		   fRhs->resultType().colDataType == CalpontSystemCatalog::CHAR ||
+           fRhs->resultType().colDataType == CalpontSystemCatalog::BLOB ||
 		   fRhs->resultType().colDataType == CalpontSystemCatalog::VARBINARY ||
 		   fRhs->resultType().colDataType == CalpontSystemCatalog::DATE ||
 		   fRhs->resultType().colDataType == CalpontSystemCatalog::DATETIME))
@@ -185,6 +186,7 @@ const string SimpleFilter::data() const
 	if (dynamic_cast<ConstantColumn*>(fLhs) &&
 		  (fLhs->resultType().colDataType == CalpontSystemCatalog::VARCHAR ||
 		   fLhs->resultType().colDataType == CalpontSystemCatalog::CHAR ||
+           fLhs->resultType().colDataType == CalpontSystemCatalog::BLOB ||
 		   fLhs->resultType().colDataType == CalpontSystemCatalog::VARBINARY ||
 		   fLhs->resultType().colDataType == CalpontSystemCatalog::DATE ||
 		   fLhs->resultType().colDataType == CalpontSystemCatalog::DATETIME))

dbcon/execplan/treenode.h

@@ -394,6 +394,7 @@ inline bool TreeNode::getBoolVal()
 			return (atoi(fResult.strVal.c_str()) != 0);
 		//FIXME: Huh???
 		case CalpontSystemCatalog::VARBINARY:
+        case CalpontSystemCatalog::BLOB:
 			if (fResultType.colWidth <= 7)
 				return (atoi((char*)(&fResult.origIntVal)) != 0);
 			return (atoi(fResult.strVal.c_str()) != 0);
@@ -440,6 +441,7 @@ inline const std::string& TreeNode::getStrVal()
 			break;
 		//FIXME: ???
 		case CalpontSystemCatalog::VARBINARY:
+        case CalpontSystemCatalog::BLOB:
 			if (fResultType.colWidth <= 7)
 				fResult.strVal = (char*)(&fResult.origIntVal);
 			break;
@@ -573,6 +575,7 @@ inline int64_t TreeNode::getIntVal()
 			return atoll(fResult.strVal.c_str());
 		//FIXME: ???
 		case CalpontSystemCatalog::VARBINARY:
+        case CalpontSystemCatalog::BLOB:
 			if (fResultType.colWidth <= 7)
 				return fResult.intVal;
 			return atoll(fResult.strVal.c_str());
@@ -656,6 +659,7 @@ inline float TreeNode::getFloatVal()
 			return atof(fResult.strVal.c_str());
 		//FIXME: ???
 		case CalpontSystemCatalog::VARBINARY:
+        case CalpontSystemCatalog::BLOB:
 			if (fResultType.colWidth <= 7)
 				return atof((char*)(&fResult.origIntVal));
 			return atof(fResult.strVal.c_str());
@@ -703,6 +707,7 @@ inline double TreeNode::getDoubleVal()
 			return strtod(fResult.strVal.c_str(), NULL);
 		//FIXME: ???
 		case CalpontSystemCatalog::VARBINARY:
+        case CalpontSystemCatalog::BLOB:
 			if (fResultType.colWidth <= 7)
 				return strtod((char*)(&fResult.origIntVal), NULL);
 			return strtod(fResult.strVal.c_str(), NULL);
@@ -746,6 +751,7 @@ inline IDB_Decimal TreeNode::getDecimalVal()
 		case CalpontSystemCatalog::VARCHAR:
 			throw logging::InvalidConversionExcept("TreeNode::getDecimalVal: non-support conversion from string");
 		case CalpontSystemCatalog::VARBINARY:
+        case CalpontSystemCatalog::BLOB:
 			throw logging::InvalidConversionExcept("TreeNode::getDecimalVal: non-support conversion from binary string");
 		case CalpontSystemCatalog::BIGINT:
 		case CalpontSystemCatalog::MEDINT:

dbcon/joblist/expressionstep.cpp

@@ -324,7 +324,8 @@ void ExpressionStep::addColumn(ReturnedColumn* rc, JobInfo& jobInfo)
 void ExpressionStep::populateColumnInfo(ReturnedColumn* rc, JobInfo& jobInfo)
 {
 	// As of bug3695, make sure varbinary is not used in function expression.
-	if (rc->resultType().colDataType == CalpontSystemCatalog::VARBINARY && !fVarBinOK)
+	if ((rc->resultType().colDataType == CalpontSystemCatalog::VARBINARY ||
+         rc->resultType().colDataType == CalpontSystemCatalog::BLOB) && !fVarBinOK)
 		throw runtime_error("VARBINARY in filter or function is not supported.");
 
 	SimpleColumn* sc = dynamic_cast<SimpleColumn*>(rc);
@@ -344,7 +345,8 @@ void ExpressionStep::populateColumnInfo(ReturnedColumn* rc, JobInfo& jobInfo)
 void ExpressionStep::populateColumnInfo(SimpleColumn* sc, JobInfo& jobInfo)
 {
 	// As of bug3695, make sure varbinary is not used in function expression.
-	if (sc->resultType().colDataType == CalpontSystemCatalog::VARBINARY && !fVarBinOK)
+	if ((sc->resultType().colDataType == CalpontSystemCatalog::VARBINARY ||
+         sc->resultType().colDataType == CalpontSystemCatalog::BLOB) && !fVarBinOK)
 		throw runtime_error ("VARBINARY in filter or function is not supported.");
 
 	CalpontSystemCatalog::OID tblOid = joblist::tableOid(sc, jobInfo.csc);
@@ -409,7 +411,8 @@ void ExpressionStep::populateColumnInfo(SimpleColumn* sc, JobInfo& jobInfo)
 void ExpressionStep::populateColumnInfo(WindowFunctionColumn* wc, JobInfo& jobInfo)
 {
 	// As of bug3695, make sure varbinary is not used in function expression.
-	if (wc->resultType().colDataType == CalpontSystemCatalog::VARBINARY && !fVarBinOK)
+	if ((wc->resultType().colDataType == CalpontSystemCatalog::VARBINARY ||
+         wc->resultType().colDataType == CalpontSystemCatalog::BLOB) && !fVarBinOK)
 		throw runtime_error("VARBINARY in filter or function is not supported.");
 
 	// This is for window function in IN/EXISTS sub-query.
@@ -434,7 +437,8 @@ void ExpressionStep::populateColumnInfo(WindowFunctionColumn* wc, JobInfo& jobIn
 void ExpressionStep::populateColumnInfo(AggregateColumn* ac, JobInfo& jobInfo)
 {
 	// As of bug3695, make sure varbinary is not used in function expression.
-	if (ac->resultType().colDataType == CalpontSystemCatalog::VARBINARY && !fVarBinOK)
+	if ((ac->resultType().colDataType == CalpontSystemCatalog::VARBINARY ||
+         ac->resultType().colDataType == CalpontSystemCatalog::BLOB) && !fVarBinOK)
 		throw runtime_error("VARBINARY in filter or function is not supported.");
 
 	// This is for aggregate function in IN/EXISTS sub-query.

primitives/linux-port/column.cpp

@@ -249,6 +249,7 @@ inline bool isEmptyVal<8>(uint8_t type, const uint8_t* ival)
 	case CalpontSystemCatalog::DATE:
 	case CalpontSystemCatalog::DATETIME:
 	case CalpontSystemCatalog::VARBINARY:
+    case CalpontSystemCatalog::BLOB:
 		return (*val == joblist::CHAR8EMPTYROW);
 	case CalpontSystemCatalog::UBIGINT:
 		return (joblist::UBIGINTEMPTYROW == *val);
@@ -271,6 +272,7 @@ inline bool isEmptyVal<4>(uint8_t type, const uint8_t* ival)
 		return (joblist::FLOATEMPTYROW == *val);
 	case CalpontSystemCatalog::CHAR:
 	case CalpontSystemCatalog::VARCHAR:
+    case CalpontSystemCatalog::BLOB:
 	case CalpontSystemCatalog::DATE:
 	case CalpontSystemCatalog::DATETIME:
 		return (joblist::CHAR4EMPTYROW == *val);
@@ -292,6 +294,7 @@ inline bool isEmptyVal<2>(uint8_t type, const uint8_t* ival)
 	{
 	case CalpontSystemCatalog::CHAR:
 	case CalpontSystemCatalog::VARCHAR:
+    case CalpontSystemCatalog::BLOB:
 	case CalpontSystemCatalog::DATE:
 	case CalpontSystemCatalog::DATETIME:
 		return (joblist::CHAR2EMPTYROW == *val);
@@ -313,6 +316,7 @@ inline bool isEmptyVal<1>(uint8_t type, const uint8_t* ival)
 	{
 	case CalpontSystemCatalog::CHAR:
 	case CalpontSystemCatalog::VARCHAR:
+    case CalpontSystemCatalog::BLOB:
 	case CalpontSystemCatalog::DATE:
 	case CalpontSystemCatalog::DATETIME:
 		return (*val == joblist::CHAR1EMPTYROW);
@@ -343,6 +347,7 @@ inline bool isNullVal<8>(uint8_t type, const uint8_t* ival)
 	case CalpontSystemCatalog::DATE:
 	case CalpontSystemCatalog::DATETIME:
 	case CalpontSystemCatalog::VARBINARY:
+    case CalpontSystemCatalog::BLOB:
 		//@bug 339 might be a token here
 		//TODO: what's up with the second const here?
 		return (*val == joblist::CHAR8NULL || 0xFFFFFFFFFFFFFFFELL == *val);
@@ -367,6 +372,7 @@ inline bool isNullVal<4>(uint8_t type, const uint8_t* ival)
 		return (joblist::FLOATNULL == *val);
 	case CalpontSystemCatalog::CHAR:
 	case CalpontSystemCatalog::VARCHAR:
+    case CalpontSystemCatalog::BLOB:
 		return (joblist::CHAR4NULL == *val);
 	case CalpontSystemCatalog::DATE:
 	case CalpontSystemCatalog::DATETIME:
@@ -389,6 +395,7 @@ inline bool isNullVal<2>(uint8_t type, const uint8_t* ival)
 	{
 	case CalpontSystemCatalog::CHAR:
 	case CalpontSystemCatalog::VARCHAR:
+    case CalpontSystemCatalog::BLOB:
 	case CalpontSystemCatalog::DATE:
 	case CalpontSystemCatalog::DATETIME:
 		return (joblist::CHAR2NULL == *val);
@@ -410,6 +417,7 @@ inline bool isNullVal<1>(uint8_t type, const uint8_t* ival)
 	{
 	case CalpontSystemCatalog::CHAR:
 	case CalpontSystemCatalog::VARCHAR:
+    case CalpontSystemCatalog::BLOB:
 	case CalpontSystemCatalog::DATE:
 	case CalpontSystemCatalog::DATETIME:
 		return (*val == joblist::CHAR1NULL);
@@ -451,6 +459,7 @@ inline bool isMinMaxValid(const NewColRequestHeader *in) {
 		case CalpontSystemCatalog::CHAR:
 			return (in->DataSize<9);
 		case CalpontSystemCatalog::VARCHAR:
+        case CalpontSystemCatalog::BLOB:
 			return (in->DataSize<8);
 		case CalpontSystemCatalog::TINYINT:
 		case CalpontSystemCatalog::SMALLINT:
@@ -505,7 +514,7 @@ inline bool colCompare(int64_t val1, int64_t val2, uint8_t COP, uint8_t rf, int
 		return colCompare_(dVal1, dVal2, COP);
 	}
 
-	else if ( (type == CalpontSystemCatalog::CHAR || type == CalpontSystemCatalog::VARCHAR) && !isNull )
+	else if ( (type == CalpontSystemCatalog::CHAR || type == CalpontSystemCatalog::VARCHAR || type == CalpontSystemCatalog::BLOB) && !isNull )
 	{
 		if (!regex.used && !rf)
 			return colCompare_(order_swap(val1), order_swap(val2), COP);
@@ -1180,7 +1189,7 @@ inline void p_Col_ridArray(NewColRequestHeader *in,
 		if (out->ValidMinMax && !isNull && !isEmpty)
 		{
 
-			if ((in->DataType == CalpontSystemCatalog::CHAR || in->DataType == CalpontSystemCatalog::VARCHAR ) && 1 < W)
+			if ((in->DataType == CalpontSystemCatalog::CHAR || in->DataType == CalpontSystemCatalog::VARCHAR || in->DataType == CalpontSystemCatalog::BLOB ) && 1 < W)
 			{
 				if (colCompare(out->Min, val, COMPARE_GT, false, in->DataType, W, placeholderRegex))
 					out->Min = val;

primitives/primproc/columncommand.cpp

@@ -866,6 +866,7 @@ const uint64_t ColumnCommand::getEmptyRowValue( const execplan::CalpontSystemCat
 	case execplan::CalpontSystemCatalog::DATE :
 	case execplan::CalpontSystemCatalog::DATETIME :
 	case execplan::CalpontSystemCatalog::VARBINARY :
+    case execplan::CalpontSystemCatalog::BLOB :
 	default:
 		emptyVal = joblist::CHAR1EMPTYROW;
 		if ( width == (2 + offset) )

primitives/primproc/dictstep.cpp

@@ -509,7 +509,8 @@ void DictStep::_projectToRG(RowGroup &rg, uint32_t col)
 
 		// bug 4901 - move this inside the loop and call incrementally
 		// to save the unnecessary string copy
-		if (rg.getColTypes()[col] != execplan::CalpontSystemCatalog::VARBINARY) {
+		if ((rg.getColTypes()[col] != execplan::CalpontSystemCatalog::VARBINARY) &&
+            (rg.getColTypes()[col] != execplan::CalpontSystemCatalog::BLOB)) {
 			for (i = curResultCounter; i < tmpResultCounter; i++) {
 				rg.getRow(newRidList[i].pos, &r);
 				//cout << "serializing " << tmpStrings[i] << endl;
@@ -517,9 +518,39 @@ void DictStep::_projectToRG(RowGroup &rg, uint32_t col)
 			}
 		}
 		else {
-			for (i = curResultCounter; i < tmpResultCounter; i++) {
+            uint32_t firstTmpResultCounter = tmpResultCounter;
+			for (i = curResultCounter; i < firstTmpResultCounter; i++) {
 				rg.getRow(newRidList[i].pos, &r);
-				r.setVarBinaryField(tmpStrings[i].ptr, tmpStrings[i].len, col);
+                // If this is a multi-block blob, get all the blocks
+                // We do string copy here, should maybe have a RowGroup
+                // function to append strings or something?
+                if ((newRidList[i].token != 0xffffffffffffffffLL) &&
+                    ((newRidList[i].token >> 46) > 0))
+                {
+                    StringPtr multi_part[1];
+                    uint16_t old_offset = primMsg->tokens[0].offset;
+                    string result((char*)tmpStrings[i].ptr, tmpStrings[i].len);
+                    uint64_t origin_lbid = primMsg->LBID;
+                    uint32_t lbid_count = newRidList[i].token >> 46;
+                    primMsg->tokens[0].offset = 1; // first offset of a sig
+                    for (uint32_t j = 1; j <= lbid_count; j++)
+                    {
+                        tmpResultCounter = 0;
+                        primMsg->LBID = origin_lbid + j;
+                        primMsg->NVALS = 1;
+                        primMsg->tokens[0].LBID = origin_lbid + j;
+                        issuePrimitive(false);
+                        projectResult(multi_part);
+                        result.append((char*)multi_part[0].ptr, multi_part[0].len);
+                    }
+                    primMsg->tokens[0].offset = old_offset;
+                    tmpResultCounter = firstTmpResultCounter;
+                    r.setVarBinaryField((unsigned char*)result.c_str(), result.length(), col);
+                }
+                else
+                {
+                    r.setVarBinaryField(tmpStrings[i].ptr, tmpStrings[i].len, col);
+                }
 			}
 		}
 		curResultCounter = tmpResultCounter;

primitives/primproc/filtercommand.cpp

@@ -94,7 +94,8 @@ Command* FilterCommand::makeFilterCommand(ByteStream& bs, vector<SCommand>& cmds
 		// char[] is stored as int, but cannot directly compare if length is different
 		// due to endian issue
 		if (cmd0->getColType().colDataType == execplan::CalpontSystemCatalog::CHAR ||
-			cmd0->getColType().colDataType == execplan::CalpontSystemCatalog::VARCHAR)
+			cmd0->getColType().colDataType == execplan::CalpontSystemCatalog::VARCHAR ||
+            cmd0->getColType().colDataType == execplan::CalpontSystemCatalog::BLOB)
 		{
 			StrFilterCmd* sc = new StrFilterCmd();
 			sc->setCompareFunc(CC);

utils/funcexp/func_hex.cpp

@@ -96,6 +96,7 @@ string Func_hex::getStrVal(rowgroup::Row& row,
 			break;
 		}
 		case CalpontSystemCatalog::VARBINARY:
+        case CalpontSystemCatalog::BLOB:
 		{
 			const string& arg = parm[0]->data()->getStrVal(row, isNull);
 			uint64_t hexLen = arg.size() * 2;

utils/funcexp/func_length.cpp

@@ -48,7 +48,8 @@ int64_t Func_length::getIntVal(rowgroup::Row& row,
 							bool& isNull,
 							CalpontSystemCatalog::ColType&)
 {
-	if (fp[0]->data()->resultType().colDataType == CalpontSystemCatalog::VARBINARY)
+	if ((fp[0]->data()->resultType().colDataType == CalpontSystemCatalog::VARBINARY) ||
+        (fp[0]->data()->resultType().colDataType == CalpontSystemCatalog::BLOB))
 		return fp[0]->data()->getStrVal(row, isNull).length();
 
 	return strlen(fp[0]->data()->getStrVal(row, isNull).c_str());

utils/funcexp/funcexp.cpp

@@ -265,7 +265,9 @@ void FuncExp::evaluate(rowgroup::Row& row, std::vector<execplan::SRCP>& expressi
 				break;
 			}
 			case CalpontSystemCatalog::CHAR:
-			case CalpontSystemCatalog::VARCHAR:			
+			case CalpontSystemCatalog::VARCHAR:
+            // TODO: might not be right thing for BLOB
+            case CalpontSystemCatalog::BLOB:
 			{			
 				const std::string& val = expression[i]->getStrVal(row, isNull);
 				if (isNull)

utils/rowgroup/rowgroup.cpp

@@ -1106,13 +1106,13 @@ void applyMapping(const int *mapping, const Row &in, Row *out)
 	for (i = 0; i < in.getColumnCount(); i++)
 		if (mapping[i] != -1)
 		{
-			if (UNLIKELY(in.isLongString(i)))
+            if (UNLIKELY(in.getColTypes()[i] == execplan::CalpontSystemCatalog::VARBINARY || in.getColTypes()[i] == execplan::CalpontSystemCatalog::BLOB))
+				out->setVarBinaryField(in.getVarBinaryField(i), in.getVarBinaryLength(i), mapping[i]);
+            else if (UNLIKELY(in.isLongString(i)))
 				out->setStringField(in.getStringPointer(i), in.getStringLength(i), mapping[i]);
 				//out->setStringField(in.getStringField(i), mapping[i]);
 			else if (UNLIKELY(in.isShortString(i)))
 				out->setUintField(in.getUintField(i), mapping[i]);
-			else if (UNLIKELY(in.getColTypes()[i] == execplan::CalpontSystemCatalog::VARBINARY || in.getColTypes()[i] == execplan::CalpontSystemCatalog::BLOB))
-				out->setVarBinaryField(in.getVarBinaryField(i), in.getVarBinaryLength(i), mapping[i]);
 			else if (UNLIKELY(in.getColTypes()[i] == execplan::CalpontSystemCatalog::LONGDOUBLE))
 				out->setLongDoubleField(in.getLongDoubleField(i), mapping[i]);
 			else if (in.isUnsigned(i))

utils/rowgroup/rowgroup.h

@@ -797,7 +797,8 @@ inline void Row::copyField(uint32_t destIndex, uint32_t srcIndex) const
 
 inline void Row::copyField(Row &out, uint32_t destIndex, uint32_t srcIndex) const
 {
-	if (UNLIKELY(types[srcIndex] == execplan::CalpontSystemCatalog::VARBINARY))
+	if (UNLIKELY(types[srcIndex] == execplan::CalpontSystemCatalog::VARBINARY ||
+                 types[srcIndex] == execplan::CalpontSystemCatalog::BLOB))
 		out.setVarBinaryField(getVarBinaryStringField(srcIndex), destIndex);
 	else if (UNLIKELY(isLongString(srcIndex)))
 		out.setStringField(getStringPointer(srcIndex), getStringLength(srcIndex), destIndex);
@@ -1268,7 +1269,8 @@ inline bool RowGroup::isLongString(uint32_t colIndex) const
 {
 	return ((getColumnWidth(colIndex) > 7 && types[colIndex] == execplan::CalpontSystemCatalog::VARCHAR) ||
 		(getColumnWidth(colIndex) > 8 && types[colIndex] == execplan::CalpontSystemCatalog::CHAR) ||
-		types[colIndex] == execplan::CalpontSystemCatalog::VARBINARY);
+		types[colIndex] == execplan::CalpontSystemCatalog::VARBINARY ||
+        types[colIndex] == execplan::CalpontSystemCatalog::BLOB);
 }
 
 inline bool RowGroup::usesStringTable() const

writeengine/dictionary/we_dctnry.cpp

@@ -64,7 +64,7 @@ namespace
     const int START_HDR1  =                    // start loc of 2nd offset (HDR1)
           HDR_UNIT_SIZE + NEXT_PTR_BYTES + HDR_UNIT_SIZE;
     const int PSEUDO_COL_WIDTH = DICT_COL_WIDTH; // used to convert row count to block count
-
+    const int MAX_BLOB_SIZE = 2100000000; // for safety, we use an 18bit block count of 8KB blocks
 }
 
 namespace WriteEngine
@@ -620,17 +620,104 @@ bool Dctnry::getTokenFromArray(Signature& sig)
  *       token - token that was assigned to the inserted signature
  *
  * RETURN:
- *    none
+ *    success    - successfully write the signature to the block
+ *    failure    - failed to extend/create an extent for the block
  ******************************************************************************/
-void Dctnry::insertDctnry2(Signature& sig)
+int Dctnry::insertDctnry2(Signature& sig)
 {
-    insertDctnryHdr(m_curBlock.data,
+    int rc = 0;
-                    sig.size);
+    int write_size;
-    insertSgnture(m_curBlock.data, sig.size, (unsigned char*)sig.signature);
+    bool lbid_in_token;
 
-    sig.token.fbo = m_curLbid;
+    sig.token.bc = 0;
-    sig.token.op  = m_curOp;
+
-    sig.token.bc = 0U;
+    while (sig.size > 0)
+    {
+        if (sig.size > (m_freeSpace - m_totalHdrBytes))
+        {
+            write_size = (m_freeSpace - m_totalHdrBytes);
+        }
+        else
+        {
+            write_size = sig.size;
+        }
+
+        insertDctnryHdr(m_curBlock.data, write_size);
+        insertSgnture(m_curBlock.data, write_size, (unsigned char*)sig.signature);
+        
+        sig.size -= write_size;
+        sig.signature += write_size;
+
+        if (!lbid_in_token)
+        {
+            sig.token.fbo = m_curLbid;
+            sig.token.op  = m_curOp;
+            lbid_in_token = true;
+        }
+        
+        if (sig.size > 0)
+        {
+            CommBlock cb;
+            cb.file.oid = m_dctnryOID;
+            cb.file.pFile = m_dFile;
+            sig.token.bc++;
+            
+            RETURN_ON_ERROR( writeDBFile(cb, &m_curBlock, m_curLbid) );
+            memset( m_curBlock.data, 0, sizeof(m_curBlock.data));
+            memcpy( m_curBlock.data, &m_dctnryHeader2, m_totalHdrBytes);
+            m_freeSpace = BYTE_PER_BLOCK - m_totalHdrBytes;
+            m_curBlock.state = BLK_WRITE;
+            m_curOp =0;
+            m_lastFbo++;
+            m_curFbo = m_lastFbo;
+ 
+            //...Expand current extent if it is an abbreviated initial extent
+            if ((m_curFbo    == m_numBlocks) &&
+                (m_numBlocks == NUM_BLOCKS_PER_INITIAL_EXTENT))
+            {
+                 RETURN_ON_ERROR( expandDctnryExtent() );
+            }
+    
+            //...Allocate a new extent if we have reached the last block in the
+            //   current extent.
+            if (m_curFbo == m_numBlocks)
+            {//last block
+                //for roll back the extent to use
+                //Save those empty extents in case of failure to rollback
+                std::vector<ExtentInfo> dictExtentInfo;
+                ExtentInfo info;
+                info.oid = m_dctnryOID;
+                info.partitionNum = m_partition;
+                info.segmentNum = m_segment;
+                info.dbRoot = m_dbRoot;
+                info.hwm = m_hwm;
+                info.newFile = false;
+                dictExtentInfo.push_back (info);
+                LBID_t startLbid;
+                // Add an extent.
+                rc =  createDctnry(m_dctnryOID,
+                                   0,             // dummy column width
+                                   m_dbRoot,
+                                   m_partition,
+                                   m_segment,
+                                   startLbid,
+                                   false) ;
+                if ( rc != NO_ERROR )
+                {
+                    //roll back the extent             
+                    BRMWrapper::getInstance()->deleteEmptyDictStoreExtents(
+                        dictExtentInfo);
+                    return rc;
+                }
+            }
+            RETURN_ON_ERROR( BRMWrapper::getInstance()->getBrmInfo(m_dctnryOID,
+                                                        m_partition, m_segment,
+                                                        m_curFbo,    m_curLbid) );
+            m_curBlock.lbid = m_curLbid;
+     
+        }
+    }
+    return NO_ERROR;
 }
 
 /*******************************************************************************
@@ -705,7 +792,8 @@ int Dctnry::insertDctnry(const char* buf,
         // it is too late to reject the row.  However, as a precaution, we
         // still check against max size & set to null token if needed.
         if ((curSig.size == 0) ||
-            (curSig.size == COLPOSPAIR_NULL_TOKEN_OFFSET))
+            (curSig.size == COLPOSPAIR_NULL_TOKEN_OFFSET) ||
+            (curSig.size > MAX_BLOB_SIZE))
         {
             if (m_defVal.length() > 0) // use default string if available
             {
@@ -735,6 +823,7 @@ int Dctnry::insertDctnry(const char* buf,
         }
 
         //...Search for the string in our string cache
+        //if it fits into one block (< 8KB)
         if ((m_arraySize < MAX_STRING_CACHE_SIZE) &&
             (curSig.size <= MAX_SIGNATURE_SIZE))
         {
@@ -750,14 +839,15 @@ int Dctnry::insertDctnry(const char* buf,
             }
             //Stats::stopParseEvent("getTokenFromArray");
         }
-        totalUseSize = HDR_UNIT_SIZE + curSig.size;
+        totalUseSize = m_totalHdrBytes + curSig.size;
 
         //...String not found in cache, so proceed.
         //   If room is available in current block then insert into block.
         // @bug 3960: Add MAX_OP_COUNT check to handle case after bulk rollback
-        if( (totalUseSize <= m_freeSpace) &&
+        if( ((totalUseSize <= m_freeSpace) ||
+            ((curSig.size > 8176) && (m_freeSpace > m_totalHdrBytes))) &&
             (m_curOp      < (MAX_OP_COUNT-1)) ) {
-            insertDctnry2(curSig); //m_freeSpace updated!
+            RETURN_ON_ERROR(insertDctnry2(curSig)); //m_freeSpace updated!
             m_curBlock.state = BLK_WRITE;
             memcpy( pOut + outOffset, &curSig.token, 8 );
             outOffset += 8;
@@ -880,7 +970,7 @@ int Dctnry::insertDctnry(const char* buf,
             //   we need to add the string to the new block.
             if (!found)
             {
-                insertDctnry2(curSig); //m_freeSpace updated!
+                RETURN_ON_ERROR(insertDctnry2(curSig)); //m_freeSpace updated!
                 m_curBlock.state = BLK_WRITE;
                 memcpy( pOut + outOffset, &curSig.token, 8 );
                 outOffset += 8;
@@ -949,7 +1039,7 @@ int Dctnry::insertDctnry(const int& sgnature_size,
     int write_size;
     bool lbid_in_token = false;
     // Round down for safety. In theory we can take 262143 * 8176 bytes
-    if (sgnature_size > (2100000000))
+    if (sgnature_size > MAX_BLOB_SIZE)
     {
         return ERR_DICT_SIZE_GT_2G;
     }

writeengine/dictionary/we_dctnry.h

@@ -239,7 +239,7 @@ protected:
     // insertDctnryHdr inserts the new value info into the header.
     // insertSgnture   inserts the new value into the block.
     //
-    void         insertDctnry2(Signature& sig);
+    int          insertDctnry2(Signature& sig);
     void         insertDctnryHdr( unsigned char* blockBuf, const int& size);
     void         insertSgnture(unsigned char* blockBuf,
                                const int& size, unsigned char*value);