diff --git a/dbcon/execplan/calpontsystemcatalog.h b/dbcon/execplan/calpontsystemcatalog.h index 18164497c..3bb912de5 100644 --- a/dbcon/execplan/calpontsystemcatalog.h +++ b/dbcon/execplan/calpontsystemcatalog.h @@ -905,7 +905,8 @@ const CalpontSystemCatalog::TableAliasName make_aliasview(const std::string& s, inline bool isCharType(const execplan::CalpontSystemCatalog::ColDataType type) { return (execplan::CalpontSystemCatalog::VARCHAR == type || - execplan::CalpontSystemCatalog::CHAR == type); + execplan::CalpontSystemCatalog::CHAR == type || + execplan::CalpontSystemCatalog::BLOB == type); } /** convenience function to determine if column type is an diff --git a/dbcon/execplan/simplecolumn.cpp b/dbcon/execplan/simplecolumn.cpp index bfbf96a2c..3c641804c 100644 --- a/dbcon/execplan/simplecolumn.cpp +++ b/dbcon/execplan/simplecolumn.cpp @@ -583,6 +583,7 @@ void SimpleColumn::evaluate(Row& row, bool& isNull) break; } case CalpontSystemCatalog::VARBINARY: + case CalpontSystemCatalog::BLOB: { fResult.strVal = row.getVarBinaryStringField(fInputIndex); break; diff --git a/dbcon/execplan/simplefilter.cpp b/dbcon/execplan/simplefilter.cpp index 692da7513..55b22cdb2 100644 --- a/dbcon/execplan/simplefilter.cpp +++ b/dbcon/execplan/simplefilter.cpp @@ -176,6 +176,7 @@ const string SimpleFilter::data() const if (dynamic_cast(fRhs) && (fRhs->resultType().colDataType == CalpontSystemCatalog::VARCHAR || fRhs->resultType().colDataType == CalpontSystemCatalog::CHAR || + fRhs->resultType().colDataType == CalpontSystemCatalog::BLOB || fRhs->resultType().colDataType == CalpontSystemCatalog::VARBINARY || fRhs->resultType().colDataType == CalpontSystemCatalog::DATE || fRhs->resultType().colDataType == CalpontSystemCatalog::DATETIME)) @@ -185,6 +186,7 @@ const string SimpleFilter::data() const if (dynamic_cast(fLhs) && (fLhs->resultType().colDataType == CalpontSystemCatalog::VARCHAR || fLhs->resultType().colDataType == CalpontSystemCatalog::CHAR || + fLhs->resultType().colDataType == CalpontSystemCatalog::BLOB || fLhs->resultType().colDataType == CalpontSystemCatalog::VARBINARY || fLhs->resultType().colDataType == CalpontSystemCatalog::DATE || fLhs->resultType().colDataType == CalpontSystemCatalog::DATETIME)) diff --git a/dbcon/execplan/treenode.h b/dbcon/execplan/treenode.h index db6302530..8fdcc6076 100644 --- a/dbcon/execplan/treenode.h +++ b/dbcon/execplan/treenode.h @@ -394,6 +394,7 @@ inline bool TreeNode::getBoolVal() return (atoi(fResult.strVal.c_str()) != 0); //FIXME: Huh??? case CalpontSystemCatalog::VARBINARY: + case CalpontSystemCatalog::BLOB: if (fResultType.colWidth <= 7) return (atoi((char*)(&fResult.origIntVal)) != 0); return (atoi(fResult.strVal.c_str()) != 0); @@ -440,6 +441,7 @@ inline const std::string& TreeNode::getStrVal() break; //FIXME: ??? case CalpontSystemCatalog::VARBINARY: + case CalpontSystemCatalog::BLOB: if (fResultType.colWidth <= 7) fResult.strVal = (char*)(&fResult.origIntVal); break; @@ -573,6 +575,7 @@ inline int64_t TreeNode::getIntVal() return atoll(fResult.strVal.c_str()); //FIXME: ??? case CalpontSystemCatalog::VARBINARY: + case CalpontSystemCatalog::BLOB: if (fResultType.colWidth <= 7) return fResult.intVal; return atoll(fResult.strVal.c_str()); @@ -656,6 +659,7 @@ inline float TreeNode::getFloatVal() return atof(fResult.strVal.c_str()); //FIXME: ??? case CalpontSystemCatalog::VARBINARY: + case CalpontSystemCatalog::BLOB: if (fResultType.colWidth <= 7) return atof((char*)(&fResult.origIntVal)); return atof(fResult.strVal.c_str()); @@ -703,6 +707,7 @@ inline double TreeNode::getDoubleVal() return strtod(fResult.strVal.c_str(), NULL); //FIXME: ??? case CalpontSystemCatalog::VARBINARY: + case CalpontSystemCatalog::BLOB: if (fResultType.colWidth <= 7) return strtod((char*)(&fResult.origIntVal), NULL); return strtod(fResult.strVal.c_str(), NULL); @@ -746,6 +751,7 @@ inline IDB_Decimal TreeNode::getDecimalVal() case CalpontSystemCatalog::VARCHAR: throw logging::InvalidConversionExcept("TreeNode::getDecimalVal: non-support conversion from string"); case CalpontSystemCatalog::VARBINARY: + case CalpontSystemCatalog::BLOB: throw logging::InvalidConversionExcept("TreeNode::getDecimalVal: non-support conversion from binary string"); case CalpontSystemCatalog::BIGINT: case CalpontSystemCatalog::MEDINT: diff --git a/dbcon/joblist/expressionstep.cpp b/dbcon/joblist/expressionstep.cpp index 96616ee0b..6eae7872a 100644 --- a/dbcon/joblist/expressionstep.cpp +++ b/dbcon/joblist/expressionstep.cpp @@ -324,7 +324,8 @@ void ExpressionStep::addColumn(ReturnedColumn* rc, JobInfo& jobInfo) void ExpressionStep::populateColumnInfo(ReturnedColumn* rc, JobInfo& jobInfo) { // As of bug3695, make sure varbinary is not used in function expression. - if (rc->resultType().colDataType == CalpontSystemCatalog::VARBINARY && !fVarBinOK) + if ((rc->resultType().colDataType == CalpontSystemCatalog::VARBINARY || + rc->resultType().colDataType == CalpontSystemCatalog::BLOB) && !fVarBinOK) throw runtime_error("VARBINARY in filter or function is not supported."); SimpleColumn* sc = dynamic_cast(rc); @@ -344,7 +345,8 @@ void ExpressionStep::populateColumnInfo(ReturnedColumn* rc, JobInfo& jobInfo) void ExpressionStep::populateColumnInfo(SimpleColumn* sc, JobInfo& jobInfo) { // As of bug3695, make sure varbinary is not used in function expression. - if (sc->resultType().colDataType == CalpontSystemCatalog::VARBINARY && !fVarBinOK) + if ((sc->resultType().colDataType == CalpontSystemCatalog::VARBINARY || + sc->resultType().colDataType == CalpontSystemCatalog::BLOB) && !fVarBinOK) throw runtime_error ("VARBINARY in filter or function is not supported."); CalpontSystemCatalog::OID tblOid = joblist::tableOid(sc, jobInfo.csc); @@ -409,7 +411,8 @@ void ExpressionStep::populateColumnInfo(SimpleColumn* sc, JobInfo& jobInfo) void ExpressionStep::populateColumnInfo(WindowFunctionColumn* wc, JobInfo& jobInfo) { // As of bug3695, make sure varbinary is not used in function expression. - if (wc->resultType().colDataType == CalpontSystemCatalog::VARBINARY && !fVarBinOK) + if ((wc->resultType().colDataType == CalpontSystemCatalog::VARBINARY || + wc->resultType().colDataType == CalpontSystemCatalog::BLOB) && !fVarBinOK) throw runtime_error("VARBINARY in filter or function is not supported."); // This is for window function in IN/EXISTS sub-query. @@ -434,7 +437,8 @@ void ExpressionStep::populateColumnInfo(WindowFunctionColumn* wc, JobInfo& jobIn void ExpressionStep::populateColumnInfo(AggregateColumn* ac, JobInfo& jobInfo) { // As of bug3695, make sure varbinary is not used in function expression. - if (ac->resultType().colDataType == CalpontSystemCatalog::VARBINARY && !fVarBinOK) + if ((ac->resultType().colDataType == CalpontSystemCatalog::VARBINARY || + ac->resultType().colDataType == CalpontSystemCatalog::BLOB) && !fVarBinOK) throw runtime_error("VARBINARY in filter or function is not supported."); // This is for aggregate function in IN/EXISTS sub-query. diff --git a/primitives/linux-port/column.cpp b/primitives/linux-port/column.cpp index 7474239fd..7c57d15e9 100644 --- a/primitives/linux-port/column.cpp +++ b/primitives/linux-port/column.cpp @@ -249,6 +249,7 @@ inline bool isEmptyVal<8>(uint8_t type, const uint8_t* ival) case CalpontSystemCatalog::DATE: case CalpontSystemCatalog::DATETIME: case CalpontSystemCatalog::VARBINARY: + case CalpontSystemCatalog::BLOB: return (*val == joblist::CHAR8EMPTYROW); case CalpontSystemCatalog::UBIGINT: return (joblist::UBIGINTEMPTYROW == *val); @@ -271,6 +272,7 @@ inline bool isEmptyVal<4>(uint8_t type, const uint8_t* ival) return (joblist::FLOATEMPTYROW == *val); case CalpontSystemCatalog::CHAR: case CalpontSystemCatalog::VARCHAR: + case CalpontSystemCatalog::BLOB: case CalpontSystemCatalog::DATE: case CalpontSystemCatalog::DATETIME: return (joblist::CHAR4EMPTYROW == *val); @@ -292,6 +294,7 @@ inline bool isEmptyVal<2>(uint8_t type, const uint8_t* ival) { case CalpontSystemCatalog::CHAR: case CalpontSystemCatalog::VARCHAR: + case CalpontSystemCatalog::BLOB: case CalpontSystemCatalog::DATE: case CalpontSystemCatalog::DATETIME: return (joblist::CHAR2EMPTYROW == *val); @@ -313,6 +316,7 @@ inline bool isEmptyVal<1>(uint8_t type, const uint8_t* ival) { case CalpontSystemCatalog::CHAR: case CalpontSystemCatalog::VARCHAR: + case CalpontSystemCatalog::BLOB: case CalpontSystemCatalog::DATE: case CalpontSystemCatalog::DATETIME: return (*val == joblist::CHAR1EMPTYROW); @@ -343,6 +347,7 @@ inline bool isNullVal<8>(uint8_t type, const uint8_t* ival) case CalpontSystemCatalog::DATE: case CalpontSystemCatalog::DATETIME: case CalpontSystemCatalog::VARBINARY: + case CalpontSystemCatalog::BLOB: //@bug 339 might be a token here //TODO: what's up with the second const here? return (*val == joblist::CHAR8NULL || 0xFFFFFFFFFFFFFFFELL == *val); @@ -367,6 +372,7 @@ inline bool isNullVal<4>(uint8_t type, const uint8_t* ival) return (joblist::FLOATNULL == *val); case CalpontSystemCatalog::CHAR: case CalpontSystemCatalog::VARCHAR: + case CalpontSystemCatalog::BLOB: return (joblist::CHAR4NULL == *val); case CalpontSystemCatalog::DATE: case CalpontSystemCatalog::DATETIME: @@ -389,6 +395,7 @@ inline bool isNullVal<2>(uint8_t type, const uint8_t* ival) { case CalpontSystemCatalog::CHAR: case CalpontSystemCatalog::VARCHAR: + case CalpontSystemCatalog::BLOB: case CalpontSystemCatalog::DATE: case CalpontSystemCatalog::DATETIME: return (joblist::CHAR2NULL == *val); @@ -410,6 +417,7 @@ inline bool isNullVal<1>(uint8_t type, const uint8_t* ival) { case CalpontSystemCatalog::CHAR: case CalpontSystemCatalog::VARCHAR: + case CalpontSystemCatalog::BLOB: case CalpontSystemCatalog::DATE: case CalpontSystemCatalog::DATETIME: return (*val == joblist::CHAR1NULL); @@ -451,6 +459,7 @@ inline bool isMinMaxValid(const NewColRequestHeader *in) { case CalpontSystemCatalog::CHAR: return (in->DataSize<9); case CalpontSystemCatalog::VARCHAR: + case CalpontSystemCatalog::BLOB: return (in->DataSize<8); case CalpontSystemCatalog::TINYINT: case CalpontSystemCatalog::SMALLINT: @@ -505,7 +514,7 @@ inline bool colCompare(int64_t val1, int64_t val2, uint8_t COP, uint8_t rf, int return colCompare_(dVal1, dVal2, COP); } - else if ( (type == CalpontSystemCatalog::CHAR || type == CalpontSystemCatalog::VARCHAR) && !isNull ) + else if ( (type == CalpontSystemCatalog::CHAR || type == CalpontSystemCatalog::VARCHAR || type == CalpontSystemCatalog::BLOB) && !isNull ) { if (!regex.used && !rf) return colCompare_(order_swap(val1), order_swap(val2), COP); @@ -1180,7 +1189,7 @@ inline void p_Col_ridArray(NewColRequestHeader *in, if (out->ValidMinMax && !isNull && !isEmpty) { - if ((in->DataType == CalpontSystemCatalog::CHAR || in->DataType == CalpontSystemCatalog::VARCHAR ) && 1 < W) + if ((in->DataType == CalpontSystemCatalog::CHAR || in->DataType == CalpontSystemCatalog::VARCHAR || in->DataType == CalpontSystemCatalog::BLOB ) && 1 < W) { if (colCompare(out->Min, val, COMPARE_GT, false, in->DataType, W, placeholderRegex)) out->Min = val; diff --git a/primitives/primproc/columncommand.cpp b/primitives/primproc/columncommand.cpp index 8cad317b3..891f33373 100644 --- a/primitives/primproc/columncommand.cpp +++ b/primitives/primproc/columncommand.cpp @@ -866,6 +866,7 @@ const uint64_t ColumnCommand::getEmptyRowValue( const execplan::CalpontSystemCat case execplan::CalpontSystemCatalog::DATE : case execplan::CalpontSystemCatalog::DATETIME : case execplan::CalpontSystemCatalog::VARBINARY : + case execplan::CalpontSystemCatalog::BLOB : default: emptyVal = joblist::CHAR1EMPTYROW; if ( width == (2 + offset) ) diff --git a/primitives/primproc/dictstep.cpp b/primitives/primproc/dictstep.cpp index aa8e3614c..295cfa1af 100644 --- a/primitives/primproc/dictstep.cpp +++ b/primitives/primproc/dictstep.cpp @@ -509,7 +509,8 @@ void DictStep::_projectToRG(RowGroup &rg, uint32_t col) // bug 4901 - move this inside the loop and call incrementally // to save the unnecessary string copy - if (rg.getColTypes()[col] != execplan::CalpontSystemCatalog::VARBINARY) { + if ((rg.getColTypes()[col] != execplan::CalpontSystemCatalog::VARBINARY) && + (rg.getColTypes()[col] != execplan::CalpontSystemCatalog::BLOB)) { for (i = curResultCounter; i < tmpResultCounter; i++) { rg.getRow(newRidList[i].pos, &r); //cout << "serializing " << tmpStrings[i] << endl; @@ -517,9 +518,39 @@ void DictStep::_projectToRG(RowGroup &rg, uint32_t col) } } else { - for (i = curResultCounter; i < tmpResultCounter; i++) { + uint32_t firstTmpResultCounter = tmpResultCounter; + for (i = curResultCounter; i < firstTmpResultCounter; i++) { rg.getRow(newRidList[i].pos, &r); - r.setVarBinaryField(tmpStrings[i].ptr, tmpStrings[i].len, col); + // If this is a multi-block blob, get all the blocks + // We do string copy here, should maybe have a RowGroup + // function to append strings or something? + if ((newRidList[i].token != 0xffffffffffffffffLL) && + ((newRidList[i].token >> 46) > 0)) + { + StringPtr multi_part[1]; + uint16_t old_offset = primMsg->tokens[0].offset; + string result((char*)tmpStrings[i].ptr, tmpStrings[i].len); + uint64_t origin_lbid = primMsg->LBID; + uint32_t lbid_count = newRidList[i].token >> 46; + primMsg->tokens[0].offset = 1; // first offset of a sig + for (uint32_t j = 1; j <= lbid_count; j++) + { + tmpResultCounter = 0; + primMsg->LBID = origin_lbid + j; + primMsg->NVALS = 1; + primMsg->tokens[0].LBID = origin_lbid + j; + issuePrimitive(false); + projectResult(multi_part); + result.append((char*)multi_part[0].ptr, multi_part[0].len); + } + primMsg->tokens[0].offset = old_offset; + tmpResultCounter = firstTmpResultCounter; + r.setVarBinaryField((unsigned char*)result.c_str(), result.length(), col); + } + else + { + r.setVarBinaryField(tmpStrings[i].ptr, tmpStrings[i].len, col); + } } } curResultCounter = tmpResultCounter; diff --git a/primitives/primproc/filtercommand.cpp b/primitives/primproc/filtercommand.cpp index 91a89a235..018fc7853 100644 --- a/primitives/primproc/filtercommand.cpp +++ b/primitives/primproc/filtercommand.cpp @@ -94,7 +94,8 @@ Command* FilterCommand::makeFilterCommand(ByteStream& bs, vector& cmds // char[] is stored as int, but cannot directly compare if length is different // due to endian issue if (cmd0->getColType().colDataType == execplan::CalpontSystemCatalog::CHAR || - cmd0->getColType().colDataType == execplan::CalpontSystemCatalog::VARCHAR) + cmd0->getColType().colDataType == execplan::CalpontSystemCatalog::VARCHAR || + cmd0->getColType().colDataType == execplan::CalpontSystemCatalog::BLOB) { StrFilterCmd* sc = new StrFilterCmd(); sc->setCompareFunc(CC); diff --git a/utils/funcexp/func_hex.cpp b/utils/funcexp/func_hex.cpp index 3d6630245..0af09feb4 100644 --- a/utils/funcexp/func_hex.cpp +++ b/utils/funcexp/func_hex.cpp @@ -96,6 +96,7 @@ string Func_hex::getStrVal(rowgroup::Row& row, break; } case CalpontSystemCatalog::VARBINARY: + case CalpontSystemCatalog::BLOB: { const string& arg = parm[0]->data()->getStrVal(row, isNull); uint64_t hexLen = arg.size() * 2; diff --git a/utils/funcexp/func_length.cpp b/utils/funcexp/func_length.cpp index 18a2f7232..25833d0a5 100644 --- a/utils/funcexp/func_length.cpp +++ b/utils/funcexp/func_length.cpp @@ -48,7 +48,8 @@ int64_t Func_length::getIntVal(rowgroup::Row& row, bool& isNull, CalpontSystemCatalog::ColType&) { - if (fp[0]->data()->resultType().colDataType == CalpontSystemCatalog::VARBINARY) + if ((fp[0]->data()->resultType().colDataType == CalpontSystemCatalog::VARBINARY) || + (fp[0]->data()->resultType().colDataType == CalpontSystemCatalog::BLOB)) return fp[0]->data()->getStrVal(row, isNull).length(); return strlen(fp[0]->data()->getStrVal(row, isNull).c_str()); diff --git a/utils/funcexp/funcexp.cpp b/utils/funcexp/funcexp.cpp index 675208f9a..6995cfa36 100644 --- a/utils/funcexp/funcexp.cpp +++ b/utils/funcexp/funcexp.cpp @@ -265,7 +265,9 @@ void FuncExp::evaluate(rowgroup::Row& row, std::vector& expressi break; } case CalpontSystemCatalog::CHAR: - case CalpontSystemCatalog::VARCHAR: + case CalpontSystemCatalog::VARCHAR: + // TODO: might not be right thing for BLOB + case CalpontSystemCatalog::BLOB: { const std::string& val = expression[i]->getStrVal(row, isNull); if (isNull) diff --git a/utils/rowgroup/rowgroup.cpp b/utils/rowgroup/rowgroup.cpp index 4d4d657c1..a6bccca66 100644 --- a/utils/rowgroup/rowgroup.cpp +++ b/utils/rowgroup/rowgroup.cpp @@ -1106,13 +1106,13 @@ void applyMapping(const int *mapping, const Row &in, Row *out) for (i = 0; i < in.getColumnCount(); i++) if (mapping[i] != -1) { - if (UNLIKELY(in.isLongString(i))) + if (UNLIKELY(in.getColTypes()[i] == execplan::CalpontSystemCatalog::VARBINARY || in.getColTypes()[i] == execplan::CalpontSystemCatalog::BLOB)) + out->setVarBinaryField(in.getVarBinaryField(i), in.getVarBinaryLength(i), mapping[i]); + else if (UNLIKELY(in.isLongString(i))) out->setStringField(in.getStringPointer(i), in.getStringLength(i), mapping[i]); //out->setStringField(in.getStringField(i), mapping[i]); else if (UNLIKELY(in.isShortString(i))) out->setUintField(in.getUintField(i), mapping[i]); - else if (UNLIKELY(in.getColTypes()[i] == execplan::CalpontSystemCatalog::VARBINARY || in.getColTypes()[i] == execplan::CalpontSystemCatalog::BLOB)) - out->setVarBinaryField(in.getVarBinaryField(i), in.getVarBinaryLength(i), mapping[i]); else if (UNLIKELY(in.getColTypes()[i] == execplan::CalpontSystemCatalog::LONGDOUBLE)) out->setLongDoubleField(in.getLongDoubleField(i), mapping[i]); else if (in.isUnsigned(i)) diff --git a/utils/rowgroup/rowgroup.h b/utils/rowgroup/rowgroup.h index 79d925a4c..b37a6f5f4 100644 --- a/utils/rowgroup/rowgroup.h +++ b/utils/rowgroup/rowgroup.h @@ -797,7 +797,8 @@ inline void Row::copyField(uint32_t destIndex, uint32_t srcIndex) const inline void Row::copyField(Row &out, uint32_t destIndex, uint32_t srcIndex) const { - if (UNLIKELY(types[srcIndex] == execplan::CalpontSystemCatalog::VARBINARY)) + if (UNLIKELY(types[srcIndex] == execplan::CalpontSystemCatalog::VARBINARY || + types[srcIndex] == execplan::CalpontSystemCatalog::BLOB)) out.setVarBinaryField(getVarBinaryStringField(srcIndex), destIndex); else if (UNLIKELY(isLongString(srcIndex))) out.setStringField(getStringPointer(srcIndex), getStringLength(srcIndex), destIndex); @@ -1268,7 +1269,8 @@ inline bool RowGroup::isLongString(uint32_t colIndex) const { return ((getColumnWidth(colIndex) > 7 && types[colIndex] == execplan::CalpontSystemCatalog::VARCHAR) || (getColumnWidth(colIndex) > 8 && types[colIndex] == execplan::CalpontSystemCatalog::CHAR) || - types[colIndex] == execplan::CalpontSystemCatalog::VARBINARY); + types[colIndex] == execplan::CalpontSystemCatalog::VARBINARY || + types[colIndex] == execplan::CalpontSystemCatalog::BLOB); } inline bool RowGroup::usesStringTable() const diff --git a/writeengine/dictionary/we_dctnry.cpp b/writeengine/dictionary/we_dctnry.cpp index 20917f9f7..3b6b8ee20 100644 --- a/writeengine/dictionary/we_dctnry.cpp +++ b/writeengine/dictionary/we_dctnry.cpp @@ -64,7 +64,7 @@ namespace const int START_HDR1 = // start loc of 2nd offset (HDR1) HDR_UNIT_SIZE + NEXT_PTR_BYTES + HDR_UNIT_SIZE; const int PSEUDO_COL_WIDTH = DICT_COL_WIDTH; // used to convert row count to block count - + const int MAX_BLOB_SIZE = 2100000000; // for safety, we use an 18bit block count of 8KB blocks } namespace WriteEngine @@ -620,17 +620,104 @@ bool Dctnry::getTokenFromArray(Signature& sig) * token - token that was assigned to the inserted signature * * RETURN: - * none + * success - successfully write the signature to the block + * failure - failed to extend/create an extent for the block ******************************************************************************/ -void Dctnry::insertDctnry2(Signature& sig) +int Dctnry::insertDctnry2(Signature& sig) { - insertDctnryHdr(m_curBlock.data, - sig.size); - insertSgnture(m_curBlock.data, sig.size, (unsigned char*)sig.signature); + int rc = 0; + int write_size; + bool lbid_in_token; - sig.token.fbo = m_curLbid; - sig.token.op = m_curOp; - sig.token.bc = 0U; + sig.token.bc = 0; + + while (sig.size > 0) + { + if (sig.size > (m_freeSpace - m_totalHdrBytes)) + { + write_size = (m_freeSpace - m_totalHdrBytes); + } + else + { + write_size = sig.size; + } + + insertDctnryHdr(m_curBlock.data, write_size); + insertSgnture(m_curBlock.data, write_size, (unsigned char*)sig.signature); + + sig.size -= write_size; + sig.signature += write_size; + + if (!lbid_in_token) + { + sig.token.fbo = m_curLbid; + sig.token.op = m_curOp; + lbid_in_token = true; + } + + if (sig.size > 0) + { + CommBlock cb; + cb.file.oid = m_dctnryOID; + cb.file.pFile = m_dFile; + sig.token.bc++; + + RETURN_ON_ERROR( writeDBFile(cb, &m_curBlock, m_curLbid) ); + memset( m_curBlock.data, 0, sizeof(m_curBlock.data)); + memcpy( m_curBlock.data, &m_dctnryHeader2, m_totalHdrBytes); + m_freeSpace = BYTE_PER_BLOCK - m_totalHdrBytes; + m_curBlock.state = BLK_WRITE; + m_curOp =0; + m_lastFbo++; + m_curFbo = m_lastFbo; + + //...Expand current extent if it is an abbreviated initial extent + if ((m_curFbo == m_numBlocks) && + (m_numBlocks == NUM_BLOCKS_PER_INITIAL_EXTENT)) + { + RETURN_ON_ERROR( expandDctnryExtent() ); + } + + //...Allocate a new extent if we have reached the last block in the + // current extent. + if (m_curFbo == m_numBlocks) + {//last block + //for roll back the extent to use + //Save those empty extents in case of failure to rollback + std::vector dictExtentInfo; + ExtentInfo info; + info.oid = m_dctnryOID; + info.partitionNum = m_partition; + info.segmentNum = m_segment; + info.dbRoot = m_dbRoot; + info.hwm = m_hwm; + info.newFile = false; + dictExtentInfo.push_back (info); + LBID_t startLbid; + // Add an extent. + rc = createDctnry(m_dctnryOID, + 0, // dummy column width + m_dbRoot, + m_partition, + m_segment, + startLbid, + false) ; + if ( rc != NO_ERROR ) + { + //roll back the extent + BRMWrapper::getInstance()->deleteEmptyDictStoreExtents( + dictExtentInfo); + return rc; + } + } + RETURN_ON_ERROR( BRMWrapper::getInstance()->getBrmInfo(m_dctnryOID, + m_partition, m_segment, + m_curFbo, m_curLbid) ); + m_curBlock.lbid = m_curLbid; + + } + } + return NO_ERROR; } /******************************************************************************* @@ -705,7 +792,8 @@ int Dctnry::insertDctnry(const char* buf, // it is too late to reject the row. However, as a precaution, we // still check against max size & set to null token if needed. if ((curSig.size == 0) || - (curSig.size == COLPOSPAIR_NULL_TOKEN_OFFSET)) + (curSig.size == COLPOSPAIR_NULL_TOKEN_OFFSET) || + (curSig.size > MAX_BLOB_SIZE)) { if (m_defVal.length() > 0) // use default string if available { @@ -735,6 +823,7 @@ int Dctnry::insertDctnry(const char* buf, } //...Search for the string in our string cache + //if it fits into one block (< 8KB) if ((m_arraySize < MAX_STRING_CACHE_SIZE) && (curSig.size <= MAX_SIGNATURE_SIZE)) { @@ -750,14 +839,15 @@ int Dctnry::insertDctnry(const char* buf, } //Stats::stopParseEvent("getTokenFromArray"); } - totalUseSize = HDR_UNIT_SIZE + curSig.size; + totalUseSize = m_totalHdrBytes + curSig.size; //...String not found in cache, so proceed. // If room is available in current block then insert into block. // @bug 3960: Add MAX_OP_COUNT check to handle case after bulk rollback - if( (totalUseSize <= m_freeSpace) && + if( ((totalUseSize <= m_freeSpace) || + ((curSig.size > 8176) && (m_freeSpace > m_totalHdrBytes))) && (m_curOp < (MAX_OP_COUNT-1)) ) { - insertDctnry2(curSig); //m_freeSpace updated! + RETURN_ON_ERROR(insertDctnry2(curSig)); //m_freeSpace updated! m_curBlock.state = BLK_WRITE; memcpy( pOut + outOffset, &curSig.token, 8 ); outOffset += 8; @@ -880,7 +970,7 @@ int Dctnry::insertDctnry(const char* buf, // we need to add the string to the new block. if (!found) { - insertDctnry2(curSig); //m_freeSpace updated! + RETURN_ON_ERROR(insertDctnry2(curSig)); //m_freeSpace updated! m_curBlock.state = BLK_WRITE; memcpy( pOut + outOffset, &curSig.token, 8 ); outOffset += 8; @@ -949,7 +1039,7 @@ int Dctnry::insertDctnry(const int& sgnature_size, int write_size; bool lbid_in_token = false; // Round down for safety. In theory we can take 262143 * 8176 bytes - if (sgnature_size > (2100000000)) + if (sgnature_size > MAX_BLOB_SIZE) { return ERR_DICT_SIZE_GT_2G; } diff --git a/writeengine/dictionary/we_dctnry.h b/writeengine/dictionary/we_dctnry.h index d3b55896c..9faa1da9e 100644 --- a/writeengine/dictionary/we_dctnry.h +++ b/writeengine/dictionary/we_dctnry.h @@ -239,7 +239,7 @@ protected: // insertDctnryHdr inserts the new value info into the header. // insertSgnture inserts the new value into the block. // - void insertDctnry2(Signature& sig); + int insertDctnry2(Signature& sig); void insertDctnryHdr( unsigned char* blockBuf, const int& size); void insertSgnture(unsigned char* blockBuf, const int& size, unsigned char*value);