diff --git a/dbcon/joblist/columncommand-jl.cpp b/dbcon/joblist/columncommand-jl.cpp index 63a57743c..10d4ea21e 100644 --- a/dbcon/joblist/columncommand-jl.cpp +++ b/dbcon/joblist/columncommand-jl.cpp @@ -160,6 +160,7 @@ void ColumnCommandJL::createCommand(ByteStream& bs) const bs << (uint8_t) colType.colWidth; bs << (uint8_t) colType.scale; bs << (uint8_t) colType.compressionType; + bs << (uint32_t) colType.charsetNumber; bs << BOP; bs << filterCount; serializeInlineVector(bs, fLastLbid); diff --git a/dbcon/joblist/lbidlist.cpp b/dbcon/joblist/lbidlist.cpp index 60180c8f8..03ab1fed7 100644 --- a/dbcon/joblist/lbidlist.cpp +++ b/dbcon/joblist/lbidlist.cpp @@ -658,6 +658,48 @@ inline bool LBIDList::compareVal(const T& Min, const T& Max, const T& value, cha return true; } + +static inline bool compareStr(const datatypes::Charset &cs, + const utils::ConstString &Min, + const utils::ConstString &Max, + const utils::ConstString &value, + char op, uint8_t lcf) +{ + switch (op) + { + case COMPARE_LT: + case COMPARE_NGE: + return cs.strnncollsp(value, Min) > 0; + + case COMPARE_LE: + case COMPARE_NGT: + return cs.strnncollsp(value, Min) >= 0; + + case COMPARE_GT: + case COMPARE_NLE: + return cs.strnncollsp(value, Max) < 0; + + case COMPARE_GE: + case COMPARE_NLT: + return cs.strnncollsp(value, Max) <= 0; + + case COMPARE_EQ: + return cs.strnncollsp(value, Min) >= 0 && + cs.strnncollsp(value, Max) <= 0 && + lcf <= 0; + + case COMPARE_NE: + + // @bug 3087 + return cs.strnncollsp(value, Min) != 0 || + cs.strnncollsp(value, Max) != 0 || + lcf != 0; + } + + return false; +} + + template bool LBIDList::checkSingleValue(T min, T max, T value, execplan::CalpontSystemCatalog::ColDataType type) @@ -824,15 +866,13 @@ bool LBIDList::CasualPartitionPredicate(const BRM::EMCasualPartition_t& cpRange, if (bIsChar && 1 < ct.colWidth) { - // MCOL-1246 Trim trailing whitespace for matching so that we have - // the same as InnoDB behaviour - int64_t tMin = cpRange.loVal; - int64_t tMax = cpRange.hiVal; - dataconvert::DataConvert::trimWhitespace(tMin); - dataconvert::DataConvert::trimWhitespace(tMax); - - scan = compareVal(order_swap(tMin), order_swap(tMax), order_swap(value), - op, lcf); + datatypes::Charset cs(ct.charsetNumber); + utils::ConstString sMin((const char *) &cpRange.loVal, 8); + utils::ConstString sMax((const char *) &cpRange.hiVal, 8); + utils::ConstString sVal((const char *) &value, 8); + scan = compareStr(cs, sMin.rtrimZero(), + sMax.rtrimZero(), + sVal.rtrimZero(), op, lcf); // cout << "scan=" << (uint32_t) scan << endl; } else if (bIsUnsigned) diff --git a/dbcon/joblist/pcolscan.cpp b/dbcon/joblist/pcolscan.cpp index d7453cca9..3e5fcc540 100644 --- a/dbcon/joblist/pcolscan.cpp +++ b/dbcon/joblist/pcolscan.cpp @@ -123,6 +123,7 @@ pColScanStep::pColScanStep( const JobInfo& jobInfo) : JobStep(jobInfo), fRm(jobInfo.rm), + fMsgHeader(), fNumThreads(fRm->getJlNumScanReceiveThreads()), fFilterCount(0), fOid(o), @@ -154,8 +155,6 @@ pColScanStep::pColScanStep( rDoNothing = false; fIsDict = false; - memset(&fMsgHeader, 0, sizeof(fMsgHeader)); - //If this is a dictionary column, fudge the numbers... if ( fColType.colDataType == CalpontSystemCatalog::VARCHAR ) { @@ -1008,7 +1007,8 @@ uint64_t pColScanStep::getFBO(uint64_t lbid) pColScanStep::pColScanStep(const pColStep& rhs) : JobStep(rhs), - fRm(rhs.resourceManager()) + fRm(rhs.resourceManager()), + fMsgHeader() { fNumThreads = fRm->getJlNumScanReceiveThreads(); fFilterCount = rhs.filterCount(); @@ -1040,8 +1040,6 @@ pColScanStep::pColScanStep(const pColStep& rhs) : int err; - memset(&fMsgHeader, 0, sizeof(fMsgHeader)); - err = dbrm.lookup(fOid, lbidRanges); if (err) diff --git a/dbcon/joblist/primitivemsg.h b/dbcon/joblist/primitivemsg.h index 7660f38ef..6e78a9092 100644 --- a/dbcon/joblist/primitivemsg.h +++ b/dbcon/joblist/primitivemsg.h @@ -283,6 +283,27 @@ struct ColLoopback PrimitiveHeader Hdr; // 64 bit header }; + +struct ColRequestHeaderDataType: public datatypes::Charset +{ + int32_t CompType; + uint16_t DataSize; + uint8_t DataType; // enum ColDataType defined in calpont system catalog header file + ColRequestHeaderDataType() + :Charset(my_charset_bin), + CompType(0), + DataSize(0), + DataType(0) + { } + ColRequestHeaderDataType(const execplan::CalpontSystemCatalog::ColType &rhs) + :Charset(rhs.charsetNumber), + CompType(rhs.compressionType), + DataSize(rhs.colWidth), + DataType(rhs.colDataType) + { } +}; + + // COL_BY_SCAN //Tied to ColByScanRangeRequestHeader and NewColRequestHeader. Check other headers if modifying. @@ -290,15 +311,19 @@ struct ColByScanRequestHeader { PrimitiveHeader Hdr; // 64 bit header uint64_t LBID; - int32_t CompType; - uint16_t DataSize; - uint8_t DataType; // enum ColDataType defined in calpont system catalog header file + ColRequestHeaderDataType colType; uint8_t OutputType; // 1 = RID, 2 = Token, 3 = Both uint8_t BOP; // 0 = N/A, 1 = AND, 2 = OR uint8_t RidFlags; // a bitmap indicating the rid ranges in the resultM SB => row 7168-8191 uint16_t NOPS; uint16_t NVALS; uint8_t sort; + ColByScanRequestHeader() + :LBID(0), OutputType(0), BOP(0), + RidFlags(0), NOPS(0), NVALS(0), sort(0) + { + memset(&Hdr, 0, sizeof(Hdr)); + } }; // COL_BY_SCAN_RANGE @@ -308,9 +333,7 @@ struct ColByScanRangeRequestHeader { PrimitiveHeader Hdr; // 64 bit header uint64_t LBID; // starting LBID - int32_t CompType; - uint16_t DataSize; - uint8_t DataType; // enum ColDataType defined in calpont system catalog header file + ColRequestHeaderDataType colType; uint8_t OutputType; // 1 = RID, 2 = Token, 3 = Both uint8_t BOP; // 0 = N/A, 1 = AND, 2 = OR uint8_t RidFlags; // a bitmap indicating the rid ranges in the result MSB => row 7168-8191 @@ -318,6 +341,13 @@ struct ColByScanRangeRequestHeader uint16_t NVALS; uint8_t sort; uint16_t Count; //Number of LBID's + ColByScanRangeRequestHeader() + :LBID(0), OutputType(0), + BOP(0), RidFlags(0), NOPS(0), NVALS(0), + sort(0), Count(0) + { + memset(&Hdr, 0, sizeof(Hdr)); + } }; // COL_BY_RID @@ -326,15 +356,14 @@ struct ColByRIDRequestHeader { PrimitiveHeader Hdr; // 64 bit header uint64_t LBID; - int32_t CompType; - uint16_t DataSize; - uint8_t DataType; // enum ColDataType defined in calpont system catalog header file + ColRequestHeaderDataType colType; uint8_t OutputType; // 1 = RID, 2 = Token, 3 = Both uint8_t BOP; // 0 = N/A, 1 = AND, 2 = OR uint8_t InputFlags; // 1 = interpret each NOP & RID as a pair uint16_t NOPS; uint16_t NVALS; uint8_t sort; + ColByRIDRequestHeader(); // QQ? Not used? }; // COL_AGG_BY_SCAN @@ -358,14 +387,13 @@ struct ColAggByRIDRequestHeader { PrimitiveHeader Hdr; // 64 bit header uint64_t LBID; - int32_t CompType; - uint16_t DataSize; - uint8_t DataType; // enum ColDataType defined in calpont system catalog header file + ColRequestHeaderDataType colType; uint8_t OutputType; // 1 = RID, 2 = Token, 3 = Both uint8_t BOP; // 0 = N/A, 1 = AND, 2 = OR uint8_t ExtraNotUsed; uint16_t NOPS; uint16_t NVALS; + ColAggByRIDRequestHeader(); // Not used? }; // Loopback Results @@ -663,9 +691,7 @@ struct NewColRequestHeader ISMPacketHeader ism; PrimitiveHeader hdr; uint64_t LBID; - int32_t CompType; - uint16_t DataSize; - uint8_t DataType; + ColRequestHeaderDataType colType; uint8_t OutputType; // OT_DATAVALUE, OT_RID, or OT_BOTH uint8_t BOP; // uint8_t InputFlags; // 1 = interpret each NOP & RID as a pair (deprecated) @@ -676,6 +702,10 @@ struct NewColRequestHeader // this follows the header // ColArgs ArgList[NOPS] (where the val field is DataSize bytes long) // uint16_t Rids[NVALS] (each rid is relative to the given block) + + // QQ: The below constructor is never used. + // This struct is used in a cast only, in a hackish way. + NewColRequestHeader(); }; struct NewColAggRequestHeader @@ -683,9 +713,7 @@ struct NewColAggRequestHeader ISMPacketHeader ism; PrimitiveHeader hdr; uint64_t LBID; - int32_t CompType; - uint8_t DataSize; - uint8_t DataType; + ColRequestHeaderDataType colType; uint8_t OutputType; uint8_t BOP; uint8_t ExtraNotUsed; @@ -694,6 +722,7 @@ struct NewColAggRequestHeader // this follows the header // ColArgs ArgList[NOPS] (where the val field is DataSize bytes long) // uint16_t Rids[NVALS] (each rid is relative to the given block) + NewColAggRequestHeader(); // QQ: not used }; struct NewColResultHeader diff --git a/primitives/linux-port/column.cpp b/primitives/linux-port/column.cpp index bd7b0527e..d62e5f7fb 100644 --- a/primitives/linux-port/column.cpp +++ b/primitives/linux-port/column.cpp @@ -140,6 +140,44 @@ inline bool colCompare_(const T& val1, const T& val2, uint8_t COP) } } + +inline bool colCompareStr(const ColRequestHeaderDataType &type, + uint8_t COP, + const utils::ConstString &val1, + const utils::ConstString &val2) +{ + int res = type.strnncollsp(val1, val2); + + switch (COP) + { + case COMPARE_NIL: + return false; + + case COMPARE_LT: + return res < 0; + + case COMPARE_EQ: + return res == 0; + + case COMPARE_LE: + return res <= 0; + + case COMPARE_GT: + return res > 0; + + case COMPARE_NE: + return res != 0; + + case COMPARE_GE: + return res >= 0; + + default: + logIt(34, COP, "colCompareStr"); + return false; // throw an exception here? + } +} + + template inline bool colCompare_(const T& val1, const T& val2, uint8_t COP, uint8_t rf) { @@ -589,15 +627,15 @@ inline bool isMinMaxValid(const NewColRequestHeader* in) } else { - switch (in->DataType) + switch (in->colType.DataType) { case CalpontSystemCatalog::CHAR: - return (in->DataSize < 9); + return (in->colType.DataSize < 9); case CalpontSystemCatalog::VARCHAR: case CalpontSystemCatalog::BLOB: case CalpontSystemCatalog::TEXT: - return (in->DataSize < 8); + return (in->colType.DataSize < 8); case CalpontSystemCatalog::TINYINT: case CalpontSystemCatalog::SMALLINT: @@ -617,7 +655,7 @@ inline bool isMinMaxValid(const NewColRequestHeader* in) case CalpontSystemCatalog::DECIMAL: case CalpontSystemCatalog::UDECIMAL: - return (in->DataSize <= datatypes::MAXDECIMALWIDTH); + return (in->colType.DataSize <= datatypes::MAXDECIMALWIDTH); default: return false; @@ -636,8 +674,13 @@ inline string fixChar(int64_t intval) return string(chval); } -inline bool colCompare(int64_t val1, int64_t val2, uint8_t COP, uint8_t rf, int type, uint8_t width, const idb_regex_t& regex, bool isNull = false) +inline bool colCompare(int64_t val1, int64_t val2, uint8_t COP, uint8_t rf, + const ColRequestHeaderDataType &typeHolder, uint8_t width, + const idb_regex_t& regex, bool isNull = false) { + uint8_t type = typeHolder.DataType; +// cout << "comparing " << hex << val1 << " to " << val2 << endl; + if (COMPARE_NIL == COP) return false; //@bug 425 added isNull condition @@ -664,11 +707,9 @@ inline bool colCompare(int64_t val1, int64_t val2, uint8_t COP, uint8_t rf, int { if (!regex.used && !rf) { - // MCOL-1246 Trim trailing whitespace for matching, but not for - // regex - dataconvert::DataConvert::trimWhitespace(val1); - dataconvert::DataConvert::trimWhitespace(val2); - return colCompare_(order_swap(val1), order_swap(val2), COP); + utils::ConstString s1 = {reinterpret_cast(&val1), 8}; + utils::ConstString s2 = {reinterpret_cast(&val2), 8}; + return colCompareStr(typeHolder, COP, s1.rtrimZero(), s2.rtrimZero()); } else return colStrCompare_(order_swap(val1), order_swap(val2), COP, rf, ®ex); @@ -742,7 +783,7 @@ inline void store(const NewColRequestHeader* in, { #ifdef PRIM_DEBUG - if (*written + in->DataSize > outSize) + if (*written + in->colType.DataSize > outSize) { logIt(35, 2); throw logic_error("PrimitiveProcessor::store(): output buffer is too small"); @@ -753,7 +794,7 @@ inline void store(const NewColRequestHeader* in, void* ptr1 = &out8[*written]; const uint8_t* ptr2 = &block8[0]; - switch (in->DataSize) + switch (in->colType.DataSize) { case 32: std::cout << __func__ << " WARNING!!! Not implemented for 32 byte data types." << std::endl; @@ -788,7 +829,8 @@ inline void store(const NewColRequestHeader* in, memcpy(ptr1, ptr2, 1); break; } - *written += in->DataSize; + + *written += in->colType.DataSize; } out->NVALS++; @@ -1146,7 +1188,7 @@ inline void p_Col_noprid(const NewColRequestHeader* in, NewColResultHeader* out, rid = *reinterpret_cast(&in8[argOffset + sizeof(ColArgs) + in->DataSize]); - if (isUnsigned((CalpontSystemCatalog::ColDataType)in->DataType)) + if (isUnsigned((CalpontSystemCatalog::ColDataType)in->colType.DataType)) { switch (in->DataSize) { @@ -1178,7 +1220,7 @@ inline void p_Col_noprid(const NewColRequestHeader* in, NewColResultHeader* out, return; } - if (colCompare(ucolVal, uargVal, args->COP, args->rf, in->DataType, in->DataSize, placeholderRegex)) + if (colCompare(ucolVal, uargVal, args->COP, args->rf, in->colType.DataType, in->DataSize, placeholderRegex)) store(in, out, outSize, written, rid, reinterpret_cast(block)); } else @@ -1213,7 +1255,7 @@ inline void p_Col_noprid(const NewColRequestHeader* in, NewColResultHeader* out, return; } - if (colCompare(colVal, argVal, args->COP, args->rf, in->DataType, in->DataSize, placeholderRegex)) + if (colCompare(colVal, argVal, args->COP, args->rf, in->colType.DataType, in->DataSize, placeholderRegex)) store(in, out, outSize, written, rid, reinterpret_cast(block)); } } @@ -1255,7 +1297,7 @@ inline void p_Col_ridArray(NewColRequestHeader* in, if (out->ValidMinMax) { - if (isUnsigned((CalpontSystemCatalog::ColDataType)in->DataType)) + if (isUnsigned((CalpontSystemCatalog::ColDataType)in->colType.DataType)) { out->Min = static_cast(numeric_limits::max()); out->Max = 0; @@ -1298,7 +1340,7 @@ inline void p_Col_ridArray(NewColRequestHeader* in, std_regex.reset(new idb_regex_t[in->NOPS]); regex = &(std_regex[0]); - if (isUnsigned((CalpontSystemCatalog::ColDataType)in->DataType)) + if (isUnsigned((CalpontSystemCatalog::ColDataType)in->colType.DataType)) { uargVals = reinterpret_cast(std_argVals); cops = std_cops; @@ -1358,7 +1400,7 @@ inline void p_Col_ridArray(NewColRequestHeader* in, case 4: #if 0 - if (in->DataType == CalpontSystemCatalog::FLOAT) + if (in->colType.DataType == CalpontSystemCatalog::FLOAT) { double dTmp; @@ -1409,14 +1451,14 @@ inline void p_Col_ridArray(NewColRequestHeader* in, // else we have a pre-parsed filter, and it's an unordered set for quick == comparisons - if (isUnsigned((CalpontSystemCatalog::ColDataType)in->DataType)) + if (isUnsigned((CalpontSystemCatalog::ColDataType)in->colType.DataType)) { - uval = nextUnsignedColValue(in->DataType, ridArray, in->NVALS, &nextRidIndex, &done, &isNull, + uval = nextUnsignedColValue(in->colType.DataType, ridArray, in->NVALS, &nextRidIndex, &done, &isNull, &isEmpty, &rid, in->OutputType, reinterpret_cast(block), itemsPerBlk); } else { - val = nextColValue(in->DataType, ridArray, in->NVALS, &nextRidIndex, &done, &isNull, + val = nextColValue(in->colType.DataType, ridArray, in->NVALS, &nextRidIndex, &done, &isNull, &isEmpty, &rid, in->OutputType, reinterpret_cast(block), itemsPerBlk); } @@ -1427,7 +1469,7 @@ inline void p_Col_ridArray(NewColRequestHeader* in, /* bug 1920: ignore NULLs in the set and in the column data */ if (!(isNull && in->BOP == BOP_AND)) { - if (isUnsigned((CalpontSystemCatalog::ColDataType)in->DataType)) + if (isUnsigned((CalpontSystemCatalog::ColDataType)in->colType.DataType)) { it = parsedColumnFilter->prestored_set->find(*reinterpret_cast(&uval)); } @@ -1458,15 +1500,15 @@ inline void p_Col_ridArray(NewColRequestHeader* in, { for (argIndex = 0; argIndex < in->NOPS; argIndex++) { - if (isUnsigned((CalpontSystemCatalog::ColDataType)in->DataType)) + if (isUnsigned((CalpontSystemCatalog::ColDataType)in->colType.DataType)) { cmp = colCompareUnsigned(uval, uargVals[argIndex], cops[argIndex], - rfs[argIndex], in->DataType, W, regex[argIndex], isNull); + rfs[argIndex], in->colType.DataType, W, regex[argIndex], isNull); } else { cmp = colCompare(val, argVals[argIndex], cops[argIndex], - rfs[argIndex], in->DataType, W, regex[argIndex], isNull); + rfs[argIndex], in->colType, W, regex[argIndex], isNull); } if (in->NOPS == 1) @@ -1499,16 +1541,18 @@ inline void p_Col_ridArray(NewColRequestHeader* in, if (out->ValidMinMax && !isNull && !isEmpty) { - if ((in->DataType == CalpontSystemCatalog::CHAR || in->DataType == CalpontSystemCatalog::VARCHAR || - in->DataType == CalpontSystemCatalog::BLOB || in->DataType == CalpontSystemCatalog::TEXT ) && 1 < W) + if ((in->colType.DataType == CalpontSystemCatalog::CHAR || + in->colType.DataType == CalpontSystemCatalog::VARCHAR || + in->colType.DataType == CalpontSystemCatalog::BLOB || + in->colType.DataType == CalpontSystemCatalog::TEXT ) && 1 < W) { - if (colCompare(out->Min, val, COMPARE_GT, false, in->DataType, W, placeholderRegex)) + if (colCompare(out->Min, val, COMPARE_GT, false, in->colType, W, placeholderRegex)) out->Min = val; - if (colCompare(out->Max, val, COMPARE_LT, false, in->DataType, W, placeholderRegex)) + if (colCompare(out->Max, val, COMPARE_LT, false, in->colType, W, placeholderRegex)) out->Max = val; } - else if (isUnsigned((CalpontSystemCatalog::ColDataType)in->DataType)) + else if (isUnsigned((CalpontSystemCatalog::ColDataType)in->colType.DataType)) { if (static_cast(out->Min) > uval) out->Min = static_cast(uval); @@ -1526,15 +1570,15 @@ inline void p_Col_ridArray(NewColRequestHeader* in, } } - if (isUnsigned((CalpontSystemCatalog::ColDataType)in->DataType)) + if (isUnsigned((CalpontSystemCatalog::ColDataType)in->colType.DataType)) { - uval = nextUnsignedColValue(in->DataType, ridArray, in->NVALS, &nextRidIndex, &done, + uval = nextUnsignedColValue(in->colType.DataType, ridArray, in->NVALS, &nextRidIndex, &done, &isNull, &isEmpty, &rid, in->OutputType, reinterpret_cast(block), itemsPerBlk); } else { - val = nextColValue(in->DataType, ridArray, in->NVALS, &nextRidIndex, &done, + val = nextColValue(in->colType.DataType, ridArray, in->NVALS, &nextRidIndex, &done, &isNull, &isEmpty, &rid, in->OutputType, reinterpret_cast(block), itemsPerBlk); } @@ -1589,7 +1633,7 @@ inline void p_Col_bin_ridArray(NewColRequestHeader* in, if (out->ValidMinMax) { // Assume that isUnsigned returns true for 8-bytes DTs only - if (isUnsigned((CalpontSystemCatalog::ColDataType)in->DataType)) + if (isUnsigned((CalpontSystemCatalog::ColDataType)in->colType.DataType)) { out->Min = -1; out->Max = 0; @@ -1653,7 +1697,7 @@ inline void p_Col_bin_ridArray(NewColRequestHeader* in, // else we have a pre-parsed filter, and it's an unordered set for quick == comparisons - bval = (binWtype*)nextBinColValue(in->DataType, ridArray, in->NVALS, &nextRidIndex, &done, &isNull, + bval = (binWtype*)nextBinColValue(in->colType.DataType, ridArray, in->NVALS, &nextRidIndex, &done, &isNull, &isEmpty, &rid, in->OutputType, reinterpret_cast(block), itemsPerBlk); T val; @@ -1696,7 +1740,7 @@ inline void p_Col_bin_ridArray(NewColRequestHeader* in, T filterVal = *reinterpret_cast(argVals[argIndex]); cmp = colCompare(val, filterVal, cops[argIndex], - rfs[argIndex], in->DataType, W, isNull); + rfs[argIndex], in->colType.DataType, W, isNull); if (in->NOPS == 1) { @@ -1727,15 +1771,16 @@ inline void p_Col_bin_ridArray(NewColRequestHeader* in, if (out->ValidMinMax && !isNull && !isEmpty) { - if (in->DataType == CalpontSystemCatalog::CHAR || in->DataType == CalpontSystemCatalog::VARCHAR) + if (in->colType.DataType == CalpontSystemCatalog::CHAR || + in->colType.DataType == CalpontSystemCatalog::VARCHAR) { // !!! colCompare is overloaded with int128_t only yet. - if (colCompare(out->Min, val, COMPARE_GT, false, in->DataType, W, placeholderRegex)) + if (colCompare(out->Min, val, COMPARE_GT, false, in->colType, W, placeholderRegex)) { out->Min = val; } - if (colCompare(out->Max, val, COMPARE_LT, false, in->DataType, W, placeholderRegex)) + if (colCompare(out->Max, val, COMPARE_LT, false, in->colType, W, placeholderRegex)) { out->Max = val; } @@ -1754,7 +1799,7 @@ inline void p_Col_bin_ridArray(NewColRequestHeader* in, } } - bval = (binWtype*)nextBinColValue(in->DataType, ridArray, in->NVALS, &nextRidIndex, &done, &isNull, + bval = (binWtype*)nextBinColValue(in->colType.DataType, ridArray, in->NVALS, &nextRidIndex, &done, &isNull, &isEmpty, &rid, in->OutputType, reinterpret_cast(block), itemsPerBlk); } @@ -1789,7 +1834,7 @@ void PrimitiveProcessor::p_Col(NewColRequestHeader* in, NewColResultHeader* out, if (logicalBlockMode) itemsPerBlk = BLOCK_SIZE; else - itemsPerBlk = BLOCK_SIZE / in->DataSize; + itemsPerBlk = BLOCK_SIZE / in->colType.DataSize; //...Initialize I/O counts; out->CacheIO = 0; @@ -1816,7 +1861,7 @@ void PrimitiveProcessor::p_Col(NewColRequestHeader* in, NewColResultHeader* out, fStatsPtr->markEvent(in->LBID, pthread_self(), in->hdr.SessionID, 'B'); #endif - switch (in->DataSize) + switch (in->colType.DataSize) { case 8: p_Col_ridArray<8>(in, out, outSize, written, block, fStatsPtr, itemsPerBlk, parsedColumnFilter); diff --git a/primitives/primproc/columncommand.cpp b/primitives/primproc/columncommand.cpp index ab6fbe223..be5d91a99 100644 --- a/primitives/primproc/columncommand.cpp +++ b/primitives/primproc/columncommand.cpp @@ -298,7 +298,7 @@ void ColumnCommand::issuePrimitive() { ostringstream os; os << " WARNING!!! Not implemented for "; - os << primMsg->DataSize << " column."; + os << primMsg->colType.DataSize << " column."; throw PrimitiveColumnProjectResultExcept(os.str()); } } @@ -497,6 +497,7 @@ void ColumnCommand::processResult() void ColumnCommand::createCommand(ByteStream& bs) { uint8_t tmp8; + uint32_t tmp32; bs.advance(1); bs >> tmp8; @@ -519,6 +520,8 @@ void ColumnCommand::createCommand(ByteStream& bs) colType.scale = tmp8; bs >> tmp8; colType.compressionType = tmp8; + bs >> tmp32; + colType.charsetNumber = tmp32; bs >> BOP; bs >> filterCount; deserializeInlineVector(bs, lastLbid); @@ -576,9 +579,7 @@ void ColumnCommand::prep(int8_t outputType, bool absRids) primMsg->hdr.TransactionID = bpp->txnID; primMsg->hdr.VerID = bpp->versionInfo.currentScn; primMsg->hdr.StepID = bpp->stepID; - primMsg->DataSize = colType.colWidth; - primMsg->DataType = colType.colDataType; - primMsg->CompType = colType.compressionType; + primMsg->colType = ColRequestHeaderDataType(colType); primMsg->OutputType = outputType; primMsg->BOP = BOP; primMsg->NOPS = (suppressFilter ? 0 : filterCount); diff --git a/utils/common/collation.h b/utils/common/collation.h index e007a6407..d26129933 100644 --- a/utils/common/collation.h +++ b/utils/common/collation.h @@ -19,6 +19,7 @@ #define COLLATION_H_INCLUDED #include "exceptclasses.h" +#include "conststring.h" /* Redefine definitions used by MariaDB m_ctype.h. @@ -92,9 +93,9 @@ public: MariaDBHasher() :mPart1(1), mPart2(4) { } - MariaDBHasher & add(CHARSET_INFO & cs, const char *str, size_t length) + MariaDBHasher & add(CHARSET_INFO * cs, const char *str, size_t length) { - cs.hash_sort((const uchar *) str, length, &mPart1, &mPart2); + cs->hash_sort((const uchar *) str, length, &mPart1, &mPart2); return *this; } uint32_t finalize() const @@ -109,21 +110,33 @@ public: class Charset { protected: - const struct charset_info_st & mCharset; + const struct charset_info_st * mCharset; public: - Charset(CHARSET_INFO & cs) :mCharset(cs) { } + Charset(CHARSET_INFO & cs) :mCharset(&cs) { } Charset(uint32_t charsetNumber); - CHARSET_INFO & getCharset() const { return mCharset; } + CHARSET_INFO & getCharset() const { return *mCharset; } uint32_t hash(const char *data, uint64_t len) const { return MariaDBHasher().add(mCharset, data, len).finalize(); } bool eq(const std::string & str1, const std::string & str2) const { - return mCharset.strnncollsp(str1.data(), str1.length(), - str2.data(), str2.length()) == 0; + return mCharset->strnncollsp(str1.data(), str1.length(), + str2.data(), str2.length()) == 0; } + int strnncollsp(const utils::ConstString &str1, + const utils::ConstString &str2) const + { + return mCharset->strnncollsp(str1.str(), str1.length(), + str2.str(), str2.length()); + } + bool test_if_important_data(const char *str, const char *end) const + { + if (mCharset->state & MY_CS_NOPAD) + return str < end; + return str + mCharset->scan(str, end, MY_SEQ_SPACES) < end; + } }; diff --git a/utils/common/conststring.h b/utils/common/conststring.h new file mode 100644 index 000000000..67415882a --- /dev/null +++ b/utils/common/conststring.h @@ -0,0 +1,47 @@ +/* Copyright (C) 2020 MariaDB Corporation. + + This program is free software; you can redistribute it and/or + modify it under the terms of the GNU General Public License + as published by the Free Software Foundation; version 2 of + the License. + + This program is distributed in the hope that it will be useful, + but WITHOUT ANY WARRANTY; without even the implied warranty of + MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the + GNU General Public License for more details. + + You should have received a copy of the GNU General Public License + along with this program; if not, write to the Free Software + Foundation, Inc., 51 Franklin Street, Fifth Floor, Boston, + MA 02110-1301, USA. */ + + +#ifndef MARIADB_CONSTSTRING_H +#define MARIADB_CONSTSTRING_H + + +namespace utils +{ + +class ConstString +{ + const char *mStr; + size_t mLength; +public: + ConstString(const char *str, size_t length) + :mStr(str), mLength(length) + { } + const char *str() const { return mStr; } + size_t length() const { return mLength; } + ConstString & rtrimZero() + { + for ( ; mLength && mStr[mLength - 1] == '\0'; mLength--) + { } + return *this; + } +}; + + +} // namespace utils + +#endif // MARIADB_CONSTSTRING_H diff --git a/utils/common/utils_utf8.cpp b/utils/common/utils_utf8.cpp index 0d053dcf1..7ed4003aa 100644 --- a/utils/common/utils_utf8.cpp +++ b/utils/common/utils_utf8.cpp @@ -31,7 +31,7 @@ static inline CHARSET_INFO & get_charset_or_bin(int32_t charsetNumber) Charset::Charset(uint32_t charsetNumber) - :mCharset(get_charset_or_bin(charsetNumber)) + :mCharset(&get_charset_or_bin(charsetNumber)) { } diff --git a/utils/dataconvert/dataconvert.cpp b/utils/dataconvert/dataconvert.cpp index 11c40c221..c0bcc8e90 100644 --- a/utils/dataconvert/dataconvert.cpp +++ b/utils/dataconvert/dataconvert.cpp @@ -1542,8 +1542,14 @@ DataConvert::StringToString(const datatypes::SystemCatalog::TypeAttributesStd& c //check data length if ( data.length() > (unsigned int)colType.colWidth ) { + // TODO: charsetNumber should be moved to TypeStdAttributes ASAP + const execplan::CalpontSystemCatalog::ColType &colType2= + static_cast(colType); + datatypes::Charset cs(colType2.charsetNumber); + const char *newEnd = data.data() + colType.colWidth; + const char *origEnd = data.data() + data.length(); + pushWarning = cs.test_if_important_data(newEnd, origEnd); data = data.substr(0, colType.colWidth); - pushWarning = true; boost::any value = data; return value; }