database/mariadb-columnstore-engine
1
0
Fork 0
mirror of https://github.com/mariadb-corporation/mariadb-columnstore-engine.git synced 2025-07-30 19:23:07 +03:00
Code Activity

Part#2 MCOL-495 Make string comparison not case sensitive

Browse Source 
Fixing field='str' for short (non-Dict) CHAR and VARCHAR data types.
This commit is contained in:
Alexander Barkov
2020-12-01 12:50:54 +04:00
parent 0ff6a6ec20
commit 52c5af054a
10 changed files with 268 additions and 88 deletions
Download Patch File Download Diff File Expand all files Collapse all files

1
dbcon/joblist/columncommand-jl.cpp
View File

@ -160,6 +160,7 @@ void ColumnCommandJL::createCommand(ByteStream& bs) const
bs << (uint8_t) colType.colWidth;
bs << (uint8_t) colType.scale;
bs << (uint8_t) colType.compressionType;
bs << (uint32_t) colType.charsetNumber;
bs << BOP;
bs << filterCount;
serializeInlineVector(bs, fLastLbid);

58
dbcon/joblist/lbidlist.cpp
View File

@ -658,6 +658,48 @@ inline bool LBIDList::compareVal(const T& Min, const T& Max, const T& value, cha
return true;
}
static inline bool compareStr(const datatypes::Charset &cs,
const utils::ConstString &Min,
const utils::ConstString &Max,
const utils::ConstString &value,
char op, uint8_t lcf)
{
switch (op)
{
case COMPARE_LT:
case COMPARE_NGE:
return cs.strnncollsp(value, Min) > 0;
case COMPARE_LE:
case COMPARE_NGT:
return cs.strnncollsp(value, Min) >= 0;
case COMPARE_GT:
case COMPARE_NLE:
return cs.strnncollsp(value, Max) < 0;
case COMPARE_GE:
case COMPARE_NLT:
return cs.strnncollsp(value, Max) <= 0;
case COMPARE_EQ:
return cs.strnncollsp(value, Min) >= 0 &&
cs.strnncollsp(value, Max) <= 0 &&
lcf <= 0;
case COMPARE_NE:
// @bug 3087
return cs.strnncollsp(value, Min) != 0 ||
cs.strnncollsp(value, Max) != 0 ||
lcf != 0;
}
return false;
}
template<typename T>
bool LBIDList::checkSingleValue(T min, T max, T value,
execplan::CalpontSystemCatalog::ColDataType type)
@ -824,15 +866,13 @@ bool LBIDList::CasualPartitionPredicate(const BRM::EMCasualPartition_t& cpRange,
if (bIsChar && 1 < ct.colWidth)
{
// MCOL-1246 Trim trailing whitespace for matching so that we have
// the same as InnoDB behaviour
int64_t tMin = cpRange.loVal;
int64_t tMax = cpRange.hiVal;
dataconvert::DataConvert::trimWhitespace(tMin);
dataconvert::DataConvert::trimWhitespace(tMax);
scan = compareVal(order_swap(tMin), order_swap(tMax), order_swap(value),
op, lcf);
datatypes::Charset cs(ct.charsetNumber);
utils::ConstString sMin((const char *) &cpRange.loVal, 8);
utils::ConstString sMax((const char *) &cpRange.hiVal, 8);
utils::ConstString sVal((const char *) &value, 8);
scan = compareStr(cs, sMin.rtrimZero(),
sMax.rtrimZero(),
sVal.rtrimZero(), op, lcf);
// cout << "scan=" << (uint32_t) scan << endl;
}
else if (bIsUnsigned)

8
dbcon/joblist/pcolscan.cpp
View File

@ -123,6 +123,7 @@ pColScanStep::pColScanStep(
const JobInfo& jobInfo) :
JobStep(jobInfo),
fRm(jobInfo.rm),
fMsgHeader(),
fNumThreads(fRm->getJlNumScanReceiveThreads()),
fFilterCount(0),
fOid(o),
@ -154,8 +155,6 @@ pColScanStep::pColScanStep(
rDoNothing = false;
fIsDict = false;
memset(&fMsgHeader, 0, sizeof(fMsgHeader));
//If this is a dictionary column, fudge the numbers...
if ( fColType.colDataType == CalpontSystemCatalog::VARCHAR )
{
@ -1008,7 +1007,8 @@ uint64_t pColScanStep::getFBO(uint64_t lbid)
pColScanStep::pColScanStep(const pColStep& rhs) :
JobStep(rhs),
fRm(rhs.resourceManager())
fRm(rhs.resourceManager()),
fMsgHeader()
{
fNumThreads = fRm->getJlNumScanReceiveThreads();
fFilterCount = rhs.filterCount();
@ -1040,8 +1040,6 @@ pColScanStep::pColScanStep(const pColStep& rhs) :
int err;
memset(&fMsgHeader, 0, sizeof(fMsgHeader));
err = dbrm.lookup(fOid, lbidRanges);
if (err)

65
dbcon/joblist/primitivemsg.h
View File

@ -283,6 +283,27 @@ struct ColLoopback
PrimitiveHeader Hdr; // 64 bit header
};
struct ColRequestHeaderDataType: public datatypes::Charset
{
int32_t CompType;
uint16_t DataSize;
uint8_t DataType; // enum ColDataType defined in calpont system catalog header file
ColRequestHeaderDataType()
:Charset(my_charset_bin),
CompType(0),
DataSize(0),
DataType(0)
{ }
ColRequestHeaderDataType(const execplan::CalpontSystemCatalog::ColType &rhs)
:Charset(rhs.charsetNumber),
CompType(rhs.compressionType),
DataSize(rhs.colWidth),
DataType(rhs.colDataType)
{ }
};
// COL_BY_SCAN
//Tied to ColByScanRangeRequestHeader and NewColRequestHeader. Check other headers if modifying.
@ -290,15 +311,19 @@ struct ColByScanRequestHeader
{
PrimitiveHeader Hdr; // 64 bit header
uint64_t LBID;
int32_t CompType;
uint16_t DataSize;
uint8_t DataType; // enum ColDataType defined in calpont system catalog header file
ColRequestHeaderDataType colType;
uint8_t OutputType; // 1 = RID, 2 = Token, 3 = Both
uint8_t BOP; // 0 = N/A, 1 = AND, 2 = OR
uint8_t RidFlags; // a bitmap indicating the rid ranges in the resultM SB => row 7168-8191
uint16_t NOPS;
uint16_t NVALS;
uint8_t sort;
ColByScanRequestHeader()
:LBID(0), OutputType(0), BOP(0),
RidFlags(0), NOPS(0), NVALS(0), sort(0)
{
memset(&Hdr, 0, sizeof(Hdr));
}
};
// COL_BY_SCAN_RANGE
@ -308,9 +333,7 @@ struct ColByScanRangeRequestHeader
{
PrimitiveHeader Hdr; // 64 bit header
uint64_t LBID; // starting LBID
int32_t CompType;
uint16_t DataSize;
uint8_t DataType; // enum ColDataType defined in calpont system catalog header file
ColRequestHeaderDataType colType;
uint8_t OutputType; // 1 = RID, 2 = Token, 3 = Both
uint8_t BOP; // 0 = N/A, 1 = AND, 2 = OR
uint8_t RidFlags; // a bitmap indicating the rid ranges in the result MSB => row 7168-8191
@ -318,6 +341,13 @@ struct ColByScanRangeRequestHeader
uint16_t NVALS;
uint8_t sort;
uint16_t Count; //Number of LBID's
ColByScanRangeRequestHeader()
:LBID(0), OutputType(0),
BOP(0), RidFlags(0), NOPS(0), NVALS(0),
sort(0), Count(0)
{
memset(&Hdr, 0, sizeof(Hdr));
}
};
// COL_BY_RID
@ -326,15 +356,14 @@ struct ColByRIDRequestHeader
{
PrimitiveHeader Hdr; // 64 bit header
uint64_t LBID;
int32_t CompType;
uint16_t DataSize;
uint8_t DataType; // enum ColDataType defined in calpont system catalog header file
ColRequestHeaderDataType colType;
uint8_t OutputType; // 1 = RID, 2 = Token, 3 = Both
uint8_t BOP; // 0 = N/A, 1 = AND, 2 = OR
uint8_t InputFlags; // 1 = interpret each NOP & RID as a pair
uint16_t NOPS;
uint16_t NVALS;
uint8_t sort;
ColByRIDRequestHeader(); // QQ? Not used?
};
// COL_AGG_BY_SCAN
@ -358,14 +387,13 @@ struct ColAggByRIDRequestHeader
{
PrimitiveHeader Hdr; // 64 bit header
uint64_t LBID;
int32_t CompType;
uint16_t DataSize;
uint8_t DataType; // enum ColDataType defined in calpont system catalog header file
ColRequestHeaderDataType colType;
uint8_t OutputType; // 1 = RID, 2 = Token, 3 = Both
uint8_t BOP; // 0 = N/A, 1 = AND, 2 = OR
uint8_t ExtraNotUsed;
uint16_t NOPS;
uint16_t NVALS;
ColAggByRIDRequestHeader(); // Not used?
};
// Loopback Results
@ -663,9 +691,7 @@ struct NewColRequestHeader
ISMPacketHeader ism;
PrimitiveHeader hdr;
uint64_t LBID;
int32_t CompType;
uint16_t DataSize;
uint8_t DataType;
ColRequestHeaderDataType colType;
uint8_t OutputType; // OT_DATAVALUE, OT_RID, or OT_BOTH
uint8_t BOP;
// uint8_t InputFlags; // 1 = interpret each NOP & RID as a pair (deprecated)
@ -676,6 +702,10 @@ struct NewColRequestHeader
// this follows the header
// ColArgs ArgList[NOPS] (where the val field is DataSize bytes long)
// uint16_t Rids[NVALS] (each rid is relative to the given block)
// QQ: The below constructor is never used.
// This struct is used in a cast only, in a hackish way.
NewColRequestHeader();
};
struct NewColAggRequestHeader
@ -683,9 +713,7 @@ struct NewColAggRequestHeader
ISMPacketHeader ism;
PrimitiveHeader hdr;
uint64_t LBID;
int32_t CompType;
uint8_t DataSize;
uint8_t DataType;
ColRequestHeaderDataType colType;
uint8_t OutputType;
uint8_t BOP;
uint8_t ExtraNotUsed;
@ -694,6 +722,7 @@ struct NewColAggRequestHeader
// this follows the header
// ColArgs ArgList[NOPS] (where the val field is DataSize bytes long)
// uint16_t Rids[NVALS] (each rid is relative to the given block)
NewColAggRequestHeader(); // QQ: not used
};
struct NewColResultHeader

131
primitives/linux-port/column.cpp
View File

@ -140,6 +140,44 @@ inline bool colCompare_(const T& val1, const T& val2, uint8_t COP)
}
}
inline bool colCompareStr(const ColRequestHeaderDataType &type,
uint8_t COP,
const utils::ConstString &val1,
const utils::ConstString &val2)
{
int res = type.strnncollsp(val1, val2);
switch (COP)
{
case COMPARE_NIL:
return false;
case COMPARE_LT:
return res < 0;
case COMPARE_EQ:
return res == 0;
case COMPARE_LE:
return res <= 0;
case COMPARE_GT:
return res > 0;
case COMPARE_NE:
return res != 0;
case COMPARE_GE:
return res >= 0;
default:
logIt(34, COP, "colCompareStr");
return false; // throw an exception here?
}
}
template<class T>
inline bool colCompare_(const T& val1, const T& val2, uint8_t COP, uint8_t rf)
{
@ -589,15 +627,15 @@ inline bool isMinMaxValid(const NewColRequestHeader* in)
}
else
{
switch (in->DataType)
switch (in->colType.DataType)
{
case CalpontSystemCatalog::CHAR:
return (in->DataSize < 9);
return (in->colType.DataSize < 9);
case CalpontSystemCatalog::VARCHAR:
case CalpontSystemCatalog::BLOB:
case CalpontSystemCatalog::TEXT:
return (in->DataSize < 8);
return (in->colType.DataSize < 8);
case CalpontSystemCatalog::TINYINT:
case CalpontSystemCatalog::SMALLINT:
@ -617,7 +655,7 @@ inline bool isMinMaxValid(const NewColRequestHeader* in)
case CalpontSystemCatalog::DECIMAL:
case CalpontSystemCatalog::UDECIMAL:
return (in->DataSize <= datatypes::MAXDECIMALWIDTH);
return (in->colType.DataSize <= datatypes::MAXDECIMALWIDTH);
default:
return false;
@ -636,8 +674,13 @@ inline string fixChar(int64_t intval)
return string(chval);
}
inline bool colCompare(int64_t val1, int64_t val2, uint8_t COP, uint8_t rf, int type, uint8_t width, const idb_regex_t& regex, bool isNull = false)
inline bool colCompare(int64_t val1, int64_t val2, uint8_t COP, uint8_t rf,
const ColRequestHeaderDataType &typeHolder, uint8_t width,
const idb_regex_t& regex, bool isNull = false)
{
uint8_t type = typeHolder.DataType;
// cout << "comparing " << hex << val1 << " to " << val2 << endl;
if (COMPARE_NIL == COP) return false;
//@bug 425 added isNull condition
@ -664,11 +707,9 @@ inline bool colCompare(int64_t val1, int64_t val2, uint8_t COP, uint8_t rf, int
{
if (!regex.used && !rf)
{
// MCOL-1246 Trim trailing whitespace for matching, but not for
// regex
dataconvert::DataConvert::trimWhitespace(val1);
dataconvert::DataConvert::trimWhitespace(val2);
return colCompare_(order_swap(val1), order_swap(val2), COP);
utils::ConstString s1 = {reinterpret_cast<const char*>(&val1), 8};
utils::ConstString s2 = {reinterpret_cast<const char*>(&val2), 8};
return colCompareStr(typeHolder, COP, s1.rtrimZero(), s2.rtrimZero());
}
else
return colStrCompare_(order_swap(val1), order_swap(val2), COP, rf, &regex);
@ -742,7 +783,7 @@ inline void store(const NewColRequestHeader* in,
{
#ifdef PRIM_DEBUG
if (*written + in->DataSize > outSize)
if (*written + in->colType.DataSize > outSize)
{
logIt(35, 2);
throw logic_error("PrimitiveProcessor::store(): output buffer is too small");
@ -753,7 +794,7 @@ inline void store(const NewColRequestHeader* in,
void* ptr1 = &out8[*written];
const uint8_t* ptr2 = &block8[0];
switch (in->DataSize)
switch (in->colType.DataSize)
{
case 32:
std::cout << __func__ << " WARNING!!! Not implemented for 32 byte data types." << std::endl;
@ -788,7 +829,8 @@ inline void store(const NewColRequestHeader* in,
memcpy(ptr1, ptr2, 1);
break;
}
*written += in->DataSize;
*written += in->colType.DataSize;
}
out->NVALS++;
@ -1146,7 +1188,7 @@ inline void p_Col_noprid(const NewColRequestHeader* in, NewColResultHeader* out,
rid = *reinterpret_cast<const uint16_t*>(&in8[argOffset + sizeof(ColArgs) +
in->DataSize]);
if (isUnsigned((CalpontSystemCatalog::ColDataType)in->DataType))
if (isUnsigned((CalpontSystemCatalog::ColDataType)in->colType.DataType))
{
switch (in->DataSize)
{
@ -1178,7 +1220,7 @@ inline void p_Col_noprid(const NewColRequestHeader* in, NewColResultHeader* out,
return;
}
if (colCompare(ucolVal, uargVal, args->COP, args->rf, in->DataType, in->DataSize, placeholderRegex))
if (colCompare(ucolVal, uargVal, args->COP, args->rf, in->colType.DataType, in->DataSize, placeholderRegex))
store(in, out, outSize, written, rid, reinterpret_cast<const uint8_t*>(block));
}
else
@ -1213,7 +1255,7 @@ inline void p_Col_noprid(const NewColRequestHeader* in, NewColResultHeader* out,
return;
}
if (colCompare(colVal, argVal, args->COP, args->rf, in->DataType, in->DataSize, placeholderRegex))
if (colCompare(colVal, argVal, args->COP, args->rf, in->colType.DataType, in->DataSize, placeholderRegex))
store(in, out, outSize, written, rid, reinterpret_cast<const uint8_t*>(block));
}
}
@ -1255,7 +1297,7 @@ inline void p_Col_ridArray(NewColRequestHeader* in,
if (out->ValidMinMax)
{
if (isUnsigned((CalpontSystemCatalog::ColDataType)in->DataType))
if (isUnsigned((CalpontSystemCatalog::ColDataType)in->colType.DataType))
{
out->Min = static_cast<int64_t>(numeric_limits<uint64_t>::max());
out->Max = 0;
@ -1298,7 +1340,7 @@ inline void p_Col_ridArray(NewColRequestHeader* in,
std_regex.reset(new idb_regex_t[in->NOPS]);
regex = &(std_regex[0]);
if (isUnsigned((CalpontSystemCatalog::ColDataType)in->DataType))
if (isUnsigned((CalpontSystemCatalog::ColDataType)in->colType.DataType))
{
uargVals = reinterpret_cast<uint64_t*>(std_argVals);
cops = std_cops;
@ -1358,7 +1400,7 @@ inline void p_Col_ridArray(NewColRequestHeader* in,
case 4:
#if 0
if (in->DataType == CalpontSystemCatalog::FLOAT)
if (in->colType.DataType == CalpontSystemCatalog::FLOAT)
{
double dTmp;
@ -1409,14 +1451,14 @@ inline void p_Col_ridArray(NewColRequestHeader* in,
// else we have a pre-parsed filter, and it's an unordered set for quick == comparisons
if (isUnsigned((CalpontSystemCatalog::ColDataType)in->DataType))
if (isUnsigned((CalpontSystemCatalog::ColDataType)in->colType.DataType))
{
uval = nextUnsignedColValue<W>(in->DataType, ridArray, in->NVALS, &nextRidIndex, &done, &isNull,
uval = nextUnsignedColValue<W>(in->colType.DataType, ridArray, in->NVALS, &nextRidIndex, &done, &isNull,
&isEmpty, &rid, in->OutputType, reinterpret_cast<uint8_t*>(block), itemsPerBlk);
}
else
{
val = nextColValue<W>(in->DataType, ridArray, in->NVALS, &nextRidIndex, &done, &isNull,
val = nextColValue<W>(in->colType.DataType, ridArray, in->NVALS, &nextRidIndex, &done, &isNull,
&isEmpty, &rid, in->OutputType, reinterpret_cast<uint8_t*>(block), itemsPerBlk);
}
@ -1427,7 +1469,7 @@ inline void p_Col_ridArray(NewColRequestHeader* in,
/* bug 1920: ignore NULLs in the set and in the column data */
if (!(isNull && in->BOP == BOP_AND))
{
if (isUnsigned((CalpontSystemCatalog::ColDataType)in->DataType))
if (isUnsigned((CalpontSystemCatalog::ColDataType)in->colType.DataType))
{
it = parsedColumnFilter->prestored_set->find(*reinterpret_cast<int64_t*>(&uval));
}
@ -1458,15 +1500,15 @@ inline void p_Col_ridArray(NewColRequestHeader* in,
{
for (argIndex = 0; argIndex < in->NOPS; argIndex++)
{
if (isUnsigned((CalpontSystemCatalog::ColDataType)in->DataType))
if (isUnsigned((CalpontSystemCatalog::ColDataType)in->colType.DataType))
{
cmp = colCompareUnsigned(uval, uargVals[argIndex], cops[argIndex],
rfs[argIndex], in->DataType, W, regex[argIndex], isNull);
rfs[argIndex], in->colType.DataType, W, regex[argIndex], isNull);
}
else
{
cmp = colCompare(val, argVals[argIndex], cops[argIndex],
rfs[argIndex], in->DataType, W, regex[argIndex], isNull);
rfs[argIndex], in->colType, W, regex[argIndex], isNull);
}
if (in->NOPS == 1)
@ -1499,16 +1541,18 @@ inline void p_Col_ridArray(NewColRequestHeader* in,
if (out->ValidMinMax && !isNull && !isEmpty)
{
if ((in->DataType == CalpontSystemCatalog::CHAR || in->DataType == CalpontSystemCatalog::VARCHAR ||
in->DataType == CalpontSystemCatalog::BLOB || in->DataType == CalpontSystemCatalog::TEXT ) && 1 < W)
if ((in->colType.DataType == CalpontSystemCatalog::CHAR ||
in->colType.DataType == CalpontSystemCatalog::VARCHAR ||
in->colType.DataType == CalpontSystemCatalog::BLOB ||
in->colType.DataType == CalpontSystemCatalog::TEXT ) && 1 < W)
{
if (colCompare(out->Min, val, COMPARE_GT, false, in->DataType, W, placeholderRegex))
if (colCompare(out->Min, val, COMPARE_GT, false, in->colType, W, placeholderRegex))
out->Min = val;
if (colCompare(out->Max, val, COMPARE_LT, false, in->DataType, W, placeholderRegex))
if (colCompare(out->Max, val, COMPARE_LT, false, in->colType, W, placeholderRegex))
out->Max = val;
}
else if (isUnsigned((CalpontSystemCatalog::ColDataType)in->DataType))
else if (isUnsigned((CalpontSystemCatalog::ColDataType)in->colType.DataType))
{
if (static_cast<uint64_t>(out->Min) > uval)
out->Min = static_cast<int64_t>(uval);
@ -1526,15 +1570,15 @@ inline void p_Col_ridArray(NewColRequestHeader* in,
}
}
if (isUnsigned((CalpontSystemCatalog::ColDataType)in->DataType))
if (isUnsigned((CalpontSystemCatalog::ColDataType)in->colType.DataType))
{
uval = nextUnsignedColValue<W>(in->DataType, ridArray, in->NVALS, &nextRidIndex, &done,
uval = nextUnsignedColValue<W>(in->colType.DataType, ridArray, in->NVALS, &nextRidIndex, &done,
&isNull, &isEmpty, &rid, in->OutputType, reinterpret_cast<uint8_t*>(block),
itemsPerBlk);
}
else
{
val = nextColValue<W>(in->DataType, ridArray, in->NVALS, &nextRidIndex, &done,
val = nextColValue<W>(in->colType.DataType, ridArray, in->NVALS, &nextRidIndex, &done,
&isNull, &isEmpty, &rid, in->OutputType, reinterpret_cast<uint8_t*>(block),
itemsPerBlk);
}
@ -1589,7 +1633,7 @@ inline void p_Col_bin_ridArray(NewColRequestHeader* in,
if (out->ValidMinMax)
{
// Assume that isUnsigned returns true for 8-bytes DTs only
if (isUnsigned((CalpontSystemCatalog::ColDataType)in->DataType))
if (isUnsigned((CalpontSystemCatalog::ColDataType)in->colType.DataType))
{
out->Min = -1;
out->Max = 0;
@ -1653,7 +1697,7 @@ inline void p_Col_bin_ridArray(NewColRequestHeader* in,
// else we have a pre-parsed filter, and it's an unordered set for quick == comparisons
bval = (binWtype*)nextBinColValue<W>(in->DataType, ridArray, in->NVALS, &nextRidIndex, &done, &isNull,
bval = (binWtype*)nextBinColValue<W>(in->colType.DataType, ridArray, in->NVALS, &nextRidIndex, &done, &isNull,
&isEmpty, &rid, in->OutputType, reinterpret_cast<uint8_t*>(block), itemsPerBlk);
T val;
@ -1696,7 +1740,7 @@ inline void p_Col_bin_ridArray(NewColRequestHeader* in,
T filterVal = *reinterpret_cast<T*>(argVals[argIndex]);
cmp = colCompare(val, filterVal, cops[argIndex],
rfs[argIndex], in->DataType, W, isNull);
rfs[argIndex], in->colType.DataType, W, isNull);
if (in->NOPS == 1)
{
@ -1727,15 +1771,16 @@ inline void p_Col_bin_ridArray(NewColRequestHeader* in,
if (out->ValidMinMax && !isNull && !isEmpty)
{
if (in->DataType == CalpontSystemCatalog::CHAR || in->DataType == CalpontSystemCatalog::VARCHAR)
if (in->colType.DataType == CalpontSystemCatalog::CHAR ||
in->colType.DataType == CalpontSystemCatalog::VARCHAR)
{
// !!! colCompare is overloaded with int128_t only yet.
if (colCompare(out->Min, val, COMPARE_GT, false, in->DataType, W, placeholderRegex))
if (colCompare(out->Min, val, COMPARE_GT, false, in->colType, W, placeholderRegex))
{
out->Min = val;
}
if (colCompare(out->Max, val, COMPARE_LT, false, in->DataType, W, placeholderRegex))
if (colCompare(out->Max, val, COMPARE_LT, false, in->colType, W, placeholderRegex))
{
out->Max = val;
}
@ -1754,7 +1799,7 @@ inline void p_Col_bin_ridArray(NewColRequestHeader* in,
}
}
bval = (binWtype*)nextBinColValue<W>(in->DataType, ridArray, in->NVALS, &nextRidIndex, &done, &isNull,
bval = (binWtype*)nextBinColValue<W>(in->colType.DataType, ridArray, in->NVALS, &nextRidIndex, &done, &isNull,
&isEmpty, &rid, in->OutputType, reinterpret_cast<uint8_t*>(block), itemsPerBlk);
}
@ -1789,7 +1834,7 @@ void PrimitiveProcessor::p_Col(NewColRequestHeader* in, NewColResultHeader* out,
if (logicalBlockMode)
itemsPerBlk = BLOCK_SIZE;
else
itemsPerBlk = BLOCK_SIZE / in->DataSize;
itemsPerBlk = BLOCK_SIZE / in->colType.DataSize;
//...Initialize I/O counts;
out->CacheIO = 0;
@ -1816,7 +1861,7 @@ void PrimitiveProcessor::p_Col(NewColRequestHeader* in, NewColResultHeader* out,
fStatsPtr->markEvent(in->LBID, pthread_self(), in->hdr.SessionID, 'B');
#endif
switch (in->DataSize)
switch (in->colType.DataSize)
{
case 8:
p_Col_ridArray<8>(in, out, outSize, written, block, fStatsPtr, itemsPerBlk, parsedColumnFilter);

9
primitives/primproc/columncommand.cpp
View File

@ -298,7 +298,7 @@ void ColumnCommand::issuePrimitive()
{
ostringstream os;
os << " WARNING!!! Not implemented for ";
os << primMsg->DataSize << " column.";
os << primMsg->colType.DataSize << " column.";
throw PrimitiveColumnProjectResultExcept(os.str());
}
}
@ -497,6 +497,7 @@ void ColumnCommand::processResult()
void ColumnCommand::createCommand(ByteStream& bs)
{
uint8_t tmp8;
uint32_t tmp32;
bs.advance(1);
bs >> tmp8;
@ -519,6 +520,8 @@ void ColumnCommand::createCommand(ByteStream& bs)
colType.scale = tmp8;
bs >> tmp8;
colType.compressionType = tmp8;
bs >> tmp32;
colType.charsetNumber = tmp32;
bs >> BOP;
bs >> filterCount;
deserializeInlineVector(bs, lastLbid);
@ -576,9 +579,7 @@ void ColumnCommand::prep(int8_t outputType, bool absRids)
primMsg->hdr.TransactionID = bpp->txnID;
primMsg->hdr.VerID = bpp->versionInfo.currentScn;
primMsg->hdr.StepID = bpp->stepID;
primMsg->DataSize = colType.colWidth;
primMsg->DataType = colType.colDataType;
primMsg->CompType = colType.compressionType;
primMsg->colType = ColRequestHeaderDataType(colType);
primMsg->OutputType = outputType;
primMsg->BOP = BOP;
primMsg->NOPS = (suppressFilter ? 0 : filterCount);

25
utils/common/collation.h
View File

@ -19,6 +19,7 @@
#define COLLATION_H_INCLUDED
#include "exceptclasses.h"
#include "conststring.h"
/*
Redefine definitions used by MariaDB m_ctype.h.
@ -92,9 +93,9 @@ public:
MariaDBHasher()
:mPart1(1), mPart2(4)
{ }
MariaDBHasher & add(CHARSET_INFO & cs, const char *str, size_t length)
MariaDBHasher & add(CHARSET_INFO * cs, const char *str, size_t length)
{
cs.hash_sort((const uchar *) str, length, &mPart1, &mPart2);
cs->hash_sort((const uchar *) str, length, &mPart1, &mPart2);
return *this;
}
uint32_t finalize() const
@ -109,21 +110,33 @@ public:
class Charset
{
protected:
const struct charset_info_st & mCharset;
const struct charset_info_st * mCharset;
public:
Charset(CHARSET_INFO & cs) :mCharset(cs) { }
Charset(CHARSET_INFO & cs) :mCharset(&cs) { }
Charset(uint32_t charsetNumber);
CHARSET_INFO & getCharset() const { return mCharset; }
CHARSET_INFO & getCharset() const { return *mCharset; }
uint32_t hash(const char *data, uint64_t len) const
{
return MariaDBHasher().add(mCharset, data, len).finalize();
}
bool eq(const std::string & str1, const std::string & str2) const
{
return mCharset.strnncollsp(str1.data(), str1.length(),
return mCharset->strnncollsp(str1.data(), str1.length(),
str2.data(), str2.length()) == 0;
}
int strnncollsp(const utils::ConstString &str1,
const utils::ConstString &str2) const
{
return mCharset->strnncollsp(str1.str(), str1.length(),
str2.str(), str2.length());
}
bool test_if_important_data(const char *str, const char *end) const
{
if (mCharset->state & MY_CS_NOPAD)
return str < end;
return str + mCharset->scan(str, end, MY_SEQ_SPACES) < end;
}
};

47
utils/common/conststring.h Normal file
View File

@ -0,0 +1,47 @@
/* Copyright (C) 2020 MariaDB Corporation.
This program is free software; you can redistribute it and/or
modify it under the terms of the GNU General Public License
as published by the Free Software Foundation; version 2 of
the License.
This program is distributed in the hope that it will be useful,
but WITHOUT ANY WARRANTY; without even the implied warranty of
MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
GNU General Public License for more details.
You should have received a copy of the GNU General Public License
along with this program; if not, write to the Free Software
Foundation, Inc., 51 Franklin Street, Fifth Floor, Boston,
MA 02110-1301, USA. */
#ifndef MARIADB_CONSTSTRING_H
#define MARIADB_CONSTSTRING_H
namespace utils
{
class ConstString
{
const char *mStr;
size_t mLength;
public:
ConstString(const char *str, size_t length)
:mStr(str), mLength(length)
{ }
const char *str() const { return mStr; }
size_t length() const { return mLength; }
ConstString & rtrimZero()
{
for ( ; mLength && mStr[mLength - 1] == '\0'; mLength--)
{ }
return *this;
}
};
} // namespace utils
#endif // MARIADB_CONSTSTRING_H

2
utils/common/utils_utf8.cpp
View File

@ -31,7 +31,7 @@ static inline CHARSET_INFO & get_charset_or_bin(int32_t charsetNumber)
Charset::Charset(uint32_t charsetNumber)
:mCharset(get_charset_or_bin(charsetNumber))
:mCharset(&get_charset_or_bin(charsetNumber))
{
}

8
utils/dataconvert/dataconvert.cpp
View File

@ -1542,8 +1542,14 @@ DataConvert::StringToString(const datatypes::SystemCatalog::TypeAttributesStd& c
//check data length
if ( data.length() > (unsigned int)colType.colWidth )
{
// TODO: charsetNumber should be moved to TypeStdAttributes ASAP
const execplan::CalpontSystemCatalog::ColType &colType2=
static_cast<const execplan::CalpontSystemCatalog::ColType &>(colType);
datatypes::Charset cs(colType2.charsetNumber);
const char *newEnd = data.data() + colType.colWidth;
const char *origEnd = data.data() + data.length();
pushWarning = cs.test_if_important_data(newEnd, origEnd);
data = data.substr(0, colType.colWidth);
pushWarning = true;
boost::any value = data;
return value;
}