database/mariadb-columnstore-engine
1
0
Fork 0
mirror of https://github.com/mariadb-corporation/mariadb-columnstore-engine.git synced 2025-07-30 19:23:07 +03:00
Code Activity

MCOL-271 empty strings should not be NULLs (#2794)

Browse Source 
This patch improves handling of NULLs in textual fields in ColumnStore.
Previously empty strings were considered NULLs and it could be a problem
if data scheme allows for empty strings. It was also one of major
reasons of behavior difference between ColumnStore and other engines in
MariaDB family.

Also, this patch fixes some other bugs and incorrect behavior, for
example, incorrect comparison for "column <= ''" which evaluates to
constant True for all purposes before this patch.
This commit is contained in:
Sergey Zefirov
2023-03-30 17:26:45 +01:00
committed by Roman Nozdrin
parent 0ea592da80
commit b53c231ca6
417 changed files with 12459 additions and 3520 deletions
Download Patch File Download Diff File Expand all files Collapse all files

2
utils/windowfunction/wf_count.cpp
View File

@ -62,7 +62,7 @@ boost::shared_ptr<WindowFunctionType> WF_count<T>::makeFunction(int id, const st
case CalpontSystemCatalog::VARCHAR:
case CalpontSystemCatalog::VARBINARY:
{
func.reset(new WF_count<string>(id, name));
func.reset(new WF_count<utils::NullString>(id, name));
break;
}

4
utils/windowfunction/wf_lead_lag.cpp
View File

@ -119,7 +119,7 @@ boost::shared_ptr<WindowFunctionType> WF_lead_lag<T>::makeFunction(int id, const
default:
{
func.reset(new WF_lead_lag<string>(id, name));
func.reset(new WF_lead_lag<utils::NullString>(id, name));
break;
}
}
@ -303,6 +303,6 @@ template void WF_lead_lag<uint64_t>::parseParms(const std::vector<execplan::SRCP
template void WF_lead_lag<int128_t>::parseParms(const std::vector<execplan::SRCP>&);
template void WF_lead_lag<float>::parseParms(const std::vector<execplan::SRCP>&);
template void WF_lead_lag<double>::parseParms(const std::vector<execplan::SRCP>&);
template void WF_lead_lag<string>::parseParms(const std::vector<execplan::SRCP>&);
template void WF_lead_lag<utils::NullString>::parseParms(const std::vector<execplan::SRCP>&);
} // namespace windowfunction

2
utils/windowfunction/wf_min_max.cpp
View File

@ -120,7 +120,7 @@ boost::shared_ptr<WindowFunctionType> WF_min_max<T>::makeFunction(int id, const
default:
{
func.reset(new WF_min_max<string>(id, name));
func.reset(new WF_min_max<utils::NullString>(id, name));
break;
}
}

2
utils/windowfunction/wf_nth_value.cpp
View File

@ -120,7 +120,7 @@ boost::shared_ptr<WindowFunctionType> WF_nth_value<T>::makeFunction(int id, cons
default:
{
func.reset(new WF_nth_value<string>(id, name));
func.reset(new WF_nth_value<utils::NullString>(id, name));
break;
}
}

7
utils/windowfunction/wf_percentile.cpp
View File

@ -129,7 +129,7 @@ boost::shared_ptr<WindowFunctionType> WF_percentile<T>::makeFunction(int id, con
{
if (id == WF__PERCENTILE_DISC)
{
func.reset(new WF_percentile<string>(id, name));
func.reset(new WF_percentile<utils::NullString>(id, name));
}
else
{
@ -334,11 +334,12 @@ void WF_percentile<T>::operator()(int64_t b, int64_t e, int64_t c)
implicit2T(idx, cv, 0);
vd = (crn - rn) * fv + (rn - frn) * cv;
}
double tempvd[2];
tempvd[0] = vd;
tempvd[1] = 0;
v = *(reinterpret_cast<T*>(&tempvd[0])); // old code that referred to 'vd' var triggered partial out-of-bounds warning.
v = *(reinterpret_cast<T*>(&tempvd[0]));
//v = *(reinterpret_cast<T*>(&vd)); // XXX old code that produced out-of-bounds difference.
p = &v;
}
else // (fFunctionId == WF__PERCENTILE_DISC)

25
utils/windowfunction/wf_udaf.cpp
View File

@ -162,7 +162,7 @@ bool WF_udaf::dropValues(int64_t b, int64_t e)
// Turn on Null flags or skip based on respect nulls
flags[k] = 0;
if ((!cc && fRow.isNullValue(colIn) == true) || (cc && cc->type() == ConstantColumn::NULLDATA))
if ((!cc && fRow.isNullValue(colIn) == true) || (cc && cc->isNull()))
{
if (!bRespectNulls)
{
@ -450,11 +450,11 @@ bool WF_udaf::dropValues(int64_t b, int64_t e)
case CalpontSystemCatalog::TEXT:
case CalpontSystemCatalog::BLOB:
{
string valIn;
utils::NullString valIn;
if (cc)
{
valIn = cc->getStrVal(fRow, isNull);
valIn = cc->getStrVal(fRow, isNull); // XXX: we probably need to change Distinctmap.
}
else
{
@ -548,7 +548,7 @@ void WF_udaf::SetUDAFValue(static_any::any& valOut, int64_t colOut, int64_t b, i
static const static_any::any& ullTypeId = (unsigned long long)1;
static const static_any::any& floatTypeId = (float)1;
static const static_any::any& doubleTypeId = (double)1;
static const std::string typeStr("");
static const std::string typeStr;
static const static_any::any& strTypeId = typeStr;
CDT colDataType = fRow.getColType(colOut);
@ -661,7 +661,7 @@ void WF_udaf::SetUDAFValue(static_any::any& valOut, int64_t colOut, int64_t b, i
if (valOut.compatible(strTypeId))
{
std::string strOut = valOut.cast<std::string>();
strOut = valOut.cast<std::string>();
// Convert the string to numeric type, just in case.
intOut = atol(strOut.c_str());
uintOut = strtoul(strOut.c_str(), NULL, 10);
@ -756,11 +756,12 @@ void WF_udaf::SetUDAFValue(static_any::any& valOut, int64_t colOut, int64_t b, i
case execplan::CalpontSystemCatalog::BLOB:
if (valOut.empty())
{
setValue(colDataType, b, e, c, (string*)NULL);
setValue(colDataType, b, e, c, (utils::NullString*)NULL);
}
else
{
setValue(colDataType, b, e, c, &strOut);
utils::NullString nullStrOut(strOut);
setValue(colDataType, b, e, c, &nullStrOut);
}
break;
@ -845,7 +846,7 @@ void WF_udaf::operator()(int64_t b, int64_t e, int64_t c)
// Turn on Null flags or skip based on respect nulls
flags[k] = 0;
if ((!cc && fRow.isNullValue(colIn) == true) || (cc && cc->type() == ConstantColumn::NULLDATA))
if ((!cc && fRow.isNullValue(colIn) == true) || (cc && cc->isNull()))
{
if (!bRespectNulls)
{
@ -1112,11 +1113,11 @@ void WF_udaf::operator()(int64_t b, int64_t e, int64_t c)
case CalpontSystemCatalog::TEXT:
case CalpontSystemCatalog::BLOB:
{
string valIn;
utils::NullString valIn;
if (cc)
{
valIn = cc->getStrVal(fRow, isNull);
valIn = cc->getStrVal(fRow, isNull); // XXX: the same problem with distinct.
}
else
{
@ -1127,7 +1128,7 @@ void WF_udaf::operator()(int64_t b, int64_t e, int64_t c)
// Currently, distinct only works on the first parameter.
if (k == 0 && fDistinct)
{
std::pair<static_any::any, uint64_t> val = make_pair(valIn, 1);
std::pair<static_any::any, uint64_t> val = make_pair(valIn.isNull() ? nullptr : valIn.safeString(""), 1);
std::pair<DistinctMap::iterator, bool> distinct;
distinct = fDistinctMap.insert(val);
if (distinct.second == false)
@ -1138,7 +1139,7 @@ void WF_udaf::operator()(int64_t b, int64_t e, int64_t c)
}
}
datum.columnData = valIn;
datum.columnData = valIn.isNull() ? nullptr : valIn.safeString("");
break;
}

18
utils/windowfunction/windowfunctiontype.cpp
View File

@ -297,7 +297,7 @@ void WindowFunctionType::getValue<long double>(uint64_t i, long double& t, CDT*
}
template <>
void WindowFunctionType::getValue<string>(uint64_t i, string& t, CDT* cdt)
void WindowFunctionType::getValue<utils::NullString>(uint64_t i, utils::NullString& t, CDT* cdt)
{
t = fRow.getStringField(i);
// By not setting cdt, we let it default to the column's type
@ -356,14 +356,14 @@ void WindowFunctionType::setValue<int128_t>(uint64_t i, int128_t& t)
}
template <>
void WindowFunctionType::setValue<string>(uint64_t i, string& t)
void WindowFunctionType::setValue<utils::NullString>(uint64_t i, utils::NullString& t)
{
fRow.setStringField(t, i);
}
// MCOL-1676 Need a separate specialization for string now.
template <>
void WindowFunctionType::setValue<string>(int ct, int64_t b, int64_t e, int64_t c, string* v)
void WindowFunctionType::setValue<utils::NullString>(int ct, int64_t b, int64_t e, int64_t c, utils::NullString* v)
{
if (c != WF__BOUND_ALL)
b = e = c;
@ -371,13 +371,12 @@ void WindowFunctionType::setValue<string>(int ct, int64_t b, int64_t e, int64_t
uint64_t i = fFieldIndex[0];
if (v == NULL)
v = (string*)getNullValueByType(ct, i);
v = (utils::NullString*)getNullValueByType(ct, i);
for (int64_t j = b; j <= e; j++)
{
if (j % 1000 == 0 && fStep->cancelled())
break;
fRow.setData(getPointer((*fRowData)[j]));
setValue(i, *v);
}
@ -574,7 +573,7 @@ void WindowFunctionType::implicit2T(uint64_t i, T& t, int s)
}
template <>
void WindowFunctionType::implicit2T<string>(uint64_t i, string& t, int)
void WindowFunctionType::implicit2T<utils::NullString>(uint64_t i, utils::NullString& t, int)
{
t = fRow.getStringField(i);
}
@ -621,7 +620,7 @@ void WindowFunctionType::getConstValue<float>(ConstantColumn* cc, float& t, bool
}
template <>
void WindowFunctionType::getConstValue<string>(ConstantColumn* cc, string& t, bool& b)
void WindowFunctionType::getConstValue<utils::NullString>(ConstantColumn* cc, utils::NullString& t, bool& b)
{
t = cc->getStrVal(fRow, b);
}
@ -660,7 +659,7 @@ void* WindowFunctionType::getNullValueByType(int ct, int pos)
// static uint64_t char2Null = joblist::CHAR2NULL;
// static uint64_t char4Null = joblist::CHAR4NULL;
// static uint64_t char8Null = joblist::CHAR8NULL;
static string stringNull("");
static utils::NullString stringNull;
static int128_t int128Null; // Set at runtime;
void* v = NULL;
@ -692,6 +691,8 @@ void* WindowFunctionType::getNullValueByType(int ct, int pos)
case CalpontSystemCatalog::CHAR:
case CalpontSystemCatalog::VARCHAR:
case CalpontSystemCatalog::TEXT:
case CalpontSystemCatalog::VARBINARY: // XXX: I guess it is right to do that. TODO: we can add TEXT here too.
{
// uint64_t len = fRow.getColumnWidth(pos);
#if 0
@ -764,7 +765,6 @@ void* WindowFunctionType::getNullValueByType(int ct, int pos)
case CalpontSystemCatalog::LONGDOUBLE: v = &longDoubleNull; break;
case CalpontSystemCatalog::VARBINARY:
default:
std::ostringstream oss;
oss << "not supported data type: " << colType2String[ct];

1
utils/windowfunction/windowfunctiontype.h
View File

@ -32,6 +32,7 @@
#include "windowframe.h"
#include "constantcolumn.h"
#include "mcs_decimal.h"
#include "nullstring.h"
namespace ordering
{