database/mariadb-columnstore-engine
1
0
Fork 0
mirror of https://github.com/mariadb-corporation/mariadb-columnstore-engine.git synced 2025-07-29 08:21:15 +03:00
Code Activity

MCOL-271 empty strings should not be NULLs (#2794)

Browse Source 
This patch improves handling of NULLs in textual fields in ColumnStore.
Previously empty strings were considered NULLs and it could be a problem
if data scheme allows for empty strings. It was also one of major
reasons of behavior difference between ColumnStore and other engines in
MariaDB family.

Also, this patch fixes some other bugs and incorrect behavior, for
example, incorrect comparison for "column <= ''" which evaluates to
constant True for all purposes before this patch.
This commit is contained in:
Sergey Zefirov
2023-03-30 17:26:45 +01:00
committed by Roman Nozdrin
parent 0ea592da80
commit b53c231ca6
417 changed files with 12459 additions and 3520 deletions
Download Patch File Download Diff File Expand all files Collapse all files

4
writeengine/bulk/we_bulkloadbuffer.cpp
View File

@ -483,8 +483,8 @@ void BulkLoadBuffer::convert(char* field, int fieldLength, bool nullFlag, unsign
{
if (column.fWithDefault)
{
int defLen = column.fDefaultChr.size();
const char* defData = column.fDefaultChr.c_str();
int defLen = column.fDefaultChr.length();
const char* defData = column.fDefaultChr.str();
if (defLen > column.definedWidth)
memcpy(charTmpBuf, defData, column.definedWidth);

11
writeengine/dictionary/we_dctnry.cpp
View File

@ -657,7 +657,7 @@ int Dctnry::insertDctnry2(Signature& sig)
sig.token.bc = 0;
while (sig.size > 0)
while (sig.size > 0 || !lbid_in_token)
{
if (sig.size > (m_freeSpace - HDR_UNIT_SIZE))
{
@ -819,7 +819,7 @@ int Dctnry::insertDctnry(const char* buf, ColPosPair** pos, const int totalRow,
{
if (m_defVal.length() > 0) // use default string if available
{
pIn = m_defVal.c_str();
pIn = m_defVal.str();
curSig.signature = (unsigned char*)pIn;
curSig.size = m_defVal.length();
}
@ -1061,13 +1061,6 @@ int Dctnry::insertDctnry(const int& sgnature_size, const unsigned char* sgnature
return ERR_DICT_SIZE_GT_2G;
}
if (sgnature_size == 0)
{
WriteEngine::Token nullToken;
memcpy(&token, &nullToken, 8);
return NO_ERROR;
}
CommBlock cb;
cb.file.oid = m_dctnryOID;
cb.file.pFile = m_dFile;

5
writeengine/dictionary/we_dctnry.h
View File

@ -35,6 +35,7 @@
#include "we_type.h"
#include "we_brm.h"
#include "bytestream.h"
#include "nullstring.h"
#define EXPORT
@ -214,7 +215,7 @@ class Dctnry : public DbFileOp
/**
* @brief Set dictionary default for this column
*/
void setDefault(const std::string& defVal)
void setDefault(const utils::NullString& defVal)
{
m_defVal = defVal;
}
@ -325,7 +326,7 @@ class Dctnry : public DbFileOp
DataBlock m_curBlock; // current "raw" (uncompressed) data block
Log* m_logger; // logger, mainly for bulk load
int m_colWidth; // width of this dictionary column
std::string m_defVal; // optional default string value
utils::NullString m_defVal; // optional default string value
ImportDataMode m_importDataMode; // Import data in text or binary mode
}; // end of class

49
writeengine/server/we_ddlcommandproc.cpp
View File

@ -171,24 +171,26 @@ uint8_t WE_DDLCommandProc::writeSystable(ByteStream& bs, std::string& err)
getColumnsForTable(sessionID, tableName.schema, tableName.table, columns);
column_iterator = columns.begin();
std::string tmpStr("");
NullString tmpStr;
while (column_iterator != columns.end())
{
column = *column_iterator;
boost::to_lower(column.tableColName.column);
tmpStr.dropString();
if (TABLENAME_COL == column.tableColName.column)
{
std::string tablename = tableDef.fQualifiedName->fName;
colTuple.data = tablename;
tmpStr = tablename;
tmpStr.assign(tablename);
}
else if (SCHEMA_COL == column.tableColName.column)
{
std::string schema = tableDef.fQualifiedName->fSchema;
colTuple.data = schema;
tmpStr = schema;
tmpStr.assign(schema);
}
else if (OBJECTID_COL == column.tableColName.column)
{
@ -433,7 +435,7 @@ uint8_t WE_DDLCommandProc::writeCreateSyscolumn(ByteStream& bs, std::string& err
ddlpackage::QualifiedName qualifiedName = *(tableDef.fQualifiedName);
iter = tableDefCols.begin();
// colpos = 0;
std::string tmpStr("");
NullString tmpStr;
for (unsigned int ii = 0; ii < numCols; ii++)
{
@ -517,18 +519,18 @@ uint8_t WE_DDLCommandProc::writeCreateSyscolumn(ByteStream& bs, std::string& err
if (SCHEMA_COL == column.tableColName.column)
{
colTuple.data = qualifiedName.fSchema;
tmpStr = qualifiedName.fSchema;
tmpStr.assign(qualifiedName.fSchema);
}
else if (TABLENAME_COL == column.tableColName.column)
{
colTuple.data = qualifiedName.fName;
tmpStr = qualifiedName.fName;
tmpStr.assign(qualifiedName.fName);
}
else if (COLNAME_COL == column.tableColName.column)
{
boost::to_lower(colDefPtr->fName);
colTuple.data = colDefPtr->fName;
tmpStr = colDefPtr->fName;
tmpStr.assign(colDefPtr->fName);
}
else if (OBJECTID_COL == column.tableColName.column)
{
@ -564,16 +566,15 @@ uint8_t WE_DDLCommandProc::writeCreateSyscolumn(ByteStream& bs, std::string& err
}
else if (DEFAULTVAL_COL == column.tableColName.column)
{
if (colDefPtr->fDefaultValue)
if (colDefPtr->fDefaultValue && !colDefPtr->fDefaultValue->fNull)
{
colTuple.data = colDefPtr->fDefaultValue->fValue;
tmpStr = colDefPtr->fDefaultValue->fValue;
tmpStr.assign(colDefPtr->fDefaultValue->fValue);
}
else
{
tmpStr = "";
// colTuple.data = column.colType.getNullValueForType();
tmpStr.dropString();
}
colTuple.data = tmpStr;
}
else if (NULLABLE_COL == column.tableColName.column)
{
@ -625,11 +626,11 @@ uint8_t WE_DDLCommandProc::writeCreateSyscolumn(ByteStream& bs, std::string& err
}
else if (MINVAL_COL == column.tableColName.column)
{
tmpStr = "";
tmpStr.dropString();
}
else if (MAXVAL_COL == column.tableColName.column)
{
tmpStr = "";
tmpStr.dropString();
}
else if (COMPRESSIONTYPE_COL == column.tableColName.column)
{
@ -834,7 +835,6 @@ uint8_t WE_DDLCommandProc::writeSyscolumn(ByteStream& bs, std::string& err)
std::map<uint32_t, uint32_t> oids;
std::vector<BRM::OID_t> oidsToFlush;
// colpos = 0;
std::string tmpStr("");
for (unsigned int ii = 0; ii < numCols; ii++)
{
@ -900,24 +900,25 @@ uint8_t WE_DDLCommandProc::writeSyscolumn(ByteStream& bs, std::string& err)
while (column_iterator != columns.end())
{
NullString tmpStr;
column = *column_iterator;
boost::to_lower(column.tableColName.column);
if (SCHEMA_COL == column.tableColName.column)
{
colTuple.data = schema;
tmpStr = schema;
tmpStr.assign(schema);
}
else if (TABLENAME_COL == column.tableColName.column)
{
colTuple.data = tablename;
tmpStr = tablename;
tmpStr.assign(tablename);
}
else if (COLNAME_COL == column.tableColName.column)
{
boost::to_lower(colDefPtr->fName);
colTuple.data = colDefPtr->fName;
tmpStr = colDefPtr->fName;
tmpStr.assign(colDefPtr->fName);
}
else if (OBJECTID_COL == column.tableColName.column)
{
@ -950,16 +951,16 @@ uint8_t WE_DDLCommandProc::writeSyscolumn(ByteStream& bs, std::string& err)
}
else if (DEFAULTVAL_COL == column.tableColName.column)
{
if (colDefPtr->fDefaultValue)
if (colDefPtr->fDefaultValue && !colDefPtr->fDefaultValue->fNull)
{
colTuple.data = colDefPtr->fDefaultValue->fValue;
tmpStr = colDefPtr->fDefaultValue->fValue;
tmpStr.assign(colDefPtr->fDefaultValue->fValue);
}
else
{
tmpStr = "";
tmpStr.dropString();
// colTuple.data = column.colType.getNullValueForType();
}
colTuple.data = tmpStr;
}
else if (NULLABLE_COL == column.tableColName.column)
{
@ -1011,11 +1012,11 @@ uint8_t WE_DDLCommandProc::writeSyscolumn(ByteStream& bs, std::string& err)
}
else if (MINVAL_COL == column.tableColName.column)
{
tmpStr = "";
tmpStr.dropString();
}
else if (MAXVAL_COL == column.tableColName.column)
{
tmpStr = "";
tmpStr.dropString();
}
else if (COMPRESSIONTYPE_COL == column.tableColName.column)
{

285
writeengine/server/we_dmlcommandproc.cpp
View File

@ -54,6 +54,7 @@ using namespace BRM;
#include "columnwidth.h"
using namespace std;
using namespace utils;
namespace WriteEngine
{
@ -91,7 +92,7 @@ WE_DMLCommandProc::~WE_DMLCommandProc()
dbRootExtTrackerVec.clear();
}
void WE_DMLCommandProc::processAuxCol(const std::vector<std::string>& origVals,
void WE_DMLCommandProc::processAuxCol(const std::vector<utils::NullString>& origVals,
WriteEngine::ColValueList& colValuesList,
WriteEngine::DictStrList& dicStringList)
{
@ -104,7 +105,8 @@ void WE_DMLCommandProc::processAuxCol(const std::vector<std::string>& origVals,
auxColTuple.data = (uint8_t)1;
auxColTuples.push_back(auxColTuple);
//@Bug 2515. Only pass string values to write engine
auxDicStrings.push_back("");
utils::NullString ns;
auxDicStrings.push_back(ns);
}
colValuesList.push_back(auxColTuples);
@ -206,7 +208,7 @@ uint8_t WE_DMLCommandProc::processSingleInsert(messageqcpp::ByteStream& bs, std:
dctnryStruct.fCharsetNumber = colType.charsetNumber;
if (colStruct.tokenFlag)
if (colStruct.tokenFlag)
{
dctnryStruct.dctnryOid = colType.ddn.dictOID;
dctnryStruct.columnOid = colStruct.dataOid;
@ -255,7 +257,7 @@ uint8_t WE_DMLCommandProc::processSingleInsert(messageqcpp::ByteStream& bs, std:
cscColTypeList.push_back(colType);
}
std::string tmpStr("");
NullString tmpStr;
for (unsigned int i = 0; i < numcols; i++)
{
@ -278,7 +280,7 @@ uint8_t WE_DMLCommandProc::processSingleInsert(messageqcpp::ByteStream& bs, std:
boost::any datavalue;
bool isNULL = false;
bool pushWarning = false;
std::vector<std::string> origVals;
std::vector<NullString> origVals;
origVals = columnPtr->get_DataVector();
WriteEngine::dictStr dicStrings;
@ -291,14 +293,11 @@ uint8_t WE_DMLCommandProc::processSingleInsert(messageqcpp::ByteStream& bs, std:
isNULL = columnPtr->get_isnull();
if (isNULL || (tmpStr.length() == 0))
isNULL = true;
else
isNULL = false;
isNULL = isNULL ? true : tmpStr.isNull();
if (colType.constraintType == CalpontSystemCatalog::NOTNULL_CONSTRAINT)
{
if (isNULL && colType.defaultValue.empty()) // error out
if (isNULL && colType.defaultValue.isNull()) // error out
{
Message::Args args;
args.add(tableColName.column);
@ -306,7 +305,7 @@ uint8_t WE_DMLCommandProc::processSingleInsert(messageqcpp::ByteStream& bs, std:
rc = 1;
return rc;
}
else if (isNULL && !(colType.defaultValue.empty()))
else if (isNULL && !(colType.defaultValue.isNull()))
{
tmpStr = colType.defaultValue;
}
@ -314,7 +313,7 @@ uint8_t WE_DMLCommandProc::processSingleInsert(messageqcpp::ByteStream& bs, std:
if (tmpStr.length() > (unsigned int)colType.colWidth)
{
tmpStr = tmpStr.substr(0, colType.colWidth);
tmpStr.assign(tmpStr.unsafeStringRef().substr(0, colType.colWidth));
if (!pushWarning)
{
@ -343,18 +342,13 @@ uint8_t WE_DMLCommandProc::processSingleInsert(messageqcpp::ByteStream& bs, std:
else
{
string x;
std::string indata;
NullString indata;
for (uint32_t i = 0; i < origVals.size(); i++)
{
indata = origVals[i];
isNULL = columnPtr->get_isnull();
if (isNULL || (indata.length() == 0))
isNULL = true;
else
isNULL = false;
isNULL = columnPtr->get_isnull() ? true : indata.isNull();
// check if autoincrement column and value is 0 or null
uint64_t nextVal = 1;
@ -376,7 +370,7 @@ uint8_t WE_DMLCommandProc::processSingleInsert(messageqcpp::ByteStream& bs, std:
}
}
if (colType.autoincrement && (isNULL || (indata.compare("0") == 0)))
if (colType.autoincrement && (isNULL || (indata.safeString("").compare("0") == 0)))
{
try
{
@ -398,13 +392,13 @@ uint8_t WE_DMLCommandProc::processSingleInsert(messageqcpp::ByteStream& bs, std:
ostringstream oss;
oss << nextVal;
indata = oss.str();
indata.assign(oss.str());
isNULL = false;
}
if (colType.constraintType == CalpontSystemCatalog::NOTNULL_CONSTRAINT)
{
if (isNULL && colType.defaultValue.empty()) // error out
if (isNULL && colType.defaultValue.isNull()) // error out
{
Message::Args args;
args.add(tableColName.column);
@ -412,7 +406,7 @@ uint8_t WE_DMLCommandProc::processSingleInsert(messageqcpp::ByteStream& bs, std:
rc = 1;
return rc;
}
else if (isNULL && !(colType.defaultValue.empty()))
else if (isNULL && !(colType.defaultValue.isNull()))
{
indata = colType.defaultValue;
isNULL = false;
@ -421,14 +415,14 @@ uint8_t WE_DMLCommandProc::processSingleInsert(messageqcpp::ByteStream& bs, std:
try
{
datavalue = colType.convertColumnData(indata, pushWarning, insertPkg.get_TimeZone(), isNULL,
datavalue = colType.convertColumnData(indata, pushWarning, insertPkg.get_TimeZone(),
false, false);
}
catch (exception&)
{
rc = 1;
Message::Args args;
args.add(string("'") + indata + string("'"));
args.add(string("'") + indata.safeString("<<null>>") + string("'"));
err = IDBErrorInfo::instance()->errorMsg(ERR_NON_NUMERIC_DATA, args);
}
@ -1185,7 +1179,7 @@ uint8_t WE_DMLCommandProc::processBatchInsert(messageqcpp::ByteStream& bs, std::
}
unsigned int numcols = rowPtr->get_NumberOfColumns();
std::string tmpStr("");
NullString tmpStr;
try
{
@ -1209,7 +1203,7 @@ uint8_t WE_DMLCommandProc::processBatchInsert(messageqcpp::ByteStream& bs, std::
boost::any datavalue;
bool isNULL = false;
std::vector<std::string> origVals;
std::vector<NullString> origVals;
origVals = columnPtr->get_DataVector();
WriteEngine::dictStr dicStrings;
@ -1220,14 +1214,11 @@ uint8_t WE_DMLCommandProc::processBatchInsert(messageqcpp::ByteStream& bs, std::
{
tmpStr = origVals[i];
if (tmpStr.length() == 0)
isNULL = true;
else
isNULL = false;
isNULL = tmpStr.isNull();
if (colType.constraintType == CalpontSystemCatalog::NOTNULL_CONSTRAINT)
{
if (isNULL && colType.defaultValue.empty()) // error out
if (isNULL && colType.defaultValue.isNull()) // error out
{
Message::Args args;
args.add(tableColName.column);
@ -1235,7 +1226,7 @@ uint8_t WE_DMLCommandProc::processBatchInsert(messageqcpp::ByteStream& bs, std::
rc = 1;
return rc;
}
else if (isNULL && !(colType.defaultValue.empty()))
else if (isNULL && !(colType.defaultValue.isNull()))
{
tmpStr = colType.defaultValue;
}
@ -1243,7 +1234,7 @@ uint8_t WE_DMLCommandProc::processBatchInsert(messageqcpp::ByteStream& bs, std::
if (tmpStr.length() > (unsigned int)colType.colWidth)
{
tmpStr = tmpStr.substr(0, colType.colWidth);
tmpStr.assign(tmpStr.unsafeStringRef().substr(0, colType.colWidth));
if (!pushWarning)
pushWarning = true;
@ -1264,7 +1255,7 @@ uint8_t WE_DMLCommandProc::processBatchInsert(messageqcpp::ByteStream& bs, std::
else
{
string x;
std::string indata;
NullString indata;
// scan once to check how many autoincrement value needed
uint32_t nextValNeeded = 0;
uint64_t nextVal = 1;
@ -1287,12 +1278,12 @@ uint8_t WE_DMLCommandProc::processBatchInsert(messageqcpp::ByteStream& bs, std::
{
indata = origVals[i];
if (indata.length() == 0)
if (indata.isNull())
isNULL = true;
else
isNULL = false;
if (isNULL || (indata.compare("0") == 0))
if (isNULL || (indata.safeString("").compare("0") == 0))
nextValNeeded++;
}
}
@ -1322,23 +1313,23 @@ uint8_t WE_DMLCommandProc::processBatchInsert(messageqcpp::ByteStream& bs, std::
{
indata = origVals[i];
if (indata.length() == 0)
if (indata.isNull())
isNULL = true;
else
isNULL = false;
// check if autoincrement column and value is 0 or null
if (colType.autoincrement && (isNULL || (indata.compare("0") == 0)))
if (colType.autoincrement && (isNULL || (indata.safeString("").compare("0") == 0)))
{
ostringstream oss;
oss << nextVal++;
indata = oss.str();
indata.assign(oss.str());
isNULL = false;
}
if (colType.constraintType == CalpontSystemCatalog::NOTNULL_CONSTRAINT)
{
if (isNULL && colType.defaultValue.empty()) // error out
if (isNULL && colType.defaultValue.isNull()) // error out
{
Message::Args args;
args.add(tableColName.column);
@ -1346,7 +1337,7 @@ uint8_t WE_DMLCommandProc::processBatchInsert(messageqcpp::ByteStream& bs, std::
rc = 1;
return rc;
}
else if (isNULL && !(colType.defaultValue.empty()))
else if (isNULL && !(colType.defaultValue.isNull()))
{
indata = colType.defaultValue;
isNULL = false;
@ -1355,14 +1346,14 @@ uint8_t WE_DMLCommandProc::processBatchInsert(messageqcpp::ByteStream& bs, std::
try
{
datavalue = colType.convertColumnData(indata, pushWarning, insertPkg.get_TimeZone(), isNULL,
datavalue = colType.convertColumnData(indata, pushWarning, insertPkg.get_TimeZone(),
false, false);
}
catch (exception&)
{
rc = 1;
Message::Args args;
args.add(string("'") + indata + string("'"));
args.add(string("'") + indata.safeString("<<null>>") + string("'"));
err = IDBErrorInfo::instance()->errorMsg(ERR_NON_NUMERIC_DATA, args);
}
@ -1772,7 +1763,7 @@ uint8_t WE_DMLCommandProc::processBatchInsertBinary(messageqcpp::ByteStream& bs,
return rc;
}
std::string tmpStr("");
NullString tmpStr;
uint32_t valuesPerColumn;
bs >> valuesPerColumn;
colValuesList.reserve(columnCount * valuesPerColumn);
@ -1798,16 +1789,13 @@ uint8_t WE_DMLCommandProc::processBatchInsertBinary(messageqcpp::ByteStream& bs,
{
for (uint32_t i = 0; i < valuesPerColumn; i++)
{
bs >> tmp8;
isNULL = tmp8;
bs >> tmpStr;
if (tmpStr.length() == 0)
isNULL = true;
isNULL = tmpStr.isNull();
if (colType.constraintType == CalpontSystemCatalog::NOTNULL_CONSTRAINT)
{
if (isNULL && colType.defaultValue.empty()) // error out
if (isNULL && colType.defaultValue.isNull()) // error out
{
Message::Args args;
args.add(tableColName.column);
@ -1815,7 +1803,7 @@ uint8_t WE_DMLCommandProc::processBatchInsertBinary(messageqcpp::ByteStream& bs,
rc = 1;
return rc;
}
else if (isNULL && !(colType.defaultValue.empty()))
else if (isNULL && !(colType.defaultValue.isNull()))
{
tmpStr = colType.defaultValue;
}
@ -1823,7 +1811,7 @@ uint8_t WE_DMLCommandProc::processBatchInsertBinary(messageqcpp::ByteStream& bs,
if (tmpStr.length() > (unsigned int)colType.colWidth)
{
tmpStr = tmpStr.substr(0, colType.colWidth);
tmpStr.assign(tmpStr.unsafeStringRef().substr(0, colType.colWidth));
if (!pushWarning)
pushWarning = true;
@ -1871,7 +1859,7 @@ uint8_t WE_DMLCommandProc::processBatchInsertBinary(messageqcpp::ByteStream& bs,
uint64_t colValue;
float valF;
double valD;
std::string valStr;
NullString valStr;
bool valZero = false; // Needed for autoinc check
switch (colType.colDataType)
@ -2058,22 +2046,25 @@ uint8_t WE_DMLCommandProc::processBatchInsertBinary(messageqcpp::ByteStream& bs,
case execplan::CalpontSystemCatalog::BLOB:
bs >> valStr;
if (valStr.length() > (unsigned int)colType.colWidth)
if (!valStr.isNull()) // null values do not require any padding or truncation.
{
valStr = valStr.substr(0, colType.colWidth);
pushWarning = true;
}
else
{
if ((unsigned int)colType.colWidth > valStr.length())
if (valStr.length() > (unsigned int)colType.colWidth)
{
// Pad null character to the string
valStr.resize(colType.colWidth, 0);
valStr = NullString(valStr.unsafeStringRef().substr(0, colType.colWidth));
pushWarning = true;
}
else
{
if ((unsigned int)colType.colWidth > valStr.length())
{
// Pad null character to the string
valStr.resize(colType.colWidth, 0);
}
}
}
// FIXME: colValue is uint64_t (8 bytes)
memcpy(&colValue, valStr.c_str(), valStr.length());
memcpy(&colValue, valStr.str(), valStr.length());
break;
default:
@ -2113,7 +2104,7 @@ uint8_t WE_DMLCommandProc::processBatchInsertBinary(messageqcpp::ByteStream& bs,
if (colType.constraintType == CalpontSystemCatalog::NOTNULL_CONSTRAINT)
{
if (isNULL && colType.defaultValue.empty()) // error out
if (isNULL && colType.defaultValue.isNull()) // error out
{
Message::Args args;
args.add(tableColName.column);
@ -2121,9 +2112,9 @@ uint8_t WE_DMLCommandProc::processBatchInsertBinary(messageqcpp::ByteStream& bs,
rc = 1;
return rc;
}
else if (isNULL && !(colType.defaultValue.empty()))
else if (isNULL && !(colType.defaultValue.isNull()))
{
memcpy(&colValue, colType.defaultValue.c_str(), colType.defaultValue.length());
memcpy(&colValue, colType.defaultValue.str(), colType.defaultValue.length());
isNULL = false;
}
}
@ -2771,7 +2762,7 @@ uint8_t WE_DMLCommandProc::processUpdate(messageqcpp::ByteStream& bs, std::strin
// get rows and values
rowgroup::Row row;
rowGroups[txnId]->initRow(&row);
string value("");
utils::NullString value;
uint32_t rowsThisRowgroup = rowGroups[txnId]->getRowCount();
uint32_t columnsSelected = rowGroups[txnId]->getColumnCount();
std::vector<execplan::CalpontSystemCatalog::ColDataType> fetchColTypes = rowGroups[txnId]->getColTypes();
@ -3000,6 +2991,7 @@ uint8_t WE_DMLCommandProc::processUpdate(messageqcpp::ByteStream& bs, std::strin
{
for (unsigned i = 0; i < rowsThisRowgroup; i++)
{
value.dropString();
rowGroups[txnId]->getRow(i, &row);
if (row.isNullValue(fetchColPos))
@ -3019,7 +3011,7 @@ uint8_t WE_DMLCommandProc::processUpdate(messageqcpp::ByteStream& bs, std::strin
if (value.length() > (unsigned int)colType.colWidth)
{
value = value.substr(0, colType.colWidth);
value.assign(value.safeString("").substr(0, colType.colWidth));
pushWarn = true;
if (!pushWarning)
@ -3032,7 +3024,7 @@ uint8_t WE_DMLCommandProc::processUpdate(messageqcpp::ByteStream& bs, std::strin
}
WriteEngine::DctnryTuple dctTuple;
dctTuple.sigValue = (unsigned char*)value.c_str();
dctTuple.sigValue = (unsigned char*)value.str();
dctTuple.sigSize = value.length();
dctTuple.isNull = false;
@ -3062,28 +3054,28 @@ uint8_t WE_DMLCommandProc::processUpdate(messageqcpp::ByteStream& bs, std::strin
case CalpontSystemCatalog::DATE:
{
intColVal = row.getUintField<4>(fetchColPos);
value = DataConvert::dateToString(intColVal);
value.assign(DataConvert::dateToString(intColVal));
break;
}
case CalpontSystemCatalog::DATETIME:
{
intColVal = row.getUintField<8>(fetchColPos);
value = DataConvert::datetimeToString(intColVal, colType.precision);
value.assign(DataConvert::datetimeToString(intColVal, colType.precision));
break;
}
case CalpontSystemCatalog::TIMESTAMP:
{
intColVal = row.getUintField<8>(fetchColPos);
value = DataConvert::timestampToString(intColVal, timeZone, colType.precision);
value.assign(DataConvert::timestampToString(intColVal, timeZone, colType.precision));
break;
}
case CalpontSystemCatalog::TIME:
{
intColVal = row.getIntField<8>(fetchColPos);
value = DataConvert::timeToString(intColVal, colType.precision);
value.assign(DataConvert::timeToString(intColVal, colType.precision));
break;
}
@ -3091,8 +3083,11 @@ uint8_t WE_DMLCommandProc::processUpdate(messageqcpp::ByteStream& bs, std::strin
case CalpontSystemCatalog::VARCHAR:
{
value = row.getStringField(fetchColPos);
unsigned i = strlen(value.c_str());
value = value.substr(0, i);
if (!value.isNull())
{
unsigned i = strlen(value.str());
value.assign(value.unsafeStringRef().substr(0, i));
}
break;
}
@ -3100,7 +3095,7 @@ uint8_t WE_DMLCommandProc::processUpdate(messageqcpp::ByteStream& bs, std::strin
case CalpontSystemCatalog::BLOB:
case CalpontSystemCatalog::TEXT:
{
value = row.getVarBinaryStringField(fetchColPos);
value.assign(row.getVarBinaryField(fetchColPos), row.getVarBinaryLength(fetchColPos));
break;
}
@ -3111,7 +3106,7 @@ uint8_t WE_DMLCommandProc::processUpdate(messageqcpp::ByteStream& bs, std::strin
{
datatypes::Decimal dec(row.getTSInt128Field(fetchColPos), fetchColScales[fetchColPos],
rowGroups[txnId]->getPrecision()[fetchColPos]);
value = dec.toString(true);
value.assign(dec.toString(true));
break;
}
}
@ -3145,13 +3140,13 @@ uint8_t WE_DMLCommandProc::processUpdate(messageqcpp::ByteStream& bs, std::strin
else
os << intColVal;
value = os.str();
value.assign(os.str());
}
else
{
datatypes::Decimal dec(intColVal, fetchColScales[fetchColPos],
rowGroups[txnId]->getPrecision()[fetchColPos]);
value = dec.toString();
value.assign(dec.toString());
}
}
break;
@ -3175,7 +3170,7 @@ uint8_t WE_DMLCommandProc::processUpdate(messageqcpp::ByteStream& bs, std::strin
ostringstream os;
//@Bug 3350 fix the precision.
os << setprecision(7) << dl;
value = os.str();
value.assign(os.str());
break;
}
@ -3195,7 +3190,7 @@ uint8_t WE_DMLCommandProc::processUpdate(messageqcpp::ByteStream& bs, std::strin
ostringstream os;
//@Bug 3350 fix the precision.
os << setprecision(16) << dl;
value = os.str();
value.assign(os.str());
break;
}
@ -3209,7 +3204,7 @@ uint8_t WE_DMLCommandProc::processUpdate(messageqcpp::ByteStream& bs, std::strin
ostringstream os;
//@Bug 3350 fix the precision.
os << setprecision(19) << dll;
value = os.str();
value.assign(os.str());
break;
}
@ -3218,7 +3213,7 @@ uint8_t WE_DMLCommandProc::processUpdate(messageqcpp::ByteStream& bs, std::strin
ostringstream os;
intColVal = row.getUintField<8>(fetchColPos);
os << intColVal;
value = os.str();
value.assign(os.str());
break;
}
}
@ -3227,27 +3222,30 @@ uint8_t WE_DMLCommandProc::processUpdate(messageqcpp::ByteStream& bs, std::strin
if (funcScale != 0)
{
string::size_type pos = value.find_first_of("."); // decimal point
string str = value.safeString("");
string::size_type pos = str.find_first_of("."); // decimal point
if (pos >= value.length())
value.insert(value.length(), ".");
if (pos >= str.length())
str.insert(str.length(), ".");
// padding 0 if needed
pos = value.find_first_of(".");
uint32_t digitsAfterPoint = value.length() - pos - 1;
pos = str.find_first_of(".");
uint32_t digitsAfterPoint = str.length() - pos - 1;
if (digitsAfterPoint < funcScale)
{
for (uint32_t i = 0; i < (funcScale - digitsAfterPoint); i++)
value += "0";
str += "0";
}
value.assign(str);
}
// check data length
// trim the string if needed
if (value.length() > (unsigned int)colType.colWidth)
{
value = value.substr(0, colType.colWidth);
value.assign(value.unsafeStringRef().substr(0, colType.colWidth));
if (!pushWarn)
pushWarn = true;
@ -3260,7 +3258,7 @@ uint8_t WE_DMLCommandProc::processUpdate(messageqcpp::ByteStream& bs, std::strin
}
WriteEngine::DctnryTuple dctTuple;
dctTuple.sigValue = (unsigned char*)value.c_str();
dctTuple.sigValue = (unsigned char*)value.str();
dctTuple.sigSize = value.length();
dctTuple.isNull = false;
@ -3305,7 +3303,7 @@ uint8_t WE_DMLCommandProc::processUpdate(messageqcpp::ByteStream& bs, std::strin
if (value.length() > (unsigned int)colType.colWidth)
{
value = value.substr(0, colType.colWidth);
value.assign(value.safeString("").substr(0, colType.colWidth));
pushWarn = true;
if (!pushWarning)
@ -3318,7 +3316,7 @@ uint8_t WE_DMLCommandProc::processUpdate(messageqcpp::ByteStream& bs, std::strin
}
WriteEngine::DctnryTuple dctTuple;
dctTuple.sigValue = (unsigned char*)value.c_str();
dctTuple.sigValue = (unsigned char*)value.str();
dctTuple.sigSize = value.length();
dctTuple.isNull = false;
error = fWEWrapper.tokenize(txnId, dctTuple, colType.compressionType);
@ -3348,11 +3346,12 @@ uint8_t WE_DMLCommandProc::processUpdate(messageqcpp::ByteStream& bs, std::strin
}
else
{
value = columnsUpdated[j]->get_Data();
idbassert(!columnsUpdated[j]->get_DataVector()[0].isNull());
value = columnsUpdated[j]->get_DataVector()[0];
if (value.length() > (unsigned int)colType.colWidth)
{
value = value.substr(0, colType.colWidth);
value.assign(value.unsafeStringRef().substr(0, colType.colWidth));
pushWarn = true;
if (!pushWarning)
@ -3365,7 +3364,7 @@ uint8_t WE_DMLCommandProc::processUpdate(messageqcpp::ByteStream& bs, std::strin
}
WriteEngine::DctnryTuple dctTuple;
dctTuple.sigValue = (unsigned char*)value.c_str();
dctTuple.sigValue = (unsigned char*)value.str();
dctTuple.sigSize = value.length();
dctTuple.isNull = false;
error = fWEWrapper.tokenize(txnId, dctTuple, colType.compressionType);
@ -3405,7 +3404,7 @@ uint8_t WE_DMLCommandProc::processUpdate(messageqcpp::ByteStream& bs, std::strin
if (row.isNullValue(fetchColPos))
{
isNull = true;
value = "";
value.dropString();
}
else
{
@ -3416,28 +3415,28 @@ uint8_t WE_DMLCommandProc::processUpdate(messageqcpp::ByteStream& bs, std::strin
case CalpontSystemCatalog::DATE:
{
intColVal = row.getUintField<4>(fetchColPos);
value = DataConvert::dateToString(intColVal);
value.assign(DataConvert::dateToString(intColVal));
break;
}
case CalpontSystemCatalog::DATETIME:
{
intColVal = row.getUintField<8>(fetchColPos);
value = DataConvert::datetimeToString(intColVal, colType.precision);
value.assign(DataConvert::datetimeToString(intColVal, colType.precision));
break;
}
case CalpontSystemCatalog::TIMESTAMP:
{
intColVal = row.getUintField<8>(fetchColPos);
value = DataConvert::timestampToString(intColVal, timeZone, colType.precision);
value.assign(DataConvert::timestampToString(intColVal, timeZone, colType.precision));
break;
}
case CalpontSystemCatalog::TIME:
{
intColVal = row.getIntField<8>(fetchColPos);
value = DataConvert::timeToString(intColVal, colType.precision);
value.assign(DataConvert::timeToString(intColVal, colType.precision));
break;
}
@ -3445,8 +3444,11 @@ uint8_t WE_DMLCommandProc::processUpdate(messageqcpp::ByteStream& bs, std::strin
case CalpontSystemCatalog::VARCHAR:
{
value = row.getStringField(fetchColPos);
unsigned i = strlen(value.c_str());
value = value.substr(0, i);
if (!value.isNull())
{
unsigned i = strlen(value.str());
value.assign(value.safeString().substr(0, i)); // XXX: why???
}
break;
}
@ -3454,7 +3456,7 @@ uint8_t WE_DMLCommandProc::processUpdate(messageqcpp::ByteStream& bs, std::strin
case CalpontSystemCatalog::BLOB:
case CalpontSystemCatalog::TEXT:
{
value = row.getVarBinaryStringField(fetchColPos);
value.assign(row.getVarBinaryField(fetchColPos), row.getVarBinaryLength(fetchColPos));
break;
}
@ -3465,7 +3467,7 @@ uint8_t WE_DMLCommandProc::processUpdate(messageqcpp::ByteStream& bs, std::strin
{
datatypes::Decimal dec(row.getTSInt128Field(fetchColPos), fetchColScales[fetchColPos],
rowGroups[txnId]->getPrecision()[fetchColPos]);
value = dec.toString(true);
value.assign(dec.toString(true));
break;
}
}
@ -3499,13 +3501,13 @@ uint8_t WE_DMLCommandProc::processUpdate(messageqcpp::ByteStream& bs, std::strin
else
os << intColVal;
value = os.str();
value.assign(os.str());
}
else
{
datatypes::Decimal dec(intColVal, fetchColScales[fetchColPos],
rowGroups[txnId]->getPrecision()[fetchColPos]);
value = dec.toString();
value.assign(dec.toString());
}
}
break;
@ -3529,7 +3531,7 @@ uint8_t WE_DMLCommandProc::processUpdate(messageqcpp::ByteStream& bs, std::strin
ostringstream os;
//@Bug 3350 fix the precision.
os << setprecision(7) << dl;
value = os.str();
value.assign(os.str());
break;
}
@ -3549,7 +3551,7 @@ uint8_t WE_DMLCommandProc::processUpdate(messageqcpp::ByteStream& bs, std::strin
ostringstream os;
//@Bug 3350 fix the precision.
os << setprecision(16) << dl;
value = os.str();
value.assign(os.str());
break;
}
@ -3563,7 +3565,7 @@ uint8_t WE_DMLCommandProc::processUpdate(messageqcpp::ByteStream& bs, std::strin
ostringstream os;
//@Bug 3350 fix the precision.
os << setprecision(19) << dll;
value = os.str();
value.assign(os.str());
break;
}
@ -3572,7 +3574,7 @@ uint8_t WE_DMLCommandProc::processUpdate(messageqcpp::ByteStream& bs, std::strin
ostringstream os;
intColVal = row.getUintField<8>(fetchColPos);
os << intColVal;
value = os.str();
value.assign(os.str());
break;
}
}
@ -3582,24 +3584,26 @@ uint8_t WE_DMLCommandProc::processUpdate(messageqcpp::ByteStream& bs, std::strin
if (funcScale != 0)
{
string::size_type pos = value.find_first_of("."); // decimal point
string str = value.safeString("");
string::size_type pos = str.find_first_of("."); // decimal point
if (pos >= value.length())
value.insert(value.length(), ".");
if (pos >= str.length())
str.insert(str.length(), ".");
// padding 0 if needed
pos = value.find_first_of(".");
uint32_t digitsAfterPoint = value.length() - pos - 1;
pos = str.find_first_of(".");
uint32_t digitsAfterPoint = str.length() - pos - 1;
if (digitsAfterPoint < funcScale)
{
for (uint32_t i = 0; i < (funcScale - digitsAfterPoint); i++)
value += "0";
str += "0";
}
value.assign(str);
}
// Check NOT NULL constraint and default value
if ((isNull) && (colType.defaultValue.length() <= 0) &&
if ((isNull) && (colType.defaultValue.isNull()) &&
(colType.constraintType == CalpontSystemCatalog::NOTNULL_CONSTRAINT))
{
rc = 1;
@ -3608,7 +3612,7 @@ uint8_t WE_DMLCommandProc::processUpdate(messageqcpp::ByteStream& bs, std::strin
err = IDBErrorInfo::instance()->errorMsg(ERR_NOT_NULL_CONSTRAINTS, args);
return rc;
}
else if ((isNull) && (colType.defaultValue.length() > 0))
else if ((isNull) && (!colType.defaultValue.isNull()))
{
isNull = false;
bool oneWarn = false;
@ -3616,14 +3620,14 @@ uint8_t WE_DMLCommandProc::processUpdate(messageqcpp::ByteStream& bs, std::strin
try
{
datavalue =
colType.convertColumnData(colType.defaultValue, pushWarn, timeZone, isNull, false, false);
colType.convertColumnData(colType.defaultValue.safeString(""), pushWarn, timeZone, isNull, false, false);
}
catch (exception&)
{
//@Bug 2624. Error out on conversion failure
rc = 1;
Message::Args args;
args.add(string("'") + colType.defaultValue + string("'"));
args.add(string("'") + colType.defaultValue.safeString() + string("'"));
err = IDBErrorInfo::instance()->errorMsg(ERR_NON_NUMERIC_DATA, args);
}
@ -3648,14 +3652,14 @@ uint8_t WE_DMLCommandProc::processUpdate(messageqcpp::ByteStream& bs, std::strin
{
try
{
datavalue = colType.convertColumnData(value, pushWarn, timeZone, isNull, false, false);
datavalue = colType.convertColumnData(value.safeString(""), pushWarn, timeZone, isNull, false, false);
}
catch (exception&)
{
//@Bug 2624. Error out on conversion failure
rc = 1;
Message::Args args;
args.add(string("'") + value + string("'"));
args.add(string("'") + value.safeString() + string("'"));
err = IDBErrorInfo::instance()->errorMsg(ERR_NON_NUMERIC_DATA, args);
return rc;
}
@ -3686,14 +3690,19 @@ uint8_t WE_DMLCommandProc::processUpdate(messageqcpp::ByteStream& bs, std::strin
isNull = false;
}
string inData(columnsUpdated[j]->get_Data());
if (((colType.colDataType == execplan::CalpontSystemCatalog::DATE) && (inData == "0000-00-00")) ||
((colType.colDataType == execplan::CalpontSystemCatalog::DATETIME) &&
(inData == "0000-00-00 00:00:00")) ||
((colType.colDataType == execplan::CalpontSystemCatalog::TIMESTAMP) &&
(inData == "0000-00-00 00:00:00")))
NullString inData;
if (!isNull)
{
inData = columnsUpdated[j]->get_DataVector()[0];
}
if (((colType.colDataType == execplan::CalpontSystemCatalog::DATE) && (inData.safeString("").compare("0000-00-00") == 0)) ||
((colType.colDataType == execplan::CalpontSystemCatalog::DATETIME) &&
(inData.safeString("").compare("0000-00-00 00:00:00") == 0)) ||
((colType.colDataType == execplan::CalpontSystemCatalog::TIMESTAMP) &&
(inData.safeString("").compare("0000-00-00 00:00:00") == 0)))
{
inData.dropString();
isNull = true;
}
@ -3714,7 +3723,7 @@ uint8_t WE_DMLCommandProc::processUpdate(messageqcpp::ByteStream& bs, std::strin
}
}
if (colType.autoincrement && (isNull || (inData.compare("0") == 0)))
if (colType.autoincrement && (isNull || (inData.safeString("").compare("0") == 0)))
{
// reserve nextVal
try
@ -3742,18 +3751,18 @@ uint8_t WE_DMLCommandProc::processUpdate(messageqcpp::ByteStream& bs, std::strin
{
ostringstream oss;
oss << nextVal++;
inData = oss.str();
inData.assign(oss.str());
try
{
datavalue = colType.convertColumnData(inData, pushWarn, timeZone, isNull, false, false);
datavalue = colType.convertColumnData(inData, pushWarn, timeZone, false, false);
}
catch (exception&)
{
//@Bug 2624. Error out on conversion failure
rc = 1;
Message::Args args;
args.add(string("'") + inData + string("'"));
args.add(string("'") + inData.safeString() + string("'"));
err = IDBErrorInfo::instance()->errorMsg(ERR_NON_NUMERIC_DATA, args);
}
@ -3794,14 +3803,14 @@ uint8_t WE_DMLCommandProc::processUpdate(messageqcpp::ByteStream& bs, std::strin
try
{
datavalue =
colType.convertColumnData(colType.defaultValue, pushWarn, timeZone, isNull, false, false);
colType.convertColumnData(colType.defaultValue.safeString(), pushWarn, timeZone, isNull, false, false);
}
catch (exception&)
{
//@Bug 2624. Error out on conversion failure
rc = 1;
Message::Args args;
args.add(string("'") + colType.defaultValue + string("'"));
args.add(string("'") + colType.defaultValue.safeString() + string("'"));
err = IDBErrorInfo::instance()->errorMsg(ERR_NON_NUMERIC_DATA, args);
}
@ -3827,7 +3836,7 @@ uint8_t WE_DMLCommandProc::processUpdate(messageqcpp::ByteStream& bs, std::strin
{
try
{
datavalue = colType.convertColumnData(inData, pushWarn, timeZone, isNull, false, true);
datavalue = colType.convertColumnData(inData, pushWarn, timeZone, false, true);
}
catch (exception& ex)
{
@ -3835,7 +3844,7 @@ uint8_t WE_DMLCommandProc::processUpdate(messageqcpp::ByteStream& bs, std::strin
rc = 1;
cout << ex.what() << endl;
Message::Args args;
args.add(string("'") + inData + string("'"));
args.add(string("'") + inData.safeString() + string("'"));
err = IDBErrorInfo::instance()->errorMsg(ERR_NON_NUMERIC_DATA, args);
return rc;
}

3
writeengine/server/we_dmlcommandproc.h
View File

@ -39,6 +39,7 @@
#include "we_rbmetawriter.h"
#include "rowgroup.h"
#include "we_log.h"
#include "nullstring.h"
#define EXPORT
@ -123,7 +124,7 @@ class WE_DMLCommandProc
const std::vector<BRM::FileInfo>& files,
const std::vector<BRM::OID_t>& oidsToFlush, std::string& err);
void processAuxCol(const std::vector<std::string>& origVals,
void processAuxCol(const std::vector<utils::NullString>& origVals,
WriteEngine::ColValueList& colValuesList,
WriteEngine::DictStrList& dicStringList);

5
writeengine/shared/we_type.h
View File

@ -39,6 +39,7 @@
#include "calpontsystemcatalog.h"
#include "IDBDataFile.h"
#include "IDBPolicy.h"
#include "nullstring.h"
#undef EXPORT
#undef DELETE
@ -331,7 +332,7 @@ typedef std::vector<ColTupleList> ColValueList; /** @
typedef std::vector<RID> RIDList; /** @brief RID list */
typedef std::vector<execplan::CalpontSystemCatalog::ColType> CSCTypesList; /** @brief CSC column types list */
typedef std::vector<std::string> dictStr;
typedef std::vector<NullString> dictStr;
typedef std::vector<dictStr> DictStrList;
// dictionary
@ -408,7 +409,7 @@ struct JobColumn /** @brief Job Column Structure */
unsigned long long fDefaultUInt; /** @brief UnsignedInt col default*/
double fDefaultDbl; /** @brief Dbl/Flt column default */
int128_t fDefaultWideDecimal; /** @brief Wide decimal column default */
std::string fDefaultChr; /** @brief Char column default */
utils::NullString fDefaultChr; /** @brief Char column default */
JobColumn()
: mapOid(0)
, dataType(execplan::CalpontSystemCatalog::INT)

40
writeengine/wrapper/writeengine.cpp
View File

@ -791,7 +791,9 @@ void WriteEngineWrapper::convertValue(const CalpontSystemCatalog::ColType& cscCo
case WriteEngine::WR_ULONGLONG: ((uint64_t*)valArray)[pos] = boost::any_cast<uint64_t>(data); break;
case WriteEngine::WR_TOKEN: ((Token*)valArray)[pos] = boost::any_cast<Token>(data); break;
case WriteEngine::WR_TOKEN:
((Token*)valArray)[pos] = boost::any_cast<Token>(data);
break;
case WriteEngine::WR_BINARY:
size_t size = cscColType.colWidth;
@ -1714,7 +1716,7 @@ int WriteEngineWrapper::insertColumnRecs(
#if defined(XXX_WRITEENGINE_TOKENS_RANGES_XXX)
int64_t strPrefix;
#endif
if (dctStr_iter->length() == 0)
if (dctStr_iter->isNull())
{
Token nullToken;
col_iter->data = nullToken;
@ -1728,10 +1730,10 @@ int WriteEngineWrapper::insertColumnRecs(
timer.start("tokenize");
#endif
DctnryTuple dctTuple;
dctTuple.sigValue = (unsigned char*)dctStr_iter->c_str();
dctTuple.sigValue = (unsigned char*)dctStr_iter->str();
dctTuple.sigSize = dctStr_iter->length();
#if defined(XXX_WRITEENGINE_TOKENS_RANGES_XXX)
strPrefix = encodeStringPrefix_check_null(dctTuple.sigValue, dctTuple.sigSize,
strPrefix = encodeStringPrefix(dctTuple.sigValue, dctTuple.sigSize,
dctnryStructList[i].fCharsetNumber);
#endif
dctTuple.isNull = false;
@ -1782,7 +1784,7 @@ int WriteEngineWrapper::insertColumnRecs(
#if defined(XXX_WRITEENGINE_TOKENS_RANGES_XXX)
int64_t strPrefix;
#endif
if (dctStr_iter->length() == 0)
if (dctStr_iter->isNull())
{
Token nullToken;
col_iter->data = nullToken;
@ -1796,7 +1798,7 @@ int WriteEngineWrapper::insertColumnRecs(
timer.start("tokenize");
#endif
DctnryTuple dctTuple;
dctTuple.sigValue = (unsigned char*)dctStr_iter->c_str();
dctTuple.sigValue = (unsigned char*)dctStr_iter->str();
dctTuple.sigSize = dctStr_iter->length();
#if defined(XXX_WRITEENGINE_TOKENS_RANGES_XXX)
strPrefix = encodeStringPrefix_check_null(dctTuple.sigValue, dctTuple.sigSize,
@ -2532,7 +2534,7 @@ int WriteEngineWrapper::insertColumnRecsBinary(
{
colValPtr = &colValueList[(i * rowsPerColumn) + rows];
if (dctStr_iter->length() == 0)
if (dctStr_iter->isNull())
{
Token nullToken;
memcpy(colValPtr, &nullToken, 8);
@ -2543,7 +2545,7 @@ int WriteEngineWrapper::insertColumnRecsBinary(
timer.start("tokenize");
#endif
DctnryTuple dctTuple;
dctTuple.sigValue = (unsigned char*)dctStr_iter->c_str();
dctTuple.sigValue = (unsigned char*)dctStr_iter->str();
dctTuple.sigSize = dctStr_iter->length();
dctTuple.isNull = false;
rc = tokenize(txnid, dctTuple, dctnryStructList[i].fCompressionType);
@ -2589,7 +2591,7 @@ int WriteEngineWrapper::insertColumnRecsBinary(
{
colValPtr = &colValueList[(i * rowsPerColumn) + rows];
if (dctStr_iter->length() == 0)
if (dctStr_iter->isNull())
{
Token nullToken;
memcpy(colValPtr, &nullToken, 8);
@ -2600,7 +2602,7 @@ int WriteEngineWrapper::insertColumnRecsBinary(
timer.start("tokenize");
#endif
DctnryTuple dctTuple;
dctTuple.sigValue = (unsigned char*)dctStr_iter->c_str();
dctTuple.sigValue = (unsigned char*)dctStr_iter->str();
dctTuple.sigSize = dctStr_iter->length();
dctTuple.isNull = false;
rc = tokenize(txnid, dctTuple, newDctnryStructList[i].fCompressionType);
@ -3107,7 +3109,7 @@ int WriteEngineWrapper::insertColumnRec_SYS(const TxnID& txnid, const CSCTypesLi
for (uint32_t rows = 0; rows < (totalRow - rowsLeft); rows++)
{
if (dctStr_iter->length() == 0)
if (dctStr_iter->isNull())
{
Token nullToken;
col_iter->data = nullToken;
@ -3118,7 +3120,7 @@ int WriteEngineWrapper::insertColumnRec_SYS(const TxnID& txnid, const CSCTypesLi
timer.start("tokenize");
#endif
DctnryTuple dctTuple;
dctTuple.sigValue = (unsigned char*)dctStr_iter->c_str();
dctTuple.sigValue = (unsigned char*)dctStr_iter->str();
dctTuple.sigSize = dctStr_iter->length();
dctTuple.isNull = false;
rc = tokenize(txnid, dctTuple, dctnryStructList[i].fCompressionType);
@ -3181,7 +3183,7 @@ int WriteEngineWrapper::insertColumnRec_SYS(const TxnID& txnid, const CSCTypesLi
for (uint32_t rows = 0; rows < rowsLeft; rows++)
{
if (dctStr_iter->length() == 0)
if (dctStr_iter->isNull())
{
Token nullToken;
col_iter->data = nullToken;
@ -3192,7 +3194,7 @@ int WriteEngineWrapper::insertColumnRec_SYS(const TxnID& txnid, const CSCTypesLi
timer.start("tokenize");
#endif
DctnryTuple dctTuple;
dctTuple.sigValue = (unsigned char*)dctStr_iter->c_str();
dctTuple.sigValue = (unsigned char*)dctStr_iter->str();
dctTuple.sigSize = dctStr_iter->length();
dctTuple.isNull = false;
rc = tokenize(txnid, dctTuple, newDctnryStructList[i].fCompressionType);
@ -3769,7 +3771,7 @@ int WriteEngineWrapper::insertColumnRec_Single(const TxnID& txnid, const CSCType
for (uint32_t rows = 0; rows < (totalRow - rowsLeft); rows++)
{
if (dctStr_iter->length() == 0)
if (dctStr_iter->isNull())
{
Token nullToken;
col_iter->data = nullToken;
@ -3780,7 +3782,7 @@ int WriteEngineWrapper::insertColumnRec_Single(const TxnID& txnid, const CSCType
timer.start("tokenize");
#endif
DctnryTuple dctTuple;
dctTuple.sigValue = (unsigned char*)dctStr_iter->c_str();
dctTuple.sigValue = (unsigned char*)dctStr_iter->str();
dctTuple.sigSize = dctStr_iter->length();
dctTuple.isNull = false;
rc = tokenize(txnid, dctTuple, dctnryStructList[i].fCompressionType);
@ -3844,7 +3846,7 @@ int WriteEngineWrapper::insertColumnRec_Single(const TxnID& txnid, const CSCType
for (uint32_t rows = 0; rows < rowsLeft; rows++)
{
if (dctStr_iter->length() == 0)
if (dctStr_iter->isNull())
{
Token nullToken;
col_iter->data = nullToken;
@ -3855,7 +3857,7 @@ int WriteEngineWrapper::insertColumnRec_Single(const TxnID& txnid, const CSCType
timer.start("tokenize");
#endif
DctnryTuple dctTuple;
dctTuple.sigValue = (unsigned char*)dctStr_iter->c_str();
dctTuple.sigValue = (unsigned char*)dctStr_iter->str();
dctTuple.sigSize = dctStr_iter->length();
dctTuple.isNull = false;
rc = tokenize(txnid, dctTuple, newDctnryStructList[i].fCompressionType);
@ -4218,7 +4220,7 @@ void WriteEngineWrapper::printInputValue(const ColStructList& colStructList, con
{
// We presume there will be a value.
auto tokenOidIdx = oidToIdxMap[dctnryStructList[i].columnOid];
std::cerr << "string [" << dictStrList[i][j] << "]" << std::endl;
std::cerr << "string [" << dictStrList[i][j].safeString("<<null>>") << "]" << std::endl;
bool isToken = colStructList[tokenOidIdx].colType == WriteEngine::WR_TOKEN &&
colStructList[tokenOidIdx].tokenFlag;
if (isToken && !colValueList[tokenOidIdx][j].data.empty())

9
writeengine/xml/we_xmlgenproc.cpp
View File

@ -325,23 +325,24 @@ bool XMLGenProc::makeColumnData(const CalpontSystemCatalog::TableName& table)
}
// Include NotNull and Default value
const std::string col_defaultValue(col->colType.defaultValue);
const NullString col_defaultValue(col->colType.defaultValue);
if (col->colType.constraintType == execplan::CalpontSystemCatalog::NOTNULL_CONSTRAINT)
{
int notNull = 1;
xmlTextWriterWriteFormatAttribute(fWriter, BAD_CAST xmlTagTable[TAG_NOT_NULL], "%d", notNull);
if (!col_defaultValue.empty())
if (!col_defaultValue.isNull())
{
xmlTextWriterWriteAttribute(fWriter, BAD_CAST xmlTagTable[TAG_DEFAULT_VALUE],
BAD_CAST col_defaultValue.c_str());
BAD_CAST col_defaultValue.unsafeStringRef().c_str());
}
}
else if (col->colType.constraintType == execplan::CalpontSystemCatalog::DEFAULT_CONSTRAINT)
{
idbassert(!col_defaultValue.isNull()); // good enough for now. I have to figure out how to store the null.
xmlTextWriterWriteAttribute(fWriter, BAD_CAST xmlTagTable[TAG_DEFAULT_VALUE],
BAD_CAST col_defaultValue.c_str());
BAD_CAST (col_defaultValue.unsafeStringRef().c_str()));
}
} // end of "if fSysCatRpt"

22
writeengine/xml/we_xmljob.cpp
View File

@ -930,13 +930,13 @@ void XMLJob::fillInXMLDataAsLoaded(execplan::CalpontSystemCatalog::RIDList& colR
void XMLJob::fillInXMLDataNotNullDefault(const std::string& fullTblName,
execplan::CalpontSystemCatalog::ColType& colType, JobColumn& col)
{
const std::string col_defaultValue(colType.defaultValue);
const NullString col_defaultValue(colType.defaultValue);
if (colType.constraintType == execplan::CalpontSystemCatalog::NOTNULL_CONSTRAINT)
{
col.fNotNull = true;
if (!col_defaultValue.empty())
if (!col_defaultValue.isNull())
col.fWithDefault = true;
}
else if (colType.constraintType == execplan::CalpontSystemCatalog::DEFAULT_CONSTRAINT)
@ -963,7 +963,7 @@ void XMLJob::fillInXMLDataNotNullDefault(const std::string& fullTblName,
case execplan::CalpontSystemCatalog::BIGINT:
{
errno = 0;
col.fDefaultInt = strtoll(col_defaultValue.c_str(), 0, 10);
col.fDefaultInt = strtoll(col_defaultValue.str(), 0, 10);
if (errno == ERANGE)
bDefaultConvertError = true;
@ -978,7 +978,7 @@ void XMLJob::fillInXMLDataNotNullDefault(const std::string& fullTblName,
case execplan::CalpontSystemCatalog::UBIGINT:
{
errno = 0;
col.fDefaultUInt = strtoull(col_defaultValue.c_str(), 0, 10);
col.fDefaultUInt = strtoull(col_defaultValue.str(), 0, 10);
if (errno == ERANGE)
bDefaultConvertError = true;
@ -991,11 +991,11 @@ void XMLJob::fillInXMLDataNotNullDefault(const std::string& fullTblName,
{
if (LIKELY(colType.colWidth == datatypes::MAXDECIMALWIDTH))
{
col.fDefaultWideDecimal = colType.decimal128FromString(col_defaultValue, &bDefaultConvertError);
col.fDefaultWideDecimal = colType.decimal128FromString(col_defaultValue.safeString(), &bDefaultConvertError);
}
else
{
col.fDefaultInt = Convertor::convertDecimalString(col_defaultValue.c_str(),
col.fDefaultInt = Convertor::convertDecimalString(col_defaultValue.str(),
col_defaultValue.length(), colType.scale);
if (errno == ERANGE)
@ -1008,7 +1008,7 @@ void XMLJob::fillInXMLDataNotNullDefault(const std::string& fullTblName,
case execplan::CalpontSystemCatalog::DATE:
{
int convertStatus;
int32_t dt = dataconvert::DataConvert::convertColumnDate(col_defaultValue.c_str(),
int32_t dt = dataconvert::DataConvert::convertColumnDate(col_defaultValue.str(),
dataconvert::CALPONTDATE_ENUM, convertStatus,
col_defaultValue.length());
@ -1023,7 +1023,7 @@ void XMLJob::fillInXMLDataNotNullDefault(const std::string& fullTblName,
{
int convertStatus;
int64_t dt = dataconvert::DataConvert::convertColumnDatetime(
col_defaultValue.c_str(), dataconvert::CALPONTDATETIME_ENUM, convertStatus,
col_defaultValue.str(), dataconvert::CALPONTDATETIME_ENUM, convertStatus,
col_defaultValue.length());
if (convertStatus != 0)
@ -1037,7 +1037,7 @@ void XMLJob::fillInXMLDataNotNullDefault(const std::string& fullTblName,
{
int convertStatus;
int64_t dt = dataconvert::DataConvert::convertColumnTimestamp(
col_defaultValue.c_str(), dataconvert::CALPONTDATETIME_ENUM, convertStatus,
col_defaultValue.str(), dataconvert::CALPONTDATETIME_ENUM, convertStatus,
col_defaultValue.length(), fTimeZone);
if (convertStatus != 0)
@ -1050,7 +1050,7 @@ void XMLJob::fillInXMLDataNotNullDefault(const std::string& fullTblName,
case execplan::CalpontSystemCatalog::TIME:
{
int convertStatus;
int64_t dt = dataconvert::DataConvert::convertColumnTime(col_defaultValue.c_str(),
int64_t dt = dataconvert::DataConvert::convertColumnTime(col_defaultValue.str(),
dataconvert::CALPONTTIME_ENUM, convertStatus,
col_defaultValue.length());
@ -1067,7 +1067,7 @@ void XMLJob::fillInXMLDataNotNullDefault(const std::string& fullTblName,
case execplan::CalpontSystemCatalog::UDOUBLE:
{
errno = 0;
col.fDefaultDbl = strtod(col_defaultValue.c_str(), 0);
col.fDefaultDbl = strtod(col_defaultValue.str(), 0);
if (errno == ERANGE)
bDefaultConvertError = true;