database/mariadb-columnstore-engine
1
0
Fork 0
mirror of https://github.com/mariadb-corporation/mariadb-columnstore-engine.git synced 2025-07-27 21:01:50 +03:00
Code Activity

Add support for zero date separate to NULL

Browse Source 
NULL is now pushed through the MariaDB storage engine plugin down to the
insert processing. A '0000-00-00' date is now a separate value to NULL.

This is more in-line with MariaDB's handling.
This commit is contained in:
Andrew Hutchings
2016-08-30 16:26:36 +01:00
parent 9a76bfc0f3
commit 25851df89d
16 changed files with 34 additions and 37 deletions
Download Patch File Download Diff File Expand all files Collapse all files

4
dbcon/dmlpackage/calpontdmlfactory.cpp
View File

@ -174,7 +174,7 @@ dmlpackage::CalpontDMLPackage* CalpontDMLFactory::makeCalpontDMLPackageFromMysql
{ {
case DML_INSERT: case DML_INSERT:
packagePtr = new InsertDMLPackage(vpackage.get_SchemaName(), vpackage.get_TableName(), vpackage.get_DMLStatement(), vpackage.get_SessionID()); packagePtr = new InsertDMLPackage(vpackage.get_SchemaName(), vpackage.get_TableName(), vpackage.get_DMLStatement(), vpackage.get_SessionID());
(void)packagePtr->buildFromMysqlBuffer(vpackage.get_ColNames(), vpackage.get_values(), vpackage.get_Columns(), vpackage.get_Rows()); (void)packagePtr->buildFromMysqlBuffer(vpackage.get_ColNames(), vpackage.get_values(), vpackage.get_Columns(), vpackage.get_Rows(), vpackage.get_nullValues());
break; break;
case DML_COMMAND: case DML_COMMAND:
packagePtr = new CommandDMLPackage(vpackage.get_DMLStatement(), vpackage.get_SessionID() ); packagePtr = new CommandDMLPackage(vpackage.get_DMLStatement(), vpackage.get_SessionID() );
@ -182,7 +182,7 @@ dmlpackage::CalpontDMLPackage* CalpontDMLFactory::makeCalpontDMLPackageFromMysql
case DML_DELETE: case DML_DELETE:
packagePtr = new DeleteDMLPackage(vpackage.get_SchemaName(), vpackage.get_TableName(), packagePtr = new DeleteDMLPackage(vpackage.get_SchemaName(), vpackage.get_TableName(),
vpackage.get_DMLStatement(), vpackage.get_SessionID() ); vpackage.get_DMLStatement(), vpackage.get_SessionID() );
(void)packagePtr->buildFromMysqlBuffer(vpackage.get_ColNames(), vpackage.get_values(), vpackage.get_Columns(), vpackage.get_Rows()); (void)packagePtr->buildFromMysqlBuffer(vpackage.get_ColNames(), vpackage.get_values(), vpackage.get_Columns(), vpackage.get_Rows(), vpackage.get_nullValues());
break; break;
default: default:
cerr << "makeCalpontDMLPackage: invalid statement type" << endl; cerr << "makeCalpontDMLPackage: invalid statement type" << endl;

2
dbcon/dmlpackage/calpontdmlpackage.h
View File

@ -94,7 +94,7 @@ namespace dmlpackage
* @param columns number of columns in the table * @param columns number of columns in the table
* @param rows number of rows to be touched * @param rows number of rows to be touched
*/ */
virtual int buildFromMysqlBuffer(ColNameList& colNameList, TableValuesMap& tableValuesMap, int columns, int rows) = 0; virtual int buildFromMysqlBuffer(ColNameList& colNameList, TableValuesMap& tableValuesMap, int columns, int rows, NullValuesBitset& nullValues) = 0;
/** @brief get the table object /** @brief get the table object
*/ */

2
dbcon/dmlpackage/commanddmlpackage.h
View File

@ -86,7 +86,7 @@ namespace dmlpackage
* @param colNameList, tableValuesMap * @param colNameList, tableValuesMap
* @param rows the number of rows in the buffer * @param rows the number of rows in the buffer
*/ */
int buildFromMysqlBuffer(ColNameList& colNameList, TableValuesMap& tableValuesMap, int columns, int rows) int buildFromMysqlBuffer(ColNameList& colNameList, TableValuesMap& tableValuesMap, int columns, int rows, NullValuesBitset& nullValues)
{ {
return 1; return 1;
}; };

2
dbcon/dmlpackage/deletedmlpackage.cpp
View File

@ -189,7 +189,7 @@ int DeleteDMLPackage::buildFromBuffer(std::string& buffer, int columns, int rows
} }
int DeleteDMLPackage::buildFromMysqlBuffer(ColNameList& colNameList, TableValuesMap& tableValuesMap, int columns, int rows ) int DeleteDMLPackage::buildFromMysqlBuffer(ColNameList& colNameList, TableValuesMap& tableValuesMap, int columns, int rows, NullValuesBitset& nullValues )
{ {
int retval = 1; int retval = 1;

2
dbcon/dmlpackage/deletedmlpackage.h
View File

@ -92,7 +92,7 @@ namespace dmlpackage
* @param colNameList, tableValuesMap * @param colNameList, tableValuesMap
* @param rows the number of rows in the buffer * @param rows the number of rows in the buffer
*/ */
EXPORT int buildFromMysqlBuffer(ColNameList& colNameList, TableValuesMap& tableValuesMap, int columns, int rows); EXPORT int buildFromMysqlBuffer(ColNameList& colNameList, TableValuesMap& tableValuesMap, int columns, int rows, NullValuesBitset& nullValues);
protected: protected:

2
dbcon/dmlpackage/dmlpkg.h
View File

@ -29,6 +29,7 @@
#include <map> #include <map>
#include <utility> #include <utility>
#include <iostream> #include <iostream>
#include <bitset>
#include <stdint.h> #include <stdint.h>
namespace dmlpackage namespace dmlpackage
@ -71,6 +72,7 @@ typedef std::vector<char*> QueryBuffer;
typedef std::vector<std::string> ColValuesList; typedef std::vector<std::string> ColValuesList;
typedef std::vector<std::string> ColNameList; typedef std::vector<std::string> ColNameList;
typedef std::map<uint32_t, ColValuesList> TableValuesMap; typedef std::map<uint32_t, ColValuesList> TableValuesMap;
typedef std::bitset<4096> NullValuesBitset;
std::ostream& operator<<(std::ostream& os, const SqlStatementList& ct); std::ostream& operator<<(std::ostream& os, const SqlStatementList& ct);
std::ostream& operator<<(std::ostream& os, const SqlStatement& stmt); std::ostream& operator<<(std::ostream& os, const SqlStatement& stmt);

4
dbcon/dmlpackage/insertdmlpackage.cpp
View File

@ -151,7 +151,7 @@ int InsertDMLPackage::buildFromBuffer(std::string& buffer, int columns, int rows
return retval; return retval;
} }
int InsertDMLPackage::buildFromMysqlBuffer(ColNameList& colNameList, TableValuesMap& tableValuesMap, int columns, int rows ) int InsertDMLPackage::buildFromMysqlBuffer(ColNameList& colNameList, TableValuesMap& tableValuesMap, int columns, int rows, NullValuesBitset& nullValues )
{ {
int retval = 1; int retval = 1;
@ -166,7 +166,7 @@ int InsertDMLPackage::buildFromMysqlBuffer(ColNameList& colNameList, TableValues
colValList = tableValuesMap[j]; colValList = tableValuesMap[j];
DMLColumn* aColumn = new DMLColumn(colName, colValList, false); DMLColumn* aColumn = new DMLColumn(colName, colValList, false, 0, nullValues[j]);
(aRowPtr->get_ColumnList()).push_back(aColumn); (aRowPtr->get_ColumnList()).push_back(aColumn);
} }
//build a row list for a table //build a row list for a table

2
dbcon/dmlpackage/insertdmlpackage.h
View File

@ -88,7 +88,7 @@ public:
* @param columns number of columns in the table * @param columns number of columns in the table
* @param rows number of rows to be touched * @param rows number of rows to be touched
*/ */
EXPORT int buildFromMysqlBuffer(ColNameList& colNameList, TableValuesMap& tableValuesMap, int columns, int rows); EXPORT int buildFromMysqlBuffer(ColNameList& colNameList, TableValuesMap& tableValuesMap, int columns, int rows, NullValuesBitset& nullValues);
/** @brief build a InsertDMLPackage from a InsertSqlStatement /** @brief build a InsertDMLPackage from a InsertSqlStatement
* *

4
dbcon/dmlpackage/updatedmlpackage.cpp
View File

@ -206,7 +206,7 @@ int UpdateDMLPackage::buildFromBuffer(std::string& buffer, int columns, int rows
return retval; return retval;
} }
int UpdateDMLPackage::buildFromMysqlBuffer(ColNameList& colNameList, TableValuesMap& tableValuesMap, int columns, int rows ) int UpdateDMLPackage::buildFromMysqlBuffer(ColNameList& colNameList, TableValuesMap& tableValuesMap, int columns, int rows, NullValuesBitset& nullValues )
{ {
int retval = 1; int retval = 1;
@ -221,7 +221,7 @@ int UpdateDMLPackage::buildFromMysqlBuffer(ColNameList& colNameList, TableValues
colValList = tableValuesMap[j]; colValList = tableValuesMap[j];
DMLColumn* aColumn = new DMLColumn(colName, colValList, false); DMLColumn* aColumn = new DMLColumn(colName, colValList, false, 0, nullValues[j]);
(aRowPtr->get_ColumnList()).push_back(aColumn); (aRowPtr->get_ColumnList()).push_back(aColumn);
} }
//build a row list for a table //build a row list for a table

2
dbcon/dmlpackage/updatedmlpackage.h
View File

@ -92,7 +92,7 @@ namespace dmlpackage
* @param colNameList, tableValuesMap * @param colNameList, tableValuesMap
* @param rows the number of rows in the buffer * @param rows the number of rows in the buffer
*/ */
EXPORT int buildFromMysqlBuffer(ColNameList& colNameList, TableValuesMap& tableValuesMap, int columns, int rows); EXPORT int buildFromMysqlBuffer(ColNameList& colNameList, TableValuesMap& tableValuesMap, int columns, int rows, NullValuesBitset& nullValues);
void buildUpdateFromMysqlBuffer(UpdateSqlStatement& updateStmt ); void buildUpdateFromMysqlBuffer(UpdateSqlStatement& updateStmt );

4
dbcon/dmlpackage/vendordmlstatement.cpp
View File

@ -48,10 +48,10 @@ namespace dmlpackage
{} {}
VendorDMLStatement::VendorDMLStatement(std::string dmlstatement, int stmttype, std::string tName, std::string schema, int rows, int columns, VendorDMLStatement::VendorDMLStatement(std::string dmlstatement, int stmttype, std::string tName, std::string schema, int rows, int columns,
ColNameList& colNameList, TableValuesMap& tableValuesMap, int sessionID) ColNameList& colNameList, TableValuesMap& tableValuesMap, NullValuesBitset& nullValues, int sessionID)
:fDMLStatement(dmlstatement), fDMLStatementType(stmttype), :fDMLStatement(dmlstatement), fDMLStatementType(stmttype),
fTableName(tName), fSchema(schema), fRows(rows), fColumns(columns), fTableName(tName), fSchema(schema), fRows(rows), fColumns(columns),
fColNameList(colNameList), fTableValuesMap(tableValuesMap), fSessionID(sessionID), fLogging(true),fLogending(true) fColNameList(colNameList), fTableValuesMap(tableValuesMap), fNullValues(nullValues), fSessionID(sessionID), fLogging(true),fLogending(true)
{} {}
VendorDMLStatement::~VendorDMLStatement() VendorDMLStatement::~VendorDMLStatement()

7
dbcon/dmlpackage/vendordmlstatement.h
View File

@ -26,6 +26,7 @@
#include <string> #include <string>
#include <vector> #include <vector>
#include <map> #include <map>
#include <bitset>
#include <stdint.h> #include <stdint.h>
#if defined(_MSC_VER) && defined(xxxVENDORDMLSTATEMENT_DLLEXPORT) #if defined(_MSC_VER) && defined(xxxVENDORDMLSTATEMENT_DLLEXPORT)
@ -38,6 +39,7 @@ namespace dmlpackage
typedef std::vector<std::string> ColValuesList; typedef std::vector<std::string> ColValuesList;
typedef std::vector<std::string> ColNameList; typedef std::vector<std::string> ColNameList;
typedef std::map<uint32_t, ColValuesList> TableValuesMap; typedef std::map<uint32_t, ColValuesList> TableValuesMap;
typedef std::bitset<4096> NullValuesBitset;
/** @brief describes the general interface /** @brief describes the general interface
* and implementation of a Vendor DML Statement * and implementation of a Vendor DML Statement
@ -63,7 +65,7 @@ namespace dmlpackage
/** @brief ctor for mysql /** @brief ctor for mysql
*/ */
EXPORT VendorDMLStatement(std::string dmlstatement, int stmttype, std::string tName, std::string schema, int rows, int columns, EXPORT VendorDMLStatement(std::string dmlstatement, int stmttype, std::string tName, std::string schema, int rows, int columns,
ColNameList& colNameList, TableValuesMap& tableValuesMap, int sessionID); ColNameList& colNameList, TableValuesMap& tableValuesMap, NullValuesBitset& nullValues, int sessionID);
/** @brief destructor /** @brief destructor
*/ */
@ -128,6 +130,8 @@ namespace dmlpackage
*/ */
inline int get_SessionID() { return fSessionID; } inline int get_SessionID() { return fSessionID; }
inline NullValuesBitset& get_nullValues() { return fNullValues; }
/** @brief Set the session ID /** @brief Set the session ID
*/ */
inline void set_SessionID( int value ) { fSessionID = value; } inline void set_SessionID( int value ) { fSessionID = value; }
@ -172,6 +176,7 @@ namespace dmlpackage
std::string fDataBuffer; std::string fDataBuffer;
ColNameList fColNameList; ColNameList fColNameList;
TableValuesMap fTableValuesMap; TableValuesMap fTableValuesMap;
NullValuesBitset fNullValues;
int fSessionID; int fSessionID;
bool fLogging; bool fLogging;
bool fLogending; bool fLogending;

5
dbcon/mysql/ha_calpont_dml.cpp
View File

@ -187,15 +187,18 @@ uint32_t buildValueList (TABLE* table, cal_connection_info& ci )
uint32_t size=0; uint32_t size=0;
int columnPos = 0; int columnPos = 0;
double dbval; double dbval;
ci.nullValuesBitset.reset();
for (Field** field = table->field; *field; field++) for (Field** field = table->field; *field; field++)
{ {
if((*field)->is_null()) if((*field)->is_null())
{ {
ci.tableValuesMap[columnPos].push_back (""); //currently, empty string is treated as null. ci.tableValuesMap[columnPos].push_back (""); //currently, empty string is treated as null.
ci.nullValuesBitset[columnPos] = true;
} }
else else
{ {
bitmap_set_bit(table->read_set, (*field)->field_index); bitmap_set_bit(table->read_set, (*field)->field_index);
ci.nullValuesBitset[columnPos] = false;
// @bug 3798 get real value for float/double type // @bug 3798 get real value for float/double type
if ((*field)->result_type() == REAL_RESULT) if ((*field)->result_type() == REAL_RESULT)
{ {
@ -330,7 +333,7 @@ int doProcessInsertValues ( TABLE* table, uint32_t size, cal_connection_info& ci
VendorDMLStatement dmlStmts(idb_mysql_query_str(thd), DML_INSERT, table->s->table_name.str, VendorDMLStatement dmlStmts(idb_mysql_query_str(thd), DML_INSERT, table->s->table_name.str,
table->s->db.str, size, ci.colNameList.size(), ci.colNameList, table->s->db.str, size, ci.colNameList.size(), ci.colNameList,
ci.tableValuesMap, sessionID); ci.tableValuesMap, ci.nullValuesBitset, sessionID);
CalpontDMLPackage* pDMLPackage = CalpontDMLFactory::makeCalpontDMLPackageFromMysqlBuffer(dmlStmts); CalpontDMLPackage* pDMLPackage = CalpontDMLFactory::makeCalpontDMLPackageFromMysqlBuffer(dmlStmts);
//@Bug 2466 Move the clean up earlier to avoid the second insert in another session to get the data //@Bug 2466 Move the clean up earlier to avoid the second insert in another session to get the data

2
dbcon/mysql/ha_calpont_impl_if.h
View File

@ -195,6 +195,7 @@ typedef std::tr1::unordered_map<TABLE*, cal_table_info> CalTableMap;
typedef std::vector<std::string> ColValuesList; typedef std::vector<std::string> ColValuesList;
typedef std::vector<std::string> ColNameList; typedef std::vector<std::string> ColNameList;
typedef std::map<uint32_t, ColValuesList> TableValuesMap; typedef std::map<uint32_t, ColValuesList> TableValuesMap;
typedef std::bitset<4096> NullValuesBitset;
struct cal_connection_info struct cal_connection_info
{ {
enum AlterTableState { NOT_ALTER, ALTER_SECOND_RENAME, ALTER_FIRST_RENAME }; enum AlterTableState { NOT_ALTER, ALTER_SECOND_RENAME, ALTER_FIRST_RENAME };
@ -257,6 +258,7 @@ struct cal_connection_info
ha_rows rowsHaveInserted; ha_rows rowsHaveInserted;
ColNameList colNameList; ColNameList colNameList;
TableValuesMap tableValuesMap; TableValuesMap tableValuesMap;
NullValuesBitset nullValuesBitset;
int rc; int rc;
uint32_t tableOid; uint32_t tableOid;
querystats::QueryStats stats; querystats::QueryStats stats;

14
utils/dataconvert/dataconvert.cpp
View File

@ -1105,13 +1105,6 @@ boost::any
case CalpontSystemCatalog::DATE: case CalpontSystemCatalog::DATE:
{ {
if (data == "0000-00-00") //@Bug 3210 Treat blank date as null
{
uint32_t d = joblist::DATENULL;
value = d;
break;
}
Date aDay; Date aDay;
if (stringToDateStruct(data, aDay)) if (stringToDateStruct(data, aDay))
{ {
@ -1135,13 +1128,6 @@ boost::any
case CalpontSystemCatalog::DATETIME: case CalpontSystemCatalog::DATETIME:
{ {
if (data == "0000-00-00 00:00:00") //@Bug 3210 Treat blank date as null
{
uint64_t d = joblist::DATETIMENULL;
value = d;
break;
}
DateTime aDatetime; DateTime aDatetime;
if (stringToDatetimeStruct(data, aDatetime, 0)) if (stringToDatetimeStruct(data, aDatetime, 0))
{ {

13
writeengine/server/we_dmlcommandproc.cpp
View File

@ -215,7 +215,9 @@ uint8_t WE_DMLCommandProc::processSingleInsert(messageqcpp::ByteStream& bs, std:
for ( uint32_t i=0; i < origVals.size(); i++ ) for ( uint32_t i=0; i < origVals.size(); i++ )
{ {
tmpStr = origVals[i]; tmpStr = origVals[i];
if ( tmpStr.length() == 0 )
isNULL = columnPtr->get_isnull();
if ( isNULL || ( tmpStr.length() == 0 ) )
isNULL = true; isNULL = true;
else else
isNULL = false; isNULL = false;
@ -266,7 +268,9 @@ uint8_t WE_DMLCommandProc::processSingleInsert(messageqcpp::ByteStream& bs, std:
for ( uint32_t i=0; i < origVals.size(); i++ ) for ( uint32_t i=0; i < origVals.size(); i++ )
{ {
indata = origVals[i]; indata = origVals[i];
if ( indata.length() == 0 )
isNULL = columnPtr->get_isnull();
if ( isNULL || ( indata.length() == 0 ) )
isNULL = true; isNULL = true;
else else
isNULL = false; isNULL = false;
@ -313,11 +317,6 @@ uint8_t WE_DMLCommandProc::processSingleInsert(messageqcpp::ByteStream& bs, std:
if (colType.constraintType == CalpontSystemCatalog::NOTNULL_CONSTRAINT) if (colType.constraintType == CalpontSystemCatalog::NOTNULL_CONSTRAINT)
{ {
if (((colType.colDataType == execplan::CalpontSystemCatalog::DATE) && (indata =="0000-00-00")) ||
((colType.colDataType == execplan::CalpontSystemCatalog::DATETIME) && (indata =="0000-00-00 00:00:00")))
{
isNULL = true;
}
if (isNULL && colType.defaultValue.empty()) //error out if (isNULL && colType.defaultValue.empty()) //error out
{ {
Message::Args args; Message::Args args;