diff --git a/dbcon/dmlpackage/calpontdmlfactory.cpp b/dbcon/dmlpackage/calpontdmlfactory.cpp index 52644d5f4..9f5e36d0d 100644 --- a/dbcon/dmlpackage/calpontdmlfactory.cpp +++ b/dbcon/dmlpackage/calpontdmlfactory.cpp @@ -174,7 +174,7 @@ dmlpackage::CalpontDMLPackage* CalpontDMLFactory::makeCalpontDMLPackageFromMysql { case DML_INSERT: packagePtr = new InsertDMLPackage(vpackage.get_SchemaName(), vpackage.get_TableName(), vpackage.get_DMLStatement(), vpackage.get_SessionID()); - (void)packagePtr->buildFromMysqlBuffer(vpackage.get_ColNames(), vpackage.get_values(), vpackage.get_Columns(), vpackage.get_Rows()); + (void)packagePtr->buildFromMysqlBuffer(vpackage.get_ColNames(), vpackage.get_values(), vpackage.get_Columns(), vpackage.get_Rows(), vpackage.get_nullValues()); break; case DML_COMMAND: packagePtr = new CommandDMLPackage(vpackage.get_DMLStatement(), vpackage.get_SessionID() ); @@ -182,7 +182,7 @@ dmlpackage::CalpontDMLPackage* CalpontDMLFactory::makeCalpontDMLPackageFromMysql case DML_DELETE: packagePtr = new DeleteDMLPackage(vpackage.get_SchemaName(), vpackage.get_TableName(), vpackage.get_DMLStatement(), vpackage.get_SessionID() ); - (void)packagePtr->buildFromMysqlBuffer(vpackage.get_ColNames(), vpackage.get_values(), vpackage.get_Columns(), vpackage.get_Rows()); + (void)packagePtr->buildFromMysqlBuffer(vpackage.get_ColNames(), vpackage.get_values(), vpackage.get_Columns(), vpackage.get_Rows(), vpackage.get_nullValues()); break; default: cerr << "makeCalpontDMLPackage: invalid statement type" << endl; diff --git a/dbcon/dmlpackage/calpontdmlpackage.h b/dbcon/dmlpackage/calpontdmlpackage.h index 01e94068b..983d9d392 100644 --- a/dbcon/dmlpackage/calpontdmlpackage.h +++ b/dbcon/dmlpackage/calpontdmlpackage.h @@ -94,7 +94,7 @@ namespace dmlpackage * @param columns number of columns in the table * @param rows number of rows to be touched */ - virtual int buildFromMysqlBuffer(ColNameList& colNameList, TableValuesMap& tableValuesMap, int columns, int rows) = 0; + virtual int buildFromMysqlBuffer(ColNameList& colNameList, TableValuesMap& tableValuesMap, int columns, int rows, NullValuesBitset& nullValues) = 0; /** @brief get the table object */ diff --git a/dbcon/dmlpackage/commanddmlpackage.h b/dbcon/dmlpackage/commanddmlpackage.h index bbd45f387..da1ecd94b 100644 --- a/dbcon/dmlpackage/commanddmlpackage.h +++ b/dbcon/dmlpackage/commanddmlpackage.h @@ -86,7 +86,7 @@ namespace dmlpackage * @param colNameList, tableValuesMap * @param rows the number of rows in the buffer */ - int buildFromMysqlBuffer(ColNameList& colNameList, TableValuesMap& tableValuesMap, int columns, int rows) + int buildFromMysqlBuffer(ColNameList& colNameList, TableValuesMap& tableValuesMap, int columns, int rows, NullValuesBitset& nullValues) { return 1; }; diff --git a/dbcon/dmlpackage/deletedmlpackage.cpp b/dbcon/dmlpackage/deletedmlpackage.cpp index a6858c0c7..b7a445cac 100644 --- a/dbcon/dmlpackage/deletedmlpackage.cpp +++ b/dbcon/dmlpackage/deletedmlpackage.cpp @@ -189,7 +189,7 @@ int DeleteDMLPackage::buildFromBuffer(std::string& buffer, int columns, int rows } -int DeleteDMLPackage::buildFromMysqlBuffer(ColNameList& colNameList, TableValuesMap& tableValuesMap, int columns, int rows ) +int DeleteDMLPackage::buildFromMysqlBuffer(ColNameList& colNameList, TableValuesMap& tableValuesMap, int columns, int rows, NullValuesBitset& nullValues ) { int retval = 1; diff --git a/dbcon/dmlpackage/deletedmlpackage.h b/dbcon/dmlpackage/deletedmlpackage.h index 8ba708896..4356b5f22 100644 --- a/dbcon/dmlpackage/deletedmlpackage.h +++ b/dbcon/dmlpackage/deletedmlpackage.h @@ -92,7 +92,7 @@ namespace dmlpackage * @param colNameList, tableValuesMap * @param rows the number of rows in the buffer */ - EXPORT int buildFromMysqlBuffer(ColNameList& colNameList, TableValuesMap& tableValuesMap, int columns, int rows); + EXPORT int buildFromMysqlBuffer(ColNameList& colNameList, TableValuesMap& tableValuesMap, int columns, int rows, NullValuesBitset& nullValues); protected: diff --git a/dbcon/dmlpackage/dmlpkg.h b/dbcon/dmlpackage/dmlpkg.h index 37bebd5ba..3bef09343 100644 --- a/dbcon/dmlpackage/dmlpkg.h +++ b/dbcon/dmlpackage/dmlpkg.h @@ -29,6 +29,7 @@ #include #include #include +#include #include namespace dmlpackage @@ -71,6 +72,7 @@ typedef std::vector QueryBuffer; typedef std::vector ColValuesList; typedef std::vector ColNameList; typedef std::map TableValuesMap; +typedef std::bitset<4096> NullValuesBitset; std::ostream& operator<<(std::ostream& os, const SqlStatementList& ct); std::ostream& operator<<(std::ostream& os, const SqlStatement& stmt); diff --git a/dbcon/dmlpackage/insertdmlpackage.cpp b/dbcon/dmlpackage/insertdmlpackage.cpp index 9caf5eb3a..243027cb4 100644 --- a/dbcon/dmlpackage/insertdmlpackage.cpp +++ b/dbcon/dmlpackage/insertdmlpackage.cpp @@ -151,7 +151,7 @@ int InsertDMLPackage::buildFromBuffer(std::string& buffer, int columns, int rows return retval; } -int InsertDMLPackage::buildFromMysqlBuffer(ColNameList& colNameList, TableValuesMap& tableValuesMap, int columns, int rows ) +int InsertDMLPackage::buildFromMysqlBuffer(ColNameList& colNameList, TableValuesMap& tableValuesMap, int columns, int rows, NullValuesBitset& nullValues ) { int retval = 1; @@ -166,7 +166,7 @@ int InsertDMLPackage::buildFromMysqlBuffer(ColNameList& colNameList, TableValues colValList = tableValuesMap[j]; - DMLColumn* aColumn = new DMLColumn(colName, colValList, false); + DMLColumn* aColumn = new DMLColumn(colName, colValList, false, 0, nullValues[j]); (aRowPtr->get_ColumnList()).push_back(aColumn); } //build a row list for a table diff --git a/dbcon/dmlpackage/insertdmlpackage.h b/dbcon/dmlpackage/insertdmlpackage.h index 9f890dada..5787ec55b 100644 --- a/dbcon/dmlpackage/insertdmlpackage.h +++ b/dbcon/dmlpackage/insertdmlpackage.h @@ -88,7 +88,7 @@ public: * @param columns number of columns in the table * @param rows number of rows to be touched */ - EXPORT int buildFromMysqlBuffer(ColNameList& colNameList, TableValuesMap& tableValuesMap, int columns, int rows); + EXPORT int buildFromMysqlBuffer(ColNameList& colNameList, TableValuesMap& tableValuesMap, int columns, int rows, NullValuesBitset& nullValues); /** @brief build a InsertDMLPackage from a InsertSqlStatement * diff --git a/dbcon/dmlpackage/updatedmlpackage.cpp b/dbcon/dmlpackage/updatedmlpackage.cpp index de634d321..05ca4f117 100644 --- a/dbcon/dmlpackage/updatedmlpackage.cpp +++ b/dbcon/dmlpackage/updatedmlpackage.cpp @@ -206,7 +206,7 @@ int UpdateDMLPackage::buildFromBuffer(std::string& buffer, int columns, int rows return retval; } -int UpdateDMLPackage::buildFromMysqlBuffer(ColNameList& colNameList, TableValuesMap& tableValuesMap, int columns, int rows ) +int UpdateDMLPackage::buildFromMysqlBuffer(ColNameList& colNameList, TableValuesMap& tableValuesMap, int columns, int rows, NullValuesBitset& nullValues ) { int retval = 1; @@ -221,7 +221,7 @@ int UpdateDMLPackage::buildFromMysqlBuffer(ColNameList& colNameList, TableValues colValList = tableValuesMap[j]; - DMLColumn* aColumn = new DMLColumn(colName, colValList, false); + DMLColumn* aColumn = new DMLColumn(colName, colValList, false, 0, nullValues[j]); (aRowPtr->get_ColumnList()).push_back(aColumn); } //build a row list for a table diff --git a/dbcon/dmlpackage/updatedmlpackage.h b/dbcon/dmlpackage/updatedmlpackage.h index f67c24acb..7efc34c32 100644 --- a/dbcon/dmlpackage/updatedmlpackage.h +++ b/dbcon/dmlpackage/updatedmlpackage.h @@ -92,7 +92,7 @@ namespace dmlpackage * @param colNameList, tableValuesMap * @param rows the number of rows in the buffer */ - EXPORT int buildFromMysqlBuffer(ColNameList& colNameList, TableValuesMap& tableValuesMap, int columns, int rows); + EXPORT int buildFromMysqlBuffer(ColNameList& colNameList, TableValuesMap& tableValuesMap, int columns, int rows, NullValuesBitset& nullValues); void buildUpdateFromMysqlBuffer(UpdateSqlStatement& updateStmt ); diff --git a/dbcon/dmlpackage/vendordmlstatement.cpp b/dbcon/dmlpackage/vendordmlstatement.cpp index 0d680df2c..cc34bd2f4 100644 --- a/dbcon/dmlpackage/vendordmlstatement.cpp +++ b/dbcon/dmlpackage/vendordmlstatement.cpp @@ -48,10 +48,10 @@ namespace dmlpackage {} VendorDMLStatement::VendorDMLStatement(std::string dmlstatement, int stmttype, std::string tName, std::string schema, int rows, int columns, - ColNameList& colNameList, TableValuesMap& tableValuesMap, int sessionID) + ColNameList& colNameList, TableValuesMap& tableValuesMap, NullValuesBitset& nullValues, int sessionID) :fDMLStatement(dmlstatement), fDMLStatementType(stmttype), fTableName(tName), fSchema(schema), fRows(rows), fColumns(columns), - fColNameList(colNameList), fTableValuesMap(tableValuesMap), fSessionID(sessionID), fLogging(true),fLogending(true) + fColNameList(colNameList), fTableValuesMap(tableValuesMap), fNullValues(nullValues), fSessionID(sessionID), fLogging(true),fLogending(true) {} VendorDMLStatement::~VendorDMLStatement() diff --git a/dbcon/dmlpackage/vendordmlstatement.h b/dbcon/dmlpackage/vendordmlstatement.h index e9f499a04..f59654a9f 100644 --- a/dbcon/dmlpackage/vendordmlstatement.h +++ b/dbcon/dmlpackage/vendordmlstatement.h @@ -26,6 +26,7 @@ #include #include #include +#include #include #if defined(_MSC_VER) && defined(xxxVENDORDMLSTATEMENT_DLLEXPORT) @@ -38,6 +39,7 @@ namespace dmlpackage typedef std::vector ColValuesList; typedef std::vector ColNameList; typedef std::map TableValuesMap; + typedef std::bitset<4096> NullValuesBitset; /** @brief describes the general interface * and implementation of a Vendor DML Statement @@ -63,7 +65,7 @@ namespace dmlpackage /** @brief ctor for mysql */ EXPORT VendorDMLStatement(std::string dmlstatement, int stmttype, std::string tName, std::string schema, int rows, int columns, - ColNameList& colNameList, TableValuesMap& tableValuesMap, int sessionID); + ColNameList& colNameList, TableValuesMap& tableValuesMap, NullValuesBitset& nullValues, int sessionID); /** @brief destructor */ @@ -128,6 +130,8 @@ namespace dmlpackage */ inline int get_SessionID() { return fSessionID; } + inline NullValuesBitset& get_nullValues() { return fNullValues; } + /** @brief Set the session ID */ inline void set_SessionID( int value ) { fSessionID = value; } @@ -172,6 +176,7 @@ namespace dmlpackage std::string fDataBuffer; ColNameList fColNameList; TableValuesMap fTableValuesMap; + NullValuesBitset fNullValues; int fSessionID; bool fLogging; bool fLogending; diff --git a/dbcon/mysql/ha_calpont_dml.cpp b/dbcon/mysql/ha_calpont_dml.cpp index ce93e3720..2f1147c9b 100755 --- a/dbcon/mysql/ha_calpont_dml.cpp +++ b/dbcon/mysql/ha_calpont_dml.cpp @@ -187,15 +187,18 @@ uint32_t buildValueList (TABLE* table, cal_connection_info& ci ) uint32_t size=0; int columnPos = 0; double dbval; + ci.nullValuesBitset.reset(); for (Field** field = table->field; *field; field++) { if((*field)->is_null()) { ci.tableValuesMap[columnPos].push_back (""); //currently, empty string is treated as null. + ci.nullValuesBitset[columnPos] = true; } else { bitmap_set_bit(table->read_set, (*field)->field_index); + ci.nullValuesBitset[columnPos] = false; // @bug 3798 get real value for float/double type if ((*field)->result_type() == REAL_RESULT) { @@ -330,7 +333,7 @@ int doProcessInsertValues ( TABLE* table, uint32_t size, cal_connection_info& ci VendorDMLStatement dmlStmts(idb_mysql_query_str(thd), DML_INSERT, table->s->table_name.str, table->s->db.str, size, ci.colNameList.size(), ci.colNameList, - ci.tableValuesMap, sessionID); + ci.tableValuesMap, ci.nullValuesBitset, sessionID); CalpontDMLPackage* pDMLPackage = CalpontDMLFactory::makeCalpontDMLPackageFromMysqlBuffer(dmlStmts); //@Bug 2466 Move the clean up earlier to avoid the second insert in another session to get the data diff --git a/dbcon/mysql/ha_calpont_impl_if.h b/dbcon/mysql/ha_calpont_impl_if.h index 3f3704e5a..9a4fd8bd7 100644 --- a/dbcon/mysql/ha_calpont_impl_if.h +++ b/dbcon/mysql/ha_calpont_impl_if.h @@ -195,6 +195,7 @@ typedef std::tr1::unordered_map CalTableMap; typedef std::vector ColValuesList; typedef std::vector ColNameList; typedef std::map TableValuesMap; +typedef std::bitset<4096> NullValuesBitset; struct cal_connection_info { enum AlterTableState { NOT_ALTER, ALTER_SECOND_RENAME, ALTER_FIRST_RENAME }; @@ -257,6 +258,7 @@ struct cal_connection_info ha_rows rowsHaveInserted; ColNameList colNameList; TableValuesMap tableValuesMap; + NullValuesBitset nullValuesBitset; int rc; uint32_t tableOid; querystats::QueryStats stats; diff --git a/utils/dataconvert/dataconvert.cpp b/utils/dataconvert/dataconvert.cpp index cb54bb185..de92c2d89 100644 --- a/utils/dataconvert/dataconvert.cpp +++ b/utils/dataconvert/dataconvert.cpp @@ -1105,13 +1105,6 @@ boost::any case CalpontSystemCatalog::DATE: { - if (data == "0000-00-00") //@Bug 3210 Treat blank date as null - { - uint32_t d = joblist::DATENULL; - value = d; - break; - } - Date aDay; if (stringToDateStruct(data, aDay)) { @@ -1135,13 +1128,6 @@ boost::any case CalpontSystemCatalog::DATETIME: { - if (data == "0000-00-00 00:00:00") //@Bug 3210 Treat blank date as null - { - uint64_t d = joblist::DATETIMENULL; - value = d; - break; - } - DateTime aDatetime; if (stringToDatetimeStruct(data, aDatetime, 0)) { diff --git a/writeengine/server/we_dmlcommandproc.cpp b/writeengine/server/we_dmlcommandproc.cpp index bb4bd89ec..3eeead69c 100644 --- a/writeengine/server/we_dmlcommandproc.cpp +++ b/writeengine/server/we_dmlcommandproc.cpp @@ -215,7 +215,9 @@ uint8_t WE_DMLCommandProc::processSingleInsert(messageqcpp::ByteStream& bs, std: for ( uint32_t i=0; i < origVals.size(); i++ ) { tmpStr = origVals[i]; - if ( tmpStr.length() == 0 ) + + isNULL = columnPtr->get_isnull(); + if ( isNULL || ( tmpStr.length() == 0 ) ) isNULL = true; else isNULL = false; @@ -266,7 +268,9 @@ uint8_t WE_DMLCommandProc::processSingleInsert(messageqcpp::ByteStream& bs, std: for ( uint32_t i=0; i < origVals.size(); i++ ) { indata = origVals[i]; - if ( indata.length() == 0 ) + + isNULL = columnPtr->get_isnull(); + if ( isNULL || ( indata.length() == 0 ) ) isNULL = true; else isNULL = false; @@ -313,11 +317,6 @@ uint8_t WE_DMLCommandProc::processSingleInsert(messageqcpp::ByteStream& bs, std: if (colType.constraintType == CalpontSystemCatalog::NOTNULL_CONSTRAINT) { - if (((colType.colDataType == execplan::CalpontSystemCatalog::DATE) && (indata =="0000-00-00")) || - ((colType.colDataType == execplan::CalpontSystemCatalog::DATETIME) && (indata =="0000-00-00 00:00:00"))) - { - isNULL = true; - } if (isNULL && colType.defaultValue.empty()) //error out { Message::Args args;