diff --git a/dbcon/ddlpackage/CMakeLists.txt b/dbcon/ddlpackage/CMakeLists.txt index 4f4fb995f..084aeeac0 100644 --- a/dbcon/ddlpackage/CMakeLists.txt +++ b/dbcon/ddlpackage/CMakeLists.txt @@ -9,6 +9,8 @@ ADD_CUSTOM_COMMAND( DEPENDS ddl.y ddl.l ) +set_source_files_properties(ddl-scan.cpp PROPERTIES COMPILE_FLAGS -Wno-sign-compare) + # Parser puts extra info to stderr. MY_CHECK_AND_SET_COMPILER_FLAG("-DYYDEBUG=1" DEBUG) diff --git a/dbcon/ddlpackage/ddl.l b/dbcon/ddlpackage/ddl.l index edc962a10..a15773bc8 100644 --- a/dbcon/ddlpackage/ddl.l +++ b/dbcon/ddlpackage/ddl.l @@ -125,6 +125,7 @@ CURRENT_USER {return CURRENT_USER;} DATE {ddlget_lval(yyscanner)->str=strdup("date"); return DATE;} DATETIME {return DATETIME;} TIME {ddlget_lval(yyscanner)->str=strdup("time"); return TIME;} +TIMESTAMP {return TIMESTAMP;} DECIMAL {return DECIMAL;} DEC {return DECIMAL;} DEFAULT {return DEFAULT;} diff --git a/dbcon/ddlpackage/ddl.y b/dbcon/ddlpackage/ddl.y index cdd439927..177a64caa 100644 --- a/dbcon/ddlpackage/ddl.y +++ b/dbcon/ddlpackage/ddl.y @@ -112,7 +112,7 @@ MIN_ROWS MODIFY NO NOT NULL_TOK NUMBER NUMERIC ON PARTIAL PRECISION PRIMARY REFERENCES RENAME RESTRICT SET SMALLINT TABLE TEXT TINYBLOB TINYTEXT TINYINT TO UNIQUE UNSIGNED UPDATE USER SESSION_USER SYSTEM_USER VARCHAR VARBINARY VARYING WITH ZONE DOUBLE IDB_FLOAT REAL CHARSET IDB_IF EXISTS CHANGE TRUNCATE -BOOL BOOLEAN MEDIUMINT +BOOL BOOLEAN MEDIUMINT TIMESTAMP %token DQ_IDENT IDENT FCONST SCONST CP_SEARCH_CONDITION_TEXT ICONST DATE TIME @@ -1138,7 +1138,7 @@ datetime_type: { $$ = new ColumnType(DDL_DATETIME); $$->fLength = DDLDatatypeLength[DDL_DATETIME]; - $$->fPrecision = $2; + $$->fPrecision = $2; } | DATE @@ -1153,6 +1153,13 @@ datetime_type: $$->fLength = DDLDatatypeLength[DDL_TIME]; $$->fPrecision = $2; } + | + TIMESTAMP opt_time_precision + { + $$ = new ColumnType(DDL_TIMESTAMP); + $$->fLength = DDLDatatypeLength[DDL_TIMESTAMP]; + $$->fPrecision = $2; + } opt_time_precision: '(' ICONST ')' {$$ = atoi($2);} diff --git a/dbcon/ddlpackage/ddlpkg.h b/dbcon/ddlpackage/ddlpkg.h index 779577f83..0c7edd389 100644 --- a/dbcon/ddlpackage/ddlpkg.h +++ b/dbcon/ddlpackage/ddlpkg.h @@ -237,6 +237,7 @@ enum DDL_DATATYPES DDL_UNSIGNED_NUMERIC, DDL_TEXT, DDL_TIME, + DDL_TIMESTAMP, DDL_INVALID_DATATYPE }; @@ -274,7 +275,8 @@ const std::string DDLDatatypeString[] = "unsigned-double", "unsigned-numeric", "text", - "time" + "time", + "timestamp" "" }; @@ -331,6 +333,7 @@ const int DDLDatatypeLength[] = 2, // UNSIGNED_NUMERIC, 8, // TEXT 8, // TIME + 8, // TIMESTAMP -1 // INVALID LENGTH }; @@ -1396,6 +1399,7 @@ struct AlterTableStatement : public SqlStatement QualifiedName* fTableName; AlterTableActionList fActions; + std::string fTimeZone; }; diff --git a/dbcon/ddlpackage/serialize.cpp b/dbcon/ddlpackage/serialize.cpp index 12c87b9b0..f81cef077 100644 --- a/dbcon/ddlpackage/serialize.cpp +++ b/dbcon/ddlpackage/serialize.cpp @@ -121,6 +121,8 @@ int AlterTableStatement::unserialize(ByteStream& bytestream) // read table name fTableName->unserialize( bytestream ); + bytestream >> fTimeZone; + // read alter action list quadbyte action_count; bytestream >> action_count; @@ -228,6 +230,8 @@ int AlterTableStatement::serialize(ByteStream& bytestream) // write table name fTableName->serialize( bytestream ); + bytestream << fTimeZone; + write_vec(fActions, bytestream); // write sessionid diff --git a/dbcon/ddlpackageproc/altertableprocessor.cpp b/dbcon/ddlpackageproc/altertableprocessor.cpp index 989dffce2..743779c68 100644 --- a/dbcon/ddlpackageproc/altertableprocessor.cpp +++ b/dbcon/ddlpackageproc/altertableprocessor.cpp @@ -214,6 +214,11 @@ bool typesAreSame(const CalpontSystemCatalog::ColType& colType, const ColumnType break; + case (CalpontSystemCatalog::TIMESTAMP): + if (newType.fType == DDL_TIMESTAMP) return true; + + break; + case (CalpontSystemCatalog::TIME): if (newType.fType == DDL_TIME) return true; @@ -989,6 +994,7 @@ void AlterTableProcessor::addColumn (uint32_t sessionID, execplan::CalpontSystem bs << (ByteStream::byte) column_iterator->colType.colDataType; bs << (uint32_t) column_iterator->colType.colWidth; bs << (ByteStream::byte) column_iterator->colType.compressionType; + bs << fTimeZone; //cout << "sending command fillcolumn " << endl; uint32_t msgRecived = 0; fWEClient->write_to_all(bs); diff --git a/dbcon/ddlpackageproc/altertableprocessor.h b/dbcon/ddlpackageproc/altertableprocessor.h index 42d66a8e4..645b003d0 100644 --- a/dbcon/ddlpackageproc/altertableprocessor.h +++ b/dbcon/ddlpackageproc/altertableprocessor.h @@ -146,6 +146,7 @@ public: ddlpackage::AtaTableComment& ataTableComment, ddlpackage::QualifiedName& fTableName, const uint64_t uniqueId); + std::string fTimeZone; protected: void rollBackAlter(const std::string& error, BRM::TxnID txnID, int sessionId, DDLResult& result, uint64_t uniqueId); diff --git a/dbcon/ddlpackageproc/ddlindexpopulator.cpp b/dbcon/ddlpackageproc/ddlindexpopulator.cpp index cb58df5bc..be2549084 100644 --- a/dbcon/ddlpackageproc/ddlindexpopulator.cpp +++ b/dbcon/ddlpackageproc/ddlindexpopulator.cpp @@ -331,6 +331,7 @@ boost::any DDLIndexPopulator::convertData(const CalpontSystemCatalog::ColType& case execplan::CalpontSystemCatalog::DATETIME: // @bug 375 case execplan::CalpontSystemCatalog::TIME: + case execplan::CalpontSystemCatalog::TIMESTAMP: case execplan::CalpontSystemCatalog::BIGINT: return *reinterpret_cast(&data); @@ -526,6 +527,7 @@ bool DDLIndexPopulator::checkNotNull(const IdxTuple& data, const CalpontSystemCa case execplan::CalpontSystemCatalog::DATETIME: case execplan::CalpontSystemCatalog::TIME: + case execplan::CalpontSystemCatalog::TIMESTAMP: isNull = any_cast(data.data) == any_cast(nullvalue); break; diff --git a/dbcon/ddlpackageproc/ddlpackageprocessor.cpp b/dbcon/ddlpackageproc/ddlpackageprocessor.cpp index 2a64dd1f7..0419fc70e 100644 --- a/dbcon/ddlpackageproc/ddlpackageprocessor.cpp +++ b/dbcon/ddlpackageproc/ddlpackageprocessor.cpp @@ -236,6 +236,10 @@ execplan::CalpontSystemCatalog::ColDataType DDLPackageProcessor::convertDataType colDataType = CalpontSystemCatalog::TIME; break; + case ddlpackage::DDL_TIMESTAMP: + colDataType = CalpontSystemCatalog::TIMESTAMP; + break; + case ddlpackage::DDL_CLOB: colDataType = CalpontSystemCatalog::CLOB; break; @@ -487,6 +491,13 @@ DDLPackageProcessor::getNullValueForType(const execplan::CalpontSystemCatalog::C } break; + case execplan::CalpontSystemCatalog::TIMESTAMP: + { + long long d = joblist::TIMESTAMPNULL; + value = d; + } + break; + case execplan::CalpontSystemCatalog::CHAR: { std::string charnull; diff --git a/dbcon/dmlpackage/CMakeLists.txt b/dbcon/dmlpackage/CMakeLists.txt index 945215249..4e711c967 100644 --- a/dbcon/dmlpackage/CMakeLists.txt +++ b/dbcon/dmlpackage/CMakeLists.txt @@ -10,6 +10,7 @@ ADD_CUSTOM_COMMAND( DEPENDS dml.y dml.l ) +set_source_files_properties(dml-scan.cpp PROPERTIES COMPILE_FLAGS -Wno-sign-compare) ########### next target ############### diff --git a/dbcon/dmlpackage/calpontdmlpackage.h b/dbcon/dmlpackage/calpontdmlpackage.h index 5de3ea23a..839fb87b1 100644 --- a/dbcon/dmlpackage/calpontdmlpackage.h +++ b/dbcon/dmlpackage/calpontdmlpackage.h @@ -220,6 +220,22 @@ public: return fSchemaName; } + /** @brief set the timezone + * + * @param the timezone to set + */ + void set_TimeZone( const std::string& timeZone ) + { + fTimeZone = timeZone; + } + + /** @brief get the timezone + */ + const std::string get_TimeZone() const + { + return fTimeZone; + } + /** @brief does this dml statement have a filter */ bool HasFilter() const @@ -345,6 +361,7 @@ protected: void initializeTable(); std::string fSchemaName; + std::string fTimeZone; std::string fTableName; std::string fDMLStatement; std::string fSQLStatement; diff --git a/dbcon/dmlpackage/commanddmlpackage.cpp b/dbcon/dmlpackage/commanddmlpackage.cpp index 719e2106e..6bbdc0693 100644 --- a/dbcon/dmlpackage/commanddmlpackage.cpp +++ b/dbcon/dmlpackage/commanddmlpackage.cpp @@ -58,6 +58,7 @@ int CommandDMLPackage::write(messageqcpp::ByteStream& bytestream) bytestream << fSQLStatement; // for cleartablelock, this is table lockID bytestream << (uint8_t)fLogging; bytestream << fSchemaName; + bytestream << fTimeZone; bytestream << fTableName; bytestream << fTableOid; bytestream << static_cast(fIsAutocommitOn); @@ -80,6 +81,7 @@ int CommandDMLPackage::read(messageqcpp::ByteStream& bytestream) bytestream >> logging; fLogging = (logging != 0); bytestream >> fSchemaName; + bytestream >> fTimeZone; bytestream >> fTableName; bytestream >> fTableOid; bytestream >> reinterpret_cast< messageqcpp::ByteStream::byte&>(fIsAutocommitOn); diff --git a/dbcon/dmlpackage/deletedmlpackage.cpp b/dbcon/dmlpackage/deletedmlpackage.cpp index f841a32b0..af0a77bde 100644 --- a/dbcon/dmlpackage/deletedmlpackage.cpp +++ b/dbcon/dmlpackage/deletedmlpackage.cpp @@ -67,6 +67,7 @@ int DeleteDMLPackage::write(messageqcpp::ByteStream& bytestream) bytestream << fDMLStatement; bytestream << fSQLStatement; bytestream << fSchemaName; + bytestream << fTimeZone; if (fTable != 0) { @@ -102,6 +103,7 @@ int DeleteDMLPackage::read(messageqcpp::ByteStream& bytestream) bytestream >> fDMLStatement; bytestream >> fSQLStatement; bytestream >> fSchemaName; + bytestream >> fTimeZone; fTable = new DMLTable(); retval = fTable->read(bytestream); diff --git a/dbcon/dmlpackage/insertdmlpackage.cpp b/dbcon/dmlpackage/insertdmlpackage.cpp index 6df53b6d0..8c00ed2fc 100644 --- a/dbcon/dmlpackage/insertdmlpackage.cpp +++ b/dbcon/dmlpackage/insertdmlpackage.cpp @@ -62,6 +62,7 @@ int InsertDMLPackage::write(messageqcpp::ByteStream& bytestream) bytestream << fDMLStatement; bytestream << fDMLStatement; bytestream << fSchemaName; + bytestream << fTimeZone; bytestream << (uint8_t)fLogging; bytestream << (uint8_t)fLogending; @@ -91,6 +92,7 @@ int InsertDMLPackage::read(messageqcpp::ByteStream& bytestream) bytestream >> fDMLStatement; bytestream >> fSQLStatement; bytestream >> fSchemaName; + bytestream >> fTimeZone; uint8_t logging; bytestream >> logging; fLogging = (logging != 0); diff --git a/dbcon/dmlpackage/updatedmlpackage.cpp b/dbcon/dmlpackage/updatedmlpackage.cpp index 7a7421915..a68e42fba 100644 --- a/dbcon/dmlpackage/updatedmlpackage.cpp +++ b/dbcon/dmlpackage/updatedmlpackage.cpp @@ -66,6 +66,7 @@ int UpdateDMLPackage::write(messageqcpp::ByteStream& bytestream) bytestream << fDMLStatement; bytestream << fSQLStatement; bytestream << fSchemaName; + bytestream << fTimeZone; bytestream << (uint8_t)fIsFromCol; if (fTable != 0) @@ -102,6 +103,7 @@ int UpdateDMLPackage::read(messageqcpp::ByteStream& bytestream) bytestream >> fDMLStatement; bytestream >> fSQLStatement; bytestream >> fSchemaName; + bytestream >> fTimeZone; uint8_t isFromCol; bytestream >> isFromCol; fIsFromCol = (isFromCol != 0); diff --git a/dbcon/execplan/aggregatecolumn.cpp b/dbcon/execplan/aggregatecolumn.cpp index 982841e4e..8583459da 100644 --- a/dbcon/execplan/aggregatecolumn.cpp +++ b/dbcon/execplan/aggregatecolumn.cpp @@ -119,7 +119,8 @@ AggregateColumn::AggregateColumn( const AggregateColumn& rhs, const uint32_t ses fTableAlias(rhs.tableAlias()), fAsc(rhs.asc()), fData(rhs.data()), - fConstCol(rhs.fConstCol) + fConstCol(rhs.fConstCol), + fTimeZone(rhs.timeZone()) { fAlias = rhs.alias(); fAggParms = rhs.fAggParms; @@ -186,6 +187,7 @@ void AggregateColumn::serialize(messageqcpp::ByteStream& b) const (*rcit)->serialize(b); b << fData; + b << fTimeZone; //b << fAlias; b << fTableAlias; b << static_cast(fAsc); @@ -238,6 +240,7 @@ void AggregateColumn::unserialize(messageqcpp::ByteStream& b) } b >> fData; + b >> fTimeZone; //b >> fAlias; b >> fTableAlias; b >> reinterpret_cast< ByteStream::doublebyte&>(fAsc); @@ -292,6 +295,9 @@ bool AggregateColumn::operator==(const AggregateColumn& t) const *(fConstCol.get()) != t.fConstCol.get())) return false; + if (fTimeZone != t.fTimeZone) + return false; + return true; } @@ -343,6 +349,14 @@ void AggregateColumn::evaluate(Row& row, bool& isNull) break; + case CalpontSystemCatalog::TIMESTAMP: + if (row.equals<8>(TIMESTAMPNULL, fInputIndex)) + isNull = true; + else + fResult.intVal = row.getUintField<8>(fInputIndex); + + break; + case CalpontSystemCatalog::TIME: if (row.equals<8>(TIMENULL, fInputIndex)) isNull = true; diff --git a/dbcon/execplan/aggregatecolumn.h b/dbcon/execplan/aggregatecolumn.h index 79df22e4d..0825f00ff 100644 --- a/dbcon/execplan/aggregatecolumn.h +++ b/dbcon/execplan/aggregatecolumn.h @@ -312,6 +312,16 @@ public: return false; } + inline const std::string timeZone () const + { + return fTimeZone; + } + + inline void timeZone (const std::string& timeZone) + { + fTimeZone = timeZone; + } + protected: std::string fFunctionName; // deprecated field uint8_t fAggOp; @@ -335,6 +345,7 @@ protected: ColumnList fGroupByColList; ColumnList fProjectColList; SRCP fConstCol; + std::string fTimeZone; public: /*********************************************************** @@ -346,7 +357,7 @@ public: virtual const std::string& getStrVal(rowgroup::Row& row, bool& isNull) { evaluate(row, isNull); - return TreeNode::getStrVal(); + return TreeNode::getStrVal(fTimeZone); } /** @@ -426,6 +437,15 @@ public: evaluate(row, isNull); return TreeNode::getDatetimeIntVal(); } + /** + * F&E + */ + virtual int64_t getTimestampIntVal(rowgroup::Row& row, bool& isNull) + { + evaluate(row, isNull); + return TreeNode::getTimestampIntVal(); + } + private: void evaluate(rowgroup::Row& row, bool& isNull); diff --git a/dbcon/execplan/arithmeticcolumn.h b/dbcon/execplan/arithmeticcolumn.h index 14d838bad..d7e895969 100644 --- a/dbcon/execplan/arithmeticcolumn.h +++ b/dbcon/execplan/arithmeticcolumn.h @@ -254,6 +254,11 @@ public: return fExpression->getDatetimeIntVal(row, isNull); } + virtual int64_t getTimestampIntVal(rowgroup::Row& row, bool& isNull) + { + return fExpression->getTimestampIntVal(row, isNull); + } + virtual int64_t getTimeIntVal(rowgroup::Row& row, bool& isNull) { return fExpression->getTimeIntVal(row, isNull); diff --git a/dbcon/execplan/arithmeticoperator.cpp b/dbcon/execplan/arithmeticoperator.cpp index 57b77381a..cf47554d9 100644 --- a/dbcon/execplan/arithmeticoperator.cpp +++ b/dbcon/execplan/arithmeticoperator.cpp @@ -56,7 +56,9 @@ ArithmeticOperator::ArithmeticOperator(const string& operatorName): Operator(ope { } -ArithmeticOperator::ArithmeticOperator(const ArithmeticOperator& rhs): Operator(rhs) +ArithmeticOperator::ArithmeticOperator(const ArithmeticOperator& rhs): + Operator(rhs), + fTimeZone(rhs.timeZone()) { } @@ -84,21 +86,26 @@ ostream& operator<<(ostream& output, const ArithmeticOperator& rhs) void ArithmeticOperator::serialize(messageqcpp::ByteStream& b) const { b << (ObjectReader::id_t) ObjectReader::ARITHMETICOPERATOR; + b << fTimeZone; Operator::serialize(b); } void ArithmeticOperator::unserialize(messageqcpp::ByteStream& b) { ObjectReader::checkType(b, ObjectReader::ARITHMETICOPERATOR); + b >> fTimeZone; Operator::unserialize(b); } bool ArithmeticOperator::operator==(const ArithmeticOperator& t) const { - if (data() == t.data()) - return true; + if (data() != t.data()) + return false; - return false; + if (timeZone() != t.timeZone()) + return false; + + return true; } bool ArithmeticOperator::operator==(const TreeNode* t) const diff --git a/dbcon/execplan/arithmeticoperator.h b/dbcon/execplan/arithmeticoperator.h index 0f54b1e82..cc6cc70da 100644 --- a/dbcon/execplan/arithmeticoperator.h +++ b/dbcon/execplan/arithmeticoperator.h @@ -60,6 +60,15 @@ public: return new ArithmeticOperator (*this); } + inline const std::string& timeZone() const + { + return fTimeZone; + } + inline void timeZone(const std::string& timeZone) + { + fTimeZone = timeZone; + } + /** * The serialization interface */ @@ -102,7 +111,7 @@ public: virtual const std::string& getStrVal(rowgroup::Row& row, bool& isNull, ParseTree* lop, ParseTree* rop) { evaluate(row, isNull, lop, rop); - return TreeNode::getStrVal(); + return TreeNode::getStrVal(fTimeZone); } virtual int64_t getIntVal(rowgroup::Row& row, bool& isNull, ParseTree* lop, ParseTree* rop) { @@ -157,6 +166,11 @@ public: evaluate(row, isNull, lop, rop); return TreeNode::getDatetimeIntVal(); } + virtual int64_t getTimestampIntVal(rowgroup::Row& row, bool& isNull, ParseTree* lop, ParseTree* rop) + { + evaluate(row, isNull, lop, rop); + return TreeNode::getTimestampIntVal(); + } virtual int64_t getTimeIntVal(rowgroup::Row& row, bool& isNull, ParseTree* lop, ParseTree* rop) { evaluate(row, isNull, lop, rop); @@ -173,6 +187,7 @@ private: template inline result_t execute(result_t op1, result_t op2, bool& isNull); inline void execute(IDB_Decimal& result, IDB_Decimal op1, IDB_Decimal op2, bool& isNull); + std::string fTimeZone; }; #include "parsetree.h" diff --git a/dbcon/execplan/calpontselectexecutionplan.cpp b/dbcon/execplan/calpontselectexecutionplan.cpp index 0e2417588..fd12b8179 100644 --- a/dbcon/execplan/calpontselectexecutionplan.cpp +++ b/dbcon/execplan/calpontselectexecutionplan.cpp @@ -497,6 +497,7 @@ void CalpontSelectExecutionPlan::serialize(messageqcpp::ByteStream& b) const b << fDJSPartitionSize; b << fUMMemLimit; b << (uint8_t) fIsDML; + b << fTimeZone; } void CalpontSelectExecutionPlan::unserialize(messageqcpp::ByteStream& b) @@ -670,6 +671,7 @@ void CalpontSelectExecutionPlan::unserialize(messageqcpp::ByteStream& b) b >> fUMMemLimit; b >> tmp8; fIsDML = tmp8; + b >> fTimeZone; } bool CalpontSelectExecutionPlan::operator==(const CalpontSelectExecutionPlan& t) const diff --git a/dbcon/execplan/calpontselectexecutionplan.h b/dbcon/execplan/calpontselectexecutionplan.h index 5d1f2fbb6..3e4eedcbf 100644 --- a/dbcon/execplan/calpontselectexecutionplan.h +++ b/dbcon/execplan/calpontselectexecutionplan.h @@ -694,6 +694,15 @@ public: return fIsDML; } + void timeZone(const std::string& timezone) + { + fTimeZone = timezone; + } + const std::string timeZone() const + { + return fTimeZone; + } + /** * The serialization interface */ @@ -895,6 +904,8 @@ private: uint64_t fDJSPartitionSize; int64_t fUMMemLimit; bool fIsDML; + + std::string fTimeZone; }; /** diff --git a/dbcon/execplan/calpontsystemcatalog.cpp b/dbcon/execplan/calpontsystemcatalog.cpp index 119678cf6..234fb6235 100644 --- a/dbcon/execplan/calpontsystemcatalog.cpp +++ b/dbcon/execplan/calpontsystemcatalog.cpp @@ -167,6 +167,10 @@ const string colDataTypeToString(CalpontSystemCatalog::ColDataType cdt) return "time"; break; + case CalpontSystemCatalog::TIMESTAMP: + return "timestamp"; + break; + case CalpontSystemCatalog::VARCHAR: return "varchar"; break; diff --git a/dbcon/execplan/calpontsystemcatalog.h b/dbcon/execplan/calpontsystemcatalog.h index d55ce93fa..188dae4e6 100644 --- a/dbcon/execplan/calpontsystemcatalog.h +++ b/dbcon/execplan/calpontsystemcatalog.h @@ -161,6 +161,7 @@ public: UDOUBLE, /*!< Unsigned DOUBLE type */ TEXT, /*!< TEXT type */ TIME, /*!< TIME type */ + TIMESTAMP, /*!< TIMESTAMP type */ NUM_OF_COL_DATA_TYPE, /* NEW TYPES ABOVE HERE */ LONGDOUBLE, /* @bug3241, dev and variance calculation only */ STRINT, /* @bug3532, string as int for fast comparison */ @@ -1159,6 +1160,13 @@ inline bool isNull(int64_t val, const execplan::CalpontSystemCatalog::ColType& c break; } + case execplan::CalpontSystemCatalog::TIMESTAMP: + { + if ((int64_t)joblist::TIMESTAMPNULL == val) ret = true; + + break; + } + case execplan::CalpontSystemCatalog::VARCHAR: { int colWidth = ct.colWidth; diff --git a/dbcon/execplan/constantcolumn.cpp b/dbcon/execplan/constantcolumn.cpp index 3a816b114..aa2676816 100644 --- a/dbcon/execplan/constantcolumn.cpp +++ b/dbcon/execplan/constantcolumn.cpp @@ -195,7 +195,8 @@ ConstantColumn::ConstantColumn( const ConstantColumn& rhs): ReturnedColumn(rhs), fConstval (rhs.constval()), fType (rhs.type()), - fData (rhs.data()) + fData (rhs.data()), + fTimeZone (rhs.timeZone()) { sequence(rhs.sequence()); fAlias = rhs.alias(); @@ -306,6 +307,7 @@ void ConstantColumn::serialize(messageqcpp::ByteStream& b) const b << (uint32_t) fType; //b << fAlias; b << fData; + b << fTimeZone; b << static_cast(fReturnAll); b << (uint64_t)fResult.intVal; b << fResult.uintVal; @@ -329,6 +331,7 @@ void ConstantColumn::unserialize(messageqcpp::ByteStream& b) b >> fConstval; b >> (uint32_t&) fType; b >> fData; + b >> fTimeZone; b >> reinterpret_cast< ByteStream::doublebyte&>(fReturnAll); b >> (uint64_t&)fResult.intVal; b >> fResult.uintVal; @@ -365,6 +368,9 @@ bool ConstantColumn::operator==(const ConstantColumn& t) const if (fReturnAll != t.fReturnAll) return false; + if (fTimeZone != t.fTimeZone) + return false; + return true; } diff --git a/dbcon/execplan/constantcolumn.h b/dbcon/execplan/constantcolumn.h index ed8315202..51b914be0 100644 --- a/dbcon/execplan/constantcolumn.h +++ b/dbcon/execplan/constantcolumn.h @@ -112,6 +112,20 @@ public: { fConstval = constval; } + /** + * accessor + */ + inline const std::string& timeZone() const + { + return fTimeZone; + } + /** + * mutator + */ + inline void timeZone(const std::string& timeZone) + { + fTimeZone = timeZone; + } /** * accessor */ @@ -192,6 +206,7 @@ private: std::string fConstval; int fType; std::string fData; + std::string fTimeZone; /*********************************************************** * F&E framework * @@ -315,6 +330,21 @@ public: return fResult.intVal; } + /** + * F&E + */ + virtual int64_t getTimestampIntVal(rowgroup::Row& row, bool& isNull) + { + isNull = isNull || (fType == NULLDATA); + + if (!fResult.valueConverted) + { + fResult.intVal = dataconvert::DataConvert::stringToTimestamp(fResult.strVal, fTimeZone); + fResult.valueConverted = true; + } + + return fResult.intVal; + } /** * F&E */ diff --git a/dbcon/execplan/functioncolumn.cpp b/dbcon/execplan/functioncolumn.cpp index 487d42c0b..c815a3cb2 100644 --- a/dbcon/execplan/functioncolumn.cpp +++ b/dbcon/execplan/functioncolumn.cpp @@ -74,6 +74,7 @@ FunctionColumn::FunctionColumn( const FunctionColumn& rhs, const uint32_t sessio fFunctionName(rhs.functionName()), fTableAlias (rhs.tableAlias()), fData (rhs.data()), + fTimeZone(rhs.timeZone()), fFunctor(rhs.fFunctor) { fFunctionParms.clear(); @@ -266,6 +267,7 @@ void FunctionColumn::serialize(messageqcpp::ByteStream& b) const b << fTableAlias; b << fData; + b << fTimeZone; } void FunctionColumn::unserialize(messageqcpp::ByteStream& b) @@ -297,8 +299,10 @@ void FunctionColumn::unserialize(messageqcpp::ByteStream& b) b >> fTableAlias; b >> fData; + b >> fTimeZone; FuncExp* funcExp = FuncExp::instance(); fFunctor = funcExp->getFunctor(fFunctionName); + fFunctor->timeZone(fTimeZone); // @bug 3506. Special treatment for rand() function. reset the seed Func_rand* rand = dynamic_cast(fFunctor); @@ -338,8 +342,10 @@ bool FunctionColumn::operator==(const FunctionColumn& t) const if (fData != t.fData) return false; + if (fTimeZone != t.fTimeZone) + return false; + return true; - return false; } bool FunctionColumn::operator==(const TreeNode* t) const diff --git a/dbcon/execplan/functioncolumn.h b/dbcon/execplan/functioncolumn.h index cb4a397fb..99e3c9be7 100644 --- a/dbcon/execplan/functioncolumn.h +++ b/dbcon/execplan/functioncolumn.h @@ -123,6 +123,16 @@ public: fTableAlias = tableAlias; } + inline const std::string timeZone () const + { + return fTimeZone; + } + + inline void timeZone (const std::string& timeZone) + { + fTimeZone = timeZone; + } + virtual const std::string data() const; virtual void data(const std::string data) { @@ -172,6 +182,7 @@ private: std::string fFunctionName; /// function name std::string fTableAlias; /// table alias which has the column std::string fData; /// SQL representation + std::string fTimeZone; /** @brief Do a deep, strict (as opposed to semantic) equivalence test * @@ -260,6 +271,10 @@ public: { return fFunctor->getDatetimeIntVal(row, fFunctionParms, isNull, fOperationType); } + virtual int64_t getTimestampIntVal(rowgroup::Row& row, bool& isNull) + { + return fFunctor->getTimestampIntVal(row, fFunctionParms, isNull, fOperationType); + } virtual int64_t getTimeIntVal(rowgroup::Row& row, bool& isNull) { return fFunctor->getTimeIntVal(row, fFunctionParms, isNull, fOperationType); diff --git a/dbcon/execplan/operator.h b/dbcon/execplan/operator.h index 13dea8d38..b8d2ab18c 100644 --- a/dbcon/execplan/operator.h +++ b/dbcon/execplan/operator.h @@ -195,6 +195,10 @@ public: { return fResult.intVal; } + virtual int64_t getTimestampIntVal(rowgroup::Row& row, bool& isNull, ParseTree* lop, ParseTree* rop) + { + return fResult.intVal; + } virtual int64_t getTimeIntVal(rowgroup::Row& row, bool& isNull, ParseTree* lop, ParseTree* rop) { return fResult.intVal; diff --git a/dbcon/execplan/parsetree.h b/dbcon/execplan/parsetree.h index 044d54fd6..3561d9dc6 100644 --- a/dbcon/execplan/parsetree.h +++ b/dbcon/execplan/parsetree.h @@ -291,6 +291,14 @@ public: return fData->getDatetimeIntVal(row, isNull); } + inline int64_t getTimestampIntVal(rowgroup::Row& row, bool& isNull) + { + if (fLeft && fRight) + return (reinterpret_cast(fData))->getTimestampIntVal(row, isNull, fLeft, fRight); + else + return fData->getTimestampIntVal(row, isNull); + } + inline int64_t getTimeIntVal(rowgroup::Row& row, bool& isNull) { if (fLeft && fRight) diff --git a/dbcon/execplan/predicateoperator.cpp b/dbcon/execplan/predicateoperator.cpp index b32de94c2..501bd8a73 100644 --- a/dbcon/execplan/predicateoperator.cpp +++ b/dbcon/execplan/predicateoperator.cpp @@ -182,6 +182,7 @@ void PredicateOperator::setOpType(Type& l, Type& r) { if ( l.colDataType == execplan::CalpontSystemCatalog::DATETIME || l.colDataType == execplan::CalpontSystemCatalog::TIME || + l.colDataType == execplan::CalpontSystemCatalog::TIMESTAMP || l.colDataType == execplan::CalpontSystemCatalog::DATE ) { switch (r.colDataType) @@ -196,6 +197,11 @@ void PredicateOperator::setOpType(Type& l, Type& r) fOperationType.colWidth = 8; break; + case execplan::CalpontSystemCatalog::TIMESTAMP: + fOperationType.colDataType = execplan::CalpontSystemCatalog::TIMESTAMP; + fOperationType.colWidth = 8; + break; + case execplan::CalpontSystemCatalog::TIME: fOperationType.colDataType = execplan::CalpontSystemCatalog::TIME; fOperationType.colWidth = 8; @@ -213,6 +219,7 @@ void PredicateOperator::setOpType(Type& l, Type& r) } else if ( r.colDataType == execplan::CalpontSystemCatalog::DATETIME || r.colDataType == execplan::CalpontSystemCatalog::TIME || + r.colDataType == execplan::CalpontSystemCatalog::TIMESTAMP || r.colDataType == execplan::CalpontSystemCatalog::DATE ) { switch (l.colDataType) @@ -228,6 +235,11 @@ void PredicateOperator::setOpType(Type& l, Type& r) fOperationType.colWidth = 8; break; + case execplan::CalpontSystemCatalog::TIMESTAMP: + fOperationType.colDataType = execplan::CalpontSystemCatalog::TIMESTAMP; + fOperationType.colWidth = 8; + break; + case execplan::CalpontSystemCatalog::TIME: fOperationType.colDataType = execplan::CalpontSystemCatalog::TIME; fOperationType.colWidth = 8; diff --git a/dbcon/execplan/predicateoperator.h b/dbcon/execplan/predicateoperator.h index 66a12ec32..75cde2051 100644 --- a/dbcon/execplan/predicateoperator.h +++ b/dbcon/execplan/predicateoperator.h @@ -406,6 +406,35 @@ inline bool PredicateOperator::getBoolVal(rowgroup::Row& row, bool& isNull, Retu return numericCompare(val1, rop->getDatetimeIntVal(row, isNull)) && !isNull; } + case execplan::CalpontSystemCatalog::TIMESTAMP: + { + if (fOp == OP_ISNULL) + { + lop->getTimestampIntVal(row, isNull); + bool ret = isNull; + isNull = false; + return ret; + } + + if (fOp == OP_ISNOTNULL) + { + lop->getTimestampIntVal(row, isNull); + bool ret = isNull; + isNull = false; + return !ret; + } + + if (isNull) + return false; + + int64_t val1 = lop->getTimestampIntVal(row, isNull); + + if (isNull) + return false; + + return numericCompare(val1, rop->getTimestampIntVal(row, isNull)) && !isNull; + } + case execplan::CalpontSystemCatalog::TIME: { if (fOp == OP_ISNULL) diff --git a/dbcon/execplan/simplecolumn.cpp b/dbcon/execplan/simplecolumn.cpp index 206194954..68f09ee26 100644 --- a/dbcon/execplan/simplecolumn.cpp +++ b/dbcon/execplan/simplecolumn.cpp @@ -196,6 +196,7 @@ SimpleColumn::SimpleColumn (const SimpleColumn& rhs, const uint32_t sessionID): fData (rhs.data()), fIndexName (rhs.indexName()), fViewName (rhs.viewName()), + fTimeZone (rhs.timeZone()), fIsInfiniDB (rhs.isInfiniDB()) { } @@ -229,6 +230,7 @@ SimpleColumn& SimpleColumn::operator=(const SimpleColumn& rhs) fAsc = rhs.asc(); fIndexName = rhs.indexName(); fViewName = rhs.viewName(); + fTimeZone = rhs.timeZone(); fData = rhs.data(); fSequence = rhs.sequence(); fDistinct = rhs.distinct(); @@ -345,6 +347,7 @@ void SimpleColumn::serialize(messageqcpp::ByteStream& b) const b << fColumnName; b << fIndexName; b << fViewName; + b << fTimeZone; b << (uint32_t) fOid; b << fData; b << fTableAlias; @@ -361,6 +364,7 @@ void SimpleColumn::unserialize(messageqcpp::ByteStream& b) b >> fColumnName; b >> fIndexName; b >> fViewName; + b >> fTimeZone; b >> (uint32_t&) fOid; b >> fData; b >> fTableAlias; @@ -390,6 +394,9 @@ bool SimpleColumn::operator==(const SimpleColumn& t) const if (fViewName != t.fViewName) return false; + if (fTimeZone != t.fTimeZone) + return false; + if (fOid != t.fOid) return false; @@ -497,6 +504,7 @@ void SimpleColumn::evaluate(Row& row, bool& isNull) } case CalpontSystemCatalog::DATETIME: + case CalpontSystemCatalog::TIMESTAMP: case CalpontSystemCatalog::TIME: { fResult.intVal = row.getUintField<8>(fInputIndex); diff --git a/dbcon/execplan/simplecolumn.h b/dbcon/execplan/simplecolumn.h index 6694863d6..cb63e9683 100644 --- a/dbcon/execplan/simplecolumn.h +++ b/dbcon/execplan/simplecolumn.h @@ -152,6 +152,14 @@ public: { fViewName = viewName; } + inline const std::string& timeZone() const + { + return fTimeZone; + } + inline void timeZone(const std::string& timeZone) + { + fTimeZone = timeZone; + } inline const bool isInfiniDB() const { return fIsInfiniDB; @@ -252,6 +260,7 @@ protected: std::string fIndexName; // if belong to view, view name is non-empty std::string fViewName; + std::string fTimeZone; bool fIsInfiniDB; /** @brief parse SimpleColumn text @@ -274,7 +283,7 @@ public: virtual const std::string& getStrVal(rowgroup::Row& row, bool& isNull) { evaluate(row, isNull); - return TreeNode::getStrVal(); + return TreeNode::getStrVal(fTimeZone); } virtual int64_t getIntVal(rowgroup::Row& row, bool& isNull) @@ -348,6 +357,12 @@ public: return TreeNode::getDatetimeIntVal(); } + inline int64_t getTimestampIntVal(rowgroup::Row& row, bool& isNull) + { + evaluate(row, isNull); + return TreeNode::getTimestampIntVal(); + } + inline int64_t getTimeIntVal(rowgroup::Row& row, bool& isNull) { evaluate(row, isNull); diff --git a/dbcon/execplan/simplefilter.cpp b/dbcon/execplan/simplefilter.cpp index 639a04945..5a4fd996d 100644 --- a/dbcon/execplan/simplefilter.cpp +++ b/dbcon/execplan/simplefilter.cpp @@ -60,8 +60,8 @@ SimpleFilter::SimpleFilter(const string& sql): parse(sql); } -SimpleFilter::SimpleFilter(const SOP& op, ReturnedColumn* lhs, ReturnedColumn* rhs) : - fOp(op), fLhs(lhs), fRhs(rhs), fIndexFlag(NOINDEX), fJoinFlag(EQUA) +SimpleFilter::SimpleFilter(const SOP& op, ReturnedColumn* lhs, ReturnedColumn* rhs, const string& timeZone) : + fOp(op), fLhs(lhs), fRhs(rhs), fIndexFlag(NOINDEX), fJoinFlag(EQUA), fTimeZone(timeZone) { convertConstant(); } @@ -69,7 +69,8 @@ SimpleFilter::SimpleFilter(const SOP& op, ReturnedColumn* lhs, ReturnedColumn* r SimpleFilter::SimpleFilter(const SimpleFilter& rhs) : fOp(rhs.op()), fIndexFlag(rhs.indexFlag()), - fJoinFlag(rhs.joinFlag()) + fJoinFlag(rhs.joinFlag()), + fTimeZone(rhs.timeZone()) { fLhs = rhs.lhs()->clone(); fRhs = rhs.rhs()->clone(); @@ -210,6 +211,7 @@ const string SimpleFilter::data() const fRhs->resultType().colDataType == CalpontSystemCatalog::VARBINARY || fRhs->resultType().colDataType == CalpontSystemCatalog::DATE || fRhs->resultType().colDataType == CalpontSystemCatalog::DATETIME || + fRhs->resultType().colDataType == CalpontSystemCatalog::TIMESTAMP || fRhs->resultType().colDataType == CalpontSystemCatalog::TIME)) rhs = "'" + SimpleFilter::escapeString(fRhs->data()) + "'"; else @@ -223,6 +225,7 @@ const string SimpleFilter::data() const fLhs->resultType().colDataType == CalpontSystemCatalog::VARBINARY || fLhs->resultType().colDataType == CalpontSystemCatalog::DATE || fLhs->resultType().colDataType == CalpontSystemCatalog::TIME || + fLhs->resultType().colDataType == CalpontSystemCatalog::TIMESTAMP || fLhs->resultType().colDataType == CalpontSystemCatalog::DATETIME)) lhs = "'" + SimpleFilter::escapeString(fLhs->data()) + "'"; else @@ -313,6 +316,7 @@ void SimpleFilter::serialize(messageqcpp::ByteStream& b) const b << static_cast(fIndexFlag); b << static_cast(fJoinFlag); + b << fTimeZone; } void SimpleFilter::unserialize(messageqcpp::ByteStream& b) @@ -328,6 +332,7 @@ void SimpleFilter::unserialize(messageqcpp::ByteStream& b) fRhs = dynamic_cast(ObjectReader::createTreeNode(b)); b >> reinterpret_cast(fIndexFlag); b >> reinterpret_cast(fJoinFlag); + b >> fTimeZone; fSimpleColumnList.clear(); fAggColumnList.clear(); @@ -459,6 +464,9 @@ bool SimpleFilter::operator==(const SimpleFilter& t) const else if (fJoinFlag != t.fJoinFlag) return false; + else if (fTimeZone != t.fTimeZone) + return false; + return true; } @@ -546,6 +554,19 @@ void SimpleFilter::convertConstant() result.intVal = dataconvert::DataConvert::datetimeToInt(result.strVal); } } + else if (fRhs->resultType().colDataType == CalpontSystemCatalog::TIMESTAMP) + { + if (lcc->constval().empty()) + { + lcc->constval("0000-00-00 00:00:00"); + result.intVal = 0; + result.strVal = lcc->constval(); + } + else + { + result.intVal = dataconvert::DataConvert::timestampToInt(result.strVal, fTimeZone); + } + } else if (fRhs->resultType().colDataType == CalpontSystemCatalog::TIME) { if (lcc->constval().empty()) @@ -593,6 +614,19 @@ void SimpleFilter::convertConstant() result.intVal = dataconvert::DataConvert::datetimeToInt(result.strVal); } } + else if (fLhs->resultType().colDataType == CalpontSystemCatalog::TIMESTAMP) + { + if (rcc->constval().empty()) + { + rcc->constval("0000-00-00 00:00:00"); + result.intVal = 0; + result.strVal = rcc->constval(); + } + else + { + result.intVal = dataconvert::DataConvert::timestampToInt(result.strVal, fTimeZone); + } + } else if (fLhs->resultType().colDataType == CalpontSystemCatalog::TIME) { if (rcc->constval().empty()) diff --git a/dbcon/execplan/simplefilter.h b/dbcon/execplan/simplefilter.h index 04afb087a..e448aa3ed 100644 --- a/dbcon/execplan/simplefilter.h +++ b/dbcon/execplan/simplefilter.h @@ -69,7 +69,7 @@ public: SimpleFilter(); SimpleFilter(const std::string& sql); - SimpleFilter(const SOP& op, ReturnedColumn* lhs, ReturnedColumn* rhs); + SimpleFilter(const SOP& op, ReturnedColumn* lhs, ReturnedColumn* rhs, const std::string& timeZone = ""); SimpleFilter(const SimpleFilter& rhs); virtual ~SimpleFilter(); @@ -94,6 +94,16 @@ public: return fLhs; } + inline const std::string& timeZone() const + { + return fTimeZone; + } + + inline void timeZone(const std::string& timeZone) + { + fTimeZone = timeZone; + } + virtual const std::string data() const; /** assign fLhs @@ -203,6 +213,7 @@ private: ReturnedColumn* fRhs; /// right operand int fIndexFlag; /// which side col is index int fJoinFlag; /// hash join type + std::string fTimeZone; void parse (std::string); diff --git a/dbcon/execplan/treenode.h b/dbcon/execplan/treenode.h index 9c4283042..84d2ccfac 100644 --- a/dbcon/execplan/treenode.h +++ b/dbcon/execplan/treenode.h @@ -395,6 +395,10 @@ public: { return fResult.intVal; } + virtual int64_t getTimestampIntVal(rowgroup::Row& row, bool& isNull) + { + return fResult.intVal; + } virtual int64_t getTimeIntVal(rowgroup::Row& row, bool& isNull) { return fResult.intVal; @@ -402,7 +406,7 @@ public: virtual void evaluate(rowgroup::Row& row, bool& isNull) {} inline bool getBoolVal(); - inline const std::string& getStrVal(); + inline const std::string& getStrVal(const std::string& timeZone); inline int64_t getIntVal(); inline uint64_t getUintVal(); inline float getFloatVal(); @@ -411,6 +415,7 @@ public: inline IDB_Decimal getDecimalVal(); inline int32_t getDateIntVal(); inline int64_t getDatetimeIntVal(); + inline int64_t getTimestampIntVal(); inline int64_t getTimeIntVal(); virtual const execplan::CalpontSystemCatalog::ColType& resultType() const @@ -503,6 +508,7 @@ inline bool TreeNode::getBoolVal() case CalpontSystemCatalog::INT: case CalpontSystemCatalog::DATE: case CalpontSystemCatalog::DATETIME: + case CalpontSystemCatalog::TIMESTAMP: case CalpontSystemCatalog::TIME: return (fResult.intVal != 0); @@ -535,7 +541,7 @@ inline bool TreeNode::getBoolVal() return fResult.boolVal; } -inline const std::string& TreeNode::getStrVal() +inline const std::string& TreeNode::getStrVal(const std::string& timeZone) { switch (fResultType.colDataType) { @@ -716,6 +722,13 @@ inline const std::string& TreeNode::getStrVal() break; } + case CalpontSystemCatalog::TIMESTAMP: + { + dataconvert::DataConvert::timestampToString(fResult.intVal, tmp, 255, timeZone, fResultType.precision); + fResult.strVal = std::string(tmp); + break; + } + case CalpontSystemCatalog::TIME: { dataconvert::DataConvert::timeToString(fResult.intVal, tmp, 255, fResultType.precision); @@ -788,6 +801,7 @@ inline int64_t TreeNode::getIntVal() case CalpontSystemCatalog::DATE: case CalpontSystemCatalog::DATETIME: + case CalpontSystemCatalog::TIMESTAMP: case CalpontSystemCatalog::TIME: return fResult.intVal; @@ -834,6 +848,7 @@ inline uint64_t TreeNode::getUintVal() case CalpontSystemCatalog::DATE: case CalpontSystemCatalog::DATETIME: + case CalpontSystemCatalog::TIMESTAMP: case CalpontSystemCatalog::TIME: return fResult.intVal; @@ -900,6 +915,7 @@ inline float TreeNode::getFloatVal() case CalpontSystemCatalog::DATE: case CalpontSystemCatalog::DATETIME: + case CalpontSystemCatalog::TIMESTAMP: case CalpontSystemCatalog::TIME: return (float)fResult.intVal; @@ -968,6 +984,7 @@ inline double TreeNode::getDoubleVal() case CalpontSystemCatalog::DATE: case CalpontSystemCatalog::DATETIME: + case CalpontSystemCatalog::TIMESTAMP: case CalpontSystemCatalog::TIME: return (double)fResult.intVal; @@ -1101,6 +1118,9 @@ inline IDB_Decimal TreeNode::getDecimalVal() case CalpontSystemCatalog::DATETIME: throw logging::InvalidConversionExcept("TreeNode::getDecimalVal: Invalid conversion from datetime."); + case CalpontSystemCatalog::TIMESTAMP: + throw logging::InvalidConversionExcept("TreeNode::getDecimalVal: Invalid conversion from timestamp."); + case CalpontSystemCatalog::TIME: throw logging::InvalidConversionExcept("TreeNode::getDecimalVal: Invalid conversion from time."); @@ -1162,6 +1182,14 @@ inline int64_t TreeNode::getDatetimeIntVal() return getIntVal(); } +inline int64_t TreeNode::getTimestampIntVal() +{ + if (fResultType.colDataType == execplan::CalpontSystemCatalog::TIMESTAMP) + return fResult.intVal; + else + return getIntVal(); +} + inline int64_t TreeNode::getTimeIntVal() { if (fResultType.colDataType == execplan::CalpontSystemCatalog::DATETIME) diff --git a/dbcon/execplan/windowfunctioncolumn.cpp b/dbcon/execplan/windowfunctioncolumn.cpp index e01c5c010..09ba50ddc 100644 --- a/dbcon/execplan/windowfunctioncolumn.cpp +++ b/dbcon/execplan/windowfunctioncolumn.cpp @@ -235,7 +235,8 @@ WindowFunctionColumn::WindowFunctionColumn( const WindowFunctionColumn& rhs, con fFunctionParms(rhs.functionParms()), fPartitions (rhs.partitions()), fOrderBy (rhs.orderBy()), - udafContext(rhs.getUDAFContext()) + udafContext(rhs.getUDAFContext()), + fTimeZone(rhs.timeZone()) {} const string WindowFunctionColumn::toString() const @@ -288,6 +289,7 @@ void WindowFunctionColumn::serialize(messageqcpp::ByteStream& b) const fOrderBy.serialize(b); udafContext.serialize(b); + b << fTimeZone; } void WindowFunctionColumn::unserialize(messageqcpp::ByteStream& b) @@ -319,6 +321,7 @@ void WindowFunctionColumn::unserialize(messageqcpp::ByteStream& b) fOrderBy.unserialize(b); udafContext.unserialize(b); + b >> fTimeZone; } void WindowFunctionColumn::addToPartition(vector& groupByList) @@ -413,6 +416,16 @@ void WindowFunctionColumn::evaluate(Row& row, bool& isNull) break; } + case CalpontSystemCatalog::TIMESTAMP: + { + if (row.equals<8>(TIMESTAMPNULL, fInputIndex)) + isNull = true; + else + fResult.intVal = row.getUintField<8>(fInputIndex); + + break; + } + case CalpontSystemCatalog::TIME: { if (row.equals<8>(TIMENULL, fInputIndex)) diff --git a/dbcon/execplan/windowfunctioncolumn.h b/dbcon/execplan/windowfunctioncolumn.h index 55f062990..8dee14ece 100644 --- a/dbcon/execplan/windowfunctioncolumn.h +++ b/dbcon/execplan/windowfunctioncolumn.h @@ -146,6 +146,16 @@ public: return udafContext; } + inline const std::string timeZone () const + { + return fTimeZone; + } + + inline void timeZone (const std::string& timeZone) + { + fTimeZone = timeZone; + } + private: /** * Fields @@ -169,6 +179,8 @@ private: // UDAnF support mcsv1sdk::mcsv1Context udafContext; + + std::string fTimeZone; /*********************************************************** * F&E framework * ***********************************************************/ @@ -176,7 +188,7 @@ public: virtual const std::string& getStrVal(rowgroup::Row& row, bool& isNull) { evaluate(row, isNull); - return TreeNode::getStrVal(); + return TreeNode::getStrVal(fTimeZone); } virtual int64_t getIntVal(rowgroup::Row& row, bool& isNull) @@ -224,6 +236,11 @@ public: evaluate(row, isNull); return TreeNode::getDatetimeIntVal(); } + virtual int64_t getTimestampIntVal(rowgroup::Row& row, bool& isNull) + { + evaluate(row, isNull); + return TreeNode::getTimestampIntVal(); + } virtual int64_t getTimeIntVal(rowgroup::Row& row, bool& isNull) { evaluate(row, isNull); diff --git a/dbcon/joblist/crossenginestep.cpp b/dbcon/joblist/crossenginestep.cpp index e9b573713..07ab611e7 100644 --- a/dbcon/joblist/crossenginestep.cpp +++ b/dbcon/joblist/crossenginestep.cpp @@ -247,7 +247,7 @@ int64_t CrossEngineStep::convertValueNum( // bool nulFlag, // bool noRoundup ) bool pushWarning = false; - boost::any anyVal = DataConvert::convertColumnData(ct, str, pushWarning, false, true); + boost::any anyVal = DataConvert::convertColumnData(ct, str, pushWarning, fTimeZone, false, true, false); // Out of range values are treated as NULL as discussed during design review. if (pushWarning) @@ -351,6 +351,10 @@ int64_t CrossEngineStep::convertValueNum( rv = boost::any_cast(anyVal); break; + case CalpontSystemCatalog::TIMESTAMP: + rv = boost::any_cast(anyVal); + break; + case CalpontSystemCatalog::TIME: rv = boost::any_cast(anyVal); break; diff --git a/dbcon/joblist/expressionstep.cpp b/dbcon/joblist/expressionstep.cpp index a2efad74a..17dc3d68a 100644 --- a/dbcon/joblist/expressionstep.cpp +++ b/dbcon/joblist/expressionstep.cpp @@ -414,9 +414,10 @@ void ExpressionStep::populateColumnInfo(SimpleColumn* sc, JobInfo& jobInfo) TupleInfo ti(setTupleInfo(ct, sc->oid(), jobInfo, tblOid, sc, alias)); fColumnKeys.push_back(ti.key); - // @bug 2990, MySQL time/date/datetime type is different from IDB type + // @bug 2990, MySQL timestamp/time/date/datetime type is different from IDB type if (ti.dtype == CalpontSystemCatalog::DATE || ti.dtype == CalpontSystemCatalog::DATETIME || - ti.dtype == CalpontSystemCatalog::TIME) + ti.dtype == CalpontSystemCatalog::TIME || + ti.dtype == CalpontSystemCatalog::TIMESTAMP) { if (ti.dtype != ct.colDataType) { diff --git a/dbcon/joblist/groupconcat.cpp b/dbcon/joblist/groupconcat.cpp index fe7dd8482..1433bd012 100644 --- a/dbcon/joblist/groupconcat.cpp +++ b/dbcon/joblist/groupconcat.cpp @@ -87,6 +87,7 @@ void GroupConcatInfo::prepGroupConcat(JobInfo& jobInfo) groupConcat->fSize = gcc->resultType().colWidth; groupConcat->fRm = jobInfo.rm; groupConcat->fSessionMemLimit = jobInfo.umMemLimit; + groupConcat->fTimeZone = jobInfo.timeZone; int key = -1; const vector& cols = rcp->columnVec(); @@ -396,6 +397,7 @@ void GroupConcator::initialize(const rowgroup::SP_GroupConcat& gcc) // too high(3MB) to allocate it for every instance. fGroupConcatLen = gcc->fSize; fCurrentLength -= strlen(gcc->fSeparator.c_str()); + fTimeZone = gcc->fTimeZone; fConstCols = gcc->fConstCols; fConstantLen = strlen(gcc->fSeparator.c_str()); @@ -510,6 +512,12 @@ void GroupConcator::outputRow(std::ostringstream& oss, const rowgroup::Row& row) break; } + case CalpontSystemCatalog::TIMESTAMP: + { + oss << DataConvert::timestampToString(row.getUintField(*i), fTimeZone); + break; + } + case CalpontSystemCatalog::TIME: { oss << DataConvert::timeToString(row.getUintField(*i)); @@ -643,6 +651,7 @@ int64_t GroupConcator::lengthEstimate(const rowgroup::Row& row) } case CalpontSystemCatalog::DATETIME: + case CalpontSystemCatalog::TIMESTAMP: { fieldLen = 19; // YYYY-MM-DD HH24:MI:SS // Decimal point and milliseconds diff --git a/dbcon/joblist/groupconcat.h b/dbcon/joblist/groupconcat.h index 752d09304..40a4d7e53 100644 --- a/dbcon/joblist/groupconcat.h +++ b/dbcon/joblist/groupconcat.h @@ -132,6 +132,7 @@ protected: int64_t fGroupConcatLen; int64_t fConstantLen; boost::scoped_array fOutputString; + std::string fTimeZone; }; diff --git a/dbcon/joblist/jlf_common.cpp b/dbcon/joblist/jlf_common.cpp index 7ce066427..bdaa18252 100644 --- a/dbcon/joblist/jlf_common.cpp +++ b/dbcon/joblist/jlf_common.cpp @@ -792,6 +792,11 @@ bool compatibleColumnTypes(const CalpontSystemCatalog::ColDataType& dt1, uint32_ break; + case CalpontSystemCatalog::TIMESTAMP: + if (dt2 != CalpontSystemCatalog::TIMESTAMP) return false; + + break; + case CalpontSystemCatalog::TIME: if (dt2 != CalpontSystemCatalog::TIME) return false; diff --git a/dbcon/joblist/jlf_common.h b/dbcon/joblist/jlf_common.h index 18c1ea396..76538eef9 100644 --- a/dbcon/joblist/jlf_common.h +++ b/dbcon/joblist/jlf_common.h @@ -360,6 +360,7 @@ struct JobInfo int64_t largeSideLimit; uint64_t partitionSize; bool isDML; + std::string timeZone; private: //defaults okay diff --git a/dbcon/joblist/jlf_execplantojoblist.cpp b/dbcon/joblist/jlf_execplantojoblist.cpp index 7bbb8a136..54cfaf726 100644 --- a/dbcon/joblist/jlf_execplantojoblist.cpp +++ b/dbcon/joblist/jlf_execplantojoblist.cpp @@ -130,11 +130,11 @@ const JobStepVector doSimpleFilter(SimpleFilter* sf, JobInfo& jobInfo); /* This looks like an inefficient way to get NULL values. Much easier ways to do it. */ -int64_t valueNullNum(const CalpontSystemCatalog::ColType& ct) +int64_t valueNullNum(const CalpontSystemCatalog::ColType& ct, const string& timeZone) { int64_t n = 0; bool pushWarning = false; - boost::any anyVal = DataConvert::convertColumnData(ct, "", pushWarning, true); + boost::any anyVal = DataConvert::convertColumnData(ct, "", pushWarning, timeZone, true, false, false); switch (ct.colDataType) { @@ -282,6 +282,10 @@ int64_t valueNullNum(const CalpontSystemCatalog::ColType& ct) n = boost::any_cast(anyVal); break; + case CalpontSystemCatalog::TIMESTAMP: + n = boost::any_cast(anyVal); + break; + case CalpontSystemCatalog::TIME: n = boost::any_cast(anyVal); break; @@ -308,14 +312,14 @@ int64_t valueNullNum(const CalpontSystemCatalog::ColType& ct) return n; } -int64_t convertValueNum(const string& str, const CalpontSystemCatalog::ColType& ct, bool isNull, uint8_t& rf) +int64_t convertValueNum(const string& str, const CalpontSystemCatalog::ColType& ct, bool isNull, uint8_t& rf, const string& timeZone) { - if (str.size() == 0 || isNull ) return valueNullNum(ct); + if (str.size() == 0 || isNull ) return valueNullNum(ct, timeZone); int64_t v = 0; rf = 0; bool pushWarning = false; - boost::any anyVal = DataConvert::convertColumnData(ct, str, pushWarning, false, true); + boost::any anyVal = DataConvert::convertColumnData(ct, str, pushWarning, timeZone, false, true, false); switch (ct.colDataType) { @@ -425,6 +429,10 @@ int64_t convertValueNum(const string& str, const CalpontSystemCatalog::ColType& v = boost::any_cast(anyVal); break; + case CalpontSystemCatalog::TIMESTAMP: + v = boost::any_cast(anyVal); + break; + case CalpontSystemCatalog::TIME: v = boost::any_cast(anyVal); break; @@ -1823,7 +1831,7 @@ const JobStepVector doSimpleFilter(SimpleFilter* sf, JobInfo& jobInfo) try { bool isNull = ConstantColumn::NULLDATA == cc->type(); - value = convertValueNum(constval, ct, isNull, rf); + value = convertValueNum(constval, ct, isNull, rf, jobInfo.timeZone); if (ct.colDataType == CalpontSystemCatalog::FLOAT && !isNull) { @@ -1862,7 +1870,7 @@ const JobStepVector doSimpleFilter(SimpleFilter* sf, JobInfo& jobInfo) #else bool isNull = ConstantColumn::NULLDATA == cc->type(); - value = convertValueNum(constval, ct, isNull, rf); + value = convertValueNum(constval, ct, isNull, rf, jobInfo.timeZone); if (ct.colDataType == CalpontSystemCatalog::FLOAT && !isNull) { @@ -2963,7 +2971,7 @@ const JobStepVector doConstantFilter(const ConstantFilter* cf, JobInfo& jobInfo) // @bug 1151 string longer than colwidth of char/varchar. uint8_t rf = 0; bool isNull = ConstantColumn::NULLDATA == cc->type(); - value = convertValueNum(constval, ct, isNull, rf); + value = convertValueNum(constval, ct, isNull, rf, jobInfo.timeZone); if (ct.colDataType == CalpontSystemCatalog::FLOAT && !isNull) { diff --git a/dbcon/joblist/jlf_subquery.cpp b/dbcon/joblist/jlf_subquery.cpp index 56dba64d7..a7f99d575 100644 --- a/dbcon/joblist/jlf_subquery.cpp +++ b/dbcon/joblist/jlf_subquery.cpp @@ -67,7 +67,7 @@ using namespace joblist; namespace { -void getColumnValue(ConstantColumn** cc, uint64_t i, const Row& row) +void getColumnValue(ConstantColumn** cc, uint64_t i, const Row& row, const string& timeZone) { ostringstream oss; int64_t data = 0; @@ -141,6 +141,11 @@ void getColumnValue(ConstantColumn** cc, uint64_t i, const Row& row) *cc = new ConstantColumn(oss.str()); break; + case CalpontSystemCatalog::TIMESTAMP: + oss << dataconvert::DataConvert::timestampToString(row.getUintField<8>(i), timeZone); + *cc = new ConstantColumn(oss.str()); + break; + case CalpontSystemCatalog::TIME: oss << dataconvert::DataConvert::timeToString(row.getUintField<8>(i)); *cc = new ConstantColumn(oss.str()); @@ -502,7 +507,7 @@ const SRCP doSelectSubquery(CalpontExecutionPlan* ep, SRCP& sc, JobInfo& jobInfo const Row& row = simpleTransformer.resultRow(); if (!row.isNullValue(0)) - getColumnValue(&cc, 0, row); + getColumnValue(&cc, 0, row, jobInfo.timeZone); } // Empty set or null value @@ -577,7 +582,7 @@ bool simpleScalarFilterToParseTree(SimpleScalarFilter* sf, ParseTree*& pt, JobIn // set fResult for cc ConstantColumn* cc = NULL; - getColumnValue(&cc, i, row); + getColumnValue(&cc, i, row, jobInfo.timeZone); sop->setOpType(cols[i]->resultType(), cc->resultType()); SimpleFilter* sf = new SimpleFilter(sop, cols[i]->clone(), cc); diff --git a/dbcon/joblist/joblistfactory.cpp b/dbcon/joblist/joblistfactory.cpp index 30e683988..9f2ca5e3e 100644 --- a/dbcon/joblist/joblistfactory.cpp +++ b/dbcon/joblist/joblistfactory.cpp @@ -1858,6 +1858,7 @@ SJLP makeJobList_( jobInfo.stringTableThreshold = csep->stringTableThreshold(); jobInfo.localQuery = csep->localQuery(); jobInfo.uuid = csep->uuid(); + jobInfo.timeZone = csep->timeZone(); /* disk-based join vars */ jobInfo.smallSideLimit = csep->djsSmallSideLimit(); diff --git a/dbcon/joblist/joblisttypes.h b/dbcon/joblist/joblisttypes.h index 6fd099330..c40c949c1 100644 --- a/dbcon/joblist/joblisttypes.h +++ b/dbcon/joblist/joblisttypes.h @@ -61,6 +61,8 @@ const uint64_t DATETIMENULL = 0xFFFFFFFFFFFFFFFEULL; const uint64_t DATETIMEEMPTYROW = 0xFFFFFFFFFFFFFFFFULL; const uint64_t TIMENULL = 0xFFFFFFFFFFFFFFFEULL; const uint64_t TIMEEMPTYROW = 0xFFFFFFFFFFFFFFFFULL; +const uint64_t TIMESTAMPNULL = 0xFFFFFFFFFFFFFFFEULL; +const uint64_t TIMESTAMPEMPTYROW = 0xFFFFFFFFFFFFFFFFULL; const uint8_t CHAR1NULL = 0xFE; const uint8_t CHAR1EMPTYROW = 0xFF; diff --git a/dbcon/joblist/jobstep.cpp b/dbcon/joblist/jobstep.cpp index 8e90bd2a6..3816476d1 100644 --- a/dbcon/joblist/jobstep.cpp +++ b/dbcon/joblist/jobstep.cpp @@ -88,7 +88,8 @@ JobStep::JobStep(const JobInfo& j) : fLocalQuery(j.localQuery), fQueryUuid(j.uuid), fProgress(0), - fStartTime(-1) + fStartTime(-1), + fTimeZone(j.timeZone) { QueryTeleServerParms tsp; string teleServerHost(Config::makeConfig()->getConfig("QueryTele", "Host")); diff --git a/dbcon/joblist/jobstep.h b/dbcon/joblist/jobstep.h index d4f5143f4..7a13b6757 100644 --- a/dbcon/joblist/jobstep.h +++ b/dbcon/joblist/jobstep.h @@ -421,6 +421,15 @@ public: fOnClauseFilter = b; } + void timeZone(const std::string& timezone) + { + fTimeZone = timezone; + } + const std::string timeZone() const + { + return fTimeZone; + } + static threadpool::ThreadPool jobstepThreadPool; protected: @@ -488,6 +497,7 @@ protected: uint64_t fProgress; int64_t fStartTime; int64_t fLastStepTeleTime; + std::string fTimeZone; private: static boost::mutex fLogMutex; diff --git a/dbcon/joblist/lbidlist.cpp b/dbcon/joblist/lbidlist.cpp index 7852562ef..9ef9c8a11 100644 --- a/dbcon/joblist/lbidlist.cpp +++ b/dbcon/joblist/lbidlist.cpp @@ -511,6 +511,7 @@ bool LBIDList::CasualPartitionDataType(const CalpontSystemCatalog::ColDataType t case CalpontSystemCatalog::DATE: case CalpontSystemCatalog::DATETIME: case CalpontSystemCatalog::TIME: + case CalpontSystemCatalog::TIMESTAMP: case CalpontSystemCatalog::UTINYINT: case CalpontSystemCatalog::USMALLINT: case CalpontSystemCatalog::UDECIMAL: diff --git a/dbcon/joblist/pcolscan.cpp b/dbcon/joblist/pcolscan.cpp index 6cea4fc03..da35ad8ce 100644 --- a/dbcon/joblist/pcolscan.cpp +++ b/dbcon/joblist/pcolscan.cpp @@ -1168,6 +1168,7 @@ bool pColScanStep::isEmptyVal(const uint8_t* val8) const case CalpontSystemCatalog::DATE: case CalpontSystemCatalog::DATETIME: case CalpontSystemCatalog::TIME: + case CalpontSystemCatalog::TIMESTAMP: if (width == 1) { return (*val8 == joblist::CHAR1EMPTYROW); diff --git a/dbcon/joblist/rowestimator.cpp b/dbcon/joblist/rowestimator.cpp index e69c66d79..4e5e97274 100644 --- a/dbcon/joblist/rowestimator.cpp +++ b/dbcon/joblist/rowestimator.cpp @@ -119,6 +119,12 @@ uint64_t RowEstimator::adjustValue(const execplan::CalpontSystemCatalog::ColType dtm.hour * 3600 + dtm.minute * 60 + dtm.second; } + case CalpontSystemCatalog::TIMESTAMP: + { + dataconvert::TimeStamp ts(value); + return ts.second; + } + // Use the first character only for estimating chars and varchar ranges. // TODO: Use dictionary column HWM for dictionary columns. case CalpontSystemCatalog::CHAR: @@ -179,6 +185,7 @@ uint32_t RowEstimator::estimateDistinctValues(const execplan::CalpontSystemCatal // Use 1000 for dates. case CalpontSystemCatalog::DATE: case CalpontSystemCatalog::DATETIME: + case CalpontSystemCatalog::TIMESTAMP: return 1000; // Use 10 for CHARs and VARCHARs. We'll use 10 for whatever else. diff --git a/dbcon/joblist/subquerytransformer.cpp b/dbcon/joblist/subquerytransformer.cpp index 753438387..1118e92e8 100644 --- a/dbcon/joblist/subquerytransformer.cpp +++ b/dbcon/joblist/subquerytransformer.cpp @@ -245,9 +245,10 @@ SJSTEP& SubQueryTransformer::makeSubQueryStep(execplan::CalpontSelectExecutionPl fVtable.columnType(ct, i); } } - // MySQL time/date/datetime type is different from IDB type + // MySQL timestamp/time/date/datetime type is different from IDB type else if (colDataTypeInRg == CalpontSystemCatalog::DATE || colDataTypeInRg == CalpontSystemCatalog::DATETIME || + colDataTypeInRg == CalpontSystemCatalog::TIMESTAMP || colDataTypeInRg == CalpontSystemCatalog::TIME) { ct.colWidth = row.getColumnWidth(i); diff --git a/dbcon/joblist/tupleaggregatestep.cpp b/dbcon/joblist/tupleaggregatestep.cpp index b4aec4013..990a694b1 100644 --- a/dbcon/joblist/tupleaggregatestep.cpp +++ b/dbcon/joblist/tupleaggregatestep.cpp @@ -273,6 +273,9 @@ inline string colTypeIdString(CalpontSystemCatalog::ColDataType type) case CalpontSystemCatalog::DATETIME: return string("DATETIME"); + case CalpontSystemCatalog::TIMESTAMP: + return string("TIMESTAMP"); + case CalpontSystemCatalog::TIME: return string("TIME"); @@ -1390,6 +1393,7 @@ void TupleAggregateStep::prep1PhaseAggregate( typeProj[colProj] == CalpontSystemCatalog::TEXT || typeProj[colProj] == CalpontSystemCatalog::DATE || typeProj[colProj] == CalpontSystemCatalog::DATETIME || + typeProj[colProj] == CalpontSystemCatalog::TIMESTAMP || typeProj[colProj] == CalpontSystemCatalog::TIME) { Message::Args args; @@ -1431,6 +1435,7 @@ void TupleAggregateStep::prep1PhaseAggregate( typeProj[colProj] == CalpontSystemCatalog::BLOB || typeProj[colProj] == CalpontSystemCatalog::DATE || typeProj[colProj] == CalpontSystemCatalog::DATETIME || + typeProj[colProj] == CalpontSystemCatalog::TIMESTAMP || typeProj[colProj] == CalpontSystemCatalog::TIME) { Message::Args args; @@ -1626,6 +1631,7 @@ void TupleAggregateStep::prep1PhaseAggregate( RowGroup aggRG(oidsAgg.size(), posAgg, oidsAgg, keysAgg, typeAgg, scaleAgg, precisionAgg, jobInfo.stringTableThreshold); SP_ROWAGG_UM_t rowAgg(new RowAggregationUM(groupBy, functionVec, jobInfo.rm, jobInfo.umMemLimit)); + rowAgg->timeZone(jobInfo.timeZone); rowgroups.push_back(aggRG); aggregators.push_back(rowAgg); @@ -1919,6 +1925,7 @@ void TupleAggregateStep::prep1PhaseDistinctAggregate( typeProj[colProj] == CalpontSystemCatalog::TEXT || typeProj[colProj] == CalpontSystemCatalog::DATE || typeProj[colProj] == CalpontSystemCatalog::DATETIME || + typeProj[colProj] == CalpontSystemCatalog::TIMESTAMP || typeProj[colProj] == CalpontSystemCatalog::TIME) { Message::Args args; @@ -1977,6 +1984,7 @@ void TupleAggregateStep::prep1PhaseDistinctAggregate( typeProj[colProj] == CalpontSystemCatalog::TEXT || typeProj[colProj] == CalpontSystemCatalog::DATE || typeProj[colProj] == CalpontSystemCatalog::DATETIME || + typeProj[colProj] == CalpontSystemCatalog::TIMESTAMP || typeProj[colProj] == CalpontSystemCatalog::TIME) { Message::Args args; @@ -2239,6 +2247,7 @@ void TupleAggregateStep::prep1PhaseDistinctAggregate( typeAgg[colAgg] == CalpontSystemCatalog::TEXT || typeAgg[colAgg] == CalpontSystemCatalog::DATE || typeAgg[colAgg] == CalpontSystemCatalog::DATETIME || + typeAgg[colAgg] == CalpontSystemCatalog::TIMESTAMP || typeAgg[colAgg] == CalpontSystemCatalog::TIME) { Message::Args args; @@ -2650,6 +2659,7 @@ void TupleAggregateStep::prep1PhaseDistinctAggregate( RowGroup aggRG(oidsAgg.size(), posAgg, oidsAgg, keysAgg, typeAgg, scaleAgg, precisionAgg, jobInfo.stringTableThreshold); SP_ROWAGG_UM_t rowAgg(new RowAggregationUM(groupBy, functionVec1, jobInfo.rm, jobInfo.umMemLimit)); + rowAgg->timeZone(jobInfo.timeZone); posAggDist.push_back(2); // rid @@ -2659,6 +2669,7 @@ void TupleAggregateStep::prep1PhaseDistinctAggregate( RowGroup aggRgDist(oidsAggDist.size(), posAggDist, oidsAggDist, keysAggDist, typeAggDist, scaleAggDist, precisionAggDist, jobInfo.stringTableThreshold); SP_ROWAGG_DIST rowAggDist(new RowAggregationDistinct(groupByNoDist, functionVec2, jobInfo.rm, jobInfo.umMemLimit)); + rowAggDist->timeZone(jobInfo.timeZone); // mapping the group_concat columns, if any. if (jobInfo.groupConcatInfo.groupConcat().size() > 0) @@ -2675,6 +2686,7 @@ void TupleAggregateStep::prep1PhaseDistinctAggregate( { RowAggregationMultiDistinct* multiDistinctAggregator = new RowAggregationMultiDistinct(groupByNoDist, functionVec2, jobInfo.rm, jobInfo.umMemLimit); + multiDistinctAggregator->timeZone(jobInfo.timeZone); rowAggDist.reset(multiDistinctAggregator); rowAggDist->groupConcat(jobInfo.groupConcatInfo.groupConcat()); @@ -2804,6 +2816,7 @@ void TupleAggregateStep::prep1PhaseDistinctAggregate( // construct sub-aggregator SP_ROWAGG_UM_t subAgg( new RowAggregationSubDistinct(groupBySub, functionSub1, jobInfo.rm, jobInfo.umMemLimit)); + subAgg->timeZone(jobInfo.timeZone); subAgg->groupConcat(jobInfo.groupConcatInfo.groupConcat()); // add to rowAggDist @@ -2879,6 +2892,7 @@ void TupleAggregateStep::prep1PhaseDistinctAggregate( // construct sub-aggregator SP_ROWAGG_UM_t subAgg( new RowAggregationUM(groupBySubNoDist, functionSub1, jobInfo.rm, jobInfo.umMemLimit)); + subAgg->timeZone(jobInfo.timeZone); subAgg->groupConcat(jobInfo.groupConcatInfo.groupConcat()); // add to rowAggDist @@ -3159,6 +3173,7 @@ void TupleAggregateStep::prep2PhasesAggregate( typeProj[colProj] == CalpontSystemCatalog::TEXT || typeProj[colProj] == CalpontSystemCatalog::DATE || typeProj[colProj] == CalpontSystemCatalog::DATETIME || + typeProj[colProj] == CalpontSystemCatalog::TIMESTAMP || typeProj[colProj] == CalpontSystemCatalog::TIME) { Message::Args args; @@ -3206,6 +3221,7 @@ void TupleAggregateStep::prep2PhasesAggregate( typeProj[colProj] == CalpontSystemCatalog::TEXT || typeProj[colProj] == CalpontSystemCatalog::DATE || typeProj[colProj] == CalpontSystemCatalog::DATETIME || + typeProj[colProj] == CalpontSystemCatalog::TIMESTAMP || typeProj[colProj] == CalpontSystemCatalog::TIME) { Message::Args args; @@ -3695,6 +3711,7 @@ void TupleAggregateStep::prep2PhasesAggregate( RowGroup aggRgUm(oidsAggUm.size(), posAggUm, oidsAggUm, keysAggUm, typeAggUm, scaleAggUm, precisionAggUm, jobInfo.stringTableThreshold); SP_ROWAGG_UM_t rowAggUm(new RowAggregationUMP2(groupByUm, functionVecUm, jobInfo.rm, jobInfo.umMemLimit)); + rowAggUm->timeZone(jobInfo.timeZone); rowgroups.push_back(aggRgUm); aggregators.push_back(rowAggUm); @@ -3706,6 +3723,7 @@ void TupleAggregateStep::prep2PhasesAggregate( RowGroup aggRgPm(oidsAggPm.size(), posAggPm, oidsAggPm, keysAggPm, typeAggPm, scaleAggPm, precisionAggPm, jobInfo.stringTableThreshold); SP_ROWAGG_PM_t rowAggPm(new RowAggregation(groupByPm, functionVecPm)); + rowAggPm->timeZone(jobInfo.timeZone); rowgroups.push_back(aggRgPm); aggregators.push_back(rowAggPm); @@ -4011,6 +4029,7 @@ void TupleAggregateStep::prep2PhasesDistinctAggregate( typeProj[colProj] == CalpontSystemCatalog::TEXT || typeProj[colProj] == CalpontSystemCatalog::DATE || typeProj[colProj] == CalpontSystemCatalog::DATETIME || + typeProj[colProj] == CalpontSystemCatalog::TIMESTAMP || typeProj[colProj] == CalpontSystemCatalog::TIME) { Message::Args args; @@ -4069,6 +4088,7 @@ void TupleAggregateStep::prep2PhasesDistinctAggregate( typeProj[colProj] == CalpontSystemCatalog::TEXT || typeProj[colProj] == CalpontSystemCatalog::DATE || typeProj[colProj] == CalpontSystemCatalog::DATETIME || + typeProj[colProj] == CalpontSystemCatalog::TIMESTAMP || typeProj[colProj] == CalpontSystemCatalog::TIME) { Message::Args args; @@ -4378,6 +4398,7 @@ void TupleAggregateStep::prep2PhasesDistinctAggregate( typeAggUm[colUm] == CalpontSystemCatalog::TEXT || typeAggUm[colUm] == CalpontSystemCatalog::DATE || typeAggUm[colUm] == CalpontSystemCatalog::DATETIME || + typeAggUm[colUm] == CalpontSystemCatalog::TIMESTAMP || typeAggUm[colUm] == CalpontSystemCatalog::TIME) { Message::Args args; @@ -4733,6 +4754,7 @@ void TupleAggregateStep::prep2PhasesDistinctAggregate( RowGroup aggRgUm(oidsAggUm.size(), posAggUm, oidsAggUm, keysAggUm, typeAggUm, scaleAggUm, precisionAggUm, jobInfo.stringTableThreshold); SP_ROWAGG_UM_t rowAggUm(new RowAggregationUMP2(groupByUm, functionNoDistVec, jobInfo.rm, jobInfo.umMemLimit)); + rowAggUm->timeZone(jobInfo.timeZone); posAggDist.push_back(2); // rid @@ -4742,6 +4764,7 @@ void TupleAggregateStep::prep2PhasesDistinctAggregate( RowGroup aggRgDist(oidsAggDist.size(), posAggDist, oidsAggDist, keysAggDist, typeAggDist, scaleAggDist, precisionAggDist, jobInfo.stringTableThreshold); SP_ROWAGG_DIST rowAggDist(new RowAggregationDistinct(groupByNoDist, functionVecUm, jobInfo.rm, jobInfo.umMemLimit)); + rowAggDist->timeZone(jobInfo.timeZone); // if distinct key word applied to more than one aggregate column, reset rowAggDist vector subRgVec; @@ -4750,6 +4773,7 @@ void TupleAggregateStep::prep2PhasesDistinctAggregate( { RowAggregationMultiDistinct* multiDistinctAggregator = new RowAggregationMultiDistinct(groupByNoDist, functionVecUm, jobInfo.rm, jobInfo.umMemLimit); + multiDistinctAggregator->timeZone(jobInfo.timeZone); rowAggDist.reset(multiDistinctAggregator); // construct and add sub-aggregators to rowAggDist @@ -4878,6 +4902,7 @@ void TupleAggregateStep::prep2PhasesDistinctAggregate( // construct sub-aggregator SP_ROWAGG_UM_t subAgg(new RowAggregationSubDistinct(groupBySub, functionSub1, jobInfo.rm, jobInfo.umMemLimit)); + subAgg->timeZone(jobInfo.timeZone); // add to rowAggDist multiDistinctAggregator->addSubAggregator(subAgg, subRg, functionSub2); @@ -4953,6 +4978,7 @@ void TupleAggregateStep::prep2PhasesDistinctAggregate( // construct sub-aggregator SP_ROWAGG_UM_t subAgg( new RowAggregationUMP2(groupBySubNoDist, functionSub1, jobInfo.rm, jobInfo.umMemLimit)); + subAgg->timeZone(jobInfo.timeZone); // add to rowAggDist multiDistinctAggregator->addSubAggregator(subAgg, aggRgUm, functionSub2); @@ -4973,6 +4999,7 @@ void TupleAggregateStep::prep2PhasesDistinctAggregate( RowGroup aggRgPm(oidsAggPm.size(), posAggPm, oidsAggPm, keysAggPm, typeAggPm, scaleAggPm, precisionAggPm, jobInfo.stringTableThreshold); SP_ROWAGG_PM_t rowAggPm(new RowAggregation(groupByPm, functionVecPm)); + rowAggPm->timeZone(jobInfo.timeZone); rowgroups.push_back(aggRgPm); aggregators.push_back(rowAggPm); diff --git a/dbcon/joblist/tupleconstantstep.cpp b/dbcon/joblist/tupleconstantstep.cpp index d03996586..02f61eb9c 100644 --- a/dbcon/joblist/tupleconstantstep.cpp +++ b/dbcon/joblist/tupleconstantstep.cpp @@ -225,6 +225,7 @@ void TupleConstantStep::constructContanstRow(const JobInfo& jobInfo) case CalpontSystemCatalog::DATE: case CalpontSystemCatalog::DATETIME: case CalpontSystemCatalog::TIME: + case CalpontSystemCatalog::TIMESTAMP: { fRowConst.setIntField(c.intVal, *i); break; diff --git a/dbcon/joblist/tupleunion.cpp b/dbcon/joblist/tupleunion.cpp index 0ef83c33b..666b0d514 100644 --- a/dbcon/joblist/tupleunion.cpp +++ b/dbcon/joblist/tupleunion.cpp @@ -101,7 +101,8 @@ TupleUnion::TupleUnion(CalpontSystemCatalog::OID tableOID, const JobInfo& jobInf fRowsReturned(0), runRan(false), joinRan(false), - sessionMemLimit(jobInfo.umMemLimit) + sessionMemLimit(jobInfo.umMemLimit), + fTimeZone(jobInfo.timeZone) { uniquer.reset(new Uniquer_t(10, Hasher(this), Eq(this), allocator)); fExtendedInfo = "TUN: "; @@ -475,7 +476,8 @@ void TupleUnion::normalize(const Row& in, Row* out) case CalpontSystemCatalog::DATE: case CalpontSystemCatalog::DATETIME: case CalpontSystemCatalog::TIME: - throw logic_error("TupleUnion::normalize(): tried to normalize an int to a time, date or datetime"); + case CalpontSystemCatalog::TIMESTAMP: + throw logic_error("TupleUnion::normalize(): tried to normalize an int to a timestamp, time, date or datetime"); case CalpontSystemCatalog::FLOAT: case CalpontSystemCatalog::UFLOAT: @@ -597,7 +599,8 @@ dec1: case CalpontSystemCatalog::DATE: case CalpontSystemCatalog::DATETIME: case CalpontSystemCatalog::TIME: - throw logic_error("TupleUnion::normalize(): tried to normalize an int to a time, date or datetime"); + case CalpontSystemCatalog::TIMESTAMP: + throw logic_error("TupleUnion::normalize(): tried to normalize an int to a timestamp, time, date or datetime"); case CalpontSystemCatalog::FLOAT: case CalpontSystemCatalog::UFLOAT: @@ -712,6 +715,37 @@ dec2: break; } + case CalpontSystemCatalog::TIMESTAMP: + { + dataconvert::Date date(in.getUintField(i)); + dataconvert::MySQLTime m_time; + m_time.year = date.year; + m_time.month = date.month; + m_time.day = date.day; + m_time.hour = 0; + m_time.minute = 0; + m_time.second = 0; + m_time.second_part = 0; + + dataconvert::TimeStamp timeStamp; + bool isValid = true; + int64_t seconds = dataconvert::mySQLTimeToGmtSec(m_time, fTimeZone, isValid); + + if (!isValid) + { + timeStamp.reset(); + } + else + { + timeStamp.second = seconds; + timeStamp.msecond = m_time.second_part; + } + + uint64_t outValue = (uint64_t) *(reinterpret_cast(&timeStamp)); + out->setUintField(outValue, i); + break; + } + case CalpontSystemCatalog::CHAR: case CalpontSystemCatalog::TEXT: case CalpontSystemCatalog::VARCHAR: @@ -747,6 +781,39 @@ dec2: break; } + case CalpontSystemCatalog::TIMESTAMP: + { + uint64_t val = in.getUintField(i); + dataconvert::DateTime dtime(val); + dataconvert::MySQLTime m_time; + dataconvert::TimeStamp timeStamp; + + m_time.year = dtime.year; + m_time.month = dtime.month; + m_time.day = dtime.day; + m_time.hour = dtime.hour; + m_time.minute = dtime.minute; + m_time.second = dtime.second; + m_time.second_part = dtime.msecond; + + bool isValid = true; + int64_t seconds = mySQLTimeToGmtSec(m_time, fTimeZone, isValid); + + if (!isValid) + { + timeStamp.reset(); + } + else + { + timeStamp.second = seconds; + timeStamp.msecond = m_time.second_part; + } + + uint64_t outValue = (uint64_t) *(reinterpret_cast(&timeStamp)); + out->setUintField(outValue, i); + break; + } + case CalpontSystemCatalog::CHAR: case CalpontSystemCatalog::TEXT: case CalpontSystemCatalog::VARCHAR: @@ -767,6 +834,70 @@ dec2: break; + case CalpontSystemCatalog::TIMESTAMP: + switch (out->getColTypes()[i]) + { + case CalpontSystemCatalog::TIMESTAMP: + out->setIntField(in.getIntField(i), i); + break; + + case CalpontSystemCatalog::DATE: + case CalpontSystemCatalog::DATETIME: + { + uint64_t val = in.getUintField(i); + dataconvert::TimeStamp timestamp(val); + int64_t seconds = timestamp.second; + uint64_t outValue; + + dataconvert::MySQLTime time; + dataconvert::gmtSecToMySQLTime(seconds, time, fTimeZone); + + if (out->getColTypes()[i] == CalpontSystemCatalog::DATE) + { + dataconvert::Date date; + date.year = time.year; + date.month = time.month; + date.day = time.day; + date.spare = 0; + outValue = (uint32_t) *(reinterpret_cast(&date)); + } + else + { + dataconvert::DateTime datetime; + datetime.year = time.year; + datetime.month = time.month; + datetime.day = time.day; + datetime.hour = time.hour; + datetime.minute = time.minute; + datetime.second = time.second; + datetime.msecond = timestamp.msecond; + outValue = (uint64_t) *(reinterpret_cast(&datetime)); + } + + out->setUintField(outValue, i); + break; + } + + case CalpontSystemCatalog::CHAR: + case CalpontSystemCatalog::TEXT: + case CalpontSystemCatalog::VARCHAR: + { + string d = DataConvert::timestampToString(in.getUintField(i), fTimeZone); + out->setStringField(d, i); + break; + } + + default: + { + ostringstream os; + os << "TupleUnion::normalize(): tried an illegal conversion: timestamp to " + << out->getColTypes()[i]; + throw logic_error(os.str()); + } + } + + break; + case CalpontSystemCatalog::TIME: switch (out->getColTypes()[i]) { @@ -1217,6 +1348,10 @@ void TupleUnion::writeNull(Row* out, uint32_t col) out->setUintField<8>(joblist::DATETIMENULL, col); break; + case CalpontSystemCatalog::TIMESTAMP: + out->setUintField<8>(joblist::TIMESTAMPNULL, col); + break; + case CalpontSystemCatalog::TIME: out->setUintField<8>(joblist::TIMENULL, col); break; diff --git a/dbcon/joblist/tupleunion.h b/dbcon/joblist/tupleunion.h index b6a445da9..c3c226672 100644 --- a/dbcon/joblist/tupleunion.h +++ b/dbcon/joblist/tupleunion.h @@ -201,6 +201,7 @@ private: bool runRan, joinRan; boost::shared_ptr sessionMemLimit; + std::string fTimeZone; }; } diff --git a/dbcon/joblist/windowfunctionstep.cpp b/dbcon/joblist/windowfunctionstep.cpp index fe43faa77..610026e33 100644 --- a/dbcon/joblist/windowfunctionstep.cpp +++ b/dbcon/joblist/windowfunctionstep.cpp @@ -1193,6 +1193,7 @@ boost::shared_ptr WindowFunctionStep::parseFrameBoundRows( case execplan::CalpontSystemCatalog::DATE: case execplan::CalpontSystemCatalog::DATETIME: case execplan::CalpontSystemCatalog::TIME: + case execplan::CalpontSystemCatalog::TIMESTAMP: { fb.reset(new FrameBoundExpressionRow(type, id, idx)); break; @@ -1344,6 +1345,7 @@ boost::shared_ptr WindowFunctionStep::parseFrameBoundRange(const exe case execplan::CalpontSystemCatalog::DATE: case execplan::CalpontSystemCatalog::DATETIME: case execplan::CalpontSystemCatalog::TIME: + case execplan::CalpontSystemCatalog::TIMESTAMP: { if (isConstant) { diff --git a/dbcon/mysql/ha_calpont_ddl.cpp b/dbcon/mysql/ha_calpont_ddl.cpp index 191ff610a..833a0ab38 100644 --- a/dbcon/mysql/ha_calpont_ddl.cpp +++ b/dbcon/mysql/ha_calpont_ddl.cpp @@ -178,6 +178,10 @@ uint32_t convertDataType(int dataType) calpontDataType = CalpontSystemCatalog::TIME; break; + case ddlpackage::DDL_TIMESTAMP: + calpontDataType = CalpontSystemCatalog::TIMESTAMP; + break; + case ddlpackage::DDL_CLOB: calpontDataType = CalpontSystemCatalog::CLOB; break; @@ -524,7 +528,177 @@ bool anyRowInTable(string& schema, string& tableName, int sessionID) } } -bool anyNullInTheColumn (string& schema, string& table, string& columnName, int sessionID) +bool anyTimestampColumn(string& schema, string& tableName, int sessionID) +{ + boost::shared_ptr csc = CalpontSystemCatalog::makeCalpontSystemCatalog(sessionID); + csc->identity(execplan::CalpontSystemCatalog::FE); + CalpontSystemCatalog::TableName aTableName; + algorithm::to_lower(schema); + algorithm::to_lower(tableName); + + // select columnname from calpontsys.syscolumn + // where schema = schema and tablename = tableName + // and datatype = 'timestamp' + CalpontSelectExecutionPlan csep; + CalpontSelectExecutionPlan::ReturnedColumnList returnedColumnList; + CalpontSelectExecutionPlan::FilterTokenList filterTokenList; + CalpontSelectExecutionPlan::ColumnMap colMap; + + SessionManager sm; + BRM::TxnID txnID; + txnID = sm.getTxnID(sessionID); + + if (!txnID.valid) + { + txnID.id = 0; + txnID.valid = true; + } + + QueryContext verID; + verID = sm.verID(); + csep.txnID(txnID.id); + csep.verID(verID); + csep.sessionID(sessionID); + + string sysTable = "calpontsys.syscolumn."; + string firstCol = sysTable + "columnname"; + SimpleColumn* c1 = new SimpleColumn(firstCol, sessionID); + string secondCol = sysTable + "schema"; + SimpleColumn* c2 = new SimpleColumn(secondCol, sessionID); + string thirdCol = sysTable + "tablename"; + SimpleColumn* c3 = new SimpleColumn(thirdCol, sessionID); + string fourthCol = sysTable + "datatype"; + SimpleColumn* c4 = new SimpleColumn(fourthCol, sessionID); + SRCP srcp; + srcp.reset(c1); + colMap.insert(CMVT_(firstCol, srcp)); + srcp.reset(c2); + colMap.insert(CMVT_(secondCol, srcp)); + srcp.reset(c3); + colMap.insert(CMVT_(thirdCol, srcp)); + srcp.reset(c4); + colMap.insert(CMVT_(fourthCol, srcp)); + csep.columnMapNonStatic(colMap); + srcp.reset(c1->clone()); + returnedColumnList.push_back(srcp); + csep.returnedCols(returnedColumnList); + + // Filters + const SOP opeq(new Operator("=")); + SimpleFilter* f1 = new SimpleFilter (opeq, + c2->clone(), + new ConstantColumn(schema, ConstantColumn::LITERAL)); + filterTokenList.push_back(f1); + filterTokenList.push_back(new Operator("and")); + + SimpleFilter* f2 = new SimpleFilter (opeq, + c3->clone(), + new ConstantColumn(tableName, ConstantColumn::LITERAL)); + filterTokenList.push_back(f2); + filterTokenList.push_back(new Operator("and")); + + SimpleFilter* f3 = new SimpleFilter (opeq, + c4->clone(), + new ConstantColumn((uint64_t) execplan::CalpontSystemCatalog::TIMESTAMP, ConstantColumn::NUM)); + filterTokenList.push_back(f3); + csep.filterTokenList(filterTokenList); + + CalpontSelectExecutionPlan::TableList tablelist; + tablelist.push_back(make_aliastable("calpontsys", "syscolumn", "")); + csep.tableList(tablelist); + + boost::shared_ptr exemgrClient (new messageqcpp::MessageQueueClient("ExeMgr1")); + ByteStream msg, emsgBs; + rowgroup::RGData rgData; + ByteStream::quadbyte qb = 4; + msg << qb; + rowgroup::RowGroup* rowGroup = 0; + bool anyRow = false; + + exemgrClient->write(msg); + ByteStream msgPlan; + csep.serialize(msgPlan); + exemgrClient->write(msgPlan); + msg.restart(); + msg = exemgrClient->read(); //error handling + emsgBs = exemgrClient->read(); + ByteStream::quadbyte qb1; + + if (emsgBs.length() == 0) + { + //exemgrClient->shutdown(); + //delete exemgrClient; + //exemgrClient = 0; + throw runtime_error("Lost conection to ExeMgr."); + } + + string emsgStr; + emsgBs >> emsgStr; + + if (msg.length() == 4) + { + msg >> qb1; + + if (qb1 != 0) + { + //exemgrClient->shutdown(); + //delete exemgrClient; + //exemgrClient = 0; + throw runtime_error(emsgStr); + } + } + + while (true) + { + msg.restart(); + msg = exemgrClient->read(); + + if ( msg.length() == 0 ) + { + //exemgrClient->shutdown(); + //delete exemgrClient; + //exemgrClient = 0; + throw runtime_error("Lost conection to ExeMgr."); + } + else + { + if (!rowGroup) + { + //This is mete data + rowGroup = new rowgroup::RowGroup(); + rowGroup->deserialize(msg); + qb = 100; + msg.restart(); + msg << qb; + exemgrClient->write(msg); + continue; + } + + rgData.deserialize(msg); + rowGroup->setData(&rgData); + + if (rowGroup->getStatus() != 0) + { + //msg.advance(rowGroup->getDataSize()); + msg >> emsgStr; + //exemgrClient->shutdown(); + //delete exemgrClient; + //exemgrClient = 0; + throw runtime_error(emsgStr); + } + + if (rowGroup->getRowCount() > 0) + anyRow = true; + + //exemgrClient->shutdown(); + //delete exemgrClient; + //exemgrClient = 0; + return anyRow; + } + } +} + +bool anyNullInTheColumn (THD* thd, string& schema, string& table, string& columnName, int sessionID) { CalpontSelectExecutionPlan csep; CalpontSelectExecutionPlan::ReturnedColumnList returnedColumnList; @@ -561,9 +735,11 @@ bool anyNullInTheColumn (string& schema, string& table, string& columnName, int csep.returnedCols(returnedColumnList); SimpleFilter* sf = new SimpleFilter(); + sf->timeZone(thd->variables.time_zone->get_name()->ptr()); boost::shared_ptr sop(new PredicateOperator("isnull")); sf->op(sop); ConstantColumn* rhs = new ConstantColumn("", ConstantColumn::NULLDATA); + rhs->timeZone(thd->variables.time_zone->get_name()->ptr()); sf->lhs(col[0]->clone()); sf->rhs(rhs); @@ -725,6 +901,7 @@ int ProcessDDLStatement(string& ddlStatement, string& schema, const string& tabl } bool matchedCol = false; + bool isFirstTimestamp = true; for ( unsigned i = 0; i < createTable->fTableDef->fColumns.size(); i++ ) { @@ -775,6 +952,13 @@ int ProcessDDLStatement(string& ddlStatement, string& schema, const string& tabl return rc; } + // For TIMESTAMP, if no constraint is given, default to NOT NULL + if (createTable->fTableDef->fColumns[i]->fType->fType == ddlpackage::DDL_TIMESTAMP && + createTable->fTableDef->fColumns[i]->fConstraints.empty()) + { + createTable->fTableDef->fColumns[i]->fConstraints.push_back(new ColumnConstraintDef(DDL_NOT_NULL)); + } + if (createTable->fTableDef->fColumns[i]->fDefaultValue) { if ((!createTable->fTableDef->fColumns[i]->fDefaultValue->fNull) && (createTable->fTableDef->fColumns[i]->fType->fType == ddlpackage::DDL_VARBINARY)) @@ -803,7 +987,7 @@ int ProcessDDLStatement(string& ddlStatement, string& schema, const string& tabl try { - convertedVal = DataConvert::convertColumnData(colType, createTable->fTableDef->fColumns[i]->fDefaultValue->fValue, pushWarning, false, false ); + convertedVal = DataConvert::convertColumnData(colType, createTable->fTableDef->fColumns[i]->fDefaultValue->fValue, pushWarning, thd->variables.time_zone->get_name()->ptr(), false, false, false); } catch (std::exception&) { @@ -824,6 +1008,35 @@ int ProcessDDLStatement(string& ddlStatement, string& schema, const string& tabl ci->isAlter = false; return rc; } + if (createTable->fTableDef->fColumns[i]->fType->fType == ddlpackage::DDL_TIMESTAMP) + { + if (createTable->fTableDef->fColumns[i]->fDefaultValue->fValue == "0") + { + createTable->fTableDef->fColumns[i]->fDefaultValue->fValue = "0000-00-00 00:00:00"; + } + if (isFirstTimestamp) + { + isFirstTimestamp = false; + } + } + } + } + // If no default value exists for TIMESTAMP, we apply + // automatic TIMESTAMP properties. + // TODO: If no default value exists but the constraint is NULL, + // default value should be set to NULL. But this is currently + // not supported since columnstore does not track whether user + // specified a NULL or not + else if (createTable->fTableDef->fColumns[i]->fType->fType == ddlpackage::DDL_TIMESTAMP) + { + if (isFirstTimestamp) + { + isFirstTimestamp = false; + createTable->fTableDef->fColumns[i]->fDefaultValue = new ColumnDefaultValue("current_timestamp() ON UPDATE current_timestamp()"); + } + else + { + createTable->fTableDef->fColumns[i]->fDefaultValue = new ColumnDefaultValue("0000-00-00 00:00:00"); } } @@ -990,6 +1203,8 @@ int ProcessDDLStatement(string& ddlStatement, string& schema, const string& tabl { AlterTableStatement* alterTable = dynamic_cast ( &stmt ); + alterTable->fTimeZone = thd->variables.time_zone->get_name()->ptr(); + if ( schema.length() == 0 ) { schema = alterTable->fTableName->fSchema; @@ -1065,7 +1280,7 @@ int ProcessDDLStatement(string& ddlStatement, string& schema, const string& tabl } //if not null constraint, user has to provide a default value - if ((addColumnPtr->fColumnDef->fConstraints[j]->fConstraintType == DDL_NOT_NULL) && (!addColumnPtr->fColumnDef->fDefaultValue)) + if ((addColumnPtr->fColumnDef->fConstraints[j]->fConstraintType == DDL_NOT_NULL) && (!addColumnPtr->fColumnDef->fDefaultValue) && (addColumnPtr->fColumnDef->fType->fType != ddlpackage::DDL_TIMESTAMP)) { //do select count(*) from the table to check whether there are existing rows. if there is, error out. @@ -1143,7 +1358,7 @@ int ProcessDDLStatement(string& ddlStatement, string& schema, const string& tabl try { - convertedVal = DataConvert::convertColumnData(colType, addColumnPtr->fColumnDef->fDefaultValue->fValue, pushWarning, false, false ); + convertedVal = DataConvert::convertColumnData(colType, addColumnPtr->fColumnDef->fDefaultValue->fValue, pushWarning, thd->variables.time_zone->get_name()->ptr(), false, false, false); } catch (std::exception&) { @@ -1164,6 +1379,39 @@ int ProcessDDLStatement(string& ddlStatement, string& schema, const string& tabl ci->isAlter = false; return rc; } + if (addColumnPtr->fColumnDef->fType->fType == ddlpackage::DDL_TIMESTAMP && + addColumnPtr->fColumnDef->fDefaultValue->fValue == "0") + { + addColumnPtr->fColumnDef->fDefaultValue->fValue = "0000-00-00 00:00:00"; + } + } + } + + // For TIMESTAMP, if no constraint is given, default to NOT NULL + if (addColumnPtr->fColumnDef->fType->fType == ddlpackage::DDL_TIMESTAMP && + addColumnPtr->fColumnDef->fConstraints.empty()) + { + addColumnPtr->fColumnDef->fConstraints.push_back(new ColumnConstraintDef(DDL_NOT_NULL)); + } + + // If no default value exists for TIMESTAMP, we apply + // automatic TIMESTAMP properties. + // TODO: If no default value exists but the constraint is NULL, + // default value should be set to NULL. But this is currently + // not supported since columnstore does not track whether user + // specified a NULL or not + if (addColumnPtr->fColumnDef->fType->fType == ddlpackage::DDL_TIMESTAMP && + !addColumnPtr->fColumnDef->fDefaultValue) + { + // Query calpontsys.syscolumn to see + // if a timestamp column already exists in this table + if (!anyTimestampColumn(alterTable->fTableName->fSchema, alterTable->fTableName->fName, sessionID)) + { + addColumnPtr->fColumnDef->fDefaultValue = new ColumnDefaultValue("current_timestamp() ON UPDATE current_timestamp()"); + } + else + { + addColumnPtr->fColumnDef->fDefaultValue = new ColumnDefaultValue("0000-00-00 00:00:00"); } } @@ -1386,7 +1634,7 @@ int ProcessDDLStatement(string& ddlStatement, string& schema, const string& tabl } //if not null constraint, user has to provide a default value - if ((addColumnsPtr->fColumns[0]->fConstraints[j]->fConstraintType == DDL_NOT_NULL) && (!addColumnsPtr->fColumns[0]->fDefaultValue)) + if ((addColumnsPtr->fColumns[0]->fConstraints[j]->fConstraintType == DDL_NOT_NULL) && (!addColumnsPtr->fColumns[0]->fDefaultValue) && (addColumnsPtr->fColumns[0]->fType->fType != ddlpackage::DDL_TIMESTAMP)) { //do select count(*) from the table to check whether there are existing rows. if there is, error out. @@ -1464,7 +1712,7 @@ int ProcessDDLStatement(string& ddlStatement, string& schema, const string& tabl try { - convertedVal = DataConvert::convertColumnData(colType, addColumnsPtr->fColumns[0]->fDefaultValue->fValue, pushWarning, false, false ); + convertedVal = DataConvert::convertColumnData(colType, addColumnsPtr->fColumns[0]->fDefaultValue->fValue, pushWarning, thd->variables.time_zone->get_name()->ptr(), false, false, false); } catch (std::exception&) { @@ -1485,6 +1733,39 @@ int ProcessDDLStatement(string& ddlStatement, string& schema, const string& tabl ci->isAlter = false; return rc; } + if (addColumnsPtr->fColumns[0]->fType->fType == ddlpackage::DDL_TIMESTAMP && + addColumnsPtr->fColumns[0]->fDefaultValue->fValue == "0") + { + addColumnsPtr->fColumns[0]->fDefaultValue->fValue = "0000-00-00 00:00:00"; + } + } + } + + // For TIMESTAMP, if no constraint is given, default to NOT NULL + if (addColumnsPtr->fColumns[0]->fType->fType == ddlpackage::DDL_TIMESTAMP && + addColumnsPtr->fColumns[0]->fConstraints.empty()) + { + addColumnsPtr->fColumns[0]->fConstraints.push_back(new ColumnConstraintDef(DDL_NOT_NULL)); + } + + // If no default value exists for TIMESTAMP, we apply + // automatic TIMESTAMP properties. + // TODO: If no default value exists but the constraint is NULL, + // default value should be set to NULL. But this is currently + // not supported since columnstore does not track whether user + // specified a NULL or not + if (addColumnsPtr->fColumns[0]->fType->fType == ddlpackage::DDL_TIMESTAMP && + !addColumnsPtr->fColumns[0]->fDefaultValue) + { + // Query calpontsys.syscolumn to see + // if a timestamp column already exists in this table + if (!anyTimestampColumn(alterTable->fTableName->fSchema, alterTable->fTableName->fName, sessionID)) + { + addColumnsPtr->fColumns[0]->fDefaultValue = new ColumnDefaultValue("current_timestamp() ON UPDATE current_timestamp()"); + } + else + { + addColumnsPtr->fColumns[0]->fDefaultValue = new ColumnDefaultValue("0000-00-00 00:00:00"); } } @@ -1734,7 +2015,7 @@ int ProcessDDLStatement(string& ddlStatement, string& schema, const string& tabl try { - anyNullVal = anyNullInTheColumn (alterTable->fTableName->fSchema, alterTable->fTableName->fName, renameColumnsPtr->fName, sessionID); + anyNullVal = anyNullInTheColumn (thd, alterTable->fTableName->fSchema, alterTable->fTableName->fName, renameColumnsPtr->fName, sessionID); } catch (runtime_error& ex) { diff --git a/dbcon/mysql/ha_calpont_dml.cpp b/dbcon/mysql/ha_calpont_dml.cpp index e5f82f62b..20462e2c8 100644 --- a/dbcon/mysql/ha_calpont_dml.cpp +++ b/dbcon/mysql/ha_calpont_dml.cpp @@ -88,6 +88,108 @@ inline uint32_t tid2sid(const uint32_t tid) } //StopWatch timer; +int buildBuffer(uchar* buf, string& buffer, int& columns, TABLE* table) +{ + char attribute_buffer[1024]; + String attribute(attribute_buffer, sizeof(attribute_buffer), + &my_charset_bin); + + std::string cols = " ("; + std::string vals = " values ("; + columns = 0; + + for (Field** field = table->field; *field; field++) + { + const char* ptr; + const char* end_ptr; + + if ((*field)->is_null()) + ptr = end_ptr = 0; + else + { + bitmap_set_bit(table->read_set, (*field)->field_index); + (*field)->val_str(&attribute, &attribute); + ptr = attribute.ptr(); + end_ptr = attribute.length() + ptr; + } + + if (columns > 0) + { + cols.append(","); + vals.append(","); + } + + columns++; + + cols.append((*field)->field_name.str); + + if (ptr == end_ptr) + { + vals.append ("NULL"); + } + else + { + + if ( (*field)->type() == MYSQL_TYPE_VARCHAR || + /*FIXME: (*field)->type() == MYSQL_TYPE_VARBINARY || */ + (*field)->type() == MYSQL_TYPE_VAR_STRING || + (*field)->type() == MYSQL_TYPE_STRING || + (*field)->type() == MYSQL_TYPE_DATE || + (*field)->type() == MYSQL_TYPE_DATETIME || + (*field)->type() == MYSQL_TYPE_DATETIME2 || + (*field)->type() == MYSQL_TYPE_TIMESTAMP || + (*field)->type() == MYSQL_TYPE_TIMESTAMP2 || + (*field)->type() == MYSQL_TYPE_TIME ) + vals.append("'"); + + while (ptr < end_ptr) + { + + if (*ptr == '\r') + { + ptr++; + } + else if (*ptr == '\n') + { + ptr++; + } + else if (*ptr == '\'' ) + { + //@Bug 1820. Replace apostrophe with strange character to pass parser. + vals += '\252'; + ptr++; + } + else + vals += *ptr++; + } + + if ( (*field)->type() == MYSQL_TYPE_VARCHAR || + /*FIXME: (*field)->type() == MYSQL_TYPE_VARBINARY || */ + (*field)->type() == MYSQL_TYPE_VAR_STRING || + (*field)->type() == MYSQL_TYPE_STRING || + (*field)->type() == MYSQL_TYPE_DATE || + (*field)->type() == MYSQL_TYPE_DATETIME || + (*field)->type() == MYSQL_TYPE_DATETIME2 || + (*field)->type() == MYSQL_TYPE_TIMESTAMP || + (*field)->type() == MYSQL_TYPE_TIMESTAMP2 || + (*field)->type() == MYSQL_TYPE_TIME ) + vals.append("'"); + } + } + + if (columns) + { + cols.append(") "); + vals.append(") "); + buffer = "INSERT INTO "; + buffer.append(table->s->table_name.str); + buffer.append(cols); + buffer.append(vals); + } + + return columns; +} + uint32_t buildValueList (TABLE* table, cal_connection_info& ci ) { @@ -312,6 +414,7 @@ int doProcessInsertValues ( TABLE* table, uint32_t size, cal_connection_info& ci pDMLPackage->set_TableName(name); name = table->s->db.str; pDMLPackage->set_SchemaName(name); + pDMLPackage->set_TimeZone(thd->variables.time_zone->get_name()->ptr()); if (thd->lex->sql_command == SQLCOM_INSERT_SELECT) pDMLPackage->set_isInsertSelect(true); @@ -821,6 +924,41 @@ int ha_calpont_impl_write_batch_row_(uchar* buf, TABLE* table, cal_impl_if::cal_ break; } + case CalpontSystemCatalog::TIMESTAMP: + { + if (nullVal && (ci.columnTypes[colpos].constraintType != CalpontSystemCatalog::NOTNULL_CONSTRAINT)) + { + fprintf(ci.filePtr, "%c", ci.delimiter); + } + else + { + const uchar* pos = buf; + struct timeval tm; + my_timestamp_from_binary(&tm, pos, table->field[colpos]->decimals()); + + MySQLTime time; + gmtSecToMySQLTime(tm.tv_sec, time, current_thd->variables.time_zone->get_name()->ptr()); + + if (!tm.tv_usec) + { + fprintf(ci.filePtr, "%04d-%02d-%02d %02d:%02d:%02d%c", + time.year, time.month, time.day, + time.hour, time.minute, time.second, ci.delimiter); + } + else + { + fprintf(ci.filePtr, "%04d-%02d-%02d %02d:%02d:%02d.%ld%c", + time.year, time.month, time.day, + time.hour, time.minute, time.second, + tm.tv_usec, ci.delimiter); + } + } + + buf += table->field[colpos]->pack_length(); + + break; + } + case CalpontSystemCatalog::CHAR: { if (nullVal && (ci.columnTypes[colpos].constraintType != CalpontSystemCatalog::NOTNULL_CONSTRAINT)) diff --git a/dbcon/mysql/ha_calpont_execplan.cpp b/dbcon/mysql/ha_calpont_execplan.cpp index cb31cc75a..d40584109 100644 --- a/dbcon/mysql/ha_calpont_execplan.cpp +++ b/dbcon/mysql/ha_calpont_execplan.cpp @@ -1237,7 +1237,7 @@ uint32_t buildOuterJoin(gp_walk_info& gwi, SELECT_LEX& select_lex) return 0; } -ParseTree* buildRowPredicate(RowColumn* lhs, RowColumn* rhs, string predicateOp) +ParseTree* buildRowPredicate(THD* thd, RowColumn* lhs, RowColumn* rhs, string predicateOp) { PredicateOperator* po = new PredicateOperator(predicateOp); boost::shared_ptr sop(po); @@ -1251,6 +1251,7 @@ ParseTree* buildRowPredicate(RowColumn* lhs, RowColumn* rhs, string predicateOp) ParseTree* pt = new ParseTree(lo); sop->setOpType(lhs->columnVec()[0]->resultType(), rhs->columnVec()[0]->resultType()); SimpleFilter* sf = new SimpleFilter(sop, lhs->columnVec()[0].get(), rhs->columnVec()[0].get()); + sf->timeZone(thd->variables.time_zone->get_name()->ptr()); pt->left(new ParseTree(sf)); for (uint32_t i = 1; i < lhs->columnVec().size(); i++) @@ -1258,6 +1259,7 @@ ParseTree* buildRowPredicate(RowColumn* lhs, RowColumn* rhs, string predicateOp) sop.reset(po->clone()); sop->setOpType(lhs->columnVec()[i]->resultType(), rhs->columnVec()[i]->resultType()); SimpleFilter* sf = new SimpleFilter(sop, lhs->columnVec()[i].get(), rhs->columnVec()[i].get()); + sf->timeZone(thd->variables.time_zone->get_name()->ptr()); pt->right(new ParseTree(sf)); if (i + 1 < lhs->columnVec().size()) @@ -1277,7 +1279,7 @@ bool buildRowColumnFilter(gp_walk_info* gwip, RowColumn* rhs, RowColumn* lhs, It { // (c1,c2,..) = (v1,v2,...) transform to: c1=v1 and c2=v2 and ... assert (!lhs->columnVec().empty() && lhs->columnVec().size() == rhs->columnVec().size()); - gwip->ptWorkStack.push(buildRowPredicate(rhs, lhs, ifp->func_name())); + gwip->ptWorkStack.push(buildRowPredicate(gwip->thd, rhs, lhs, ifp->func_name())); } else if (ifp->functype() == Item_func::IN_FUNC) { @@ -1321,7 +1323,7 @@ bool buildRowColumnFilter(gp_walk_info* gwip, RowColumn* rhs, RowColumn* lhs, It RowColumn* vals = dynamic_cast(tmpStack.top()); valVec.push_back(vals); tmpStack.pop(); - ParseTree* pt = buildRowPredicate(columns, vals, predicateOp); + ParseTree* pt = buildRowPredicate(gwip->thd, columns, vals, predicateOp); while (!tmpStack.empty()) { @@ -1330,7 +1332,7 @@ bool buildRowColumnFilter(gp_walk_info* gwip, RowColumn* rhs, RowColumn* lhs, It vals = dynamic_cast(tmpStack.top()); valVec.push_back(vals); tmpStack.pop(); - pt1->right(buildRowPredicate(columns->clone(), vals, predicateOp)); + pt1->right(buildRowPredicate(gwip->thd, columns->clone(), vals, predicateOp)); pt = pt1; } @@ -1373,7 +1375,8 @@ bool buildRowColumnFilter(gp_walk_info* gwip, RowColumn* rhs, RowColumn* lhs, It sop->setOpType(sc->resultType(), valVec[j]->resultType()); cf->pushFilter(new SimpleFilter(sop, sc->clone(), - valVec[j]->columnVec()[i]->clone())); + valVec[j]->columnVec()[i]->clone(), + gwip->thd->variables.time_zone->get_name()->ptr())); } if (j < valVec.size()) @@ -1468,9 +1471,11 @@ bool buildPredicateItem(Item_func* ifp, gp_walk_info* gwip) sop.reset(new PredicateOperator(">")); sop->setOpType(filterCol->resultType(), rhs->resultType()); sfr = new SimpleFilter(sop, filterCol, rhs); + sfr->timeZone(gwip->thd->variables.time_zone->get_name()->ptr()); sop.reset(new PredicateOperator("<")); sop->setOpType(filterCol->resultType(), lhs->resultType()); sfl = new SimpleFilter(sop, filterCol->clone(), lhs); + sfl->timeZone(gwip->thd->variables.time_zone->get_name()->ptr()); ParseTree* ptp = new ParseTree(new LogicOperator("or")); ptp->left(sfr); ptp->right(sfl); @@ -1481,9 +1486,11 @@ bool buildPredicateItem(Item_func* ifp, gp_walk_info* gwip) sop.reset(new PredicateOperator("<=")); sop->setOpType(filterCol->resultType(), rhs->resultType()); sfr = new SimpleFilter(sop, filterCol, rhs); + sfr->timeZone(gwip->thd->variables.time_zone->get_name()->ptr()); sop.reset(new PredicateOperator(">=")); sop->setOpType(filterCol->resultType(), lhs->resultType()); sfl = new SimpleFilter(sop, filterCol->clone(), lhs); + sfl->timeZone(gwip->thd->variables.time_zone->get_name()->ptr()); ParseTree* ptp = new ParseTree(new LogicOperator("and")); ptp->left(sfr); ptp->right(sfl); @@ -1543,7 +1550,7 @@ bool buildPredicateItem(Item_func* ifp, gp_walk_info* gwip) cf->op(sop); sop.reset(new PredicateOperator(eqop)); sop->setOpType(gwip->scsp->resultType(), lhs->resultType()); - cf->pushFilter(new SimpleFilter(sop, gwip->scsp->clone(), lhs)); + cf->pushFilter(new SimpleFilter(sop, gwip->scsp->clone(), lhs, gwip->thd->variables.time_zone->get_name()->ptr())); while (!gwip->rcWorkStack.empty()) { @@ -1554,7 +1561,7 @@ bool buildPredicateItem(Item_func* ifp, gp_walk_info* gwip) gwip->rcWorkStack.pop(); sop.reset(new PredicateOperator(eqop)); sop->setOpType(gwip->scsp->resultType(), lhs->resultType()); - cf->pushFilter(new SimpleFilter(sop, gwip->scsp->clone(), lhs)); + cf->pushFilter(new SimpleFilter(sop, gwip->scsp->clone(), lhs, gwip->thd->variables.time_zone->get_name()->ptr())); } if (!gwip->rcWorkStack.empty()) @@ -1621,6 +1628,7 @@ bool buildPredicateItem(Item_func* ifp, gp_walk_info* gwip) { gwip->rcWorkStack.push(new ConstantColumn((int64_t)udf->val_int())); } + (dynamic_cast(gwip->rcWorkStack.top()))->timeZone(gwip->thd->variables.time_zone->get_name()->ptr()); } else { @@ -1642,6 +1650,7 @@ bool buildPredicateItem(Item_func* ifp, gp_walk_info* gwip) { gwip->rcWorkStack.push(new ConstantColumn(buf.ptr(), ConstantColumn::NUM)); } + (dynamic_cast(gwip->rcWorkStack.top()))->timeZone(gwip->thd->variables.time_zone->get_name()->ptr()); return false; } @@ -1814,15 +1823,19 @@ bool buildPredicateItem(Item_func* ifp, gp_walk_info* gwip) SimpleFilter* sfo = 0; // b IS NULL ConstantColumn* nlhs1 = new ConstantColumn("", ConstantColumn::NULLDATA); + nlhs1->timeZone(gwip->thd->variables.time_zone->get_name()->ptr()); sop.reset(new PredicateOperator("isnull")); sop->setOpType(lhs->resultType(), rhs->resultType()); sfn1 = new SimpleFilter(sop, rhs, nlhs1); + sfn1->timeZone(gwip->thd->variables.time_zone->get_name()->ptr()); ParseTree* ptpl = new ParseTree(sfn1); // a IS NULL ConstantColumn* nlhs2 = new ConstantColumn("", ConstantColumn::NULLDATA); + nlhs2->timeZone(gwip->thd->variables.time_zone->get_name()->ptr()); sop.reset(new PredicateOperator("isnull")); sop->setOpType(lhs->resultType(), rhs->resultType()); sfn2 = new SimpleFilter(sop, lhs, nlhs2); + sfn2->timeZone(gwip->thd->variables.time_zone->get_name()->ptr()); ParseTree* ptpr = new ParseTree(sfn2); // AND them both ParseTree* ptpn = new ParseTree(new LogicOperator("and")); @@ -1832,6 +1845,7 @@ bool buildPredicateItem(Item_func* ifp, gp_walk_info* gwip) sop.reset(new PredicateOperator("=")); sop->setOpType(lhs->resultType(), rhs->resultType()); sfo = new SimpleFilter(sop, lhs->clone(), rhs->clone()); + sfo->timeZone(gwip->thd->variables.time_zone->get_name()->ptr()); // OR with the NULL comparison tree ParseTree* ptp = new ParseTree(new LogicOperator("or")); ptp->left(sfo); @@ -1930,6 +1944,7 @@ bool buildPredicateItem(Item_func* ifp, gp_walk_info* gwip) } SimpleFilter* sf = new SimpleFilter(); + sf->timeZone(gwip->thd->variables.time_zone->get_name()->ptr()); //@bug 2101 for when there are only constants in a delete or update where clause (eg "where 5 < 6"). //There will be no field column and it will get here only if the comparison is true. @@ -2037,6 +2052,7 @@ bool buildPredicateItem(Item_func* ifp, gp_walk_info* gwip) bool buildConstPredicate(Item_func* ifp, ReturnedColumn* rhs, gp_walk_info* gwip) { SimpleFilter* sf = new SimpleFilter(); + sf->timeZone(gwip->thd->variables.time_zone->get_name()->ptr()); boost::shared_ptr sop(new PredicateOperator(ifp->func_name())); ConstantColumn* lhs = 0; @@ -2055,6 +2071,7 @@ bool buildConstPredicate(Item_func* ifp, ReturnedColumn* rhs, gp_walk_info* gwip lhs = new ConstantColumn((int64_t)0, ConstantColumn::NUM); sop.reset(new PredicateOperator("=")); } + lhs->timeZone(gwip->thd->variables.time_zone->get_name()->ptr()); CalpontSystemCatalog::ColType opType = rhs->resultType(); @@ -2128,6 +2145,7 @@ SimpleColumn* buildSimpleColFromDerivedTable(gp_walk_info& gwi, Item_field* ifp) sc->tableAlias(lower(gwi.tbList[i].alias)); sc->viewName(lower(viewName)); sc->resultType(ct); + sc->timeZone(gwi.thd->variables.time_zone->get_name()->ptr()); break; } } @@ -2186,6 +2204,7 @@ SimpleColumn* buildSimpleColFromDerivedTable(gp_walk_info& gwi, Item_field* ifp) string tableAlias(csep->derivedTbAlias()); sc->tableAlias(lower(tableAlias)); sc->viewName(lower(viewName)); + sc->timeZone(gwi.thd->variables.time_zone->get_name()->ptr()); sc->resultType(cols[j]->resultType()); sc->hasAggregate(cols[j]->hasAggregate()); @@ -2302,6 +2321,7 @@ void collectAllCols(gp_walk_info& gwi, Item_field* ifp) sc->tableAlias(lower(tableAlias)); sc->viewName(lower(gwi.tbList[i].view)); sc->resultType(cols[j]->resultType()); + sc->timeZone(gwi.thd->variables.time_zone->get_name()->ptr()); // @bug5634 derived table optimization cols[j]->incRefCount(); @@ -2336,6 +2356,7 @@ void collectAllCols(gp_walk_info& gwi, Item_field* ifp) sc->resultType(ct); sc->tableAlias(lower(gwi.tbList[i].alias)); sc->viewName(lower(viewName)); + sc->timeZone(gwi.thd->variables.time_zone->get_name()->ptr()); srcp.reset(sc); gwi.returnedCols.push_back(srcp); gwi.columnMap.insert(CalpontSelectExecutionPlan::ColumnMap::value_type(sc->columnName(), srcp)); @@ -2630,6 +2651,7 @@ SimpleColumn* getSmallestColumn(boost::shared_ptr csc, sc->columnName(rc->alias()); sc->sequence(0); sc->tableAlias(lower(tan.alias)); + sc->timeZone(gwi.thd->variables.time_zone->get_name()->ptr()); // @bug5634 derived table optimization. rc->incRefCount(); sc->derivedTable(csep->derivedTbAlias()); @@ -2650,6 +2672,7 @@ SimpleColumn* getSmallestColumn(boost::shared_ptr csc, string alias(table->alias.ptr()); sc->tableAlias(lower(alias)); sc->isInfiniDB(false); + sc->timeZone(gwi.thd->variables.time_zone->get_name()->ptr()); sc->resultType(fieldType_MysqlToIDB(field)); sc->oid(field->field_index + 1); return sc; @@ -2678,6 +2701,7 @@ SimpleColumn* getSmallestColumn(boost::shared_ptr csc, SimpleColumn* sc = new SimpleColumn(tcn.schema, tcn.table, tcn.column, csc->sessionID()); sc->tableAlias(lower(tan.alias)); sc->viewName(lower(tan.view)); + sc->timeZone(gwi.thd->variables.time_zone->get_name()->ptr()); sc->resultType(csc->colType(oidlist[minWidthColOffset].objnum)); return sc; } @@ -2778,14 +2802,19 @@ CalpontSystemCatalog::ColType colType_MysqlToIDB (const Item* item) ct.colWidth = 4; } else if (item->field_type() == MYSQL_TYPE_DATETIME || - item->field_type() == MYSQL_TYPE_DATETIME2 || - item->field_type() == MYSQL_TYPE_TIMESTAMP || - item->field_type() == MYSQL_TYPE_TIMESTAMP2 + item->field_type() == MYSQL_TYPE_DATETIME2 ) { ct.colDataType = CalpontSystemCatalog::DATETIME; ct.colWidth = 8; } + else if (item->field_type() == MYSQL_TYPE_TIMESTAMP || + item->field_type() == MYSQL_TYPE_TIMESTAMP2 + ) + { + ct.colDataType = CalpontSystemCatalog::TIMESTAMP; + ct.colWidth = 8; + } else if (item->field_type() == MYSQL_TYPE_TIME) { ct.colDataType = CalpontSystemCatalog::TIME; @@ -2884,6 +2913,7 @@ ReturnedColumn* buildReturnedColumn(Item* item, gp_walk_info& gwi, bool& nonSupp { rc = new ConstantColumn(valStr, (int64_t)item->val_int(), ConstantColumn::NUM); } + (dynamic_cast(rc))->timeZone(gwi.thd->variables.time_zone->get_name()->ptr()); //return cc; break; @@ -2895,6 +2925,7 @@ ReturnedColumn* buildReturnedColumn(Item* item, gp_walk_info& gwi, bool& nonSupp string valStr; valStr.assign(str->ptr(), str->length()); rc = new ConstantColumn(valStr); + (dynamic_cast(rc))->timeZone(gwi.thd->variables.time_zone->get_name()->ptr()); break; } @@ -2904,6 +2935,7 @@ ReturnedColumn* buildReturnedColumn(Item* item, gp_walk_info& gwi, bool& nonSupp string valStr; valStr.assign(str->ptr(), str->length()); rc = new ConstantColumn(valStr, item->val_real()); + (dynamic_cast(rc))->timeZone(gwi.thd->variables.time_zone->get_name()->ptr()); break; } @@ -2948,10 +2980,12 @@ ReturnedColumn* buildReturnedColumn(Item* item, gp_walk_info& gwi, bool& nonSupp if (!str) { rc = new ConstantColumn("", ConstantColumn::NULLDATA); + (dynamic_cast(rc))->timeZone(gwi.thd->variables.time_zone->get_name()->ptr()); } else if (ifp->result_type() == STRING_RESULT) { rc = new ConstantColumn(valStr, ConstantColumn::LITERAL); + (dynamic_cast(rc))->timeZone(gwi.thd->variables.time_zone->get_name()->ptr()); rc->resultType(colType_MysqlToIDB(item)); } else if (ifp->result_type() == DECIMAL_RESULT) @@ -2959,6 +2993,7 @@ ReturnedColumn* buildReturnedColumn(Item* item, gp_walk_info& gwi, bool& nonSupp else { rc = new ConstantColumn(valStr, ConstantColumn::NUM); + (dynamic_cast(rc))->timeZone(gwi.thd->variables.time_zone->get_name()->ptr()); rc->resultType(colType_MysqlToIDB(item)); } @@ -3009,7 +3044,9 @@ ReturnedColumn* buildReturnedColumn(Item* item, gp_walk_info& gwi, bool& nonSupp if (gwi.condPush) return new SimpleColumn("noop"); - return new ConstantColumn("", ConstantColumn::NULLDATA); + ConstantColumn *tmp = new ConstantColumn("", ConstantColumn::NULLDATA); + tmp->timeZone(gwi.thd->variables.time_zone->get_name()->ptr()); + return tmp; } case Item::CACHE_ITEM: @@ -3047,6 +3084,7 @@ ReturnedColumn* buildReturnedColumn(Item* item, gp_walk_info& gwi, bool& nonSupp string valStr; valStr.assign(str->ptr(), str->length()); rc = new ConstantColumn(valStr); + (dynamic_cast(rc))->timeZone(gwi.thd->variables.time_zone->get_name()->ptr()); break; } @@ -3114,6 +3152,7 @@ ArithmeticColumn* buildArithmeticColumn( ArithmeticColumn* ac = new ArithmeticColumn(); Item** sfitempp = item->arguments(); ArithmeticOperator* aop = new ArithmeticOperator(item->func_name()); + aop->timeZone(gwi.thd->variables.time_zone->get_name()->ptr()); ParseTree* pt = new ParseTree(aop); //ReturnedColumn *lhs = 0, *rhs = 0; ParseTree* lhs = 0, *rhs = 0; @@ -3238,6 +3277,7 @@ ArithmeticColumn* buildArithmeticColumn( else { ConstantColumn* cc = new ConstantColumn(string("0"), (int64_t)0); + cc->timeZone(gwi.thd->variables.time_zone->get_name()->ptr()); if (gwi.clauseType == SELECT || gwi.clauseType == HAVING || gwi.clauseType == GROUP_BY) // select clause { @@ -3553,6 +3593,7 @@ ReturnedColumn* buildFunctionColumn( { THD* thd = current_thd; sptp.reset(new ParseTree(new ConstantColumn(static_cast(thd->variables.default_week_format)))); + (dynamic_cast(sptp->data()))->timeZone(gwi.thd->variables.time_zone->get_name()->ptr()); funcParms.push_back(sptp); } @@ -3560,7 +3601,7 @@ ReturnedColumn* buildFunctionColumn( if (funcName == "date_add_interval" || funcName == "extract" || funcName == "timestampdiff") { - addIntervalArgs(ifp, funcParms); + addIntervalArgs(gwi.thd, ifp, funcParms); } // check for unsupported arguments add the keyword unit argument for extract functions @@ -3611,19 +3652,19 @@ ReturnedColumn* buildFunctionColumn( // add the keyword unit argument and char length for cast functions if (funcName == "cast_as_char" ) { - castCharArgs(ifp, funcParms); + castCharArgs(gwi.thd, ifp, funcParms); } // add the length and scale arguments if (funcName == "decimal_typecast" ) { - castDecimalArgs(ifp, funcParms); + castDecimalArgs(gwi.thd, ifp, funcParms); } // add the type argument if (funcName == "get_format") { - castTypeArgs(ifp, funcParms); + castTypeArgs(gwi.thd, ifp, funcParms); } // add my_time_zone @@ -3638,6 +3679,7 @@ ReturnedColumn* buildFunctionColumn( //FIXME: Get GMT offset (in seconds east of GMT) in Windows... sptp.reset(new ParseTree(new ConstantColumn(static_cast(0), ConstantColumn::NUM))); #endif + (dynamic_cast(sptp->data()))->timeZone(gwi.thd->variables.time_zone->get_name()->ptr()); funcParms.push_back(sptp); } @@ -3647,8 +3689,10 @@ ReturnedColumn* buildFunctionColumn( if (funcParms.size() == 0) { sptp.reset(new ParseTree(new ConstantColumn((int64_t)gwi.thd->rand.seed1, ConstantColumn::NUM))); + (dynamic_cast(sptp->data()))->timeZone(gwi.thd->variables.time_zone->get_name()->ptr()); funcParms.push_back(sptp); sptp.reset(new ParseTree(new ConstantColumn((int64_t)gwi.thd->rand.seed2, ConstantColumn::NUM))); + (dynamic_cast(sptp->data()))->timeZone(gwi.thd->variables.time_zone->get_name()->ptr()); funcParms.push_back(sptp); gwi.no_parm_func_list.push_back(fc); } @@ -3676,6 +3720,7 @@ ReturnedColumn* buildFunctionColumn( sign = -1; } sptp.reset(new ParseTree(new ConstantColumn(sign))); + (dynamic_cast(sptp->data()))->timeZone(gwi.thd->variables.time_zone->get_name()->ptr()); funcParms.push_back(sptp); } @@ -3686,15 +3731,21 @@ ReturnedColumn* buildFunctionColumn( // MySQL give string result type for date function, but has the flag set. // we should set the result type to be datetime for comparision. if (ifp->field_type() == MYSQL_TYPE_DATETIME || - ifp->field_type() == MYSQL_TYPE_DATETIME2 || - ifp->field_type() == MYSQL_TYPE_TIMESTAMP || - ifp->field_type() == MYSQL_TYPE_TIMESTAMP2) + ifp->field_type() == MYSQL_TYPE_DATETIME2) { CalpontSystemCatalog::ColType ct; ct.colDataType = CalpontSystemCatalog::DATETIME; ct.colWidth = 8; fc->resultType(ct); } + if (ifp->field_type() == MYSQL_TYPE_TIMESTAMP || + ifp->field_type() == MYSQL_TYPE_TIMESTAMP2) + { + CalpontSystemCatalog::ColType ct; + ct.colDataType = CalpontSystemCatalog::TIMESTAMP; + ct.colWidth = 8; + fc->resultType(ct); + } else if (ifp->field_type() == MYSQL_TYPE_DATE) { CalpontSystemCatalog::ColType ct; @@ -3797,6 +3848,8 @@ ReturnedColumn* buildFunctionColumn( } } + fc->timeZone(gwi.thd->variables.time_zone->get_name()->ptr()); + return fc; } @@ -3942,6 +3995,7 @@ FunctionColumn* buildCaseFunction(Item_func* item, gp_walk_info& gwi, bool& nonS fc->functionName(funcName); fc->functionParms(funcParms); fc->expressionId(ci->expressionId++); + fc->timeZone(gwi.thd->variables.time_zone->get_name()->ptr()); // For function join. If any argument has non-zero joininfo, set it to the function. fc->setSimpleColumnList(); @@ -3995,7 +4049,9 @@ ConstantColumn* buildDecimalColumn(Item* item, gp_walk_info& gwi) infinidb_decimal.scale = idp->decimals; infinidb_decimal.precision = idp->max_length - idp->decimals; - return new ConstantColumn(valStr, infinidb_decimal); + ConstantColumn* cc = new ConstantColumn(valStr, infinidb_decimal); + cc->timeZone(gwi.thd->variables.time_zone->get_name()->ptr()); + return cc; } SimpleColumn* buildSimpleColumn(Item_field* ifp, gp_walk_info& gwi) @@ -4131,6 +4187,7 @@ SimpleColumn* buildSimpleColumn(Item_field* ifp, gp_walk_info& gwi) sc->alias(ifp->name.str); sc->isInfiniDB(infiniDB); + sc->timeZone(gwi.thd->variables.time_zone->get_name()->ptr()); if (!infiniDB && ifp->field) sc->oid(ifp->field->field_index + 1); // ExeMgr requires offset started from 1 @@ -4237,6 +4294,8 @@ ReturnedColumn* buildAggregateColumn(Item* item, gp_walk_info& gwi) ac = new AggregateColumn(gwi.sessionid); } + ac->timeZone(gwi.thd->variables.time_zone->get_name()->ptr()); + if (isp->name.length) ac->alias(isp->name.str); @@ -4380,6 +4439,7 @@ ReturnedColumn* buildAggregateColumn(Item* item, gp_walk_info& gwi) case Item::NULL_ITEM: { parm.reset(new ConstantColumn("", ConstantColumn::NULLDATA)); + (dynamic_cast(parm.get()))->timeZone(gwi.thd->variables.time_zone->get_name()->ptr()); ac->constCol(SRCP(buildReturnedColumn(sfitemp, gwi, gwi.fatalParseError))); break; } @@ -4711,7 +4771,7 @@ ReturnedColumn* buildAggregateColumn(Item* item, gp_walk_info& gwi) return ac; } -void addIntervalArgs(Item_func* ifp, FunctionParm& functionParms) +void addIntervalArgs(THD* thd, Item_func* ifp, FunctionParm& functionParms) { string funcName = ifp->func_name(); int interval_type = -1; @@ -4723,7 +4783,7 @@ void addIntervalArgs(Item_func* ifp, FunctionParm& functionParms) else if (funcName == "extract") interval_type = ((Item_extract*)ifp)->int_type; - functionParms.push_back(getIntervalType(interval_type)); + functionParms.push_back(getIntervalType(thd, interval_type)); SPTP sptp; if (funcName == "date_add_interval") @@ -4731,37 +4791,42 @@ void addIntervalArgs(Item_func* ifp, FunctionParm& functionParms) if (((Item_date_add_interval*)ifp)->date_sub_interval) { sptp.reset(new ParseTree(new ConstantColumn((int64_t)OP_SUB))); + (dynamic_cast(sptp->data()))->timeZone(thd->variables.time_zone->get_name()->ptr()); functionParms.push_back(sptp); } else { sptp.reset(new ParseTree(new ConstantColumn((int64_t)OP_ADD))); + (dynamic_cast(sptp->data()))->timeZone(thd->variables.time_zone->get_name()->ptr()); functionParms.push_back(sptp); } } } -SPTP getIntervalType(int interval_type) +SPTP getIntervalType(THD* thd, int interval_type) { SPTP sptp; sptp.reset(new ParseTree(new ConstantColumn((int64_t)interval_type))); + (dynamic_cast(sptp->data()))->timeZone(thd->variables.time_zone->get_name()->ptr()); return sptp; } -void castCharArgs(Item_func* ifp, FunctionParm& functionParms) +void castCharArgs(THD* thd, Item_func* ifp, FunctionParm& functionParms) { Item_char_typecast* idai = (Item_char_typecast*)ifp; SPTP sptp; sptp.reset(new ParseTree(new ConstantColumn((int64_t)idai->castLength()))); + (dynamic_cast(sptp->data()))->timeZone(thd->variables.time_zone->get_name()->ptr()); functionParms.push_back(sptp); } -void castDecimalArgs(Item_func* ifp, FunctionParm& functionParms) +void castDecimalArgs(THD* thd, Item_func* ifp, FunctionParm& functionParms) { Item_decimal_typecast* idai = (Item_decimal_typecast*)ifp; SPTP sptp; sptp.reset(new ParseTree(new ConstantColumn((int64_t)idai->decimals))); + (dynamic_cast(sptp->data()))->timeZone(thd->variables.time_zone->get_name()->ptr()); functionParms.push_back(sptp); // max length including sign and/or decimal points @@ -4769,11 +4834,12 @@ void castDecimalArgs(Item_func* ifp, FunctionParm& functionParms) sptp.reset(new ParseTree(new ConstantColumn((int64_t)idai->max_length - 1))); else sptp.reset(new ParseTree(new ConstantColumn((int64_t)idai->max_length - 2))); + (dynamic_cast(sptp->data()))->timeZone(thd->variables.time_zone->get_name()->ptr()); functionParms.push_back(sptp); } -void castTypeArgs(Item_func* ifp, FunctionParm& functionParms) +void castTypeArgs(THD* thd, Item_func* ifp, FunctionParm& functionParms) { Item_func_get_format* get_format = (Item_func_get_format*)ifp; SPTP sptp; @@ -4782,6 +4848,7 @@ void castTypeArgs(Item_func* ifp, FunctionParm& functionParms) sptp.reset(new ParseTree(new ConstantColumn("DATE"))); else sptp.reset(new ParseTree(new ConstantColumn("DATETIME"))); + (dynamic_cast(sptp->data()))->timeZone(thd->variables.time_zone->get_name()->ptr()); functionParms.push_back(sptp); } @@ -4872,6 +4939,7 @@ void gp_walk(const Item* item, void* arg) } gwip->rcWorkStack.push(new ConstantColumn(cval)); + (dynamic_cast(gwip->rcWorkStack.top()))->timeZone(gwip->thd->variables.time_zone->get_name()->ptr()); break; } @@ -4913,6 +4981,7 @@ void gp_walk(const Item* item, void* arg) } gwip->rcWorkStack.push(new ConstantColumn("", ConstantColumn::NULLDATA)); + (dynamic_cast(gwip->rcWorkStack.top()))->timeZone(gwip->thd->variables.time_zone->get_name()->ptr()); break; } @@ -5003,10 +5072,12 @@ void gp_walk(const Item* item, void* arg) if (!str) //@ bug 2844 check whether parameter is defined { cc = new ConstantColumn("", ConstantColumn::NULLDATA); + cc->timeZone(gwip->thd->variables.time_zone->get_name()->ptr()); } else if (ifp->result_type() == STRING_RESULT) { cc = new ConstantColumn(valStr, ConstantColumn::LITERAL); + cc->timeZone(gwip->thd->variables.time_zone->get_name()->ptr()); } else if (ifp->result_type() == DECIMAL_RESULT) { @@ -5015,6 +5086,7 @@ void gp_walk(const Item* item, void* arg) else { cc = new ConstantColumn(valStr, ConstantColumn::NUM); + cc->timeZone(gwip->thd->variables.time_zone->get_name()->ptr()); cc->resultType(colType_MysqlToIDB(item)); } @@ -5737,6 +5809,7 @@ int getSelectPlan(gp_walk_info& gwi, SELECT_LEX& select_lex, SCSEP& csep, bool i gwi.sessionid = sessionID; boost::shared_ptr csc = CalpontSystemCatalog::makeCalpontSystemCatalog(sessionID); csc->identity(CalpontSystemCatalog::FE); + csep->timeZone(gwi.thd->variables.time_zone->get_name()->ptr()); gwi.csc = csc; // @bug 2123. Override large table estimate if infinidb_ordered hint was used. @@ -5998,6 +6071,7 @@ int getSelectPlan(gp_walk_info& gwi, SELECT_LEX& select_lex, SCSEP& csep, bool i else if (join && join->zero_result_cause) { gwi.rcWorkStack.push(new ConstantColumn((int64_t)0, ConstantColumn::NUM)); + (dynamic_cast(gwi.rcWorkStack.top()))->timeZone(gwi.thd->variables.time_zone->get_name()->ptr()); } @@ -6385,10 +6459,12 @@ int getSelectPlan(gp_walk_info& gwi, SELECT_LEX& select_lex, SCSEP& csep, bool i if (!str) { cc = new ConstantColumn("", ConstantColumn::NULLDATA); + cc->timeZone(gwi.thd->variables.time_zone->get_name()->ptr()); } else if (ifp->result_type() == STRING_RESULT) { cc = new ConstantColumn(valStr, ConstantColumn::LITERAL); + cc->timeZone(gwi.thd->variables.time_zone->get_name()->ptr()); } else if (ifp->result_type() == DECIMAL_RESULT) { @@ -6397,6 +6473,7 @@ int getSelectPlan(gp_walk_info& gwi, SELECT_LEX& select_lex, SCSEP& csep, bool i else { cc = new ConstantColumn(valStr, ConstantColumn::NUM); + cc->timeZone(gwi.thd->variables.time_zone->get_name()->ptr()); cc->resultType(colType_MysqlToIDB(item)); } @@ -6620,6 +6697,7 @@ int getSelectPlan(gp_walk_info& gwi, SELECT_LEX& select_lex, SCSEP& csep, bool i selectSubList.push_back(ssub); SimpleColumn* rc = new SimpleColumn(); rc->colSource(rc->colSource() | SELECT_SUB); + rc->timeZone(gwi.thd->variables.time_zone->get_name()->ptr()); if (sub->get_select_lex()->get_table_list()) rc->viewName(lower(getViewName(sub->get_select_lex()->get_table_list()))); @@ -7999,6 +8077,7 @@ int getSelectPlan(gp_walk_info& gwi, SELECT_LEX& select_lex, SCSEP& csep, bool i sc1->tableAlias(sc->tableAlias()); sc1->viewName(lower(sc->viewName())); sc1->colPosition(0); + sc1->timeZone(gwi.thd->variables.time_zone->get_name()->ptr()); minSc.reset(sc1); } } @@ -8080,7 +8159,7 @@ int cp_get_plan(THD* thd, SCSEP& csep) #endif // Derived table projection and filter optimization. - derivedTableOptimization(csep); + derivedTableOptimization(thd, csep); return 0; } @@ -8113,6 +8192,7 @@ int cp_get_table_plan(THD* thd, SCSEP& csep, cal_table_info& ti) SimpleColumn* sc = new SimpleColumn(table->s->db.str, table->s->table_name.str, field->field_name.str, sessionID); string alias(table->alias.c_ptr()); sc->tableAlias(lower(alias)); + sc->timeZone(gwi->thd->variables.time_zone->get_name()->ptr()); assert (sc); boost::shared_ptr spsc(sc); gwi->returnedCols.push_back(spsc); @@ -8486,6 +8566,7 @@ int getGroupPlan(gp_walk_info& gwi, SELECT_LEX& select_lex, SCSEP& csep, cal_gro else if (join && join->zero_result_cause) { gwi.rcWorkStack.push(new ConstantColumn((int64_t)0, ConstantColumn::NUM)); + (dynamic_cast(gwi.rcWorkStack.top()))->timeZone(gwi.thd->variables.time_zone->get_name()->ptr()); } SELECT_LEX tmp_select_lex; @@ -8838,10 +8919,12 @@ int getGroupPlan(gp_walk_info& gwi, SELECT_LEX& select_lex, SCSEP& csep, cal_gro if (!str) { cc = new ConstantColumn("", ConstantColumn::NULLDATA); + cc->timeZone(gwi.thd->variables.time_zone->get_name()->ptr()); } else if (ifp->result_type() == STRING_RESULT) { cc = new ConstantColumn(valStr, ConstantColumn::LITERAL); + cc->timeZone(gwi.thd->variables.time_zone->get_name()->ptr()); } else if (ifp->result_type() == DECIMAL_RESULT) { @@ -8850,6 +8933,7 @@ int getGroupPlan(gp_walk_info& gwi, SELECT_LEX& select_lex, SCSEP& csep, cal_gro else { cc = new ConstantColumn(valStr, ConstantColumn::NUM); + cc->timeZone(gwi.thd->variables.time_zone->get_name()->ptr()); cc->resultType(colType_MysqlToIDB(item)); } @@ -9071,6 +9155,7 @@ int getGroupPlan(gp_walk_info& gwi, SELECT_LEX& select_lex, SCSEP& csep, cal_gro selectSubList.push_back(ssub); SimpleColumn* rc = new SimpleColumn(); rc->colSource(rc->colSource() | SELECT_SUB); + rc->timeZone(gwi.thd->variables.time_zone->get_name()->ptr()); if (sub->get_select_lex()->get_table_list()) rc->viewName(lower(getViewName(sub->get_select_lex()->get_table_list()))); @@ -10159,6 +10244,7 @@ int getGroupPlan(gp_walk_info& gwi, SELECT_LEX& select_lex, SCSEP& csep, cal_gro sc1->tableName(sc->tableName()); sc1->tableAlias(sc->tableAlias()); sc1->viewName(lower(sc->viewName())); + sc1->timeZone(gwi.thd->variables.time_zone->get_name()->ptr()); sc1->colPosition(0); minSc.reset(sc1); } diff --git a/dbcon/mysql/ha_calpont_impl.cpp b/dbcon/mysql/ha_calpont_impl.cpp index cc80f9898..1d3c98003 100644 --- a/dbcon/mysql/ha_calpont_impl.cpp +++ b/dbcon/mysql/ha_calpont_impl.cpp @@ -600,6 +600,19 @@ int fetchNextRow(uchar* buf, cal_table_info& ti, cal_connection_info* ci, bool h break; } + case CalpontSystemCatalog::TIMESTAMP: + { + if ((*f)->null_ptr) + *(*f)->null_ptr &= ~(*f)->null_bit; + + intColVal = row.getUintField<8>(s); + DataConvert::timestampToString(intColVal, tmp, 255, current_thd->variables.time_zone->get_name()->ptr(), colType.precision); + + Field_varstring* f2 = (Field_varstring*)*f; + f2->store(tmp, strlen(tmp), f2->charset()); + break; + } + case CalpontSystemCatalog::CHAR: case CalpontSystemCatalog::VARCHAR: { @@ -1005,6 +1018,205 @@ void makeUpdateSemiJoin(const ParseTree* n, void* obj) return; } +vector getOnUpdateTimestampColumns(string& schema, string& tableName, int sessionID) +{ + vector returnVal; + typedef CalpontSelectExecutionPlan::ColumnMap::value_type CMVT_; + boost::shared_ptr csc = CalpontSystemCatalog::makeCalpontSystemCatalog(sessionID); + csc->identity(execplan::CalpontSystemCatalog::FE); + CalpontSystemCatalog::TableName aTableName; + boost::algorithm::to_lower(schema); + boost::algorithm::to_lower(tableName); + + // select columnname from calpontsys.syscolumn + // where schema = schema and tablename = tableName + // and datatype = 'timestamp' + // and defaultvalue = 'current_timestamp() ON UPDATE current_timestamp()' + CalpontSelectExecutionPlan csep; + CalpontSelectExecutionPlan::ReturnedColumnList returnedColumnList; + CalpontSelectExecutionPlan::FilterTokenList filterTokenList; + CalpontSelectExecutionPlan::ColumnMap colMap; + + SessionManager sm; + BRM::TxnID txnID; + txnID = sm.getTxnID(sessionID); + + if (!txnID.valid) + { + txnID.id = 0; + txnID.valid = true; + } + + QueryContext verID; + verID = sm.verID(); + csep.txnID(txnID.id); + csep.verID(verID); + csep.sessionID(sessionID); + + string sysTable = "calpontsys.syscolumn."; + string firstCol = sysTable + "columnname"; + SimpleColumn* c1 = new SimpleColumn(firstCol, sessionID); + string secondCol = sysTable + "schema"; + SimpleColumn* c2 = new SimpleColumn(secondCol, sessionID); + string thirdCol = sysTable + "tablename"; + SimpleColumn* c3 = new SimpleColumn(thirdCol, sessionID); + string fourthCol = sysTable + "datatype"; + SimpleColumn* c4 = new SimpleColumn(fourthCol, sessionID); + string fifthCol = sysTable + "defaultvalue"; + SimpleColumn* c5 = new SimpleColumn(fifthCol, sessionID); + SRCP srcp; + srcp.reset(c1); + colMap.insert(CMVT_(firstCol, srcp)); + srcp.reset(c2); + colMap.insert(CMVT_(secondCol, srcp)); + srcp.reset(c3); + colMap.insert(CMVT_(thirdCol, srcp)); + srcp.reset(c4); + colMap.insert(CMVT_(fourthCol, srcp)); + srcp.reset(c5); + colMap.insert(CMVT_(fifthCol, srcp)); + csep.columnMapNonStatic(colMap); + srcp.reset(c1->clone()); + returnedColumnList.push_back(srcp); + csep.returnedCols(returnedColumnList); + + // Filters + const SOP opeq(new Operator("=")); + SimpleFilter* f1 = new SimpleFilter (opeq, + c2->clone(), + new ConstantColumn(schema, ConstantColumn::LITERAL)); + filterTokenList.push_back(f1); + filterTokenList.push_back(new Operator("and")); + + SimpleFilter* f2 = new SimpleFilter (opeq, + c3->clone(), + new ConstantColumn(tableName, ConstantColumn::LITERAL)); + filterTokenList.push_back(f2); + filterTokenList.push_back(new Operator("and")); + + SimpleFilter* f3 = new SimpleFilter (opeq, + c4->clone(), + new ConstantColumn((uint64_t) execplan::CalpontSystemCatalog::TIMESTAMP, ConstantColumn::NUM)); + filterTokenList.push_back(f3); + filterTokenList.push_back(new Operator("and")); + + string defaultValue = "current_timestamp() ON UPDATE current_timestamp()"; + SimpleFilter* f4 = new SimpleFilter (opeq, + c5->clone(), + new ConstantColumn(defaultValue, ConstantColumn::LITERAL)); + filterTokenList.push_back(f4); + csep.filterTokenList(filterTokenList); + + CalpontSelectExecutionPlan::TableList tablelist; + tablelist.push_back(make_aliastable("calpontsys", "syscolumn", "")); + csep.tableList(tablelist); + + boost::shared_ptr exemgrClient (new messageqcpp::MessageQueueClient("ExeMgr1")); + ByteStream msg, emsgBs; + rowgroup::RGData rgData; + ByteStream::quadbyte qb = 4; + msg << qb; + rowgroup::RowGroup* rowGroup = 0; + uint32_t rowCount; + + exemgrClient->write(msg); + ByteStream msgPlan; + csep.serialize(msgPlan); + exemgrClient->write(msgPlan); + msg.restart(); + msg = exemgrClient->read(); //error handling + emsgBs = exemgrClient->read(); + ByteStream::quadbyte qb1; + + if (emsgBs.length() == 0) + { + //exemgrClient->shutdown(); + //delete exemgrClient; + //exemgrClient = 0; + throw runtime_error("Lost conection to ExeMgr."); + } + + string emsgStr; + emsgBs >> emsgStr; + + if (msg.length() == 4) + { + msg >> qb1; + + if (qb1 != 0) + { + //exemgrClient->shutdown(); + //delete exemgrClient; + //exemgrClient = 0; + throw runtime_error(emsgStr); + } + } + + while (true) + { + msg.restart(); + msg = exemgrClient->read(); + + if ( msg.length() == 0 ) + { + //exemgrClient->shutdown(); + //delete exemgrClient; + //exemgrClient = 0; + throw runtime_error("Lost conection to ExeMgr."); + } + else + { + if (!rowGroup) + { + //This is mete data + rowGroup = new rowgroup::RowGroup(); + rowGroup->deserialize(msg); + qb = 100; + msg.restart(); + msg << qb; + exemgrClient->write(msg); + continue; + } + + rgData.deserialize(msg); + rowGroup->setData(&rgData); + + if (rowGroup->getStatus() != 0) + { + //msg.advance(rowGroup->getDataSize()); + msg >> emsgStr; + //exemgrClient->shutdown(); + //delete exemgrClient; + //exemgrClient = 0; + throw runtime_error(emsgStr); + } + + rowCount = rowGroup->getRowCount(); + if (rowCount > 0) + { + rowgroup::Row row; + rowGroup->initRow(&row); + for (uint32_t i = 0; i < rowCount; i++) + { + rowGroup->getRow(i, &row); + // we are only fetching a single column + returnVal.push_back(row.getStringField(0)); + } + } + else + { + break; + } + + //exemgrClient->shutdown(); + //delete exemgrClient; + //exemgrClient = 0; + } + } + + return returnVal; +} + uint32_t doUpdateDelete(THD* thd) { if (get_fe_conn_info_ptr() == NULL) @@ -1129,6 +1341,7 @@ uint32_t doUpdateDelete(THD* thd) updateCP->queryType(CalpontSelectExecutionPlan::UPDATE); ci->stats.fQueryType = updateCP->queryType(); uint32_t cnt = 0; + tr1::unordered_set timeStampColumnNames; // for (; table_ptr; table_ptr= table_ptr->next_local) // { @@ -1182,6 +1395,12 @@ uint32_t doUpdateDelete(THD* thd) else schemaName = string(item->db_name); + if (item->field_type() == MYSQL_TYPE_TIMESTAMP || + item->field_type() == MYSQL_TYPE_TIMESTAMP2) + { + timeStampColumnNames.insert(string(item->name.str)); + } + columnAssignmentPtr = new ColumnAssignment(); columnAssignmentPtr->fColumn = string(item->name.str); columnAssignmentPtr->fOperator = "="; @@ -1370,6 +1589,30 @@ uint32_t doUpdateDelete(THD* thd) // if (cnt < thd->lex->select_lex.item_list.elements) // dmlStmt += ", "; } + + // Support for on update current_timestamp() for timestamp fields + // Query calpontsys.syscolumn to get all timestamp columns with + // ON UPDATE current_timestamp() property + vector onUpdateTimeStampColumns = getOnUpdateTimestampColumns(schemaName, tableName, tid2sid(thd->thread_id)); + for (size_t i = 0; i < onUpdateTimeStampColumns.size(); i++) + { + if (timeStampColumnNames.find(onUpdateTimeStampColumns[i]) == timeStampColumnNames.end()) + { + columnAssignmentPtr = new ColumnAssignment(); + columnAssignmentPtr->fColumn = string(onUpdateTimeStampColumns[i]); + columnAssignmentPtr->fOperator = "="; + columnAssignmentPtr->fFuncScale = 0; + columnAssignmentPtr->fFromCol = false; + struct timeval tv; + char buf[64]; + gettimeofday(&tv, 0); + MySQLTime time; + gmtSecToMySQLTime(tv.tv_sec, time, thd->variables.time_zone->get_name()->ptr()); + sprintf(buf, "%04d-%02d-%02d %02d:%02d:%02d.%06ld", time.year, time.month, time.day, time.hour, time.minute, time.second, tv.tv_usec); + columnAssignmentPtr->fScalarExpression = buf; + colAssignmentListPtr->push_back ( columnAssignmentPtr ); + } + } } else { @@ -1498,6 +1741,7 @@ uint32_t doUpdateDelete(THD* thd) pDMLPackage->set_TableName(tableName); pDMLPackage->set_SchemaName(schemaName); + pDMLPackage->set_TimeZone(thd->variables.time_zone->get_name()->ptr()); pDMLPackage->set_IsFromCol( true ); //cout << " setting isFromCol to " << isFromCol << endl; @@ -1697,6 +1941,7 @@ uint32_t doUpdateDelete(THD* thd) CalpontSystemCatalog::TableColName tcn = csc->colName(colrids[minWidthColOffset].objnum); SimpleColumn* sc = new SimpleColumn(tcn.schema, tcn.table, tcn.column, csc->sessionID()); sc->tableAlias(aliasName); + sc->timeZone(thd->variables.time_zone->get_name()->ptr()); sc->resultType(csc->colType(colrids[minWidthColOffset].objnum)); SRCP srcp; srcp.reset(sc); @@ -1826,6 +2071,7 @@ uint32_t doUpdateDelete(THD* thd) //cout << "doUpdateDelete start new DMLProc client for ctrl-c " << " for session " << sessionID << endl; VendorDMLStatement cmdStmt( "CTRL+C", DML_COMMAND, sessionID); CalpontDMLPackage* pDMLPackage = CalpontDMLFactory::makeCalpontDMLPackageFromMysqlBuffer(cmdStmt); + pDMLPackage->set_TimeZone(thd->variables.time_zone->get_name()->ptr()); ByteStream bytestream; bytestream << static_cast(sessionID); pDMLPackage->write(bytestream); @@ -1947,6 +2193,7 @@ uint32_t doUpdateDelete(THD* thd) { VendorDMLStatement cmdStmt(command, DML_COMMAND, sessionID); CalpontDMLPackage* pDMLPackage = CalpontDMLFactory::makeCalpontDMLPackageFromMysqlBuffer(cmdStmt); + pDMLPackage->set_TimeZone(thd->variables.time_zone->get_name()->ptr()); pDMLPackage->setTableOid (ci->tableOid); ByteStream bytestream; bytestream << static_cast(sessionID); @@ -3097,9 +3344,6 @@ int ha_calpont_impl_write_row(uchar* buf, TABLE* table) int ha_calpont_impl_update_row() { - //@Bug 2540. Return the correct error code. - THD* thd = current_thd; - if (get_fe_conn_info_ptr() == NULL) set_fe_conn_info_ptr((void*)new cal_connection_info()); @@ -3114,9 +3358,6 @@ int ha_calpont_impl_update_row() int ha_calpont_impl_delete_row() { - //@Bug 2540. Return the correct error code. - THD* thd = current_thd; - if (get_fe_conn_info_ptr() == NULL) set_fe_conn_info_ptr((void*)new cal_connection_info()); @@ -3311,7 +3552,7 @@ void ha_calpont_impl_start_bulk_insert(ha_rows rows, TABLE* table) #ifdef _MSC_VER aCmdLine = aCmdLine + "/bin/cpimport.exe -N -P " + to_string(localModuleId) + " -s " + ci->delimiter + " -e 0" + " -E " + escapechar + ci->enclosed_by + " "; #else - aCmdLine = aCmdLine + "/bin/cpimport -m 1 -N -P " + boost::to_string(localModuleId) + " -s " + ci->delimiter + " -e 0" + " -E " + escapechar + ci->enclosed_by + " "; + aCmdLine = aCmdLine + "/bin/cpimport -m 1 -N -P " + boost::to_string(localModuleId) + " -s " + ci->delimiter + " -e 0" + " -T " + thd->variables.time_zone->get_name()->ptr() + " -E " + escapechar + ci->enclosed_by + " "; #endif } } @@ -3320,7 +3561,7 @@ void ha_calpont_impl_start_bulk_insert(ha_rows rows, TABLE* table) #ifdef _MSC_VER aCmdLine = aCmdLine + "/bin/cpimport.exe -N -s " + ci->delimiter + " -e 0" + " -E " + escapechar + ci->enclosed_by + " "; #else - aCmdLine = aCmdLine + "/bin/cpimport -m 1 -N -s " + ci->delimiter + " -e 0" + " -E " + escapechar + ci->enclosed_by + " "; + aCmdLine = aCmdLine + "/bin/cpimport -m 1 -N -s " + ci->delimiter + " -e 0" + " -T " + thd->variables.time_zone->get_name()->ptr() + " -E " + escapechar + ci->enclosed_by + " "; #endif } diff --git a/dbcon/mysql/ha_calpont_impl_if.h b/dbcon/mysql/ha_calpont_impl_if.h index d53d6e652..6d8945ad8 100644 --- a/dbcon/mysql/ha_calpont_impl_if.h +++ b/dbcon/mysql/ha_calpont_impl_if.h @@ -357,13 +357,13 @@ execplan::ParseTree* buildParseTree(Item_func* item, gp_walk_info& gwi, bool& no execplan::ReturnedColumn* buildAggregateColumn(Item* item, gp_walk_info& gwi); execplan::ReturnedColumn* buildWindowFunctionColumn(Item* item, gp_walk_info& gwi, bool& nonSupport); execplan::ReturnedColumn* buildPseudoColumn(Item* item, gp_walk_info& gwi, bool& nonSupport, uint32_t pseudoType); -void addIntervalArgs(Item_func* ifp, funcexp::FunctionParm& functionParms); -void castCharArgs(Item_func* ifp, funcexp::FunctionParm& functionParms); -void castDecimalArgs(Item_func* ifp, funcexp::FunctionParm& functionParms); -void castTypeArgs(Item_func* ifp, funcexp::FunctionParm& functionParms); +void addIntervalArgs(THD* thd, Item_func* ifp, funcexp::FunctionParm& functionParms); +void castCharArgs(THD* thd, Item_func* ifp, funcexp::FunctionParm& functionParms); +void castDecimalArgs(THD* thd, Item_func* ifp, funcexp::FunctionParm& functionParms); +void castTypeArgs(THD* thd, Item_func* ifp, funcexp::FunctionParm& functionParms); //void parse_item (Item* item, std::vector& field_vec, bool& hasNonSupportItem, uint16& parseInfo); bool isPredicateFunction(Item* item, gp_walk_info* gwip); -execplan::ParseTree* buildRowPredicate(execplan::RowColumn* lhs, execplan::RowColumn* rhs, std::string predicateOp); +execplan::ParseTree* buildRowPredicate(THD* thd, execplan::RowColumn* lhs, execplan::RowColumn* rhs, std::string predicateOp); bool buildRowColumnFilter(gp_walk_info* gwip, execplan::RowColumn* rhs, execplan::RowColumn* lhs, Item_func* ifp); bool buildPredicateItem(Item_func* ifp, gp_walk_info* gwip); void collectAllCols(gp_walk_info& gwi, Item_field* ifp); @@ -373,13 +373,13 @@ std::string getViewName(TABLE_LIST* table_ptr); bool buildConstPredicate(Item_func* ifp, execplan::ReturnedColumn* rhs, gp_walk_info* gwip); execplan::CalpontSystemCatalog::ColType fieldType_MysqlToIDB (const Field* field); execplan::CalpontSystemCatalog::ColType colType_MysqlToIDB (const Item* item); -execplan::SPTP getIntervalType(int interval_type); +execplan::SPTP getIntervalType(THD* thd, int interval_type); uint32_t isPseudoColumn(std::string funcName); void setDerivedTable(execplan::ParseTree* n); -execplan::ParseTree* setDerivedFilter(execplan::ParseTree*& n, +execplan::ParseTree* setDerivedFilter(THD* thd, execplan::ParseTree*& n, std::map& obj, execplan::CalpontSelectExecutionPlan::SelectList& derivedTbList); -void derivedTableOptimization(execplan::SCSEP& csep); +void derivedTableOptimization(THD* thd, execplan::SCSEP& csep); #ifdef DEBUG_WALK_COND void debug_walk(const Item* item, void* arg); diff --git a/dbcon/mysql/ha_calpont_partition.cpp b/dbcon/mysql/ha_calpont_partition.cpp index 7d2317661..5ee2311d0 100644 --- a/dbcon/mysql/ha_calpont_partition.cpp +++ b/dbcon/mysql/ha_calpont_partition.cpp @@ -128,6 +128,9 @@ string name(CalpontSystemCatalog::ColType& ct) case CalpontSystemCatalog::TIME: return "TIME"; + case CalpontSystemCatalog::TIMESTAMP: + return "TIMESTAMP"; + case CalpontSystemCatalog::DECIMAL: return "DECIMAL"; @@ -208,6 +211,7 @@ bool CP_type(CalpontSystemCatalog::ColType& ct) ct.colDataType == CalpontSystemCatalog::DATE || ct.colDataType == CalpontSystemCatalog::DATETIME || ct.colDataType == CalpontSystemCatalog::TIME || + ct.colDataType == CalpontSystemCatalog::TIMESTAMP || ct.colDataType == CalpontSystemCatalog::DECIMAL || ct.colDataType == CalpontSystemCatalog::UTINYINT || ct.colDataType == CalpontSystemCatalog::USMALLINT || @@ -253,6 +257,10 @@ const string format(int64_t v, CalpontSystemCatalog::ColType& ct) oss << DataConvert::datetimeToString(v); break; + case CalpontSystemCatalog::TIMESTAMP: + oss << DataConvert::timestampToString(v, current_thd->variables.time_zone->get_name()->ptr()); + break; + case CalpontSystemCatalog::TIME: oss << DataConvert::timeToString(v); @@ -311,7 +319,7 @@ const int64_t IDB_format(char* str, CalpontSystemCatalog::ColType& ct, uint8_t& int64_t v = 0; bool pushWarning = false; rf = 0; - boost::any anyVal = DataConvert::convertColumnData(ct, str, pushWarning, false, true); + boost::any anyVal = DataConvert::convertColumnData(ct, str, pushWarning, current_thd->variables.time_zone->get_name()->ptr(), false, true, false); switch (ct.colDataType) { @@ -378,6 +386,7 @@ const int64_t IDB_format(char* str, CalpontSystemCatalog::ColType& ct, uint8_t& v = boost::any_cast(anyVal); break; + case CalpontSystemCatalog::TIMESTAMP: case CalpontSystemCatalog::DATETIME: v = boost::any_cast(anyVal); break; diff --git a/dbcon/mysql/ha_from_sub.cpp b/dbcon/mysql/ha_from_sub.cpp index 4add7afcb..94f4e1d18 100644 --- a/dbcon/mysql/ha_from_sub.cpp +++ b/dbcon/mysql/ha_from_sub.cpp @@ -44,7 +44,7 @@ using namespace execplan; namespace cal_impl_if { -void derivedTableOptimization(SCSEP& csep) +void derivedTableOptimization(THD* thd, SCSEP& csep) { // @bug5634. replace the unused column with ConstantColumn from derived table column list, // ExeMgr will not project ConstantColumn. Only count for local derived column. @@ -102,9 +102,13 @@ void derivedTableOptimization(SCSEP& csep) cols[i]->derivedRefCol()->decRefCount(); cols[i].reset(new ConstantColumn(val)); + (dynamic_cast(cols[i].get()))->timeZone(thd->variables.time_zone->get_name()->ptr()); for (uint j = 0; j < unionColVec.size(); j++) + { unionColVec[j][i].reset(new ConstantColumn(val)); + (dynamic_cast(unionColVec[j][i].get()))->timeZone(thd->variables.time_zone->get_name()->ptr()); + } } } @@ -136,7 +140,7 @@ void derivedTableOptimization(SCSEP& csep) if (horizontalOptimization && pt) { pt->walk(setDerivedTable); - setDerivedFilter(pt, derivedTbFilterMap, derivedTbList); + setDerivedFilter(thd, pt, derivedTbFilterMap, derivedTbList); csep->filters(pt); } @@ -208,7 +212,7 @@ void derivedTableOptimization(SCSEP& csep) for (uint i = 0; i < csep->subSelectList().size(); i++) { SCSEP subselect(boost::dynamic_pointer_cast(csep->subSelectList()[i])); - derivedTableOptimization(subselect); + derivedTableOptimization(thd, subselect); } } @@ -246,7 +250,8 @@ void setDerivedTable(execplan::ParseTree* n) } } -ParseTree* setDerivedFilter(ParseTree*& n, map& filterMap, +ParseTree* setDerivedFilter(THD* thd, ParseTree*& n, + map& filterMap, CalpontSelectExecutionPlan::SelectList& derivedTbList) { if (!(n->derivedTable().empty())) @@ -288,6 +293,7 @@ ParseTree* setDerivedFilter(ParseTree*& n, map& filterMap, int64_t val = 1; n = new ParseTree(new ConstantColumn(val)); + (dynamic_cast(n->data()))->timeZone(thd->variables.time_zone->get_name()->ptr()); } else { @@ -303,10 +309,10 @@ ParseTree* setDerivedFilter(ParseTree*& n, map& filterMap, ParseTree* rhs = n->right(); if (lhs) - n->left(setDerivedFilter(lhs, filterMap, derivedTbList)); + n->left(setDerivedFilter(thd, lhs, filterMap, derivedTbList)); if (rhs) - n->right(setDerivedFilter(rhs, filterMap, derivedTbList)); + n->right(setDerivedFilter(thd, rhs, filterMap, derivedTbList)); } } diff --git a/dbcon/mysql/ha_pseudocolumn.cpp b/dbcon/mysql/ha_pseudocolumn.cpp index c87196e00..57683b9bf 100644 --- a/dbcon/mysql/ha_pseudocolumn.cpp +++ b/dbcon/mysql/ha_pseudocolumn.cpp @@ -53,8 +53,6 @@ void bailout(char* error, const string& funcName) int64_t idblocalpm() { - THD* thd = current_thd; - if (get_fe_conn_info_ptr() == NULL) set_fe_conn_info_ptr((void*)new cal_connection_info()); @@ -503,6 +501,7 @@ execplan::ReturnedColumn* buildPseudoColumn(Item* item, cc = new ConstantColumn(localPm); else cc = new ConstantColumn("", ConstantColumn::NULLDATA); + cc->timeZone(gwi.thd->variables.time_zone->get_name()->ptr()); cc->alias(ifp->full_name() ? ifp->full_name() : ""); return cc; @@ -566,6 +565,7 @@ execplan::ReturnedColumn* buildPseudoColumn(Item* item, parms.push_back(sptp); fc->functionParms(parms); fc->expressionId(ci->expressionId++); + fc->timeZone(gwi.thd->variables.time_zone->get_name()->ptr()); // string result type CalpontSystemCatalog::ColType ct; diff --git a/dbcon/mysql/ha_scalar_sub.cpp b/dbcon/mysql/ha_scalar_sub.cpp index 2635f6116..8669303f9 100644 --- a/dbcon/mysql/ha_scalar_sub.cpp +++ b/dbcon/mysql/ha_scalar_sub.cpp @@ -106,6 +106,7 @@ execplan::ParseTree* ScalarSub::transform() { fSub = (Item_subselect*)(fFunc->arguments()[0]); fColumn.reset(new ConstantColumn("", ConstantColumn::NULLDATA)); + (dynamic_cast(fColumn.get()))->timeZone(fGwip.thd->variables.time_zone->get_name()->ptr()); delete rhs; return buildParseTree(op); } @@ -171,6 +172,7 @@ execplan::ParseTree* ScalarSub::transform_between() SOP sop; sop.reset(op_LE); rhs = new ParseTree(new SimpleFilter(sop, fColumn.get(), op3)); + (dynamic_cast(rhs->data()))->timeZone(fGwip.thd->variables.time_zone->get_name()->ptr()); } SubSelect* sub1 = dynamic_cast(op2); @@ -186,6 +188,7 @@ execplan::ParseTree* ScalarSub::transform_between() SOP sop; sop.reset(op_GE); lhs = new ParseTree(new SimpleFilter(sop, fColumn.get(), op2)); + (dynamic_cast(lhs->data()))->timeZone(fGwip.thd->variables.time_zone->get_name()->ptr()); } if (!rhs || !lhs) diff --git a/dbcon/mysql/ha_window_function.cpp b/dbcon/mysql/ha_window_function.cpp index 417edf61c..eb02b77a3 100644 --- a/dbcon/mysql/ha_window_function.cpp +++ b/dbcon/mysql/ha_window_function.cpp @@ -144,22 +144,25 @@ ReturnedColumn* buildBoundExp(WF_Boundary& bound, SRCP& order, gp_walk_info& gwi // put interval val column to bound (dynamic_cast(rc))->functionName(funcName); + (dynamic_cast(rc))->timeZone(gwi.thd->variables.time_zone->get_name()->ptr()); sptp.reset(new ParseTree(order->clone())); funcParms.push_back(sptp); sptp.reset(new ParseTree(intervalCol->val()->clone())); funcParms.push_back(sptp); - funcParms.push_back(getIntervalType(intervalCol->intervalType())); + funcParms.push_back(getIntervalType(gwi.thd, intervalCol->intervalType())); SRCP srcp(intervalCol->val()); bound.fVal = srcp; if (addOp) { sptp.reset(new ParseTree(new ConstantColumn("ADD"))); + (dynamic_cast(sptp->data()))->timeZone(gwi.thd->variables.time_zone->get_name()->ptr()); funcParms.push_back(sptp); } else { sptp.reset(new ParseTree(new ConstantColumn("SUB"))); + (dynamic_cast(sptp->data()))->timeZone(gwi.thd->variables.time_zone->get_name()->ptr()); funcParms.push_back(sptp); } @@ -181,6 +184,7 @@ ReturnedColumn* buildBoundExp(WF_Boundary& bound, SRCP& order, gp_walk_info& gwi else aop = new ArithmeticOperator("-"); + aop->timeZone(gwi.thd->variables.time_zone->get_name()->ptr()); ParseTree* pt = new ParseTree(aop); ParseTree* lhs = 0, *rhs = 0; lhs = new ParseTree(order->clone()); @@ -348,6 +352,7 @@ ReturnedColumn* buildWindowFunctionColumn(Item* item, gp_walk_info& gwi, bool& n Item_sum* item_sum = wf->window_func(); string funcName = ConvertFuncName(item_sum); WindowFunctionColumn* ac = new WindowFunctionColumn(funcName); + ac->timeZone(gwi.thd->variables.time_zone->get_name()->ptr()); ac->distinct(item_sum->has_with_distinct()); Window_spec* win_spec = wf->window_spec; SRCP srcp; @@ -441,33 +446,42 @@ ReturnedColumn* buildWindowFunctionColumn(Item* item, gp_walk_info& gwi, bool& n char sRespectNulls[18]; sprintf(sRespectNulls, "%lu", bRespectNulls); srcp.reset(new ConstantColumn(sRespectNulls, (uint64_t)bRespectNulls, ConstantColumn::NUM)); // IGNORE/RESPECT NULLS. 1 => RESPECT + (dynamic_cast(srcp.get()))->timeZone(gwi.thd->variables.time_zone->get_name()->ptr()); funcParms.push_back(srcp); break; } case Item_sum::FIRST_VALUE_FUNC: srcp.reset(new ConstantColumn("1", (uint64_t)1, ConstantColumn::NUM)); // OFFSET (always one) + (dynamic_cast(srcp.get()))->timeZone(gwi.thd->variables.time_zone->get_name()->ptr()); funcParms.push_back(srcp); srcp.reset(new ConstantColumn("1", (uint64_t)1, ConstantColumn::NUM)); // FROM_FIRST + (dynamic_cast(srcp.get()))->timeZone(gwi.thd->variables.time_zone->get_name()->ptr()); funcParms.push_back(srcp); srcp.reset(new ConstantColumn("1", (uint64_t)1, ConstantColumn::NUM)); // IGNORE/RESPECT NULLS. 1 => RESPECT + (dynamic_cast(srcp.get()))->timeZone(gwi.thd->variables.time_zone->get_name()->ptr()); funcParms.push_back(srcp); break; case Item_sum::LAST_VALUE_FUNC: srcp.reset(new ConstantColumn("1", (uint64_t)1, ConstantColumn::NUM)); // OFFSET (always one) + (dynamic_cast(srcp.get()))->timeZone(gwi.thd->variables.time_zone->get_name()->ptr()); funcParms.push_back(srcp); srcp.reset(new ConstantColumn("0", (uint64_t)0, ConstantColumn::NUM)); // FROM_LAST + (dynamic_cast(srcp.get()))->timeZone(gwi.thd->variables.time_zone->get_name()->ptr()); funcParms.push_back(srcp); srcp.reset(new ConstantColumn("1", (uint64_t)1, ConstantColumn::NUM)); // IGNORE/RESPECT NULLS. 1 => RESPECT + (dynamic_cast(srcp.get()))->timeZone(gwi.thd->variables.time_zone->get_name()->ptr()); funcParms.push_back(srcp); break; case Item_sum::NTH_VALUE_FUNC: // When the front end supports these paramters, this needs modification srcp.reset(new ConstantColumn("1", (uint64_t)1, ConstantColumn::NUM)); // FROM FIRST/LAST 1 => FIRST + (dynamic_cast(srcp.get()))->timeZone(gwi.thd->variables.time_zone->get_name()->ptr()); funcParms.push_back(srcp); srcp.reset(new ConstantColumn("1", (uint64_t)1, ConstantColumn::NUM)); // IGNORE/RESPECT NULLS. 1 => RESPECT + (dynamic_cast(srcp.get()))->timeZone(gwi.thd->variables.time_zone->get_name()->ptr()); funcParms.push_back(srcp); break; @@ -475,8 +489,10 @@ ReturnedColumn* buildWindowFunctionColumn(Item* item, gp_walk_info& gwi, bool& n case Item_sum::LAG_FUNC: // When the front end supports these paramters, this needs modification srcp.reset(new ConstantColumn("", ConstantColumn::NULLDATA)); // Default to fill in for NULL values + (dynamic_cast(srcp.get()))->timeZone(gwi.thd->variables.time_zone->get_name()->ptr()); funcParms.push_back(srcp); srcp.reset(new ConstantColumn("1", (uint64_t)1, ConstantColumn::NUM)); // IGNORE/RESPECT NULLS. 1 => RESPECT + (dynamic_cast(srcp.get()))->timeZone(gwi.thd->variables.time_zone->get_name()->ptr()); funcParms.push_back(srcp); break; @@ -622,6 +638,7 @@ ReturnedColumn* buildWindowFunctionColumn(Item* item, gp_walk_info& gwi, bool& n case CalpontSystemCatalog::DATE: case CalpontSystemCatalog::DATETIME: case CalpontSystemCatalog::TIME: + case CalpontSystemCatalog::TIMESTAMP: if (!frm.fIsRange) boundTypeErr = true; else if (dynamic_cast(frm.fStart.fVal.get()) == NULL) @@ -674,6 +691,7 @@ ReturnedColumn* buildWindowFunctionColumn(Item* item, gp_walk_info& gwi, bool& n case CalpontSystemCatalog::DATE: case CalpontSystemCatalog::DATETIME: case CalpontSystemCatalog::TIME: + case CalpontSystemCatalog::TIMESTAMP: if (!frm.fIsRange) boundTypeErr = true; else if (dynamic_cast(frm.fEnd.fVal.get()) == NULL) @@ -826,6 +844,7 @@ ReturnedColumn* buildWindowFunctionColumn(Item* item, gp_walk_info& gwi, bool& n bound = 1; srcp.reset(new ConstantColumn((int64_t)bound)); + (dynamic_cast(srcp.get()))->timeZone(gwi.thd->variables.time_zone->get_name()->ptr()); frm.fStart.fVal = srcp; frm.fStart.fBound.reset(buildBoundExp(frm.fStart, srcp, gwi)); @@ -841,6 +860,7 @@ ReturnedColumn* buildWindowFunctionColumn(Item* item, gp_walk_info& gwi, bool& n bound = 1; srcp.reset(new ConstantColumn((int64_t)bound)); + (dynamic_cast(srcp.get()))->timeZone(gwi.thd->variables.time_zone->get_name()->ptr()); frm.fEnd.fVal = srcp; frm.fEnd.fBound.reset(buildBoundExp(frm.fEnd, srcp, gwi)); diff --git a/dbcon/mysql/idb_mysql.h b/dbcon/mysql/idb_mysql.h index cce7bf9f4..25f3effcf 100644 --- a/dbcon/mysql/idb_mysql.h +++ b/dbcon/mysql/idb_mysql.h @@ -69,6 +69,7 @@ template bool isnan(T); #include "mysqld_error.h" #include "item_windowfunc.h" #include "sql_cte.h" +#include "tztime.h" // Now clean up the pollution as best we can... #undef min diff --git a/ddlproc/ddlprocessor.cpp b/ddlproc/ddlprocessor.cpp index 45bae8bac..4f3588707 100644 --- a/ddlproc/ddlprocessor.cpp +++ b/ddlproc/ddlprocessor.cpp @@ -144,6 +144,8 @@ struct PackageHandler qts.schema_name = alterTableStmt.schemaName(); fQtc.postQueryTele(qts); + processor->fTimeZone = alterTableStmt.fTimeZone; + result = processor->processPackage(alterTableStmt); systemCatalogPtr->removeCalpontSystemCatalog( alterTableStmt.fSessionID ); diff --git a/primitives/linux-port/column.cpp b/primitives/linux-port/column.cpp index afa411f08..a8b3dc838 100644 --- a/primitives/linux-port/column.cpp +++ b/primitives/linux-port/column.cpp @@ -288,6 +288,7 @@ inline bool isEmptyVal<8>(uint8_t type, const uint8_t* ival) case CalpontSystemCatalog::VARCHAR: case CalpontSystemCatalog::DATE: case CalpontSystemCatalog::DATETIME: + case CalpontSystemCatalog::TIMESTAMP: case CalpontSystemCatalog::TIME: case CalpontSystemCatalog::VARBINARY: case CalpontSystemCatalog::BLOB: @@ -321,6 +322,7 @@ inline bool isEmptyVal<4>(uint8_t type, const uint8_t* ival) case CalpontSystemCatalog::TEXT: case CalpontSystemCatalog::DATE: case CalpontSystemCatalog::DATETIME: + case CalpontSystemCatalog::TIMESTAMP: case CalpontSystemCatalog::TIME: return (joblist::CHAR4EMPTYROW == *val); @@ -348,6 +350,7 @@ inline bool isEmptyVal<2>(uint8_t type, const uint8_t* ival) case CalpontSystemCatalog::TEXT: case CalpontSystemCatalog::DATE: case CalpontSystemCatalog::DATETIME: + case CalpontSystemCatalog::TIMESTAMP: case CalpontSystemCatalog::TIME: return (joblist::CHAR2EMPTYROW == *val); @@ -374,6 +377,7 @@ inline bool isEmptyVal<1>(uint8_t type, const uint8_t* ival) case CalpontSystemCatalog::TEXT: case CalpontSystemCatalog::DATE: case CalpontSystemCatalog::DATETIME: + case CalpontSystemCatalog::TIMESTAMP: case CalpontSystemCatalog::TIME: return (*val == joblist::CHAR1EMPTYROW); @@ -405,6 +409,7 @@ inline bool isNullVal<8>(uint8_t type, const uint8_t* ival) case CalpontSystemCatalog::VARCHAR: case CalpontSystemCatalog::DATE: case CalpontSystemCatalog::DATETIME: + case CalpontSystemCatalog::TIMESTAMP: case CalpontSystemCatalog::TIME: case CalpontSystemCatalog::VARBINARY: case CalpontSystemCatalog::BLOB: @@ -442,6 +447,7 @@ inline bool isNullVal<4>(uint8_t type, const uint8_t* ival) case CalpontSystemCatalog::DATE: case CalpontSystemCatalog::DATETIME: + case CalpontSystemCatalog::TIMESTAMP: case CalpontSystemCatalog::TIME: return (joblist::DATENULL == *val); @@ -469,6 +475,7 @@ inline bool isNullVal<2>(uint8_t type, const uint8_t* ival) case CalpontSystemCatalog::TEXT: case CalpontSystemCatalog::DATE: case CalpontSystemCatalog::DATETIME: + case CalpontSystemCatalog::TIMESTAMP: case CalpontSystemCatalog::TIME: return (joblist::CHAR2NULL == *val); @@ -495,6 +502,7 @@ inline bool isNullVal<1>(uint8_t type, const uint8_t* ival) case CalpontSystemCatalog::TEXT: case CalpontSystemCatalog::DATE: case CalpontSystemCatalog::DATETIME: + case CalpontSystemCatalog::TIMESTAMP: case CalpontSystemCatalog::TIME: return (*val == joblist::CHAR1NULL); @@ -558,6 +566,7 @@ inline bool isMinMaxValid(const NewColRequestHeader* in) case CalpontSystemCatalog::BIGINT: case CalpontSystemCatalog::DATETIME: case CalpontSystemCatalog::TIME: + case CalpontSystemCatalog::TIMESTAMP: case CalpontSystemCatalog::UTINYINT: case CalpontSystemCatalog::USMALLINT: case CalpontSystemCatalog::UMEDINT: diff --git a/primitives/primproc/batchprimitiveprocessor.cpp b/primitives/primproc/batchprimitiveprocessor.cpp index bf34e9cb4..bd7c7b406 100644 --- a/primitives/primproc/batchprimitiveprocessor.cpp +++ b/primitives/primproc/batchprimitiveprocessor.cpp @@ -2264,6 +2264,7 @@ SBPP BatchPrimitiveProcessor::duplicate() bpp->fAggregateRG = fAggregateRG; bpp->fAggregator.reset(new RowAggregation( fAggregator->getGroupByCols(), fAggregator->getAggFunctions())); + bpp->fAggregator->timeZone(fAggregator->timeZone()); } bpp->sendRidsAtDelivery = sendRidsAtDelivery; diff --git a/primitives/primproc/columncommand.cpp b/primitives/primproc/columncommand.cpp index f5db52a2d..95c614323 100644 --- a/primitives/primproc/columncommand.cpp +++ b/primitives/primproc/columncommand.cpp @@ -969,6 +969,7 @@ const uint64_t ColumnCommand::getEmptyRowValue( const execplan::CalpontSystemCat case execplan::CalpontSystemCatalog::VARCHAR : case execplan::CalpontSystemCatalog::DATE : case execplan::CalpontSystemCatalog::DATETIME : + case execplan::CalpontSystemCatalog::TIMESTAMP : case execplan::CalpontSystemCatalog::TIME : case execplan::CalpontSystemCatalog::VARBINARY : case execplan::CalpontSystemCatalog::BLOB : diff --git a/tools/dbloadxml/colxml.cpp b/tools/dbloadxml/colxml.cpp index 69be8d61e..1ac41d7bf 100644 --- a/tools/dbloadxml/colxml.cpp +++ b/tools/dbloadxml/colxml.cpp @@ -47,7 +47,7 @@ int main(int argc, char** argv) // set effective ID to root if( setuid( 0 ) < 0 ) { - std::cerr << " colxml: setuid failed " << std::endl; + std::cerr << " colxml: couldn't set uid " << std::endl; } setlocale(LC_ALL, ""); WriteEngine::Config::initConfigCache(); // load Columnstore.xml config settings diff --git a/tools/pingproc/pingproc.cpp b/tools/pingproc/pingproc.cpp index b40eec905..f1dd7b8bc 100644 --- a/tools/pingproc/pingproc.cpp +++ b/tools/pingproc/pingproc.cpp @@ -308,6 +308,7 @@ bool OidOperation::isIntegralDataType() DataType() == CalpontSystemCatalog::DATE || DataType() == CalpontSystemCatalog::BIGINT || DataType() == CalpontSystemCatalog::DATETIME || + DataType() == CalpontSystemCatalog::TIMESTAMP || DataType() == CalpontSystemCatalog::TIME || DataType() == CalpontSystemCatalog::UTINYINT || DataType() == CalpontSystemCatalog::USMALLINT || diff --git a/utils/common/nullvaluemanip.cpp b/utils/common/nullvaluemanip.cpp index a72e8ad00..44ee2273a 100644 --- a/utils/common/nullvaluemanip.cpp +++ b/utils/common/nullvaluemanip.cpp @@ -57,6 +57,9 @@ uint64_t getNullValue(CalpontSystemCatalog::ColDataType t, uint32_t colWidth) case CalpontSystemCatalog::DATETIME: return joblist::DATETIMENULL; + case CalpontSystemCatalog::TIMESTAMP: + return joblist::TIMESTAMPNULL; + case CalpontSystemCatalog::TIME: return joblist::TIMENULL; @@ -164,6 +167,9 @@ int64_t getSignedNullValue(CalpontSystemCatalog::ColDataType t, uint32_t colWidt case CalpontSystemCatalog::DATETIME: return joblist::DATETIMENULL; + case CalpontSystemCatalog::TIMESTAMP: + return joblist::TIMESTAMPNULL; + case CalpontSystemCatalog::TIME: return joblist::TIMENULL; diff --git a/utils/dataconvert/dataconvert.cpp b/utils/dataconvert/dataconvert.cpp index 15a200d3b..b17b430fb 100644 --- a/utils/dataconvert/dataconvert.cpp +++ b/utils/dataconvert/dataconvert.cpp @@ -1051,7 +1051,6 @@ bool mysql_str_to_time( const string& input, Time& output, long decimals ) return true; } - bool stringToDateStruct( const string& data, Date& date ) { bool isDate; @@ -1095,9 +1094,76 @@ bool stringToTimeStruct(const string& data, Time& dtime, long decimals) return true; } +bool stringToTimestampStruct(const string& data, TimeStamp& timeStamp, const string& timeZone) +{ + // special handling for 0000-00-00 00:00:00 + // "0" is sent by the server when checking for default value + // in the DDL. This is equivalent of 0000-00-00 00:00:00 + if (data.substr(0, 19) == "0000-00-00 00:00:00" || data == "0") + { + timeStamp.second = 0; + timeStamp.msecond = 0; + return true; + } + + // for alter table add column timestamp, + // if the table is non-empty, then columnstore will apply + // default value to populate the new column + if (data == "current_timestamp() ON UPDATE current_timestamp()") + { + struct timeval tv; + gettimeofday(&tv, 0); + timeStamp.second = tv.tv_sec; + timeStamp.msecond = tv.tv_usec; + return true; + } + + bool isDate; + + DateTime dtime; + + if ( !mysql_str_to_datetime( data, dtime, isDate ) ) + { + timeStamp.reset(); + return false; + } + + if ( isDate ) + { + dtime.hour = 0; + dtime.minute = 0; + dtime.second = 0; + dtime.msecond = 0; + } + + MySQLTime m_time; + m_time.year = dtime.year; + m_time.month = dtime.month; + m_time.day = dtime.day; + m_time.hour = dtime.hour; + m_time.minute = dtime.minute; + m_time.second = dtime.second; + m_time.second_part = dtime.msecond; + + bool isValid = true; + int64_t seconds = mySQLTimeToGmtSec(m_time, timeZone, isValid); + + if (!isValid) + { + timeStamp.reset(); + return false; + } + + timeStamp.second = seconds; + timeStamp.msecond = m_time.second_part; + + return true; + +} + boost::any DataConvert::convertColumnData(const CalpontSystemCatalog::ColType& colType, - const std::string& dataOrig, bool& pushWarning, bool nulFlag, bool noRoundup, bool isUpdate ) + const std::string& dataOrig, bool& pushWarning, const std::string& timeZone, bool nulFlag, bool noRoundup, bool isUpdate) { boost::any value; std::string data( dataOrig ); @@ -1443,6 +1509,18 @@ DataConvert::convertColumnData(const CalpontSystemCatalog::ColType& colType, } break; + case CalpontSystemCatalog::TIMESTAMP: + { + TimeStamp aTimestamp; + + if (!stringToTimestampStruct(data, aTimestamp, timeZone)) + { + pushWarning = true; + } + + value = (uint64_t) *(reinterpret_cast(&aTimestamp)); + } + break; case CalpontSystemCatalog::BLOB: case CalpontSystemCatalog::CLOB: @@ -1560,6 +1638,13 @@ DataConvert::convertColumnData(const CalpontSystemCatalog::ColType& colType, } break; + case CalpontSystemCatalog::TIMESTAMP: + { + uint64_t d = joblist::TIMESTAMPNULL; + value = d; + } + break; + case CalpontSystemCatalog::TIME: { uint64_t d = joblist::TIMENULL; @@ -1915,6 +2000,186 @@ int64_t DataConvert::convertColumnDatetime( return value; } +//------------------------------------------------------------------------------ +// Convert timestamp string to binary timestamp. Used by BulkLoad. +// Most of this code is taken from DataConvert::convertColumnDatetime +//------------------------------------------------------------------------------ +int64_t DataConvert::convertColumnTimestamp( + const char* dataOrg, + CalpontDateTimeFormat datetimeFormat, + int& status, + unsigned int dataOrgLen, + const std::string& timeZone ) +{ + + std::string dataOrgTemp = dataOrg; + if (dataOrgTemp.substr(0, 19) == "0000-00-00 00:00:00") + { + return 0; + } + + // this is the default value of the first timestamp field in a table, + // which is stored in the system catalog + if (strcmp(dataOrg, "current_timestamp() ON UPDATE current_timestamp()") == 0) + { + struct timeval tv; + char tmbuf[64]; + gettimeofday(&tv, 0); + MySQLTime time; + gmtSecToMySQLTime(tv.tv_sec, time, timeZone); + sprintf(tmbuf, "%04d-%02d-%02d %02d:%02d:%02d.%06ld", time.year, time.month, time.day, time.hour, time.minute, time.second, tv.tv_usec); + dataOrg = tmbuf; + dataOrgLen = strlen(tmbuf); + } + + status = 0; + const char* p; + p = dataOrg; + char fld[10]; + int64_t value = 0; + + if ( datetimeFormat != CALPONTDATETIME_ENUM ) + { + status = -1; + return value; + } + + unsigned int dataLen = dataOrgLen; + + if ((dataOrgLen > 0) && (dataOrg[0] == ' ')) + { + unsigned nblanks = 0; + + for (unsigned nn = 0; nn < dataOrgLen; nn++) + { + if (dataOrg[nn] == ' ') + nblanks++; + else + break; + } + + p = dataOrg + nblanks; + dataLen = dataOrgLen - nblanks; + } + + if ( dataLen < 10) + { + status = -1; + return value; + } + + int inYear, inMonth, inDay, inHour, inMinute, inSecond, inMicrosecond; + memcpy( fld, p, 4); + fld[4] = '\0'; + + inYear = strtol(fld, 0, 10); + + memcpy( fld, p + 5, 2); + fld[2] = '\0'; + + inMonth = strtol(fld, 0, 10); + + memcpy( fld, p + 8, 2); + fld[2] = '\0'; + + inDay = strtol(fld, 0, 10); + + inHour = 0; + inMinute = 0; + inSecond = 0; + inMicrosecond = 0; + + if (dataLen > 12) + { + // For backwards compatability we still allow leading blank + if ((!isdigit(p[11]) && (p[11] != ' ')) || + !isdigit(p[12])) + { + status = -1; + return value; + } + + memcpy( fld, p + 11, 2); + fld[2] = '\0'; + + inHour = strtol(fld, 0, 10); + + if (dataLen > 15) + { + if (!isdigit(p[14]) || !isdigit(p[15])) + { + status = -1; + return value; + } + + memcpy( fld, p + 14, 2); + fld[2] = '\0'; + + inMinute = strtol(fld, 0, 10); + + if (dataLen > 18) + { + if (!isdigit(p[17]) || !isdigit(p[18])) + { + status = -1; + return value; + } + + memcpy( fld, p + 17, 2); + fld[2] = '\0'; + + inSecond = strtol(fld, 0, 10); + + if (dataLen > 20) + { + unsigned int microFldLen = dataLen - 20; + + if (microFldLen > (sizeof(fld) - 1)) + microFldLen = sizeof(fld) - 1; + + memcpy( fld, p + 20, microFldLen); + fld[microFldLen] = '\0'; + inMicrosecond = strtol(fld, 0, 10); + } + } + } + } + + if ( isDateValid (inDay, inMonth, inYear) && + isDateTimeValid (inHour, inMinute, inSecond, inMicrosecond) ) + { + MySQLTime m_time; + m_time.year = inYear; + m_time.month = inMonth; + m_time.day = inDay; + m_time.hour = inHour; + m_time.minute = inMinute; + m_time.second = inSecond; + m_time.second_part = inMicrosecond; + + bool isValid = true; + int64_t seconds = mySQLTimeToGmtSec(m_time, timeZone, isValid); + + if (!isValid) + { + status = -1; + return value; + } + + TimeStamp timestamp; + timestamp.second = seconds; + timestamp.msecond = m_time.second_part; + + memcpy( &value, ×tamp, 8 ); + } + else + { + status = -1; + } + + return value; +} + //------------------------------------------------------------------------------ // Convert time string to binary time. Used by BulkLoad. // Most of this is taken from str_to_time in sql-common/my_time.c @@ -2086,6 +2351,14 @@ bool DataConvert::isColumnTimeValid( int64_t time ) return isTimeValid(dt.hour, dt.minute, dt.second, dt.msecond); } +bool DataConvert::isColumnTimeStampValid( int64_t timeStamp ) +{ + TimeStamp dt; + memcpy(&dt, &timeStamp, sizeof(uint64_t)); + + return isTimestampValid(dt.second, dt.msecond); +} + std::string DataConvert::dateToString( int datevalue ) { // @bug 4703 abandon multiple ostringstream's for conversion @@ -2121,6 +2394,34 @@ std::string DataConvert::datetimeToString( long long datetimevalue, long decima return buf; } +std::string DataConvert::timestampToString( long long timestampvalue, const std::string& timezone, long decimals ) +{ + // 10 is default which means we don't need microseconds + if (decimals > 6 || decimals < 0) + { + decimals = 0; + } + + TimeStamp timestamp(timestampvalue); + int64_t seconds = timestamp.second; + + MySQLTime time; + gmtSecToMySQLTime(seconds, time, timezone); + + const int TIMESTAMPTOSTRING_LEN = 28; // YYYY-MM-DD HH:MM:SS.mmmmmm\0 + char buf[TIMESTAMPTOSTRING_LEN]; + + sprintf(buf, "%04d-%02d-%02d %02d:%02d:%02d", time.year, time.month, time.day, time.hour, time.minute, time.second); + + if (timestamp.msecond && decimals) + { + // Pad start with zeros + sprintf(buf + strlen(buf), ".%0*d", (int)decimals, timestamp.msecond); + } + + return buf; +} + std::string DataConvert::timeToString( long long timevalue, long decimals ) { // 10 is default which means we don't need microseconds @@ -2175,6 +2476,20 @@ std::string DataConvert::datetimeToString1( long long datetimevalue ) return buf; } +std::string DataConvert::timestampToString1( long long timestampvalue, const std::string& timezone ) +{ + const int TIMESTAMPTOSTRING1_LEN = 22; // YYYYMMDDHHMMSSmmmmmm\0 + char buf[TIMESTAMPTOSTRING1_LEN]; + + TimeStamp timestamp(timestampvalue); + int64_t seconds = timestamp.second; + MySQLTime time; + gmtSecToMySQLTime(seconds, time, timezone); + + sprintf(buf, "%04d%02d%02d%02d%02d%02d%06d", time.year, time.month, time.day, time.hour, time.minute, time.second, timestamp.msecond); + return buf; +} + std::string DataConvert::timeToString1( long long datetimevalue ) { // @bug 4703 abandon multiple ostringstream's for conversion @@ -2408,6 +2723,11 @@ int64_t DataConvert::datetimeToInt(const string& datetime) return stringToDatetime(datetime); } +int64_t DataConvert::timestampToInt(const string& timestamp, const string& timeZone) +{ + return stringToTimestamp(timestamp, timeZone); +} + int64_t DataConvert::timeToInt(const string& time) { return stringToTime(time); @@ -2433,6 +2753,16 @@ int64_t DataConvert::stringToDatetime(const string& data, bool* date) return -1; } +int64_t DataConvert::stringToTimestamp(const string& data, const string& timeZone) +{ + TimeStamp aTimestamp; + + if ( stringToTimestampStruct( data, aTimestamp, timeZone ) ) + return *(reinterpret_cast(&aTimestamp)); + else + return -1; +} + /* This is really painful and expensive b/c it seems the input is not normalized or sanitized. That should really be done on ingestion. */ int64_t DataConvert::intToDate(int64_t data) @@ -2450,7 +2780,7 @@ int64_t DataConvert::intToDate(int64_t data) // this snprintf call causes a compiler warning b/c we're potentially copying a 20-digit # // into 15 bytes, however, that appears to be intentional. -#if defined(__GNUC__) && __GNUC__ >= 6 +#if defined(__GNUC__) && __GNUC__ >= 7 #pragma GCC diagnostic push #pragma GCC diagnostic ignored "-Wformat-truncation=" snprintf( buf, 15, "%llu", (long long unsigned int)data); @@ -2586,7 +2916,7 @@ int64_t DataConvert::intToDatetime(int64_t data, bool* date) // this snprintf call causes a compiler warning b/c we're potentially copying a 20-digit # // into 15 bytes, however, that appears to be intentional. -#if defined(__GNUC__) && __GNUC__ >= 6 +#if defined(__GNUC__) && __GNUC__ >= 7 #pragma GCC diagnostic push #pragma GCC diagnostic ignored "-Wformat-truncation=" snprintf( buf, 15, "%llu", (long long unsigned int)data); @@ -2724,7 +3054,7 @@ int64_t DataConvert::intToTime(int64_t data, bool fromString) // this snprintf call causes a compiler warning b/c we're potentially copying a 20-digit # // into 15 bytes, however, that appears to be intentional. -#if defined(__GNUC__) && __GNUC__ >= 6 +#if defined(__GNUC__) && __GNUC__ >= 7 #pragma GCC diagnostic push #pragma GCC diagnostic ignored "-Wformat-truncation=" snprintf( buf, 15, "%llu", (long long unsigned int)data); @@ -3019,6 +3349,7 @@ CalpontSystemCatalog::ColType DataConvert::convertUnionColType(vector 26) ? types[i].colWidth : 26; break; diff --git a/utils/dataconvert/dataconvert.h b/utils/dataconvert/dataconvert.h index d563f5fea..cb4aca771 100644 --- a/utils/dataconvert/dataconvert.h +++ b/utils/dataconvert/dataconvert.h @@ -106,6 +106,22 @@ const int64_t IDB_pow[19] = }; +const int32_t SECS_PER_MIN = 60; +const int32_t MINS_PER_HOUR = 60; +const int32_t HOURS_PER_DAY = 24; +const int32_t DAYS_PER_WEEK = 7; +const int32_t DAYS_PER_NYEAR = 365; +const int32_t DAYS_PER_LYEAR = 366; +const int32_t SECS_PER_HOUR = SECS_PER_MIN * MINS_PER_HOUR; +const int32_t SECS_PER_DAY = SECS_PER_HOUR * HOURS_PER_DAY; +const int32_t EPOCH_YEAR = 1970; +const int32_t MONS_PER_YEAR = 12; +const int32_t MAX_TIMESTAMP_YEAR = 2038; +const int32_t MIN_TIMESTAMP_YEAR = 1969; +const int32_t MAX_TIMESTAMP_VALUE = (1ULL << 31) - 1; +const int32_t MIN_TIMESTAMP_VALUE = 0; + + namespace dataconvert { @@ -116,6 +132,485 @@ enum CalpontDateTimeFormat CALPONTTIME_ENUM = 3 }; +/** @brief a structure that represents a timestamp in broken down + * representation + */ +struct MySQLTime +{ + unsigned int year, month, day, hour, minute, second; + unsigned long second_part; + CalpontDateTimeFormat time_type; + void reset() + { + year = month = day = 0; + hour = minute = second = second_part = 0; + time_type = CALPONTDATETIME_ENUM; + } +}; + +/** + * This function converts the timezone represented as a string + * in the format "+HH:MM" or "-HH:MM" to a signed offset in seconds + * Most of this code is taken from tztime.cc:str_to_offset + */ +inline +bool timeZoneToOffset(const char *str, std::string::size_type length, long *offset) +{ + const char *end = str + length; + bool negative; + unsigned long number_tmp; + long offset_tmp; + + if (length < 4) + return 1; + + if (*str == '+') + negative = 0; + else if (*str == '-') + negative = 1; + else + return 1; + str++; + + number_tmp = 0; + + while (str < end && isdigit(*str)) + { + number_tmp = number_tmp * 10 + *str - '0'; + str++; + } + + if (str + 1 >= end || *str != ':') + return 1; + str++; + + offset_tmp = number_tmp * 60L; + number_tmp = 0; + + while (str < end && isdigit(*str)) + { + number_tmp = number_tmp * 10 + *str - '0'; + str++; + } + + if (str != end) + return 1; + + offset_tmp = (offset_tmp + number_tmp) * 60L; + + if (negative) + offset_tmp = -offset_tmp; + + /* + Check if offset is in range prescribed by standard + (from -12:59 to 13:00). + */ + + if (number_tmp > 59 || offset_tmp < -13 * 3600L + 1 || + offset_tmp > 13 * 3600L) + return 1; + + *offset = offset_tmp; + + return 0; +} + +const int32_t year_lengths[2] = +{ + DAYS_PER_NYEAR, DAYS_PER_LYEAR +}; + +const unsigned int mon_lengths[2][MONS_PER_YEAR]= +{ + { 31, 28, 31, 30, 31, 30, 31, 31, 30, 31, 30, 31 }, + { 31, 29, 31, 30, 31, 30, 31, 31, 30, 31, 30, 31 } +}; + +const unsigned int mon_starts[2][MONS_PER_YEAR]= +{ + { 0, 31, 59, 90, 120, 151, 181, 212, 243, 273, 304, 334 }, + { 0, 31, 60, 91, 121, 152, 182, 213, 244, 274, 305, 335 } +}; + +inline int32_t leapsThruEndOf(int32_t year) +{ + return (year / 4 - year / 100 + year / 400); +} + +inline bool isLeapYear ( int year) +{ + if ( year % 400 == 0 ) + return true; + + if ( ( year % 4 == 0 ) && ( year % 100 != 0 ) ) + return true; + + return false; +} + +static uint32_t daysInMonth[13] = {31, 28, 31, 30, 31, 30, 31, 31, 30, 31, 30, 31, 0}; + +inline uint32_t getDaysInMonth(uint32_t month, int year) +{ + if (month < 1 || month > 12) + return 0; + + uint32_t days = daysInMonth[month - 1]; + + if ((month == 2) && isLeapYear(year)) + days++; + + return days; +} + +inline +bool isDateValid ( int day, int month, int year) +{ + bool valid = true; + + if ( year == 0 && month == 0 && year == 0 ) + { + return true; + } + + int daycheck = getDaysInMonth( month, year ); + + if ( ( year < 1000 ) || ( year > 9999 ) ) + valid = false; + else if ( month < 1 || month > 12 ) + valid = false; + else if ( day < 1 || day > daycheck ) + valid = false; + + return ( valid ); +} + +inline +bool isDateTimeValid ( int hour, int minute, int second, int microSecond) +{ + bool valid = false; + + if ( hour >= 0 && hour <= 24 ) + { + if ( minute >= 0 && minute < 60 ) + { + if ( second >= 0 && second < 60 ) + { + if ( microSecond >= 0 && microSecond <= 999999 ) + { + valid = true; + } + } + } + } + + return valid; +} + +inline +bool isTimeValid ( int hour, int minute, int second, int microSecond) +{ + bool valid = false; + + if ( hour >= -838 && hour <= 838 ) + { + if ( minute >= 0 && minute < 60 ) + { + if ( second >= 0 && second < 60 ) + { + if ( microSecond >= 0 && microSecond <= 999999 ) + { + valid = true; + } + } + } + } + + return valid; +} + +inline +bool isTimestampValid ( uint64_t second, uint64_t microsecond ) +{ + bool valid = false; + + // MariaDB server currently sets the upper limit on timestamp to + // 0x7FFFFFFF. So enforce the same restriction here. + // TODO: We however store the seconds portion of the timestamp in + // 44 bits, so change this limit when the server supports higher values. + if ( second >= MIN_TIMESTAMP_VALUE && second <= MAX_TIMESTAMP_VALUE ) + { + if ( microsecond >= 0 && microsecond <= 999999 ) + { + valid = true; + } + } + + return valid; +} + +/** + * @brief converts a timestamp (seconds in UTC since Epoch) + * to broken-down representation. Most of this code is taken + * from sec_to_TIME and Time_zone_system::gmt_sec_to_TIME + * functions in tztime.cc in the server + * + * @param seconds the value to be converted + * @param time the broken-down representation of the timestamp + * @param timeZone a string with the server timezone of the machine + * which initiated the query + */ +inline void gmtSecToMySQLTime(int64_t seconds, MySQLTime& time, + const std::string& timeZone) +{ + if (seconds == 0) + { + time.reset(); + return; + } + + if (timeZone == "SYSTEM") + { + struct tm tmp_tm; + time_t tmp_t = (time_t)seconds; + localtime_r(&tmp_t, &tmp_tm); + time.second_part = 0; + time.year = (int) ((tmp_tm.tm_year + 1900) % 10000); + time.month = (int) tmp_tm.tm_mon + 1; + time.day = (int) tmp_tm.tm_mday; + time.hour = (int) tmp_tm.tm_hour; + time.minute = (int) tmp_tm.tm_min; + time.second = (int) tmp_tm.tm_sec; + time.time_type = CALPONTDATETIME_ENUM; + if (time.second == 60 || time.second == 61) + time.second = 59; + } + else + { + long offset; + if (timeZoneToOffset(timeZone.c_str(), timeZone.size(), &offset)) + { + time.reset(); + return; + } + + int64_t days; + int32_t rem; + int32_t y; + int32_t yleap; + const unsigned int *ip; + + days = (int64_t) (seconds / SECS_PER_DAY); + rem = (int32_t) (seconds % SECS_PER_DAY); + + rem += offset; + while (rem < 0) + { + rem += SECS_PER_DAY; + days--; + } + while (rem >= SECS_PER_DAY) + { + rem -= SECS_PER_DAY; + days++; + } + time.hour = (unsigned int) (rem / SECS_PER_HOUR); + rem = rem % SECS_PER_HOUR; + time.minute = (unsigned int) (rem / SECS_PER_MIN); + time.second = (unsigned int) (rem % SECS_PER_MIN); + + y = EPOCH_YEAR; + while (days < 0 || days >= (int64_t) (year_lengths[yleap = isLeapYear(y)])) + { + int32_t newy; + + newy = y + days / DAYS_PER_NYEAR; + if (days < 0) + newy--; + days -= (newy - y) * DAYS_PER_NYEAR + + leapsThruEndOf(newy - 1) - + leapsThruEndOf(y - 1); + y = newy; + } + time.year = y; + + ip = mon_lengths[yleap]; + for (time.month = 0; days >= (int64_t) ip[time.month]; time.month++) + days -= (int64_t) ip[time.month]; + time.month++; + time.day = (unsigned int) (days + 1); + + time.second_part = 0; + time.time_type = CALPONTDATETIME_ENUM; + } +} + +/** + * @brief function that provides a rough estimate if a broken-down + * representation of timestamp is in range + * + * @param t the broken-down representation of timestamp + */ +inline bool validateTimestampRange(const MySQLTime& t) +{ + if ((t.year > MAX_TIMESTAMP_YEAR || t.year < MIN_TIMESTAMP_YEAR) || + (t.year == MAX_TIMESTAMP_YEAR && (t.month > 1 || t.day > 19))) + return false; + + return true; +} + +inline +int64_t secSinceEpoch(int year, int month, int day, int hour, int min, int sec) +{ + int64_t days = (year - EPOCH_YEAR) * DAYS_PER_NYEAR + + leapsThruEndOf(year - 1) - + leapsThruEndOf(EPOCH_YEAR - 1); + days += mon_starts[isLeapYear(year)][month - 1]; + days += day - 1; + + return ((days * HOURS_PER_DAY + hour) * MINS_PER_HOUR + min) * + SECS_PER_MIN + sec; +} + +// This is duplicate of funchelpers.h:calc_mysql_daynr, +// with one additional function parameter +inline uint32_t calc_mysql_daynr( uint32_t year, uint32_t month, uint32_t day, bool& isValid ) +{ + int temp; + int y = year; + long delsum; + + if ( !isDateValid( day, month, year ) ) + { + isValid = false; + return 0; + } + + delsum = (long) (365 * y + 31 * ((int) month - 1) + (int) day); + + if (month <= 2) + y--; + else + delsum -= (long) ((int) month * 4 + 23) / 10; + + temp = (int) ((y / 100 + 1) * 3) / 4; + + return delsum + (int) y / 4 - temp; +} + +/** + * @brief converts a timestamp from broken-down representation + * to seconds since UTC epoch + * + * @param time the broken-down representation of the timestamp + @param timeZone a string with the server timezone of the machine + which initiated the query + */ +inline int64_t mySQLTimeToGmtSec(const MySQLTime& time, + const std::string& timeZone, bool& isValid) +{ + int64_t seconds; + + if (!validateTimestampRange(time)) + { + isValid = false; + return 0; + } + + if (timeZone == "SYSTEM") + { + // This is mirror of code in func_unix_timestamp.cpp + uint32_t loop; + time_t tmp_t = 0; + int shift = 0; + struct tm* l_time, tm_tmp; + int64_t diff; + localtime_r(&tmp_t, &tm_tmp); + // Get the system timezone offset at 0 seconds since epoch + int64_t my_time_zone = tm_tmp.tm_gmtoff; + int day = time.day; + + if ((time.year == MAX_TIMESTAMP_YEAR) && (time.month == 1) && (day > 4)) + { + day -= 2; + shift = 2; + } + + tmp_t = (time_t)(((calc_mysql_daynr(time.year, time.month, day, isValid) - + 719528) * 86400L + (int64_t)time.hour * 3600L + + (int64_t)(time.minute * 60 + time.second)) - (time_t)my_time_zone); + if (!isValid) + return 0; + + localtime_r(&tmp_t, &tm_tmp); + l_time = &tm_tmp; + + for (loop = 0; loop < 2 && (time.hour != (uint32_t) l_time->tm_hour || + time.minute != (uint32_t) l_time->tm_min || + time.second != (uint32_t)l_time->tm_sec); loop++) + { + int days = day - l_time->tm_mday; + + if (days < -1) + days = 1; /* Month has wrapped */ + else if (days > 1) + days = -1; + + diff = (3600L * (int64_t) (days * 24 + ((int) time.hour - (int) l_time->tm_hour)) + + (int64_t) (60 * ((int) time.minute - (int) l_time->tm_min)) + + (int64_t) ((int) time.second - (int) l_time->tm_sec)); + tmp_t += (time_t) diff; + localtime_r(&tmp_t, &tm_tmp); + l_time = &tm_tmp; + } + + if (loop == 2 && time.hour != (uint32_t)l_time->tm_hour) + { + int days = day - l_time->tm_mday; + + if (days < -1) + days = 1; /* Month has wrapped */ + else if (days > 1) + days = -1; + + diff = (3600L * (int64_t) (days * 24 + ((int) time.hour - (int) l_time->tm_hour)) + + (int64_t) (60 * ((int) time.minute - (int) l_time->tm_min)) + + (int64_t) ((int) time.second - (int) l_time->tm_sec)); + + if (diff == 3600) + tmp_t += 3600 - time.minute * 60 - time.second; /* Move to next hour */ + else if (diff == -3600) + tmp_t -= time.minute * 60 + time.second; /* Move to previous hour */ + } + + + /* shift back, if we were dealing with boundary dates */ + tmp_t += shift * 86400L; + + seconds = (int64_t)tmp_t; + } + else + { + long offset; + if (timeZoneToOffset(timeZone.c_str(), timeZone.size(), &offset)) + { + isValid = false; + return 0; + } + seconds = secSinceEpoch(time.year, time.month, time.day, + time.hour, time.minute, time.second) - offset; + } + + /* make sure we have legit timestamps (i.e. we didn't over/underflow anywhere above) */ + if (seconds >= MIN_TIMESTAMP_VALUE && seconds <= MAX_TIMESTAMP_VALUE) + return seconds; + + isValid = false; + return 0; + +} + /** @brief a structure to hold a date */ @@ -258,96 +753,44 @@ int64_t Time::convertToMySQLint() const } } -static uint32_t daysInMonth[13] = {31, 28, 31, 30, 31, 30, 31, 31, 30, 31, 30, 31, 0}; - -inline bool isLeapYear ( int year) +/** @brief a structure to hold a timestamp + */ +struct TimeStamp { - if ( year % 400 == 0 ) - return true; + unsigned msecond : 20; + unsigned long long second : 44; + // NULL column value = 0xFFFFFFFFFFFFFFFE + TimeStamp( ) : + msecond(0xFFFFE), second(0xFFFFFFFFFFF) {} + // Construct a TimeStamp from a 64 bit integer Calpont timestamp. + TimeStamp(uint64_t val) : + msecond(val & 0xFFFFF), second(val >> 20) {} + TimeStamp(unsigned msec, unsigned long long sec) : + msecond(msec), second(sec) {} - if ( ( year % 4 == 0 ) && ( year % 100 != 0 ) ) - return true; + int64_t convertToMySQLint(const std::string& timeZone) const; + void reset(); +}; - return false; -} - -inline uint32_t getDaysInMonth(uint32_t month, int year) +inline +int64_t TimeStamp::convertToMySQLint(const std::string& timeZone) const { - if (month < 1 || month > 12) - return 0; + const int TIMESTAMPTOSTRING1_LEN = 22; // YYYYMMDDHHMMSSmmmmmm\0 + char buf[TIMESTAMPTOSTRING1_LEN]; - uint32_t days = daysInMonth[month - 1]; + MySQLTime time; + gmtSecToMySQLTime(second, time, timeZone); - if ((month == 2) && isLeapYear(year)) - days++; + sprintf(buf, "%04d%02d%02d%02d%02d%02d", time.year, time.month, time.day, time.hour, time.minute, time.second); - return days; + return (int64_t) atoll(buf); } inline -bool isDateValid ( int day, int month, int year) +void TimeStamp::reset() { - bool valid = true; - - if ( year == 0 && month == 0 && year == 0 ) - { - return true; - } - - int daycheck = getDaysInMonth( month, year ); - - if ( ( year < 1000 ) || ( year > 9999 ) ) - valid = false; - else if ( month < 1 || month > 12 ) - valid = false; - else if ( day < 1 || day > daycheck ) - valid = false; - - return ( valid ); -} - -inline -bool isDateTimeValid ( int hour, int minute, int second, int microSecond) -{ - bool valid = false; - - if ( hour >= 0 && hour <= 24 ) - { - if ( minute >= 0 && minute < 60 ) - { - if ( second >= 0 && second < 60 ) - { - if ( microSecond >= 0 && microSecond <= 999999 ) - { - valid = true; - } - } - } - } - - return valid; -} - -inline -bool isTimeValid ( int hour, int minute, int second, int microSecond) -{ - bool valid = false; - - if ( hour >= -838 && hour <= 838 ) - { - if ( minute >= 0 && minute < 60 ) - { - if ( second >= 0 && second < 60 ) - { - if ( microSecond >= 0 && microSecond <= 999999 ) - { - valid = true; - } - } - } - } - - return valid; + msecond = 0xFFFFE; + second = 0xFFFFFFFFFFF; } inline @@ -412,7 +855,7 @@ public: * @param data the columns string representation of it's data */ EXPORT static boost::any convertColumnData( const execplan::CalpontSystemCatalog::ColType& colType, - const std::string& dataOrig, bool& bSaturate, + const std::string& dataOrig, bool& bSaturate, const std::string& timeZone, bool nulFlag = false, bool noRoundup = false, bool isUpdate = false); /** @@ -433,6 +876,15 @@ public: EXPORT static std::string datetimeToString( long long datetimevalue, long decimals = 0 ); static inline void datetimeToString( long long datetimevalue, char* buf, unsigned int buflen, long decimals = 0 ); + /** + * @brief convert a columns data from native format to a string + * + * @param type the columns database type + * @param data the columns string representation of it's data + */ + EXPORT static std::string timestampToString( long long timestampvalue, const std::string& timezone, long decimals = 0 ); + static inline void timestampToString( long long timestampvalue, char* buf, unsigned int buflen, const std::string& timezone, long decimals = 0 ); + /** * @brief convert a columns data from native format to a string * @@ -460,6 +912,15 @@ public: EXPORT static std::string datetimeToString1( long long datetimevalue ); static inline void datetimeToString1( long long datetimevalue, char* buf, unsigned int buflen ); + /** + * @brief convert a columns data from native format to a string + * + * @param type the columns database type + * @param data the columns string representation of it's data + */ + EXPORT static std::string timestampToString1( long long timestampvalue, const std::string& timezone ); + static inline void timestampToString1( long long timestampvalue, char* buf, unsigned int buflen, const std::string& timezone ); + /** * @brief convert a columns data from native format to a string * @@ -502,6 +963,21 @@ public: CalpontDateTimeFormat datetimeFormat, int& status, unsigned int dataOrgLen ); + /** + * @brief convert a timestamp column data, represented as a string, + * to it's native format. This function is for bulkload to use. + * + * @param dataOrg the columns string representation of it's data + * @param datetimeFormat the format the date value in + * @param status 0 - success, -1 - fail + * @param dataOrgLen length specification of dataOrg + * @param timeZone the timezone used for conversion to native format + */ + EXPORT static int64_t convertColumnTimestamp( const char* dataOrg, + CalpontDateTimeFormat datetimeFormat, + int& status, unsigned int dataOrgLen, + const std::string& timeZone ); + /** * @brief convert a time column data, represented as a string, * to it's native format. This function is for bulkload to use. @@ -521,6 +997,7 @@ public: */ EXPORT static bool isColumnDateTimeValid( int64_t dateTime ); EXPORT static bool isColumnTimeValid( int64_t time ); + EXPORT static bool isColumnTimeStampValid( int64_t timeStamp ); EXPORT static bool isNullData(execplan::ColumnResult* cr, int rownum, execplan::CalpontSystemCatalog::ColType colType); static inline std::string decimalToString(int64_t value, uint8_t scale, execplan::CalpontSystemCatalog::ColDataType colDataType); @@ -536,6 +1013,8 @@ public: EXPORT static int64_t stringToDate(const std::string& data); // convert string to datetime EXPORT static int64_t stringToDatetime(const std::string& data, bool* isDate = NULL); + // convert string to timestamp + EXPORT static int64_t stringToTimestamp(const std::string& data, const std::string& timeZone); // convert integer to date EXPORT static int64_t intToDate(int64_t data); // convert integer to datetime @@ -546,6 +1025,7 @@ public: EXPORT static int64_t dateToInt(const std::string& date); // convert string to datetime. alias to datetimeToInt EXPORT static int64_t datetimeToInt(const std::string& datetime); + EXPORT static int64_t timestampToInt(const std::string& timestamp, const std::string& timeZone); EXPORT static int64_t timeToInt(const std::string& time); EXPORT static int64_t stringToTime (const std::string& data); // bug4388, union type conversion @@ -591,6 +1071,31 @@ inline void DataConvert::datetimeToString( long long datetimevalue, char* buf, u } } +inline void DataConvert::timestampToString( long long timestampvalue, char* buf, unsigned int buflen, const std::string& timezone, long decimals ) +{ + // 10 is default which means we don't need microseconds + if (decimals > 6 || decimals < 0) + { + decimals = 0; + } + + TimeStamp timestamp(timestampvalue); + int64_t seconds = timestamp.second; + + MySQLTime time; + gmtSecToMySQLTime(seconds, time, timezone); + + snprintf( buf, buflen, "%04d-%02d-%02d %02d:%02d:%02d", + time.year, time.month, time.day, + time.hour, time.minute, time.second + ); + + if (timestamp.msecond || decimals) + { + snprintf(buf + strlen(buf), buflen - strlen(buf), ".%0*d", (int)decimals, timestamp.msecond); + } +} + inline void DataConvert::timeToString( long long timevalue, char* buf, unsigned int buflen, long decimals ) { // 10 is default which means we don't need microseconds @@ -655,6 +1160,20 @@ inline void DataConvert::datetimeToString1( long long datetimevalue, char* buf, ); } +inline void DataConvert::timestampToString1( long long timestampvalue, char* buf, unsigned int buflen, const std::string& timezone ) +{ + TimeStamp timestamp(timestampvalue); + int64_t seconds = timestamp.second; + + MySQLTime time; + gmtSecToMySQLTime(seconds, time, timezone); + + snprintf( buf, buflen, "%04d%02d%02d%02d%02d%02d", + time.year, time.month, time.day, + time.hour, time.minute, time.second + ); +} + inline void DataConvert::timeToString1( long long timevalue, char* buf, unsigned int buflen ) { // Handle negative correctly @@ -675,7 +1194,7 @@ inline void DataConvert::timeToString1( long long timevalue, char* buf, unsigned } // this snprintf call causes a compiler warning b/c buffer size is less // then maximum string size. -#if defined(__GNUC__) && __GNUC__ >= 6 +#if defined(__GNUC__) && __GNUC__ >= 7 #pragma GCC diagnostic push #pragma GCC diagnostic ignored "-Wformat-truncation=" snprintf( buf, buflen, "%02d%02d%02d", diff --git a/utils/funcexp/func_add_time.cpp b/utils/funcexp/func_add_time.cpp index 3974190b4..a95eff53d 100644 --- a/utils/funcexp/func_add_time.cpp +++ b/utils/funcexp/func_add_time.cpp @@ -80,6 +80,9 @@ int64_t Func_add_time::getDatetimeIntVal(rowgroup::Row& row, bool& isNull, CalpontSystemCatalog::ColType& ct) { + if (parm[0]->data()->resultType().colDataType == CalpontSystemCatalog::TIMESTAMP) + return getTimestampIntVal(row, parm, isNull, ct); + int64_t val1 = parm[0]->data()->getDatetimeIntVal(row, isNull); if (isNull) @@ -155,6 +158,90 @@ int64_t Func_add_time::getDatetimeIntVal(rowgroup::Row& row, return addTime(dt1, t2); } +int64_t Func_add_time::getTimestampIntVal(rowgroup::Row& row, + FunctionParm& parm, + bool& isNull, + CalpontSystemCatalog::ColType& ct) +{ + int64_t val1 = parm[0]->data()->getTimestampIntVal(row, isNull); + + if (isNull) + return -1; + + // Adding a zero date to a time is always NULL + if (val1 == 0) + { + isNull = true; + return -1; + } + + const string& val2 = parm[1]->data()->getStrVal(row, isNull); + int sign = parm[2]->data()->getIntVal(row, isNull); + DateTime dt1; + TimeStamp timestamp(val1); + int64_t seconds = timestamp.second; + MySQLTime m_time; + gmtSecToMySQLTime(seconds, m_time, fTimeZone); + dt1.year = m_time.year; + dt1.month = m_time.month; + dt1.day = m_time.day; + dt1.hour = m_time.hour; + dt1.minute = m_time.minute; + dt1.second = m_time.second; + dt1.msecond = timestamp.msecond; + + int64_t time = DataConvert::stringToTime(val2); + + if (time == -1) + { + isNull = true; + return -1; + } + + Time t2 = *(reinterpret_cast(&time)); + + // MySQL TIME type range '-838:59:59' and '838:59:59' + if (t2.minute > 59 || t2.second > 59 || t2.msecond > 999999) + { + isNull = true; + return -1; + } + + int val_sign = 1; + + if (t2.hour < 0) + { + val_sign = -1; + } + + if (abs(t2.hour) > 838) + { + t2.hour = 838; + t2.minute = 59; + t2.second = 59; + t2.msecond = 999999; + } + + if (val_sign * sign < 0) + { + t2.hour = -abs(t2.hour); + t2.minute = -abs(t2.minute); + t2.second = -abs(t2.second); + t2.msecond = -abs(t2.msecond); + } + else + { + t2.hour = abs(t2.hour); + t2.minute = abs(t2.minute); + t2.second = abs(t2.second); + t2.msecond = abs(t2.msecond); + } + + t2.day = 0; + + return addTime(dt1, t2); +} + int64_t Func_add_time::getTimeIntVal(rowgroup::Row& row, FunctionParm& parm, bool& isNull, diff --git a/utils/funcexp/func_between.cpp b/utils/funcexp/func_between.cpp index 43bde349d..44908ad4b 100644 --- a/utils/funcexp/func_between.cpp +++ b/utils/funcexp/func_between.cpp @@ -157,6 +157,24 @@ inline bool getBool(rowgroup::Row& row, numericLE(val, pm[2]->data()->getDatetimeIntVal(row, isNull)); } + case execplan::CalpontSystemCatalog::TIMESTAMP: + { + int64_t val = pm[0]->data()->getTimestampIntVal(row, isNull); + + if (notBetween) + { + if (!numericGE(val, pm[1]->data()->getTimestampIntVal(row, isNull)) && !isNull) + return true; + + isNull = false; + return (!numericLE(val, pm[2]->data()->getTimestampIntVal(row, isNull)) && !isNull); + } + + return !isNull && + numericGE(val, pm[1]->data()->getTimestampIntVal(row, isNull)) && + numericLE(val, pm[2]->data()->getTimestampIntVal(row, isNull)); + } + case execplan::CalpontSystemCatalog::TIME: { // Shift out unused day for compare @@ -286,6 +304,7 @@ CalpontSystemCatalog::ColType Func_between::operationType( FunctionParm& fp, Cal fp[i]->data()->resultType().colDataType != CalpontSystemCatalog::VARCHAR) || ct.colDataType == CalpontSystemCatalog::DATE || ct.colDataType == CalpontSystemCatalog::DATETIME || + ct.colDataType == CalpontSystemCatalog::TIMESTAMP || ct.colDataType == CalpontSystemCatalog::TIME) { allString = false; @@ -314,6 +333,22 @@ CalpontSystemCatalog::ColType Func_between::operationType( FunctionParm& fp, Cal } } } + else if (op.operationType().colDataType == CalpontSystemCatalog::TIMESTAMP) + { + ConstantColumn* cc = NULL; + + for (uint32_t i = 1; i < fp.size(); i++) + { + cc = dynamic_cast(fp[i]->data()); + + if (cc) + { + Result result = cc->result(); + result.intVal = dataconvert::DataConvert::timestampToInt(result.strVal, fTimeZone); + cc->result(result); + } + } + } else if (op.operationType().colDataType == CalpontSystemCatalog::TIME) { ConstantColumn* cc = NULL; diff --git a/utils/funcexp/func_bitand.cpp b/utils/funcexp/func_bitand.cpp index 69429bfb8..64fa4078b 100644 --- a/utils/funcexp/func_bitand.cpp +++ b/utils/funcexp/func_bitand.cpp @@ -157,6 +157,17 @@ int64_t Func_bitand::getIntVal(Row& row, } break; + case execplan::CalpontSystemCatalog::TIMESTAMP: + { + TimeStamp timestamp(parm[i]->data()->getTimestampIntVal(row, isNull)); + int64_t seconds = timestamp.second; + MySQLTime m_time; + gmtSecToMySQLTime(seconds, m_time, fTimeZone); + + values.push_back((m_time.month * 100000000000000) + (m_time.day * 1000000000000) + (m_time.hour * 10000000000) + (m_time.minute * 100000000) + (m_time.second * 1000000) + timestamp.msecond); + } + break; + case execplan::CalpontSystemCatalog::TIME: { int64_t time = parm[i]->data()->getTimeIntVal(row, isNull); diff --git a/utils/funcexp/func_bitwise.cpp b/utils/funcexp/func_bitwise.cpp index 752c28561..2cc3dbb77 100644 --- a/utils/funcexp/func_bitwise.cpp +++ b/utils/funcexp/func_bitwise.cpp @@ -55,7 +55,8 @@ bool getUIntValFromParm( Row& row, const execplan::SPTP& parm, uint64_t& value, - bool& isNull) + bool& isNull, + const string& timeZone) { switch (parm->data()->resultType().colDataType) { @@ -143,6 +144,15 @@ bool getUIntValFromParm( } break; + case execplan::CalpontSystemCatalog::TIMESTAMP: + { + int64_t time = parm->data()->getTimestampIntVal(row, isNull); + + TimeStamp dt(time); + value = dt.convertToMySQLint(timeZone); + } + break; + case execplan::CalpontSystemCatalog::TIME: { int64_t time = parm->data()->getTimeIntVal(row, isNull); @@ -189,8 +199,8 @@ int64_t Func_bitand::getIntVal(Row& row, uint64_t val1 = 0; uint64_t val2 = 0; - if (!getUIntValFromParm(row, parm[0], val1, isNull) || - !getUIntValFromParm(row, parm[1], val2, isNull)) + if (!getUIntValFromParm(row, parm[0], val1, isNull, fTimeZone) || + !getUIntValFromParm(row, parm[1], val2, isNull, fTimeZone)) { std::ostringstream oss; oss << "bitand: datatype of " << execplan::colDataTypeToString(operationColType.colDataType); @@ -225,8 +235,8 @@ int64_t Func_leftshift::getIntVal(Row& row, uint64_t val1 = 0; uint64_t val2 = 0; - if (!getUIntValFromParm(row, parm[0], val1, isNull) || - !getUIntValFromParm(row, parm[1], val2, isNull)) + if (!getUIntValFromParm(row, parm[0], val1, isNull, fTimeZone) || + !getUIntValFromParm(row, parm[1], val2, isNull, fTimeZone)) { std::ostringstream oss; oss << "leftshift: datatype of " << execplan::colDataTypeToString(operationColType.colDataType); @@ -261,8 +271,8 @@ int64_t Func_rightshift::getIntVal(Row& row, uint64_t val1 = 0; uint64_t val2 = 0; - if (!getUIntValFromParm(row, parm[0], val1, isNull) || - !getUIntValFromParm(row, parm[1], val2, isNull)) + if (!getUIntValFromParm(row, parm[0], val1, isNull, fTimeZone) || + !getUIntValFromParm(row, parm[1], val2, isNull, fTimeZone)) { std::ostringstream oss; oss << "rightshift: datatype of " << execplan::colDataTypeToString(operationColType.colDataType); @@ -297,8 +307,8 @@ int64_t Func_bitor::getIntVal(Row& row, uint64_t val1 = 0; uint64_t val2 = 0; - if (!getUIntValFromParm(row, parm[0], val1, isNull) || - !getUIntValFromParm(row, parm[1], val2, isNull)) + if (!getUIntValFromParm(row, parm[0], val1, isNull, fTimeZone) || + !getUIntValFromParm(row, parm[1], val2, isNull, fTimeZone)) { std::ostringstream oss; oss << "bitor: datatype of " << execplan::colDataTypeToString(operationColType.colDataType); @@ -341,8 +351,8 @@ int64_t Func_bitxor::getIntVal(Row& row, uint64_t val1 = 0; uint64_t val2 = 0; - if (!getUIntValFromParm(row, parm[0], val1, isNull) || - !getUIntValFromParm(row, parm[1], val2, isNull)) + if (!getUIntValFromParm(row, parm[0], val1, isNull, fTimeZone) || + !getUIntValFromParm(row, parm[1], val2, isNull, fTimeZone)) { std::ostringstream oss; oss << "bitxor: datatype of " << execplan::colDataTypeToString(operationColType.colDataType); diff --git a/utils/funcexp/func_case.cpp b/utils/funcexp/func_case.cpp index 571c66ec7..22705cdd0 100644 --- a/utils/funcexp/func_case.cpp +++ b/utils/funcexp/func_case.cpp @@ -107,6 +107,27 @@ inline uint64_t simple_case_cmp(Row& row, break; } + case execplan::CalpontSystemCatalog::TIMESTAMP: + { + int64_t ev = parm[n]->data()->getTimestampIntVal(row, isNull); + + if (isNull) + break; + + for (i = 1; i <= whereCount; i++) + { + if (ev == parm[i]->data()->getTimestampIntVal(row, isNull) && !isNull) + { + foundIt = true; + break; + } + else + isNull = false; + } + + break; + } + case execplan::CalpontSystemCatalog::TIME: { int64_t ev = parm[n]->data()->getTimeIntVal(row, isNull); @@ -574,6 +595,20 @@ int64_t Func_simple_case::getDatetimeIntVal(rowgroup::Row& row, } +int64_t Func_simple_case::getTimestampIntVal(rowgroup::Row& row, + FunctionParm& parm, + bool& isNull, + execplan::CalpontSystemCatalog::ColType& op_ct) +{ + uint64_t i = simple_case_cmp(row, parm, isNull, op_ct); + + if (isNull) + return joblist::TIMESTAMPNULL; + + return parm[i]->data()->getTimestampIntVal(row, isNull); +} + + int64_t Func_simple_case::getTimeIntVal(rowgroup::Row& row, FunctionParm& parm, bool& isNull, @@ -726,6 +761,20 @@ int64_t Func_searched_case::getDatetimeIntVal(rowgroup::Row& row, } +int64_t Func_searched_case::getTimestampIntVal(rowgroup::Row& row, + FunctionParm& parm, + bool& isNull, + execplan::CalpontSystemCatalog::ColType& op_ct) +{ + uint64_t i = simple_case_cmp(row, parm, isNull, op_ct); + + if (isNull) + return joblist::TIMESTAMPNULL; + + return parm[i]->data()->getTimestampIntVal(row, isNull); +} + + int64_t Func_searched_case::getTimeIntVal(rowgroup::Row& row, FunctionParm& parm, bool& isNull, diff --git a/utils/funcexp/func_cast.cpp b/utils/funcexp/func_cast.cpp index 97c5db075..d2db69144 100644 --- a/utils/funcexp/func_cast.cpp +++ b/utils/funcexp/func_cast.cpp @@ -212,6 +212,15 @@ int64_t Func_cast_signed::getIntVal(Row& row, } break; + case execplan::CalpontSystemCatalog::TIMESTAMP: + { + int64_t time = parm[0]->data()->getTimestampIntVal(row, isNull); + + TimeStamp dt(time); + return dt.convertToMySQLint(fTimeZone); + } + break; + case execplan::CalpontSystemCatalog::TIME: { int64_t time = parm[0]->data()->getTimeIntVal(row, isNull); @@ -363,6 +372,15 @@ uint64_t Func_cast_unsigned::getUintVal(Row& row, } break; + case execplan::CalpontSystemCatalog::TIMESTAMP: + { + int64_t time = parm[0]->data()->getTimestampIntVal(row, isNull); + + TimeStamp dt(time); + return dt.convertToMySQLint(fTimeZone); + } + break; + case execplan::CalpontSystemCatalog::TIME: { int64_t time = parm[0]->data()->getTimeIntVal(row, isNull); @@ -487,6 +505,12 @@ string Func_cast_char::getStrVal(Row& row, } break; + case execplan::CalpontSystemCatalog::TIMESTAMP: + { + return dataconvert::DataConvert::timestampToString(parm[0]->data()->getTimestampIntVal(row, isNull), fTimeZone).substr(0, length); + } + break; + default: { std::ostringstream oss; @@ -647,6 +671,13 @@ int32_t Func_cast_date::getDateIntVal(rowgroup::Row& row, { return parm[0]->data()->getDateIntVal(row, isNull); } + case execplan::CalpontSystemCatalog::TIMESTAMP: + { + int64_t val1 = parm[0]->data()->getTimestampIntVal(row, isNull); + string value = dataconvert::DataConvert::timestampToString(val1, fTimeZone); + value = value.substr(0, 10); + return dataconvert::DataConvert::stringToDate(value); + } case execplan::CalpontSystemCatalog::TIME: { int64_t val1; @@ -765,6 +796,22 @@ int64_t Func_cast_date::getDatetimeIntVal(rowgroup::Row& row, val1.msecond = 0; return *(reinterpret_cast(&val1)); } + case execplan::CalpontSystemCatalog::TIMESTAMP: + { + TimeStamp timestamp(parm[0]->data()->getTimestampIntVal(row, isNull)); + int64_t seconds = timestamp.second; + MySQLTime m_time; + gmtSecToMySQLTime(seconds, m_time, fTimeZone); + DateTime dt; + dt.year = m_time.year; + dt.month = m_time.month; + dt.day = m_time.day; + dt.hour = 0; + dt.minute = 0; + dt.second = 0; + dt.msecond = 0; + return *(reinterpret_cast(&dt)); + } case CalpontSystemCatalog::TIME: { DateTime aDateTime = static_cast(nowDatetime()); @@ -931,6 +978,23 @@ int64_t Func_cast_datetime::getDatetimeIntVal(rowgroup::Row& row, return parm[0]->data()->getDatetimeIntVal(row, isNull); } + case execplan::CalpontSystemCatalog::TIMESTAMP: + { + TimeStamp timestamp(parm[0]->data()->getTimestampIntVal(row, isNull)); + int64_t seconds = timestamp.second; + MySQLTime m_time; + gmtSecToMySQLTime(seconds, m_time, fTimeZone); + DateTime dt; + dt.year = m_time.year; + dt.month = m_time.month; + dt.day = m_time.day; + dt.hour = m_time.hour; + dt.minute = m_time.minute; + dt.second = m_time.second; + dt.msecond = timestamp.msecond; + return *(reinterpret_cast(&dt)); + } + case CalpontSystemCatalog::TIME: { DateTime aDateTime = static_cast(nowDatetime()); @@ -1028,6 +1092,22 @@ int64_t Func_cast_datetime::getTimeIntVal(rowgroup::Row& row, return parm[0]->data()->getTimeIntVal(row, isNull); } + case execplan::CalpontSystemCatalog::TIMESTAMP: + { + TimeStamp timestamp(parm[0]->data()->getTimestampIntVal(row, isNull)); + int64_t seconds = timestamp.second; + MySQLTime m_time; + gmtSecToMySQLTime(seconds, m_time, fTimeZone); + Time time; + time.hour = m_time.hour; + time.minute = m_time.minute; + time.second = m_time.second; + time.is_neg = 0; + time.day = 0; + time.msecond = 0; + return *(reinterpret_cast(&time)); + } + default: { isNull = true; @@ -1392,6 +1472,25 @@ IDB_Decimal Func_cast_decimal::getDecimalVal(Row& row, } break; + case execplan::CalpontSystemCatalog::TIMESTAMP: + { + int32_t s = 0; + + string value = dataconvert::DataConvert::timestampToString1(parm[0]->data()->getTimestampIntVal(row, isNull), fTimeZone); + + //strip off micro seconds + string date = value.substr(0, 14); + + int64_t x = atoll(date.c_str()); + + if (!isNull) + { + decimal.value = x; + decimal.scale = s; + } + } + break; + case execplan::CalpontSystemCatalog::TIME: { int32_t s = 0; @@ -1495,6 +1594,18 @@ double Func_cast_double::getDoubleVal(Row& row, } break; + case execplan::CalpontSystemCatalog::TIMESTAMP: + { + string str = + DataConvert::timestampToString1(parm[0]->data()->getTimestampIntVal(row, isNull), fTimeZone); + + // strip off micro seconds + str = str.substr(0, 14); + + dblval = atof(str.c_str()); + } + break; + case execplan::CalpontSystemCatalog::UBIGINT: case execplan::CalpontSystemCatalog::UINT: case execplan::CalpontSystemCatalog::UMEDINT: diff --git a/utils/funcexp/func_ceil.cpp b/utils/funcexp/func_ceil.cpp index 4cc916b42..cd76df3e3 100644 --- a/utils/funcexp/func_ceil.cpp +++ b/utils/funcexp/func_ceil.cpp @@ -159,6 +159,15 @@ int64_t Func_ceil::getIntVal(Row& row, } break; + case CalpontSystemCatalog::TIMESTAMP: + { + TimeStamp dt(parm[0]->data()->getTimestampIntVal(row, isNull)); + + if (!isNull) + ret = dt.convertToMySQLint(fTimeZone); + } + break; + case CalpontSystemCatalog::TIME: { Time dt(parm[0]->data()->getTimeIntVal(row, isNull)); @@ -255,6 +264,15 @@ uint64_t Func_ceil::getUintVal(Row& row, } break; + case CalpontSystemCatalog::TIMESTAMP: + { + TimeStamp dt(parm[0]->data()->getTimestampIntVal(row, isNull)); + + if (!isNull) + ret = dt.convertToMySQLint(fTimeZone); + } + break; + case CalpontSystemCatalog::TIME: { Time dt(parm[0]->data()->getTimeIntVal(row, isNull)); diff --git a/utils/funcexp/func_char.cpp b/utils/funcexp/func_char.cpp index bc54fe5c0..86cec9f32 100644 --- a/utils/funcexp/func_char.cpp +++ b/utils/funcexp/func_char.cpp @@ -173,6 +173,7 @@ string Func_char::getStrVal(Row& row, case execplan::CalpontSystemCatalog::DATE: case execplan::CalpontSystemCatalog::DATETIME: + case execplan::CalpontSystemCatalog::TIMESTAMP: { isNull = true; return ""; diff --git a/utils/funcexp/func_char_length.cpp b/utils/funcexp/func_char_length.cpp index b47afd58a..bacef974c 100644 --- a/utils/funcexp/func_char_length.cpp +++ b/utils/funcexp/func_char_length.cpp @@ -101,6 +101,12 @@ int64_t Func_char_length::getIntVal(rowgroup::Row& row, return (int64_t)date.size(); } + case execplan::CalpontSystemCatalog::TIMESTAMP: + { + string date = dataconvert::DataConvert::timestampToString(parm[0]->data()->getTimestampIntVal(row, isNull), fTimeZone); + return (int64_t)date.size(); + } + case execplan::CalpontSystemCatalog::TIME: { string date = dataconvert::DataConvert::timeToString(parm[0]->data()->getTimeIntVal(row, isNull)); diff --git a/utils/funcexp/func_coalesce.cpp b/utils/funcexp/func_coalesce.cpp index 7df883757..4c3faca85 100644 --- a/utils/funcexp/func_coalesce.cpp +++ b/utils/funcexp/func_coalesce.cpp @@ -140,6 +140,30 @@ int64_t Func_coalesce::getDatetimeIntVal(rowgroup::Row& row, return val; } +int64_t Func_coalesce::getTimestampIntVal(rowgroup::Row& row, + FunctionParm& parm, + bool& isNull, + CalpontSystemCatalog::ColType& ct) +{ + int64_t val = 0; + + for (uint32_t i = 0; i < parm.size(); i++) + { + val = parm[i]->data()->getTimestampIntVal(row, isNull); + + if (isNull) + { + isNull = false; + continue; + } + + return val; + } + + isNull = true; + return val; +} + int64_t Func_coalesce::getTimeIntVal(rowgroup::Row& row, FunctionParm& parm, bool& isNull, diff --git a/utils/funcexp/func_date.cpp b/utils/funcexp/func_date.cpp index 719b068df..7280e838a 100644 --- a/utils/funcexp/func_date.cpp +++ b/utils/funcexp/func_date.cpp @@ -75,6 +75,14 @@ int64_t Func_date::getIntVal(rowgroup::Row& row, break; } + case execplan::CalpontSystemCatalog::TIMESTAMP: + { + int64_t val1 = parm[0]->data()->getTimestampIntVal(row, isNull); + value = dataconvert::DataConvert::timestampToString(val1, fTimeZone); + value = value.substr(0, 10); + break; + } + // Time adds to now() and then gets value case CalpontSystemCatalog::TIME: { diff --git a/utils/funcexp/func_date_add.cpp b/utils/funcexp/func_date_add.cpp index 6e4ca0543..5ed920e44 100644 --- a/utils/funcexp/func_date_add.cpp +++ b/utils/funcexp/func_date_add.cpp @@ -777,6 +777,24 @@ int64_t Func_date_add::getIntVal(rowgroup::Row& row, break; } + case execplan::CalpontSystemCatalog::TIMESTAMP: + { + TimeStamp timestamp(parm[0]->data()->getTimestampIntVal(row, isNull)); + int64_t seconds = timestamp.second; + MySQLTime m_time; + gmtSecToMySQLTime(seconds, m_time, fTimeZone); + DateTime dt; + dt.year = m_time.year; + dt.month = m_time.month; + dt.day = m_time.day; + dt.hour = m_time.hour; + dt.minute = m_time.minute; + dt.second = m_time.second; + dt.msecond = timestamp.msecond; + val = *(reinterpret_cast(&dt)); + break; + } + default: { isNull = true; diff --git a/utils/funcexp/func_date_format.cpp b/utils/funcexp/func_date_format.cpp index 9a25cb941..e62bc3952 100644 --- a/utils/funcexp/func_date_format.cpp +++ b/utils/funcexp/func_date_format.cpp @@ -269,6 +269,23 @@ string Func_date_format::getStrVal(rowgroup::Row& row, dt.msecond = (uint32_t)((val & 0xfffff)); break; + case CalpontSystemCatalog::TIMESTAMP: + { + val = parm[0]->data()->getTimestampIntVal(row, isNull); + TimeStamp timestamp(val); + int64_t seconds = timestamp.second; + MySQLTime time; + gmtSecToMySQLTime(seconds, time, fTimeZone); + dt.year = time.year; + dt.month = time.month; + dt.day = time.day; + dt.hour = time.hour; + dt.minute = time.minute; + dt.second = time.second; + dt.msecond = timestamp.msecond; + break; + } + case CalpontSystemCatalog::TIME: { DateTime aDateTime = static_cast(nowDatetime()); @@ -397,6 +414,14 @@ int64_t Func_date_format::getDatetimeIntVal(rowgroup::Row& row, return dataconvert::DataConvert::datetimeToInt(getStrVal(row, parm, isNull, ct)); } +int64_t Func_date_format::getTimestampIntVal(rowgroup::Row& row, + FunctionParm& parm, + bool& isNull, + CalpontSystemCatalog::ColType& ct) +{ + return dataconvert::DataConvert::timestampToInt(getStrVal(row, parm, isNull, ct), fTimeZone); +} + } // namespace funcexp // vim:ts=4 sw=4: diff --git a/utils/funcexp/func_day.cpp b/utils/funcexp/func_day.cpp index 6adfc9c25..c8ba97e6c 100644 --- a/utils/funcexp/func_day.cpp +++ b/utils/funcexp/func_day.cpp @@ -62,6 +62,15 @@ int64_t Func_day::getIntVal(rowgroup::Row& row, val = parm[0]->data()->getIntVal(row, isNull); return (uint32_t)((val >> 38) & 0x3f); + case CalpontSystemCatalog::TIMESTAMP: + { + dataconvert::TimeStamp timestamp(parm[0]->data()->getTimestampIntVal(row, isNull)); + int64_t seconds = timestamp.second; + dataconvert::MySQLTime m_time; + dataconvert::gmtSecToMySQLTime(seconds, m_time, fTimeZone); + return m_time.day; + } + // Time adds to now() and then gets value case CalpontSystemCatalog::TIME: aDateTime = static_cast(nowDatetime()); diff --git a/utils/funcexp/func_dayname.cpp b/utils/funcexp/func_dayname.cpp index 15933080e..02955fcf4 100644 --- a/utils/funcexp/func_dayname.cpp +++ b/utils/funcexp/func_dayname.cpp @@ -73,6 +73,19 @@ int64_t Func_dayname::getIntVal(rowgroup::Row& row, day = (uint32_t)((val >> 38) & 0x3f); break; + case CalpontSystemCatalog::TIMESTAMP: + { + val = parm[0]->data()->getIntVal(row, isNull); + dataconvert::TimeStamp timestamp(val); + int64_t seconds = timestamp.second; + dataconvert::MySQLTime time; + dataconvert::gmtSecToMySQLTime(seconds, time, fTimeZone); + year = time.year; + month = time.month; + day = time.day; + break; + } + // Time adds to now() and then gets value case CalpontSystemCatalog::TIME: aDateTime = static_cast(nowDatetime()); diff --git a/utils/funcexp/func_dayofweek.cpp b/utils/funcexp/func_dayofweek.cpp index e9cb9f0e4..e66e1d864 100644 --- a/utils/funcexp/func_dayofweek.cpp +++ b/utils/funcexp/func_dayofweek.cpp @@ -71,6 +71,19 @@ int64_t Func_dayofweek::getIntVal(rowgroup::Row& row, day = (uint32_t)((val >> 38) & 0x3f); break; + case CalpontSystemCatalog::TIMESTAMP: + { + val = parm[0]->data()->getIntVal(row, isNull); + dataconvert::TimeStamp timestamp(val); + int64_t seconds = timestamp.second; + dataconvert::MySQLTime time; + dataconvert::gmtSecToMySQLTime(seconds, time, fTimeZone); + year = time.year; + month = time.month; + day = time.day; + break; + } + // Time adds to now() and then gets value case CalpontSystemCatalog::TIME: aDateTime = static_cast(nowDatetime()); diff --git a/utils/funcexp/func_dayofyear.cpp b/utils/funcexp/func_dayofyear.cpp index 782e7e1af..6d4fc9fcf 100644 --- a/utils/funcexp/func_dayofyear.cpp +++ b/utils/funcexp/func_dayofyear.cpp @@ -71,6 +71,18 @@ int64_t Func_dayofyear::getIntVal(rowgroup::Row& row, day = (uint32_t)((val >> 38) & 0x3f); break; + case execplan::CalpontSystemCatalog::TIMESTAMP: + { + dataconvert::TimeStamp timestamp(parm[0]->data()->getIntVal(row, isNull)); + int64_t seconds = timestamp.second; + dataconvert::MySQLTime m_time; + dataconvert::gmtSecToMySQLTime(seconds, m_time, fTimeZone); + year = m_time.year; + month = m_time.month; + day = m_time.day; + break; + } + // Time adds to now() and then gets value case CalpontSystemCatalog::TIME: aDateTime = static_cast(nowDatetime()); diff --git a/utils/funcexp/func_extract.cpp b/utils/funcexp/func_extract.cpp index 332c05441..b8b1f90bc 100644 --- a/utils/funcexp/func_extract.cpp +++ b/utils/funcexp/func_extract.cpp @@ -247,6 +247,24 @@ int64_t Func_extract::getIntVal(rowgroup::Row& row, time = parm[0]->data()->getDatetimeIntVal(row, isNull); break; + case CalpontSystemCatalog::TIMESTAMP: + { + dataconvert::TimeStamp timestamp(parm[0]->data()->getTimestampIntVal(row, isNull)); + int64_t seconds = timestamp.second; + dataconvert::MySQLTime m_time; + dataconvert::gmtSecToMySQLTime(seconds, m_time, fTimeZone); + dataconvert::DateTime dt; + dt.year = m_time.year; + dt.month = m_time.month; + dt.day = m_time.day; + dt.hour = m_time.hour; + dt.minute = m_time.minute; + dt.second = m_time.second; + dt.msecond = timestamp.msecond; + time = *(reinterpret_cast(&dt)); + break; + } + case CalpontSystemCatalog::TIME: time = parm[0]->data()->getTimeIntVal(row, isNull); isTime = true; diff --git a/utils/funcexp/func_floor.cpp b/utils/funcexp/func_floor.cpp index 75954f128..2ec854b8c 100644 --- a/utils/funcexp/func_floor.cpp +++ b/utils/funcexp/func_floor.cpp @@ -158,6 +158,19 @@ int64_t Func_floor::getIntVal(Row& row, } break; + case execplan::CalpontSystemCatalog::TIMESTAMP: + { + string str = + DataConvert::timestampToString1(parm[0]->data()->getTimestampIntVal(row, isNull), fTimeZone); + + // strip off micro seconds + str = str.substr(0, 14); + + if (!isNull) + ret = atoll(str.c_str()); + } + break; + case execplan::CalpontSystemCatalog::TIME: { string str = @@ -262,6 +275,19 @@ uint64_t Func_floor::getUintVal(Row& row, } break; + case execplan::CalpontSystemCatalog::TIMESTAMP: + { + string str = + DataConvert::timestampToString1(parm[0]->data()->getTimestampIntVal(row, isNull), fTimeZone); + + // strip off micro seconds + str = str.substr(0, 14); + + if (!isNull) + ret = strtoull(str.c_str(), NULL, 10); + } + break; + case execplan::CalpontSystemCatalog::TIME: { string str = diff --git a/utils/funcexp/func_greatest.cpp b/utils/funcexp/func_greatest.cpp index 57c3b82fb..388a35f20 100644 --- a/utils/funcexp/func_greatest.cpp +++ b/utils/funcexp/func_greatest.cpp @@ -227,6 +227,26 @@ int64_t Func_greatest::getDatetimeIntVal(rowgroup::Row& row, return greatestStr; } +int64_t Func_greatest::getTimestampIntVal(rowgroup::Row& row, + FunctionParm& fp, + bool& isNull, + execplan::CalpontSystemCatalog::ColType& ct) +{ + int64_t str = fp[0]->data()->getTimestampIntVal(row, isNull); + + int64_t greatestStr = str; + + for (uint32_t i = 1; i < fp.size(); i++) + { + int64_t str1 = fp[i]->data()->getTimestampIntVal(row, isNull); + + if ( greatestStr < str1 ) + greatestStr = str1; + } + + return greatestStr; +} + int64_t Func_greatest::getTimeIntVal(rowgroup::Row& row, FunctionParm& fp, bool& isNull, diff --git a/utils/funcexp/func_hex.cpp b/utils/funcexp/func_hex.cpp index 462d8e779..7f7c7eff9 100644 --- a/utils/funcexp/func_hex.cpp +++ b/utils/funcexp/func_hex.cpp @@ -78,6 +78,7 @@ string Func_hex::getStrVal(rowgroup::Row& row, case CalpontSystemCatalog::TEXT: case CalpontSystemCatalog::VARCHAR: case CalpontSystemCatalog::DATETIME: + case CalpontSystemCatalog::TIMESTAMP: case CalpontSystemCatalog::DATE: case CalpontSystemCatalog::TIME: { diff --git a/utils/funcexp/func_hour.cpp b/utils/funcexp/func_hour.cpp index 4750829ad..6959ce526 100644 --- a/utils/funcexp/func_hour.cpp +++ b/utils/funcexp/func_hour.cpp @@ -109,6 +109,15 @@ int64_t Func_hour::getIntVal(rowgroup::Row& row, break; } + case execplan::CalpontSystemCatalog::TIMESTAMP: + { + dataconvert::TimeStamp timestamp(parm[0]->data()->getTimestampIntVal(row, isNull)); + int64_t seconds = timestamp.second; + dataconvert::MySQLTime m_time; + dataconvert::gmtSecToMySQLTime(seconds, m_time, fTimeZone); + return m_time.hour; + } + case execplan::CalpontSystemCatalog::TIME: { isTime = true; diff --git a/utils/funcexp/func_if.cpp b/utils/funcexp/func_if.cpp index 5da8293e6..cc46f4a27 100644 --- a/utils/funcexp/func_if.cpp +++ b/utils/funcexp/func_if.cpp @@ -36,7 +36,7 @@ using namespace rowgroup; namespace { -bool boolVal(SPTP& parm, Row& row) +bool boolVal(SPTP& parm, Row& row, const string& timeZone) { bool ret = true; bool isNull = false; // Keep it local. We don't want to mess with the global one here. @@ -52,7 +52,7 @@ bool boolVal(SPTP& parm, Row& row) case CalpontSystemCatalog::CHAR: case CalpontSystemCatalog::TEXT: case CalpontSystemCatalog::VARCHAR: - ret = (atoi((char*)(parm->data()->getStrVal().c_str())) != 0); + ret = (atoi((char*)(parm->data()->getStrVal(timeZone).c_str())) != 0); case CalpontSystemCatalog::FLOAT: case CalpontSystemCatalog::UFLOAT: @@ -79,6 +79,7 @@ bool boolVal(SPTP& parm, Row& row) case CalpontSystemCatalog::UINT: case CalpontSystemCatalog::DATE: case CalpontSystemCatalog::DATETIME: + case CalpontSystemCatalog::TIMESTAMP: case CalpontSystemCatalog::TIME: default: ret = (parm->data()->getIntVal(row, isNull) != 0); @@ -133,7 +134,7 @@ int64_t Func_if::getIntVal(Row& row, bool& isNull, CalpontSystemCatalog::ColType&) { - if (boolVal(parm[0], row)) + if (boolVal(parm[0], row, fTimeZone)) { return parm[1]->data()->getIntVal(row, isNull); } @@ -150,7 +151,7 @@ string Func_if::getStrVal(Row& row, CalpontSystemCatalog::ColType&) { - if (boolVal(parm[0], row)) + if (boolVal(parm[0], row, fTimeZone)) { return parm[1]->data()->getStrVal(row, isNull); } @@ -166,7 +167,7 @@ IDB_Decimal Func_if::getDecimalVal(Row& row, bool& isNull, CalpontSystemCatalog::ColType&) { - if (boolVal(parm[0], row)) + if (boolVal(parm[0], row, fTimeZone)) { return parm[1]->data()->getDecimalVal(row, isNull); } @@ -182,7 +183,7 @@ double Func_if::getDoubleVal(Row& row, bool& isNull, CalpontSystemCatalog::ColType&) { - if (boolVal(parm[0], row)) + if (boolVal(parm[0], row, fTimeZone)) { return parm[1]->data()->getDoubleVal(row, isNull); } @@ -197,7 +198,7 @@ long double Func_if::getLongDoubleVal(Row& row, bool& isNull, CalpontSystemCatalog::ColType&) { - if (boolVal(parm[0], row)) + if (boolVal(parm[0], row, fTimeZone)) { return parm[1]->data()->getLongDoubleVal(row, isNull); } @@ -213,7 +214,7 @@ int32_t Func_if::getDateIntVal(Row& row, bool& isNull, CalpontSystemCatalog::ColType&) { - if (boolVal(parm[0], row)) + if (boolVal(parm[0], row, fTimeZone)) { return parm[1]->data()->getDateIntVal(row, isNull); } @@ -229,7 +230,7 @@ int64_t Func_if::getDatetimeIntVal(Row& row, bool& isNull, CalpontSystemCatalog::ColType&) { - if (boolVal(parm[0], row)) + if (boolVal(parm[0], row, fTimeZone)) { return parm[1]->data()->getDatetimeIntVal(row, isNull); } @@ -239,12 +240,29 @@ int64_t Func_if::getDatetimeIntVal(Row& row, } } + +int64_t Func_if::getTimestampIntVal(Row& row, + FunctionParm& parm, + bool& isNull, + CalpontSystemCatalog::ColType&) +{ + if (boolVal(parm[0], row, fTimeZone)) + { + return parm[1]->data()->getTimestampIntVal(row, isNull); + } + else + { + return parm[2]->data()->getTimestampIntVal(row, isNull); + } +} + + int64_t Func_if::getTimeIntVal(Row& row, FunctionParm& parm, bool& isNull, CalpontSystemCatalog::ColType&) { - if (boolVal(parm[0], row)) + if (boolVal(parm[0], row, fTimeZone)) { return parm[1]->data()->getTimeIntVal(row, isNull); } diff --git a/utils/funcexp/func_ifnull.cpp b/utils/funcexp/func_ifnull.cpp index 00aaa682b..6309171fd 100644 --- a/utils/funcexp/func_ifnull.cpp +++ b/utils/funcexp/func_ifnull.cpp @@ -188,6 +188,25 @@ int64_t Func_ifnull::getDatetimeIntVal(Row& row, return r; } +int64_t Func_ifnull::getTimestampIntVal(Row& row, + FunctionParm& parm, + bool& isNull, + CalpontSystemCatalog::ColType&) +{ + if (isNull) + return 0; + + int64_t r = parm[0]->data()->getTimestampIntVal(row, isNull); + + if (isNull) + { + isNull = false; + return parm[1]->data()->getTimestampIntVal(row, isNull); + } + + return r; +} + int64_t Func_ifnull::getTimeIntVal(Row& row, FunctionParm& parm, bool& isNull, diff --git a/utils/funcexp/func_in.cpp b/utils/funcexp/func_in.cpp index 966b30ae5..5b1aa015c 100644 --- a/utils/funcexp/func_in.cpp +++ b/utils/funcexp/func_in.cpp @@ -159,6 +159,27 @@ inline bool getBoolForIn(rowgroup::Row& row, return false; } + case execplan::CalpontSystemCatalog::TIMESTAMP: + { + int64_t val = pm[0]->data()->getTimestampIntVal(row, isNull); + + if (isNull) + return false; + + for (uint32_t i = 1; i < pm.size(); i++) + { + isNull = false; + + if ( val == pm[i]->data()->getTimestampIntVal(row, isNull) && !isNull ) + return true; + + if (isNull && isNotIn) + return true; // will be reversed to false by the caller + } + + return false; + } + case execplan::CalpontSystemCatalog::TIME: { int64_t val = pm[0]->data()->getTimeIntVal(row, isNull); diff --git a/utils/funcexp/func_inet_aton.cpp b/utils/funcexp/func_inet_aton.cpp index 9692cf850..4bcdc3115 100644 --- a/utils/funcexp/func_inet_aton.cpp +++ b/utils/funcexp/func_inet_aton.cpp @@ -222,6 +222,26 @@ int64_t Func_inet_aton::getDatetimeIntVal(rowgroup::Row& row, return iValue; } +int64_t Func_inet_aton::getTimestampIntVal(rowgroup::Row& row, + FunctionParm& fp, + bool& isNull, + execplan::CalpontSystemCatalog::ColType& op_ct) +{ + int64_t iValue = joblist::TIMESTAMPNULL; + + const std::string& sValue = fp[0]->data()->getStrVal(row, isNull); + + if (!isNull) + { + int64_t iVal = convertAton( sValue, isNull ); + + if (!isNull) + iValue = iVal; + } + + return iValue; +} + int64_t Func_inet_aton::getTimeIntVal(rowgroup::Row& row, FunctionParm& fp, bool& isNull, diff --git a/utils/funcexp/func_inet_ntoa.cpp b/utils/funcexp/func_inet_ntoa.cpp index ff8c889ed..b9a31b08e 100644 --- a/utils/funcexp/func_inet_ntoa.cpp +++ b/utils/funcexp/func_inet_ntoa.cpp @@ -256,6 +256,20 @@ int64_t Func_inet_ntoa::getDatetimeIntVal(rowgroup::Row& row, return iValue; } +int64_t Func_inet_ntoa::getTimestampIntVal(rowgroup::Row& row, + FunctionParm& fp, + bool& isNull, + execplan::CalpontSystemCatalog::ColType& op_ct) +{ +// std::cout << "In Func_inet_ntoa::getTimestampVal" << std::endl; + +// int64t iValue = fp[0]->data()->getTimestampIntVal(row, isNull); + int64_t iValue = joblist::TIMESTAMPNULL; + isNull = true; + + return iValue; +} + int64_t Func_inet_ntoa::getTimeIntVal(rowgroup::Row& row, FunctionParm& fp, bool& isNull, diff --git a/utils/funcexp/func_last_day.cpp b/utils/funcexp/func_last_day.cpp index 28b4c01e2..c201898db 100644 --- a/utils/funcexp/func_last_day.cpp +++ b/utils/funcexp/func_last_day.cpp @@ -72,6 +72,18 @@ int64_t Func_last_day::getIntVal(rowgroup::Row& row, day = (uint32_t)((val >> 38) & 0x3f); break; + case execplan::CalpontSystemCatalog::TIMESTAMP: + { + TimeStamp timestamp(parm[0]->data()->getIntVal(row, isNull)); + int64_t seconds = timestamp.second; + MySQLTime m_time; + gmtSecToMySQLTime(seconds, m_time, fTimeZone); + year = m_time.year; + month = m_time.month; + day = m_time.day; + break; + } + // Time adds to now() and then gets value case CalpontSystemCatalog::TIME: aDateTime = static_cast(nowDatetime()); diff --git a/utils/funcexp/func_least.cpp b/utils/funcexp/func_least.cpp index cfb590291..3675e940d 100644 --- a/utils/funcexp/func_least.cpp +++ b/utils/funcexp/func_least.cpp @@ -203,6 +203,26 @@ int64_t Func_least::getDatetimeIntVal(rowgroup::Row& row, return leastStr; } +int64_t Func_least::getTimestampIntVal(rowgroup::Row& row, + FunctionParm& fp, + bool& isNull, + execplan::CalpontSystemCatalog::ColType& op_ct) +{ + int64_t str = fp[0]->data()->getTimestampIntVal(row, isNull); + + int64_t leastStr = str; + + for (uint32_t i = 1; i < fp.size(); i++) + { + int64_t str1 = fp[i]->data()->getTimestampIntVal(row, isNull); + + if ( leastStr > str1 ) + leastStr = str1; + } + + return leastStr; +} + int64_t Func_least::getTimeIntVal(rowgroup::Row& row, FunctionParm& fp, bool& isNull, diff --git a/utils/funcexp/func_math.cpp b/utils/funcexp/func_math.cpp index 7af31901a..ff4d908da 100644 --- a/utils/funcexp/func_math.cpp +++ b/utils/funcexp/func_math.cpp @@ -155,6 +155,20 @@ double Func_acos::getDoubleVal(Row& row, } break; + case execplan::CalpontSystemCatalog::TIMESTAMP: + { + int64_t value = parm[0]->data()->getTimestampIntVal(row, isNull); + + if (isNull || (value < -1.0 || value > 1.0)) + { + isNull = true; + return doubleNullVal(); + } + + return acos((double)value); + } + break; + case execplan::CalpontSystemCatalog::TIME: { int64_t value = parm[0]->data()->getTimeIntVal(row, isNull); @@ -259,6 +273,20 @@ double Func_asin::getDoubleVal(Row& row, } break; + case execplan::CalpontSystemCatalog::TIMESTAMP: + { + int64_t value = parm[0]->data()->getTimestampIntVal(row, isNull); + + if (isNull || (value < -1.0 || value > 1.0)) + { + isNull = true; + return doubleNullVal(); + } + + return asin((double)value); + } + break; + case execplan::CalpontSystemCatalog::TIME: { int64_t value = parm[0]->data()->getTimeIntVal(row, isNull); @@ -402,6 +430,33 @@ double Func_atan::getDoubleVal(Row& row, } break; + case execplan::CalpontSystemCatalog::TIMESTAMP: + { + int64_t value = parm[0]->data()->getTimestampIntVal(row, isNull); + + if (isNull) + { + isNull = true; + return doubleNullVal(); + } + + if (parm.size() > 1 ) + { + double value2 = parm[1]->data()->getDoubleVal(row, isNull); + + if (isNull) + { + isNull = true; + return doubleNullVal(); + } + + return atan2(value, value2); + } + + return atan((double)value); + } + break; + case execplan::CalpontSystemCatalog::TIME: { @@ -519,6 +574,20 @@ double Func_cos::getDoubleVal(Row& row, } break; + case execplan::CalpontSystemCatalog::TIMESTAMP: + { + int64_t value = parm[0]->data()->getTimestampIntVal(row, isNull); + + if (isNull) + { + isNull = true; + return doubleNullVal(); + } + + return cos((double)value); + } + break; + case execplan::CalpontSystemCatalog::TIME: { int64_t value = parm[0]->data()->getTimeIntVal(row, isNull); @@ -649,6 +718,29 @@ double Func_cot::getDoubleVal(Row& row, } break; + case execplan::CalpontSystemCatalog::TIMESTAMP: + { + int64_t value = parm[0]->data()->getTimestampIntVal(row, isNull); + + if (value == 0) + { + Message::Args args; + args.add("cot"); + args.add((uint64_t)value); + unsigned errcode = ERR_FUNC_OUT_OF_RANGE_RESULT; + throw IDBExcept(IDBErrorInfo::instance()->errorMsg(errcode, args), errcode); + } + + if (isNull) + { + isNull = true; + return doubleNullVal(); + } + + return 1.0 / tan((double)value); + } + break; + case execplan::CalpontSystemCatalog::TIME: { int64_t value = parm[0]->data()->getTimeIntVal(row, isNull); @@ -798,6 +890,33 @@ double Func_log::getDoubleVal(Row& row, } break; + case execplan::CalpontSystemCatalog::TIMESTAMP: + { + int64_t value = parm[0]->data()->getTimestampIntVal(row, isNull); + + if (isNull || value <= 0.0) + { + isNull = true; + return doubleNullVal(); + } + + if (parm.size() > 1 ) + { + double value2 = parm[1]->data()->getDoubleVal(row, isNull); + + if (isNull || (value2 <= 0.0 || value == 1.0) ) + { + isNull = true; + return doubleNullVal(); + } + + return log(value2) / log((double)value); + } + + return log((double)value); + } + break; + case execplan::CalpontSystemCatalog::TIME: { int64_t value = parm[0]->data()->getTimeIntVal(row, isNull); @@ -919,6 +1038,20 @@ double Func_log2::getDoubleVal(Row& row, } break; + case execplan::CalpontSystemCatalog::TIMESTAMP: + { + int64_t value = parm[0]->data()->getTimestampIntVal(row, isNull); + + if (isNull || value <= 0.0) + { + isNull = true; + return doubleNullVal(); + } + + return log2(value); + } + break; + case execplan::CalpontSystemCatalog::TIME: { int64_t value = parm[0]->data()->getTimeIntVal(row, isNull); @@ -1027,6 +1160,20 @@ double Func_log10::getDoubleVal(Row& row, } break; + case execplan::CalpontSystemCatalog::TIMESTAMP: + { + int64_t value = parm[0]->data()->getTimestampIntVal(row, isNull); + + if (isNull || value <= 0.0) + { + isNull = true; + return doubleNullVal(); + } + + return log10((double)value); + } + break; + case execplan::CalpontSystemCatalog::TIME: { int64_t value = parm[0]->data()->getTimeIntVal(row, isNull); @@ -1138,6 +1285,20 @@ double Func_sin::getDoubleVal(Row& row, } break; + case execplan::CalpontSystemCatalog::TIMESTAMP: + { + int64_t value = parm[0]->data()->getTimestampIntVal(row, isNull); + + if (isNull) + { + isNull = true; + return doubleNullVal(); + } + + return sin((double)value); + } + break; + case execplan::CalpontSystemCatalog::TIME: { int64_t value = parm[0]->data()->getTimeIntVal(row, isNull); @@ -1241,6 +1402,20 @@ double Func_sqrt::getDoubleVal(Row& row, } break; + case execplan::CalpontSystemCatalog::TIMESTAMP: + { + int64_t value = parm[0]->data()->getTimestampIntVal(row, isNull); + + if (isNull || value < 0) + { + isNull = true; + return doubleNullVal(); + } + + return sqrt((double)value); + } + break; + case execplan::CalpontSystemCatalog::TIME: { int64_t value = parm[0]->data()->getTimeIntVal(row, isNull); @@ -1344,6 +1519,20 @@ double Func_tan::getDoubleVal(Row& row, } break; + case execplan::CalpontSystemCatalog::TIMESTAMP: + { + int64_t value = parm[0]->data()->getTimestampIntVal(row, isNull); + + if (isNull) + { + isNull = true; + return doubleNullVal(); + } + + return tan((double)value); + } + break; + case execplan::CalpontSystemCatalog::TIME: { int64_t value = parm[0]->data()->getTimeIntVal(row, isNull); @@ -1445,6 +1634,12 @@ string Func_format::getStrVal(Row& row, } break; + case execplan::CalpontSystemCatalog::TIMESTAMP: + { + value = dataconvert::DataConvert::timestampToString1(parm[0]->data()->getTimestampIntVal(row, isNull), fTimeZone); + } + break; + case execplan::CalpontSystemCatalog::TIME: { value = dataconvert::DataConvert::timeToString1(parm[0]->data()->getTimeIntVal(row, isNull)); @@ -1692,6 +1887,20 @@ double Func_radians::getDoubleVal(Row& row, } break; + case execplan::CalpontSystemCatalog::TIMESTAMP: + { + int64_t value = parm[0]->data()->getTimestampIntVal(row, isNull); + + if (isNull) + { + isNull = true; + return doubleNullVal(); + } + + return radians((double)value); + } + break; + case execplan::CalpontSystemCatalog::TIME: { int64_t value = parm[0]->data()->getTimeIntVal(row, isNull); @@ -1795,6 +2004,20 @@ double Func_degrees::getDoubleVal(Row& row, } break; + case execplan::CalpontSystemCatalog::TIMESTAMP: + { + int64_t value = parm[0]->data()->getTimestampIntVal(row, isNull); + + if (isNull) + { + isNull = true; + return doubleNullVal(); + } + + return degrees((double)value); + } + break; + case execplan::CalpontSystemCatalog::TIME: { int64_t value = parm[0]->data()->getTimeIntVal(row, isNull); diff --git a/utils/funcexp/func_microsecond.cpp b/utils/funcexp/func_microsecond.cpp index f292e26a5..12b2beb70 100644 --- a/utils/funcexp/func_microsecond.cpp +++ b/utils/funcexp/func_microsecond.cpp @@ -63,6 +63,11 @@ int64_t Func_microsecond::getIntVal(rowgroup::Row& row, microSecond = (uint32_t)((val & 0xfffff)); break; + case CalpontSystemCatalog::TIMESTAMP: + val = parm[0]->data()->getIntVal(row, isNull); + microSecond = (uint32_t)((val & 0xfffff)); + break; + case CalpontSystemCatalog::TIME: val = parm[0]->data()->getIntVal(row, isNull); microSecond = (uint32_t)((val & 0xffffff)); diff --git a/utils/funcexp/func_minute.cpp b/utils/funcexp/func_minute.cpp index 299dd6ae2..10b3f1ef6 100644 --- a/utils/funcexp/func_minute.cpp +++ b/utils/funcexp/func_minute.cpp @@ -109,6 +109,15 @@ int64_t Func_minute::getIntVal(rowgroup::Row& row, break; } + case execplan::CalpontSystemCatalog::TIMESTAMP: + { + dataconvert::TimeStamp timestamp(parm[0]->data()->getTimestampIntVal(row, isNull)); + int64_t seconds = timestamp.second; + dataconvert::MySQLTime m_time; + dataconvert::gmtSecToMySQLTime(seconds, m_time, fTimeZone); + return m_time.minute; + } + default: { isNull = true; diff --git a/utils/funcexp/func_month.cpp b/utils/funcexp/func_month.cpp index e11cee296..94708fb1b 100644 --- a/utils/funcexp/func_month.cpp +++ b/utils/funcexp/func_month.cpp @@ -61,6 +61,15 @@ int64_t Func_month::getIntVal(rowgroup::Row& row, val = parm[0]->data()->getIntVal(row, isNull); return (unsigned)((val >> 44) & 0xf); + case execplan::CalpontSystemCatalog::TIMESTAMP: + { + dataconvert::TimeStamp timestamp(parm[0]->data()->getTimestampIntVal(row, isNull)); + int64_t seconds = timestamp.second; + dataconvert::MySQLTime m_time; + dataconvert::gmtSecToMySQLTime(seconds, m_time, fTimeZone); + return m_time.month; + } + // Time adds to now() and then gets value case CalpontSystemCatalog::TIME: aDateTime = static_cast(nowDatetime()); diff --git a/utils/funcexp/func_monthname.cpp b/utils/funcexp/func_monthname.cpp index 70bba26e0..831d16586 100644 --- a/utils/funcexp/func_monthname.cpp +++ b/utils/funcexp/func_monthname.cpp @@ -73,6 +73,15 @@ int64_t Func_monthname::getDatetimeIntVal(rowgroup::Row& row, return val; } +int64_t Func_monthname::getTimestampIntVal(rowgroup::Row& row, + FunctionParm& parm, + bool& isNull, + CalpontSystemCatalog::ColType& ct) +{ + uint32_t val = getIntVal(row, parm, isNull, ct); + return val; +} + int64_t Func_monthname::getIntVal(rowgroup::Row& row, FunctionParm& parm, bool& isNull, @@ -92,6 +101,16 @@ int64_t Func_monthname::getIntVal(rowgroup::Row& row, val = parm[0]->data()->getIntVal(row, isNull); return (unsigned)((val >> 44) & 0xf); + case CalpontSystemCatalog::TIMESTAMP: + { + val = parm[0]->data()->getIntVal(row, isNull); + dataconvert::TimeStamp timestamp(val); + int64_t seconds = timestamp.second; + dataconvert::MySQLTime time; + dataconvert::gmtSecToMySQLTime(seconds, time, fTimeZone); + return time.month; + } + // Time adds to now() and then gets value case CalpontSystemCatalog::TIME: aDateTime = static_cast(nowDatetime()); diff --git a/utils/funcexp/func_nullif.cpp b/utils/funcexp/func_nullif.cpp index 4bac44c45..224c2d435 100644 --- a/utils/funcexp/func_nullif.cpp +++ b/utils/funcexp/func_nullif.cpp @@ -158,6 +158,41 @@ int64_t Func_nullif::getIntVal(rowgroup::Row& row, break; } + case execplan::CalpontSystemCatalog::TIMESTAMP: + { + exp2 = parm[1]->data()->getTimestampIntVal(row, isNull); + + if ((parm[0]->data()->resultType().colDataType == + execplan::CalpontSystemCatalog::TIMESTAMP) || + (parm[0]->data()->resultType().colDataType == + execplan::CalpontSystemCatalog::DATETIME)) + + { + // NULLIF arg0 is DATETIME, arg1 is TIMESTAMP, + // Upgrade arg0 to timestamp + // When comparing exp1 as a Date, we can't simply promote. We have + // to be careful of the return value in case of not null return. + int64_t exp1 = parm[0]->data()->getTimestampIntVal(row, isNull); + + if ( exp1 == exp2 ) + { + isNull = true; + return 0; + } + + // since exp1 here is inside the block, when we leave the block, the + // original (Date) value is restored. + } + + if (isNull) + { + isNull = false; + return exp1; + } + + break; + } + case execplan::CalpontSystemCatalog::TIME: { exp2 = parm[1]->data()->getTimeIntVal(row, isNull); @@ -281,6 +316,19 @@ uint64_t Func_nullif::getUintVal(rowgroup::Row& row, break; } + case execplan::CalpontSystemCatalog::TIMESTAMP: + { + exp2 = parm[1]->data()->getTimestampIntVal(row, isNull); + + if (isNull) + { + isNull = false; + return exp1; + } + + break; + } + case execplan::CalpontSystemCatalog::TIME: { exp2 = parm[1]->data()->getTimeIntVal(row, isNull); @@ -334,13 +382,15 @@ string Func_nullif::getStrVal(rowgroup::Row& row, int datatype1 = parm[1]->data()->resultType().colDataType; if ( datatype0 == execplan::CalpontSystemCatalog::DATE && - datatype1 == execplan::CalpontSystemCatalog::DATETIME ) + (datatype1 == execplan::CalpontSystemCatalog::DATETIME || + datatype1 == execplan::CalpontSystemCatalog::TIMESTAMP)) { exp1 = exp1 + " 00:00:00"; } if ( datatype1 == execplan::CalpontSystemCatalog::DATE && - datatype0 == execplan::CalpontSystemCatalog::DATETIME ) + (datatype0 == execplan::CalpontSystemCatalog::DATETIME || + datatype0 == execplan::CalpontSystemCatalog::TIMESTAMP)) { exp2 = exp2 + " 00:00:00"; } @@ -414,6 +464,19 @@ int32_t Func_nullif::getDateIntVal(rowgroup::Row& row, break; } + case execplan::CalpontSystemCatalog::TIMESTAMP: + { + exp2 = parm[1]->data()->getTimestampIntVal(row, isNull); + + if (isNull) + { + isNull = false; + return exp1; + } + + break; + } + default: { isNull = true; @@ -467,6 +530,7 @@ int64_t Func_nullif::getDatetimeIntVal(rowgroup::Row& row, case execplan::CalpontSystemCatalog::TIME: case execplan::CalpontSystemCatalog::DATETIME: + case execplan::CalpontSystemCatalog::TIMESTAMP: { exp2 = parm[1]->data()->getDatetimeIntVal(row, isNull); @@ -517,6 +581,7 @@ int64_t Func_nullif::getTimeIntVal(rowgroup::Row& row, case execplan::CalpontSystemCatalog::TEXT: case execplan::CalpontSystemCatalog::TIME: case execplan::CalpontSystemCatalog::DATETIME: + case execplan::CalpontSystemCatalog::TIMESTAMP: { exp2 = parm[1]->data()->getTimeIntVal(row, isNull); @@ -545,6 +610,58 @@ int64_t Func_nullif::getTimeIntVal(rowgroup::Row& row, return exp1; } +int64_t Func_nullif::getTimestampIntVal(rowgroup::Row& row, + FunctionParm& parm, + bool& isNull, + CalpontSystemCatalog::ColType& ct) +{ + int64_t exp1 = parm[0]->data()->getTimestampIntVal(row, isNull); + int64_t exp2 = 0; + + switch (parm[1]->data()->resultType().colDataType) + { + case execplan::CalpontSystemCatalog::BIGINT: + case execplan::CalpontSystemCatalog::INT: + case execplan::CalpontSystemCatalog::MEDINT: + case execplan::CalpontSystemCatalog::TINYINT: + case execplan::CalpontSystemCatalog::SMALLINT: + case execplan::CalpontSystemCatalog::DOUBLE: + case execplan::CalpontSystemCatalog::FLOAT: + case execplan::CalpontSystemCatalog::DECIMAL: + case execplan::CalpontSystemCatalog::VARCHAR: + case execplan::CalpontSystemCatalog::CHAR: + case execplan::CalpontSystemCatalog::TEXT: + case execplan::CalpontSystemCatalog::TIME: + case execplan::CalpontSystemCatalog::DATETIME: + case execplan::CalpontSystemCatalog::TIMESTAMP: + { + exp2 = parm[1]->data()->getTimestampIntVal(row, isNull); + + if (isNull) + { + isNull = false; + return exp1; + } + + break; + } + + default: + { + isNull = false; + return exp1; + } + } + + if ( exp1 == exp2 ) + { + isNull = true; + return 0; + } + + return exp1; +} + double Func_nullif::getDoubleVal(rowgroup::Row& row, FunctionParm& parm, @@ -710,6 +827,19 @@ long double Func_nullif::getLongDoubleVal(rowgroup::Row& row, break; } + case execplan::CalpontSystemCatalog::TIMESTAMP: + { + exp2 = parm[1]->data()->getTimestampIntVal(row, isNull); + + if (isNull) + { + isNull = false; + return exp1; + } + + break; + } + default: { isNull = true; @@ -841,12 +971,16 @@ execplan::IDB_Decimal Func_nullif::getDecimalVal(rowgroup::Row& row, case execplan::CalpontSystemCatalog::TIME: case execplan::CalpontSystemCatalog::DATETIME: + case execplan::CalpontSystemCatalog::TIMESTAMP: { int32_t s = 0; int64_t x = 0; + string value; - string value = - DataConvert::datetimeToString1(parm[1]->data()->getDatetimeIntVal(row, isNull)); + if (parm[1]->data()->resultType().colDataType == execplan::CalpontSystemCatalog::TIMESTAMP) + value = DataConvert::timestampToString1(parm[1]->data()->getTimestampIntVal(row, isNull), fTimeZone); + else + value = DataConvert::datetimeToString1(parm[1]->data()->getDatetimeIntVal(row, isNull)); if (!isNull) { diff --git a/utils/funcexp/func_period_diff.cpp b/utils/funcexp/func_period_diff.cpp index faf15dbb4..d66d50de7 100644 --- a/utils/funcexp/func_period_diff.cpp +++ b/utils/funcexp/func_period_diff.cpp @@ -76,6 +76,7 @@ int64_t Func_period_diff::getIntVal(rowgroup::Row& row, case execplan::CalpontSystemCatalog::SMALLINT: case execplan::CalpontSystemCatalog::DATE: case execplan::CalpontSystemCatalog::DATETIME: + case execplan::CalpontSystemCatalog::TIMESTAMP: { period1 = parm[0]->data()->getIntVal(row, isNull); break; @@ -124,6 +125,7 @@ int64_t Func_period_diff::getIntVal(rowgroup::Row& row, case execplan::CalpontSystemCatalog::SMALLINT: case execplan::CalpontSystemCatalog::DATE: case execplan::CalpontSystemCatalog::DATETIME: + case execplan::CalpontSystemCatalog::TIMESTAMP: { period2 = parm[1]->data()->getIntVal(row, isNull); break; diff --git a/utils/funcexp/func_quarter.cpp b/utils/funcexp/func_quarter.cpp index 4f476e2ff..e402444a7 100644 --- a/utils/funcexp/func_quarter.cpp +++ b/utils/funcexp/func_quarter.cpp @@ -65,6 +65,16 @@ int64_t Func_quarter::getIntVal(rowgroup::Row& row, month = (val >> 44) & 0xf; break; + case execplan::CalpontSystemCatalog::TIMESTAMP: + { + dataconvert::TimeStamp timestamp(parm[0]->data()->getTimestampIntVal(row, isNull)); + int64_t seconds = timestamp.second; + dataconvert::MySQLTime m_time; + dataconvert::gmtSecToMySQLTime(seconds, m_time, fTimeZone); + month = m_time.month; + break; + } + // Time adds to now() and then gets value case CalpontSystemCatalog::TIME: aDateTime = static_cast(nowDatetime()); diff --git a/utils/funcexp/func_regexp.cpp b/utils/funcexp/func_regexp.cpp index 420c792cf..809cb239a 100644 --- a/utils/funcexp/func_regexp.cpp +++ b/utils/funcexp/func_regexp.cpp @@ -50,7 +50,8 @@ namespace inline bool getBool(rowgroup::Row& row, funcexp::FunctionParm& pm, bool& isNull, - CalpontSystemCatalog::ColType& ct) + CalpontSystemCatalog::ColType& ct, + const string& timeZone) { string expr; @@ -94,6 +95,14 @@ inline bool getBool(rowgroup::Row& row, break; } + case execplan::CalpontSystemCatalog::TIMESTAMP: + { + expr = dataconvert::DataConvert::timestampToString(pm[0]->data()->getTimestampIntVal(row, isNull), timeZone); + //strip off micro seconds + expr = expr.substr(0, 19); + break; + } + case execplan::CalpontSystemCatalog::TIME: { expr = dataconvert::DataConvert::timeToString(pm[0]->data()->getTimeIntVal(row, isNull)); @@ -161,6 +170,14 @@ inline bool getBool(rowgroup::Row& row, break; } + case execplan::CalpontSystemCatalog::TIMESTAMP: + { + pattern = dataconvert::DataConvert::timestampToString(pm[1]->data()->getTimestampIntVal(row, isNull), timeZone); + //strip off micro seconds + pattern = pattern.substr(0, 19); + break; + } + case execplan::CalpontSystemCatalog::TIME: { pattern = dataconvert::DataConvert::timeToString(pm[1]->data()->getTimeIntVal(row, isNull)); @@ -227,7 +244,7 @@ bool Func_regexp::getBoolVal(rowgroup::Row& row, bool& isNull, CalpontSystemCatalog::ColType& ct) { - return getBool(row, pm, isNull, ct) && !isNull; + return getBool(row, pm, isNull, ct, fTimeZone) && !isNull; } diff --git a/utils/funcexp/func_round.cpp b/utils/funcexp/func_round.cpp index 2a843f429..c57dda167 100644 --- a/utils/funcexp/func_round.cpp +++ b/utils/funcexp/func_round.cpp @@ -484,10 +484,15 @@ IDB_Decimal Func_round::getDecimalVal(Row& row, case execplan::CalpontSystemCatalog::TIME: case execplan::CalpontSystemCatalog::DATETIME: + case execplan::CalpontSystemCatalog::TIMESTAMP: { int32_t s = 0; - string value = dataconvert::DataConvert::datetimeToString1(parm[0]->data()->getDatetimeIntVal(row, isNull)); + string value; + if (op_ct.colDataType == execplan::CalpontSystemCatalog::TIMESTAMP) + value = dataconvert::DataConvert::timestampToString1(parm[0]->data()->getTimestampIntVal(row, isNull), fTimeZone); + else + value = dataconvert::DataConvert::datetimeToString1(parm[0]->data()->getDatetimeIntVal(row, isNull)); //strip off micro seconds value = value.substr(0, 14); diff --git a/utils/funcexp/func_second.cpp b/utils/funcexp/func_second.cpp index ae12b157c..4f12cfca8 100644 --- a/utils/funcexp/func_second.cpp +++ b/utils/funcexp/func_second.cpp @@ -108,6 +108,15 @@ int64_t Func_second::getIntVal(rowgroup::Row& row, break; } + case execplan::CalpontSystemCatalog::TIMESTAMP: + { + dataconvert::TimeStamp timestamp(parm[0]->data()->getTimestampIntVal(row, isNull)); + int64_t seconds = timestamp.second; + dataconvert::MySQLTime m_time; + dataconvert::gmtSecToMySQLTime(seconds, m_time, fTimeZone); + return m_time.second; + } + case execplan::CalpontSystemCatalog::TIME: { val = parm[0]->data()->getTimeIntVal(row, isNull); diff --git a/utils/funcexp/func_str_to_date.cpp b/utils/funcexp/func_str_to_date.cpp index 31bbbf6ca..3432ba100 100644 --- a/utils/funcexp/func_str_to_date.cpp +++ b/utils/funcexp/func_str_to_date.cpp @@ -44,7 +44,8 @@ using namespace funcexp; dataconvert::DateTime getDateTime (rowgroup::Row& row, FunctionParm& parm, bool& isNull, - CalpontSystemCatalog::ColType&) + CalpontSystemCatalog::ColType& ct, + const string& timeZone) { TimeExtractor extractor; dataconvert::DateTime dateTime; @@ -92,6 +93,21 @@ dataconvert::DateTime getDateTime (rowgroup::Row& row, break; } + case CalpontSystemCatalog::TIMESTAMP: + { + val = parm[0]->data()->getIntVal(row, isNull); + valStr = dataconvert::DataConvert::timestampToString (val, timeZone); + rc = extractor.extractTime (valStr, formatStr, dateTime); + + if ( rc < 0) + { + isNull = true; + return -1; + } + + break; + } + case CalpontSystemCatalog::CHAR: case CalpontSystemCatalog::TEXT: case CalpontSystemCatalog::VARCHAR: @@ -171,7 +187,7 @@ string Func_str_to_date::getStrVal(rowgroup::Row& row, CalpontSystemCatalog::ColType& ct) { dataconvert::DateTime dateTime; - dateTime = getDateTime(row, parm, isNull, ct); + dateTime = getDateTime(row, parm, isNull, ct, fTimeZone); string convertedDate = dataconvert::DataConvert::datetimeToString(*((long long*) &dateTime)); return convertedDate; } @@ -182,7 +198,7 @@ int32_t Func_str_to_date::getDateIntVal(rowgroup::Row& row, CalpontSystemCatalog::ColType& ct) { dataconvert::DateTime dateTime; - dateTime = getDateTime(row, parm, isNull, ct); + dateTime = getDateTime(row, parm, isNull, ct, fTimeZone); int64_t time = *(reinterpret_cast(&dateTime)); return ((((int32_t)(time >> 32)) & 0xFFFFFFC0) | 0x3E); } @@ -193,11 +209,42 @@ int64_t Func_str_to_date::getDatetimeIntVal(rowgroup::Row& row, CalpontSystemCatalog::ColType& ct) { dataconvert::DateTime dateTime; - dateTime = getDateTime(row, parm, isNull, ct); + dateTime = getDateTime(row, parm, isNull, ct, fTimeZone); int64_t time = *(reinterpret_cast(&dateTime)); return time; } +int64_t Func_str_to_date::getTimestampIntVal(rowgroup::Row& row, + FunctionParm& parm, + bool& isNull, + CalpontSystemCatalog::ColType& ct) +{ + dataconvert::DateTime dateTime; + dateTime = getDateTime(row, parm, isNull, ct, fTimeZone); + dataconvert::TimeStamp timestamp; + dataconvert::MySQLTime m_time; + m_time.year = dateTime.year; + m_time.month = dateTime.month; + m_time.day = dateTime.day; + m_time.hour = dateTime.hour; + m_time.minute = dateTime.minute; + m_time.second = dateTime.second; + bool isValid = true; + int64_t seconds = mySQLTimeToGmtSec(m_time, fTimeZone, isValid); + if (!isValid) + { + timestamp = -1; + isNull = true; + } + else + { + timestamp.second = seconds; + timestamp.msecond = dateTime.msecond; + } + int64_t time = *(reinterpret_cast(×tamp)); + return time; +} + int64_t Func_str_to_date::getTimeIntVal(rowgroup::Row& row, FunctionParm& parm, bool& isNull, @@ -205,7 +252,7 @@ int64_t Func_str_to_date::getTimeIntVal(rowgroup::Row& row, { dataconvert::DateTime dateTime; dataconvert::Time retTime; - dateTime = getDateTime(row, parm, isNull, ct); + dateTime = getDateTime(row, parm, isNull, ct, fTimeZone); retTime.day = 0; retTime.is_neg = false; retTime.hour = dateTime.hour; @@ -222,7 +269,7 @@ int64_t Func_str_to_date::getIntVal(rowgroup::Row& row, CalpontSystemCatalog::ColType& ct) { dataconvert::DateTime dateTime; - dateTime = getDateTime(row, parm, isNull, ct); + dateTime = getDateTime(row, parm, isNull, ct, fTimeZone); int64_t time = *(reinterpret_cast(&dateTime)); return time; } diff --git a/utils/funcexp/func_sysdate.cpp b/utils/funcexp/func_sysdate.cpp index bf71e0415..0019b3110 100644 --- a/utils/funcexp/func_sysdate.cpp +++ b/utils/funcexp/func_sysdate.cpp @@ -100,6 +100,14 @@ int64_t Func_sysdate::getDatetimeIntVal(rowgroup::Row& row, return getIntVal(row, parm, isNull, operationColType); } +int64_t Func_sysdate::getTimestampIntVal(rowgroup::Row& row, + FunctionParm& parm, + bool& isNull, + CalpontSystemCatalog::ColType& operationColType) +{ + return getIntVal(row, parm, isNull, operationColType); +} + int64_t Func_sysdate::getTimeIntVal(rowgroup::Row& row, FunctionParm& parm, bool& isNull, diff --git a/utils/funcexp/func_time.cpp b/utils/funcexp/func_time.cpp index 4a90e0fe2..b83c678db 100644 --- a/utils/funcexp/func_time.cpp +++ b/utils/funcexp/func_time.cpp @@ -118,6 +118,23 @@ string Func_time::getStrVal(rowgroup::Row& row, break; } + case execplan::CalpontSystemCatalog::TIMESTAMP: + { + dataconvert::TimeStamp timestamp(parm[0]->data()->getTimestampIntVal(row, isNull)); + int64_t seconds = timestamp.second; + dataconvert::MySQLTime m_time; + dataconvert::gmtSecToMySQLTime(seconds, m_time, fTimeZone); + dataconvert::Time time; + time.hour = m_time.hour; + time.minute = m_time.minute; + time.second = m_time.second; + time.is_neg = 0; + time.day = 0; + time.msecond = 0; + val = *(reinterpret_cast(&time)); + break; + } + default: { isNull = true; diff --git a/utils/funcexp/func_time_format.cpp b/utils/funcexp/func_time_format.cpp index 539a712c8..08739b046 100644 --- a/utils/funcexp/func_time_format.cpp +++ b/utils/funcexp/func_time_format.cpp @@ -72,6 +72,19 @@ string Func_time_format::getStrVal(rowgroup::Row& row, msec = (uint32_t)((val & 0xfffff)); break; + case execplan::CalpontSystemCatalog::TIMESTAMP: + { + dataconvert::TimeStamp timestamp(parm[0]->data()->getIntVal(row, isNull)); + int64_t seconds = timestamp.second; + dataconvert::MySQLTime m_time; + dataconvert::gmtSecToMySQLTime(seconds, m_time, fTimeZone); + hour = m_time.hour; + min = m_time.minute; + sec = m_time.second; + msec = timestamp.msecond; + break; + } + case CalpontSystemCatalog::CHAR: case CalpontSystemCatalog::VARCHAR: case CalpontSystemCatalog::TEXT: diff --git a/utils/funcexp/func_time_to_sec.cpp b/utils/funcexp/func_time_to_sec.cpp index ee5c4b669..f8391d12f 100644 --- a/utils/funcexp/func_time_to_sec.cpp +++ b/utils/funcexp/func_time_to_sec.cpp @@ -70,6 +70,19 @@ int64_t Func_time_to_sec::getIntVal(rowgroup::Row& row, sec = (int32_t)((val >> 20) & 0x3f); break; + case CalpontSystemCatalog::TIMESTAMP: + { + val = parm[0]->data()->getIntVal(row, isNull); + dataconvert::TimeStamp timestamp(val); + int64_t seconds = timestamp.second; + dataconvert::MySQLTime time; + dataconvert::gmtSecToMySQLTime(seconds, time, fTimeZone); + hour = time.hour; + min = time.minute; + sec = time.second; + break; + } + case CalpontSystemCatalog::TIME: val = parm[0]->data()->getTimeIntVal(row, isNull); diff --git a/utils/funcexp/func_timediff.cpp b/utils/funcexp/func_timediff.cpp index 369bb80a1..1254f2d1b 100644 --- a/utils/funcexp/func_timediff.cpp +++ b/utils/funcexp/func_timediff.cpp @@ -126,6 +126,31 @@ string Func_timediff::getStrVal(rowgroup::Row& row, val1 = parm[0]->data()->getDatetimeIntVal(row, isNull); break; + case execplan::CalpontSystemCatalog::TIMESTAMP: + { + if (type1 != type2) + { + isNull = true; + break; + } + int64_t temp = parm[0]->data()->getTimestampIntVal(row, isNull); + dataconvert::TimeStamp timestamp(temp); + int64_t seconds = timestamp.second; + dataconvert::MySQLTime time; + dataconvert::gmtSecToMySQLTime(seconds, time, fTimeZone); + dataconvert::DateTime dt; + dt.year = time.year; + dt.month = time.month; + dt.day = time.day; + dt.hour = time.hour; + dt.minute = time.minute; + dt.second = time.second; + dt.msecond = timestamp.msecond; + val1 = (int64_t) *(reinterpret_cast(&dt)); + + break; + } + case execplan::CalpontSystemCatalog::VARCHAR: case execplan::CalpontSystemCatalog::CHAR: case execplan::CalpontSystemCatalog::TEXT: @@ -168,6 +193,25 @@ string Func_timediff::getStrVal(rowgroup::Row& row, val2 = parm[1]->data()->getDatetimeIntVal(row, isNull); break; + case execplan::CalpontSystemCatalog::TIMESTAMP: + { + int64_t temp = parm[1]->data()->getTimestampIntVal(row, isNull); + dataconvert::TimeStamp timestamp(temp); + int64_t seconds = timestamp.second; + dataconvert::MySQLTime time; + dataconvert::gmtSecToMySQLTime(seconds, time, fTimeZone); + dataconvert::DateTime dt; + dt.year = time.year; + dt.month = time.month; + dt.day = time.day; + dt.hour = time.hour; + dt.minute = time.minute; + dt.second = time.second; + dt.msecond = timestamp.msecond; + val2 = (int64_t) *(reinterpret_cast(&dt)); + break; + } + case execplan::CalpontSystemCatalog::VARCHAR: case execplan::CalpontSystemCatalog::CHAR: case execplan::CalpontSystemCatalog::TEXT: @@ -220,6 +264,14 @@ int64_t Func_timediff::getDatetimeIntVal(rowgroup::Row& row, return dataconvert::DataConvert::datetimeToInt(getStrVal(row, parm, isNull, ct)); } +int64_t Func_timediff::getTimestampIntVal(rowgroup::Row& row, + FunctionParm& parm, + bool& isNull, + CalpontSystemCatalog::ColType& ct) +{ + return dataconvert::DataConvert::timestampToInt(getStrVal(row, parm, isNull, ct), fTimeZone); +} + int64_t Func_timediff::getTimeIntVal(rowgroup::Row& row, FunctionParm& parm, bool& isNull, diff --git a/utils/funcexp/func_timestampdiff.cpp b/utils/funcexp/func_timestampdiff.cpp index 9394203d7..60b18cdc7 100644 --- a/utils/funcexp/func_timestampdiff.cpp +++ b/utils/funcexp/func_timestampdiff.cpp @@ -50,25 +50,63 @@ int64_t Func_timestampdiff::getIntVal(rowgroup::Row& row, bool& isNull, CalpontSystemCatalog::ColType& op_ct) { - int64_t val1 = parm[0]->data()->getDatetimeIntVal(row, isNull); - int64_t val2 = parm[1]->data()->getDatetimeIntVal(row, isNull); - IntervalColumn::interval_type unit = static_cast(parm[2]->data()->getIntVal(row, isNull)); + int64_t val1, val2; DateTime dt1, dt2; - dt1.year = (val1 >> 48) & 0xffff; - dt1.month = (val1 >> 44) & 0xf; - dt1.day = (val1 >> 38) & 0x3f; - dt1.hour = (val1 >> 32) & 0x3f; - dt1.minute = (val1 >> 26) & 0x3f; - dt1.second = (val1 >> 20) & 0x3f; - dt1.msecond = val1 & 0xfffff; + if (parm[0]->data()->resultType().colDataType == execplan::CalpontSystemCatalog::TIMESTAMP) + { + TimeStamp timestamp(parm[0]->data()->getTimestampIntVal(row, isNull)); + int64_t seconds = timestamp.second; + MySQLTime m_time; + gmtSecToMySQLTime(seconds, m_time, fTimeZone); + dt1.year = m_time.year; + dt1.month = m_time.month; + dt1.day = m_time.day; + dt1.hour = m_time.hour; + dt1.minute = m_time.minute; + dt1.second = m_time.second; + dt1.msecond = timestamp.msecond; + val1 = *(reinterpret_cast(&dt1)); + } + else + { + val1 = parm[0]->data()->getDatetimeIntVal(row, isNull); + dt1.year = (val1 >> 48) & 0xffff; + dt1.month = (val1 >> 44) & 0xf; + dt1.day = (val1 >> 38) & 0x3f; + dt1.hour = (val1 >> 32) & 0x3f; + dt1.minute = (val1 >> 26) & 0x3f; + dt1.second = (val1 >> 20) & 0x3f; + dt1.msecond = val1 & 0xfffff; + } - dt2.year = (val2 >> 48) & 0xffff; - dt2.month = (val2 >> 44) & 0xf; - dt2.day = (val2 >> 38) & 0x3f; - dt2.hour = (val2 >> 32) & 0x3f; - dt2.minute = (val2 >> 26) & 0x3f; - dt2.second = (val2 >> 20) & 0x3f; - dt2.msecond = val2 & 0xfffff; + if (parm[1]->data()->resultType().colDataType == execplan::CalpontSystemCatalog::TIMESTAMP) + { + TimeStamp timestamp(parm[1]->data()->getTimestampIntVal(row, isNull)); + int64_t seconds = timestamp.second; + MySQLTime m_time; + gmtSecToMySQLTime(seconds, m_time, fTimeZone); + dt2.year = m_time.year; + dt2.month = m_time.month; + dt2.day = m_time.day; + dt2.hour = m_time.hour; + dt2.minute = m_time.minute; + dt2.second = m_time.second; + dt2.msecond = timestamp.msecond; + val2 = *(reinterpret_cast(&dt2)); + } + else + { + val2 = parm[1]->data()->getDatetimeIntVal(row, isNull); + dt2.year = (val2 >> 48) & 0xffff; + dt2.month = (val2 >> 44) & 0xf; + dt2.day = (val2 >> 38) & 0x3f; + dt2.hour = (val2 >> 32) & 0x3f; + dt2.minute = (val2 >> 26) & 0x3f; + dt2.second = (val2 >> 20) & 0x3f; + dt2.msecond = val2 & 0xfffff; + } + + IntervalColumn::interval_type unit = static_cast(parm[2]->data()->getIntVal(row, isNull)); int64_t diff = 0; @@ -152,6 +190,14 @@ int64_t Func_timestampdiff::getDatetimeIntVal(rowgroup::Row& row, return getIntVal(row, parm, isNull, ct); } +int64_t Func_timestampdiff::getTimestampIntVal(rowgroup::Row& row, + FunctionParm& parm, + bool& isNull, + CalpontSystemCatalog::ColType& ct) +{ + return getIntVal(row, parm, isNull, ct); +} + int64_t Func_timestampdiff::getTimeIntVal(rowgroup::Row& row, FunctionParm& parm, bool& isNull, diff --git a/utils/funcexp/func_to_days.cpp b/utils/funcexp/func_to_days.cpp index 33f72b22d..b51eb5aff 100644 --- a/utils/funcexp/func_to_days.cpp +++ b/utils/funcexp/func_to_days.cpp @@ -85,6 +85,17 @@ int64_t Func_to_days::getIntVal(rowgroup::Row& row, break; } + case execplan::CalpontSystemCatalog::TIMESTAMP: + { + dataconvert::TimeStamp timestamp(parm[0]->data()->getTimestampIntVal(row, isNull)); + int64_t seconds = timestamp.second; + dataconvert::MySQLTime m_time; + dataconvert::gmtSecToMySQLTime(seconds, m_time, fTimeZone); + year = m_time.year; + month = m_time.month; + day = m_time.day; + return helpers::calc_mysql_daynr(year, month, day); + } // Time adds to now() and then gets value case CalpontSystemCatalog::TIME: { diff --git a/utils/funcexp/func_truncate.cpp b/utils/funcexp/func_truncate.cpp index 1b8383b28..a2f73753a 100644 --- a/utils/funcexp/func_truncate.cpp +++ b/utils/funcexp/func_truncate.cpp @@ -491,6 +491,52 @@ IDB_Decimal Func_truncate::getDecimalVal(Row& row, } break; + case execplan::CalpontSystemCatalog::TIMESTAMP: + { + int32_t s = 0; + int64_t x = 0; + + string value = + DataConvert::timestampToString1(parm[0]->data()->getTimestampIntVal(row, isNull), fTimeZone); + + s = parm[1]->data()->getIntVal(row, isNull); + + if (!isNull) + { + //strip off micro seconds + value = value.substr(0, 14); + x = atoll(value.c_str()); + + if ( s > 5 ) + s = 0; + + if ( s > 0 ) + { + x *= helpers::powerOf10_c[s]; + } + else if (s < 0) + { + s = -s; + + if ( s >= (int32_t) value.size() ) + { + x = 0; + } + else + { + x /= helpers::powerOf10_c[s]; + x *= helpers::powerOf10_c[s]; + } + + s = 0; + } + } + + decimal.value = x; + decimal.scale = s; + } + break; + case execplan::CalpontSystemCatalog::TIME: { int32_t s = 0; diff --git a/utils/funcexp/func_unix_timestamp.cpp b/utils/funcexp/func_unix_timestamp.cpp index 02d90d92a..29a8f483a 100644 --- a/utils/funcexp/func_unix_timestamp.cpp +++ b/utils/funcexp/func_unix_timestamp.cpp @@ -84,6 +84,12 @@ int64_t Func_unix_timestamp::getIntVal(rowgroup::Row& row, sec = (uint32_t)((val >> 20) & 0x3f); break; + case CalpontSystemCatalog::TIMESTAMP: + val = parm[0]->data()->getIntVal(row, isNull); + //TimeStamp timeStamp(val); + return ((val >> 20) & 0xFFFFFFFFFFFULL); + break; + case CalpontSystemCatalog::CHAR: case CalpontSystemCatalog::VARCHAR: case CalpontSystemCatalog::TEXT: diff --git a/utils/funcexp/func_week.cpp b/utils/funcexp/func_week.cpp index ec6131b26..326f1c13a 100644 --- a/utils/funcexp/func_week.cpp +++ b/utils/funcexp/func_week.cpp @@ -75,6 +75,18 @@ int64_t Func_week::getIntVal(rowgroup::Row& row, day = (uint32_t)((val >> 38) & 0x3f); break; + case CalpontSystemCatalog::TIMESTAMP: + { + dataconvert::TimeStamp timestamp(parm[0]->data()->getTimestampIntVal(row, isNull)); + int64_t seconds = timestamp.second; + dataconvert::MySQLTime m_time; + dataconvert::gmtSecToMySQLTime(seconds, m_time, fTimeZone); + year = m_time.year; + month = m_time.month; + day = m_time.day; + break; + } + // Time adds to now() and then gets value case CalpontSystemCatalog::TIME: aDateTime = static_cast(nowDatetime()); diff --git a/utils/funcexp/func_weekday.cpp b/utils/funcexp/func_weekday.cpp index 6b5c1f55d..9b6cc8c63 100644 --- a/utils/funcexp/func_weekday.cpp +++ b/utils/funcexp/func_weekday.cpp @@ -71,6 +71,18 @@ int64_t Func_weekday::getIntVal(rowgroup::Row& row, day = (uint32_t)((val >> 38) & 0x3f); break; + case CalpontSystemCatalog::TIMESTAMP: + { + dataconvert::TimeStamp timestamp(parm[0]->data()->getTimestampIntVal(row, isNull)); + int64_t seconds = timestamp.second; + dataconvert::MySQLTime m_time; + dataconvert::gmtSecToMySQLTime(seconds, m_time, fTimeZone); + year = m_time.year; + month = m_time.month; + day = m_time.day; + break; + } + // Time adds to now() and then gets value case CalpontSystemCatalog::TIME: aDateTime = static_cast(nowDatetime()); diff --git a/utils/funcexp/func_year.cpp b/utils/funcexp/func_year.cpp index 6e71b8a03..94abdd79a 100644 --- a/utils/funcexp/func_year.cpp +++ b/utils/funcexp/func_year.cpp @@ -61,6 +61,15 @@ int64_t Func_year::getIntVal(rowgroup::Row& row, val = parm[0]->data()->getIntVal(row, isNull); return (unsigned)((val >> 48) & 0xffff); + case execplan::CalpontSystemCatalog::TIMESTAMP: + { + dataconvert::TimeStamp timestamp(parm[0]->data()->getIntVal(row, isNull)); + int64_t seconds = timestamp.second; + dataconvert::MySQLTime m_time; + dataconvert::gmtSecToMySQLTime(seconds, m_time, fTimeZone); + return m_time.year; + } + // Time adds to now() and then gets value case CalpontSystemCatalog::TIME: aDateTime = static_cast(nowDatetime()); diff --git a/utils/funcexp/func_yearweek.cpp b/utils/funcexp/func_yearweek.cpp index 5568b763c..3ae72a7f2 100644 --- a/utils/funcexp/func_yearweek.cpp +++ b/utils/funcexp/func_yearweek.cpp @@ -78,6 +78,18 @@ int64_t Func_yearweek::getIntVal(rowgroup::Row& row, day = (uint32_t)((val >> 38) & 0x3f); break; + case execplan::CalpontSystemCatalog::TIMESTAMP: + { + dataconvert::TimeStamp timestamp(parm[0]->data()->getIntVal(row, isNull)); + int64_t seconds = timestamp.second; + dataconvert::MySQLTime m_time; + dataconvert::gmtSecToMySQLTime(seconds, m_time, fTimeZone); + year = m_time.year; + month = m_time.month; + day = m_time.day; + break; + } + // Time adds to now() and then gets value case CalpontSystemCatalog::TIME: aDateTime = static_cast(nowDatetime()); diff --git a/utils/funcexp/funcexp.cpp b/utils/funcexp/funcexp.cpp index d933474f7..c119dc549 100644 --- a/utils/funcexp/funcexp.cpp +++ b/utils/funcexp/funcexp.cpp @@ -285,6 +285,18 @@ void FuncExp::evaluate(rowgroup::Row& row, std::vector& expressi break; } + case CalpontSystemCatalog::TIMESTAMP: + { + int64_t val = expression[i]->getTimestampIntVal(row, isNull); + + if (isNull) + row.setUintField<8>(TIMESTAMPNULL, expression[i]->outputIndex()); + else + row.setUintField<8>(val, expression[i]->outputIndex()); + + break; + } + case CalpontSystemCatalog::TIME: { int64_t val = expression[i]->getTimeIntVal(row, isNull); diff --git a/utils/funcexp/functor.cpp b/utils/funcexp/functor.cpp index 54060d74b..28fd64cbe 100644 --- a/utils/funcexp/functor.cpp +++ b/utils/funcexp/functor.cpp @@ -110,6 +110,22 @@ uint64_t Func::stringToDatetime(const string str) return ret; } +uint64_t Func::stringToTimestamp(const string str) +{ + int64_t ret = DataConvert::stringToTimestamp(str, fTimeZone); + + if (ret == -1) + { + Message::Args args; + args.add("timestamp"); + args.add(str); + unsigned errcode = ERR_INCORRECT_VALUE; + throw IDBExcept(IDBErrorInfo::instance()->errorMsg(errcode, args), errcode); + } + + return ret; +} + int64_t Func::stringToTime(const string str) { int64_t ret = DataConvert::stringToTime(str); @@ -143,6 +159,11 @@ uint64_t Func::intToDatetime(int64_t i) return i; } +uint64_t Func::intToTimestamp(int64_t i) +{ + return i; +} + int64_t Func::intToTime(int64_t i) { // Don't think we need to do anything here? diff --git a/utils/funcexp/functor.h b/utils/funcexp/functor.h index 890fddeae..8eb494313 100644 --- a/utils/funcexp/functor.h +++ b/utils/funcexp/functor.h @@ -70,6 +70,16 @@ public: { fFuncName = funcName; } + + const std::string timeZone() const + { + return fTimeZone; + } + void timeZone(const std::string timeZone) + { + fTimeZone = timeZone; + } + virtual execplan::CalpontSystemCatalog::ColType operationType(FunctionParm& fp, execplan::CalpontSystemCatalog::ColType& resultType) = 0; virtual int64_t getIntVal(rowgroup::Row& row, @@ -124,6 +134,14 @@ public: return intToDatetime(getIntVal(row, fp, isNull, op_ct)); } + virtual int64_t getTimestampIntVal(rowgroup::Row& row, + FunctionParm& fp, + bool& isNull, + execplan::CalpontSystemCatalog::ColType& op_ct) + { + return intToTimestamp(getIntVal(row, fp, isNull, op_ct)); + } + virtual int64_t getTimeIntVal(rowgroup::Row& row, FunctionParm& fp, bool& isNull, @@ -166,10 +184,12 @@ public: protected: virtual uint32_t stringToDate(std::string); virtual uint64_t stringToDatetime(std::string); + virtual uint64_t stringToTimestamp(std::string); virtual int64_t stringToTime(std::string); virtual uint32_t intToDate(int64_t); virtual uint64_t intToDatetime(int64_t); + virtual uint64_t intToTimestamp(int64_t); virtual int64_t intToTime(int64_t); virtual std::string intToString(int64_t); @@ -181,6 +201,7 @@ protected: virtual int64_t addTime(dataconvert::Time& dt1, dataconvert::Time& dt2); std::string fFuncName; + std::string fTimeZone; private: //defaults okay diff --git a/utils/funcexp/functor_all.h b/utils/funcexp/functor_all.h index 71d857132..a1daa0687 100644 --- a/utils/funcexp/functor_all.h +++ b/utils/funcexp/functor_all.h @@ -111,6 +111,11 @@ public: bool& isNull, execplan::CalpontSystemCatalog::ColType& op_ct); + int64_t getTimestampIntVal(rowgroup::Row& row, + FunctionParm& fp, + bool& isNull, + execplan::CalpontSystemCatalog::ColType& op_ct); + int64_t getTimeIntVal(rowgroup::Row& row, FunctionParm& fp, bool& isNull, @@ -168,6 +173,11 @@ public: bool& isNull, execplan::CalpontSystemCatalog::ColType& op_ct); + int64_t getTimestampIntVal(rowgroup::Row& row, + FunctionParm& fp, + bool& isNull, + execplan::CalpontSystemCatalog::ColType& op_ct); + int64_t getTimeIntVal(rowgroup::Row& row, FunctionParm& fp, bool& isNull, @@ -221,6 +231,11 @@ public: bool& isNull, execplan::CalpontSystemCatalog::ColType& op_ct); + int64_t getTimestampIntVal(rowgroup::Row& row, + FunctionParm& fp, + bool& isNull, + execplan::CalpontSystemCatalog::ColType& op_ct); + int64_t getTimeIntVal(rowgroup::Row& row, FunctionParm& fp, bool& isNull, @@ -273,6 +288,11 @@ public: bool& isNull, execplan::CalpontSystemCatalog::ColType& op_ct); + int64_t getTimestampIntVal(rowgroup::Row& row, + FunctionParm& fp, + bool& isNull, + execplan::CalpontSystemCatalog::ColType& op_ct); + int64_t getTimeIntVal(rowgroup::Row& row, FunctionParm& fp, bool& isNull, @@ -335,6 +355,11 @@ public: bool& isNull, execplan::CalpontSystemCatalog::ColType& op_ct); + int64_t getTimestampIntVal(rowgroup::Row& row, + FunctionParm& fp, + bool& isNull, + execplan::CalpontSystemCatalog::ColType& op_ct); + int64_t getTimeIntVal(rowgroup::Row& row, FunctionParm& fp, bool& isNull, @@ -388,6 +413,11 @@ public: bool& isNull, execplan::CalpontSystemCatalog::ColType& op_ct); + int64_t getTimestampIntVal(rowgroup::Row& row, + FunctionParm& fp, + bool& isNull, + execplan::CalpontSystemCatalog::ColType& op_ct); + int64_t getTimeIntVal(rowgroup::Row& row, FunctionParm& fp, bool& isNull, @@ -440,6 +470,11 @@ public: bool& isNull, execplan::CalpontSystemCatalog::ColType& op_ct); + int64_t getTimestampIntVal(rowgroup::Row& row, + FunctionParm& fp, + bool& isNull, + execplan::CalpontSystemCatalog::ColType& op_ct); + int64_t getTimeIntVal(rowgroup::Row& row, FunctionParm& fp, bool& isNull, @@ -477,6 +512,11 @@ public: bool& isNull, execplan::CalpontSystemCatalog::ColType& op_ct); + int64_t getTimestampIntVal(rowgroup::Row& row, + FunctionParm& fp, + bool& isNull, + execplan::CalpontSystemCatalog::ColType& op_ct); + int64_t getIntVal(rowgroup::Row& row, FunctionParm& fp, bool& isNull, diff --git a/utils/funcexp/functor_bool.h b/utils/funcexp/functor_bool.h index 27e32fbbc..28077e878 100644 --- a/utils/funcexp/functor_bool.h +++ b/utils/funcexp/functor_bool.h @@ -106,6 +106,15 @@ public: return 0; } + int64_t getTimestampIntVal(rowgroup::Row& row, + FunctionParm& fp, + bool& isNull, + execplan::CalpontSystemCatalog::ColType& op_ct) + { + isNull = true; + return 0; + } + int64_t getTimeIntVal(rowgroup::Row& row, FunctionParm& fp, bool& isNull, diff --git a/utils/funcexp/functor_dtm.h b/utils/funcexp/functor_dtm.h index bddd78202..fddd0a002 100644 --- a/utils/funcexp/functor_dtm.h +++ b/utils/funcexp/functor_dtm.h @@ -146,6 +146,11 @@ public: bool& isNull, execplan::CalpontSystemCatalog::ColType& op_ct); + int64_t getTimestampIntVal(rowgroup::Row& row, + FunctionParm& fp, + bool& isNull, + execplan::CalpontSystemCatalog::ColType& op_ct); + int64_t getTimeIntVal(rowgroup::Row& row, FunctionParm& fp, bool& isNull, @@ -342,6 +347,11 @@ public: bool& isNull, execplan::CalpontSystemCatalog::ColType& op_ct); + int64_t getTimestampIntVal(rowgroup::Row& row, + FunctionParm& fp, + bool& isNull, + execplan::CalpontSystemCatalog::ColType& op_ct); + int64_t getTimeIntVal(rowgroup::Row& row, FunctionParm& fp, bool& isNull, @@ -379,6 +389,11 @@ public: bool& isNull, execplan::CalpontSystemCatalog::ColType& op_ct); + int64_t getTimestampIntVal(rowgroup::Row& row, + FunctionParm& fp, + bool& isNull, + execplan::CalpontSystemCatalog::ColType& op_ct); + int64_t getTimeIntVal(rowgroup::Row& row, FunctionParm& fp, bool& isNull, @@ -411,6 +426,11 @@ public: bool& isNull, execplan::CalpontSystemCatalog::ColType& op_ct); + int64_t getTimestampIntVal(rowgroup::Row& row, + FunctionParm& fp, + bool& isNull, + execplan::CalpontSystemCatalog::ColType& op_ct); + int64_t getTimeIntVal(rowgroup::Row& row, FunctionParm& fp, bool& isNull, @@ -497,6 +517,12 @@ public: FunctionParm& fp, bool& isNull, execplan::CalpontSystemCatalog::ColType& op_ct); + + int64_t getTimestampIntVal(rowgroup::Row& row, + FunctionParm& fp, + bool& isNull, + execplan::CalpontSystemCatalog::ColType& op_ct); + int64_t getTimeIntVal(rowgroup::Row& row, FunctionParm& fp, bool& isNull, diff --git a/utils/funcexp/functor_real.h b/utils/funcexp/functor_real.h index a12759333..a5ae4d317 100644 --- a/utils/funcexp/functor_real.h +++ b/utils/funcexp/functor_real.h @@ -697,6 +697,11 @@ public: bool& isNull, execplan::CalpontSystemCatalog::ColType& op_ct); + int64_t getTimestampIntVal(rowgroup::Row& row, + FunctionParm& fp, + bool& isNull, + execplan::CalpontSystemCatalog::ColType& op_ct); + int64_t getTimeIntVal(rowgroup::Row& row, FunctionParm& fp, bool& isNull, diff --git a/utils/funcexp/functor_str.h b/utils/funcexp/functor_str.h index fc8de1d9f..2f772974a 100644 --- a/utils/funcexp/functor_str.h +++ b/utils/funcexp/functor_str.h @@ -97,6 +97,15 @@ public: return (isNull ? 0 : stringToDatetime(str)); } + int64_t getTimestampIntVal(rowgroup::Row& row, + FunctionParm& fp, + bool& isNull, + execplan::CalpontSystemCatalog::ColType& op_ct) + { + std::string str = getStrVal(row, fp, isNull, op_ct); + return (isNull ? 0 : stringToTimestamp(str)); + } + int64_t getTimeIntVal(rowgroup::Row& row, FunctionParm& fp, bool& isNull, @@ -214,6 +223,11 @@ public: FunctionParm& fp, bool& isNull, execplan::CalpontSystemCatalog::ColType& op_ct); + + int64_t getTimestampIntVal(rowgroup::Row& row, + FunctionParm& fp, + bool& isNull, + execplan::CalpontSystemCatalog::ColType& op_ct); }; @@ -547,6 +561,11 @@ public: FunctionParm& fp, bool& isNull, execplan::CalpontSystemCatalog::ColType& op_ct); + + int64_t getTimestampIntVal(rowgroup::Row& row, + FunctionParm& fp, + bool& isNull, + execplan::CalpontSystemCatalog::ColType& op_ct); }; @@ -696,6 +715,11 @@ public: bool& isNull, execplan::CalpontSystemCatalog::ColType& op_ct); + int64_t getTimestampIntVal(rowgroup::Row& row, + FunctionParm& fp, + bool& isNull, + execplan::CalpontSystemCatalog::ColType& op_ct); + int64_t getTimeIntVal(rowgroup::Row& row, FunctionParm& fp, bool& isNull, diff --git a/utils/idbdatafile/IDBPolicy.cpp b/utils/idbdatafile/IDBPolicy.cpp index 59fb6ea80..8eec07dd0 100644 --- a/utils/idbdatafile/IDBPolicy.cpp +++ b/utils/idbdatafile/IDBPolicy.cpp @@ -84,8 +84,15 @@ void IDBPolicy::init( bool bEnableLogging, bool bUseRdwrMemBuffer, const string& else { cout << tmpfilepath << endl; - - if (!boost::filesystem::create_directory(tmpfilepath)) + bool itWorked = false; + + try + { + itWorked = boost::filesystem::create_directories(tmpfilepath); + } + catch (...) + { } + if (!itWorked) { // We failed to create the scratch directory ostringstream oss; diff --git a/utils/rowgroup/rowaggregation.cpp b/utils/rowgroup/rowaggregation.cpp index 003d58281..a6e529274 100644 --- a/utils/rowgroup/rowaggregation.cpp +++ b/utils/rowgroup/rowaggregation.cpp @@ -129,6 +129,11 @@ inline uint64_t getUintNullValue(int colType, int colWidth = 0) return joblist::DATETIMENULL; } + case execplan::CalpontSystemCatalog::TIMESTAMP: + { + return joblist::TIMESTAMPNULL; + } + case execplan::CalpontSystemCatalog::TIME: { return joblist::TIMENULL; @@ -567,6 +572,12 @@ inline bool RowAggregation::isNull(const RowGroup* pRowGroup, const Row& row, in break; } + case execplan::CalpontSystemCatalog::TIMESTAMP: + { + ret = ((uint64_t)row.getUintField(col) == joblist::TIMESTAMPNULL); + break; + } + case execplan::CalpontSystemCatalog::TIME: { ret = ((uint64_t)row.getUintField(col) == joblist::TIMENULL); @@ -1082,6 +1093,7 @@ void RowAggregation::initMapData(const Row& rowIn) case execplan::CalpontSystemCatalog::DATE: case execplan::CalpontSystemCatalog::DATETIME: + case execplan::CalpontSystemCatalog::TIMESTAMP: case execplan::CalpontSystemCatalog::TIME: { fRow.setUintField(rowIn.getUintField(colIn), colOut); @@ -1219,6 +1231,7 @@ void RowAggregation::makeAggFieldsNull(Row& row) case execplan::CalpontSystemCatalog::DATE: case execplan::CalpontSystemCatalog::DATETIME: + case execplan::CalpontSystemCatalog::TIMESTAMP: case execplan::CalpontSystemCatalog::TIME: { row.setUintField(getUintNullValue(colDataType), colOut); @@ -1318,6 +1331,7 @@ void RowAggregation::doMinMax(const Row& rowIn, int64_t colIn, int64_t colOut, i case execplan::CalpontSystemCatalog::DATE: case execplan::CalpontSystemCatalog::DATETIME: + case execplan::CalpontSystemCatalog::TIMESTAMP: case execplan::CalpontSystemCatalog::TIME: { uint64_t valIn = rowIn.getUintField(colIn); @@ -1578,6 +1592,16 @@ void RowAggregation::doBitOp(const Row& rowIn, int64_t colIn, int64_t colOut, in break; } + case execplan::CalpontSystemCatalog::TIMESTAMP: + { + uint64_t timestamp = rowIn.getUintField(colIn); + string str = DataConvert::timestampToString1(timestamp, fTimeZone); + // strip off micro seconds + str = str.substr(0, 14); + valIn = strtoll(str.c_str(), NULL, 10); + break; + } + case execplan::CalpontSystemCatalog::TIME: { int64_t dtm = rowIn.getUintField(colIn); @@ -1641,6 +1665,8 @@ void RowAggregation::serialize(messageqcpp::ByteStream& bs) const for (uint64_t i = 0; i < functionCount; i++) fFunctionCols[i]->serialize(bs); + + bs << fTimeZone; } @@ -1685,6 +1711,8 @@ void RowAggregation::deserialize(messageqcpp::ByteStream& bs) funct->deserialize(bs); fFunctionCols.push_back(funct); } + + bs >> fTimeZone; } @@ -2128,6 +2156,22 @@ void RowAggregation::doUDAF(const Row& rowIn, int64_t colIn, int64_t colOut, break; } + case execplan::CalpontSystemCatalog::TIMESTAMP: + { + datum.dataType = execplan::CalpontSystemCatalog::UBIGINT; + + if (cc) + { + datum.columnData = cc->getTimestampIntVal(const_cast(rowIn), bIsNull); + } + else + { + datum.columnData = rowIn.getUintField(colIn); + } + + break; + } + case execplan::CalpontSystemCatalog::TIME: { datum.dataType = execplan::CalpontSystemCatalog::BIGINT; @@ -2700,6 +2744,7 @@ void RowAggregationUM::SetUDAFValue(static_any::any& valOut, int64_t colOut) case execplan::CalpontSystemCatalog::DATE: case execplan::CalpontSystemCatalog::DATETIME: + case execplan::CalpontSystemCatalog::TIMESTAMP: if (valOut.compatible(ulongTypeId)) { uintOut = valOut.cast(); @@ -2947,6 +2992,7 @@ void RowAggregationUM::SetUDAFAnyValue(static_any::any& valOut, int64_t colOut) case execplan::CalpontSystemCatalog::DATE: case execplan::CalpontSystemCatalog::DATETIME: + case execplan::CalpontSystemCatalog::TIMESTAMP: fRow.setUintField<8>(uintOut, colOut); break; @@ -3263,6 +3309,7 @@ void RowAggregationUM::doNullConstantAggregate(const ConstantAggData& aggData, u case execplan::CalpontSystemCatalog::DATE: case execplan::CalpontSystemCatalog::DATETIME: + case execplan::CalpontSystemCatalog::TIMESTAMP: { fRow.setUintField(getUintNullValue(colDataType), colOut); } @@ -3472,6 +3519,12 @@ void RowAggregationUM::doNotNullConstantAggregate(const ConstantAggData& aggData } break; + case execplan::CalpontSystemCatalog::TIMESTAMP: + { + fRow.setUintField(DataConvert::stringToTimestamp(aggData.fConstValue, fTimeZone), colOut); + } + break; + case execplan::CalpontSystemCatalog::TIME: { fRow.setIntField(DataConvert::stringToTime(aggData.fConstValue), colOut); @@ -3573,6 +3626,7 @@ void RowAggregationUM::doNotNullConstantAggregate(const ConstantAggData& aggData case execplan::CalpontSystemCatalog::DATE: case execplan::CalpontSystemCatalog::DATETIME: + case execplan::CalpontSystemCatalog::TIMESTAMP: case execplan::CalpontSystemCatalog::TIME: case execplan::CalpontSystemCatalog::CHAR: case execplan::CalpontSystemCatalog::VARCHAR: @@ -3640,6 +3694,7 @@ void RowAggregationUM::doNotNullConstantAggregate(const ConstantAggData& aggData break; case execplan::CalpontSystemCatalog::DATETIME: + case execplan::CalpontSystemCatalog::TIMESTAMP: case execplan::CalpontSystemCatalog::TIME: { fRow.setUintField(0, colOut); @@ -3782,6 +3837,12 @@ void RowAggregationUM::doNotNullConstantAggregate(const ConstantAggData& aggData } break; + case execplan::CalpontSystemCatalog::TIMESTAMP: + { + datum.columnData = DataConvert::stringToTimestamp(aggData.fConstValue, fTimeZone); + } + break; + case execplan::CalpontSystemCatalog::TIME: { datum.columnData = DataConvert::stringToTime(aggData.fConstValue); diff --git a/utils/rowgroup/rowaggregation.h b/utils/rowgroup/rowaggregation.h index 064775ba8..17e903f3f 100644 --- a/utils/rowgroup/rowaggregation.h +++ b/utils/rowgroup/rowaggregation.h @@ -423,6 +423,7 @@ struct GroupConcat std::vector > fOrderCond; // position to order by [asc/desc] joblist::ResourceManager* fRm; // resource manager boost::shared_ptr fSessionMemLimit; + std::string fTimeZone; GroupConcat() : fRm(NULL) {} }; @@ -606,6 +607,15 @@ public: return fAggMapKeyCount; } + inline void timeZone(const std::string& timeZone) + { + fTimeZone = timeZone; + } + inline const std::string& timeZone() const + { + return fTimeZone; + } + protected: virtual void initialize(); virtual void initMapData(const Row& row); @@ -686,6 +696,8 @@ protected: boost::scoped_ptr fHasher; boost::scoped_ptr fEq; + std::string fTimeZone; + //TODO: try to get rid of these friend decl's. AggHasher & Comparator //need access to rowgroup storage holding the rows to hash & ==. friend class AggHasher; diff --git a/utils/rowgroup/rowgroup.cpp b/utils/rowgroup/rowgroup.cpp index b4ab56ff1..e16f12c30 100644 --- a/utils/rowgroup/rowgroup.cpp +++ b/utils/rowgroup/rowgroup.cpp @@ -709,6 +709,10 @@ void Row::initToNull() *((uint64_t*) &data[offsets[i]]) = joblist::DATETIMENULL; break; + case CalpontSystemCatalog::TIMESTAMP: + *((uint64_t*) &data[offsets[i]]) = joblist::TIMESTAMPNULL; + break; + case CalpontSystemCatalog::TIME: *((uint64_t*) &data[offsets[i]]) = joblist::TIMENULL; break; @@ -848,6 +852,9 @@ bool Row::isNullValue(uint32_t colIndex) const case CalpontSystemCatalog::DATETIME: return (*((uint64_t*) &data[offsets[colIndex]]) == joblist::DATETIMENULL); + case CalpontSystemCatalog::TIMESTAMP: + return (*((uint64_t*) &data[offsets[colIndex]]) == joblist::TIMESTAMPNULL); + case CalpontSystemCatalog::TIME: return (*((uint64_t*) &data[offsets[colIndex]]) == joblist::TIMENULL); diff --git a/utils/threadpool/threadpool.h b/utils/threadpool/threadpool.h index 27d87fd12..f0ddf360c 100644 --- a/utils/threadpool/threadpool.h +++ b/utils/threadpool/threadpool.h @@ -77,7 +77,7 @@ public: boost::thread* create_thread(F threadfunc) { boost::lock_guard guard(m); -#if defined(__GNUC__) && __GNUC__ >= 7 +#if __cplusplus >= 201103L std::unique_ptr new_thread(new boost::thread(threadfunc)); #else std::auto_ptr new_thread(new boost::thread(threadfunc)); diff --git a/utils/windowfunction/idborderby.cpp b/utils/windowfunction/idborderby.cpp index 0cc39017c..2f05dcca1 100644 --- a/utils/windowfunction/idborderby.cpp +++ b/utils/windowfunction/idborderby.cpp @@ -318,6 +318,7 @@ void CompareRule::compileRules(const std::vector& spec, const rowgr case CalpontSystemCatalog::DATE: case CalpontSystemCatalog::DATETIME: + case CalpontSystemCatalog::TIMESTAMP: case CalpontSystemCatalog::TIME: { Compare* c = new UintCompare(*i); @@ -451,6 +452,7 @@ bool EqualCompData::operator()(Row::Pointer a, Row::Pointer b) case CalpontSystemCatalog::UBIGINT: case CalpontSystemCatalog::DATE: case CalpontSystemCatalog::DATETIME: + case CalpontSystemCatalog::TIMESTAMP: case CalpontSystemCatalog::TIME: { // equal compare. ignore sign and null diff --git a/utils/windowfunction/wf_lead_lag.cpp b/utils/windowfunction/wf_lead_lag.cpp index c720e9db2..cb15f3490 100644 --- a/utils/windowfunction/wf_lead_lag.cpp +++ b/utils/windowfunction/wf_lead_lag.cpp @@ -78,6 +78,7 @@ boost::shared_ptr WF_lead_lag::makeFunction(int id, const case CalpontSystemCatalog::UDECIMAL: case CalpontSystemCatalog::DATE: case CalpontSystemCatalog::DATETIME: + case CalpontSystemCatalog::TIMESTAMP: case CalpontSystemCatalog::TIME: { func.reset(new WF_lead_lag(id, name)); diff --git a/utils/windowfunction/wf_min_max.cpp b/utils/windowfunction/wf_min_max.cpp index 511fbb4eb..4ae76e84d 100644 --- a/utils/windowfunction/wf_min_max.cpp +++ b/utils/windowfunction/wf_min_max.cpp @@ -77,6 +77,7 @@ boost::shared_ptr WF_min_max::makeFunction(int id, const case CalpontSystemCatalog::UDECIMAL: case CalpontSystemCatalog::DATE: case CalpontSystemCatalog::DATETIME: + case CalpontSystemCatalog::TIMESTAMP: case CalpontSystemCatalog::TIME: { func.reset(new WF_min_max(id, name)); diff --git a/utils/windowfunction/wf_nth_value.cpp b/utils/windowfunction/wf_nth_value.cpp index c6eea0ace..c736c0e92 100644 --- a/utils/windowfunction/wf_nth_value.cpp +++ b/utils/windowfunction/wf_nth_value.cpp @@ -78,6 +78,7 @@ boost::shared_ptr WF_nth_value::makeFunction(int id, cons case CalpontSystemCatalog::UDECIMAL: case CalpontSystemCatalog::DATE: case CalpontSystemCatalog::DATETIME: + case CalpontSystemCatalog::TIMESTAMP: case CalpontSystemCatalog::TIME: { func.reset(new WF_nth_value(id, name)); diff --git a/utils/windowfunction/wf_percentile.cpp b/utils/windowfunction/wf_percentile.cpp index acf137c2e..0c93b1434 100644 --- a/utils/windowfunction/wf_percentile.cpp +++ b/utils/windowfunction/wf_percentile.cpp @@ -83,6 +83,7 @@ boost::shared_ptr WF_percentile::makeFunction(int id, con case CalpontSystemCatalog::UDECIMAL: case CalpontSystemCatalog::DATE: case CalpontSystemCatalog::DATETIME: + case CalpontSystemCatalog::TIMESTAMP: case CalpontSystemCatalog::TIME: { func.reset(new WF_percentile(id, name)); diff --git a/utils/windowfunction/wf_udaf.cpp b/utils/windowfunction/wf_udaf.cpp index 903dda613..0eb753152 100644 --- a/utils/windowfunction/wf_udaf.cpp +++ b/utils/windowfunction/wf_udaf.cpp @@ -271,6 +271,7 @@ bool WF_udaf::dropValues(int64_t b, int64_t e) case CalpontSystemCatalog::TIME: case CalpontSystemCatalog::DATE: case CalpontSystemCatalog::DATETIME: + case CalpontSystemCatalog::TIMESTAMP: { uint64_t valIn; @@ -687,6 +688,7 @@ void WF_udaf::SetUDAFValue(static_any::any& valOut, int64_t colOut, case execplan::CalpontSystemCatalog::UBIGINT: case execplan::CalpontSystemCatalog::DATE: case execplan::CalpontSystemCatalog::DATETIME: + case execplan::CalpontSystemCatalog::TIMESTAMP: case execplan::CalpontSystemCatalog::TIME: if (valOut.empty()) { @@ -931,6 +933,7 @@ void WF_udaf::operator()(int64_t b, int64_t e, int64_t c) case CalpontSystemCatalog::TIME: case CalpontSystemCatalog::DATE: case CalpontSystemCatalog::DATETIME: + case CalpontSystemCatalog::TIMESTAMP: { uint64_t valIn; diff --git a/utils/windowfunction/windowfunctiontype.cpp b/utils/windowfunction/windowfunctiontype.cpp index 776e63c73..cea636402 100644 --- a/utils/windowfunction/windowfunctiontype.cpp +++ b/utils/windowfunction/windowfunctiontype.cpp @@ -91,6 +91,7 @@ map colType2String = assign::map_list_of (CalpontSystemCatalog::STRINT, "INTERNAL SHORT STRING") (CalpontSystemCatalog::TEXT, "TEXT") (CalpontSystemCatalog::TIME, "TIME") + (CalpontSystemCatalog::TIMESTAMP, "TIMESTAMP") ; @@ -602,6 +603,7 @@ void* WindowFunctionType::getNullValueByType(int ct, int pos) static uint64_t dateNull = joblist::DATENULL; static uint64_t datetimeNull = joblist::DATETIMENULL; static uint64_t timeNull = joblist::TIMENULL; + static uint64_t timestampNull = joblist::TIMESTAMPNULL; // static uint64_t char1Null = joblist::CHAR1NULL; // static uint64_t char2Null = joblist::CHAR2NULL; // static uint64_t char4Null = joblist::CHAR4NULL; @@ -637,6 +639,10 @@ void* WindowFunctionType::getNullValueByType(int ct, int pos) v = &datetimeNull; break; + case CalpontSystemCatalog::TIMESTAMP: + v = ×tampNull; + break; + case CalpontSystemCatalog::TIME: v = &timeNull; break; diff --git a/versioning/BRM/extentmap.cpp b/versioning/BRM/extentmap.cpp index 14390dab3..b7733097d 100644 --- a/versioning/BRM/extentmap.cpp +++ b/versioning/BRM/extentmap.cpp @@ -1253,14 +1253,23 @@ void ExtentMap::loadVersion4(IDBDataFile* in) growEMShmseg(nrows); } - for (int i = 0; i < emNumElements; i++) + size_t progress = 0, writeSize = emNumElements * sizeof(EMEntry); + int err; + char *writePos = (char *) fExtentMap; + while (progress < writeSize) { - if (in->read((char*) &fExtentMap[i], sizeof(EMEntry)) != sizeof(EMEntry)) + err = in->read(writePos + progress, writeSize - progress); + if (err <= 0) { log_errno("ExtentMap::loadVersion4(): read "); throw runtime_error("ExtentMap::loadVersion4(): read failed. Check the error log."); } + progress += (uint) err; + } + + for (int i = 0; i < emNumElements; i++) + { reserveLBIDRange(fExtentMap[i].range.start, fExtentMap[i].range.size); //@bug 1911 - verify status value is valid @@ -1328,6 +1337,7 @@ void ExtentMap::load(const string& filename, bool fixFL) throw; } + // XXXPAT: Forcing the IDB path. Remove the fstream path once we see this works. if (true || IDBPolicy::useHdfs()) { const char* filename_p = filename.c_str(); @@ -1443,14 +1453,15 @@ void ExtentMap::save(const string& filename) throw runtime_error("ExtentMap::save(): got request to save an empty BRM"); } + // XXXPAT: I don't know why there are two options here. It can just use the IDBDataFile stuff. + // Forcing the IDB option to execute for now. Leaving the old fstream version there in case we find there's + // a case the IDB option doesn't work. if (true || IDBPolicy::useHdfs()) { - utmp = ::umask(0); const char* filename_p = filename.c_str(); scoped_ptr out(IDBDataFile::open( IDBPolicy::getType(filename_p, IDBPolicy::WRITEENG), filename_p, "wb", IDBDataFile::USE_VBUF)); - ::umask(utmp); if (!out) { @@ -1482,52 +1493,72 @@ void ExtentMap::save(const string& filename) } allocdSize = fEMShminfo->allocdSize / sizeof(EMEntry); - const int emEntrySize = sizeof(EMEntry); + //const int emEntrySize = sizeof(EMEntry); + int first = -1, last = -1, err; + size_t progress, writeSize; for (i = 0; i < allocdSize; i++) { - if (fExtentMap[i].range.size > 0) + if (fExtentMap[i].range.size > 0 && first == -1) + first = i; + else if (fExtentMap[i].range.size <= 0 && first != -1) { - try + last = i; + writeSize = (last - first) * sizeof(EMEntry); + progress = 0; + char *writePos = (char *) &fExtentMap[first]; + while (progress < writeSize) { - bytes = out->write((char*) &fExtentMap[i], emEntrySize); - - if (bytes != emEntrySize) + err = out->write(writePos + progress, writeSize - progress); + if (err < 0) + { + releaseFreeList(READ); + releaseEMEntryTable(READ); throw ios_base::failure("ExtentMap::save(): write failed. Check the error log."); + } + progress += err; } - catch (...) + first = -1; + } + } + if (first != -1) + { + writeSize = (allocdSize - first) * sizeof(EMEntry); + progress = 0; + char *writePos = (char *) &fExtentMap[first]; + while (progress < writeSize) + { + err = out->write(writePos + progress, writeSize - progress); + if (err < 0) { releaseFreeList(READ); releaseEMEntryTable(READ); - throw; + throw ios_base::failure("ExtentMap::save(): write failed. Check the error log."); } + progress += err; } } - allocdSize = fFLShminfo->allocdSize / sizeof(InlineLBIDRange); - const int inlineLbidRangeSize = sizeof(InlineLBIDRange); + //allocdSize = fFLShminfo->allocdSize / sizeof(InlineLBIDRange); + //const int inlineLbidRangeSize = sizeof(InlineLBIDRange); - for (i = 0; i < allocdSize; i++) + progress = 0; + writeSize = fFLShminfo->allocdSize; + char *writePos = (char *) fFreeList; + while (progress < writeSize) { -// if (fFreeList[i].size > 0) { - try - { - int bytes = out->write((char*) &fFreeList[i], inlineLbidRangeSize); - - if (bytes != inlineLbidRangeSize) - throw ios_base::failure("ExtentMap::save(): write failed. Check the error log."); - } - catch (...) + err = out->write(writePos + progress, writeSize - progress); + if (err < 0) { releaseFreeList(READ); releaseEMEntryTable(READ); - throw; + throw ios_base::failure("ExtentMap::save(): write failed. Check the error log."); } -// } + progress += err; } } - else + else // this is the fstream version to be expired { ofstream out; diff --git a/versioning/BRM/oidserver.cpp b/versioning/BRM/oidserver.cpp index de3e23b49..6b8887012 100644 --- a/versioning/BRM/oidserver.cpp +++ b/versioning/BRM/oidserver.cpp @@ -153,6 +153,7 @@ void OIDServer::writeData(uint8_t* buf, off_t offset, int size) const if (size == 0) return; + // XXXPAT: Forcing the IDB* path. Get rid of the fstream path when appropriate. if (true || IDBPolicy::useHdfs()) { for (errCount = 0; errCount < MaxRetries && seekerr != offset; errCount++) @@ -231,6 +232,7 @@ void OIDServer::readData(uint8_t* buf, off_t offset, int size) const if (size == 0) return; + // XXXPAT: Forcing the IDB* path. Get rid of the fstream path when appropriate. if (true || IDBPolicy::useHdfs()) { for (errCount = 0; errCount < MaxRetries && seekerr != offset; errCount++) @@ -349,14 +351,22 @@ void OIDServer::initializeBitmap() const writeData(buf, 0, HeaderSize); // reset buf to all 0's and write the bitmap - for (i = 0; i < HeaderSize; i++) - buf[i] = 0; + //for (i = 0; i < HeaderSize; i++) + // buf[i] = 0; - for (i = 0; i < bitmapSize; i += HeaderSize) - writeData(buf, HeaderSize + i, (bitmapSize - i > HeaderSize ? HeaderSize : bitmapSize - i)); + //for (i = 0; i < bitmapSize; i += HeaderSize) + // writeData(buf, HeaderSize + i, (bitmapSize - i > HeaderSize ? HeaderSize : bitmapSize - i)); + + uint8_t *bitmapbuf = new uint8_t[bitmapSize]; + memset(bitmapbuf, 0, bitmapSize); + writeData(bitmapbuf, HeaderSize, bitmapSize); + delete[] bitmapbuf; + flipOIDBlock(0, firstOID, 0); + buf[0] = 0; + buf[1] = 0; /* append a 16-bit 0 to indicate 0 entries in the vboid->dbroot mapping */ writeData(buf, StartOfVBOidSection, 2); } @@ -379,6 +389,7 @@ OIDServer::OIDServer() : fFp(NULL), fFd(-1) throw runtime_error(os.str()); } + // XXXPAT: Forcing the IDB* path. if (true || IDBPolicy::useHdfs()) { if (!IDBPolicy::exists(fFilename.c_str())) //no bitmap file diff --git a/versioning/BRM/slavecomm.cpp b/versioning/BRM/slavecomm.cpp index ef4a7a9bf..6f2a69763 100644 --- a/versioning/BRM/slavecomm.cpp +++ b/versioning/BRM/slavecomm.cpp @@ -1891,7 +1891,6 @@ void SlaveComm::do_vbRollback1(ByteStream& msg) if (!standalone) master.write(reply); - //takeSnapshot = true; doSaveDelta = true; } @@ -1931,7 +1930,6 @@ void SlaveComm::do_vbRollback2(ByteStream& msg) if (!standalone) master.write(reply); - //takeSnapshot = true; doSaveDelta = true; } @@ -1964,7 +1962,6 @@ void SlaveComm::do_vbCommit(ByteStream& msg) if (!standalone) master.write(reply); - //takeSnapshot = true; doSaveDelta = true; } @@ -2065,7 +2062,7 @@ void SlaveComm::do_confirm() saveFileToggle = !saveFileToggle; const char* filename = journalName.c_str(); - uint32_t utmp = ::umask(0); + //uint32_t utmp = ::umask(0); delete journalh; journalh = IDBDataFile::open( IDBPolicy::getType(filename, IDBPolicy::WRITEENG), filename, "w+b", 0); diff --git a/versioning/BRM/vbbm.cpp b/versioning/BRM/vbbm.cpp index ab6d2f78f..02f070278 100644 --- a/versioning/BRM/vbbm.cpp +++ b/versioning/BRM/vbbm.cpp @@ -1067,6 +1067,32 @@ void VBBM::loadVersion2(IDBDataFile* in) throw runtime_error("VBBM::load(): Failed to load vb file meta data"); } + size_t readSize = vbbmEntries * sizeof(entry); + char *readBuf = new char[readSize]; + size_t progress = 0; + int err; + while (progress < readSize) + { + err = in->read(readBuf + progress, readSize - progress); + if (err < 0) + { + log_errno("VBBM::load()"); + throw runtime_error("VBBM::load(): Failed to load, check the critical log file"); + } + else if (err == 0) + { + log("VBBM::load(): Got early EOF"); + throw runtime_error("VBBM::load(): Got early EOF"); + } + progress += err; + } + + VBBMEntry *loadedEntries = (VBBMEntry *) readBuf; + for (i = 0; i < vbbmEntries; i++) + insert(loadedEntries[i].lbid, loadedEntries[i].verID, loadedEntries[i].vbOID, + loadedEntries[i].vbFBO, true); + + /* for (i = 0; i < vbbmEntries; i++) { if (in->read((char*)&entry, sizeof(entry)) != sizeof(entry)) @@ -1077,6 +1103,7 @@ void VBBM::loadVersion2(IDBDataFile* in) insert(entry.lbid, entry.verID, entry.vbOID, entry.vbFBO, true); } + */ } @@ -1132,16 +1159,15 @@ void VBBM::load(string filename) void VBBM::save(string filename) { int i; - mode_t utmp = ::umask(0); int var; + // XXXPAT: forcing the IDB* path. Delete the fstream path when appropriate. if (true || IDBPolicy::useHdfs()) { const char* filename_p = filename.c_str(); scoped_ptr out(IDBDataFile::open( IDBPolicy::getType(filename_p, IDBPolicy::WRITEENG), filename_p, "wb", IDBDataFile::USE_VBUF)); - ::umask(utmp); if (!out) { @@ -1159,6 +1185,50 @@ void VBBM::save(string filename) bytesWritten += out->write((char*) files, sizeof(VBFileMetadata) * vbbm->nFiles); bytesToWrite += sizeof(VBFileMetadata) * vbbm->nFiles; + int first = -1, last = -1, err; + size_t progress, writeSize; + + for (i = 0; i < vbbm->vbCapacity; i++) + { + if (storage[i].lbid != -1 && first == -1) + first = i; + else if (storage[i].lbid == -1 && first != -1) + { + last = i; + writeSize = (last - first) * sizeof(VBBMEntry); + progress = 0; + char *writePos = (char *) &storage[first]; + while (progress < writeSize) + { + err = out->write(writePos + progress, writeSize - progress); + if (err < 0) + { + log_errno("VBBM::save()"); + throw runtime_error("VBBM::save(): Failed to write the file"); + } + progress += err; + } + first = -1; + } + } + if (first != -1) + { + writeSize = (vbbm->vbCapacity - first) * sizeof(VBBMEntry); + progress = 0; + char *writePos = (char *) &storage[first]; + while (progress < writeSize) + { + err = out->write(writePos + progress, writeSize - progress); + if (err < 0) + { + log_errno("VBBM::save()"); + throw runtime_error("VBBM::save(): Failed to write the file"); + } + progress += err; + } + } + + /* for (i = 0; i < vbbm->vbCapacity; i++) { if (storage[i].lbid != -1) @@ -1173,12 +1243,12 @@ void VBBM::save(string filename) log_errno("VBBM::save()"); throw runtime_error("VBBM::save(): Failed to write the file"); } + */ } else { ofstream out; out.open(filename.c_str(), ios_base::trunc | ios_base::out | ios_base::binary); - ::umask(utmp); if (!out) { diff --git a/versioning/BRM/vss.cpp b/versioning/BRM/vss.cpp index fe0276280..d75034da2 100644 --- a/versioning/BRM/vss.cpp +++ b/versioning/BRM/vss.cpp @@ -1293,15 +1293,14 @@ void VSS::save(string filename) { int i; struct Header header; - mode_t utmp = ::umask(0); + // XXXPAT: Forcing the IDB* path to run. Delete the fstream path when appropriate. if (true || IDBPolicy::useHdfs()) { const char* filename_p = filename.c_str(); scoped_ptr out(IDBDataFile::open( IDBPolicy::getType(filename_p, IDBPolicy::WRITEENG), filename_p, "wb", IDBDataFile::USE_VBUF)); - ::umask(utmp); if (!out) { @@ -1318,6 +1317,49 @@ void VSS::save(string filename) throw runtime_error("VSS::save(): Failed to write header to the file"); } + int first = -1, last = -1, err; + size_t progress, writeSize; + for (i = 0; i < vss->capacity; i++) + { + if (storage[i].lbid != -1 && first == -1) + first = i; + else if (storage[i].lbid == -1 && first != -1) + { + last = i; + writeSize = (last - first) * sizeof(VSSEntry); + progress = 0; + char *writePos = (char *) &storage[first]; + while (progress < writeSize) + { + err = out->write(writePos + progress, writeSize - progress); + if (err < 0) + { + log_errno("VSS::save()"); + throw runtime_error("VSS::save(): Failed to write the file"); + } + progress += err; + } + first = -1; + } + } + if (first != -1) + { + writeSize = (vss->capacity - first) * sizeof(VSSEntry); + progress = 0; + char *writePos = (char *) &storage[first]; + while (progress < writeSize) + { + err = out->write(writePos + progress, writeSize - progress); + if (err < 0) + { + log_errno("VSS::save()"); + throw runtime_error("VSS::save(): Failed to write the file"); + } + progress += err; + } + } + + /* for (i = 0; i < vss->capacity; i++) { if (storage[i].lbid != -1) @@ -1329,12 +1371,13 @@ void VSS::save(string filename) } } } + */ } else { ofstream out; out.open(filename.c_str(), ios_base::trunc | ios_base::out | ios_base::binary); - ::umask(utmp); + //::umask(utmp); if (!out) { @@ -1450,6 +1493,32 @@ void VSS::load(string filename) vss->LWM = 0; */ + size_t readSize = header.entries * sizeof(entry); + char *readBuf = new char[readSize]; + size_t progress = 0; + int err; + while (progress < readSize) + { + err = in->read(readBuf + progress, readSize - progress); + if (err < 0) + { + log_errno("VBBM::load()"); + throw runtime_error("VBBM::load(): Failed to load, check the critical log file"); + } + else if (err == 0) + { + log("VBBM::load(): Got early EOF"); + throw runtime_error("VBBM::load(): Got early EOF"); + } + progress += err; + } + + VSSEntry *loadedEntries = (VSSEntry *) readBuf; + for (i = 0; i < header.entries; i++) + insert(loadedEntries[i].lbid, loadedEntries[i].verID, loadedEntries[i].vbFlag, + loadedEntries[i].locked, true); + + /* for (i = 0; i < header.entries; i++) { if (in->read((char*)&entry, sizeof(entry)) != sizeof(entry)) @@ -1461,6 +1530,8 @@ void VSS::load(string filename) insert(entry.lbid, entry.verID, entry.vbFlag, entry.locked, true); } + */ + //time2 = microsec_clock::local_time(); //cout << "done loading " << time2 << " duration: " << time2-time1 << endl; } diff --git a/writeengine/bulk/cpimport.cpp b/writeengine/bulk/cpimport.cpp index 54b688d0b..edf1a8aa7 100644 --- a/writeengine/bulk/cpimport.cpp +++ b/writeengine/bulk/cpimport.cpp @@ -51,6 +51,7 @@ #include "utils_utf8.h" #include "IDBPolicy.h" #include "MonitorProcMem.h" +#include "dataconvert.h" using namespace std; using namespace WriteEngine; @@ -104,7 +105,7 @@ void printUsage() " [-c readBufSize] [-e maxErrs] [-B libBufSize] [-n NullOption] " << endl << " [-E encloseChar] [-C escapeChar] [-I binaryOpt] [-S] " "[-d debugLevel] [-i] " << endl << - " [-D] [-N] [-L rejectDir]" << endl; + " [-D] [-N] [-L rejectDir] [-T timeZone]" << endl; cout << endl << "Traditional usage without positional parameters " "(XML job file required):" << endl << @@ -114,7 +115,7 @@ void printUsage() " [-E encloseChar] [-C escapeChar] [-I binaryOpt] [-S] " "[-d debugLevel] [-i] " << endl << " [-p path] [-l loadFile]" << endl << - " [-D] [-N] [-L rejectDir]" << endl << endl; + " [-D] [-N] [-L rejectDir] [-T timeZone]" << endl << endl; cout << " Positional parameters:" << endl << " dbName Name of database to load" << endl << @@ -162,7 +163,10 @@ void printUsage() " -S Treat string truncations as errors" << endl << " -D Disable timeout when waiting for table lock" << endl << " -N Disable console output" << endl << - " -L send *.err and *.bad (reject) files here" << endl << endl; + " -L send *.err and *.bad (reject) files here" << endl << + " -T Timezone used for TIMESTAMP datatype" << endl << + " Possible values: \"SYSTEM\" (default)" << endl << + " : Offset in the form +/-HH:MM" << endl << endl; cout << " Example1:" << endl << " cpimport.bin -j 1234" << endl << @@ -313,7 +317,7 @@ void parseCmdLineArgs( std::string jobUUID; while ( (option = getopt( - argc, argv, "b:c:d:e:f:hij:kl:m:n:p:r:s:u:w:B:C:DE:I:P:R:SX:NL:")) != EOF ) + argc, argv, "b:c:d:e:f:hij:kl:m:n:p:r:s:u:w:B:C:DE:I:P:R:ST:X:NL:")) != EOF ) { switch (option) { @@ -668,6 +672,21 @@ void parseCmdLineArgs( break; } + case 'T': + { + std::string timeZone = optarg; + long offset; + + if (timeZone != "SYSTEM" && dataconvert::timeZoneToOffset(timeZone.c_str(), timeZone.size(), &offset)) + { + startupError ( std::string( + "Value for option -T is invalid"), true ); + } + + curJob.setTimeZone( timeZone ); + break; + } + case 'X': // Hidden extra options { if (!strcmp(optarg, "AllowMissingColumn")) @@ -1025,7 +1044,7 @@ int main(int argc, char** argv) // set effective ID to root if( setuid( 0 ) < 0 ) { - std::cerr << " cpimport: setuid failed " << std::endl; + std::cerr << " cpimport: couldn't set uid " << std::endl; } #endif setupSignalHandlers(); diff --git a/writeengine/bulk/we_bulkload.cpp b/writeengine/bulk/we_bulkload.cpp index 134bb07af..d8630ec70 100644 --- a/writeengine/bulk/we_bulkload.cpp +++ b/writeengine/bulk/we_bulkload.cpp @@ -156,7 +156,8 @@ BulkLoad::BulkLoad() : fImportDataMode(IMPORT_DATA_TEXT), fbContinue(false), fDisableTimeOut(false), - fUUID(boost::uuids::nil_generator()()) + fUUID(boost::uuids::nil_generator()()), + fTimeZone("SYSTEM") { fTableInfo.clear(); setDebugLevel( DEBUG_0 ); @@ -251,6 +252,7 @@ int BulkLoad::loadJobInfo( { fJobFileName = fullName; fRootDir = Config::getBulkRoot(); + fJobInfo.setTimeZone(fTimeZone); if ( !exists( fullName.c_str() ) ) { @@ -486,6 +488,7 @@ int BulkLoad::preProcess( Job& job, int tableNo, tableInfo->setEnclosedByChar(fEnclosedByChar); tableInfo->setEscapeChar(fEscapeChar); tableInfo->setImportDataMode(fImportDataMode); + tableInfo->setTimeZone(fTimeZone); tableInfo->setJobUUID(fUUID); if (fMaxErrors != -1) diff --git a/writeengine/bulk/we_bulkload.h b/writeengine/bulk/we_bulkload.h index 4c1adab09..dee60d535 100644 --- a/writeengine/bulk/we_bulkload.h +++ b/writeengine/bulk/we_bulkload.h @@ -115,6 +115,7 @@ public: void addToCmdLineImportFileList(const std::string& importFile); const std::string& getAlternateImportDir( ) const; const std::string& getErrorDir ( ) const; + const std::string& getTimeZone ( ) const; const std::string& getJobDir ( ) const; const std::string& getSchema ( ) const; const std::string& getTempJobDir ( ) const; @@ -149,6 +150,7 @@ public: void setTruncationAsError ( bool bTruncationAsError ); void setJobUUID ( const std::string& jobUUID ); void setErrorDir ( const std::string& errorDir ); + void setTimeZone ( const std::string& timeZone ); // Timer functions void startTimer ( ); void stopTimer ( ); @@ -227,6 +229,7 @@ private: bool fDisableTimeOut; // disable timeout when waiting for table lock boost::uuids::uuid fUUID; // job UUID static bool fNoConsoleOutput; // disable output to console + std::string fTimeZone; // Timezone to use for TIMESTAMP data type //-------------------------------------------------------------------------- // Private Functions @@ -319,6 +322,11 @@ inline const std::string& BulkLoad::getErrorDir( ) const return fErrorDir; } +inline const std::string& BulkLoad::getTimeZone( ) const +{ + return fTimeZone; +} + inline const std::string& BulkLoad::getJobDir( ) const { return DIR_BULK_JOB; @@ -459,6 +467,11 @@ inline void BulkLoad::setErrorDir( const std::string& errorDir ) fErrorDir = errorDir; } +inline void BulkLoad::setTimeZone( const std::string& timeZone ) +{ + fTimeZone = timeZone; +} + inline void BulkLoad::startTimer( ) { gettimeofday( &fStartTime, 0 ); diff --git a/writeengine/bulk/we_bulkloadbuffer.cpp b/writeengine/bulk/we_bulkloadbuffer.cpp index 01d6ed575..86124a783 100644 --- a/writeengine/bulk/we_bulkloadbuffer.cpp +++ b/writeengine/bulk/we_bulkloadbuffer.cpp @@ -139,6 +139,7 @@ BulkLoadBuffer::BulkLoadBuffer( fEnclosedByChar('\0'), fEscapeChar('\\'), fBufferId(bufferId), fTableName(tableName), fbTruncationAsError(false), fImportDataMode(IMPORT_DATA_TEXT), + fTimeZone("SYSTEM"), fFixedBinaryRecLen(0) { fData = new char[bufferSize]; @@ -939,6 +940,7 @@ void BulkLoadBuffer::convert(char* field, int fieldLength, bool bSatVal = false; if ( column.dataType != CalpontSystemCatalog::DATETIME && + column.dataType != CalpontSystemCatalog::TIMESTAMP && column.dataType != CalpontSystemCatalog::TIME ) { if (nullFlag) @@ -1072,6 +1074,59 @@ void BulkLoadBuffer::convert(char* field, int fieldLength, pVal = &llDate; } + else if (column.dataType == CalpontSystemCatalog::TIMESTAMP) + { + // timestamp conversion + int rc = 0; + + if (nullFlag) + { + if (column.fWithDefault) + { + llDate = column.fDefaultInt; + // fall through to update saturation and min/max + } + else + { + llDate = joblist::TIMESTAMPNULL; + pVal = &llDate; + break; + } + } + else + { + if (fImportDataMode != IMPORT_DATA_TEXT) + { + memcpy(&llDate, field, sizeof(llDate)); + + if (!dataconvert::DataConvert::isColumnTimeStampValid( + llDate)) + rc = -1; + } + else + { + llDate = dataconvert::DataConvert::convertColumnTimestamp( + field, dataconvert::CALPONTDATETIME_ENUM, + rc, fieldLength, fTimeZone ); + } + } + + if (rc == 0) + { + if (llDate < bufStats.minBufferVal) + bufStats.minBufferVal = llDate; + + if (llDate > bufStats.maxBufferVal) + bufStats.maxBufferVal = llDate; + } + else + { + llDate = 0; + bufStats.satCount++; + } + + pVal = &llDate; + } else { // datetime conversion @@ -3089,6 +3144,11 @@ bool BulkLoadBuffer::isBinaryFieldNull(void* val, if ((*(uint64_t*)val) == joblist::DATETIMENULL) isNullFlag = true; } + else if (dt == execplan::CalpontSystemCatalog::TIMESTAMP) + { + if ((*(uint64_t*)val) == joblist::TIMESTAMPNULL) + isNullFlag = true; + } else if (dt == execplan::CalpontSystemCatalog::TIME) { if ((*(uint64_t*)val) == joblist::TIMENULL) diff --git a/writeengine/bulk/we_bulkloadbuffer.h b/writeengine/bulk/we_bulkloadbuffer.h index 649e9847e..e5d93c3a1 100644 --- a/writeengine/bulk/we_bulkloadbuffer.h +++ b/writeengine/bulk/we_bulkloadbuffer.h @@ -139,6 +139,7 @@ private: // for db cols (omits default cols) bool fbTruncationAsError; // Treat string truncation as error ImportDataMode fImportDataMode; // Import data in text or binary mode + std::string fTimeZone; // Timezone used by TIMESTAMP datatype unsigned int fFixedBinaryRecLen; // Fixed rec len used in binary mode //-------------------------------------------------------------------------- @@ -381,6 +382,13 @@ public: fImportDataMode = importMode; fFixedBinaryRecLen = fixedBinaryRecLen; } + + /** @brief set timezone. + */ + void setTimeZone(const std::string& timeZone) + { + fTimeZone = timeZone; + } }; inline bool isTrueWord(const char *field, int fieldLength) diff --git a/writeengine/bulk/we_tableinfo.cpp b/writeengine/bulk/we_tableinfo.cpp index e51c8e2c2..0d93c12b7 100644 --- a/writeengine/bulk/we_tableinfo.cpp +++ b/writeengine/bulk/we_tableinfo.cpp @@ -114,6 +114,7 @@ TableInfo::TableInfo(Log* logger, const BRM::TxnID txnID, fKeepRbMetaFile(bKeepRbMetaFile), fbTruncationAsError(false), fImportDataMode(IMPORT_DATA_TEXT), + fTimeZone("SYSTEM"), fTableLocked(false), fReadFromStdin(false), fNullStringMode(false), @@ -989,6 +990,12 @@ void TableInfo::reportTotals(double elapsedTime) ossSatCnt << "invalid date/times replaced with zero value : "; } + + else if (fColumns[i].column.dataType == execplan::CalpontSystemCatalog::TIMESTAMP) + { + ossSatCnt << + "invalid timestamps replaced with zero value : "; + } else if (fColumns[i].column.dataType == execplan::CalpontSystemCatalog::TIME) { ossSatCnt << @@ -1244,6 +1251,7 @@ void TableInfo::initializeBuffers(int noOfBuffers, buffer->setTruncationAsError(getTruncationAsError()); buffer->setImportDataMode(fImportDataMode, fixedBinaryRecLen); + buffer->setTimeZone(fTimeZone); fBuffers.push_back(buffer); } } diff --git a/writeengine/bulk/we_tableinfo.h b/writeengine/bulk/we_tableinfo.h index d44ec50ff..33c45eb2a 100644 --- a/writeengine/bulk/we_tableinfo.h +++ b/writeengine/bulk/we_tableinfo.h @@ -128,6 +128,7 @@ private: // data file bool fbTruncationAsError; // Treat string truncation as error ImportDataMode fImportDataMode; // Import data in text or binary mode + std::string fTimeZone; // Timezone used by TIMESTAMP data type volatile bool fTableLocked; // Do we have db table lock @@ -253,6 +254,10 @@ public: */ ImportDataMode getImportDataMode( ) const; + /** @brief Get timezone. + */ + const std::string& getTimeZone( ) const; + /** @brief Get number of buffers */ int getNumberOfBuffers() const; @@ -310,6 +315,10 @@ public: */ void setImportDataMode( ImportDataMode importMode ); + /** @brief Set timezone. + */ + void setTimeZone( const std::string& timeZone ); + /** @brief Enable distributed mode, saving BRM updates in rptFileName */ void setBulkLoadMode(BulkModeType bulkMode, const std::string& rptFileName); @@ -481,6 +490,11 @@ inline ImportDataMode TableInfo::getImportDataMode() const return fImportDataMode; } +inline const std::string& TableInfo::getTimeZone() const +{ + return fTimeZone; +} + inline int TableInfo::getNumberOfBuffers() const { return fReadBufCount; @@ -580,6 +594,11 @@ inline void TableInfo::setImportDataMode( ImportDataMode importMode ) fImportDataMode = importMode; } +inline void TableInfo::setTimeZone( const std::string& timeZone ) +{ + fTimeZone = timeZone; +} + inline void TableInfo::setJobFileName(const std::string& jobFileName) { fjobFileName = jobFileName; diff --git a/writeengine/server/we_ddlcommandproc.cpp b/writeengine/server/we_ddlcommandproc.cpp index f536c2430..58e2ba0a3 100644 --- a/writeengine/server/we_ddlcommandproc.cpp +++ b/writeengine/server/we_ddlcommandproc.cpp @@ -3539,6 +3539,7 @@ uint8_t WE_DDLCommandProc::fillNewColumn(ByteStream& bs, std::string& err) int dataWidth, scale, precision, compressionType, refColWidth, refCompressionType; string defaultValStr; ColTuple defaultVal; + string timeZone; bs >> tmp32; txnID = tmp32; @@ -3567,6 +3568,7 @@ uint8_t WE_DDLCommandProc::fillNewColumn(ByteStream& bs, std::string& err) refColWidth = tmp32; bs >> tmp8; refCompressionType = tmp8; + bs >> timeZone; //Find the fill in value bool isNULL = false; @@ -3579,7 +3581,7 @@ uint8_t WE_DDLCommandProc::fillNewColumn(ByteStream& bs, std::string& err) colType.scale = scale; colType.precision = precision; bool pushWarning = false; - defaultVal.data = DataConvert::convertColumnData(colType, defaultValStr, pushWarning, isNULL); + defaultVal.data = DataConvert::convertColumnData(colType, defaultValStr, pushWarning, timeZone, isNULL, false, false); fWEWrapper.setTransId(txnID); fWEWrapper.setIsInsert(true); fWEWrapper.setBulkFlag(true); diff --git a/writeengine/server/we_ddlcommon.h b/writeengine/server/we_ddlcommon.h index 6af26b143..9fcc06f7b 100644 --- a/writeengine/server/we_ddlcommon.h +++ b/writeengine/server/we_ddlcommon.h @@ -277,6 +277,13 @@ inline boost::any getNullValueForType(const execplan::CalpontSystemCatalog::ColT } break; + case execplan::CalpontSystemCatalog::TIMESTAMP: + { + long long d = joblist::TIMESTAMPNULL; + value = d; + } + break; + case execplan::CalpontSystemCatalog::CHAR: { std::string charnull; @@ -440,6 +447,10 @@ inline int convertDataType(int dataType) calpontDataType = execplan::CalpontSystemCatalog::TIME; break; + case ddlpackage::DDL_TIMESTAMP: + calpontDataType = execplan::CalpontSystemCatalog::TIMESTAMP; + break; + case ddlpackage::DDL_CLOB: calpontDataType = execplan::CalpontSystemCatalog::CLOB; break; diff --git a/writeengine/server/we_dmlcommandproc.cpp b/writeengine/server/we_dmlcommandproc.cpp index 2acdd1578..bf805870e 100644 --- a/writeengine/server/we_dmlcommandproc.cpp +++ b/writeengine/server/we_dmlcommandproc.cpp @@ -356,7 +356,7 @@ uint8_t WE_DMLCommandProc::processSingleInsert(messageqcpp::ByteStream& bs, std: try { - datavalue = DataConvert::convertColumnData(colType, indata, pushWarning, isNULL); + datavalue = DataConvert::convertColumnData(colType, indata, pushWarning, insertPkg.get_TimeZone(), isNULL, false, false); } catch (exception&) { @@ -1239,7 +1239,7 @@ uint8_t WE_DMLCommandProc::processBatchInsert(messageqcpp::ByteStream& bs, std:: try { - datavalue = DataConvert::convertColumnData(colType, indata, pushWarning, isNULL); + datavalue = DataConvert::convertColumnData(colType, indata, pushWarning, insertPkg.get_TimeZone(), isNULL, false, false); } catch (exception&) { @@ -1788,6 +1788,7 @@ uint8_t WE_DMLCommandProc::processBatchInsertBinary(messageqcpp::ByteStream& bs, case execplan::CalpontSystemCatalog::BIGINT: case execplan::CalpontSystemCatalog::DATETIME: case execplan::CalpontSystemCatalog::TIME: + case execplan::CalpontSystemCatalog::TIMESTAMP: case execplan::CalpontSystemCatalog::UBIGINT: bs >> val64; @@ -2691,6 +2692,8 @@ uint8_t WE_DMLCommandProc::processUpdate(messageqcpp::ByteStream& bs, CalpontSystemCatalog::OID oid = 0; CalpontSystemCatalog::ROPair tableRO; + std::string timeZone = cpackages[txnId].get_TimeZone(); + try { tableRO = systemCatalogPtr->tableRID(tableName); @@ -2984,6 +2987,13 @@ uint8_t WE_DMLCommandProc::processUpdate(messageqcpp::ByteStream& bs, break; } + case CalpontSystemCatalog::TIMESTAMP: + { + intColVal = row.getUintField<8>(fetchColPos); + value = DataConvert::timestampToString(intColVal, timeZone, colType.precision); + break; + } + case CalpontSystemCatalog::TIME: { intColVal = row.getIntField<8>(fetchColPos); @@ -3321,6 +3331,13 @@ uint8_t WE_DMLCommandProc::processUpdate(messageqcpp::ByteStream& bs, break; } + case CalpontSystemCatalog::TIMESTAMP: + { + intColVal = row.getUintField<8>(fetchColPos); + value = DataConvert::timestampToString(intColVal, timeZone, colType.precision); + break; + } + case CalpontSystemCatalog::TIME: { intColVal = row.getIntField<8>(fetchColPos); @@ -3495,7 +3512,7 @@ uint8_t WE_DMLCommandProc::processUpdate(messageqcpp::ByteStream& bs, try { - datavalue = DataConvert::convertColumnData(colType, colType.defaultValue, pushWarn, isNull); + datavalue = DataConvert::convertColumnData(colType, colType.defaultValue, pushWarn, timeZone, isNull, false, false); } catch (exception&) { @@ -3527,7 +3544,7 @@ uint8_t WE_DMLCommandProc::processUpdate(messageqcpp::ByteStream& bs, { try { - datavalue = DataConvert::convertColumnData(colType, value, pushWarn, isNull); + datavalue = DataConvert::convertColumnData(colType, value, pushWarn, timeZone, isNull, false, false); } catch (exception&) { @@ -3568,7 +3585,8 @@ uint8_t WE_DMLCommandProc::processUpdate(messageqcpp::ByteStream& bs, string inData (columnsUpdated[j]->get_Data()); if (((colType.colDataType == execplan::CalpontSystemCatalog::DATE) && (inData == "0000-00-00")) || - ((colType.colDataType == execplan::CalpontSystemCatalog::DATETIME) && (inData == "0000-00-00 00:00:00"))) + ((colType.colDataType == execplan::CalpontSystemCatalog::DATETIME) && (inData == "0000-00-00 00:00:00")) || + ((colType.colDataType == execplan::CalpontSystemCatalog::TIMESTAMP) && (inData == "0000-00-00 00:00:00"))) { isNull = true; } @@ -3623,7 +3641,7 @@ uint8_t WE_DMLCommandProc::processUpdate(messageqcpp::ByteStream& bs, try { - datavalue = DataConvert::convertColumnData(colType, inData, pushWarn, isNull); + datavalue = DataConvert::convertColumnData(colType, inData, pushWarn, timeZone, isNull, false, false); } catch (exception&) { @@ -3669,7 +3687,7 @@ uint8_t WE_DMLCommandProc::processUpdate(messageqcpp::ByteStream& bs, { try { - datavalue = DataConvert::convertColumnData(colType, colType.defaultValue, pushWarn, isNull); + datavalue = DataConvert::convertColumnData(colType, colType.defaultValue, pushWarn, timeZone, isNull, false, false); } catch (exception&) { @@ -3702,7 +3720,7 @@ uint8_t WE_DMLCommandProc::processUpdate(messageqcpp::ByteStream& bs, { try { - datavalue = DataConvert::convertColumnData(colType, inData, pushWarn, isNull, false, true); + datavalue = DataConvert::convertColumnData(colType, inData, pushWarn, timeZone, isNull, false, true); } catch (exception& ex) { diff --git a/writeengine/shared/we_blockop.cpp b/writeengine/shared/we_blockop.cpp index 042351967..fb8dfe89b 100644 --- a/writeengine/shared/we_blockop.cpp +++ b/writeengine/shared/we_blockop.cpp @@ -164,6 +164,7 @@ uint64_t BlockOp::getEmptyRowValue( case CalpontSystemCatalog::VARCHAR : case CalpontSystemCatalog::DATE : case CalpontSystemCatalog::DATETIME : + case CalpontSystemCatalog::TIMESTAMP : default: offset = ( colDataType == CalpontSystemCatalog::VARCHAR ) ? -1 : 0; emptyVal = joblist::CHAR1EMPTYROW; diff --git a/writeengine/shared/we_chunkmanager.cpp b/writeengine/shared/we_chunkmanager.cpp index abe61ab11..c6bf3ef50 100644 --- a/writeengine/shared/we_chunkmanager.cpp +++ b/writeengine/shared/we_chunkmanager.cpp @@ -1928,7 +1928,7 @@ int ChunkManager::reallocateChunks(CompFileData* fileData) char tmText[24]; // this snprintf call causes a compiler warning b/c buffer size is less // then maximum string size. -#if defined(__GNUC__) && __GNUC__ >= 6 +#if defined(__GNUC__) && __GNUC__ >= 7 #pragma GCC diagnostic push #pragma GCC diagnostic ignored "-Wformat-truncation=" snprintf(tmText, sizeof(tmText), ".%04d%02d%02d%02d%02d%02d%06ld", @@ -2123,7 +2123,7 @@ int ChunkManager::reallocateChunks(CompFileData* fileData) char tmText[24]; // this snprintf call causes a compiler warning b/c buffer size is less // then maximum string size. -#if defined(__GNUC__) && __GNUC__ >= 6 +#if defined(__GNUC__) && __GNUC__ >= 7 #pragma GCC diagnostic push #pragma GCC diagnostic ignored "-Wformat-truncation=" snprintf(tmText, sizeof(tmText), ".%04d%02d%02d%02d%02d%02d%06ld", diff --git a/writeengine/shared/we_convertor.cpp b/writeengine/shared/we_convertor.cpp index 72d2c6429..dadec77da 100644 --- a/writeengine/shared/we_convertor.cpp +++ b/writeengine/shared/we_convertor.cpp @@ -367,9 +367,10 @@ void Convertor::convertColType(CalpontSystemCatalog::ColDataType dataType, internalType = WriteEngine::WR_FLOAT; break; - // Map BIGINT and DATETIME to WR_LONGLONG + // Map BIGINT, DATETIME, TIMESTAMP, and TIME to WR_LONGLONG case CalpontSystemCatalog::BIGINT : case CalpontSystemCatalog::DATETIME : + case CalpontSystemCatalog::TIMESTAMP : case CalpontSystemCatalog::TIME : internalType = WriteEngine::WR_LONGLONG; break; @@ -429,7 +430,7 @@ void Convertor::convertColType(CalpontSystemCatalog::ColDataType dataType, internalType = WriteEngine::WR_UINT; break; - // Map UBIGINT to WR_ULONGLONG + // Map UBIGINT, TIMESTAMP to WR_ULONGLONG case CalpontSystemCatalog::UBIGINT: internalType = WriteEngine::WR_ULONGLONG; break; @@ -481,7 +482,7 @@ void Convertor::convertWEColType(ColType internalType, dataType = CalpontSystemCatalog::FLOAT; break; - // Map BIGINT and DATETIME to WR_LONGLONG + // Map BIGINT, DATETIME, TIME, and TIMESTAMP to WR_LONGLONG case WriteEngine::WR_LONGLONG : dataType = CalpontSystemCatalog::BIGINT; break; @@ -531,7 +532,7 @@ void Convertor::convertWEColType(ColType internalType, dataType = CalpontSystemCatalog::UINT; break; - // Map UBIGINT to WR_ULONGLONG + // Map UBIGINT and TIMESTAMP to WR_ULONGLONG case WriteEngine::WR_ULONGLONG: dataType = CalpontSystemCatalog::UBIGINT; break; @@ -595,9 +596,10 @@ void Convertor::convertColType(ColStruct* curStruct) *internalType = WriteEngine::WR_FLOAT; break; - // Map BIGINT and DATETIME to WR_LONGLONG + // Map BIGINT, DATETIME, TIME, and TIMESTAMP to WR_LONGLONG case CalpontSystemCatalog::BIGINT : case CalpontSystemCatalog::DATETIME : + case CalpontSystemCatalog::TIMESTAMP : case CalpontSystemCatalog::TIME : *internalType = WriteEngine::WR_LONGLONG; break; @@ -767,6 +769,7 @@ int Convertor::getCorrectRowWidth(CalpontSystemCatalog::ColDataType dataType, in case CalpontSystemCatalog::DATETIME: case CalpontSystemCatalog::TIME: + case CalpontSystemCatalog::TIMESTAMP: newWidth = 8; break; diff --git a/writeengine/shared/we_type.h b/writeengine/shared/we_type.h index 8604b2169..71581b44b 100644 --- a/writeengine/shared/we_type.h +++ b/writeengine/shared/we_type.h @@ -164,7 +164,8 @@ const char ColDataTypeStr[execplan::CalpontSystemCatalog::NUM_OF_COL_DATA_TYPE] "unsigned-bigint", "unsigned-double", "text", - "time" + "time", + "timestamp" }; enum FuncType { FUNC_WRITE_ENGINE, FUNC_INDEX, FUNC_DICTIONARY }; diff --git a/writeengine/splitter/we_cmdargs.cpp b/writeengine/splitter/we_cmdargs.cpp index 83044fa45..7ed626f26 100644 --- a/writeengine/splitter/we_cmdargs.cpp +++ b/writeengine/splitter/we_cmdargs.cpp @@ -36,6 +36,7 @@ using namespace std; #include #include +#include "dataconvert.h" #include "liboamcpp.h" using namespace oam; @@ -74,7 +75,8 @@ WECmdArgs::WECmdArgs(int argc, char** argv) : fBlockMode3(false), fbTruncationAsError(false), fUUID(boost::uuids::nil_generator()()), - fConsoleOutput(true) + fConsoleOutput(true), + fTimeZone("SYSTEM") { try { @@ -210,6 +212,7 @@ std::string WECmdArgs::getCpImportCmdLine() } aSS << " -P " << getModuleID(); + aSS << " -T " << fTimeZone; if (fbTruncationAsError) aSS << " -S "; @@ -482,7 +485,7 @@ void WECmdArgs::usage() cout << "\t\t [-r readers] [-j JobID] [-e maxErrs] [-B libBufSize] [-w parsers]\n"; cout << "\t\t [-s c] [-E enclosedChar] [-C escapeChar] [-n NullOption]\n"; cout << "\t\t [-q batchQty] [-p jobPath] [-P list of PMs] [-S] [-i] [-v verbose]\n"; - cout << "\t\t [-I binaryOpt]\n"; + cout << "\t\t [-I binaryOpt] [-T timeZone]\n"; cout << "Traditional usage without positional parameters (XML job file required):\n"; @@ -491,7 +494,7 @@ void WECmdArgs::usage() cout << "\t\t [-b readBufs] [-p path] [-c readBufSize] [-e maxErrs] [-B libBufSize]\n"; cout << "\t\t [-n NullOption] [-E encloseChar] [-C escapeChar] [-i] [-v verbose]\n"; cout << "\t\t [-d debugLevel] [-q batchQty] [-l loadFile] [-P list of PMs] [-S]\n"; - cout << "\t\t [-I binaryOpt]\n"; + cout << "\t\t [-I binaryOpt] [-T timeZone]\n"; cout << "\n\nPositional parameters:\n"; cout << "\tdbName Name of the database to load\n"; @@ -535,7 +538,10 @@ void WECmdArgs::usage() << "\t-m\tmode\n" << "\t\t\t1 - rows will be loaded in a distributed manner across PMs.\n" << "\t\t\t2 - PM based input files loaded onto their respective PM.\n" - << "\t\t\t3 - input files will be loaded on the local PM.\n"; + << "\t\t\t3 - input files will be loaded on the local PM.\n" + << "\t-T\tTimezone used for TIMESTAMP datatype.\n" + << "\t\tPossible values: \"SYSTEM\" (default)\n" + << "\t\t : Offset in the form +/-HH:MM\n"; cout << "\nExample1: Traditional usage\n" << "\tcpimport -j 1234"; @@ -574,7 +580,7 @@ void WECmdArgs::parseCmdLineArgs(int argc, char** argv) // fPrgmName = "/home/bpaul/genii/export/bin/cpimport"; while ((aCh = getopt(argc, argv, - "d:j:w:s:v:l:r:b:e:B:f:q:ihm:E:C:P:I:n:p:c:SN")) + "d:j:w:s:v:l:r:b:e:B:f:q:ihm:E:C:P:I:n:p:c:ST:N")) != EOF) { switch (aCh) @@ -842,6 +848,21 @@ void WECmdArgs::parseCmdLineArgs(int argc, char** argv) break; } + case 'T': + { + std::string timeZone = optarg; + long offset; + + if (timeZone != "SYSTEM" && dataconvert::timeZoneToOffset(timeZone.c_str(), timeZone.size(), &offset)) + { + throw (runtime_error( + "Value for option -T is invalid")); + } + + fTimeZone = timeZone; + break; + } + case 'q': // -q: batch quantity - default value is 10000 { errno = 0; diff --git a/writeengine/splitter/we_cmdargs.h b/writeengine/splitter/we_cmdargs.h index 3186fe6c0..a7eb8ed4f 100644 --- a/writeengine/splitter/we_cmdargs.h +++ b/writeengine/splitter/we_cmdargs.h @@ -190,6 +190,10 @@ public: { return fConsoleOutput; } + const std::string& getTimeZone() const + { + return fTimeZone; + } private: // variables for SplitterApp @@ -294,6 +298,7 @@ private: // variables for SplitterApp bool fbTruncationAsError; // Treat string truncation as error boost::uuids::uuid fUUID; bool fConsoleOutput; // If false, no output to console. + std::string fTimeZone; // Timezone to use for TIMESTAMP datatype }; //---------------------------------------------------------------------- diff --git a/writeengine/splitter/we_sdhandler.cpp b/writeengine/splitter/we_sdhandler.cpp index c2120c529..14b055a33 100644 --- a/writeengine/splitter/we_sdhandler.cpp +++ b/writeengine/splitter/we_sdhandler.cpp @@ -1910,6 +1910,10 @@ void WESDHandler::onCleanupResult(int PmId, messageqcpp::SBS& Sbs) ossSatCnt << "invalid date/times replaced with zero value: "; break; + case CalpontSystemCatalog::TIMESTAMP: + ossSatCnt << "invalid timestamps replaced with zero value: "; + break; + case CalpontSystemCatalog::TIME: ossSatCnt << "invalid times replaced with zero value: "; break; diff --git a/writeengine/splitter/we_splitterapp.cpp b/writeengine/splitter/we_splitterapp.cpp index f96da7463..11d6da22f 100644 --- a/writeengine/splitter/we_splitterapp.cpp +++ b/writeengine/splitter/we_splitterapp.cpp @@ -609,7 +609,7 @@ int main(int argc, char** argv) // @BUG4343 if( setuid( 0 ) < 0 ) { - std::cerr << " we_splitterapp: setuid failed " << std::endl; + std::cerr << " we_splitterapp: couldn't set uid " << std::endl; } std::cin.sync_with_stdio(false); diff --git a/writeengine/xml/we_xmljob.cpp b/writeengine/xml/we_xmljob.cpp index 5afbe8774..4cc4c90eb 100644 --- a/writeengine/xml/we_xmljob.cpp +++ b/writeengine/xml/we_xmljob.cpp @@ -76,7 +76,8 @@ const long long infinidb_precision[19] = //------------------------------------------------------------------------------ XMLJob::XMLJob( ) : fDebugLevel( DEBUG_0 ), fDeleteTempFile(false), - fValidateColList(true) + fValidateColList(true), + fTimeZone("SYSTEM") { } @@ -1128,6 +1129,22 @@ void XMLJob::fillInXMLDataNotNullDefault( break; } + case execplan::CalpontSystemCatalog::TIMESTAMP: + { + int convertStatus; + int64_t dt = + dataconvert::DataConvert::convertColumnTimestamp( + col_defaultValue.c_str(), + dataconvert::CALPONTDATETIME_ENUM, convertStatus, + col_defaultValue.length(), fTimeZone ); + + if (convertStatus != 0) + bDefaultConvertError = true; + + col.fDefaultInt = dt; + break; + } + case execplan::CalpontSystemCatalog::TIME: { int convertStatus; diff --git a/writeengine/xml/we_xmljob.h b/writeengine/xml/we_xmljob.h index f9c996f26..3f144975b 100644 --- a/writeengine/xml/we_xmljob.h +++ b/writeengine/xml/we_xmljob.h @@ -123,6 +123,14 @@ public: */ EXPORT bool processNode( xmlNode* pParentNode ); + /** + * @brief Set timezone + */ + void setTimeZone(const std::string& timeZone) + { + fTimeZone = timeZone; + } + private: void setJobData( xmlNode* pNode, const xmlTag tag, @@ -154,6 +162,7 @@ private: JobColList fDefaultColumns; // temporary list of default cols // for table node being processed bool fValidateColList; // Validate all cols have XML tag + std::string fTimeZone; // Timezone used for TIMESTAMP datatype }; } //end of namespace