diff --git a/cpackEngineDEB.cmake b/cpackEngineDEB.cmake index 42c1f6974..5f8a83c42 100644 --- a/cpackEngineDEB.cmake +++ b/cpackEngineDEB.cmake @@ -69,7 +69,7 @@ if ("${DEBIAN_VERSION_NUMBER}" EQUAL "8") elseif ("${DEBIAN_VERSION_NUMBER}" EQUAL "9") SET(CPACK_DEBIAN_PLATFORM_PACKAGE_DEPENDS "expect, perl, openssl, file, sudo, libdbi-perl, libreadline-dev, rsync, net-tools, libboost-all-dev, mariadb-columnstore-libs, mariadb-columnstore-server, libsnappy1v5, libreadline5") else() - SET(CPACK_DEBIAN_PLATFORM_PACKAGE_DEPENDS "expect, perl, openssl, file, sudo, libdbi-perl, libboost-all-dev, libreadline-dev, rsync, snappy, net-tools") + SET(CPACK_DEBIAN_PLATFORM_PACKAGE_DEPENDS "expect, perl, openssl, file, sudo, libdbi-perl, libboost-all-dev, libreadline-dev, rsync, libsnappy1v5, net-tools") endif () SET(CPACK_DEBIAN_STORAGE-ENGINE_PACKAGE_DEPENDS "mariadb-columnstore-libs") diff --git a/dbcon/ddlpackage/ddl.l b/dbcon/ddlpackage/ddl.l index 4812bbd0d..7e7400451 100644 --- a/dbcon/ddlpackage/ddl.l +++ b/dbcon/ddlpackage/ddl.l @@ -60,6 +60,7 @@ double_quote \" grave_accent ` comment ("--"{non_newline}*) +extended_ident_cont [A-Za-z\200-\377_0-9\$#,()\[\].;\:\+\-\*\/\%\^\<\>\=!&|@\\] self [,()\[\].;\:\+\-\*\/\%\^\<\>\=] whitespace ({space}+|{comment}) @@ -67,10 +68,11 @@ digit [0-9] ident_start [A-Za-z\200-\377_0-9] ident_cont [A-Za-z\200-\377_0-9\$] identifier {ident_start}{ident_cont}* +extended_identifier {ident_start}{extended_ident_cont}* /* fully qualified names regexes */ fq_identifier {identifier}\.{identifier} -identifer_quoted {grave_accent}{identifier}{grave_accent} -identifer_double_quoted {double_quote}{identifier}{double_quote} +identifer_quoted {grave_accent}{extended_identifier}{grave_accent} +identifer_double_quoted {double_quote}{extended_identifier}{double_quote} integer [-+]?{digit}+ decimal ([-+]?({digit}*\.{digit}+)|({digit}+\.{digit}*)) @@ -116,6 +118,7 @@ CREATE {return CREATE;} CURRENT_USER {return CURRENT_USER;} DATE {ddlget_lval(yyscanner)->str=strdup("date"); return DATE;} DATETIME {return DATETIME;} +TIME {return TIME;} DECIMAL {return DECIMAL;} DEC {return DECIMAL;} DEFAULT {return DEFAULT;} diff --git a/dbcon/ddlpackage/ddl.y b/dbcon/ddlpackage/ddl.y index 8f6e713fc..68747dfc9 100644 --- a/dbcon/ddlpackage/ddl.y +++ b/dbcon/ddlpackage/ddl.y @@ -177,7 +177,6 @@ VARYING WITH ZONE DOUBLE IDB_FLOAT REAL CHARSET IDB_IF EXISTS CHANGE TRUNCATE %type opt_precision_scale %type opt_referential_triggered_action %type opt_time_precision -%type opt_with_time_zone %type qualified_name %type referential_action %type referential_triggered_action @@ -1114,10 +1113,11 @@ literal: ; datetime_type: - DATETIME + DATETIME opt_time_precision { $$ = new ColumnType(DDL_DATETIME); $$->fLength = DDLDatatypeLength[DDL_DATETIME]; + $$->fPrecision = $2; } | DATE @@ -1126,12 +1126,11 @@ datetime_type: $$->fLength = DDLDatatypeLength[DDL_DATE]; } | - TIME opt_time_precision opt_with_time_zone + TIME opt_time_precision { - $$ = new ColumnType(DDL_DATETIME); - $$->fLength = DDLDatatypeLength[DDL_DATETIME]; + $$ = new ColumnType(DDL_TIME); + $$->fLength = DDLDatatypeLength[DDL_TIME]; $$->fPrecision = $2; - $$->fWithTimezone = $3; } opt_time_precision: @@ -1139,11 +1138,6 @@ opt_time_precision: | {$$ = -1;} ; -opt_with_time_zone: - WITH TIME ZONE {$$ = true;} - | {$$ = false;} - ; - drop_column_def: DROP column_name drop_behavior {$$ = new AtaDropColumn($2, $3);} | DROP COLUMN column_name drop_behavior {$$ = new AtaDropColumn($3, $4);} diff --git a/dbcon/ddlpackage/ddlpkg.h b/dbcon/ddlpackage/ddlpkg.h index 97e971b6b..779577f83 100644 --- a/dbcon/ddlpackage/ddlpkg.h +++ b/dbcon/ddlpackage/ddlpkg.h @@ -236,6 +236,7 @@ enum DDL_DATATYPES DDL_UNSIGNED_DOUBLE, DDL_UNSIGNED_NUMERIC, DDL_TEXT, + DDL_TIME, DDL_INVALID_DATATYPE }; @@ -273,6 +274,7 @@ const std::string DDLDatatypeString[] = "unsigned-double", "unsigned-numeric", "text", + "time" "" }; @@ -328,6 +330,7 @@ const int DDLDatatypeLength[] = 8, // UNSIGNED_DOUBLE, 2, // UNSIGNED_NUMERIC, 8, // TEXT + 8, // TIME -1 // INVALID LENGTH }; diff --git a/dbcon/ddlpackageproc/altertableprocessor.cpp b/dbcon/ddlpackageproc/altertableprocessor.cpp index 0d4b25819..a398e656f 100644 --- a/dbcon/ddlpackageproc/altertableprocessor.cpp +++ b/dbcon/ddlpackageproc/altertableprocessor.cpp @@ -209,6 +209,11 @@ bool typesAreSame(const CalpontSystemCatalog::ColType& colType, const ColumnType break; + case (CalpontSystemCatalog::TIME): + if (newType.fType == DDL_TIME) return true; + + break; + case (CalpontSystemCatalog::VARCHAR): if (newType.fType == DDL_VARCHAR && colType.colWidth == newType.fLength) return true; @@ -223,7 +228,13 @@ bool typesAreSame(const CalpontSystemCatalog::ColType& colType, const ColumnType break; case (CalpontSystemCatalog::BLOB): + if (newType.fType == DDL_BLOB && colType.colWidth == newType.fLength) return true; + + break; + case (CalpontSystemCatalog::TEXT): + if (newType.fType == DDL_TEXT && colType.colWidth == newType.fLength) return true; + break; default: diff --git a/dbcon/ddlpackageproc/ddlindexpopulator.cpp b/dbcon/ddlpackageproc/ddlindexpopulator.cpp index 12d6c9854..cb58df5bc 100644 --- a/dbcon/ddlpackageproc/ddlindexpopulator.cpp +++ b/dbcon/ddlpackageproc/ddlindexpopulator.cpp @@ -330,6 +330,7 @@ boost::any DDLIndexPopulator::convertData(const CalpontSystemCatalog::ColType& return *reinterpret_cast(&data); case execplan::CalpontSystemCatalog::DATETIME: // @bug 375 + case execplan::CalpontSystemCatalog::TIME: case execplan::CalpontSystemCatalog::BIGINT: return *reinterpret_cast(&data); @@ -524,6 +525,7 @@ bool DDLIndexPopulator::checkNotNull(const IdxTuple& data, const CalpontSystemCa break; case execplan::CalpontSystemCatalog::DATETIME: + case execplan::CalpontSystemCatalog::TIME: isNull = any_cast(data.data) == any_cast(nullvalue); break; diff --git a/dbcon/ddlpackageproc/ddlpackageprocessor.cpp b/dbcon/ddlpackageproc/ddlpackageprocessor.cpp index 9b8c66658..2a64dd1f7 100644 --- a/dbcon/ddlpackageproc/ddlpackageprocessor.cpp +++ b/dbcon/ddlpackageproc/ddlpackageprocessor.cpp @@ -232,6 +232,10 @@ execplan::CalpontSystemCatalog::ColDataType DDLPackageProcessor::convertDataType colDataType = CalpontSystemCatalog::DATETIME; break; + case ddlpackage::DDL_TIME: + colDataType = CalpontSystemCatalog::TIME; + break; + case ddlpackage::DDL_CLOB: colDataType = CalpontSystemCatalog::CLOB; break; @@ -476,6 +480,13 @@ DDLPackageProcessor::getNullValueForType(const execplan::CalpontSystemCatalog::C } break; + case execplan::CalpontSystemCatalog::TIME: + { + long long d = joblist::TIMENULL; + value = d; + } + break; + case execplan::CalpontSystemCatalog::CHAR: { std::string charnull; diff --git a/dbcon/execplan/aggregatecolumn.cpp b/dbcon/execplan/aggregatecolumn.cpp index e0914beb3..18cba2607 100644 --- a/dbcon/execplan/aggregatecolumn.cpp +++ b/dbcon/execplan/aggregatecolumn.cpp @@ -353,6 +353,14 @@ void AggregateColumn::evaluate(Row& row, bool& isNull) break; + case CalpontSystemCatalog::TIME: + if (row.equals<8>(TIMENULL, fInputIndex)) + isNull = true; + else + fResult.intVal = row.getIntField<8>(fInputIndex); + + break; + case CalpontSystemCatalog::CHAR: case CalpontSystemCatalog::VARCHAR: case CalpontSystemCatalog::STRINT: diff --git a/dbcon/execplan/aggregatecolumn.h b/dbcon/execplan/aggregatecolumn.h index a70189e50..d1db7e5a4 100644 --- a/dbcon/execplan/aggregatecolumn.h +++ b/dbcon/execplan/aggregatecolumn.h @@ -411,6 +411,14 @@ public: /** * F&E */ + virtual int64_t getTimeIntVal(rowgroup::Row& row, bool& isNull) + { + evaluate(row, isNull); + return TreeNode::getTimeIntVal(); + } + /** + * F&E + */ virtual int64_t getDatetimeIntVal(rowgroup::Row& row, bool& isNull) { evaluate(row, isNull); diff --git a/dbcon/execplan/arithmeticcolumn.h b/dbcon/execplan/arithmeticcolumn.h index b6ca823c4..191416fbf 100644 --- a/dbcon/execplan/arithmeticcolumn.h +++ b/dbcon/execplan/arithmeticcolumn.h @@ -248,6 +248,11 @@ public: return fExpression->getDatetimeIntVal(row, isNull); } + virtual int64_t getTimeIntVal(rowgroup::Row& row, bool& isNull) + { + return fExpression->getTimeIntVal(row, isNull); + } + virtual bool getBoolVal(rowgroup::Row& row, bool& isNull) { return fExpression->getBoolVal(row, isNull); diff --git a/dbcon/execplan/arithmeticoperator.cpp b/dbcon/execplan/arithmeticoperator.cpp index e433db6b8..73e18b430 100644 --- a/dbcon/execplan/arithmeticoperator.cpp +++ b/dbcon/execplan/arithmeticoperator.cpp @@ -111,7 +111,7 @@ void ArithmeticOperator::unserialize(messageqcpp::ByteStream& b) bool ArithmeticOperator::operator==(const ArithmeticOperator& t) const { - if (fData == t.fData) + if (data() == t.data()) return true; return false; diff --git a/dbcon/execplan/arithmeticoperator.h b/dbcon/execplan/arithmeticoperator.h index ef7b538fa..ca254075b 100644 --- a/dbcon/execplan/arithmeticoperator.h +++ b/dbcon/execplan/arithmeticoperator.h @@ -151,6 +151,11 @@ public: evaluate(row, isNull, lop, rop); return TreeNode::getDatetimeIntVal(); } + virtual int64_t getTimeIntVal(rowgroup::Row& row, bool& isNull, ParseTree* lop, ParseTree* rop) + { + evaluate(row, isNull, lop, rop); + return TreeNode::getTimeIntVal(); + } virtual bool getBoolVal(rowgroup::Row& row, bool& isNull, ParseTree* lop, ParseTree* rop) { evaluate(row, isNull, lop, rop); diff --git a/dbcon/execplan/calpontsystemcatalog.cpp b/dbcon/execplan/calpontsystemcatalog.cpp index 501f488ce..15ccd73aa 100644 --- a/dbcon/execplan/calpontsystemcatalog.cpp +++ b/dbcon/execplan/calpontsystemcatalog.cpp @@ -156,6 +156,10 @@ const string colDataTypeToString(CalpontSystemCatalog::ColDataType cdt) return "datetime"; break; + case CalpontSystemCatalog::TIME: + return "time"; + break; + case CalpontSystemCatalog::VARCHAR: return "varchar"; break; diff --git a/dbcon/execplan/calpontsystemcatalog.h b/dbcon/execplan/calpontsystemcatalog.h index 5918f7f8c..7b828a297 100644 --- a/dbcon/execplan/calpontsystemcatalog.h +++ b/dbcon/execplan/calpontsystemcatalog.h @@ -154,6 +154,7 @@ public: UBIGINT, /*!< Unsigned BIGINT type */ UDOUBLE, /*!< Unsigned DOUBLE type */ TEXT, /*!< TEXT type */ + TIME, /*!< TIME type */ NUM_OF_COL_DATA_TYPE, /* NEW TYPES ABOVE HERE */ LONGDOUBLE, /* @bug3241, dev and variance calculation only */ STRINT, /* @bug3532, string as int for fast comparison */ diff --git a/dbcon/execplan/constantcolumn.h b/dbcon/execplan/constantcolumn.h index 8e7f178aa..04098faae 100644 --- a/dbcon/execplan/constantcolumn.h +++ b/dbcon/execplan/constantcolumn.h @@ -308,6 +308,21 @@ public: return fResult.intVal; } + /** + * F&E + */ + virtual int64_t getTimeIntVal(rowgroup::Row& row, bool& isNull) + { + isNull = isNull || (fType == NULLDATA); + + if (!fResult.valueConverted) + { + fResult.intVal = dataconvert::DataConvert::stringToTime(fResult.strVal); + fResult.valueConverted = true; + } + + return fResult.intVal; + } /** * F&E */ diff --git a/dbcon/execplan/filter.cpp b/dbcon/execplan/filter.cpp index 04b82c7b6..b76755872 100644 --- a/dbcon/execplan/filter.cpp +++ b/dbcon/execplan/filter.cpp @@ -73,7 +73,7 @@ const string Filter::toString() const bool Filter::operator==(const Filter& t) const { - if (fData == t.fData) + if (data() == t.data()) return true; return false; diff --git a/dbcon/execplan/functioncolumn.h b/dbcon/execplan/functioncolumn.h index 8f27cad75..5a099d1d2 100644 --- a/dbcon/execplan/functioncolumn.h +++ b/dbcon/execplan/functioncolumn.h @@ -255,7 +255,10 @@ public: { return fFunctor->getDatetimeIntVal(row, fFunctionParms, isNull, fOperationType); } - + virtual int64_t getTimeIntVal(rowgroup::Row& row, bool& isNull) + { + return fFunctor->getTimeIntVal(row, fFunctionParms, isNull, fOperationType); + } private: funcexp::FunctionParm fFunctionParms; diff --git a/dbcon/execplan/logicoperator.cpp b/dbcon/execplan/logicoperator.cpp index ca59b748b..a5353e472 100644 --- a/dbcon/execplan/logicoperator.cpp +++ b/dbcon/execplan/logicoperator.cpp @@ -115,7 +115,7 @@ void LogicOperator::unserialize(messageqcpp::ByteStream& b) bool LogicOperator::operator==(const LogicOperator& t) const { - if (fData == t.fData) + if (data() == t.data()) return true; return false; diff --git a/dbcon/execplan/operator.h b/dbcon/execplan/operator.h index e0e16be2f..1a33dcb94 100644 --- a/dbcon/execplan/operator.h +++ b/dbcon/execplan/operator.h @@ -190,6 +190,10 @@ public: { return fResult.intVal; } + virtual int64_t getTimeIntVal(rowgroup::Row& row, bool& isNull, ParseTree* lop, ParseTree* rop) + { + return fResult.intVal; + } virtual bool getBoolVal(rowgroup::Row& row, bool& isNull, ParseTree* lop, ParseTree* rop) { return fResult.boolVal; diff --git a/dbcon/execplan/parsetree.h b/dbcon/execplan/parsetree.h index 313f2906a..2ad41b6bb 100644 --- a/dbcon/execplan/parsetree.h +++ b/dbcon/execplan/parsetree.h @@ -282,6 +282,14 @@ public: return fData->getDatetimeIntVal(row, isNull); } + inline int64_t getTimeIntVal(rowgroup::Row& row, bool& isNull) + { + if (fLeft && fRight) + return (reinterpret_cast(fData))->getTimeIntVal(row, isNull, fLeft, fRight); + else + return fData->getTimeIntVal(row, isNull); + } + private: /** draw the tree * diff --git a/dbcon/execplan/predicateoperator.cpp b/dbcon/execplan/predicateoperator.cpp index bf77409dc..166a8edc0 100644 --- a/dbcon/execplan/predicateoperator.cpp +++ b/dbcon/execplan/predicateoperator.cpp @@ -164,7 +164,7 @@ void PredicateOperator::unserialize(messageqcpp::ByteStream& b) bool PredicateOperator::operator==(const PredicateOperator& t) const { - if (fData == t.fData) + if (data() == t.data()) return true; return false; @@ -196,6 +196,7 @@ bool PredicateOperator::operator!=(const TreeNode* t) const void PredicateOperator::setOpType(Type& l, Type& r) { if ( l.colDataType == execplan::CalpontSystemCatalog::DATETIME || + l.colDataType == execplan::CalpontSystemCatalog::TIME || l.colDataType == execplan::CalpontSystemCatalog::DATE ) { switch (r.colDataType) @@ -210,6 +211,11 @@ void PredicateOperator::setOpType(Type& l, Type& r) fOperationType.colWidth = 8; break; + case execplan::CalpontSystemCatalog::TIME: + fOperationType.colDataType = execplan::CalpontSystemCatalog::TIME; + fOperationType.colWidth = 8; + break; + case execplan::CalpontSystemCatalog::DATE: fOperationType = l; break; @@ -221,6 +227,7 @@ void PredicateOperator::setOpType(Type& l, Type& r) } } else if ( r.colDataType == execplan::CalpontSystemCatalog::DATETIME || + r.colDataType == execplan::CalpontSystemCatalog::TIME || r.colDataType == execplan::CalpontSystemCatalog::DATE ) { switch (l.colDataType) @@ -236,6 +243,11 @@ void PredicateOperator::setOpType(Type& l, Type& r) fOperationType.colWidth = 8; break; + case execplan::CalpontSystemCatalog::TIME: + fOperationType.colDataType = execplan::CalpontSystemCatalog::TIME; + fOperationType.colWidth = 8; + break; + case execplan::CalpontSystemCatalog::DATE: fOperationType = r; break; diff --git a/dbcon/execplan/predicateoperator.h b/dbcon/execplan/predicateoperator.h index 2b63548d5..6253d9389 100644 --- a/dbcon/execplan/predicateoperator.h +++ b/dbcon/execplan/predicateoperator.h @@ -344,6 +344,37 @@ inline bool PredicateOperator::getBoolVal(rowgroup::Row& row, bool& isNull, Retu return numericCompare(val1, rop->getDatetimeIntVal(row, isNull)) && !isNull; } + case execplan::CalpontSystemCatalog::TIME: + { + if (fOp == OP_ISNULL) + { + lop->getTimeIntVal(row, isNull); + bool ret = isNull; + isNull = false; + return ret; + } + + if (fOp == OP_ISNOTNULL) + { + lop->getTimeIntVal(row, isNull); + bool ret = isNull; + isNull = false; + return !ret; + } + + if (isNull) + return false; + + int64_t val1 = lop->getTimeIntVal(row, isNull); + + if (isNull) + return false; + + return numericCompare(val1, rop->getTimeIntVal(row, isNull)) && !isNull; + } + + + case execplan::CalpontSystemCatalog::VARCHAR: case execplan::CalpontSystemCatalog::CHAR: case execplan::CalpontSystemCatalog::TEXT: diff --git a/dbcon/execplan/simplecolumn.cpp b/dbcon/execplan/simplecolumn.cpp index 2482cd43d..64955401e 100644 --- a/dbcon/execplan/simplecolumn.cpp +++ b/dbcon/execplan/simplecolumn.cpp @@ -388,21 +388,19 @@ bool SimpleColumn::operator==(const SimpleColumn& t) const if (fColumnName != t.fColumnName) return false; - if (fIndexName != t.fIndexName) - return false; - +// if (fIndexName != t.fIndexName) +// return false; if (fViewName != t.fViewName) return false; if (fOid != t.fOid) return false; - if (fData != t.fData) - return false; - - if (fAlias != t.fAlias) + if (data() != t.data()) return false; +// if (fAlias != t.fAlias) +// return false; if (fTableAlias != t.fTableAlias) return false; @@ -504,6 +502,7 @@ void SimpleColumn::evaluate(Row& row, bool& isNull) } case CalpontSystemCatalog::DATETIME: + case CalpontSystemCatalog::TIME: { fResult.intVal = row.getUintField<8>(fInputIndex); break; diff --git a/dbcon/execplan/simplecolumn.h b/dbcon/execplan/simplecolumn.h index 81571e057..60eff939b 100644 --- a/dbcon/execplan/simplecolumn.h +++ b/dbcon/execplan/simplecolumn.h @@ -331,6 +331,12 @@ public: return TreeNode::getDatetimeIntVal(); } + inline int64_t getTimeIntVal(rowgroup::Row& row, bool& isNull) + { + evaluate(row, isNull); + return TreeNode::getTimeIntVal(); + } + }; typedef boost::shared_ptr SSC; diff --git a/dbcon/execplan/simplefilter.cpp b/dbcon/execplan/simplefilter.cpp index 1df673dcd..639a04945 100644 --- a/dbcon/execplan/simplefilter.cpp +++ b/dbcon/execplan/simplefilter.cpp @@ -209,7 +209,8 @@ const string SimpleFilter::data() const fRhs->resultType().colDataType == CalpontSystemCatalog::TEXT || fRhs->resultType().colDataType == CalpontSystemCatalog::VARBINARY || fRhs->resultType().colDataType == CalpontSystemCatalog::DATE || - fRhs->resultType().colDataType == CalpontSystemCatalog::DATETIME)) + fRhs->resultType().colDataType == CalpontSystemCatalog::DATETIME || + fRhs->resultType().colDataType == CalpontSystemCatalog::TIME)) rhs = "'" + SimpleFilter::escapeString(fRhs->data()) + "'"; else rhs = fRhs->data(); @@ -221,6 +222,7 @@ const string SimpleFilter::data() const fLhs->resultType().colDataType == CalpontSystemCatalog::TEXT || fLhs->resultType().colDataType == CalpontSystemCatalog::VARBINARY || fLhs->resultType().colDataType == CalpontSystemCatalog::DATE || + fLhs->resultType().colDataType == CalpontSystemCatalog::TIME || fLhs->resultType().colDataType == CalpontSystemCatalog::DATETIME)) lhs = "'" + SimpleFilter::escapeString(fLhs->data()) + "'"; else @@ -544,6 +546,19 @@ void SimpleFilter::convertConstant() result.intVal = dataconvert::DataConvert::datetimeToInt(result.strVal); } } + else if (fRhs->resultType().colDataType == CalpontSystemCatalog::TIME) + { + if (lcc->constval().empty()) + { + lcc->constval("00:00:00"); + result.intVal = 0; + result.strVal = lcc->constval(); + } + else + { + result.intVal = dataconvert::DataConvert::timeToInt(result.strVal); + } + } lcc->result(result); } @@ -578,6 +593,19 @@ void SimpleFilter::convertConstant() result.intVal = dataconvert::DataConvert::datetimeToInt(result.strVal); } } + else if (fLhs->resultType().colDataType == CalpontSystemCatalog::TIME) + { + if (rcc->constval().empty()) + { + rcc->constval("00:00:00"); + result.intVal = 0; + result.strVal = rcc->constval(); + } + else + { + result.intVal = dataconvert::DataConvert::timeToInt(result.strVal); + } + } rcc->result(result); } diff --git a/dbcon/execplan/treenode.h b/dbcon/execplan/treenode.h index 92420da3f..cef9579e9 100644 --- a/dbcon/execplan/treenode.h +++ b/dbcon/execplan/treenode.h @@ -388,6 +388,10 @@ public: { return fResult.intVal; } + virtual int64_t getTimeIntVal(rowgroup::Row& row, bool& isNull) + { + return fResult.intVal; + } virtual void evaluate(rowgroup::Row& row, bool& isNull) {} inline bool getBoolVal(); @@ -399,6 +403,7 @@ public: inline IDB_Decimal getDecimalVal(); inline int32_t getDateIntVal(); inline int64_t getDatetimeIntVal(); + inline int64_t getTimeIntVal(); virtual const execplan::CalpontSystemCatalog::ColType& resultType() const { @@ -490,6 +495,7 @@ inline bool TreeNode::getBoolVal() case CalpontSystemCatalog::INT: case CalpontSystemCatalog::DATE: case CalpontSystemCatalog::DATETIME: + case CalpontSystemCatalog::TIME: return (fResult.intVal != 0); case CalpontSystemCatalog::UBIGINT: @@ -660,7 +666,14 @@ inline const std::string& TreeNode::getStrVal() case CalpontSystemCatalog::DATETIME: { - dataconvert::DataConvert::datetimeToString(fResult.intVal, tmp, 255); + dataconvert::DataConvert::datetimeToString(fResult.intVal, tmp, 255, fResultType.precision); + fResult.strVal = std::string(tmp); + break; + } + + case CalpontSystemCatalog::TIME: + { + dataconvert::DataConvert::timeToString(fResult.intVal, tmp, 255, fResultType.precision); fResult.strVal = std::string(tmp); break; } @@ -727,6 +740,7 @@ inline int64_t TreeNode::getIntVal() case CalpontSystemCatalog::DATE: case CalpontSystemCatalog::DATETIME: + case CalpontSystemCatalog::TIME: return fResult.intVal; default: @@ -769,6 +783,7 @@ inline uint64_t TreeNode::getUintVal() case CalpontSystemCatalog::DATE: case CalpontSystemCatalog::DATETIME: + case CalpontSystemCatalog::TIME: return fResult.intVal; default: @@ -831,6 +846,7 @@ inline float TreeNode::getFloatVal() case CalpontSystemCatalog::DATE: case CalpontSystemCatalog::DATETIME: + case CalpontSystemCatalog::TIME: return (float)fResult.intVal; default: @@ -895,6 +911,7 @@ inline double TreeNode::getDoubleVal() case CalpontSystemCatalog::DATE: case CalpontSystemCatalog::DATETIME: + case CalpontSystemCatalog::TIME: return (double)fResult.intVal; default: @@ -937,9 +954,14 @@ inline IDB_Decimal TreeNode::getDecimalVal() break; case CalpontSystemCatalog::DATE: + throw logging::InvalidConversionExcept("TreeNode::getDecimalVal: Invalid conversion from date."); + case CalpontSystemCatalog::DATETIME: throw logging::InvalidConversionExcept("TreeNode::getDecimalVal: Invalid conversion from datetime."); + case CalpontSystemCatalog::TIME: + throw logging::InvalidConversionExcept("TreeNode::getDecimalVal: Invalid conversion from time."); + case CalpontSystemCatalog::FLOAT: throw logging::InvalidConversionExcept("TreeNode::getDecimalVal: non-support conversion from float"); @@ -967,6 +989,28 @@ inline int64_t TreeNode::getDatetimeIntVal() { if (fResultType.colDataType == execplan::CalpontSystemCatalog::DATE) return (fResult.intVal & 0x00000000FFFFFFC0LL) << 32; + else if (fResultType.colDataType == execplan::CalpontSystemCatalog::TIME) + { + dataconvert::Time tt; + int day = 0; + + memcpy(&tt, &fResult.intVal, 8); + + // Note, this should probably be current date +/- time + if ((tt.hour > 23) && (!tt.is_neg)) + { + day = tt.hour / 24; + tt.hour = tt.hour % 24; + } + else if ((tt.hour < 0) || (tt.is_neg)) + { + tt.hour = 0; + } + + dataconvert::DateTime dt(0, 0, day, tt.hour, tt.minute, tt.second, tt.msecond); + memcpy(&fResult.intVal, &dt, 8); + return fResult.intVal; + } else if (fResultType.colDataType == execplan::CalpontSystemCatalog::DATETIME) //return (fResult.intVal & 0xFFFFFFFFFFF00000LL); return (fResult.intVal); @@ -974,6 +1018,23 @@ inline int64_t TreeNode::getDatetimeIntVal() return getIntVal(); } +inline int64_t TreeNode::getTimeIntVal() +{ + if (fResultType.colDataType == execplan::CalpontSystemCatalog::DATETIME) + { + dataconvert::DateTime dt; + + memcpy(&dt, &fResult.intVal, 8); + dataconvert::Time tt(0, dt.hour, dt.minute, dt.second, dt.msecond, false); + memcpy(&fResult.intVal, &tt, 8); + return fResult.intVal; + } + else if (fResultType.colDataType == execplan::CalpontSystemCatalog::TIME) + return (fResult.intVal); + else + return getIntVal(); +} + inline int32_t TreeNode::getDateIntVal() { if (fResultType.colDataType == execplan::CalpontSystemCatalog::DATETIME) diff --git a/dbcon/execplan/treenodeimpl.cpp b/dbcon/execplan/treenodeimpl.cpp index 5493acf60..1b7418d3d 100644 --- a/dbcon/execplan/treenodeimpl.cpp +++ b/dbcon/execplan/treenodeimpl.cpp @@ -64,7 +64,7 @@ const string TreeNodeImpl::toString() const bool TreeNodeImpl::operator==(const TreeNodeImpl& t) const { - if (fData == t.fData) + if (data() == t.data()) return true; return false; diff --git a/dbcon/execplan/windowfunctioncolumn.cpp b/dbcon/execplan/windowfunctioncolumn.cpp index f81a6a5d4..5c84ff2d1 100644 --- a/dbcon/execplan/windowfunctioncolumn.cpp +++ b/dbcon/execplan/windowfunctioncolumn.cpp @@ -403,6 +403,16 @@ void WindowFunctionColumn::evaluate(Row& row, bool& isNull) break; } + case CalpontSystemCatalog::TIME: + { + if (row.equals<8>(TIMENULL, fInputIndex)) + isNull = true; + else + fResult.intVal = row.getIntField<8>(fInputIndex); + + break; + } + case CalpontSystemCatalog::CHAR: case CalpontSystemCatalog::VARCHAR: case CalpontSystemCatalog::STRINT: diff --git a/dbcon/execplan/windowfunctioncolumn.h b/dbcon/execplan/windowfunctioncolumn.h index 8b427947c..3fc983e55 100644 --- a/dbcon/execplan/windowfunctioncolumn.h +++ b/dbcon/execplan/windowfunctioncolumn.h @@ -213,7 +213,11 @@ public: evaluate(row, isNull); return TreeNode::getDatetimeIntVal(); } - + virtual int64_t getTimeIntVal(rowgroup::Row& row, bool& isNull) + { + evaluate(row, isNull); + return TreeNode::getTimeIntVal(); + } private: void evaluate(rowgroup::Row& row, bool& isNull); }; diff --git a/dbcon/joblist/crossenginestep.cpp b/dbcon/joblist/crossenginestep.cpp index 483249067..789e58cd4 100644 --- a/dbcon/joblist/crossenginestep.cpp +++ b/dbcon/joblist/crossenginestep.cpp @@ -351,6 +351,10 @@ int64_t CrossEngineStep::convertValueNum( rv = boost::any_cast(anyVal); break; + case CalpontSystemCatalog::TIME: + rv = boost::any_cast(anyVal); + break; + case CalpontSystemCatalog::DECIMAL: case CalpontSystemCatalog::UDECIMAL: if (ct.colWidth == CalpontSystemCatalog::ONE_BYTE) diff --git a/dbcon/joblist/expressionstep.cpp b/dbcon/joblist/expressionstep.cpp index 4a169e081..0e064c359 100644 --- a/dbcon/joblist/expressionstep.cpp +++ b/dbcon/joblist/expressionstep.cpp @@ -403,8 +403,9 @@ void ExpressionStep::populateColumnInfo(SimpleColumn* sc, JobInfo& jobInfo) TupleInfo ti(setTupleInfo(ct, sc->oid(), jobInfo, tblOid, sc, alias)); fColumnKeys.push_back(ti.key); - // @bug 2990, MySQL date/datetime type is different from IDB type - if (ti.dtype == CalpontSystemCatalog::DATE || ti.dtype == CalpontSystemCatalog::DATETIME) + // @bug 2990, MySQL time/date/datetime type is different from IDB type + if (ti.dtype == CalpontSystemCatalog::DATE || ti.dtype == CalpontSystemCatalog::DATETIME || + ti.dtype == CalpontSystemCatalog::TIME) { if (ti.dtype != ct.colDataType) { diff --git a/dbcon/joblist/groupconcat.cpp b/dbcon/joblist/groupconcat.cpp index fd122fe8d..234fc0a8e 100644 --- a/dbcon/joblist/groupconcat.cpp +++ b/dbcon/joblist/groupconcat.cpp @@ -507,6 +507,12 @@ void GroupConcator::outputRow(std::ostringstream& oss, const rowgroup::Row& row) break; } + case CalpontSystemCatalog::TIME: + { + oss << DataConvert::timeToString(row.getUintField(*i)); + break; + } + default: { break; @@ -640,6 +646,28 @@ int64_t GroupConcator::lengthEstimate(const rowgroup::Row& row) case CalpontSystemCatalog::DATETIME: { fieldLen = 19; // YYYY-MM-DD HH24:MI:SS + // Decimal point and milliseconds + uint64_t colPrecision = row.getPrecision(*i); + + if (colPrecision > 0 && colPrecision < 7) + { + fieldLen += colPrecision + 1; + } + + break; + } + + case CalpontSystemCatalog::TIME: + { + fieldLen = 10; // -HHH:MI:SS + // Decimal point and milliseconds + uint64_t colPrecision = row.getPrecision(*i); + + if (colPrecision > 0 && colPrecision < 7) + { + fieldLen += colPrecision + 1; + } + break; } diff --git a/dbcon/joblist/jlf_common.cpp b/dbcon/joblist/jlf_common.cpp index 063e24dfc..f5dbeee17 100644 --- a/dbcon/joblist/jlf_common.cpp +++ b/dbcon/joblist/jlf_common.cpp @@ -792,6 +792,11 @@ bool compatibleColumnTypes(const CalpontSystemCatalog::ColDataType& dt1, uint32_ break; + case CalpontSystemCatalog::TIME: + if (dt2 != CalpontSystemCatalog::TIME) return false; + + break; + case CalpontSystemCatalog::CHAR: case CalpontSystemCatalog::VARCHAR: case CalpontSystemCatalog::TEXT: diff --git a/dbcon/joblist/jlf_execplantojoblist.cpp b/dbcon/joblist/jlf_execplantojoblist.cpp index f4badbe57..b81a2ec8d 100644 --- a/dbcon/joblist/jlf_execplantojoblist.cpp +++ b/dbcon/joblist/jlf_execplantojoblist.cpp @@ -282,6 +282,10 @@ int64_t valueNullNum(const CalpontSystemCatalog::ColType& ct) n = boost::any_cast(anyVal); break; + case CalpontSystemCatalog::TIME: + n = boost::any_cast(anyVal); + break; + case CalpontSystemCatalog::DECIMAL: case CalpontSystemCatalog::UDECIMAL: if (ct.colWidth == execplan::CalpontSystemCatalog::ONE_BYTE) @@ -421,6 +425,10 @@ int64_t convertValueNum(const string& str, const CalpontSystemCatalog::ColType& v = boost::any_cast(anyVal); break; + case CalpontSystemCatalog::TIME: + v = boost::any_cast(anyVal); + break; + case CalpontSystemCatalog::DECIMAL: case CalpontSystemCatalog::UDECIMAL: if (ct.colWidth == execplan::CalpontSystemCatalog::ONE_BYTE) @@ -2099,6 +2107,7 @@ const JobStepVector doOuterJoinOnFilter(OuterJoinOnFilter* oj, JobInfo& jobInfo) set doneNodes; // solved joins and simple filters map cpMap; // link for node removal JobStepVector join; // join step with its projection steps + bool keepFilters = false; // keep filters for cross engine step // To compromise the front end difficulty on setting outer attributes. set tablesInOuter; @@ -2323,6 +2332,14 @@ const JobStepVector doOuterJoinOnFilter(OuterJoinOnFilter* oj, JobInfo& jobInfo) jsv.insert(jsv.end(), sfv.begin(), sfv.end()); + // MCOL-1182 if we are doing a join between a cross engine + // step and a constant then keep the filter for the cross + // engine step instead of deleting it further down. + if (!sc->isInfiniDB()) + { + keepFilters = true; + } + doneNodes.insert(cn); } } @@ -2364,7 +2381,11 @@ const JobStepVector doOuterJoinOnFilter(OuterJoinOnFilter* oj, JobInfo& jobInfo) if (p == NULL) { filters = NULL; - delete c; + + if (!keepFilters) + { + delete c; + } } else { @@ -2411,8 +2432,12 @@ const JobStepVector doOuterJoinOnFilter(OuterJoinOnFilter* oj, JobInfo& jobInfo) p->left(nullTree); p->right(nullTree); - delete p; - delete c; + + if (!keepFilters) + { + delete p; + delete c; + } } } diff --git a/dbcon/joblist/jlf_subquery.cpp b/dbcon/joblist/jlf_subquery.cpp index 5c868bee3..add3dc533 100644 --- a/dbcon/joblist/jlf_subquery.cpp +++ b/dbcon/joblist/jlf_subquery.cpp @@ -135,6 +135,11 @@ void getColumnValue(ConstantColumn** cc, uint64_t i, const Row& row) *cc = new ConstantColumn(oss.str()); break; + case CalpontSystemCatalog::TIME: + oss << dataconvert::DataConvert::timeToString(row.getUintField<8>(i)); + *cc = new ConstantColumn(oss.str()); + break; + case CalpontSystemCatalog::CHAR: case CalpontSystemCatalog::VARCHAR: case CalpontSystemCatalog::TEXT: diff --git a/dbcon/joblist/joblisttypes.h b/dbcon/joblist/joblisttypes.h index 3f3755a71..37d52f9f4 100644 --- a/dbcon/joblist/joblisttypes.h +++ b/dbcon/joblist/joblisttypes.h @@ -56,6 +56,8 @@ const uint32_t DATENULL = 0xFFFFFFFE; const uint32_t DATEEMPTYROW = 0xFFFFFFFF; const uint64_t DATETIMENULL = 0xFFFFFFFFFFFFFFFEULL; const uint64_t DATETIMEEMPTYROW = 0xFFFFFFFFFFFFFFFFULL; +const uint64_t TIMENULL = 0xFFFFFFFFFFFFFFFEULL; +const uint64_t TIMEEMPTYROW = 0xFFFFFFFFFFFFFFFFULL; const uint8_t CHAR1NULL = 0xFE; const uint8_t CHAR1EMPTYROW = 0xFF; diff --git a/dbcon/joblist/lbidlist.cpp b/dbcon/joblist/lbidlist.cpp index 58ad2dd5e..c317defc9 100644 --- a/dbcon/joblist/lbidlist.cpp +++ b/dbcon/joblist/lbidlist.cpp @@ -27,6 +27,7 @@ #include "calpontsystemcatalog.h" #include "brm.h" #include "brmtypes.h" +#include "dataconvert.h" #define IS_VERBOSE (fDebug >= 4) #define IS_DETAIL (fDebug >= 3) @@ -509,6 +510,7 @@ bool LBIDList::CasualPartitionDataType(const CalpontSystemCatalog::ColDataType t case CalpontSystemCatalog::DECIMAL: case CalpontSystemCatalog::DATE: case CalpontSystemCatalog::DATETIME: + case CalpontSystemCatalog::TIME: case CalpontSystemCatalog::UTINYINT: case CalpontSystemCatalog::USMALLINT: case CalpontSystemCatalog::UDECIMAL: @@ -741,7 +743,14 @@ bool LBIDList::CasualPartitionPredicate(const int64_t Min, if (bIsChar && 1 < ct.colWidth) { - scan = compareVal(order_swap(Min), order_swap(Max), order_swap(value), + // MCOL-1246 Trim trailing whitespace for matching so that we have + // the same as InnoDB behaviour + int64_t tMin = Min; + int64_t tMax = Max; + dataconvert::DataConvert::trimWhitespace(tMin); + dataconvert::DataConvert::trimWhitespace(tMax); + + scan = compareVal(order_swap(tMin), order_swap(tMax), order_swap(value), op, lcf); // cout << "scan=" << (uint32_t) scan << endl; } diff --git a/dbcon/joblist/pcolscan.cpp b/dbcon/joblist/pcolscan.cpp index cd19ce34c..6cea4fc03 100644 --- a/dbcon/joblist/pcolscan.cpp +++ b/dbcon/joblist/pcolscan.cpp @@ -1167,6 +1167,7 @@ bool pColScanStep::isEmptyVal(const uint8_t* val8) const case CalpontSystemCatalog::VARCHAR: case CalpontSystemCatalog::DATE: case CalpontSystemCatalog::DATETIME: + case CalpontSystemCatalog::TIME: if (width == 1) { return (*val8 == joblist::CHAR1EMPTYROW); diff --git a/dbcon/joblist/subquerytransformer.cpp b/dbcon/joblist/subquerytransformer.cpp index a5e0759fa..c212c701d 100644 --- a/dbcon/joblist/subquerytransformer.cpp +++ b/dbcon/joblist/subquerytransformer.cpp @@ -237,9 +237,10 @@ SJSTEP& SubQueryTransformer::makeSubQueryStep(execplan::CalpontSelectExecutionPl fVtable.columnType(ct, i); } } - // MySQL date/datetime type is different from IDB type + // MySQL time/date/datetime type is different from IDB type else if (colDataTypeInRg == CalpontSystemCatalog::DATE || - colDataTypeInRg == CalpontSystemCatalog::DATETIME) + colDataTypeInRg == CalpontSystemCatalog::DATETIME || + colDataTypeInRg == CalpontSystemCatalog::TIME) { ct.colWidth = row.getColumnWidth(i); ct.colDataType = row.getColTypes()[i]; diff --git a/dbcon/joblist/tupleaggregatestep.cpp b/dbcon/joblist/tupleaggregatestep.cpp index 6c8760d71..3dbd01311 100644 --- a/dbcon/joblist/tupleaggregatestep.cpp +++ b/dbcon/joblist/tupleaggregatestep.cpp @@ -232,6 +232,9 @@ inline string colTypeIdString(CalpontSystemCatalog::ColDataType type) case CalpontSystemCatalog::DATETIME: return string("DATETIME"); + case CalpontSystemCatalog::TIME: + return string("TIME"); + case CalpontSystemCatalog::VARCHAR: return string("VARCHAR"); @@ -1281,10 +1284,12 @@ void TupleAggregateStep::prep1PhaseAggregate( if (aggOp == ROWAGG_UDAF) { std::vector::iterator it = jobInfo.projectionCols.begin() + projColsUDAFIndex; + for (; it != jobInfo.projectionCols.end(); it++) { UDAFColumn* udafc = dynamic_cast((*it).get()); projColsUDAFIndex++; + if (udafc) { pUDAFFunc = udafc->getContext().getFunction(); @@ -1292,9 +1297,9 @@ void TupleAggregateStep::prep1PhaseAggregate( funct.reset(new RowUDAFFunctionCol(udafc->getContext(), colProj, i)); break; } - + } - + if (it == jobInfo.projectionCols.end()) { throw logic_error("prep1PhaseAggregate: A UDAF function is called but there's no/not enough UDAFColumn/-s"); @@ -1331,7 +1336,8 @@ void TupleAggregateStep::prep1PhaseAggregate( typeProj[colProj] == CalpontSystemCatalog::BLOB || typeProj[colProj] == CalpontSystemCatalog::TEXT || typeProj[colProj] == CalpontSystemCatalog::DATE || - typeProj[colProj] == CalpontSystemCatalog::DATETIME) + typeProj[colProj] == CalpontSystemCatalog::DATETIME || + typeProj[colProj] == CalpontSystemCatalog::TIME) { Message::Args args; args.add("sum/average"); @@ -1413,7 +1419,8 @@ void TupleAggregateStep::prep1PhaseAggregate( typeProj[colProj] == CalpontSystemCatalog::TEXT || typeProj[colProj] == CalpontSystemCatalog::BLOB || typeProj[colProj] == CalpontSystemCatalog::DATE || - typeProj[colProj] == CalpontSystemCatalog::DATETIME) + typeProj[colProj] == CalpontSystemCatalog::DATETIME || + typeProj[colProj] == CalpontSystemCatalog::TIME) { Message::Args args; args.add("variance/standard deviation"); @@ -1823,10 +1830,12 @@ void TupleAggregateStep::prep1PhaseDistinctAggregate( if (aggOp == ROWAGG_UDAF) { std::vector::iterator it = jobInfo.projectionCols.begin() + projColsUDAFIndex; + for (; it != jobInfo.projectionCols.end(); it++) { UDAFColumn* udafc = dynamic_cast((*it).get()); projColsUDAFIndex++; + if (udafc) { pUDAFFunc = udafc->getContext().getFunction(); @@ -1834,9 +1843,9 @@ void TupleAggregateStep::prep1PhaseDistinctAggregate( funct.reset(new RowUDAFFunctionCol(udafc->getContext(), colProj, colAgg)); break; } - + } - + if (it == jobInfo.projectionCols.end()) { throw logic_error("prep1PhaseDistinctAggregate: A UDAF function is called but there's no/not enough UDAFColumn/-s"); @@ -1877,7 +1886,8 @@ void TupleAggregateStep::prep1PhaseDistinctAggregate( typeProj[colProj] == CalpontSystemCatalog::BLOB || typeProj[colProj] == CalpontSystemCatalog::TEXT || typeProj[colProj] == CalpontSystemCatalog::DATE || - typeProj[colProj] == CalpontSystemCatalog::DATETIME) + typeProj[colProj] == CalpontSystemCatalog::DATETIME || + typeProj[colProj] == CalpontSystemCatalog::TIME) { Message::Args args; args.add("sum/average"); @@ -1962,7 +1972,8 @@ void TupleAggregateStep::prep1PhaseDistinctAggregate( typeProj[colProj] == CalpontSystemCatalog::BLOB || typeProj[colProj] == CalpontSystemCatalog::TEXT || typeProj[colProj] == CalpontSystemCatalog::DATE || - typeProj[colProj] == CalpontSystemCatalog::DATETIME) + typeProj[colProj] == CalpontSystemCatalog::DATETIME || + typeProj[colProj] == CalpontSystemCatalog::TIME) { Message::Args args; args.add("variance/standard deviation"); @@ -2145,7 +2156,8 @@ void TupleAggregateStep::prep1PhaseDistinctAggregate( typeAgg[colAgg] == CalpontSystemCatalog::BLOB || typeAgg[colAgg] == CalpontSystemCatalog::TEXT || typeAgg[colAgg] == CalpontSystemCatalog::DATE || - typeAgg[colAgg] == CalpontSystemCatalog::DATETIME) + typeAgg[colAgg] == CalpontSystemCatalog::DATETIME || + typeAgg[colAgg] == CalpontSystemCatalog::TIME) { Message::Args args; args.add("sum/average"); @@ -3011,10 +3023,12 @@ void TupleAggregateStep::prep2PhasesAggregate( if (aggOp == ROWAGG_UDAF) { std::vector::iterator it = jobInfo.projectionCols.begin() + projColsUDAFIndex; + for (; it != jobInfo.projectionCols.end(); it++) { UDAFColumn* udafc = dynamic_cast((*it).get()); projColsUDAFIndex++; + if (udafc) { pUDAFFunc = udafc->getContext().getFunction(); @@ -3022,9 +3036,9 @@ void TupleAggregateStep::prep2PhasesAggregate( funct.reset(new RowUDAFFunctionCol(udafc->getContext(), colProj, colAggPm)); break; } - + } - + if (it == jobInfo.projectionCols.end()) { throw logic_error("prep2PhasesAggregate: A UDAF function is called but there's no/not enough UDAFColumn/-s"); @@ -3065,7 +3079,8 @@ void TupleAggregateStep::prep2PhasesAggregate( typeProj[colProj] == CalpontSystemCatalog::BLOB || typeProj[colProj] == CalpontSystemCatalog::TEXT || typeProj[colProj] == CalpontSystemCatalog::DATE || - typeProj[colProj] == CalpontSystemCatalog::DATETIME) + typeProj[colProj] == CalpontSystemCatalog::DATETIME || + typeProj[colProj] == CalpontSystemCatalog::TIME) { Message::Args args; args.add("sum/average"); @@ -3154,7 +3169,8 @@ void TupleAggregateStep::prep2PhasesAggregate( typeProj[colProj] == CalpontSystemCatalog::BLOB || typeProj[colProj] == CalpontSystemCatalog::TEXT || typeProj[colProj] == CalpontSystemCatalog::DATE || - typeProj[colProj] == CalpontSystemCatalog::DATETIME) + typeProj[colProj] == CalpontSystemCatalog::DATETIME || + typeProj[colProj] == CalpontSystemCatalog::TIME) { Message::Args args; args.add("variance/standard deviation"); @@ -3823,19 +3839,21 @@ void TupleAggregateStep::prep2PhasesDistinctAggregate( if (aggOp == ROWAGG_UDAF) { std::vector::iterator it = jobInfo.projectionCols.begin() + projColsUDAFIndex; + for (; it != jobInfo.projectionCols.end(); it++) { UDAFColumn* udafc = dynamic_cast((*it).get()); projColsUDAFIndex++; + if (udafc) { pUDAFFunc = udafc->getContext().getFunction(); // Create a RowAggFunctionCol (UDAF subtype) with the context. funct.reset(new RowUDAFFunctionCol(udafc->getContext(), colProj, colAggPm)); break; - } + } } - + if (it == jobInfo.projectionCols.end()) { throw logic_error("prep2PhasesDistinctAggregate: A UDAF function is called but there's no/not enough UDAFColumn/-s"); @@ -3876,7 +3894,8 @@ void TupleAggregateStep::prep2PhasesDistinctAggregate( typeProj[colProj] == CalpontSystemCatalog::BLOB || typeProj[colProj] == CalpontSystemCatalog::TEXT || typeProj[colProj] == CalpontSystemCatalog::DATE || - typeProj[colProj] == CalpontSystemCatalog::DATETIME) + typeProj[colProj] == CalpontSystemCatalog::DATETIME || + typeProj[colProj] == CalpontSystemCatalog::TIME) { Message::Args args; args.add("sum/average"); @@ -3961,7 +3980,8 @@ void TupleAggregateStep::prep2PhasesDistinctAggregate( typeProj[colProj] == CalpontSystemCatalog::BLOB || typeProj[colProj] == CalpontSystemCatalog::TEXT || typeProj[colProj] == CalpontSystemCatalog::DATE || - typeProj[colProj] == CalpontSystemCatalog::DATETIME) + typeProj[colProj] == CalpontSystemCatalog::DATETIME || + typeProj[colProj] == CalpontSystemCatalog::TIME) { Message::Args args; args.add("variance/standard deviation"); @@ -4176,7 +4196,8 @@ void TupleAggregateStep::prep2PhasesDistinctAggregate( typeAggUm[colUm] == CalpontSystemCatalog::BLOB || typeAggUm[colUm] == CalpontSystemCatalog::TEXT || typeAggUm[colUm] == CalpontSystemCatalog::DATE || - typeAggUm[colUm] == CalpontSystemCatalog::DATETIME) + typeAggUm[colUm] == CalpontSystemCatalog::DATETIME || + typeAggUm[colUm] == CalpontSystemCatalog::TIME) { Message::Args args; args.add("sum/average"); diff --git a/dbcon/joblist/tupleconstantstep.cpp b/dbcon/joblist/tupleconstantstep.cpp index 6e75ad350..309bb0058 100644 --- a/dbcon/joblist/tupleconstantstep.cpp +++ b/dbcon/joblist/tupleconstantstep.cpp @@ -223,6 +223,7 @@ void TupleConstantStep::constructContanstRow(const JobInfo& jobInfo) case CalpontSystemCatalog::BIGINT: case CalpontSystemCatalog::DATE: case CalpontSystemCatalog::DATETIME: + case CalpontSystemCatalog::TIME: { fRowConst.setIntField(c.intVal, *i); break; diff --git a/dbcon/joblist/tupleunion.cpp b/dbcon/joblist/tupleunion.cpp index 51b5cde97..77665ae90 100644 --- a/dbcon/joblist/tupleunion.cpp +++ b/dbcon/joblist/tupleunion.cpp @@ -473,7 +473,8 @@ void TupleUnion::normalize(const Row& in, Row* out) case CalpontSystemCatalog::DATE: case CalpontSystemCatalog::DATETIME: - throw logic_error("TupleUnion::normalize(): tried to normalize an int to a date or datetime"); + case CalpontSystemCatalog::TIME: + throw logic_error("TupleUnion::normalize(): tried to normalize an int to a time, date or datetime"); case CalpontSystemCatalog::FLOAT: case CalpontSystemCatalog::UFLOAT: @@ -582,7 +583,8 @@ dec1: case CalpontSystemCatalog::DATE: case CalpontSystemCatalog::DATETIME: - throw logic_error("TupleUnion::normalize(): tried to normalize an int to a date or datetime"); + case CalpontSystemCatalog::TIME: + throw logic_error("TupleUnion::normalize(): tried to normalize an int to a time, date or datetime"); case CalpontSystemCatalog::FLOAT: case CalpontSystemCatalog::UFLOAT: @@ -736,6 +738,33 @@ dec2: break; + case CalpontSystemCatalog::TIME: + switch (out->getColTypes()[i]) + { + case CalpontSystemCatalog::TIME: + out->setIntField(in.getIntField(i), i); + break; + + case CalpontSystemCatalog::CHAR: + case CalpontSystemCatalog::TEXT: + case CalpontSystemCatalog::VARCHAR: + { + string d = DataConvert::timeToString(in.getIntField(i)); + out->setStringField(d, i); + break; + } + + default: + { + ostringstream os; + os << "TupleUnion::normalize(): tried an illegal conversion: time to " + << out->getColTypes()[i]; + throw logic_error(os.str()); + } + } + + break; + case CalpontSystemCatalog::FLOAT: case CalpontSystemCatalog::UFLOAT: case CalpontSystemCatalog::DOUBLE: @@ -1069,6 +1098,10 @@ void TupleUnion::writeNull(Row* out, uint32_t col) out->setUintField<8>(joblist::DATETIMENULL, col); break; + case CalpontSystemCatalog::TIME: + out->setUintField<8>(joblist::TIMENULL, col); + break; + case CalpontSystemCatalog::CHAR: case CalpontSystemCatalog::TEXT: case CalpontSystemCatalog::VARCHAR: diff --git a/dbcon/joblist/windowfunctionstep.cpp b/dbcon/joblist/windowfunctionstep.cpp index ed382ee4a..4d24f0b4b 100644 --- a/dbcon/joblist/windowfunctionstep.cpp +++ b/dbcon/joblist/windowfunctionstep.cpp @@ -1201,6 +1201,7 @@ boost::shared_ptr WindowFunctionStep::parseFrameBoundRows( case execplan::CalpontSystemCatalog::UDECIMAL: case execplan::CalpontSystemCatalog::DATE: case execplan::CalpontSystemCatalog::DATETIME: + case execplan::CalpontSystemCatalog::TIME: { fb.reset(new FrameBoundExpressionRow(type, id, idx)); break; @@ -1351,6 +1352,7 @@ boost::shared_ptr WindowFunctionStep::parseFrameBoundRange(const exe case execplan::CalpontSystemCatalog::UDECIMAL: case execplan::CalpontSystemCatalog::DATE: case execplan::CalpontSystemCatalog::DATETIME: + case execplan::CalpontSystemCatalog::TIME: { if (isConstant) { diff --git a/dbcon/mysql/columnstore_info.sql b/dbcon/mysql/columnstore_info.sql index 80b0f12b1..563052a11 100644 --- a/dbcon/mysql/columnstore_info.sql +++ b/dbcon/mysql/columnstore_info.sql @@ -52,21 +52,21 @@ CREATE PROCEDURE table_usage (IN t_schema char(64), IN t_name char(64)) CREATE TABLE columnstore_info.columnstore_files engine=myisam as (select * from information_schema.columnstore_files); ALTER TABLE columnstore_info.columnstore_files ADD INDEX `object_id` (`object_id`); IF t_name IS NOT NULL THEN -SELECT TABLE_SCHEMA, TABLE_NAME, columnstore_info.format_filesize(data) as DATA_DISK_USAGE, columnstore_info.format_filesize(dict) as DICT_DISK_USAGE, columnstore_info.format_filesize(data + dict) as TOTAL_USAGE FROM ( +SELECT TABLE_SCHEMA, TABLE_NAME, columnstore_info.format_filesize(data) as DATA_DISK_USAGE, columnstore_info.format_filesize(dict) as DICT_DISK_USAGE, columnstore_info.format_filesize(data + COALESCE(dict, 0)) as TOTAL_USAGE FROM ( SELECT TABLE_SCHEMA, TABLE_NAME, (SELECT sum(cf.file_size) as data FROM columnstore_info.columnstore_columns cc JOIN columnstore_info.columnstore_files cf ON cc.object_id = cf.object_id WHERE table_name = ics.table_name and table_schema = ics.table_schema) as data, (SELECT sum(cf.file_size) as dict FROM columnstore_info.columnstore_columns cc JOIN columnstore_info.columnstore_files cf ON cc.dictionary_object_id = cf.object_id WHERE table_name = ics.table_name and table_schema = ics.table_schema GROUP BY table_schema, table_name) as dict FROM columnstore_info.columnstore_columns ics where table_name = t_name and (table_schema = t_schema or t_schema IS NULL) group by table_schema, table_name ) q; ELSEIF t_schema IS NOT NULL THEN -SELECT TABLE_SCHEMA, TABLE_NAME, columnstore_info.format_filesize(data) as DATA_DISK_USAGE, columnstore_info.format_filesize(dict) as DICT_DISK_USAGE, columnstore_info.format_filesize(data + dict) as TOTAL_USAGE FROM ( +SELECT TABLE_SCHEMA, TABLE_NAME, columnstore_info.format_filesize(data) as DATA_DISK_USAGE, columnstore_info.format_filesize(dict) as DICT_DISK_USAGE, columnstore_info.format_filesize(data + COALESCE(dict, 0)) as TOTAL_USAGE FROM ( SELECT TABLE_SCHEMA, TABLE_NAME, (SELECT sum(cf.file_size) as data FROM columnstore_info.columnstore_columns cc JOIN columnstore_info.columnstore_files cf ON cc.object_id = cf.object_id WHERE table_name = ics.table_name and table_schema = ics.table_schema) as data, (SELECT sum(cf.file_size) as dict FROM columnstore_info.columnstore_columns cc JOIN columnstore_info.columnstore_files cf ON cc.dictionary_object_id = cf.object_id WHERE table_name = ics.table_name and table_schema = ics.table_schema GROUP BY table_schema, table_name) as dict FROM columnstore_info.columnstore_columns ics where table_schema = t_schema group by table_schema, table_name ) q; ELSE -SELECT TABLE_SCHEMA, TABLE_NAME, columnstore_info.format_filesize(data) as DATA_DISK_USAGE, columnstore_info.format_filesize(dict) as DICT_DISK_USAGE, columnstore_info.format_filesize(data + dict) as TOTAL_USAGE FROM ( +SELECT TABLE_SCHEMA, TABLE_NAME, columnstore_info.format_filesize(data) as DATA_DISK_USAGE, columnstore_info.format_filesize(dict) as DICT_DISK_USAGE, columnstore_info.format_filesize(data + COALESCE(dict, 0)) as TOTAL_USAGE FROM ( SELECT TABLE_SCHEMA, TABLE_NAME, (SELECT sum(cf.file_size) as data FROM columnstore_info.columnstore_columns cc JOIN columnstore_info.columnstore_files cf ON cc.object_id = cf.object_id WHERE table_name = ics.table_name and table_schema = ics.table_schema) as data, (SELECT sum(cf.file_size) as dict FROM columnstore_info.columnstore_columns cc JOIN columnstore_info.columnstore_files cf ON cc.dictionary_object_id = cf.object_id WHERE table_name = ics.table_name and table_schema = ics.table_schema GROUP BY table_schema, table_name) as dict FROM columnstore_info.columnstore_columns ics diff --git a/dbcon/mysql/ha_calpont_ddl.cpp b/dbcon/mysql/ha_calpont_ddl.cpp index 01a42ac5c..ba9b2c299 100644 --- a/dbcon/mysql/ha_calpont_ddl.cpp +++ b/dbcon/mysql/ha_calpont_ddl.cpp @@ -174,6 +174,10 @@ uint32_t convertDataType(int dataType) calpontDataType = CalpontSystemCatalog::DATETIME; break; + case ddlpackage::DDL_TIME: + calpontDataType = CalpontSystemCatalog::TIME; + break; + case ddlpackage::DDL_CLOB: calpontDataType = CalpontSystemCatalog::CLOB; break; @@ -2183,6 +2187,7 @@ int ha_calpont_impl_rename_table_(const char* from, const char* to, cal_connecti THD* thd = current_thd; string emsg; + ostringstream stmt1; pair fromPair; pair toPair; string stmt; @@ -2210,16 +2215,15 @@ int ha_calpont_impl_rename_table_(const char* from, const char* to, cal_connecti return -1; } - stmt = thd->query(); - stmt += ';'; + stmt1 << "alter table " << fromPair.second << " rename to " << toPair.second << ";"; + + stmt = stmt1.str(); string db; - if ( thd->db ) - db = thd->db; - else if ( fromPair.first.length() != 0 ) + if ( fromPair.first.length() != 0 ) db = fromPair.first; - else - db = toPair.first; + else if ( thd->db ) + db = thd->db; int rc = ProcessDDLStatement(stmt, db, "", tid2sid(thd->thread_id), emsg); @@ -2266,7 +2270,7 @@ extern "C" int rc = ProcessDDLStatement(stmt, db, "", tid2sid(thd->thread_id), emsg, compressiontype); if (rc != 0) - push_warning(thd, Sql_condition::WARN_LEVEL_WARN, 9999, emsg.c_str()); + push_warning(thd, Sql_condition::WARN_LEVEL_ERROR, 9999, emsg.c_str()); return rc; } diff --git a/dbcon/mysql/ha_calpont_dml.cpp b/dbcon/mysql/ha_calpont_dml.cpp index 1cef4d4da..cf103a801 100644 --- a/dbcon/mysql/ha_calpont_dml.cpp +++ b/dbcon/mysql/ha_calpont_dml.cpp @@ -136,7 +136,8 @@ int buildBuffer(uchar* buf, string& buffer, int& columns, TABLE* table) (*field)->type() == MYSQL_TYPE_STRING || (*field)->type() == MYSQL_TYPE_DATE || (*field)->type() == MYSQL_TYPE_DATETIME || - (*field)->type() == MYSQL_TYPE_DATETIME2 ) + (*field)->type() == MYSQL_TYPE_DATETIME2 || + (*field)->type() == MYSQL_TYPE_TIME ) vals.append("'"); while (ptr < end_ptr) @@ -166,7 +167,8 @@ int buildBuffer(uchar* buf, string& buffer, int& columns, TABLE* table) (*field)->type() == MYSQL_TYPE_STRING || (*field)->type() == MYSQL_TYPE_DATE || (*field)->type() == MYSQL_TYPE_DATETIME || - (*field)->type() == MYSQL_TYPE_DATETIME2 ) + (*field)->type() == MYSQL_TYPE_DATETIME2 || + (*field)->type() == MYSQL_TYPE_TIME ) vals.append("'"); } } @@ -838,11 +840,23 @@ int ha_calpont_impl_write_batch_row_(uchar* buf, TABLE* table, cal_impl_if::cal_ // mariadb 10.1 compatibility -- MYSQL_TYPE_DATETIME2 introduced in mysql 5.6 MYSQL_TIME ltime; const uchar* pos = buf; - longlong tmp = my_datetime_packed_from_binary(pos, 0); + longlong tmp = my_datetime_packed_from_binary(pos, table->field[colpos]->decimals()); TIME_from_longlong_datetime_packed(<ime, tmp); - fprintf(ci.filePtr, "%04d-%02d-%02d %02d:%02d:%02d%c", - ltime.year, ltime.month, ltime.day, - ltime.hour, ltime.minute, ltime.second, ci.delimiter); + + if (!ltime.second_part) + { + fprintf(ci.filePtr, "%04d-%02d-%02d %02d:%02d:%02d%c", + ltime.year, ltime.month, ltime.day, + ltime.hour, ltime.minute, ltime.second, ci.delimiter); + } + else + { + fprintf(ci.filePtr, "%04d-%02d-%02d %02d:%02d:%02d.%ld%c", + ltime.year, ltime.month, ltime.day, + ltime.hour, ltime.minute, ltime.second, + ltime.second_part, ci.delimiter); + } + buf += table->field[colpos]->pack_length(); } else @@ -866,6 +880,39 @@ int ha_calpont_impl_write_batch_row_(uchar* buf, TABLE* table, cal_impl_if::cal_ break; } + case CalpontSystemCatalog::TIME: + { + if (nullVal && (ci.columnTypes[colpos].constraintType != CalpontSystemCatalog::NOTNULL_CONSTRAINT)) + { + fprintf(ci.filePtr, "%c", ci.delimiter); + + buf += table->field[colpos]->pack_length(); + } + else + { + MYSQL_TIME ltime; + const uchar* pos = buf; + longlong tmp = my_time_packed_from_binary(pos, table->field[colpos]->decimals()); + TIME_from_longlong_time_packed(<ime, tmp); + + if (!ltime.second_part) + { + fprintf(ci.filePtr, "%02d:%02d:%02d%c", + ltime.hour, ltime.minute, ltime.second, ci.delimiter); + } + else + { + fprintf(ci.filePtr, "%02d:%02d:%02d.%ld%c", + ltime.hour, ltime.minute, ltime.second, + ltime.second_part, ci.delimiter); + } + + buf += table->field[colpos]->pack_length(); + } + + break; + } + case CalpontSystemCatalog::CHAR: { if (nullVal && (ci.columnTypes[colpos].constraintType != CalpontSystemCatalog::NOTNULL_CONSTRAINT)) diff --git a/dbcon/mysql/ha_calpont_execplan.cpp b/dbcon/mysql/ha_calpont_execplan.cpp index f5ba036f1..02fa4d8a4 100644 --- a/dbcon/mysql/ha_calpont_execplan.cpp +++ b/dbcon/mysql/ha_calpont_execplan.cpp @@ -2681,6 +2681,11 @@ CalpontSystemCatalog::ColType colType_MysqlToIDB (const Item* item) ct.colDataType = CalpontSystemCatalog::DATETIME; ct.colWidth = 8; } + else if (item->field_type() == MYSQL_TYPE_TIME) + { + ct.colDataType = CalpontSystemCatalog::TIME; + ct.colWidth = 8; + } if (item->field_type() == MYSQL_TYPE_BLOB) { @@ -2969,6 +2974,13 @@ ReturnedColumn* buildReturnedColumn(Item* item, gp_walk_info& gwi, bool& nonSupp break; } + case Item::COND_ITEM: + { + // MCOL-1196: Allow COND_ITEM thru. They will be picked up + // by further logic. It may become desirable to add code here. + break; + } + default: { gwi.fatalParseError = true; @@ -3530,8 +3542,7 @@ ReturnedColumn* buildFunctionColumn(Item_func* ifp, gp_walk_info& gwi, bool& non if (ifp->field_type() == MYSQL_TYPE_DATETIME || ifp->field_type() == MYSQL_TYPE_DATETIME2 || ifp->field_type() == MYSQL_TYPE_TIMESTAMP || - ifp->field_type() == MYSQL_TYPE_TIMESTAMP2 || - funcName == "add_time") + ifp->field_type() == MYSQL_TYPE_TIMESTAMP2) { CalpontSystemCatalog::ColType ct; ct.colDataType = CalpontSystemCatalog::DATETIME; @@ -3545,6 +3556,13 @@ ReturnedColumn* buildFunctionColumn(Item_func* ifp, gp_walk_info& gwi, bool& non ct.colWidth = 4; fc->resultType(ct); } + else if (ifp->field_type() == MYSQL_TYPE_TIME) + { + CalpontSystemCatalog::ColType ct; + ct.colDataType = CalpontSystemCatalog::TIME; + ct.colWidth = 8; + fc->resultType(ct); + } #if 0 @@ -3653,124 +3671,109 @@ FunctionColumn* buildCaseFunction(Item_func* item, gp_walk_info& gwi, bool& nonS if (((Item_func_case*)item)->get_first_expr_num() == -1) funcName = "case_searched"; - if (gwi.clauseType == SELECT || gwi.clauseType == HAVING || gwi.clauseType == GROUP_BY) // select clause + funcParms.reserve(item->argument_count()); + // so buildXXXcolumn function will not pop stack. + ClauseType realClauseType = gwi.clauseType; + gwi.clauseType = SELECT; + + // We ought to be able to just build from the stack, and would + // be able to if there were any way to know which stack had the + // next case item. Unfortunately, parameters may have been pushed + // onto the ptWorkStack or rcWorkStack or neither, depending on type + // and position. We can't tell which at this point, so we + // rebuild the item from the arguments directly and then try to + // figure what to pop, if anything, in order to sync the stacks. + // + // MCOL-1341 - With MariaDB 10.2.14 onwards CASE is now in the order: + // [case,]when1,when2,...,then1,then2,...[,else] + // See server commit bf1ca14ff3f3faa9f7a018097b25aa0f66d068cd for more + // information. + int32_t arg_offset = 0; + + if ((item->argument_count() - 1) % 2) { - // the first argument - if (funcName == "case_searched") + arg_offset = (item->argument_count() - 1) / 2; + } + else + { + arg_offset = item->argument_count() / 2; + } + + for (int32_t i = item->argument_count() - 1; i >= 0; i--) + { + // For case_searched, we know the items for the WHEN clause will + // not be ReturnedColumns. We do this separately just to save + // some cpu cycles trying to build a ReturnedColumn as below. + // Every even numbered arg is a WHEN. In between are the THEN. + // An odd number of args indicates an ELSE residing in the last spot. + if (funcName == "case_searched" && + (i < arg_offset)) { - for (uint32_t i = 0; i < item->argument_count(); i++) + sptp.reset(buildParseTree((Item_func*)(item->arguments()[i]), gwi, nonSupport)); + + if (!gwi.ptWorkStack.empty() && *gwi.ptWorkStack.top()->data() == sptp->data()) { - if (i % 2 == 0 && i != 1 && i != item->argument_count() - 1) - { - sptp.reset(buildParseTree((Item_func*)(item->arguments()[i]), gwi, nonSupport)); - funcParms.push_back(sptp); - } - else - { - ReturnedColumn* parm = buildReturnedColumn(item->arguments()[i], gwi, nonSupport); - - if (parm) - { - sptp.reset(new ParseTree(parm)); - } - else - { - sptp.reset(buildParseTree((Item_func*)(item->arguments()[i]), gwi, nonSupport)); - } - - funcParms.push_back(sptp); - } + gwi.ptWorkStack.pop(); } } else { - for (uint32_t i = 0; i < item->argument_count(); i++) + // First try building a ReturnedColumn. It may or may not succeed + // depending on the types involved. There's also little correlation + // between buildReturnedColumn and the existance of the item on + // rwWorkStack or ptWorkStack. + // For example, simple predicates, such as 1=1 or 1=0, land in the + // ptWorkStack but other stuff might land in the rwWorkStack + ReturnedColumn* parm = buildReturnedColumn(item->arguments()[i], gwi, nonSupport); + + if (parm) { - ReturnedColumn* parm = buildReturnedColumn(item->arguments()[i], gwi, nonSupport); + sptp.reset(new ParseTree(parm)); - if (parm) + // We need to pop whichever stack is holding it, if any. + if ((!gwi.rcWorkStack.empty()) && + *gwi.rcWorkStack.top() == parm) { - sptp.reset(new ParseTree(parm)); + gwi.rcWorkStack.pop(); } - else + else if (!gwi.ptWorkStack.empty()) { - sptp.reset(buildParseTree((Item_func*)(item->arguments()[i]), gwi, nonSupport)); - } + ReturnedColumn* ptrc = dynamic_cast(gwi.ptWorkStack.top()->data()); - funcParms.push_back(sptp); - } - } - } - else // where clause - { - // so buildXXXcolumn function will not pop stack. - gwi.clauseType = SELECT; - - if (funcName == "case_searched") - { - for (uint32_t i = 0; i < item->argument_count(); i++) - { - if (i % 2 == 0 && i != item->argument_count() - 1) - { - // build item from arguments to avoid parm sequence complexity - sptp.reset(buildParseTree((Item_func*)(item->arguments()[i]), gwi, nonSupport)); - funcParms.push_back(sptp); - - if (!gwi.ptWorkStack.empty()) + if (ptrc && *ptrc == *parm) gwi.ptWorkStack.pop(); } - else - { - ReturnedColumn* parm = buildReturnedColumn(item->arguments()[i], gwi, nonSupport); - - if (parm) - { - sptp.reset(new ParseTree(parm)); - - if (!gwi.rcWorkStack.empty()) - gwi.rcWorkStack.pop(); - } - else - { - sptp.reset(buildParseTree((Item_func*)(item->arguments()[i]), gwi, nonSupport)); - - if (!gwi.ptWorkStack.empty()) - gwi.ptWorkStack.pop(); - } - - funcParms.push_back(sptp); - } } - } - else // simple_case - { - for (uint32_t i = 0; i < item->argument_count(); i++) + else { - ReturnedColumn* parm = buildReturnedColumn(item->arguments()[i], gwi, nonSupport); + sptp.reset(buildParseTree((Item_func*)(item->arguments()[i]), gwi, nonSupport)); - if (parm) + // We need to pop whichever stack is holding it, if any. + if ((!gwi.ptWorkStack.empty()) && + *gwi.ptWorkStack.top()->data() == sptp->data()) { - sptp.reset(new ParseTree(parm)); + gwi.ptWorkStack.pop(); + } + else if (!gwi.rcWorkStack.empty()) + { + // Probably won't happen, but it might have been on the + // rcWorkStack all along. + ReturnedColumn* ptrc = dynamic_cast(sptp->data()); - if (!gwi.rcWorkStack.empty()) + if (ptrc && *ptrc == *gwi.rcWorkStack.top()) + { gwi.rcWorkStack.pop(); + } } - else - { - sptp.reset(buildParseTree((Item_func*)(item->arguments()[i]), gwi, nonSupport)); - - if (!gwi.ptWorkStack.empty()) - gwi.ptWorkStack.pop(); - } - - funcParms.push_back(sptp); } } - // recover clause type - gwi.clauseType = WHERE; + funcParms.insert(funcParms.begin(), sptp); } + // recover clause type + gwi.clauseType = realClauseType; + if (gwi.fatalParseError) { setError(gwi.thd, ER_CHECK_NOT_IMPLEMENTED, gwi.parseErrorText, gwi); @@ -4842,9 +4845,9 @@ void gp_walk(const Item* item, void* arg) // bug 3137. If filter constant like 1=0, put it to ptWorkStack // MariaDB bug 750. Breaks if compare is an argument to a function. - if ((int32_t)gwip->rcWorkStack.size() <= (gwip->rcBookMarkStack.empty() ? 0 : gwip->rcBookMarkStack.top()) - && isPredicateFunction(ifp, gwip)) -// if (isPredicateFunction(ifp, gwip)) +// if ((int32_t)gwip->rcWorkStack.size() <= (gwip->rcBookMarkStack.empty() ? 0 : gwip->rcBookMarkStack.top()) +// && isPredicateFunction(ifp, gwip)) + if (isPredicateFunction(ifp, gwip)) gwip->ptWorkStack.push(new ParseTree(cc)); else gwip->rcWorkStack.push(cc); @@ -6271,7 +6274,9 @@ int getSelectPlan(gp_walk_info& gwi, SELECT_LEX& select_lex, SCSEP& csep, bool i // @bug 1706 String funcStr; ifp->print(&funcStr, QT_INFINIDB); - gwi.selectCols.push_back(string(funcStr.c_ptr()) + " `" + escapeBackTick(ifp->name) + "`"); + string valStr; + valStr.assign(funcStr.ptr(), funcStr.length()); + gwi.selectCols.push_back(valStr + " `" + escapeBackTick(ifp->name) + "`"); // clear the error set by buildFunctionColumn gwi.fatalParseError = false; gwi.parseErrorText = ""; diff --git a/dbcon/mysql/ha_calpont_impl.cpp b/dbcon/mysql/ha_calpont_impl.cpp index f29c24383..12fa74fa5 100644 --- a/dbcon/mysql/ha_calpont_impl.cpp +++ b/dbcon/mysql/ha_calpont_impl.cpp @@ -590,8 +590,21 @@ int fetchNextRow(uchar* buf, cal_table_info& ti, cal_connection_info* ci, bool h * based on the result set. */ /* MCOL-683: UTF-8 datetime no msecs is 57, this sometimes happens! */ - if (((*f)->field_length > 19) && ((*f)->field_length != 57)) - (*f)->field_length = strlen(tmp); +// if (((*f)->field_length > 19) && ((*f)->field_length != 57)) +// (*f)->field_length = strlen(tmp); + + Field_varstring* f2 = (Field_varstring*)*f; + f2->store(tmp, strlen(tmp), f2->charset()); + break; + } + + case CalpontSystemCatalog::TIME: + { + if ((*f)->null_ptr) + *(*f)->null_ptr &= ~(*f)->null_bit; + + intColVal = row.getUintField<8>(s); + DataConvert::timeToString(intColVal, tmp, 255); Field_varstring* f2 = (Field_varstring*)*f; f2->store(tmp, strlen(tmp), f2->charset()); diff --git a/dbcon/mysql/ha_calpont_partition.cpp b/dbcon/mysql/ha_calpont_partition.cpp index 84ea65d82..99940262b 100644 --- a/dbcon/mysql/ha_calpont_partition.cpp +++ b/dbcon/mysql/ha_calpont_partition.cpp @@ -125,6 +125,9 @@ string name(CalpontSystemCatalog::ColType& ct) case CalpontSystemCatalog::DATETIME: return "DATETIME"; + case CalpontSystemCatalog::TIME: + return "TIME"; + case CalpontSystemCatalog::DECIMAL: return "DECIMAL"; @@ -201,6 +204,7 @@ bool CP_type(CalpontSystemCatalog::ColType& ct) ct.colDataType == CalpontSystemCatalog::BIGINT || ct.colDataType == CalpontSystemCatalog::DATE || ct.colDataType == CalpontSystemCatalog::DATETIME || + ct.colDataType == CalpontSystemCatalog::TIME || ct.colDataType == CalpontSystemCatalog::DECIMAL || ct.colDataType == CalpontSystemCatalog::UTINYINT || ct.colDataType == CalpontSystemCatalog::USMALLINT || @@ -261,6 +265,9 @@ const string format(int64_t v, CalpontSystemCatalog::ColType& ct) oss << DataConvert::datetimeToString(v); break; + case CalpontSystemCatalog::TIME: + oss << DataConvert::timeToString(v); + case CalpontSystemCatalog::CHAR: case CalpontSystemCatalog::VARCHAR: { @@ -387,6 +394,10 @@ const int64_t IDB_format(char* str, CalpontSystemCatalog::ColType& ct, uint8_t& v = boost::any_cast(anyVal); break; + case CalpontSystemCatalog::TIME: + v = boost::any_cast(anyVal); + break; + case CalpontSystemCatalog::DECIMAL: case CalpontSystemCatalog::UDECIMAL: if (ct.colWidth == execplan::CalpontSystemCatalog::ONE_BYTE) diff --git a/dbcon/mysql/ha_window_function.cpp b/dbcon/mysql/ha_window_function.cpp index a861cbde1..1635c815a 100644 --- a/dbcon/mysql/ha_window_function.cpp +++ b/dbcon/mysql/ha_window_function.cpp @@ -602,6 +602,7 @@ ReturnedColumn* buildWindowFunctionColumn(Item* item, gp_walk_info& gwi, bool& n case CalpontSystemCatalog::DATE: case CalpontSystemCatalog::DATETIME: + case CalpontSystemCatalog::TIME: if (!frm.fIsRange) boundTypeErr = true; else if (dynamic_cast(frm.fStart.fVal.get()) == NULL) @@ -653,6 +654,7 @@ ReturnedColumn* buildWindowFunctionColumn(Item* item, gp_walk_info& gwi, bool& n case CalpontSystemCatalog::DATE: case CalpontSystemCatalog::DATETIME: + case CalpontSystemCatalog::TIME: if (!frm.fIsRange) boundTypeErr = true; else if (dynamic_cast(frm.fEnd.fVal.get()) == NULL) diff --git a/oam/install_scripts/columnstore b/oam/install_scripts/columnstore index f4fc48157..038ad0c43 100644 --- a/oam/install_scripts/columnstore +++ b/oam/install_scripts/columnstore @@ -36,7 +36,7 @@ InstallDir=$COLUMNSTORE_INSTALL_DIR if [ $InstallDir != "/usr/local/mariadb/columnstore" ]; then export PATH=$InstallDir/bin:$InstallDir/mysql/bin:/bin:/usr/bin - export LD_LIBRARY_PATH=$LD_LIBRARY_PATH:$InstallDir/lib:$InstallDir/mysql/lib/mysql + export LD_LIBRARY_PATH=$LD_LIBRARY_PATH:$InstallDir/lib:$InstallDir/mysql/lib fi #hadoop diff --git a/oam/install_scripts/module_installer.sh b/oam/install_scripts/module_installer.sh index 10cefaf5f..6ca39b70f 100755 --- a/oam/install_scripts/module_installer.sh +++ b/oam/install_scripts/module_installer.sh @@ -47,7 +47,7 @@ shift $shiftcnt if [ $installdir != "/usr/local/mariadb/columnstore" ]; then export COLUMNSTORE_INSTALL_DIR=$installdir export PATH=$COLUMNSTORE_INSTALL_DIR/bin:$COLUMNSTORE_INSTALL_DIR/mysql/bin:/bin:/usr/bin - export LD_LIBRARY_PATH=$LD_LIBRARY_PATH:$COLUMNSTORE_INSTALL_DIR/lib:$COLUMNSTORE_INSTALL_DIR/mysql/lib/mysql + export LD_LIBRARY_PATH=$LD_LIBRARY_PATH:$COLUMNSTORE_INSTALL_DIR/lib:$COLUMNSTORE_INSTALL_DIR/mysql/lib else export COLUMNSTORE_INSTALL_DIR=$installdir fi diff --git a/oam/install_scripts/post-install b/oam/install_scripts/post-install index a295a088f..f7aeeb2ca 100755 --- a/oam/install_scripts/post-install +++ b/oam/install_scripts/post-install @@ -104,6 +104,9 @@ if [ $installdir != "/usr/local/mariadb/columnstore" ]; then fi if [ $user != "root" ]; then + sudo rm -f $profileFileEnv + sudo rm -f $profileFileAlias + sudo touch $profileFileEnv sudo chmod 666 $profileFileEnv egrep -qs 'MariaDB Columnstore Non-Root' ${profileFileEnv} @@ -112,7 +115,7 @@ if [ $user != "root" ]; then sudo echo " " >> ${profileFileEnv} sudo echo "# MariaDB Columnstore Non-Root Environment Variables" >> ${profileFileEnv} sudo echo "export COLUMNSTORE_INSTALL_DIR=$COLUMNSTORE_INSTALL_DIR" >> ${profileFileEnv} - sudo echo "export LD_LIBRARY_PATH=$LD_LIBRARY_PATH:$COLUMNSTORE_INSTALL_DIR/lib:$COLUMNSTORE_INSTALL_DIR/mysql/lib/mysql" >> ${profileFileEnv} + sudo echo "export LD_LIBRARY_PATH=$LD_LIBRARY_PATH:$COLUMNSTORE_INSTALL_DIR/lib:$COLUMNSTORE_INSTALL_DIR/mysql/lib" >> ${profileFileEnv} . ${profileFileEnv} fi @@ -183,13 +186,13 @@ else RCFILE=/etc/rc.local fi -touch $RCFILE - if [ $user = "root" ]; then + touch $RCFILE chmod +x $RCFILE else + $SUDO touch $RCFILE $SUDO chmod 777 $RCFILE - printf '%s\n' '#!/bin/bash' "#" | $SUDO tee -a $RCFILEl > /dev/null 2>&1 + $SUDO printf '%s\n' '#!/bin/bash' "#" | $SUDO tee -a $RCFILE > /dev/null 2>&1 if [ -n "$systemctl" ]; then $SUDO systemctl start rc-local >/dev/null 2>&1 diff --git a/oam/install_scripts/post-mysql-install b/oam/install_scripts/post-mysql-install index 08c38b84e..24711b048 100755 --- a/oam/install_scripts/post-mysql-install +++ b/oam/install_scripts/post-mysql-install @@ -64,7 +64,7 @@ USER=`whoami 2>/dev/null` if [ $USER != "root" ]; then sudo ldconfig >/dev/null 2>&1 export COLUMNSTORE_INSTALL_DIR=$installdir - export LD_LIBRARY_PATH=$LD_LIBRARY_PATH:$COLUMNSTORE_INSTALL_DIR/lib:$COLUMNSTORE_INSTALL_DIR/mysql/lib/mysql + export LD_LIBRARY_PATH=$LD_LIBRARY_PATH:$COLUMNSTORE_INSTALL_DIR/lib:$COLUMNSTORE_INSTALL_DIR/mysql/lib else ldconfig fi diff --git a/oam/oamcpp/liboamcpp.cpp b/oam/oamcpp/liboamcpp.cpp index 91d82eeb1..92b2db3b7 100644 --- a/oam/oamcpp/liboamcpp.cpp +++ b/oam/oamcpp/liboamcpp.cpp @@ -9486,6 +9486,105 @@ std::string Oam::updateFstab(std::string device, std::string dbrootID) return entry; } +/****************************************************************************************** +* @brief waitForActive +* +* purpose: wait for system to be active +* +******************************************************************************************/ +void Oam::waitForActive() +{ + SystemStatus systemstatus; + SystemProcessStatus systemprocessstatus; + bool bfirst = true; + int dot = 0; + + for (int i = 0 ; i < 120 ; i ++, dot ++) + { + sleep (3); + + try + { + getSystemStatus(systemstatus); + + if (systemstatus.SystemOpState == ACTIVE) + { + BRM::DBRM dbrm; + + try + { + int rc = dbrm.getSystemQueryReady(); + + if (rc == -1 ) + { + writeLog("waitForActive: getSystemQueryReady error return: startSystem failed", LOG_TYPE_ERROR); + exceptionControl("waitForActive", API_FAILURE); + } + + if ( rc != 0 ) + return; + + writeLog("waitForActive: getSystemQueryReady not ready", LOG_TYPE_DEBUG); + } + catch (...) + {} + } + + if (systemstatus.SystemOpState == FAILED) + { + exceptionControl("waitForActive", API_FAILURE); + } + + if (systemstatus.SystemOpState == MAN_OFFLINE) + { + exceptionControl("waitForActive", API_FAILURE); + } + + if (dot >= 3 ) + { + cout << "." << flush; + dot = 0; + } + + // Check DMLProc for a switch to BUSY_INIT. + // In such a case, we need to print a message that rollbacks + // are occurring and will take some time. + if (bfirst) // Once we've printed our message, no need to waste cpu looking + { + getProcessStatus(systemprocessstatus); + + for (unsigned int i = 0 ; i < systemprocessstatus.processstatus.size(); i++) + { + if (systemprocessstatus.processstatus[i].ProcessName == "DMLProc") + { + if (systemprocessstatus.processstatus[i].ProcessOpState == oam::ROLLBACK_INIT) + { + cout << endl << endl << " System Not Ready, DMLProc is checking/processing rollback of abandoned transactions. Processing could take some time, please wait..." << flush; + bfirst = false; + } + + // At this point, we've found our DMLProc, so there's no need to spin the for loop + // any further. + break; + } + } + } + } + catch (...) + { + // At some point, we need to give up, ProcMon just isn't going to respond. + if (i > 60) // 3 minutes + { + cout << endl << endl << "TIMEOUT: ProcMon not responding to getSystemStatus"; + break; + } + } + } + + exceptionControl("waitForActive", API_FAILURE); +} + + /*************************************************************************** * PRIVATE FUNCTIONS ***************************************************************************/ diff --git a/oam/oamcpp/liboamcpp.h b/oam/oamcpp/liboamcpp.h index b45cd7307..66482bc25 100644 --- a/oam/oamcpp/liboamcpp.h +++ b/oam/oamcpp/liboamcpp.h @@ -2481,6 +2481,10 @@ public: bool checkSystemRunning(); + /** @brief wait for system to be active + */ + EXPORT void waitForActive(); + private: int sendMsgToProcMgr3(messageqcpp::ByteStream::byte requestType, alarmmanager::AlarmList& alarmlist, const std::string date); diff --git a/oamapps/calpontSupport/findStranded.sh b/oamapps/calpontSupport/findStranded.sh index 6d6ca28a3..1263aace9 100755 --- a/oamapps/calpontSupport/findStranded.sh +++ b/oamapps/calpontSupport/findStranded.sh @@ -18,7 +18,7 @@ export COLUMNSTORE_INSTALL_DIR=$COLUMNSTORE_INSTALL_DIR if [ $COLUMNSTORE_INSTALL_DIR != "/usr/local/mariadb/columnstore" ]; then export PATH=$COLUMNSTORE_INSTALL_DIR/bin:$COLUMNSTORE_INSTALL_DIR/mysql/bin:/bin:/usr/bin - export LD_LIBRARY_PATH=$COLUMNSTORE_INSTALL_DIR/lib:$COLUMNSTORE_INSTALL_DIR/mysql/lib/mysql + export LD_LIBRARY_PATH=$COLUMNSTORE_INSTALL_DIR/lib:$COLUMNSTORE_INSTALL_DIR/mysql/lib fi cd $COLUMNSTORE_INSTALL_DIR diff --git a/oamapps/calpontSupport/sqlLogs.sh b/oamapps/calpontSupport/sqlLogs.sh index a1a9a403c..818ab546c 100755 --- a/oamapps/calpontSupport/sqlLogs.sh +++ b/oamapps/calpontSupport/sqlLogs.sh @@ -14,7 +14,7 @@ export COLUMNSTORE_INSTALL_DIR=$COLUMNSTORE_INSTALL_DIR if [ $COLUMNSTORE_INSTALL_DIR != "/usr/local/mariadb/columnstore" ]; then export PATH=$COLUMNSTORE_INSTALL_DIR/bin:$COLUMNSTORE_INSTALL_DIR/mysql/bin:/bin:/usr/bin - export LD_LIBRARY_PATH=$COLUMNSTORE_INSTALL_DIR/lib:$COLUMNSTORE_INSTALL_DIR/mysql/lib/mysql + export LD_LIBRARY_PATH=$COLUMNSTORE_INSTALL_DIR/lib:$COLUMNSTORE_INSTALL_DIR/mysql/lib fi diff --git a/oamapps/columnstoreSupport/findStranded.sh b/oamapps/columnstoreSupport/findStranded.sh index 8577e06e3..12e160172 100755 --- a/oamapps/columnstoreSupport/findStranded.sh +++ b/oamapps/columnstoreSupport/findStranded.sh @@ -18,7 +18,7 @@ export COLUMNSTORE_INSTALL_DIR=$COLUMNSTORE_INSTALL_DIR if [ $COLUMNSTORE_INSTALL_DIR != "/usr/local/mariadb/columnstore" ]; then export PATH=$COLUMNSTORE_INSTALL_DIR/bin:$COLUMNSTORE_INSTALL_DIR/mysql/bin:/bin:/usr/bin - export LD_LIBRARY_PATH=$COLUMNSTORE_INSTALL_DIR/lib:$COLUMNSTORE_INSTALL_DIR/mysql/lib/mysql + export LD_LIBRARY_PATH=$COLUMNSTORE_INSTALL_DIR/lib:$COLUMNSTORE_INSTALL_DIR/mysql/lib fi cd $COLUMNSTORE_INSTALL_DIR diff --git a/oamapps/columnstoreSupport/sqlLogs.sh b/oamapps/columnstoreSupport/sqlLogs.sh index 385b2d912..564973e7e 100755 --- a/oamapps/columnstoreSupport/sqlLogs.sh +++ b/oamapps/columnstoreSupport/sqlLogs.sh @@ -14,7 +14,7 @@ export COLUMNSTORE_INSTALL_DIR=$COLUMNSTORE_INSTALL_DIR if [ $COLUMNSTORE_INSTALL_DIR != "/usr/local/mariadb/columnstore" ]; then export PATH=$COLUMNSTORE_INSTALL_DIR/bin:$COLUMNSTORE_INSTALL_DIR/mysql/bin:/bin:/usr/bin - export LD_LIBRARY_PATH=$COLUMNSTORE_INSTALL_DIR/lib:$COLUMNSTORE_INSTALL_DIR/mysql/lib/mysql + export LD_LIBRARY_PATH=$COLUMNSTORE_INSTALL_DIR/lib:$COLUMNSTORE_INSTALL_DIR/mysql/lib fi diff --git a/oamapps/mcsadmin/mcsadmin.cpp b/oamapps/mcsadmin/mcsadmin.cpp index e49483c2b..c1f42f52c 100644 --- a/oamapps/mcsadmin/mcsadmin.cpp +++ b/oamapps/mcsadmin/mcsadmin.cpp @@ -77,69 +77,14 @@ bool SendToWES(Oam& oam, ByteStream bs); bool waitForActive() { Oam oam; - SystemStatus systemstatus; - SystemProcessStatus systemprocessstatus; - bool bfirst = true; - for (int i = 0 ; i < 1200 ; i ++) + try { - sleep (3); - - try - { - oam.getSystemStatus(systemstatus); - - if (systemstatus.SystemOpState == ACTIVE) - { - return true; - } - - if (systemstatus.SystemOpState == FAILED) - { - return false; - } - - if (systemstatus.SystemOpState == MAN_OFFLINE) - { - return false; - } - - cout << "." << flush; - - // Check DMLProc for a switch to BUSY_INIT. - // In such a case, we need to print a message that rollbacks - // are occurring and will take some time. - if (bfirst) // Once we've printed our message, no need to waste cpu looking - { - oam.getProcessStatus(systemprocessstatus); - - for (unsigned int i = 0 ; i < systemprocessstatus.processstatus.size(); i++) - { - if (systemprocessstatus.processstatus[i].ProcessName == "DMLProc") - { - if (systemprocessstatus.processstatus[i].ProcessOpState == oam::ROLLBACK_INIT) - { - cout << endl << endl << " System Not Ready, DMLProc is checking/processing rollback of abandoned transactions. Processing could take some time, please wait..." << flush; - bfirst = false; - } - - // At this point, we've found our DMLProc, so there's no need to spin the for loop - // any further. - break; - } - } - } - } - catch (...) - { - // At some point, we need to give up, ProcMgr just isn't going to respond. - if (i > 60) // 3 minutes - { - cout << "ProcMgr not responding while waiting for system to start"; - break; - } - } + oam.waitForActive(); + return true; } + catch (...) + {} return false; } diff --git a/oamapps/postConfigure/helpers.cpp b/oamapps/postConfigure/helpers.cpp index aa586afc2..f7d418144 100644 --- a/oamapps/postConfigure/helpers.cpp +++ b/oamapps/postConfigure/helpers.cpp @@ -77,31 +77,22 @@ void callFree(const char* ) } - bool waitForActive() { Oam oam; - const string cmd = installDir + "/bin/mcsadmin getsystemstatus > /tmp/wait.log"; - system(cmd.c_str()); - - for ( int i = 0 ; i < 120 ; i ++ ) + try { - if (oam.checkLogStatus("/tmp/wait.log", "System ACTIVE") ) - return true; - - if ( oam.checkLogStatus("/tmp/wait.log", "System FAILED") ) - return false; - - cout << "."; - cout.flush(); - sleep (10); - system(cmd.c_str()); + oam.waitForActive(); + return true; } + catch (...) + {} return false; } + void dbrmDirCheck() { @@ -544,7 +535,6 @@ int sendReplicationRequest(int IserverTypeInstall, std::string password, bool pm } else { - cout << endl << "ERROR: Module not Active, replication not done on " << (*pt).DeviceName << endl; pt++; } } diff --git a/primitives/linux-port/column.cpp b/primitives/linux-port/column.cpp index 4b5f4a28d..f31273b1c 100644 --- a/primitives/linux-port/column.cpp +++ b/primitives/linux-port/column.cpp @@ -39,6 +39,7 @@ using namespace boost; #include "we_type.h" #include "stats.h" #include "primproc.h" +#include "dataconvert.h" using namespace logging; using namespace dbbc; using namespace primitives; @@ -289,6 +290,7 @@ inline bool isEmptyVal<8>(uint8_t type, const uint8_t* ival) case CalpontSystemCatalog::VARCHAR: case CalpontSystemCatalog::DATE: case CalpontSystemCatalog::DATETIME: + case CalpontSystemCatalog::TIME: case CalpontSystemCatalog::VARBINARY: case CalpontSystemCatalog::BLOB: case CalpontSystemCatalog::TEXT: @@ -321,6 +323,7 @@ inline bool isEmptyVal<4>(uint8_t type, const uint8_t* ival) case CalpontSystemCatalog::TEXT: case CalpontSystemCatalog::DATE: case CalpontSystemCatalog::DATETIME: + case CalpontSystemCatalog::TIME: return (joblist::CHAR4EMPTYROW == *val); case CalpontSystemCatalog::UINT: @@ -346,6 +349,7 @@ inline bool isEmptyVal<2>(uint8_t type, const uint8_t* ival) case CalpontSystemCatalog::TEXT: case CalpontSystemCatalog::DATE: case CalpontSystemCatalog::DATETIME: + case CalpontSystemCatalog::TIME: return (joblist::CHAR2EMPTYROW == *val); case CalpontSystemCatalog::USMALLINT: @@ -371,6 +375,7 @@ inline bool isEmptyVal<1>(uint8_t type, const uint8_t* ival) case CalpontSystemCatalog::TEXT: case CalpontSystemCatalog::DATE: case CalpontSystemCatalog::DATETIME: + case CalpontSystemCatalog::TIME: return (*val == joblist::CHAR1EMPTYROW); case CalpontSystemCatalog::UTINYINT: @@ -401,6 +406,7 @@ inline bool isNullVal<8>(uint8_t type, const uint8_t* ival) case CalpontSystemCatalog::VARCHAR: case CalpontSystemCatalog::DATE: case CalpontSystemCatalog::DATETIME: + case CalpontSystemCatalog::TIME: case CalpontSystemCatalog::VARBINARY: case CalpontSystemCatalog::BLOB: case CalpontSystemCatalog::TEXT: @@ -437,6 +443,7 @@ inline bool isNullVal<4>(uint8_t type, const uint8_t* ival) case CalpontSystemCatalog::DATE: case CalpontSystemCatalog::DATETIME: + case CalpontSystemCatalog::TIME: return (joblist::DATENULL == *val); case CalpontSystemCatalog::UINT: @@ -462,6 +469,7 @@ inline bool isNullVal<2>(uint8_t type, const uint8_t* ival) case CalpontSystemCatalog::TEXT: case CalpontSystemCatalog::DATE: case CalpontSystemCatalog::DATETIME: + case CalpontSystemCatalog::TIME: return (joblist::CHAR2NULL == *val); case CalpontSystemCatalog::USMALLINT: @@ -487,6 +495,7 @@ inline bool isNullVal<1>(uint8_t type, const uint8_t* ival) case CalpontSystemCatalog::TEXT: case CalpontSystemCatalog::DATE: case CalpontSystemCatalog::DATETIME: + case CalpontSystemCatalog::TIME: return (*val == joblist::CHAR1NULL); case CalpontSystemCatalog::UTINYINT: @@ -547,6 +556,7 @@ inline bool isMinMaxValid(const NewColRequestHeader* in) case CalpontSystemCatalog::DATE: case CalpontSystemCatalog::BIGINT: case CalpontSystemCatalog::DATETIME: + case CalpontSystemCatalog::TIME: case CalpontSystemCatalog::UTINYINT: case CalpontSystemCatalog::USMALLINT: case CalpontSystemCatalog::UINT: @@ -603,7 +613,13 @@ inline bool colCompare(int64_t val1, int64_t val2, uint8_t COP, uint8_t rf, int type == CalpontSystemCatalog::TEXT) && !isNull ) { if (!regex.used && !rf) + { + // MCOL-1246 Trim trailing whitespace for matching, but not for + // regex + dataconvert::DataConvert::trimWhitespace(val1); + dataconvert::DataConvert::trimWhitespace(val2); return colCompare_(order_swap(val1), order_swap(val2), COP); + } else return colStrCompare_(order_swap(val1), order_swap(val2), COP, rf, ®ex); } diff --git a/primitives/linux-port/dictionary.cpp b/primitives/linux-port/dictionary.cpp index ebf439190..2860ab571 100644 --- a/primitives/linux-port/dictionary.cpp +++ b/primitives/linux-port/dictionary.cpp @@ -21,6 +21,7 @@ #include #include +#include #include using namespace std; @@ -181,7 +182,10 @@ void PrimitiveProcessor::p_TokenByScan(const TokenByScanRequestHeader* h, if (eqFilter) { - bool gotIt = eqFilter->find(string(sig, siglen)) != eqFilter->end(); + // MCOL-1246 Trim whitespace before match + string strData(sig, siglen); + boost::trim_right_if(strData, boost::is_any_of(" ")); + bool gotIt = eqFilter->find(strData) != eqFilter->end(); if ((h->COP1 == COMPARE_EQ && gotIt) || (h->COP1 == COMPARE_NE && !gotIt)) @@ -902,8 +906,10 @@ void PrimitiveProcessor::p_Dictionary(const DictInput* in, vector* out, if (eqFilter) { - bool gotIt = (eqFilter->find(string((char*) sigptr.data, sigptr.len)) - != eqFilter->end()); + // MCOL-1246 Trim whitespace before match + string strData((char*)sigptr.data, sigptr.len); + boost::trim_right_if(strData, boost::is_any_of(" ")); + bool gotIt = eqFilter->find(strData) != eqFilter->end(); if ((gotIt && eqOp == COMPARE_EQ) || (!gotIt && eqOp == COMPARE_NE)) goto store; diff --git a/primitives/primproc/columncommand.cpp b/primitives/primproc/columncommand.cpp index 19e5e9a1f..ed4b26a44 100644 --- a/primitives/primproc/columncommand.cpp +++ b/primitives/primproc/columncommand.cpp @@ -980,6 +980,7 @@ const uint64_t ColumnCommand::getEmptyRowValue( const execplan::CalpontSystemCat case execplan::CalpontSystemCatalog::VARCHAR : case execplan::CalpontSystemCatalog::DATE : case execplan::CalpontSystemCatalog::DATETIME : + case execplan::CalpontSystemCatalog::TIME : case execplan::CalpontSystemCatalog::VARBINARY : case execplan::CalpontSystemCatalog::BLOB : case execplan::CalpontSystemCatalog::TEXT : diff --git a/procmgr/main.cpp b/procmgr/main.cpp index e24b15c86..e12b96f2a 100644 --- a/procmgr/main.cpp +++ b/procmgr/main.cpp @@ -594,7 +594,10 @@ static void alarmMessageThread(Configuration config) msg = fIos.read(); if (msg.length() <= 0) + { + fIos.close(); continue; + } //log.writeLog(__LINE__, "MSG RECEIVED: Process Alarm Message"); @@ -626,16 +629,20 @@ static void alarmMessageThread(Configuration config) ALARMManager aManager; aManager.processAlarmReport(calAlarm); + + fIos.close(); } catch (exception& ex) { string error = ex.what(); log.writeLog(__LINE__, "EXCEPTION ERROR on read for ProcMgr_Alarm:" + error, LOG_TYPE_ERROR); + fIos.close(); continue; } catch (...) { log.writeLog(__LINE__, "EXCEPTION ERROR on read for ProcMgr_Alarm: Caught unknown exception!", LOG_TYPE_ERROR); + fIos.close(); continue; } } @@ -1961,7 +1968,7 @@ void pingDeviceThread() DeviceNetworkConfig devicenetworkconfig; devicenetworkconfig.DeviceName = moduleName; devicenetworklist.push_back(devicenetworkconfig); - processManager.setMySQLReplication(devicenetworklist, oam::UnassignedName, false, true); + processManager.setMySQLReplication(devicenetworklist, oam::UnassignedName, true); } } diff --git a/procmgr/processmanager.cpp b/procmgr/processmanager.cpp index 6a87495ba..2bc12c330 100644 --- a/procmgr/processmanager.cpp +++ b/procmgr/processmanager.cpp @@ -2817,7 +2817,7 @@ void processMSG(messageqcpp::IOSocket* cfIos) // target = root password oam::DeviceNetworkList devicenetworklist; - status = processManager.setMySQLReplication(devicenetworklist, oam::UnassignedName, false, true, target); + status = processManager.setMySQLReplication(devicenetworklist, oam::UnassignedName, true, target); log.writeLog(__LINE__, "Enable MySQL Replication status: " + oam.itoa(status) ); @@ -2841,7 +2841,7 @@ void processMSG(messageqcpp::IOSocket* cfIos) // target = root password oam::DeviceNetworkList devicenetworklist; - status = processManager.setMySQLReplication(devicenetworklist, oam::UnassignedName, false, false, target, false); + status = processManager.setMySQLReplication(devicenetworklist, oam::UnassignedName, false, target, false); log.writeLog(__LINE__, "Disable MySQL Replication status: " + oam.itoa(status) ); @@ -3726,12 +3726,12 @@ int ProcessManager::disableModule(string target, bool manualFlag) * purpose: recyle process, generally after some disable module is run * ******************************************************************************************/ -void ProcessManager::recycleProcess(string module) +void ProcessManager::recycleProcess(string module, bool enableModule) { Oam oam; ModuleConfig moduleconfig; - log.writeLog(__LINE__, "recycleProcess request after module was disabled: " + module, LOG_TYPE_DEBUG); + log.writeLog(__LINE__, "recycleProcess request after module status update: " + module, LOG_TYPE_DEBUG); string moduleType = module.substr(0, MAX_MODULE_TYPE_SIZE); @@ -3743,6 +3743,17 @@ void ProcessManager::recycleProcess(string module) } catch (...) {} + // restart DBRM Process and DMLProc and return if enable module is being done + if (enableModule) + { + //recycle DBRM processes in all cases + restartProcessType("DBRMControllerNode"); + restartProcessType("DBRMWorkerNode"); + + restartProcessType("DMLProc"); + return; + } + //recycle DBRM processes in all cases restartProcessType("DBRMControllerNode", module); restartProcessType("DBRMWorkerNode"); @@ -3827,6 +3838,9 @@ int ProcessManager::enableModule(string target, int state) if ( newStandbyModule == target) setStandbyModule(newStandbyModule); + //set recycle process + recycleProcess(target); + log.writeLog(__LINE__, "enableModule request for " + target + " completed", LOG_TYPE_DEBUG); return API_SUCCESS; @@ -3971,20 +3985,23 @@ int ProcessManager::startProcess(string moduleName, string processName, { Oam oam; - //skip if module is DISABLED - int opState; - bool degraded; - - try + if ( actionIndicator != oam::STATUS_UPDATE ) { - oam.getModuleStatus(moduleName, opState, degraded); - } - catch (...) - {} + //skip if module is DISABLED + int opState; + bool degraded; - //check if disabled - if (opState == oam::MAN_DISABLED || opState == oam::AUTO_DISABLED) - return API_SUCCESS; + try + { + oam.getModuleStatus(moduleName, opState, degraded); + } + catch (...) + {} + + //check if disabled + if (opState == oam::MAN_DISABLED || opState == oam::AUTO_DISABLED) + return API_SUCCESS; + } ByteStream msg; ByteStream::byte requestID = START; @@ -4942,9 +4959,9 @@ int ProcessManager::addModule(oam::DeviceNetworkList devicenetworklist, std::str } if ( packageType == "rpm") - calpontPackage = homedir + "/mariadb-columnstore*" + columnstore_version + "-" + columnstore_release + "*.rpm.tar.gz"; + calpontPackage = homedir + "/mariadb-columnstore*" + columnstore_version + "-" + columnstore_release + "*.rpm"; else if ( packageType == "deb") - calpontPackage = homedir + "/mariadb-columnstore*" + columnstore_version + "-" + columnstore_release + "*.deb.tar.gz"; + calpontPackage = homedir + "/mariadb-columnstore*" + columnstore_version + "-" + columnstore_release + "*.deb"; else calpontPackage = homedir + "/mariadb-columnstore*" + columnstore_version + "-" + columnstore_release + "*.bin.tar.gz"; @@ -5955,8 +5972,24 @@ int ProcessManager::addModule(oam::DeviceNetworkList devicenetworklist, std::str log.writeLog(__LINE__, "addModule - sleep 60 - give ProcMon time to CONFIGURE and restart", LOG_TYPE_DEBUG); sleep(60); + //start mysqld on the new modules so mysql replication can be setup + listPT = devicenetworklist.begin(); + + for ( ; listPT != devicenetworklist.end() ; listPT++) + { + processManager.startProcess((*listPT).DeviceName, "mysqld", oam::STATUS_UPDATE); + } + log.writeLog(__LINE__, "Setup MySQL Replication for new Modules being Added", LOG_TYPE_DEBUG); - processManager.setMySQLReplication(devicenetworklist, oam::UnassignedName, false, true, password ); + processManager.setMySQLReplication(devicenetworklist, oam::UnassignedName, true, password, true, true ); + + //stop mysqld + listPT = devicenetworklist.begin(); + + for ( ; listPT != devicenetworklist.end() ; listPT++) + { + processManager.stopProcess((*listPT).DeviceName, "mysqld", oam::FORCEFUL, true ); + } return API_SUCCESS; } @@ -9533,7 +9566,7 @@ int ProcessManager::switchParentOAMModule(std::string newActiveModuleName) //change master MySQL Replication setup log.writeLog(__LINE__, "Setup MySQL Replication for new Parent Module during switch-over", LOG_TYPE_DEBUG); oam::DeviceNetworkList devicenetworklist; - processManager.setMySQLReplication(devicenetworklist, newActiveModuleName, false, false, oam::UnassignedName); + processManager.setMySQLReplication(devicenetworklist, newActiveModuleName, false, oam::UnassignedName); } catch (exception& ex) @@ -10414,7 +10447,7 @@ int ProcessManager::OAMParentModuleChange() //change master MySQL Replication setup log.writeLog(__LINE__, "Setup this node as MySQL Replication Master", LOG_TYPE_DEBUG); oam::DeviceNetworkList devicenetworklist; - processManager.setMySQLReplication(devicenetworklist, config.moduleName(), true); + processManager.setMySQLReplication(devicenetworklist, config.moduleName()); } //set query system state not ready @@ -11130,7 +11163,7 @@ void ProcessManager::flushInodeCache() * * ******************************************************************************************/ -int ProcessManager::setMySQLReplication(oam::DeviceNetworkList devicenetworklist, std::string masterModule, bool failover, bool distributeDB, std::string password, bool enable) +int ProcessManager::setMySQLReplication(oam::DeviceNetworkList devicenetworklist, std::string masterModule, bool distributeDB, std::string password, bool enable, bool addModule) { Oam oam; @@ -11153,57 +11186,6 @@ int ProcessManager::setMySQLReplication(oam::DeviceNetworkList devicenetworklist return oam::API_SUCCESS; } - //also skip if single-server, multi-node seperate with 1 UM, multi-node combo with 1 PM - - /* string SingleServerInstall = "n"; - try { - oam.getSystemConfig("SingleServerInstall", SingleServerInstall); - } - catch(...) { - SingleServerInstall = "n"; - } - - //single server check - if ( SingleServerInstall == "y" ) - { - log.writeLog(__LINE__, "setMySQLReplication: Single-Server, exiting", LOG_TYPE_DEBUG); - return oam::API_SUCCESS; - } - - //combined system check - if ( config.ServerInstallType() == oam::INSTALL_COMBINE_DM_UM_PM && !failover ) { - try { - Config* sysConfig = Config::makeConfig(); - if ( sysConfig->getConfig("DBRM_Controller", "NumWorkers") == "1" ) { - log.writeLog(__LINE__, "setMySQLReplication: 1 configured module, exiting", LOG_TYPE_DEBUG); - return oam::API_SUCCESS; - } - } - catch(...) - { - log.writeLog(__LINE__, "setMySQLReplication: makeConfig exception, exiting", LOG_TYPE_DEBUG); - return oam::API_SUCCESS; - } - } - - //seperate system check - if ( ( config.ServerInstallType() != oam::INSTALL_COMBINE_DM_UM_PM ) && - (PMwithUM == "n" ) ) - { - ModuleTypeConfig moduletypeconfig; - try{ - oam.getSystemConfig("um", moduletypeconfig); - } - catch(...) - {} - - if ( moduletypeconfig.ModuleCount < 2 ) - { - log.writeLog(__LINE__, "setMySQLReplication: moduleCount = 1, exiting", LOG_TYPE_DEBUG); - return oam::API_SUCCESS; - } - } - */ log.writeLog(__LINE__, "Setup MySQL Replication", LOG_TYPE_DEBUG); //get master info @@ -11262,19 +11244,22 @@ int ProcessManager::setMySQLReplication(oam::DeviceNetworkList devicenetworklist if ( remoteModuleName == masterModule ) continue; - // skip disabled modules - int opState = oam::ACTIVE; - bool degraded; - - try + if ( !addModule ) { - oam.getModuleStatus(remoteModuleName, opState, degraded); - } - catch (...) - {} + // skip disabled modules + int opState = oam::ACTIVE; + bool degraded; - if (opState == oam::MAN_DISABLED || opState == oam::AUTO_DISABLED) - continue; + try + { + oam.getModuleStatus(remoteModuleName, opState, degraded); + } + catch (...) + {} + + if (opState == oam::MAN_DISABLED || opState == oam::AUTO_DISABLED) + continue; + } // don't do PMs unless PMwithUM flag is set if ( config.ServerInstallType() != oam::INSTALL_COMBINE_DM_UM_PM ) @@ -11362,19 +11347,22 @@ int ProcessManager::setMySQLReplication(oam::DeviceNetworkList devicenetworklist if ( remoteModuleName == masterModule ) continue; - // skip disabled modules - int opState = oam::ACTIVE; - bool degraded; - - try + if ( !addModule ) { - oam.getModuleStatus(remoteModuleName, opState, degraded); - } - catch (...) - {} + // skip disabled modules + int opState = oam::ACTIVE; + bool degraded; - if (opState == oam::MAN_DISABLED || opState == oam::AUTO_DISABLED) - continue; + try + { + oam.getModuleStatus(remoteModuleName, opState, degraded); + } + catch (...) + {} + + if (opState == oam::MAN_DISABLED || opState == oam::AUTO_DISABLED) + continue; + } // don't do PMs unless PMwithUM flag is set if ( config.ServerInstallType() != oam::INSTALL_COMBINE_DM_UM_PM ) @@ -11430,19 +11418,22 @@ int ProcessManager::setMySQLReplication(oam::DeviceNetworkList devicenetworklist if ( remoteModuleName == masterModule ) continue; - // skip disabled modules - int opState = oam::ACTIVE; - bool degraded; - - try + if ( !addModule ) { - oam.getModuleStatus(remoteModuleName, opState, degraded); - } - catch (...) - {} + // skip disabled modules + int opState = oam::ACTIVE; + bool degraded; - if (opState == oam::MAN_DISABLED || opState == oam::AUTO_DISABLED) - continue; + try + { + oam.getModuleStatus(remoteModuleName, opState, degraded); + } + catch (...) + {} + + if (opState == oam::MAN_DISABLED || opState == oam::AUTO_DISABLED) + continue; + } log.writeLog(__LINE__, "Setup Slave MySQL Replication on " + remoteModuleName, LOG_TYPE_DEBUG); diff --git a/procmgr/processmanager.h b/procmgr/processmanager.h index 61233ce28..fd2fe834e 100644 --- a/procmgr/processmanager.h +++ b/procmgr/processmanager.h @@ -304,7 +304,7 @@ public: /** *@brief recycle Processes */ - void recycleProcess(std::string module); + void recycleProcess(std::string module, bool enableModule = false); /** *@brief Enable a specified module @@ -547,7 +547,7 @@ public: /** @brief Set MySQL Replication */ - int setMySQLReplication(oam::DeviceNetworkList devicenetworklist, std::string masterModule = oam::UnassignedName, bool failover = false, bool distributeDB = false, std::string password = oam::UnassignedName, bool enable = true); + int setMySQLReplication(oam::DeviceNetworkList devicenetworklist, std::string masterModule = oam::UnassignedName, bool distributeDB = false, std::string password = oam::UnassignedName, bool enable = true, bool addModule = false); /** @brief Gluster Assign dbroot to a module */ diff --git a/procmon/processmonitor.cpp b/procmon/processmonitor.cpp index cda141e84..30b239ec1 100644 --- a/procmon/processmonitor.cpp +++ b/procmon/processmonitor.cpp @@ -484,6 +484,26 @@ void ProcessMonitor::processMessage(messageqcpp::ByteStream msg, messageqcpp::IO log.writeLog(__LINE__, "MSG RECEIVED: Stop process request on " + processName); int requestStatus = API_SUCCESS; + // check for mysql + if ( processName == "mysqld" ) + { + try + { + oam.actionMysqlCalpont(MYSQL_STOP); + } + catch (...) + {} + + ackMsg << (ByteStream::byte) ACK; + ackMsg << (ByteStream::byte) STOP; + ackMsg << (ByteStream::byte) API_SUCCESS; + mq.write(ackMsg); + + log.writeLog(__LINE__, "STOP: ACK back to ProcMgr, return status = " + oam.itoa((int) API_SUCCESS)); + + break; + } + processList::iterator listPtr; processList* aPtr = config.monitoredListPtr(); listPtr = aPtr->begin(); @@ -533,6 +553,26 @@ void ProcessMonitor::processMessage(messageqcpp::ByteStream msg, messageqcpp::IO msg >> manualFlag; log.writeLog(__LINE__, "MSG RECEIVED: Start process request on: " + processName); + // check for mysql + if ( processName == "mysqld" ) + { + try + { + oam.actionMysqlCalpont(MYSQL_START); + } + catch (...) + {} + + ackMsg << (ByteStream::byte) ACK; + ackMsg << (ByteStream::byte) START; + ackMsg << (ByteStream::byte) API_SUCCESS; + mq.write(ackMsg); + + log.writeLog(__LINE__, "START: ACK back to ProcMgr, return status = " + oam.itoa((int) API_SUCCESS)); + + break; + } + ProcessConfig processconfig; ProcessStatus processstatus; @@ -645,7 +685,7 @@ void ProcessMonitor::processMessage(messageqcpp::ByteStream msg, messageqcpp::IO int requestStatus = API_SUCCESS; // check for mysql restart - if ( processName == "mysql" ) + if ( processName == "mysqld" ) { try { @@ -5350,12 +5390,11 @@ int ProcessMonitor::runMasterRep(std::string& masterLogFile, std::string& master //skip if module is not ACTIVE - int opState = oam::ACTIVE; - bool degraded; - oam.getModuleStatus(moduleName, opState, degraded); - - if (opState != oam::ACTIVE) - continue; +// int opState = oam::ACTIVE; +// bool degraded; +// oam.getModuleStatus(moduleName, opState, degraded); +// if (opState != oam::ACTIVE) +// continue; bool passwordError = false; diff --git a/tools/pingproc/pingproc.cpp b/tools/pingproc/pingproc.cpp index cb1a7ab11..b40eec905 100644 --- a/tools/pingproc/pingproc.cpp +++ b/tools/pingproc/pingproc.cpp @@ -308,6 +308,7 @@ bool OidOperation::isIntegralDataType() DataType() == CalpontSystemCatalog::DATE || DataType() == CalpontSystemCatalog::BIGINT || DataType() == CalpontSystemCatalog::DATETIME || + DataType() == CalpontSystemCatalog::TIME || DataType() == CalpontSystemCatalog::UTINYINT || DataType() == CalpontSystemCatalog::USMALLINT || DataType() == CalpontSystemCatalog::UMEDINT || diff --git a/tools/setConfig/configxml.sh b/tools/setConfig/configxml.sh index 2c3568f00..d78953660 100755 --- a/tools/setConfig/configxml.sh +++ b/tools/setConfig/configxml.sh @@ -18,7 +18,7 @@ InstallDir=$COLUMNSTORE_INSTALL_DIR if [ $InstallDir != "/usr/local/mariadb/columnstore" ]; then export PATH=$InstallDir/bin:$InstallDir/mysql/bin:/bin:/usr/bin - export LD_LIBRARY_PATH=$InstallDir/lib:$InstallDir/mysql/lib/mysql + export LD_LIBRARY_PATH=$InstallDir/lib:$InstallDir/mysql/lib fi case "$1" in diff --git a/utils/clusterTester/columnstoreClusterTester.sh b/utils/clusterTester/columnstoreClusterTester.sh index b512eeb20..9f8f3e9a3 100755 --- a/utils/clusterTester/columnstoreClusterTester.sh +++ b/utils/clusterTester/columnstoreClusterTester.sh @@ -494,7 +494,7 @@ checkSELINUX() # SELINUX check # echo "" - echo "** Run SELINUX check - Setting should to be disabled on all nodes" + echo "** Run SELINUX check" echo "" pass=true @@ -502,9 +502,8 @@ checkSELINUX() if [ -f /etc/selinux/config ]; then `cat /etc/selinux/config | grep SELINUX | grep enforcing > /tmp/selinux_check 2>&1` if [ "$?" -eq 0 ]; then - echo "${bold}Failed${normal}, Local Node SELINUX setting is Enabled, please disable" + echo "${bold}Warning${normal}, Local Node SELINUX setting is Enabled, check port test results" pass=false - REPORTPASS=false else echo "Local Node SELINUX setting is Not Enabled" fi @@ -519,19 +518,14 @@ checkSELINUX() else `cat config | grep SELINUX | grep enforcing > /tmp/selinux_check 2>&1` if [ "$?" -eq 0 ]; then - echo "${bold}Failed${normal}, $ipadd SELINUX setting is Enabled, please disable" + echo "${bold}Warning${normal}, $ipadd SELINUX setting is Enabled, check port test results" pass=false - REPORTPASS=false else echo "$ipadd Node SELINUX setting is Not Enabled" fi `rm -f config` fi done - - if ! $pass; then - checkContinue - fi } checkFirewalls() @@ -539,28 +533,23 @@ checkFirewalls() # FIREWALL checks # echo "" - echo "** Run Firewall Services check - Firewall Services should to be Inactive on all nodes" + echo "** Run Firewall Services check" echo "" declare -a FIREWALL_LIST=("iptables" "ufw" "firewalld" "firewall") - fpass=true #check local FIREWALLS for firewall in "${FIREWALL_LIST[@]}"; do pass=true `service $firewall status > /tmp/firewall1_check 2>&1` if [ "$?" -eq 0 ]; then - echo "${bold}Failed${normal}, Local Node $firewall service is Active, please disable" + echo "${bold}Warning${normal}, Local Node $firewall service is Active, check port test results" pass=false - fpass=false - REPORTPASS=false else `systemctl status $firewall > /tmp/firewall1_check 2>&1` if [ "$?" -eq 0 ]; then - echo "${bold}Failed${normal}, Local Node $firewall service is Active, please disable" + echo "${bold}Warning${normal}, Local Node $firewall service is Active, check port test results" pass=false - fpass=false - REPORTPASS=false fi fi @@ -569,10 +558,6 @@ checkFirewalls() fi done - if ! $fpass; then - checkContinue - fi - echo "" fpass=true for ipadd in "${NODE_IPADDRESS[@]}"; do @@ -581,17 +566,13 @@ checkFirewalls() pass=true `$COLUMNSTORE_INSTALL_DIR/bin/remote_command.sh $ipadd $PASSWORD "service '$firewall' status > /tmp/firewall_check 2>&1" 1 > /tmp/remote_command_check` if [ "$?" -eq 0 ]; then - echo "${bold}Failed${normal}, $ipadd Node $firewall service is Active, please disable" + echo "${bold}Warning${normal}, $ipadd Node $firewall service is Active, check port test results" pass=false - fpass=false - REPORTPASS=false else `$COLUMNSTORE_INSTALL_DIR/bin/remote_command.sh $ipadd $PASSWORD "systemctl status '$firewall' > /tmp/firewall_check 2>&1" 1 > /tmp/remote_command_check` if [ "$?" -eq 0 ]; then - echo "${bold}Failed${normal}, $ipadd Node $firewall service is Active, please disable" + echo "${bold}Warning${normal}, $ipadd Node $firewall service is Active, check port test results" pass=false - fpass=false - REPORTPASS=false fi fi @@ -602,25 +583,20 @@ checkFirewalls() echo "" done - - if ! $fpass; then - checkContinue - fi if [ $OS == "suse12" ]; then # rcSuSEfirewall2 check # echo "" - echo "** Run rcSuSEfirewall2 check - Service should to be disabled on all nodes" + echo "** Run rcSuSEfirewall2 check" echo "" pass=true #check local IPTABLES `/sbin/rcSuSEfirewall2 status > /tmp/rcSuSEfirewall2_check 2>&1` if [ "$?" -eq 0 ]; then - echo "${bold}Failed${normal}, Local Node rcSuSEfirewall2 service is Enabled, please disable" + echo "${bold}Failed${normal}, Local Node rcSuSEfirewall2 service is Enabled, check port test results" pass=false - REPORTPASS=false else echo "Local Node rcSuSEfirewall2 service is Not Enabled" fi @@ -629,17 +605,12 @@ checkFirewalls() `$COLUMNSTORE_INSTALL_DIR/bin/remote_command.sh $ipadd $PASSWORD '/sbin/rcSuSEfirewall2 status > /tmp/rcSuSEfirewall2_check 2>&1' 1 > /tmp/remote_command_check` rc="$?" if [ $rc -eq 0 ] ; then - echo "${bold}Failed${normal}, $ipadd Node rcSuSEfirewall2 service is Enabled, please disable" + echo "${bold}Failed${normal}, $ipadd Node rcSuSEfirewall2 service is Enabled, check port test results" pass=false - REPORTPASS=false else echo "$ipadd Node rcSuSEfirewall2 service is Not Enabled" fi done - - if ! $pass; then - checkContinue - fi fi } @@ -648,17 +619,18 @@ checkPorts() # port test # echo "" - echo "** Run MariaDB ColumnStore Port (8600-8620) availibility test" + echo "** Run MariaDB ColumnStore Port (8600-8630,8700,8800,3306) availability test" echo "" pass=true for ipadd in "${NODE_IPADDRESS[@]}"; do - `nmap $ipadd -p 8600-8620 | grep 'closed unknown' > /dev/null` - if [ "$?" -eq 0 ]; then + `sudo nmap $ipadd -p 8600-8630,8700,8800,3306 | grep 'filtered' > /tmp/port_test` + if [ "$?" -ne 0 ]; then echo $ipadd " Node Passed port test" else - echo $ipadd " Node ${bold}Failed${normal} port test, check and disable any firwalls that were reported enabled" + echo $ipadd " Node ${bold}Failed${normal} port test, check and disable any firewalls or open ports in firewall" + cat /tmp/port_test pass=false REPORTPASS=false fi @@ -764,7 +736,7 @@ checkPackages() echo "** Run MariaDB ColumnStore Dependent Package Check" echo "" - declare -a CENTOS_PKG=("expect" "perl" "perl-DBI" "openssl" "zlib" "file" "sudo" "libaio" "rsync" "snappy" "net-tools" "perl-DBD-MySQL") + declare -a CENTOS_PKG=("expect" "perl" "perl-DBI" "openssl" "zlib" "file" "sudo" "libaio" "rsync" "snappy" "net-tools" "numactl-libs") declare -a CENTOS_PKG_NOT=("mariadb-libs") if [ "$OS" == "centos6" ] || [ "$OS" == "centos7" ]; then @@ -883,7 +855,7 @@ checkPackages() fi fi - declare -a SUSE_PKG=("boost-devel" "expect" "perl" "perl-DBI" "openssl" "file" "sudo" "libaio1" "rsync" "libsnappy1" "net-tools" "perl-DBD-mysql") + declare -a SUSE_PKG=("boost-devel" "expect" "perl" "perl-DBI" "openssl" "file" "sudo" "libaio1" "rsync" "libsnappy1" "net-tools" "libnuma1") declare -a SUSE_PKG_NOT=("mariadb" , "libmariadb18") if [ "$OS" == "suse12" ]; then @@ -974,7 +946,7 @@ checkPackages() fi fi - declare -a UBUNTU_PKG=("libboost-all-dev" "expect" "libdbi-perl" "perl" "openssl" "file" "sudo" "libreadline-dev" "rsync" "libsnappy1V5" "net-tools" "libdbd-mysql-perl") + declare -a UBUNTU_PKG=("libboost-all-dev" "expect" "libdbi-perl" "perl" "openssl" "file" "sudo" "libreadline-dev" "rsync" "libsnappy1V5" "net-tools" "libnuma1" ) declare -a UBUNTU_PKG_NOT=("mariadb-server" "libmariadb18") if [ "$OS" == "ubuntu16" ] ; then @@ -1091,7 +1063,7 @@ checkPackages() fi fi - declare -a DEBIAN_PKG=("libboost-all-dev" "expect" "libdbi-perl" "perl" "openssl" "file" "sudo" "libreadline-dev" "rsync" "libsnappy1" "net-tools" "libdbd-mysql-perl") + declare -a DEBIAN_PKG=("libboost-all-dev" "expect" "libdbi-perl" "perl" "openssl" "file" "sudo" "libreadline-dev" "rsync" "libsnappy1" "net-tools" "libnuma1") declare -a DEBIAN_PKG_NOT=("libmariadb18" "mariadb-server") if [ "$OS" == "debian8" ]; then @@ -1208,7 +1180,7 @@ checkPackages() fi fi - declare -a DEBIAN9_PKG=("libboost-all-dev" "expect" "libdbi-perl" "perl" "openssl" "file" "sudo" "libreadline5" "rsync" "libsnappy1V5" "net-tools" "libaio1") + declare -a DEBIAN9_PKG=("libboost-all-dev" "expect" "libdbi-perl" "perl" "openssl" "file" "sudo" "libreadline5" "rsync" "libsnappy1V5" "net-tools" "libaio1" "libnuma1") declare -a DEBIAN9_PKG_NOT=("libmariadb18" "mariadb-server") if [ "$OS" == "debian9" ]; then diff --git a/utils/common/nullvaluemanip.cpp b/utils/common/nullvaluemanip.cpp index 4eb96f77c..475f495cc 100644 --- a/utils/common/nullvaluemanip.cpp +++ b/utils/common/nullvaluemanip.cpp @@ -56,6 +56,9 @@ uint64_t getNullValue(CalpontSystemCatalog::ColDataType t, uint32_t colWidth) case CalpontSystemCatalog::DATETIME: return joblist::DATETIMENULL; + case CalpontSystemCatalog::TIME: + return joblist::TIMENULL; + case CalpontSystemCatalog::CHAR: case CalpontSystemCatalog::VARCHAR: case CalpontSystemCatalog::STRINT: @@ -163,6 +166,9 @@ int64_t getSignedNullValue(CalpontSystemCatalog::ColDataType t, uint32_t colWidt case CalpontSystemCatalog::DATETIME: return joblist::DATETIMENULL; + case CalpontSystemCatalog::TIME: + return joblist::TIMENULL; + case CalpontSystemCatalog::CHAR: case CalpontSystemCatalog::VARCHAR: case CalpontSystemCatalog::STRINT: diff --git a/utils/dataconvert/dataconvert.cpp b/utils/dataconvert/dataconvert.cpp index d0689fb0d..b1cf8bcb2 100644 --- a/utils/dataconvert/dataconvert.cpp +++ b/utils/dataconvert/dataconvert.cpp @@ -859,6 +859,180 @@ bool mysql_str_to_datetime( const string& input, DateTime& output, bool& isDate return true; } +bool mysql_str_to_time( const string& input, Time& output ) +{ + int32_t datesepct = 0; + uint32_t dtend = 0; + bool isNeg = false; + + /** + * We need to deal with the time portion. + * The rules are: + * - Time portion always ends with '\0' + * - Time portion always starts with hour + * - Without time separators (':'): + * HHMMSS + * - All Times can end with option .[microseconds] + * - With time separators there are no specific field length + * requirements + */ + while ( input[dtend] == ' ' && dtend < input.length() ) + { + ++dtend; + } + + if ( dtend == input.length() ) + { + return false; + } + + uint32_t timesep_ct = 0; + bool has_usec = false; + uint32_t len_before_msec = 0; + uint32_t tmstart = dtend; + uint32_t tmend = tmstart; + + for ( ; tmend < input.length(); ++tmend ) + { + char c = input[tmend]; + + if ( isdigit( c ) ) + { + // digits always ok + continue; + } +// else if( c == ':' ) +// { +// timesep_ct++; +// } +// else if( c == '.' ) +// { +// len_before_msec = ( tmend - tmstart ); +// has_usec = true; +// } + else if ( ispunct(c) ) + { + if ( c == '.' && timesep_ct == 2 ) + { + len_before_msec = ( tmend - tmstart ); + has_usec = true; + } + else if (c == '-' && (tmend == tmstart)) + { + isNeg = true; + ++tmstart; + } + else + { + timesep_ct++; + } + } + else + { + // some other character showed up + output.reset(); + return false; + } + } + + if ( !len_before_msec ) + len_before_msec = ( tmend - tmstart ); + + int32_t hour = -1; + int32_t min = 0; + int32_t sec = 0; + int32_t usec = 0; + const char* tstart = input.c_str() + tmstart; + + if ( timesep_ct == 2 ) + { + readDecimal(tstart, hour); + ++tstart; // skip one separator + readDecimal(tstart, min); + ++tstart; // skip one separator + readDecimal(tstart, sec); + } + else if ( timesep_ct == 1 ) + { + readDecimal(tstart, hour); + ++tstart; // skip one separator + readDecimal(tstart, min); + } + else if ( timesep_ct == 0 && len_before_msec == 6 ) + { + readDecimal(tstart, hour, 2); + readDecimal(tstart, min, 2); + readDecimal(tstart, sec, 2); + } + else if ( timesep_ct == 0 && len_before_msec == 4 ) + { + readDecimal(tstart, hour, 2); + readDecimal(tstart, min, 2); + } + else if ( timesep_ct == 0 && len_before_msec == 2 ) + { + readDecimal(tstart, hour, 2); + } + else + { + output.reset(); + return false; + } + + if ( has_usec ) + { + ++tstart; // skip '.' character. We could error check if we wanted to + uint32_t numread = readDecimal(tstart, usec); + + if ( numread > 6 || numread < 1 ) + { + // don't allow more than 6 digits when specifying microseconds + output.reset(); + return false; + } + + // usec have to be scaled up so that it always represents microseconds + for ( int i = numread; i < 6; i++ ) + usec *= 10; + } + + if ( !isTimeValid( hour, min, sec, usec ) ) + { + // Emulate MariaDB's time saturation + if (hour > 838) + { + output.hour = 838; + output.minute = 59; + output.second = 59; + output.msecond = 999999; + output.is_neg = 0; + } + else if (hour < -838) + { + output.hour = -838; + output.minute = 59; + output.second = 59; + output.msecond = 999999; + output.is_neg = 1; + } + // If neither of the above match then we return a 0 time + else + { + output.reset(); + } + + return false; + } + + output.hour = isNeg ? 0 - hour : hour; + output.minute = min; + output.second = sec; + output.msecond = usec; + output.is_neg = isNeg; + return true; +} + + bool stringToDateStruct( const string& data, Date& date ) { bool isDate; @@ -894,6 +1068,14 @@ bool stringToDatetimeStruct(const string& data, DateTime& dtime, bool* date) return true; } +bool stringToTimeStruct(const string& data, Time& dtime) +{ + if ( !mysql_str_to_time( data, dtime ) ) + return false; + + return true; +} + boost::any DataConvert::convertColumnData(const CalpontSystemCatalog::ColType& colType, const std::string& dataOrig, bool& pushWarning, bool nulFlag, bool noRoundup, bool isUpdate ) @@ -1229,6 +1411,23 @@ DataConvert::convertColumnData(const CalpontSystemCatalog::ColType& colType, } break; + case CalpontSystemCatalog::TIME: + { + Time aTime; + + if (stringToTimeStruct(data, aTime)) + { + value = (int64_t) * (reinterpret_cast(&aTime)); + } + else + { + value = (int64_t) 0; + pushWarning = true; + } + } + break; + + case CalpontSystemCatalog::BLOB: case CalpontSystemCatalog::CLOB: value = data; @@ -1345,6 +1544,13 @@ DataConvert::convertColumnData(const CalpontSystemCatalog::ColType& colType, } break; + case CalpontSystemCatalog::TIME: + { + uint64_t d = joblist::TIMENULL; + value = d; + } + break; + case CalpontSystemCatalog::CHAR: { std::string charnull; @@ -1692,6 +1898,140 @@ int64_t DataConvert::convertColumnDatetime( return value; } +//------------------------------------------------------------------------------ +// Convert time string to binary time. Used by BulkLoad. +// Most of this is taken from str_to_time in sql-common/my_time.c +//------------------------------------------------------------------------------ +int64_t DataConvert::convertColumnTime( + const char* dataOrg, + CalpontDateTimeFormat datetimeFormat, + int& status, + unsigned int dataOrgLen ) +{ + status = 0; + char* p; + char* savePoint = NULL; + p = const_cast(dataOrg); + int64_t value = 0; + int inHour, inMinute, inSecond, inMicrosecond; + inHour = 0; + inMinute = 0; + inSecond = 0; + inMicrosecond = 0; + bool isNeg = false; + + if ( datetimeFormat != CALPONTTIME_ENUM ) + { + status = -1; + return value; + } + + if (p[0] == '-') + { + isNeg = true; + } + + errno = 0; + + p = strtok_r(p, ":.", &savePoint); + inHour = strtol(p, 0, 10); + + if (errno) + { + status = -1; + return value; + } + + p = strtok_r(NULL, ":.", &savePoint); + + if (p == NULL) + { + status = -1; + return value; + } + + inMinute = strtol(p, 0, 10); + + if (errno) + { + status = -1; + return value; + } + + p = strtok_r(NULL, ":.", &savePoint); + + if (p == NULL) + { + status = -1; + return value; + } + + inSecond = strtol(p, 0, 10); + + if (errno) + { + status = -1; + return value; + } + + p = strtok_r(NULL, ":.", &savePoint); + + if (p != NULL) + { + inMicrosecond = strtol(p, 0, 10); + + if (errno) + { + status = -1; + return value; + } + } + + if ( isTimeValid (inHour, inMinute, inSecond, inMicrosecond) ) + { + Time atime; + atime.hour = inHour; + atime.minute = inMinute; + atime.second = inSecond; + atime.msecond = inMicrosecond; + atime.is_neg = isNeg; + + memcpy( &value, &atime, 8); + } + else + { + // Emulate MariaDB's time saturation + if (inHour > 838) + { + Time atime; + atime.hour = 838; + atime.minute = 59; + atime.second = 59; + atime.msecond = 999999; + atime.is_neg = false; + memcpy( &value, &atime, 8); + } + else if (inHour < -838) + { + Time atime; + atime.hour = -838; + atime.minute = 59; + atime.second = 59; + atime.msecond = 999999; + atime.is_neg = false; + memcpy( &value, &atime, 8); + } + + // If neither of the above match then we return a 0 time + + status = -1; + } + + return value; + +} + + //------------------------------------------------------------------------------ // Verify that specified datetime is valid //------------------------------------------------------------------------------ @@ -1706,6 +2046,14 @@ bool DataConvert::isColumnDateTimeValid( int64_t dateTime ) return false; } +bool DataConvert::isColumnTimeValid( int64_t time ) +{ + Time dt; + memcpy(&dt, &time, sizeof(uint64_t)); + + return isTimeValid(dt.hour, dt.minute, dt.second, dt.msecond); +} + std::string DataConvert::dateToString( int datevalue ) { // @bug 4703 abandon multiple ostringstream's for conversion @@ -1717,14 +2065,69 @@ std::string DataConvert::dateToString( int datevalue ) return buf; } -std::string DataConvert::datetimeToString( long long datetimevalue ) +std::string DataConvert::datetimeToString( long long datetimevalue, long decimals ) { + // 10 is default which means we don't need microseconds + if (decimals > 6 || decimals < 0) + { + decimals = 0; + } + // @bug 4703 abandon multiple ostringstream's for conversion DateTime dt(datetimevalue); - const int DATETIMETOSTRING_LEN = 21; // YYYY-MM-DD HH:MM:SS\0 + const int DATETIMETOSTRING_LEN = 28; // YYYY-MM-DD HH:MM:SS.mmmmmm\0 char buf[DATETIMETOSTRING_LEN]; sprintf(buf, "%04d-%02d-%02d %02d:%02d:%02d", dt.year, dt.month, dt.day, dt.hour, dt.minute, dt.second); + + if (dt.msecond && decimals) + { + snprintf(buf + strlen(buf), 21 + decimals, ".%d", dt.msecond); + + // Pad end with zeros + if (strlen(buf) < (size_t)(21 + decimals)) + { + sprintf(buf + strlen(buf), "%0*d", (int)(21 + decimals - strlen(buf)), 0); + } + } + + return buf; +} + +std::string DataConvert::timeToString( long long timevalue, long decimals ) +{ + // 10 is default which means we don't need microseconds + if (decimals > 6 || decimals < 0) + { + decimals = 0; + } + + // @bug 4703 abandon multiple ostringstream's for conversion + Time dt(timevalue); + const int TIMETOSTRING_LEN = 19; // (-H)HH:MM:SS.mmmmmm\0 + char buf[TIMETOSTRING_LEN]; + char* outbuf = buf; + + if ((dt.hour >= 0) && dt.is_neg) + { + outbuf[0] = '-'; + outbuf++; + } + + sprintf(outbuf, "%02d:%02d:%02d", dt.hour, dt.minute, dt.second); + + if (dt.msecond && decimals) + { + size_t start = strlen(buf); + snprintf(buf + strlen(buf), 12 + decimals, ".%d", dt.msecond); + + // Pad end with zeros + if (strlen(buf) - start < (size_t)decimals) + { + sprintf(buf + strlen(buf), "%0*d", (int)(decimals - (strlen(buf) - start) + 1), 0); + } + } + return buf; } @@ -1749,6 +2152,20 @@ std::string DataConvert::datetimeToString1( long long datetimevalue ) sprintf(buf, "%04d%02d%02d%02d%02d%02d%06d", dt.year, dt.month, dt.day, dt.hour, dt.minute, dt.second, dt.msecond); return buf; } + +std::string DataConvert::timeToString1( long long datetimevalue ) +{ + // @bug 4703 abandon multiple ostringstream's for conversion + DateTime dt(datetimevalue); + const int TIMETOSTRING1_LEN = 14; // HHMMSSmmmmmm\0 + char buf[TIMETOSTRING1_LEN]; + + char* outbuf = buf; + + sprintf(outbuf, "%02d%02d%02d%06d", dt.hour, dt.minute, dt.second, dt.msecond); + return buf; +} + #if 0 bool DataConvert::isNullData(ColumnResult* cr, int rownum, CalpontSystemCatalog::ColType colType) { @@ -1969,6 +2386,11 @@ int64_t DataConvert::datetimeToInt(const string& datetime) return stringToDatetime(datetime); } +int64_t DataConvert::timeToInt(const string& time) +{ + return stringToTime(time); +} + int64_t DataConvert::stringToDate(const string& data) { Date aDay; @@ -2242,18 +2664,104 @@ int64_t DataConvert::intToDatetime(int64_t data, bool* date) return *(reinterpret_cast(&adaytime)); } +int64_t DataConvert::intToTime(int64_t data) +{ + char buf[21] = {0}; + char* bufread = buf; + Time atime; + bool isNeg = false; + + if (data == 0) + { + atime.hour = 0; + atime.minute = 0; + atime.second = 0; + atime.msecond = 0; + atime.is_neg = 0; + + return *(reinterpret_cast(&atime)); + } + + snprintf( buf, 15, "%llu", (long long unsigned int)data); + //string date = buf; + string hour, min, sec, msec; + int64_t h = 0, minute = 0, s = 0, ms = 0; + + if (bufread[0] == '-') + { + isNeg = true; + bufread++; + } + + switch (strlen(bufread)) + { + case 7: + hour = string(bufread, 3); + min = string(bufread + 2, 2); + sec = string(bufread + 4, 2); + msec = string(bufread + 6, 6); + break; + + case 6: + hour = string(bufread, 2); + min = string(bufread + 2, 2); + sec = string(bufread + 4, 2); + msec = string(bufread + 6, 6); + break; + + case 4: + min = string(bufread, 2); + sec = string(bufread + 2, 2); + msec = string(bufread + 4, 6); + break; + + case 2: + sec = string(bufread, 2); + msec = string(bufread + 2, 6); + break; + + default: + return -1; + } + + h = atoi(hour.c_str()); + minute = atoi(min.c_str()); + s = atoi(sec.c_str()); + ms = atoi(msec.c_str()); + + if (!isTimeValid(h, minute, s, ms)) + return -1; + + atime.hour = h; + atime.minute = minute; + atime.second = s; + atime.msecond = ms; + atime.is_neg = isNeg; + + return *(reinterpret_cast(&atime)); +} + int64_t DataConvert::stringToTime(const string& data) { // MySQL supported time value format 'D HHH:MM:SS.fraction' // -34 <= D <= 34 // -838 <= H <= 838 uint64_t min = 0, sec = 0, msec = 0; - int64_t day = 0, hour = 0; + int64_t day = -1, hour = 0; + bool isNeg = false; string time, hms, ms; char* end = NULL; + + size_t pos = data.find("-"); + + if (pos != string::npos) + { + isNeg = true; + } + // Day - size_t pos = data.find(" "); + pos = data.find(" "); if (pos != string::npos) { @@ -2314,6 +2822,7 @@ int64_t DataConvert::stringToTime(const string& data) atime.minute = min; atime.second = sec; atime.msecond = msec; + atime.is_neg = isNeg; return *(reinterpret_cast(&atime)); } @@ -2388,6 +2897,7 @@ CalpontSystemCatalog::ColType DataConvert::convertUnionColType(vector> 24) & 0xff), minute((val >> 32) & 0xff), hour((val >> 40) & 0xfff), - day((val >> 52) & 0xfff) + day((val >> 52) & 0x7ff), + is_neg(val >> 63) {} + + Time(signed d, signed h, signed min, signed sec, signed msec, bool neg) : + msecond(msec), second(sec), minute(min), hour(h), day(d), is_neg(neg) + { + if (h < 0) + is_neg = 0b1; + } + + int64_t convertToMySQLint() const; + void reset(); }; -static uint32_t daysInMonth[13] = {31, 28, 31, 30, 31, 30, 31, 31, 30, 31, 30, 31, 0}; - -inline uint32_t getDaysInMonth(uint32_t month) +inline +void Time::reset() { - return ( (month < 1 || month > 12) ? 0 : daysInMonth[month - 1]); + msecond = 0xFFFFFE; + second = 0xFF; + minute = 0xFF; + hour = 0xFFF; + is_neg = 0b1; + day = 0x7FF; } +inline +int64_t Time::convertToMySQLint() const +{ + if ((hour >= 0) && is_neg) + { + return (int64_t) ((hour * 10000) + (minute * 100) + second) * -1; + } + else if (hour >= 0) + { + return (int64_t) (hour * 10000) + (minute * 100) + second; + } + else + { + return (int64_t) (hour * 10000) - (minute * 100) - second; + } +} + +static uint32_t daysInMonth[13] = {31, 28, 31, 30, 31, 30, 31, 31, 30, 31, 30, 31, 0}; + inline bool isLeapYear ( int year) { if ( year % 400 == 0 ) @@ -233,6 +271,19 @@ inline bool isLeapYear ( int year) return false; } +inline uint32_t getDaysInMonth(uint32_t month, int year) +{ + if (month < 1 || month > 12) + return 0; + + uint32_t days = daysInMonth[month - 1]; + + if ((month == 2) && isLeapYear(year)) + days++; + + return days; +} + inline bool isDateValid ( int day, int month, int year) { @@ -243,11 +294,7 @@ bool isDateValid ( int day, int month, int year) return true; } - int daycheck = getDaysInMonth( month ); - - if ( month == 2 && isLeapYear( year ) ) - // 29 days in February in a leap year - daycheck = 29; + int daycheck = getDaysInMonth( month, year ); if ( ( year < 1000 ) || ( year > 9999 ) ) valid = false; @@ -281,6 +328,28 @@ bool isDateTimeValid ( int hour, int minute, int second, int microSecond) return valid; } +inline +bool isTimeValid ( int hour, int minute, int second, int microSecond) +{ + bool valid = false; + + if ( hour >= -838 && hour <= 838 ) + { + if ( minute >= 0 && minute < 60 ) + { + if ( second >= 0 && second < 60 ) + { + if ( microSecond >= 0 && microSecond <= 999999 ) + { + valid = true; + } + } + } + } + + return valid; +} + inline int64_t string_to_ll( const std::string& data, bool& bSaturate ) { @@ -361,8 +430,17 @@ public: * @param type the columns database type * @param data the columns string representation of it's data */ - EXPORT static std::string datetimeToString( long long datetimevalue ); - static inline void datetimeToString( long long datetimevalue, char* buf, unsigned int buflen ); + EXPORT static std::string datetimeToString( long long datetimevalue, long decimals = 0 ); + static inline void datetimeToString( long long datetimevalue, char* buf, unsigned int buflen, long decimals = 0 ); + + /** + * @brief convert a columns data from native format to a string + * + * @param type the columns database type + * @param data the columns string representation of it's data + */ + EXPORT static std::string timeToString( long long timevalue, long decimals = 0 ); + static inline void timeToString( long long timevalue, char* buf, unsigned int buflen, long decimals = 0); /** * @brief convert a columns data from native format to a string @@ -382,6 +460,15 @@ public: EXPORT static std::string datetimeToString1( long long datetimevalue ); static inline void datetimeToString1( long long datetimevalue, char* buf, unsigned int buflen ); + /** + * @brief convert a columns data from native format to a string + * + * @param type the columns database type + * @param data the columns string representation of it's data + */ + EXPORT static std::string timeToString1( long long timevalue ); + static inline void timeToString1( long long timevalue, char* buf, unsigned int buflen ); + /** * @brief convert a date column data, represnted as a string, to it's native * format. This function is for bulkload to use. @@ -415,15 +502,31 @@ public: CalpontDateTimeFormat datetimeFormat, int& status, unsigned int dataOrgLen ); + /** + * @brief convert a time column data, represented as a string, + * to it's native format. This function is for bulkload to use. + * + * @param type the columns data type + * @param dataOrig the columns string representation of it's data + * @param timeFormat the format the time value in + * @param status 0 - success, -1 - fail + * @param dataOrgLen length specification of dataOrg + */ + EXPORT static int64_t convertColumnTime( const char* dataOrg, + CalpontDateTimeFormat datetimeFormat, + int& status, unsigned int dataOrgLen ); + /** * @brief Is specified datetime valid; used by binary bulk load */ EXPORT static bool isColumnDateTimeValid( int64_t dateTime ); + EXPORT static bool isColumnTimeValid( int64_t time ); EXPORT static bool isNullData(execplan::ColumnResult* cr, int rownum, execplan::CalpontSystemCatalog::ColType colType); static inline std::string decimalToString(int64_t value, uint8_t scale, execplan::CalpontSystemCatalog::ColDataType colDataType); static inline void decimalToString(int64_t value, uint8_t scale, char* buf, unsigned int buflen, execplan::CalpontSystemCatalog::ColDataType colDataType); static inline std::string constructRegexp(const std::string& str); + static inline void trimWhitespace(int64_t& charData); static inline bool isEscapedChar(char c) { return ('%' == c || '_' == c); @@ -437,11 +540,13 @@ public: EXPORT static int64_t intToDate(int64_t data); // convert integer to datetime EXPORT static int64_t intToDatetime(int64_t data, bool* isDate = NULL); - + // convert integer to date + EXPORT static int64_t intToTime(int64_t data); // convert string to date. alias to stringToDate EXPORT static int64_t dateToInt(const std::string& date); // convert string to datetime. alias to datetimeToInt EXPORT static int64_t datetimeToInt(const std::string& datetime); + EXPORT static int64_t timeToInt(const std::string& time); EXPORT static int64_t stringToTime (const std::string& data); // bug4388, union type conversion EXPORT static execplan::CalpontSystemCatalog::ColType convertUnionColType(std::vector&); @@ -456,8 +561,21 @@ inline void DataConvert::dateToString( int datevalue, char* buf, unsigned int bu ); } -inline void DataConvert::datetimeToString( long long datetimevalue, char* buf, unsigned int buflen ) +inline void DataConvert::datetimeToString( long long datetimevalue, char* buf, unsigned int buflen, long decimals ) { + // 10 is default which means we don't need microseconds + if (decimals > 6 || decimals < 0) + { + decimals = 0; + } + + int msec = 0; + + if ((datetimevalue & 0xfffff) > 0) + { + msec = (unsigned)((datetimevalue) & 0xfffff); + } + snprintf( buf, buflen, "%04d-%02d-%02d %02d:%02d:%02d", (unsigned)((datetimevalue >> 48) & 0xffff), (unsigned)((datetimevalue >> 44) & 0xf), @@ -466,6 +584,67 @@ inline void DataConvert::datetimeToString( long long datetimevalue, char* buf, u (unsigned)((datetimevalue >> 26) & 0x3f), (unsigned)((datetimevalue >> 20) & 0x3f) ); + + if (msec || decimals) + { + size_t start = strlen(buf); + snprintf(buf + strlen(buf), buflen - start, ".%d", msec); + + // Pad end with zeros + if (strlen(buf) - start < (size_t)decimals) + { + snprintf(buf + strlen(buf), buflen - strlen(buf), "%0*d", (int)(decimals - (strlen(buf) - start) + 1), 0); + } + } +} + +inline void DataConvert::timeToString( long long timevalue, char* buf, unsigned int buflen, long decimals ) +{ + // 10 is default which means we don't need microseconds + if (decimals > 6 || decimals < 0) + { + decimals = 0; + } + + // Handle negative correctly + int hour = 0, msec = 0; + + if ((timevalue >> 40) & 0x800) + { + hour = 0xfffff000; + } + + hour |= ((timevalue >> 40) & 0xfff); + + if ((timevalue & 0xffffff) > 0) + { + msec = (unsigned)((timevalue) & 0xffffff); + } + + if ((hour >= 0) && (timevalue >> 63)) + { + buf[0] = '-'; + buf++; + buflen--; + } + + snprintf( buf, buflen, "%02d:%02d:%02d", + hour, + (unsigned)((timevalue >> 32) & 0xff), + (unsigned)((timevalue >> 24) & 0xff) + ); + + if (msec || decimals) + { + size_t start = strlen(buf); + snprintf(buf + strlen(buf), buflen - start, ".%d", msec); + + // Pad end with zeros + if (strlen(buf) - start < (size_t)decimals) + { + snprintf(buf + strlen(buf), buflen - strlen(buf), "%0*d", (int)(decimals - (strlen(buf) - start) + 1), 0); + } + } } inline void DataConvert::dateToString1( int datevalue, char* buf, unsigned int buflen) @@ -489,6 +668,32 @@ inline void DataConvert::datetimeToString1( long long datetimevalue, char* buf, ); } +inline void DataConvert::timeToString1( long long timevalue, char* buf, unsigned int buflen ) +{ + // Handle negative correctly + int hour = 0; + + if ((timevalue >> 40) & 0x800) + { + hour = 0xfffff000; + } + + hour |= ((timevalue >> 40) & 0xfff); + + if ((hour >= 0) && (timevalue >> 63)) + { + buf[0] = '-'; + buf++; + buflen--; + } + + snprintf( buf, buflen, "%02d%02d%02d", + hour, + (unsigned)((timevalue >> 32) & 0xff), + (unsigned)((timevalue >> 14) & 0xff) + ); +} + inline std::string DataConvert::decimalToString(int64_t value, uint8_t scale, execplan::CalpontSystemCatalog::ColDataType colDataType) { char buf[80]; @@ -578,6 +783,19 @@ inline void DataConvert::decimalToString(int64_t int_val, uint8_t scale, char* b *(ptr + l1) = '.'; } +inline void DataConvert::trimWhitespace(int64_t& charData) +{ + // Trims whitespace characters off non-dict character data + char* ch_data = (char*) &charData; + + for (int8_t i = 7; i > 0; i--) + { + if (ch_data[i] == ' ' || ch_data[i] == '\0') + ch_data[i] = '\0'; + else + break; + } +} //FIXME: copy/pasted from dictionary.cpp: refactor inline std::string DataConvert::constructRegexp(const std::string& str) diff --git a/utils/funcexp/func_add_time.cpp b/utils/funcexp/func_add_time.cpp index fb4469eec..ced9beaf2 100644 --- a/utils/funcexp/func_add_time.cpp +++ b/utils/funcexp/func_add_time.cpp @@ -80,16 +80,15 @@ int64_t addTime(DateTime& dt1, Time& dt2) } hour = (signed)(dt1.hour + dt2.hour + min / 60); - dt.hour = tmp = hour % 24; -// if (tmp < -1) - if (tmp < 0) // fix for subtime dlh + if ((hour < 0) || (hour > 23)) { - dt.hour = tmp + 24; - dt2.day--; + dt.hour = hour % 24; + dt2.day = hour / 24; } - day = (signed)(dt1.day + dt2.day + hour / 24); + day = (signed)(dt1.day + dt2.day); + if (isLeapYear(dt1.year) && dt1.month == 2) day--; @@ -97,7 +96,7 @@ int64_t addTime(DateTime& dt1, Time& dt2) month = dt1.month; int addyear = 0; - if (dt2.day < 0 || dt2.hour < 0) + if (day < 0) { int monthSave = month; @@ -106,7 +105,7 @@ int64_t addTime(DateTime& dt1, Time& dt2) month = (month == 1 ? 12 : month - 1); for (; day <= 0 && month > 0; month--) - day += getDaysInMonth(month); + day += getDaysInMonth(month, dt1.year); month++; // month=12; @@ -119,10 +118,10 @@ int64_t addTime(DateTime& dt1, Time& dt2) { int monthSave = month; - while (day > getDaysInMonth(month)) + while (day > getDaysInMonth(month, dt1.year)) { - for (; day > getDaysInMonth(month) && month <= 12; month++) - day -= getDaysInMonth(month); + for (; day > getDaysInMonth(month, dt1.year) && month <= 12; month++) + day -= getDaysInMonth(month, dt1.year); if (month > 12) month = 1; @@ -138,6 +137,65 @@ int64_t addTime(DateTime& dt1, Time& dt2) return *(reinterpret_cast(&dt)); } + +int64_t addTime(Time& dt1, Time& dt2) +{ + Time dt; + dt.hour = 0; + dt.minute = 0; + dt.second = 0; + dt.msecond = 0; + + int64_t min, sec, msec, tmp; + msec = (signed)(dt1.msecond + dt2.msecond); + dt.msecond = tmp = msec % 1000000; + + if (tmp < 0) + { + dt.msecond = tmp + 1000000; + dt2.second--; + } + + sec = (signed)(dt1.second + dt2.second + msec / 1000000); + dt.second = tmp = sec % 60; + + if (tmp < 0) + { + dt.second = tmp + 60; + dt2.minute--; + } + + min = (signed)(dt1.minute + dt2.minute + sec / 60); + dt.minute = tmp = min % 60; + + if (tmp < 0) + { + dt.minute = tmp + 60; + dt2.hour--; + } + + dt.hour = (signed)(dt1.hour + dt2.hour + min / 60); + + // Saturation + if (dt.hour > 838) + { + dt.hour = 838; + dt.minute = 59; + dt.second = 59; + dt.msecond = 999999; + } + else if (dt.hour < -838) + { + dt.hour = -838; + dt.minute = 59; + dt.second = 59; + dt.msecond = 999999; + } + + return *(reinterpret_cast(&dt)); +} + + } namespace funcexp @@ -182,6 +240,13 @@ int64_t Func_add_time::getDatetimeIntVal(rowgroup::Row& row, if (isNull) return -1; + // Adding a zero date to a time is always NULL + if (val1 == 0) + { + isNull = true; + return -1; + } + const string& val2 = parm[1]->data()->getStrVal(row, isNull); int sign = parm[2]->data()->getIntVal(row, isNull); DateTime dt1; @@ -254,6 +319,86 @@ int64_t Func_add_time::getDatetimeIntVal(rowgroup::Row& row, return addTime(dt1, t2); } +int64_t Func_add_time::getTimeIntVal(rowgroup::Row& row, + FunctionParm& parm, + bool& isNull, + CalpontSystemCatalog::ColType& ct) +{ + int64_t val1 = parm[0]->data()->getTimeIntVal(row, isNull); + + if (isNull) + return -1; + + const string& val2 = parm[1]->data()->getStrVal(row, isNull); + int sign = parm[2]->data()->getIntVal(row, isNull); + Time dt1; + dt1.hour = (val1 >> 40) & 0xfff; + dt1.minute = (val1 >> 32) & 0xff; + dt1.second = (val1 >> 24) & 0xff; + dt1.msecond = val1 & 0xffffff; + + int64_t time = DataConvert::stringToTime(val2); + + if (time == -1) + { + isNull = true; + return -1; + } + + Time t2 = *(reinterpret_cast(&time)); + + // MySQL TIME type range '-838:59:59' and '838:59:59' + if (t2.minute > 59 || t2.second > 59 || t2.msecond > 999999) + { + isNull = true; + return -1; + } + + int val_sign = 1; + + if (t2.day != 0 && t2.hour < 0) + { + isNull = true; + return -1; + } + else if (t2.day < 0 || t2.hour < 0) + { + val_sign = -1; + } + + if ((abs(t2.day) * 24 + abs(t2.hour)) > 838) + { + t2.hour = 838; + t2.minute = 59; + t2.second = 59; + t2.msecond = 999999; + } + else + { + t2.hour = abs(t2.day) * 24 + t2.hour; + } + + t2.day = 0; + + if (val_sign * sign < 0) + { + t2.hour = -abs(t2.hour); + t2.minute = -abs(t2.minute); + t2.second = -abs(t2.second); + t2.msecond = -abs(t2.msecond); + } + else + { + t2.hour = abs(t2.hour); + t2.minute = abs(t2.minute); + t2.second = abs(t2.second); + t2.msecond = abs(t2.msecond); + } + + return addTime(dt1, t2); +} + + } // namespace funcexp // vim:ts=4 sw=4: diff --git a/utils/funcexp/func_between.cpp b/utils/funcexp/func_between.cpp index af950cc95..f23d67826 100644 --- a/utils/funcexp/func_between.cpp +++ b/utils/funcexp/func_between.cpp @@ -156,6 +156,25 @@ inline bool getBool(rowgroup::Row& row, numericLE(val, pm[2]->data()->getDatetimeIntVal(row, isNull)); } + case execplan::CalpontSystemCatalog::TIME: + { + // Shift out unused day for compare + int64_t val = pm[0]->data()->getTimeIntVal(row, isNull) << 12; + + if (notBetween) + { + if (!numericGE(val, pm[1]->data()->getTimeIntVal(row, isNull) << 12) && !isNull) + return true; + + isNull = false; + return (!numericLE(val, pm[2]->data()->getTimeIntVal(row, isNull) << 12) && !isNull); + } + + return !isNull && + numericGE(val, pm[1]->data()->getTimeIntVal(row, isNull) << 12) && + numericLE(val, pm[2]->data()->getTimeIntVal(row, isNull) << 12); + } + case execplan::CalpontSystemCatalog::DOUBLE: case execplan::CalpontSystemCatalog::UDOUBLE: case execplan::CalpontSystemCatalog::FLOAT: @@ -247,7 +266,8 @@ CalpontSystemCatalog::ColType Func_between::operationType( FunctionParm& fp, Cal fp[i]->data()->resultType().colDataType != CalpontSystemCatalog::TEXT && fp[i]->data()->resultType().colDataType != CalpontSystemCatalog::VARCHAR) || ct.colDataType == CalpontSystemCatalog::DATE || - ct.colDataType == CalpontSystemCatalog::DATETIME) + ct.colDataType == CalpontSystemCatalog::DATETIME || + ct.colDataType == CalpontSystemCatalog::TIME) { allString = false; } @@ -275,6 +295,23 @@ CalpontSystemCatalog::ColType Func_between::operationType( FunctionParm& fp, Cal } } } + else if (op.operationType().colDataType == CalpontSystemCatalog::TIME) + { + ConstantColumn* cc = NULL; + + for (uint32_t i = 1; i < fp.size(); i++) + { + cc = dynamic_cast(fp[i]->data()); + + if (cc) + { + Result result = cc->result(); + result.intVal = dataconvert::DataConvert::timeToInt(result.strVal); + cc->result(result); + } + } + } + return ct; } diff --git a/utils/funcexp/func_bitand.cpp b/utils/funcexp/func_bitand.cpp index 89aadf39b..69429bfb8 100644 --- a/utils/funcexp/func_bitand.cpp +++ b/utils/funcexp/func_bitand.cpp @@ -141,7 +141,8 @@ int64_t Func_bitand::getIntVal(Row& row, day = 0, hour = 0, min = 0, - sec = 0; + sec = 0, + msec = 0; year = (uint32_t)((time >> 48) & 0xffff); month = (uint32_t)((time >> 44) & 0xf); @@ -149,9 +150,39 @@ int64_t Func_bitand::getIntVal(Row& row, hour = (uint32_t)((time >> 32) & 0x3f); min = (uint32_t)((time >> 26) & 0x3f); sec = (uint32_t)((time >> 20) & 0x3f); + msec = (uint32_t)(time & 0xfffff); // return (int64_t) (year*1000000000000)+(month*100000000)+(day*1000000)+(hour*10000)+(min*100)+sec; - values.push_back((month * 100000000) + (day * 1000000) + (hour * 10000) + (min * 100) + sec); + values.push_back((month * 100000000000000) + (day * 1000000000000) + (hour * 10000000000) + (min * 100000000) + (sec * 1000000) + msec); + } + break; + + case execplan::CalpontSystemCatalog::TIME: + { + int64_t time = parm[i]->data()->getTimeIntVal(row, isNull); + + int32_t hour = 0, + min = 0, + sec = 0, + msec = 0; + + // Handle negative correctly + if ((time >> 40) & 0x800) + { + hour = 0xfffff000; + } + + hour |= ((time >> 40) & 0xfff); + + if ((hour >= 0) && (time >> 63)) + hour *= -1; + + min = (uint32_t)((time >> 32) & 0xff); + sec = (uint32_t)((time >> 24) & 0xff); + msec = (uint32_t)(time & 0xffffff); + + // return (int64_t) (year*1000000000000)+(month*100000000)+(day*1000000)+(hour*10000)+(min*100)+sec; + values.push_back((hour * 10000000000) + (min * 100000000) + (sec * 1000000) + msec); } break; diff --git a/utils/funcexp/func_bitwise.cpp b/utils/funcexp/func_bitwise.cpp index 84c83f231..752c28561 100644 --- a/utils/funcexp/func_bitwise.cpp +++ b/utils/funcexp/func_bitwise.cpp @@ -143,6 +143,15 @@ bool getUIntValFromParm( } break; + case execplan::CalpontSystemCatalog::TIME: + { + int64_t time = parm->data()->getTimeIntVal(row, isNull); + + Time dt(time); + value = dt.convertToMySQLint(); + } + break; + default: { return false; diff --git a/utils/funcexp/func_case.cpp b/utils/funcexp/func_case.cpp index 3318a9ac8..4c416360c 100644 --- a/utils/funcexp/func_case.cpp +++ b/utils/funcexp/func_case.cpp @@ -52,9 +52,10 @@ inline uint64_t simple_case_cmp(Row& row, CalpontSystemCatalog::ColType& operationColType) { uint64_t i = 0; // index to the parm list - uint64_t n = parm.size() - 1; // remove expression from count of expression_i + result_i - uint64_t hasElse = n % 2; // if 1, then ELSE exist - n -= hasElse; // index to expression + uint64_t n = 0; // remove expression from count of expression_i + result_i + uint64_t hasElse = (parm.size() - 1) % 2; // if 1, then ELSE exist + uint64_t whereCount = hasElse ? (parm.size() - 2) / 2 : (parm.size() - 1) / 2; + bool foundIt = false; switch (operationColType.colDataType) { @@ -64,17 +65,19 @@ inline uint64_t simple_case_cmp(Row& row, case execplan::CalpontSystemCatalog::INT: case execplan::CalpontSystemCatalog::BIGINT: case execplan::CalpontSystemCatalog::DATE: - case execplan::CalpontSystemCatalog::DATETIME: { int64_t ev = parm[n]->data()->getIntVal(row, isNull); if (isNull) break; - for (; i < n; i += 2) + for (i = 1; i <= whereCount; i++) { if (ev == parm[i]->data()->getIntVal(row, isNull) && !isNull) + { + foundIt = true; break; + } else isNull = false; } @@ -82,6 +85,49 @@ inline uint64_t simple_case_cmp(Row& row, break; } + case execplan::CalpontSystemCatalog::DATETIME: + { + int64_t ev = parm[n]->data()->getDatetimeIntVal(row, isNull); + + if (isNull) + break; + + for (i = 1; i <= whereCount; i++) + { + if (ev == parm[i]->data()->getDatetimeIntVal(row, isNull) && !isNull) + { + foundIt = true; + break; + } + else + isNull = false; + } + + break; + } + + case execplan::CalpontSystemCatalog::TIME: + { + int64_t ev = parm[n]->data()->getTimeIntVal(row, isNull); + + if (isNull) + break; + + for (i = 1; i <= whereCount; i++) + { + if (ev == parm[i]->data()->getTimeIntVal(row, isNull) && !isNull) + { + foundIt = true; + break; + } + else + isNull = false; + } + + break; + } + + case execplan::CalpontSystemCatalog::UBIGINT: case execplan::CalpontSystemCatalog::UINT: case execplan::CalpontSystemCatalog::UMEDINT: @@ -93,10 +139,13 @@ inline uint64_t simple_case_cmp(Row& row, if (isNull) break; - for (; i < n; i += 2) + for (i = 1; i <= whereCount; i++) { if (ev == parm[i]->data()->getUintVal(row, isNull) && !isNull) + { + foundIt = true; break; + } else isNull = false; } @@ -113,11 +162,14 @@ inline uint64_t simple_case_cmp(Row& row, if (isNull) break; - for (; i < n; i += 2) + for (i = 1; i <= whereCount; i++) { //BUG 5362 if (utf8::idb_strcoll(ev.c_str(), parm[i]->data()->getStrVal(row, isNull).c_str()) == 0 && !isNull) + { + foundIt = true; break; + } else isNull = false; } @@ -133,10 +185,13 @@ inline uint64_t simple_case_cmp(Row& row, if (isNull) break; - for (; i < n; i += 2) + for (i = 1; i <= whereCount; i++) { if (ev == parm[i]->data()->getDecimalVal(row, isNull) && !isNull) + { + foundIt = true; break; + } else isNull = false; } @@ -152,10 +207,13 @@ inline uint64_t simple_case_cmp(Row& row, if (isNull) break; - for (; i < n; i += 2) + for (i = 1; i <= whereCount; i++) { if (ev == parm[i]->data()->getDoubleVal(row, isNull) && !isNull) + { + foundIt = true; break; + } else isNull = false; } @@ -171,10 +229,13 @@ inline uint64_t simple_case_cmp(Row& row, if (isNull) break; - for (; i < n; i += 2) + for (i = 1; i <= whereCount; i++) { if (ev == parm[i]->data()->getFloatVal(row, isNull) && !isNull) + { + foundIt = true; break; + } else isNull = false; } @@ -190,16 +251,24 @@ inline uint64_t simple_case_cmp(Row& row, } } - if (i == n && !hasElse) + if (!foundIt && !hasElse) isNull = true; + else if (!foundIt && hasElse && !isNull) + { + i = parm.size() - 1; + } else if (isNull && hasElse) // BUG 5110. Only way we can exit above with isNull == true is when ev is NULL - // if so and we have else condition we need to use it by setting i = n + // if so and we have else condition we need to use it by setting i = else { - i = n; + i = parm.size() - 1; isNull = false; } + if (foundIt) + { + i += whereCount; + } return i; } @@ -210,22 +279,35 @@ inline uint64_t searched_case_cmp(Row& row, bool& isNull) { uint64_t i = 0; // index to the parm list - uint64_t n = parm.size(); // count of boolean_expression_i + result_i - uint64_t hasElse = n % 2; // if 1, then ELSE exist - n -= hasElse; // index to expression + uint64_t hasElse = parm.size() % 2; // if 1, then ELSE exist + uint64_t whereCount = hasElse ? (parm.size() - 1) / 2 : parm.size() / 2; + bool foundIt = false; - for (; i < n; i += 2) + + for (i = 0; i < whereCount; i++) { if (parm[i]->getBoolVal(row, isNull)) + { + foundIt = true; break; + } } isNull = false; - if (i == n && !hasElse) + if (!foundIt && !hasElse) isNull = true; + else if (!foundIt && hasElse) + { + i = parm.size() - 1; + } - return (i == n ? i - 1 : i); + if (foundIt) + { + i += whereCount; + } + + return i; } @@ -233,7 +315,6 @@ CalpontSystemCatalog::ColType caseOperationType(FunctionParm& fp, CalpontSystemCatalog::ColType& resultType, bool simpleCase) { - // ... expression_i + result_i + ... [[expression] + result_N] FunctionParm::size_type n = fp.size(); if (simpleCase) // simple case has an expression @@ -244,6 +325,9 @@ CalpontSystemCatalog::ColType caseOperationType(FunctionParm& fp, if (hasElse) --n; // n now is an even number + uint64_t parmCount = hasElse ? (fp.size() - 2) : (fp.size() - 1); + uint64_t whereCount = hasElse ? (fp.size() - 2 + simpleCase) / 2 : (fp.size() - 1) / 2 + simpleCase; + idbassert((n % 2) == 0); bool allStringO = true; @@ -255,10 +339,10 @@ CalpontSystemCatalog::ColType caseOperationType(FunctionParm& fp, CalpontSystemCatalog::ColType rct = resultType; bool operation = true; - for (uint64_t i = 0; i <= n; i++) + for (uint64_t i = 0; i <= parmCount; i++) { // operation or result type - operation = ((i % 2) == 0); + operation = ((i > 0) && (i <= whereCount)); // the result type of ELSE, if exists. if (i == n) @@ -353,8 +437,9 @@ namespace funcexp // END // // simple CASE parm order: -// expression1 result1 expression2 result2 ... expression [resultN] +// expression condition1 condition2 ... result1 result2 ... [resultN] // +// Note that this order changed in 10.2.14, see MCOL-1341 CalpontSystemCatalog::ColType Func_simple_case::operationType(FunctionParm& fp, CalpontSystemCatalog::ColType& resultType) { @@ -372,7 +457,7 @@ bool Func_simple_case::getBoolVal(Row& row, if (isNull) return joblist::BIGINTNULL; - return parm[i + 1]->data()->getBoolVal(row, isNull); + return parm[i]->data()->getBoolVal(row, isNull); } @@ -386,7 +471,7 @@ int64_t Func_simple_case::getIntVal(Row& row, if (isNull) return joblist::BIGINTNULL; - return parm[i + 1]->data()->getIntVal(row, isNull); + return parm[i]->data()->getIntVal(row, isNull); } @@ -400,7 +485,7 @@ string Func_simple_case::getStrVal(Row& row, if (isNull) return string(""); - return parm[i + 1]->data()->getStrVal(row, isNull); + return parm[i]->data()->getStrVal(row, isNull); } @@ -414,7 +499,7 @@ IDB_Decimal Func_simple_case::getDecimalVal(Row& row, if (isNull) return IDB_Decimal(); // need a null value for IDB_Decimal?? - return parm[i + 1]->data()->getDecimalVal(row, isNull); + return parm[i]->data()->getDecimalVal(row, isNull); } @@ -428,7 +513,7 @@ double Func_simple_case::getDoubleVal(Row& row, if (isNull) return doubleNullVal(); - return parm[i + 1]->data()->getDoubleVal(row, isNull); + return parm[i]->data()->getDoubleVal(row, isNull); } @@ -442,7 +527,7 @@ int32_t Func_simple_case::getDateIntVal(rowgroup::Row& row, if (isNull) return joblist::DATENULL; - return parm[i + 1]->data()->getDateIntVal(row, isNull); + return parm[i]->data()->getDateIntVal(row, isNull); } @@ -456,10 +541,23 @@ int64_t Func_simple_case::getDatetimeIntVal(rowgroup::Row& row, if (isNull) return joblist::DATETIMENULL; - return parm[i + 1]->data()->getDatetimeIntVal(row, isNull); + return parm[i]->data()->getDatetimeIntVal(row, isNull); } +int64_t Func_simple_case::getTimeIntVal(rowgroup::Row& row, + FunctionParm& parm, + bool& isNull, + execplan::CalpontSystemCatalog::ColType& op_ct) +{ + uint64_t i = simple_case_cmp(row, parm, isNull, op_ct); + + if (isNull) + return joblist::TIMENULL; + + return parm[i]->data()->getTimeIntVal(row, isNull); +} + // searched CASE: // SELECT CASE @@ -470,8 +568,10 @@ int64_t Func_simple_case::getDatetimeIntVal(rowgroup::Row& row, // END // // searched CASE parm order: -// boolean_expression1 result1 boolean_expression2 result2 ... [resultN] +// boolean_expression1 boolean_expression2 ... result1 result2 ... [resultN] // +// Note that this order changed in 10.2.14, see MCOL-1341 + CalpontSystemCatalog::ColType Func_searched_case::operationType(FunctionParm& fp, CalpontSystemCatalog::ColType& resultType) { // operation type not used by this functor. @@ -489,7 +589,15 @@ bool Func_searched_case::getBoolVal(Row& row, if (isNull) return joblist::BIGINTNULL; - return parm[i + 1]->data()->getBoolVal(row, isNull); + ParseTree* lop = parm[i]->left(); + ParseTree* rop = parm[i]->right(); + + if (lop && rop) + { + return (reinterpret_cast(parm[i]->data()))->getBoolVal(row, isNull, lop, rop); + } + + return parm[i]->data()->getBoolVal(row, isNull); } int64_t Func_searched_case::getIntVal(Row& row, @@ -502,7 +610,7 @@ int64_t Func_searched_case::getIntVal(Row& row, if (isNull) return joblist::BIGINTNULL; - return parm[i + 1]->data()->getIntVal(row, isNull); + return parm[i]->data()->getIntVal(row, isNull); } @@ -516,7 +624,7 @@ string Func_searched_case::getStrVal(Row& row, if (isNull) return string(""); - return parm[i + 1]->data()->getStrVal(row, isNull); + return parm[i]->data()->getStrVal(row, isNull); } @@ -530,7 +638,7 @@ IDB_Decimal Func_searched_case::getDecimalVal(Row& row, if (isNull) return IDB_Decimal(); // need a null value for IDB_Decimal?? - return parm[i + 1]->data()->getDecimalVal(row, isNull); + return parm[i]->data()->getDecimalVal(row, isNull); } @@ -544,7 +652,7 @@ double Func_searched_case::getDoubleVal(Row& row, if (isNull) return doubleNullVal(); - return parm[i + 1]->data()->getDoubleVal(row, isNull); + return parm[i]->data()->getDoubleVal(row, isNull); } @@ -558,7 +666,7 @@ int32_t Func_searched_case::getDateIntVal(rowgroup::Row& row, if (isNull) return joblist::DATENULL; - return parm[i + 1]->data()->getDateIntVal(row, isNull); + return parm[i]->data()->getDateIntVal(row, isNull); } @@ -572,9 +680,22 @@ int64_t Func_searched_case::getDatetimeIntVal(rowgroup::Row& row, if (isNull) return joblist::DATETIMENULL; - return parm[i + 1]->data()->getDatetimeIntVal(row, isNull); + return parm[i]->data()->getDatetimeIntVal(row, isNull); } +int64_t Func_searched_case::getTimeIntVal(rowgroup::Row& row, + FunctionParm& parm, + bool& isNull, + execplan::CalpontSystemCatalog::ColType& op_ct) +{ + uint64_t i = simple_case_cmp(row, parm, isNull, op_ct); + + if (isNull) + return joblist::TIMENULL; + + return parm[i]->data()->getTimeIntVal(row, isNull); +} + } // namespace funcexp // vim:ts=4 sw=4: diff --git a/utils/funcexp/func_cast.cpp b/utils/funcexp/func_cast.cpp index 9d2423293..3542e341d 100644 --- a/utils/funcexp/func_cast.cpp +++ b/utils/funcexp/func_cast.cpp @@ -191,6 +191,15 @@ int64_t Func_cast_signed::getIntVal(Row& row, } break; + case execplan::CalpontSystemCatalog::TIME: + { + int64_t time = parm[0]->data()->getTimeIntVal(row, isNull); + + Time dt(time); + return dt.convertToMySQLint(); + } + break; + default: { std::ostringstream oss; @@ -313,6 +322,15 @@ uint64_t Func_cast_unsigned::getUintVal(Row& row, } break; + case execplan::CalpontSystemCatalog::TIME: + { + int64_t time = parm[0]->data()->getTimeIntVal(row, isNull); + + Time dt(time); + return dt.convertToMySQLint(); + } + break; + default: { std::ostringstream oss; @@ -805,6 +823,85 @@ int64_t Func_cast_datetime::getDatetimeIntVal(rowgroup::Row& row, return -1; } + +int64_t Func_cast_datetime::getTimeIntVal(rowgroup::Row& row, + FunctionParm& parm, + bool& isNull, + execplan::CalpontSystemCatalog::ColType& operationColType) +{ + int64_t val; + + switch (parm[0]->data()->resultType().colDataType) + { + case execplan::CalpontSystemCatalog::BIGINT: + case execplan::CalpontSystemCatalog::INT: + case execplan::CalpontSystemCatalog::MEDINT: + case execplan::CalpontSystemCatalog::TINYINT: + case execplan::CalpontSystemCatalog::SMALLINT: + case execplan::CalpontSystemCatalog::UBIGINT: + case execplan::CalpontSystemCatalog::UINT: + case execplan::CalpontSystemCatalog::UMEDINT: + case execplan::CalpontSystemCatalog::UTINYINT: + case execplan::CalpontSystemCatalog::USMALLINT: + { + val = dataconvert::DataConvert::intToTime(parm[0]->data()->getIntVal(row, isNull)); + + if (val == -1) + isNull = true; + else + return val; + + break; + } + + case execplan::CalpontSystemCatalog::DECIMAL: + case execplan::CalpontSystemCatalog::UDECIMAL: + { + val = dataconvert::DataConvert::intToTime(parm[0]->data()->getIntVal(row, isNull)); + + if (val == -1) + isNull = true; + else + return val; + + break; + } + + case execplan::CalpontSystemCatalog::VARCHAR: + case execplan::CalpontSystemCatalog::CHAR: + case execplan::CalpontSystemCatalog::TEXT: + { + val = dataconvert::DataConvert::stringToTime(parm[0]->data()->getStrVal(row, isNull)); + + if (val == -1) + isNull = true; + else + return val; + + break; + } + + case execplan::CalpontSystemCatalog::DATE: + { + return parm[0]->data()->getTimeIntVal(row, isNull); + } + + case execplan::CalpontSystemCatalog::DATETIME: + { + return parm[0]->data()->getTimeIntVal(row, isNull); + } + + default: + { + isNull = true; + } + } + + return -1; +} + + + // // Func_cast_decimal // @@ -1138,6 +1235,25 @@ IDB_Decimal Func_cast_decimal::getDecimalVal(Row& row, } break; + case execplan::CalpontSystemCatalog::TIME: + { + int32_t s = 0; + + string value = dataconvert::DataConvert::timeToString1(parm[0]->data()->getTimeIntVal(row, isNull)); + + //strip off micro seconds + string date = value.substr(0, 14); + + int64_t x = atoll(date.c_str()); + + if (!isNull) + { + decimal.value = x; + decimal.scale = s; + } + } + break; + default: { std::ostringstream oss; diff --git a/utils/funcexp/func_ceil.cpp b/utils/funcexp/func_ceil.cpp index 7c1cafdfb..7c2f7f571 100644 --- a/utils/funcexp/func_ceil.cpp +++ b/utils/funcexp/func_ceil.cpp @@ -152,6 +152,15 @@ int64_t Func_ceil::getIntVal(Row& row, } break; + case CalpontSystemCatalog::TIME: + { + Time dt(parm[0]->data()->getTimeIntVal(row, isNull)); + + if (!isNull) + ret = dt.convertToMySQLint(); + } + break; + default: { std::ostringstream oss; @@ -233,6 +242,15 @@ uint64_t Func_ceil::getUintVal(Row& row, } break; + case CalpontSystemCatalog::TIME: + { + Time dt(parm[0]->data()->getTimeIntVal(row, isNull)); + + if (!isNull) + ret = dt.convertToMySQLint(); + } + break; + default: { std::ostringstream oss; diff --git a/utils/funcexp/func_char_length.cpp b/utils/funcexp/func_char_length.cpp index 9421554f8..b47afd58a 100644 --- a/utils/funcexp/func_char_length.cpp +++ b/utils/funcexp/func_char_length.cpp @@ -101,6 +101,12 @@ int64_t Func_char_length::getIntVal(rowgroup::Row& row, return (int64_t)date.size(); } + case execplan::CalpontSystemCatalog::TIME: + { + string date = dataconvert::DataConvert::timeToString(parm[0]->data()->getTimeIntVal(row, isNull)); + return (int64_t)date.size(); + } + default: { std::ostringstream oss; diff --git a/utils/funcexp/func_coalesce.cpp b/utils/funcexp/func_coalesce.cpp index 33a1312bc..e03040350 100644 --- a/utils/funcexp/func_coalesce.cpp +++ b/utils/funcexp/func_coalesce.cpp @@ -139,6 +139,30 @@ int64_t Func_coalesce::getDatetimeIntVal(rowgroup::Row& row, return val; } +int64_t Func_coalesce::getTimeIntVal(rowgroup::Row& row, + FunctionParm& parm, + bool& isNull, + CalpontSystemCatalog::ColType& ct) +{ + int64_t val = 0; + + for (uint32_t i = 0; i < parm.size(); i++) + { + val = parm[i]->data()->getTimeIntVal(row, isNull); + + if (isNull) + { + isNull = false; + continue; + } + + return val; + } + + isNull = true; + return val; +} + double Func_coalesce::getDoubleVal(rowgroup::Row& row, FunctionParm& parm, bool& isNull, diff --git a/utils/funcexp/func_extract.cpp b/utils/funcexp/func_extract.cpp index 08a545fde..332c05441 100644 --- a/utils/funcexp/func_extract.cpp +++ b/utils/funcexp/func_extract.cpp @@ -131,6 +131,95 @@ long long dateGet( uint64_t time, IntervalColumn::interval_type unit, bool dateT throw runtime_error("unit type is not supported: " + unit); }; } + +long long timeGet( uint64_t time, IntervalColumn::interval_type unit ) +{ + int32_t hour = 0, + min = 0, + sec = 0, + msec = 0, + day = 0; + + min = (int32_t)((time >> 32) & 0xff); + sec = (int32_t)((time >> 24) & 0xff); + msec = (int32_t)((time & 0xfffff)); + + // If negative, mask so it doesn't turn positive + int64_t mask = 0; + + if ((time >> 40) & 0x800) + mask = 0xfffffffffffff000; + + hour = mask | ((time >> 40) & 0xfff); + + if ((hour >= 0) && (time >> 63)) + hour *= -1; + + // Always positive! + day = abs(hour / 24); + + switch ( unit ) + { + case IntervalColumn::INTERVAL_YEAR: + case IntervalColumn::INTERVAL_MONTH: + return 0; + + case IntervalColumn::INTERVAL_DAY: + return day; + + case IntervalColumn::INTERVAL_HOUR: + return hour; + + case IntervalColumn::INTERVAL_MINUTE: + return min; + + case IntervalColumn::INTERVAL_SECOND: + return sec; + + case IntervalColumn::INTERVAL_MICROSECOND: + return msec; + + case IntervalColumn::INTERVAL_QUARTER: + case IntervalColumn::INTERVAL_WEEK: + case IntervalColumn::INTERVAL_YEAR_MONTH: + return 0; + + case IntervalColumn::INTERVAL_DAY_HOUR: + return (day * 100) + hour; + + case IntervalColumn::INTERVAL_DAY_MINUTE: + return (day * 10000) + (hour * 100) + min; + + case IntervalColumn::INTERVAL_DAY_SECOND: + return (day * 1000000) + (hour * 10000) + (min * 100) + sec; + + case IntervalColumn::INTERVAL_HOUR_MINUTE: + return (hour * 100) + min; + + case IntervalColumn::INTERVAL_HOUR_SECOND: + return (hour * 10000) + (min * 100) + sec; + + case IntervalColumn::INTERVAL_MINUTE_SECOND: + return (min * 100) + sec; + + case IntervalColumn::INTERVAL_DAY_MICROSECOND: + return (((day * 1000000) + (hour * 10000) + (min * 100) + sec) * 1000000) + msec; + + case IntervalColumn::INTERVAL_HOUR_MICROSECOND: + return (((hour * 10000) + (min * 100) + sec) * 1000000) + msec; + + case IntervalColumn::INTERVAL_MINUTE_MICROSECOND: + return (((min * 100) + sec) * 1000000) + msec; + + case IntervalColumn::INTERVAL_SECOND_MICROSECOND: + return (sec * 1000000) + msec; + + default: + throw runtime_error("unit type is not supported: " + unit); + }; +} + + } namespace funcexp @@ -148,6 +237,7 @@ int64_t Func_extract::getIntVal(rowgroup::Row& row, { IntervalColumn::interval_type unit = static_cast(parm[1]->data()->getIntVal(row, isNull)); uint64_t time; + bool isTime = false; //@bug4678 handle conversion from non date/datetime datatype switch (parm[0]->data()->resultType().colDataType) @@ -157,6 +247,11 @@ int64_t Func_extract::getIntVal(rowgroup::Row& row, time = parm[0]->data()->getDatetimeIntVal(row, isNull); break; + case CalpontSystemCatalog::TIME: + time = parm[0]->data()->getTimeIntVal(row, isNull); + isTime = true; + break; + case CalpontSystemCatalog::VARCHAR: case CalpontSystemCatalog::CHAR: case CalpontSystemCatalog::TEXT: @@ -181,7 +276,12 @@ int64_t Func_extract::getIntVal(rowgroup::Row& row, time = parm[0]->data()->getIntVal(row, isNull); } - long long value = dateGet( time, unit, false ); + long long value; + + if (isTime) + value = timeGet( time, unit ); + else + value = dateGet( time, unit, false ); return value; } diff --git a/utils/funcexp/func_floor.cpp b/utils/funcexp/func_floor.cpp index ad47dc988..eb05b405f 100644 --- a/utils/funcexp/func_floor.cpp +++ b/utils/funcexp/func_floor.cpp @@ -151,6 +151,19 @@ int64_t Func_floor::getIntVal(Row& row, } break; + case execplan::CalpontSystemCatalog::TIME: + { + string str = + DataConvert::timeToString1(parm[0]->data()->getTimeIntVal(row, isNull)); + + // strip off micro seconds + str = str.substr(0, 14); + + if (!isNull) + ret = atoll(str.c_str()); + } + break; + default: { std::ostringstream oss; @@ -236,6 +249,19 @@ uint64_t Func_floor::getUintVal(Row& row, } break; + case execplan::CalpontSystemCatalog::TIME: + { + string str = + DataConvert::timeToString1(parm[0]->data()->getTimeIntVal(row, isNull)); + + // strip off micro seconds + str = str.substr(0, 14); + + if (!isNull) + ret = strtoull(str.c_str(), NULL, 10); + } + break; + default: { std::ostringstream oss; diff --git a/utils/funcexp/func_from_unixtime.cpp b/utils/funcexp/func_from_unixtime.cpp index 183a75a56..9968ba1f0 100644 --- a/utils/funcexp/func_from_unixtime.cpp +++ b/utils/funcexp/func_from_unixtime.cpp @@ -148,6 +148,22 @@ int64_t Func_from_unixtime::getDatetimeIntVal(rowgroup::Row& row, return *reinterpret_cast(&dt); } +int64_t Func_from_unixtime::getTimeIntVal(rowgroup::Row& row, + FunctionParm& parm, + bool& isNull, + CalpontSystemCatalog::ColType& ct) +{ + DateTime dt = getDateTime(row, parm, isNull); + + if (*reinterpret_cast(&dt) == 0) + { + isNull = true; + return 0; + } + + return *reinterpret_cast(&dt); +} + int64_t Func_from_unixtime::getIntVal(rowgroup::Row& row, FunctionParm& parm, bool& isNull, diff --git a/utils/funcexp/func_greatest.cpp b/utils/funcexp/func_greatest.cpp index 8309456ae..de63c283e 100644 --- a/utils/funcexp/func_greatest.cpp +++ b/utils/funcexp/func_greatest.cpp @@ -206,6 +206,31 @@ int64_t Func_greatest::getDatetimeIntVal(rowgroup::Row& row, return greatestStr; } +int64_t Func_greatest::getTimeIntVal(rowgroup::Row& row, + FunctionParm& fp, + bool& isNull, + execplan::CalpontSystemCatalog::ColType& ct) +{ + // Strip off unused day + int64_t greatestStr = fp[0]->data()->getTimeIntVal(row, isNull); + + int64_t str = greatestStr << 12; + + for (uint32_t i = 1; i < fp.size(); i++) + { + int64_t str1 = fp[i]->data()->getTimeIntVal(row, isNull); + int64_t str2 = str1 << 12; + + if ( str < str2 ) + { + greatestStr = str1; + str = str2; + } + } + + return greatestStr; +} + } // namespace funcexp // vim:ts=4 sw=4: diff --git a/utils/funcexp/func_hour.cpp b/utils/funcexp/func_hour.cpp index f5058a084..685a264db 100644 --- a/utils/funcexp/func_hour.cpp +++ b/utils/funcexp/func_hour.cpp @@ -48,6 +48,7 @@ int64_t Func_hour::getIntVal(rowgroup::Row& row, CalpontSystemCatalog::ColType& op_ct) { int64_t val = 0; + bool isTime = false; switch (parm[0]->data()->resultType().colDataType) { @@ -108,6 +109,13 @@ int64_t Func_hour::getIntVal(rowgroup::Row& row, break; } + case execplan::CalpontSystemCatalog::TIME: + { + isTime = true; + val = parm[0]->data()->getTimeIntVal(row, isNull); + break; + } + default: { isNull = true; @@ -117,10 +125,31 @@ int64_t Func_hour::getIntVal(rowgroup::Row& row, if (isNull) return -1; - if ( val < 1000000000 ) - return 0; + if (isTime) + { + // If negative, mask so it doesn't turn positive + bool isNeg = false; + int64_t mask = 0; - return (uint32_t)((val >> 32) & 0x3f); + if ((val >> 40) & 0x800) + mask = 0xfffffffffffff000; + + if (!mask && (val >> 63)) + { + isNeg = true; + } + + val = mask | ((val >> 40) & 0xfff); + + if (isNeg) + val *= -1; + } + else + { + val = (val >> 32) & 0x3f; + } + + return val; } diff --git a/utils/funcexp/func_if.cpp b/utils/funcexp/func_if.cpp index 378c6d74a..ad65d10c9 100644 --- a/utils/funcexp/func_if.cpp +++ b/utils/funcexp/func_if.cpp @@ -75,6 +75,7 @@ bool boolVal(SPTP& parm, Row& row) case CalpontSystemCatalog::UINT: case CalpontSystemCatalog::DATE: case CalpontSystemCatalog::DATETIME: + case CalpontSystemCatalog::TIME: default: ret = (parm->data()->getIntVal(row, isNull) != 0); } @@ -219,6 +220,19 @@ int64_t Func_if::getDatetimeIntVal(Row& row, } } - +int64_t Func_if::getTimeIntVal(Row& row, + FunctionParm& parm, + bool& isNull, + CalpontSystemCatalog::ColType&) +{ + if (boolVal(parm[0], row)) + { + return parm[1]->data()->getTimeIntVal(row, isNull); + } + else + { + return parm[2]->data()->getTimeIntVal(row, isNull); + } +} } // namespace funcexp // vim:ts=4 sw=4: diff --git a/utils/funcexp/func_ifnull.cpp b/utils/funcexp/func_ifnull.cpp index 41db7e3b4..4b8606d4e 100644 --- a/utils/funcexp/func_ifnull.cpp +++ b/utils/funcexp/func_ifnull.cpp @@ -168,6 +168,25 @@ int64_t Func_ifnull::getDatetimeIntVal(Row& row, return r; } +int64_t Func_ifnull::getTimeIntVal(Row& row, + FunctionParm& parm, + bool& isNull, + CalpontSystemCatalog::ColType&) +{ + if (isNull) + return 0; + + int64_t r = parm[0]->data()->getTimeIntVal(row, isNull); + + if (isNull) + { + isNull = false; + return parm[1]->data()->getTimeIntVal(row, isNull); + } + + return r; +} + bool Func_ifnull::getBoolVal(Row& row, FunctionParm& parm, bool& isNull, diff --git a/utils/funcexp/func_in.cpp b/utils/funcexp/func_in.cpp index 404c0f39f..e271fa0f7 100644 --- a/utils/funcexp/func_in.cpp +++ b/utils/funcexp/func_in.cpp @@ -158,6 +158,27 @@ inline bool getBoolForIn(rowgroup::Row& row, return false; } + case execplan::CalpontSystemCatalog::TIME: + { + int64_t val = pm[0]->data()->getTimeIntVal(row, isNull); + + if (isNull) + return false; + + for (uint32_t i = 1; i < pm.size(); i++) + { + isNull = false; + + if ( val == pm[i]->data()->getTimeIntVal(row, isNull) && !isNull ) + return true; + + if (isNull && isNotIn) + return true; // will be reversed to false by the caller + } + + return false; + } + case execplan::CalpontSystemCatalog::DOUBLE: case execplan::CalpontSystemCatalog::UDOUBLE: case execplan::CalpontSystemCatalog::FLOAT: diff --git a/utils/funcexp/func_inet_aton.cpp b/utils/funcexp/func_inet_aton.cpp index bd2d2a2d7..9692cf850 100644 --- a/utils/funcexp/func_inet_aton.cpp +++ b/utils/funcexp/func_inet_aton.cpp @@ -222,6 +222,26 @@ int64_t Func_inet_aton::getDatetimeIntVal(rowgroup::Row& row, return iValue; } +int64_t Func_inet_aton::getTimeIntVal(rowgroup::Row& row, + FunctionParm& fp, + bool& isNull, + execplan::CalpontSystemCatalog::ColType& op_ct) +{ + int64_t iValue = joblist::TIMENULL; + + const std::string& sValue = fp[0]->data()->getStrVal(row, isNull); + + if (!isNull) + { + int64_t iVal = convertAton( sValue, isNull ); + + if (!isNull) + iValue = iVal; + } + + return iValue; +} + //------------------------------------------------------------------------------ // Convert an ascii IP address string to it's integer equivalent. // isNull is set to true if the IP address string has invalid content. diff --git a/utils/funcexp/func_inet_ntoa.cpp b/utils/funcexp/func_inet_ntoa.cpp index 84c476eb6..ff8c889ed 100644 --- a/utils/funcexp/func_inet_ntoa.cpp +++ b/utils/funcexp/func_inet_ntoa.cpp @@ -256,6 +256,20 @@ int64_t Func_inet_ntoa::getDatetimeIntVal(rowgroup::Row& row, return iValue; } +int64_t Func_inet_ntoa::getTimeIntVal(rowgroup::Row& row, + FunctionParm& fp, + bool& isNull, + execplan::CalpontSystemCatalog::ColType& op_ct) +{ +// std::cout << "In Func_inet_ntoa::getTimeVal" << std::endl; + +// int64t iValue = fp[0]->data()->getTimeIntVal(row, isNull); + int64_t iValue = joblist::TIMENULL; + isNull = true; + + return iValue; +} + //------------------------------------------------------------------------------ // Convert an integer IP address to its equivalent IP address string. // Source code based on MySQL source (Item_func_inet_ntoa() in item_strfunc.cc). diff --git a/utils/funcexp/func_least.cpp b/utils/funcexp/func_least.cpp index e273a4618..9ba5c98de 100644 --- a/utils/funcexp/func_least.cpp +++ b/utils/funcexp/func_least.cpp @@ -182,6 +182,30 @@ int64_t Func_least::getDatetimeIntVal(rowgroup::Row& row, return leastStr; } +int64_t Func_least::getTimeIntVal(rowgroup::Row& row, + FunctionParm& fp, + bool& isNull, + execplan::CalpontSystemCatalog::ColType& op_ct) +{ + // Strip off unused day + int64_t leastStr = fp[0]->data()->getTimeIntVal(row, isNull); + + int64_t str = leastStr << 12; + + for (uint32_t i = 1; i < fp.size(); i++) + { + int64_t str1 = fp[i]->data()->getTimeIntVal(row, isNull); + int64_t str2 = str1 << 12; + + if ( str > str2 ) + { + leastStr = str1; + str = str2; + } + } + + return leastStr; +} } // namespace funcexp diff --git a/utils/funcexp/func_math.cpp b/utils/funcexp/func_math.cpp index 8e5b84def..8592ec8b0 100644 --- a/utils/funcexp/func_math.cpp +++ b/utils/funcexp/func_math.cpp @@ -154,6 +154,20 @@ double Func_acos::getDoubleVal(Row& row, } break; + case execplan::CalpontSystemCatalog::TIME: + { + int64_t value = parm[0]->data()->getTimeIntVal(row, isNull); + + if (isNull || (value < -1.0 || value > 1.0)) + { + isNull = true; + return doubleNullVal(); + } + + return acos((double)value); + } + break; + default: { std::ostringstream oss; @@ -244,6 +258,20 @@ double Func_asin::getDoubleVal(Row& row, } break; + case execplan::CalpontSystemCatalog::TIME: + { + int64_t value = parm[0]->data()->getTimeIntVal(row, isNull); + + if (isNull || (value < -1.0 || value > 1.0)) + { + isNull = true; + return doubleNullVal(); + } + + return asin((double)value); + } + break; + default: { std::ostringstream oss; @@ -373,6 +401,34 @@ double Func_atan::getDoubleVal(Row& row, } break; + + case execplan::CalpontSystemCatalog::TIME: + { + int64_t value = parm[0]->data()->getTimeIntVal(row, isNull); + + if (isNull) + { + isNull = true; + return doubleNullVal(); + } + + if (parm.size() > 1 ) + { + double value2 = parm[1]->data()->getDoubleVal(row, isNull); + + if (isNull) + { + isNull = true; + return doubleNullVal(); + } + + return atan2(value, value2); + } + + return atan((double)value); + } + break; + default: { std::ostringstream oss; @@ -462,6 +518,20 @@ double Func_cos::getDoubleVal(Row& row, } break; + case execplan::CalpontSystemCatalog::TIME: + { + int64_t value = parm[0]->data()->getTimeIntVal(row, isNull); + + if (isNull) + { + isNull = true; + return doubleNullVal(); + } + + return cos((double)value); + } + break; + default: { std::ostringstream oss; @@ -578,6 +648,30 @@ double Func_cot::getDoubleVal(Row& row, } break; + case execplan::CalpontSystemCatalog::TIME: + { + int64_t value = parm[0]->data()->getTimeIntVal(row, isNull); + + if (value == 0) + { + Message::Args args; + args.add("cot"); + args.add((uint64_t)value); + unsigned errcode = ERR_FUNC_OUT_OF_RANGE_RESULT; + throw IDBExcept(IDBErrorInfo::instance()->errorMsg(errcode, args), errcode); + } + + if (isNull) + { + isNull = true; + return doubleNullVal(); + } + + return 1.0 / tan((double)value); + } + break; + + default: { std::ostringstream oss; @@ -703,6 +797,33 @@ double Func_log::getDoubleVal(Row& row, } break; + case execplan::CalpontSystemCatalog::TIME: + { + int64_t value = parm[0]->data()->getTimeIntVal(row, isNull); + + if (isNull || value <= 0.0) + { + isNull = true; + return doubleNullVal(); + } + + if (parm.size() > 1 ) + { + double value2 = parm[1]->data()->getDoubleVal(row, isNull); + + if (isNull || (value2 <= 0.0 || value == 1.0) ) + { + isNull = true; + return doubleNullVal(); + } + + return log(value2) / log((double)value); + } + + return log((double)value); + } + break; + case execplan::CalpontSystemCatalog::VARCHAR: case execplan::CalpontSystemCatalog::CHAR: case execplan::CalpontSystemCatalog::TEXT: @@ -797,6 +918,20 @@ double Func_log2::getDoubleVal(Row& row, } break; + case execplan::CalpontSystemCatalog::TIME: + { + int64_t value = parm[0]->data()->getTimeIntVal(row, isNull); + + if (isNull || value <= 0.0) + { + isNull = true; + return doubleNullVal(); + } + + return log2(value); + } + break; + case execplan::CalpontSystemCatalog::VARCHAR: case execplan::CalpontSystemCatalog::CHAR: case execplan::CalpontSystemCatalog::TEXT: @@ -891,6 +1026,20 @@ double Func_log10::getDoubleVal(Row& row, } break; + case execplan::CalpontSystemCatalog::TIME: + { + int64_t value = parm[0]->data()->getTimeIntVal(row, isNull); + + if (isNull || value <= 0.0) + { + isNull = true; + return doubleNullVal(); + } + + return log10((double)value); + } + break; + case execplan::CalpontSystemCatalog::VARCHAR: case execplan::CalpontSystemCatalog::CHAR: case execplan::CalpontSystemCatalog::TEXT: @@ -988,6 +1137,20 @@ double Func_sin::getDoubleVal(Row& row, } break; + case execplan::CalpontSystemCatalog::TIME: + { + int64_t value = parm[0]->data()->getTimeIntVal(row, isNull); + + if (isNull) + { + isNull = true; + return doubleNullVal(); + } + + return sin((double)value); + } + break; + default: { std::ostringstream oss; @@ -1077,6 +1240,20 @@ double Func_sqrt::getDoubleVal(Row& row, } break; + case execplan::CalpontSystemCatalog::TIME: + { + int64_t value = parm[0]->data()->getTimeIntVal(row, isNull); + + if (isNull || value < 0) + { + isNull = true; + return doubleNullVal(); + } + + return sqrt((double)value); + } + break; + default: { std::ostringstream oss; @@ -1166,6 +1343,20 @@ double Func_tan::getDoubleVal(Row& row, } break; + case execplan::CalpontSystemCatalog::TIME: + { + int64_t value = parm[0]->data()->getTimeIntVal(row, isNull); + + if (isNull) + { + isNull = true; + return doubleNullVal(); + } + + return tan((double)value); + } + break; + default: { std::ostringstream oss; @@ -1253,6 +1444,12 @@ string Func_format::getStrVal(Row& row, } break; + case execplan::CalpontSystemCatalog::TIME: + { + value = dataconvert::DataConvert::timeToString1(parm[0]->data()->getTimeIntVal(row, isNull)); + } + break; + case execplan::CalpontSystemCatalog::DECIMAL: case execplan::CalpontSystemCatalog::UDECIMAL: { @@ -1474,6 +1671,20 @@ double Func_radians::getDoubleVal(Row& row, } break; + case execplan::CalpontSystemCatalog::TIME: + { + int64_t value = parm[0]->data()->getTimeIntVal(row, isNull); + + if (isNull) + { + isNull = true; + return doubleNullVal(); + } + + return radians((double)value); + } + break; + default: { std::ostringstream oss; @@ -1563,6 +1774,20 @@ double Func_degrees::getDoubleVal(Row& row, } break; + case execplan::CalpontSystemCatalog::TIME: + { + int64_t value = parm[0]->data()->getTimeIntVal(row, isNull); + + if (isNull) + { + isNull = true; + return doubleNullVal(); + } + + return degrees((double)value); + } + break; + default: { std::ostringstream oss; diff --git a/utils/funcexp/func_microsecond.cpp b/utils/funcexp/func_microsecond.cpp index e0a8e6d11..f292e26a5 100644 --- a/utils/funcexp/func_microsecond.cpp +++ b/utils/funcexp/func_microsecond.cpp @@ -63,6 +63,11 @@ int64_t Func_microsecond::getIntVal(rowgroup::Row& row, microSecond = (uint32_t)((val & 0xfffff)); break; + case CalpontSystemCatalog::TIME: + val = parm[0]->data()->getIntVal(row, isNull); + microSecond = (uint32_t)((val & 0xffffff)); + break; + case CalpontSystemCatalog::CHAR: case CalpontSystemCatalog::TEXT: case CalpontSystemCatalog::VARCHAR: diff --git a/utils/funcexp/func_minute.cpp b/utils/funcexp/func_minute.cpp index f70af29e3..299dd6ae2 100644 --- a/utils/funcexp/func_minute.cpp +++ b/utils/funcexp/func_minute.cpp @@ -102,6 +102,7 @@ int64_t Func_minute::getIntVal(rowgroup::Row& row, break; } + case execplan::CalpontSystemCatalog::TIME: case execplan::CalpontSystemCatalog::DATETIME: { val = parm[0]->data()->getDatetimeIntVal(row, isNull); diff --git a/utils/funcexp/func_nullif.cpp b/utils/funcexp/func_nullif.cpp index 7d1647d1d..a268b0ea1 100644 --- a/utils/funcexp/func_nullif.cpp +++ b/utils/funcexp/func_nullif.cpp @@ -127,8 +127,11 @@ int64_t Func_nullif::getIntVal(rowgroup::Row& row, { exp2 = parm[1]->data()->getDatetimeIntVal(row, isNull); - if (parm[0]->data()->resultType().colDataType == - execplan::CalpontSystemCatalog::DATE) + if ((parm[0]->data()->resultType().colDataType == + execplan::CalpontSystemCatalog::DATE) || + (parm[0]->data()->resultType().colDataType == + execplan::CalpontSystemCatalog::DATETIME)) + { // NULLIF arg0 is DATE, arg1 is DATETIME, // Upgrade arg1 to datetime @@ -155,6 +158,40 @@ int64_t Func_nullif::getIntVal(rowgroup::Row& row, break; } + case execplan::CalpontSystemCatalog::TIME: + { + exp2 = parm[1]->data()->getTimeIntVal(row, isNull); + + if ((parm[0]->data()->resultType().colDataType == + execplan::CalpontSystemCatalog::DATETIME) || + (parm[0]->data()->resultType().colDataType == + execplan::CalpontSystemCatalog::TIME)) + { + // NULLIF arg0 is DATETIME, arg1 is TIME, + // Upgrade arg1 to time + // When comparing exp1 as a Date, we can't simply promote. We have + // to be careful of the return value in case of not null return. + int64_t exp1 = parm[0]->data()->getTimeIntVal(row, isNull); + + if ( exp1 == exp2 ) + { + isNull = true; + return 0; + } + + // since exp1 here is inside the block, when we leave the block, the + // original (Date) value is restored. + } + + if (isNull) + { + isNull = false; + return exp1; + } + + break; + } + default: { isNull = true; @@ -244,6 +281,19 @@ uint64_t Func_nullif::getUintVal(rowgroup::Row& row, break; } + case execplan::CalpontSystemCatalog::TIME: + { + exp2 = parm[1]->data()->getTimeIntVal(row, isNull); + + if (isNull) + { + isNull = false; + return exp1; + } + + break; + } + default: { isNull = true; @@ -400,18 +450,6 @@ int64_t Func_nullif::getDatetimeIntVal(rowgroup::Row& row, case execplan::CalpontSystemCatalog::VARCHAR: case execplan::CalpontSystemCatalog::CHAR: case execplan::CalpontSystemCatalog::TEXT: - { - exp2 = parm[1]->data()->getIntVal(row, isNull); - - if (isNull) - { - isNull = false; - return exp1; - } - - break; - } - case execplan::CalpontSystemCatalog::DATE: { // Upgrade to datetime for proper comparison @@ -427,6 +465,7 @@ int64_t Func_nullif::getDatetimeIntVal(rowgroup::Row& row, } break; + case execplan::CalpontSystemCatalog::TIME: case execplan::CalpontSystemCatalog::DATETIME: { exp2 = parm[1]->data()->getDatetimeIntVal(row, isNull); @@ -455,6 +494,57 @@ int64_t Func_nullif::getDatetimeIntVal(rowgroup::Row& row, return exp1; } +int64_t Func_nullif::getTimeIntVal(rowgroup::Row& row, + FunctionParm& parm, + bool& isNull, + CalpontSystemCatalog::ColType& ct) +{ + int64_t exp1 = parm[0]->data()->getTimeIntVal(row, isNull); + int64_t exp2 = 0; + + switch (parm[1]->data()->resultType().colDataType) + { + case execplan::CalpontSystemCatalog::BIGINT: + case execplan::CalpontSystemCatalog::INT: + case execplan::CalpontSystemCatalog::MEDINT: + case execplan::CalpontSystemCatalog::TINYINT: + case execplan::CalpontSystemCatalog::SMALLINT: + case execplan::CalpontSystemCatalog::DOUBLE: + case execplan::CalpontSystemCatalog::FLOAT: + case execplan::CalpontSystemCatalog::DECIMAL: + case execplan::CalpontSystemCatalog::VARCHAR: + case execplan::CalpontSystemCatalog::CHAR: + case execplan::CalpontSystemCatalog::TEXT: + case execplan::CalpontSystemCatalog::TIME: + case execplan::CalpontSystemCatalog::DATETIME: + { + exp2 = parm[1]->data()->getTimeIntVal(row, isNull); + + if (isNull) + { + isNull = false; + return exp1; + } + + break; + } + + default: + { + isNull = true; + } + } + + if ( exp1 == exp2 ) + { + isNull = true; + return 0; + } + + return exp1; +} + + double Func_nullif::getDoubleVal(rowgroup::Row& row, FunctionParm& parm, bool& isNull, @@ -502,6 +592,7 @@ double Func_nullif::getDoubleVal(rowgroup::Row& row, } break; + case execplan::CalpontSystemCatalog::TIME: case execplan::CalpontSystemCatalog::DATETIME: { exp2 = parm[1]->data()->getDatetimeIntVal(row, isNull); @@ -644,6 +735,7 @@ execplan::IDB_Decimal Func_nullif::getDecimalVal(rowgroup::Row& row, } break; + case execplan::CalpontSystemCatalog::TIME: case execplan::CalpontSystemCatalog::DATETIME: { int32_t s = 0; diff --git a/utils/funcexp/func_regexp.cpp b/utils/funcexp/func_regexp.cpp index 11f702a60..420c792cf 100644 --- a/utils/funcexp/func_regexp.cpp +++ b/utils/funcexp/func_regexp.cpp @@ -94,6 +94,14 @@ inline bool getBool(rowgroup::Row& row, break; } + case execplan::CalpontSystemCatalog::TIME: + { + expr = dataconvert::DataConvert::timeToString(pm[0]->data()->getTimeIntVal(row, isNull)); + //strip off micro seconds + expr = expr.substr(0, 19); + break; + } + case execplan::CalpontSystemCatalog::DECIMAL: case execplan::CalpontSystemCatalog::UDECIMAL: { @@ -153,6 +161,14 @@ inline bool getBool(rowgroup::Row& row, break; } + case execplan::CalpontSystemCatalog::TIME: + { + pattern = dataconvert::DataConvert::timeToString(pm[1]->data()->getTimeIntVal(row, isNull)); + //strip off micro seconds + pattern = pattern.substr(0, 19); + break; + } + case execplan::CalpontSystemCatalog::DECIMAL: case execplan::CalpontSystemCatalog::UDECIMAL: { diff --git a/utils/funcexp/func_round.cpp b/utils/funcexp/func_round.cpp index 18b888b2c..c107fc0fc 100644 --- a/utils/funcexp/func_round.cpp +++ b/utils/funcexp/func_round.cpp @@ -390,6 +390,7 @@ IDB_Decimal Func_round::getDecimalVal(Row& row, } break; + case execplan::CalpontSystemCatalog::TIME: case execplan::CalpontSystemCatalog::DATETIME: { int32_t s = 0; diff --git a/utils/funcexp/func_second.cpp b/utils/funcexp/func_second.cpp index 509633c46..ae12b157c 100644 --- a/utils/funcexp/func_second.cpp +++ b/utils/funcexp/func_second.cpp @@ -108,6 +108,13 @@ int64_t Func_second::getIntVal(rowgroup::Row& row, break; } + case execplan::CalpontSystemCatalog::TIME: + { + val = parm[0]->data()->getTimeIntVal(row, isNull); + return (uint32_t)((val >> 24) & 0xff); + break; + } + default: { isNull = true; diff --git a/utils/funcexp/func_sysdate.cpp b/utils/funcexp/func_sysdate.cpp index f8a5a98bd..bf71e0415 100644 --- a/utils/funcexp/func_sysdate.cpp +++ b/utils/funcexp/func_sysdate.cpp @@ -100,6 +100,15 @@ int64_t Func_sysdate::getDatetimeIntVal(rowgroup::Row& row, return getIntVal(row, parm, isNull, operationColType); } +int64_t Func_sysdate::getTimeIntVal(rowgroup::Row& row, + FunctionParm& parm, + bool& isNull, + CalpontSystemCatalog::ColType& operationColType) +{ + return getIntVal(row, parm, isNull, operationColType); +} + + } // namespace funcexp // vim:ts=4 sw=4: diff --git a/utils/funcexp/func_time.cpp b/utils/funcexp/func_time.cpp index 509caa777..4a90e0fe2 100644 --- a/utils/funcexp/func_time.cpp +++ b/utils/funcexp/func_time.cpp @@ -59,7 +59,7 @@ string Func_time::getStrVal(rowgroup::Row& row, case execplan::CalpontSystemCatalog::TINYINT: case execplan::CalpontSystemCatalog::SMALLINT: { - val = dataconvert::DataConvert::intToDatetime(parm[0]->data()->getIntVal(row, isNull)); + val = dataconvert::DataConvert::intToTime(parm[0]->data()->getIntVal(row, isNull)); if (val == -1) isNull = true; @@ -73,7 +73,7 @@ string Func_time::getStrVal(rowgroup::Row& row, { if (parm[0]->data()->resultType().scale) { - val = dataconvert::DataConvert::intToDatetime(parm[0]->data()->getIntVal(row, isNull)); + val = dataconvert::DataConvert::intToTime(parm[0]->data()->getIntVal(row, isNull)); if (val == -1) isNull = true; @@ -96,7 +96,7 @@ string Func_time::getStrVal(rowgroup::Row& row, case execplan::CalpontSystemCatalog::CHAR: case execplan::CalpontSystemCatalog::TEXT: { - val = dataconvert::DataConvert::stringToDatetime(parm[0]->data()->getStrVal(row, isNull)); + val = dataconvert::DataConvert::stringToTime(parm[0]->data()->getStrVal(row, isNull)); if (val == -1) isNull = true; @@ -106,15 +106,15 @@ string Func_time::getStrVal(rowgroup::Row& row, break; } - case execplan::CalpontSystemCatalog::DATE: + case execplan::CalpontSystemCatalog::TIME: { - val = parm[0]->data()->getDatetimeIntVal(row, isNull); + val = parm[0]->data()->getTimeIntVal(row, isNull); break; } case execplan::CalpontSystemCatalog::DATETIME: { - val = parm[0]->data()->getDatetimeIntVal(row, isNull); + val = parm[0]->data()->getTimeIntVal(row, isNull); break; } @@ -128,9 +128,9 @@ string Func_time::getStrVal(rowgroup::Row& row, return ""; char buf[30] = {'\0'}; - dataconvert::DataConvert::datetimeToString(val, buf, sizeof(buf)); + dataconvert::DataConvert::timeToString(val, buf, sizeof(buf)); string time(buf); - return time.substr(11, 80); + return time; } int64_t Func_time::getIntVal(rowgroup::Row& row, @@ -138,7 +138,7 @@ int64_t Func_time::getIntVal(rowgroup::Row& row, bool& isNull, execplan::CalpontSystemCatalog::ColType& op_ct) { - return dataconvert::DataConvert::datetimeToInt(getStrVal(row, fp, isNull, op_ct)); + return dataconvert::DataConvert::timeToInt(getStrVal(row, fp, isNull, op_ct)); } double Func_time::getDoubleVal(rowgroup::Row& row, diff --git a/utils/funcexp/func_time_to_sec.cpp b/utils/funcexp/func_time_to_sec.cpp index b87c2995a..ee5c4b669 100644 --- a/utils/funcexp/func_time_to_sec.cpp +++ b/utils/funcexp/func_time_to_sec.cpp @@ -49,12 +49,13 @@ int64_t Func_time_to_sec::getIntVal(rowgroup::Row& row, CalpontSystemCatalog::ColType& op_ct) { // assume 256 is enough. assume not allowing incomplete date - uint32_t hour = 0, - min = 0, - sec = 0; + int32_t hour = 0, + min = 0, + sec = 0; bool bIsNegative = false; // Only set to true if CHAR or VARCHAR with a '-' int64_t val = 0; + int64_t mask = 0; dataconvert::Time tval; switch (parm[0]->data()->resultType().colDataType) @@ -64,9 +65,28 @@ int64_t Func_time_to_sec::getIntVal(rowgroup::Row& row, case CalpontSystemCatalog::DATETIME: val = parm[0]->data()->getIntVal(row, isNull); - hour = (uint32_t)((val >> 32) & 0x3f); - min = (uint32_t)((val >> 26) & 0x3f); - sec = (uint32_t)((val >> 20) & 0x3f); + hour = (int32_t)((val >> 32) & 0x3f); + min = (int32_t)((val >> 26) & 0x3f); + sec = (int32_t)((val >> 20) & 0x3f); + break; + + case CalpontSystemCatalog::TIME: + val = parm[0]->data()->getTimeIntVal(row, isNull); + + // If negative, mask so it doesn't turn positive + if ((val >> 40) & 0x800) + mask = 0xfffffffffffff000; + + bIsNegative = val >> 63; + hour = (int32_t)(mask | ((val >> 40) & 0xfff)); + + if ((hour >= 0) && bIsNegative) + hour *= -1; + else + bIsNegative = false; + + min = (int32_t)((val >> 32) & 0xff); + sec = (int32_t)((val >> 24) & 0xff); break; case CalpontSystemCatalog::CHAR: @@ -112,9 +132,9 @@ int64_t Func_time_to_sec::getIntVal(rowgroup::Row& row, } else { - hour = (uint32_t)((val >> 32) & 0x3f); - min = (uint32_t)((val >> 26) & 0x3f); - sec = (uint32_t)((val >> 20) & 0x3f); + hour = (int32_t)((val >> 32) & 0x3f); + min = (int32_t)((val >> 26) & 0x3f); + sec = (int32_t)((val >> 20) & 0x3f); } break; @@ -131,9 +151,9 @@ int64_t Func_time_to_sec::getIntVal(rowgroup::Row& row, } else { - hour = (uint32_t)((val >> 32) & 0x3f); - min = (uint32_t)((val >> 26) & 0x3f); - sec = (uint32_t)((val >> 20) & 0x3f); + hour = (int32_t)((val >> 32) & 0x3f); + min = (int32_t)((val >> 26) & 0x3f); + sec = (int32_t)((val >> 20) & 0x3f); } } @@ -144,7 +164,16 @@ int64_t Func_time_to_sec::getIntVal(rowgroup::Row& row, return -1; } - int64_t rtn = (int64_t)(hour * 60 * 60) + (min * 60) + sec; + int64_t rtn; + + if (hour < 0) + { + rtn = (int64_t)(hour * 60 * 60) - (min * 60) - sec; + } + else + { + rtn = (int64_t)(hour * 60 * 60) + (min * 60) + sec; + } if (bIsNegative) { diff --git a/utils/funcexp/func_timediff.cpp b/utils/funcexp/func_timediff.cpp index c87e99b64..742e8faf7 100644 --- a/utils/funcexp/func_timediff.cpp +++ b/utils/funcexp/func_timediff.cpp @@ -116,6 +116,7 @@ string Func_timediff::getStrVal(rowgroup::Row& row, isDate1 = true; break; + case execplan::CalpontSystemCatalog::TIME: case execplan::CalpontSystemCatalog::DATETIME: val1 = parm[0]->data()->getDatetimeIntVal(row, isNull); break; @@ -157,6 +158,7 @@ string Func_timediff::getStrVal(rowgroup::Row& row, isDate2 = true; break; + case execplan::CalpontSystemCatalog::TIME: case execplan::CalpontSystemCatalog::DATETIME: val2 = parm[1]->data()->getDatetimeIntVal(row, isNull); break; @@ -213,6 +215,14 @@ int64_t Func_timediff::getDatetimeIntVal(rowgroup::Row& row, return dataconvert::DataConvert::datetimeToInt(getStrVal(row, parm, isNull, ct)); } +int64_t Func_timediff::getTimeIntVal(rowgroup::Row& row, + FunctionParm& parm, + bool& isNull, + CalpontSystemCatalog::ColType& ct) +{ + return dataconvert::DataConvert::timeToInt(getStrVal(row, parm, isNull, ct)); +} + int64_t Func_timediff::getIntVal(rowgroup::Row& row, FunctionParm& parm, bool& isNull, diff --git a/utils/funcexp/func_timestampdiff.cpp b/utils/funcexp/func_timestampdiff.cpp index f8d1fc7ac..9394203d7 100644 --- a/utils/funcexp/func_timestampdiff.cpp +++ b/utils/funcexp/func_timestampdiff.cpp @@ -152,6 +152,13 @@ int64_t Func_timestampdiff::getDatetimeIntVal(rowgroup::Row& row, return getIntVal(row, parm, isNull, ct); } +int64_t Func_timestampdiff::getTimeIntVal(rowgroup::Row& row, + FunctionParm& parm, + bool& isNull, + CalpontSystemCatalog::ColType& ct) +{ + return getIntVal(row, parm, isNull, ct); +} } // namespace funcexp // vim:ts=4 sw=4: diff --git a/utils/funcexp/func_truncate.cpp b/utils/funcexp/func_truncate.cpp index c8be27de0..b822caacd 100644 --- a/utils/funcexp/func_truncate.cpp +++ b/utils/funcexp/func_truncate.cpp @@ -387,6 +387,51 @@ IDB_Decimal Func_truncate::getDecimalVal(Row& row, } break; + case execplan::CalpontSystemCatalog::TIME: + { + int32_t s = 0; + int64_t x = 0; + + string value = + DataConvert::timeToString1(parm[0]->data()->getTimeIntVal(row, isNull)); + + s = parm[1]->data()->getIntVal(row, isNull); + + if (!isNull) + { + //strip off micro seconds + value = value.substr(0, 14); + int64_t x = atoll(value.c_str()); + + if ( s > 5 ) + s = 0; + + if ( s > 0 ) + { + x *= helpers::powerOf10_c[s]; + } + else if (s < 0) + { + s = -s; + + if ( s >= (int32_t) value.size() ) + { + x = 0; + } + else + { + x /= helpers::powerOf10_c[s]; + x *= helpers::powerOf10_c[s]; + } + + s = 0; + } + } + + decimal.value = x; + decimal.scale = s; + } + break; default: { diff --git a/utils/funcexp/funcexp.cpp b/utils/funcexp/funcexp.cpp index a59be9a7c..3c530f381 100644 --- a/utils/funcexp/funcexp.cpp +++ b/utils/funcexp/funcexp.cpp @@ -280,6 +280,18 @@ void FuncExp::evaluate(rowgroup::Row& row, std::vector& expressi break; } + case CalpontSystemCatalog::TIME: + { + int64_t val = expression[i]->getTimeIntVal(row, isNull); + + if (isNull) + row.setIntField<8>(TIMENULL, expression[i]->outputIndex()); + else + row.setIntField<8>(val, expression[i]->outputIndex()); + + break; + } + case CalpontSystemCatalog::CHAR: case CalpontSystemCatalog::VARCHAR: diff --git a/utils/funcexp/functor.cpp b/utils/funcexp/functor.cpp index 662db6448..b74812ee9 100644 --- a/utils/funcexp/functor.cpp +++ b/utils/funcexp/functor.cpp @@ -106,6 +106,21 @@ uint64_t Func::stringToDatetime(const string str) return ret; } +int64_t Func::stringToTime(const string str) +{ + int64_t ret = DataConvert::stringToTime(str); + + if (ret == -1) + { + Message::Args args; + args.add("time"); + args.add(str); + unsigned errcode = ERR_INCORRECT_VALUE; + throw IDBExcept(IDBErrorInfo::instance()->errorMsg(errcode, args), errcode); + } + + return ret; +} uint32_t Func::intToDate(int64_t i) { @@ -124,6 +139,11 @@ uint64_t Func::intToDatetime(int64_t i) return i; } +int64_t Func::intToTime(int64_t i) +{ + // Don't think we need to do anything here? + return i; +} string Func::intToString(int64_t i) { diff --git a/utils/funcexp/functor.h b/utils/funcexp/functor.h index 9e03e4af2..9edb9bf62 100644 --- a/utils/funcexp/functor.h +++ b/utils/funcexp/functor.h @@ -116,6 +116,14 @@ public: return intToDatetime(getIntVal(row, fp, isNull, op_ct)); } + virtual int64_t getTimeIntVal(rowgroup::Row& row, + FunctionParm& fp, + bool& isNull, + execplan::CalpontSystemCatalog::ColType& op_ct) + { + return intToTime(getIntVal(row, fp, isNull, op_ct)); + } + virtual bool getBoolVal(rowgroup::Row& row, FunctionParm& fp, bool& isNull, @@ -145,9 +153,11 @@ public: protected: virtual uint32_t stringToDate(std::string); virtual uint64_t stringToDatetime(std::string); + virtual int64_t stringToTime(std::string); virtual uint32_t intToDate(int64_t); virtual uint64_t intToDatetime(int64_t); + virtual int64_t intToTime(int64_t); virtual std::string intToString(int64_t); virtual std::string doubleToString(double); diff --git a/utils/funcexp/functor_all.h b/utils/funcexp/functor_all.h index 6300084cd..553abc40f 100644 --- a/utils/funcexp/functor_all.h +++ b/utils/funcexp/functor_all.h @@ -104,6 +104,11 @@ public: FunctionParm& fp, bool& isNull, execplan::CalpontSystemCatalog::ColType& op_ct); + + int64_t getTimeIntVal(rowgroup::Row& row, + FunctionParm& fp, + bool& isNull, + execplan::CalpontSystemCatalog::ColType& op_ct); }; @@ -151,6 +156,12 @@ public: FunctionParm& fp, bool& isNull, execplan::CalpontSystemCatalog::ColType& op_ct); + + int64_t getTimeIntVal(rowgroup::Row& row, + FunctionParm& fp, + bool& isNull, + execplan::CalpontSystemCatalog::ColType& op_ct); + }; @@ -193,6 +204,11 @@ public: FunctionParm& fp, bool& isNull, execplan::CalpontSystemCatalog::ColType& op_ct); + + int64_t getTimeIntVal(rowgroup::Row& row, + FunctionParm& fp, + bool& isNull, + execplan::CalpontSystemCatalog::ColType& op_ct); }; @@ -236,6 +252,11 @@ public: bool& isNull, execplan::CalpontSystemCatalog::ColType& op_ct); + int64_t getTimeIntVal(rowgroup::Row& row, + FunctionParm& fp, + bool& isNull, + execplan::CalpontSystemCatalog::ColType& op_ct); + bool getBoolVal(rowgroup::Row& row, FunctionParm& fp, bool& isNull, @@ -287,6 +308,12 @@ public: FunctionParm& fp, bool& isNull, execplan::CalpontSystemCatalog::ColType& op_ct); + + int64_t getTimeIntVal(rowgroup::Row& row, + FunctionParm& fp, + bool& isNull, + execplan::CalpontSystemCatalog::ColType& op_ct); + }; @@ -329,6 +356,11 @@ public: FunctionParm& fp, bool& isNull, execplan::CalpontSystemCatalog::ColType& op_ct); + + int64_t getTimeIntVal(rowgroup::Row& row, + FunctionParm& fp, + bool& isNull, + execplan::CalpontSystemCatalog::ColType& op_ct); }; @@ -371,6 +403,11 @@ public: FunctionParm& fp, bool& isNull, execplan::CalpontSystemCatalog::ColType& op_ct); + + int64_t getTimeIntVal(rowgroup::Row& row, + FunctionParm& fp, + bool& isNull, + execplan::CalpontSystemCatalog::ColType& op_ct); }; @@ -399,6 +436,11 @@ public: bool& isNull, execplan::CalpontSystemCatalog::ColType& op_ct); + int64_t getTimeIntVal(rowgroup::Row& row, + FunctionParm& fp, + bool& isNull, + execplan::CalpontSystemCatalog::ColType& op_ct); + int64_t getIntVal(rowgroup::Row& row, FunctionParm& fp, bool& isNull, diff --git a/utils/funcexp/functor_bool.h b/utils/funcexp/functor_bool.h index a20f925b3..b7b85106b 100644 --- a/utils/funcexp/functor_bool.h +++ b/utils/funcexp/functor_bool.h @@ -97,6 +97,15 @@ public: isNull = true; return 0; } + + int64_t getTimeIntVal(rowgroup::Row& row, + FunctionParm& fp, + bool& isNull, + execplan::CalpontSystemCatalog::ColType& op_ct) + { + isNull = true; + return 0; + } }; diff --git a/utils/funcexp/functor_dtm.h b/utils/funcexp/functor_dtm.h index dc662a176..d7837a4fe 100644 --- a/utils/funcexp/functor_dtm.h +++ b/utils/funcexp/functor_dtm.h @@ -137,6 +137,11 @@ public: bool& isNull, execplan::CalpontSystemCatalog::ColType& op_ct); + int64_t getTimeIntVal(rowgroup::Row& row, + FunctionParm& fp, + bool& isNull, + execplan::CalpontSystemCatalog::ColType& op_ct); + int64_t getIntVal(rowgroup::Row& row, FunctionParm& fp, bool& isNull, @@ -247,6 +252,12 @@ public: FunctionParm& fp, bool& isNull, execplan::CalpontSystemCatalog::ColType& op_ct); + + int64_t getTimeIntVal(rowgroup::Row& row, + FunctionParm& fp, + bool& isNull, + execplan::CalpontSystemCatalog::ColType& op_ct); + }; @@ -311,6 +322,11 @@ public: FunctionParm& fp, bool& isNull, execplan::CalpontSystemCatalog::ColType& op_ct); + + int64_t getTimeIntVal(rowgroup::Row& row, + FunctionParm& fp, + bool& isNull, + execplan::CalpontSystemCatalog::ColType& op_ct); }; @@ -343,6 +359,11 @@ public: FunctionParm& fp, bool& isNull, execplan::CalpontSystemCatalog::ColType& op_ct); + + int64_t getTimeIntVal(rowgroup::Row& row, + FunctionParm& fp, + bool& isNull, + execplan::CalpontSystemCatalog::ColType& op_ct); }; @@ -371,6 +392,11 @@ public: bool& isNull, execplan::CalpontSystemCatalog::ColType& op_ct); + int64_t getTimeIntVal(rowgroup::Row& row, + FunctionParm& fp, + bool& isNull, + execplan::CalpontSystemCatalog::ColType& op_ct); + int64_t getIntVal(rowgroup::Row& row, FunctionParm& fp, bool& isNull, @@ -411,6 +437,11 @@ public: FunctionParm& fp, bool& isNull, execplan::CalpontSystemCatalog::ColType& op_ct); + + int64_t getTimeIntVal(rowgroup::Row& row, + FunctionParm& fp, + bool& isNull, + execplan::CalpontSystemCatalog::ColType& op_ct); }; /** @brief Func_str_to_date class diff --git a/utils/funcexp/functor_real.h b/utils/funcexp/functor_real.h index 936401f80..5d40ce589 100644 --- a/utils/funcexp/functor_real.h +++ b/utils/funcexp/functor_real.h @@ -654,6 +654,12 @@ public: bool& isNull, execplan::CalpontSystemCatalog::ColType& op_ct); + int64_t getTimeIntVal(rowgroup::Row& row, + FunctionParm& fp, + bool& isNull, + execplan::CalpontSystemCatalog::ColType& op_ct); + + private: int64_t convertAton(const std::string& ipString, bool& isNull); }; diff --git a/utils/funcexp/functor_str.h b/utils/funcexp/functor_str.h index 0b40685f6..c71cdec91 100644 --- a/utils/funcexp/functor_str.h +++ b/utils/funcexp/functor_str.h @@ -89,6 +89,14 @@ public: return (isNull ? 0 : stringToDatetime(str)); } + int64_t getTimeIntVal(rowgroup::Row& row, + FunctionParm& fp, + bool& isNull, + execplan::CalpontSystemCatalog::ColType& op_ct) + { + std::string str = getStrVal(row, fp, isNull, op_ct); + return (isNull ? 0 : stringToTime(str)); + } protected: const std::string& stringValue(execplan::SPTP& fp, rowgroup::Row& row, bool& isNull) @@ -676,6 +684,10 @@ public: bool& isNull, execplan::CalpontSystemCatalog::ColType& op_ct); + int64_t getTimeIntVal(rowgroup::Row& row, + FunctionParm& fp, + bool& isNull, + execplan::CalpontSystemCatalog::ColType& op_ct); private: void convertNtoa(int64_t ipNum, std::string& ipString); }; diff --git a/utils/rowgroup/rowaggregation.cpp b/utils/rowgroup/rowaggregation.cpp index 0a98f6870..8d110cfc8 100644 --- a/utils/rowgroup/rowaggregation.cpp +++ b/utils/rowgroup/rowaggregation.cpp @@ -125,6 +125,11 @@ inline uint64_t getUintNullValue(int colType, int colWidth = 0) return joblist::DATETIMENULL; } + case execplan::CalpontSystemCatalog::TIME: + { + return joblist::TIMENULL; + } + case execplan::CalpontSystemCatalog::DECIMAL: case execplan::CalpontSystemCatalog::UDECIMAL: { @@ -640,6 +645,12 @@ inline bool RowAggregation::isNull(const RowGroup* pRowGroup, const Row& row, in break; } + case execplan::CalpontSystemCatalog::TIME: + { + ret = ((uint64_t)row.getUintField(col) == joblist::TIMENULL); + break; + } + case execplan::CalpontSystemCatalog::VARBINARY: case execplan::CalpontSystemCatalog::BLOB: { @@ -1117,6 +1128,7 @@ void RowAggregation::initMapData(const Row& rowIn) case execplan::CalpontSystemCatalog::DATE: case execplan::CalpontSystemCatalog::DATETIME: + case execplan::CalpontSystemCatalog::TIME: { fRow.setUintField(rowIn.getUintField(colIn), colOut); break; @@ -1247,6 +1259,7 @@ void RowAggregation::makeAggFieldsNull(Row& row) case execplan::CalpontSystemCatalog::DATE: case execplan::CalpontSystemCatalog::DATETIME: + case execplan::CalpontSystemCatalog::TIME: { row.setUintField(getUintNullValue(colDataType), colOut); break; @@ -1375,6 +1388,7 @@ void RowAggregation::doMinMaxSum(const Row& rowIn, int64_t colIn, int64_t colOut case execplan::CalpontSystemCatalog::DATE: case execplan::CalpontSystemCatalog::DATETIME: + case execplan::CalpontSystemCatalog::TIME: { if (funcType == ROWAGG_SUM) { @@ -1518,9 +1532,26 @@ void RowAggregation::doBitOp(const Row& rowIn, int64_t colIn, int64_t colOut, in case execplan::CalpontSystemCatalog::DATETIME: { uint64_t dtm = rowIn.getUintField(colIn); - valIn = ((dtm >> 48) * 10000000000LL) + (((dtm >> 44) & 0xF) * 100000000) + - (((dtm >> 38) & 077) * 1000000) + (((dtm >> 32) & 077) * 10000) + - (((dtm >> 26) & 077) * 100) + ((dtm >> 20) & 077); + valIn = ((dtm >> 48) * 10000000000000000LL) + (((dtm >> 44) & 0xF) * 100000000000000) + + (((dtm >> 38) & 077) * 1000000000000) + (((dtm >> 32) & 077) * 10000000000) + + (((dtm >> 26) & 077) * 100000000) + (((dtm >> 20) & 077) * 1000000) + (dtm & 0xfffff); + break; + } + + case execplan::CalpontSystemCatalog::TIME: + { + int64_t dtm = rowIn.getUintField(colIn); + // Handle negative correctly + int hour = 0; + + if ((dtm >> 40) & 0x800) + { + hour = 0xfffff000; + } + + hour |= ((dtm >> 40) & 0xfff); + valIn = (hour * 10000000000) + + (((dtm >> 32) & 0xff) * 100000000) + (((dtm >> 24) & 0xff) * 1000000) + (dtm & 0xffffff); break; } @@ -2048,6 +2079,13 @@ void RowAggregation::doUDAF(const Row& rowIn, int64_t colIn, int64_t colOut, int break; } + case execplan::CalpontSystemCatalog::TIME: + { + datum.dataType = execplan::CalpontSystemCatalog::BIGINT; + datum.columnData = rowIn.getIntField(colIn); + break; + } + case execplan::CalpontSystemCatalog::CHAR: case execplan::CalpontSystemCatalog::VARCHAR: case execplan::CalpontSystemCatalog::TEXT: @@ -2723,6 +2761,11 @@ void RowAggregationUM::SetUDAFValue(static_any::any& valOut, int64_t colOut) fRow.setUintField<8>(uintOut, colOut); break; + case execplan::CalpontSystemCatalog::TIME: + + fRow.setIntField<8>(intOut, colOut); + break; + case execplan::CalpontSystemCatalog::FLOAT: case execplan::CalpontSystemCatalog::UFLOAT: fRow.setFloatField(floatOut, colOut); @@ -3031,6 +3074,12 @@ void RowAggregationUM::doNullConstantAggregate(const ConstantAggData& aggData, u } break; + case execplan::CalpontSystemCatalog::TIME: + { + fRow.setIntField(getIntNullValue(colDataType), colOut); + } + break; + case execplan::CalpontSystemCatalog::CHAR: case execplan::CalpontSystemCatalog::VARCHAR: case execplan::CalpontSystemCatalog::TEXT: @@ -3207,6 +3256,12 @@ void RowAggregationUM::doNotNullConstantAggregate(const ConstantAggData& aggData } break; + case execplan::CalpontSystemCatalog::TIME: + { + fRow.setIntField(DataConvert::stringToTime(aggData.fConstValue), colOut); + } + break; + case execplan::CalpontSystemCatalog::CHAR: case execplan::CalpontSystemCatalog::VARCHAR: case execplan::CalpontSystemCatalog::TEXT: @@ -3295,6 +3350,7 @@ void RowAggregationUM::doNotNullConstantAggregate(const ConstantAggData& aggData case execplan::CalpontSystemCatalog::DATE: case execplan::CalpontSystemCatalog::DATETIME: + case execplan::CalpontSystemCatalog::TIME: case execplan::CalpontSystemCatalog::CHAR: case execplan::CalpontSystemCatalog::VARCHAR: case execplan::CalpontSystemCatalog::TEXT: @@ -3355,6 +3411,7 @@ void RowAggregationUM::doNotNullConstantAggregate(const ConstantAggData& aggData break; case execplan::CalpontSystemCatalog::DATETIME: + case execplan::CalpontSystemCatalog::TIME: { fRow.setUintField(0, colOut); } @@ -3492,6 +3549,12 @@ void RowAggregationUM::doNotNullConstantAggregate(const ConstantAggData& aggData } break; + case execplan::CalpontSystemCatalog::TIME: + { + datum.columnData = DataConvert::stringToTime(aggData.fConstValue); + } + break; + case execplan::CalpontSystemCatalog::CHAR: case execplan::CalpontSystemCatalog::VARCHAR: case execplan::CalpontSystemCatalog::TEXT: diff --git a/utils/rowgroup/rowgroup.cpp b/utils/rowgroup/rowgroup.cpp index c35129d03..185d3de67 100644 --- a/utils/rowgroup/rowgroup.cpp +++ b/utils/rowgroup/rowgroup.cpp @@ -699,6 +699,10 @@ void Row::initToNull() *((uint64_t*) &data[offsets[i]]) = joblist::DATETIMENULL; break; + case CalpontSystemCatalog::TIME: + *((uint64_t*) &data[offsets[i]]) = joblist::TIMENULL; + break; + case CalpontSystemCatalog::CHAR: case CalpontSystemCatalog::VARCHAR: case CalpontSystemCatalog::TEXT: @@ -841,6 +845,9 @@ bool Row::isNullValue(uint32_t colIndex) const case CalpontSystemCatalog::DATETIME: return (*((uint64_t*) &data[offsets[colIndex]]) == joblist::DATETIMENULL); + case CalpontSystemCatalog::TIME: + return (*((uint64_t*) &data[offsets[colIndex]]) == joblist::TIMENULL); + case CalpontSystemCatalog::CHAR: case CalpontSystemCatalog::VARCHAR: case CalpontSystemCatalog::STRINT: diff --git a/utils/udfsdk/docs/source/reference/ColumnDatum.rst b/utils/udfsdk/docs/source/reference/ColumnDatum.rst index 564b24e44..dd1006363 100644 --- a/utils/udfsdk/docs/source/reference/ColumnDatum.rst +++ b/utils/udfsdk/docs/source/reference/ColumnDatum.rst @@ -74,6 +74,8 @@ The provided values are: - A floating point number. Represented as a C++ double type. * - DATETIME - A Columnstore date-time stored as an eight byte unsigned integer. + * - TIME + - A Columnstore time stored as an eight byte unsigned integer. * - VARCHAR - A mariadb variable length string. Represented a std::string * - VARBINARY diff --git a/utils/udfsdk/mcsv1_udaf.cpp b/utils/udfsdk/mcsv1_udaf.cpp index a31ae71c0..349a642ec 100644 --- a/utils/udfsdk/mcsv1_udaf.cpp +++ b/utils/udfsdk/mcsv1_udaf.cpp @@ -95,6 +95,7 @@ int32_t mcsv1Context::getColWidth() case CalpontSystemCatalog::DOUBLE: case CalpontSystemCatalog::UDOUBLE: case CalpontSystemCatalog::DATETIME: + case CalpontSystemCatalog::TIME: case CalpontSystemCatalog::STRINT: fColWidth = 8; break; diff --git a/utils/udfsdk/udfsdk.cpp b/utils/udfsdk/udfsdk.cpp index ea206a167..9a8973232 100644 --- a/utils/udfsdk/udfsdk.cpp +++ b/utils/udfsdk/udfsdk.cpp @@ -101,7 +101,9 @@ CalpontSystemCatalog::ColType MCS_add::operationType (FunctionParm& fp, else if (fp[0]->data()->resultType().colDataType == CalpontSystemCatalog::DATE || fp[1]->data()->resultType().colDataType == CalpontSystemCatalog::DATE || fp[0]->data()->resultType().colDataType == CalpontSystemCatalog::DATETIME || - fp[1]->data()->resultType().colDataType == CalpontSystemCatalog::DATETIME) + fp[1]->data()->resultType().colDataType == CalpontSystemCatalog::DATETIME || + fp[0]->data()->resultType().colDataType == CalpontSystemCatalog::TIME || + fp[1]->data()->resultType().colDataType == CalpontSystemCatalog::TIME) { rt.colDataType = CalpontSystemCatalog::BIGINT; rt.colWidth = 8; diff --git a/utils/windowfunction/idborderby.cpp b/utils/windowfunction/idborderby.cpp index 5764afba5..8a021f8d8 100644 --- a/utils/windowfunction/idborderby.cpp +++ b/utils/windowfunction/idborderby.cpp @@ -281,6 +281,7 @@ void CompareRule::compileRules(const std::vector& spec, const rowgr case CalpontSystemCatalog::DATE: case CalpontSystemCatalog::DATETIME: + case CalpontSystemCatalog::TIME: { Compare* c = new UintCompare(*i); fCompares.push_back(c); @@ -413,6 +414,7 @@ bool EqualCompData::operator()(Row::Pointer a, Row::Pointer b) case CalpontSystemCatalog::UBIGINT: case CalpontSystemCatalog::DATE: case CalpontSystemCatalog::DATETIME: + case CalpontSystemCatalog::TIME: { // equal compare. ignore sign and null eq = (fRow1.getUintField(*i) == fRow2.getUintField(*i)); diff --git a/utils/windowfunction/wf_lead_lag.cpp b/utils/windowfunction/wf_lead_lag.cpp index dddfb91c6..5160b75bf 100644 --- a/utils/windowfunction/wf_lead_lag.cpp +++ b/utils/windowfunction/wf_lead_lag.cpp @@ -77,6 +77,7 @@ boost::shared_ptr WF_lead_lag::makeFunction(int id, const case CalpontSystemCatalog::UDECIMAL: case CalpontSystemCatalog::DATE: case CalpontSystemCatalog::DATETIME: + case CalpontSystemCatalog::TIME: { func.reset(new WF_lead_lag(id, name)); break; diff --git a/utils/windowfunction/wf_min_max.cpp b/utils/windowfunction/wf_min_max.cpp index 99a8fef84..b0c1fe033 100644 --- a/utils/windowfunction/wf_min_max.cpp +++ b/utils/windowfunction/wf_min_max.cpp @@ -76,6 +76,7 @@ boost::shared_ptr WF_min_max::makeFunction(int id, const case CalpontSystemCatalog::UDECIMAL: case CalpontSystemCatalog::DATE: case CalpontSystemCatalog::DATETIME: + case CalpontSystemCatalog::TIME: { func.reset(new WF_min_max(id, name)); break; diff --git a/utils/windowfunction/wf_nth_value.cpp b/utils/windowfunction/wf_nth_value.cpp index 5362ee8ec..eacd1202a 100644 --- a/utils/windowfunction/wf_nth_value.cpp +++ b/utils/windowfunction/wf_nth_value.cpp @@ -77,6 +77,7 @@ boost::shared_ptr WF_nth_value::makeFunction(int id, cons case CalpontSystemCatalog::UDECIMAL: case CalpontSystemCatalog::DATE: case CalpontSystemCatalog::DATETIME: + case CalpontSystemCatalog::TIME: { func.reset(new WF_nth_value(id, name)); break; diff --git a/utils/windowfunction/wf_percentile.cpp b/utils/windowfunction/wf_percentile.cpp index 9ccc07b9c..83d6f57d8 100644 --- a/utils/windowfunction/wf_percentile.cpp +++ b/utils/windowfunction/wf_percentile.cpp @@ -82,6 +82,7 @@ boost::shared_ptr WF_percentile::makeFunction(int id, con case CalpontSystemCatalog::UDECIMAL: case CalpontSystemCatalog::DATE: case CalpontSystemCatalog::DATETIME: + case CalpontSystemCatalog::TIME: { func.reset(new WF_percentile(id, name)); break; diff --git a/utils/windowfunction/wf_udaf.cpp b/utils/windowfunction/wf_udaf.cpp index fe394c028..f302c49cd 100644 --- a/utils/windowfunction/wf_udaf.cpp +++ b/utils/windowfunction/wf_udaf.cpp @@ -392,6 +392,7 @@ void WF_udaf::SetUDAFValue(static_any::any& valOut, int64_t colOut, case execplan::CalpontSystemCatalog::UBIGINT: case execplan::CalpontSystemCatalog::DATE: case execplan::CalpontSystemCatalog::DATETIME: + case execplan::CalpontSystemCatalog::TIME: setValue(colDataType, b, e, c, &uintOut); break; diff --git a/utils/windowfunction/windowfunctiontype.cpp b/utils/windowfunction/windowfunctiontype.cpp index 5a101f870..950045899 100644 --- a/utils/windowfunction/windowfunctiontype.cpp +++ b/utils/windowfunction/windowfunctiontype.cpp @@ -91,6 +91,7 @@ map colType2String = assign::map_list_of (CalpontSystemCatalog::LONGDOUBLE, "INTERNAL LONG DOUBLE") (CalpontSystemCatalog::STRINT, "INTERNAL SHORT STRING") (CalpontSystemCatalog::TEXT, "TEXT") + (CalpontSystemCatalog::TIME, "TIME") ; @@ -490,6 +491,7 @@ void* WindowFunctionType::getNullValueByType(int ct, int pos) static uint64_t doubleNull = joblist::DOUBLENULL; static uint64_t dateNull = joblist::DATENULL; static uint64_t datetimeNull = joblist::DATETIMENULL; + static uint64_t timeNull = joblist::TIMENULL; static uint64_t char1Null = joblist::CHAR1NULL; static uint64_t char2Null = joblist::CHAR2NULL; static uint64_t char4Null = joblist::CHAR4NULL; @@ -525,6 +527,10 @@ void* WindowFunctionType::getNullValueByType(int ct, int pos) v = &datetimeNull; break; + case CalpontSystemCatalog::TIME: + v = &timeNull; + break; + case CalpontSystemCatalog::FLOAT: case CalpontSystemCatalog::UFLOAT: v = &floatNull; diff --git a/writeengine/bulk/we_bulkloadbuffer.cpp b/writeengine/bulk/we_bulkloadbuffer.cpp index 68bf71027..506bff2eb 100644 --- a/writeengine/bulk/we_bulkloadbuffer.cpp +++ b/writeengine/bulk/we_bulkloadbuffer.cpp @@ -902,7 +902,8 @@ void BulkLoadBuffer::convert(char* field, int fieldLength, { bool bSatVal = false; - if ( column.dataType != CalpontSystemCatalog::DATETIME ) + if ( column.dataType != CalpontSystemCatalog::DATETIME && + column.dataType != CalpontSystemCatalog::TIME ) { if (nullFlag) { @@ -976,6 +977,58 @@ void BulkLoadBuffer::convert(char* field, int fieldLength, pVal = &llVal; } + else if (column.dataType == CalpontSystemCatalog::TIME) + { + // time conversion + int rc = 0; + + if (nullFlag) + { + if (column.fWithDefault) + { + llDate = column.fDefaultInt; + // fall through to update saturation and min/max + } + else + { + llDate = joblist::TIMENULL; + pVal = &llDate; + break; + } + } + else + { + if (fImportDataMode != IMPORT_DATA_TEXT) + { + memcpy(&llDate, field, sizeof(llDate)); + + if (!dataconvert::DataConvert::isColumnTimeValid( + llDate)) + rc = -1; + } + else + { + llDate = dataconvert::DataConvert::convertColumnTime( + field, dataconvert::CALPONTTIME_ENUM, + rc, fieldLength ); + } + } + + if (rc == 0) + { + if (llDate < bufStats.minBufferVal) + bufStats.minBufferVal = llDate; + + if (llDate > bufStats.maxBufferVal) + bufStats.maxBufferVal = llDate; + } + else + { + bufStats.satCount++; + } + + pVal = &llDate; + } else { // datetime conversion @@ -2973,6 +3026,11 @@ bool BulkLoadBuffer::isBinaryFieldNull(void* val, if ((*(uint64_t*)val) == joblist::DATETIMENULL) isNullFlag = true; } + else if (dt == execplan::CalpontSystemCatalog::TIME) + { + if ((*(uint64_t*)val) == joblist::TIMENULL) + isNullFlag = true; + } else { if ((*(uint64_t*)val) == joblist::BIGINTNULL) diff --git a/writeengine/bulk/we_tableinfo.cpp b/writeengine/bulk/we_tableinfo.cpp index 8da9ce968..da28204d2 100644 --- a/writeengine/bulk/we_tableinfo.cpp +++ b/writeengine/bulk/we_tableinfo.cpp @@ -978,6 +978,11 @@ void TableInfo::reportTotals(double elapsedTime) ossSatCnt << "invalid date/times replaced with zero value : "; } + else if (fColumns[i].column.dataType == CalpontSystemCatalog::TIME) + { + ossSatCnt << + "invalid times replaced with zero value : "; + } else if (fColumns[i].column.dataType == CalpontSystemCatalog::CHAR) ossSatCnt << "character strings truncated: "; diff --git a/writeengine/server/we_ddlcommon.h b/writeengine/server/we_ddlcommon.h index aca25dc39..1d482b442 100644 --- a/writeengine/server/we_ddlcommon.h +++ b/writeengine/server/we_ddlcommon.h @@ -274,6 +274,13 @@ inline boost::any getNullValueForType(const execplan::CalpontSystemCatalog::ColT } break; + case execplan::CalpontSystemCatalog::TIME: + { + long long d = joblist::TIMENULL; + value = d; + } + break; + case execplan::CalpontSystemCatalog::CHAR: { std::string charnull; @@ -433,6 +440,10 @@ inline int convertDataType(int dataType) calpontDataType = CalpontSystemCatalog::DATETIME; break; + case ddlpackage::DDL_TIME: + calpontDataType = CalpontSystemCatalog::TIME; + break; + case ddlpackage::DDL_CLOB: calpontDataType = CalpontSystemCatalog::CLOB; break; diff --git a/writeengine/server/we_dmlcommandproc.cpp b/writeengine/server/we_dmlcommandproc.cpp index bf5adf906..86625d013 100644 --- a/writeengine/server/we_dmlcommandproc.cpp +++ b/writeengine/server/we_dmlcommandproc.cpp @@ -1781,6 +1781,7 @@ uint8_t WE_DMLCommandProc::processBatchInsertBinary(messageqcpp::ByteStream& bs, case execplan::CalpontSystemCatalog::BIGINT: case execplan::CalpontSystemCatalog::DATETIME: + case execplan::CalpontSystemCatalog::TIME: case execplan::CalpontSystemCatalog::UBIGINT: bs >> val64; @@ -2977,7 +2978,14 @@ uint8_t WE_DMLCommandProc::processUpdate(messageqcpp::ByteStream& bs, case CalpontSystemCatalog::DATETIME: { intColVal = row.getUintField<8>(fetchColPos); - value = DataConvert::datetimeToString(intColVal); + value = DataConvert::datetimeToString(intColVal, colType.precision); + break; + } + + case CalpontSystemCatalog::TIME: + { + intColVal = row.getIntField<8>(fetchColPos); + value = DataConvert::timeToString(intColVal, colType.precision); break; } @@ -3305,7 +3313,14 @@ uint8_t WE_DMLCommandProc::processUpdate(messageqcpp::ByteStream& bs, case CalpontSystemCatalog::DATETIME: { intColVal = row.getUintField<8>(fetchColPos); - value = DataConvert::datetimeToString(intColVal); + value = DataConvert::datetimeToString(intColVal, colType.precision); + break; + } + + case CalpontSystemCatalog::TIME: + { + intColVal = row.getIntField<8>(fetchColPos); + value = DataConvert::timeToString(intColVal, colType.precision); break; } diff --git a/writeengine/shared/we_convertor.cpp b/writeengine/shared/we_convertor.cpp index dc9754355..8050426f3 100644 --- a/writeengine/shared/we_convertor.cpp +++ b/writeengine/shared/we_convertor.cpp @@ -420,6 +420,7 @@ void Convertor::convertColType(CalpontSystemCatalog::ColDataType dataType, // Map BIGINT and DATETIME to WR_LONGLONG case CalpontSystemCatalog::BIGINT : case CalpontSystemCatalog::DATETIME : + case CalpontSystemCatalog::TIME : internalType = WriteEngine::WR_LONGLONG; break; @@ -629,6 +630,7 @@ void Convertor::convertColType(ColStruct* curStruct) // Map BIGINT and DATETIME to WR_LONGLONG case CalpontSystemCatalog::BIGINT : case CalpontSystemCatalog::DATETIME : + case CalpontSystemCatalog::TIME : *internalType = WriteEngine::WR_LONGLONG; break; @@ -792,6 +794,7 @@ int Convertor::getCorrectRowWidth(CalpontSystemCatalog::ColDataType dataType, in break; case CalpontSystemCatalog::DATETIME: + case CalpontSystemCatalog::TIME: newWidth = 8; break; diff --git a/writeengine/shared/we_type.h b/writeengine/shared/we_type.h index 21fad3e08..06a367ecb 100644 --- a/writeengine/shared/we_type.h +++ b/writeengine/shared/we_type.h @@ -161,7 +161,8 @@ const char ColDataTypeStr[execplan::CalpontSystemCatalog::NUM_OF_COL_DATA_TYPE] "unsigned-float", "unsigned-bigint", "unsigned-double", - "text" + "text", + "time" }; enum FuncType { FUNC_WRITE_ENGINE, FUNC_INDEX, FUNC_DICTIONARY }; diff --git a/writeengine/splitter/we_sdhandler.cpp b/writeengine/splitter/we_sdhandler.cpp index 68d5bf784..802fd108b 100644 --- a/writeengine/splitter/we_sdhandler.cpp +++ b/writeengine/splitter/we_sdhandler.cpp @@ -1907,6 +1907,10 @@ void WESDHandler::onCleanupResult(int PmId, messageqcpp::SBS& Sbs) ossSatCnt << "invalid date/times replaced with zero value: "; break; + case CalpontSystemCatalog::TIME: + ossSatCnt << "invalid times replaced with zero value: "; + break; + case CalpontSystemCatalog::CHAR: ossSatCnt << "character strings truncated: "; break; diff --git a/writeengine/splitter/we_splitterapp.cpp b/writeengine/splitter/we_splitterapp.cpp index 862eb03c0..0804baf1b 100644 --- a/writeengine/splitter/we_splitterapp.cpp +++ b/writeengine/splitter/we_splitterapp.cpp @@ -165,7 +165,7 @@ void WESplitterApp::setupSignalHandlers() sa.sa_handler = SIG_IGN; sigaction(SIGPIPE, &sa, 0); sa.sa_handler = WESplitterApp::onSigHup; - sigaction(SIGPIPE, &sa, 0); + sigaction(SIGHUP, &sa, 0); sa.sa_handler = WESplitterApp::onSigInterrupt; sigaction(SIGUSR1, &sa, 0); /* diff --git a/writeengine/wrapper/writeengine.cpp b/writeengine/wrapper/writeengine.cpp index cb6eb3745..5d3dfec85 100644 --- a/writeengine/wrapper/writeengine.cpp +++ b/writeengine/wrapper/writeengine.cpp @@ -443,6 +443,8 @@ void WriteEngineWrapper::convertValue(const ColType colType, void* valArray, con case WriteEngine::WR_LONGLONG: if (data.type() == typeid(long long)) ((long long*)valArray)[pos] = boost::any_cast(data); + else if (data.type() == typeid(long)) + ((long long*)valArray)[pos] = (long long)boost::any_cast(data); else ((long long*)valArray)[pos] = boost::any_cast(data); diff --git a/writeengine/xml/we_xmljob.cpp b/writeengine/xml/we_xmljob.cpp index 135754ac1..8d755b824 100644 --- a/writeengine/xml/we_xmljob.cpp +++ b/writeengine/xml/we_xmljob.cpp @@ -1116,6 +1116,22 @@ void XMLJob::fillInXMLDataNotNullDefault( break; } + case execplan::CalpontSystemCatalog::TIME: + { + int convertStatus; + int64_t dt = + dataconvert::DataConvert::convertColumnTime( + col_defaultValue.c_str(), + dataconvert::CALPONTTIME_ENUM, convertStatus, + col_defaultValue.length() ); + + if (convertStatus != 0) + bDefaultConvertError = true; + + col.fDefaultInt = dt; + break; + } + case execplan::CalpontSystemCatalog::FLOAT: case execplan::CalpontSystemCatalog::DOUBLE: case execplan::CalpontSystemCatalog::UFLOAT: