diff --git a/.gitignore b/.gitignore index f8246dc9f..c7aa228d3 100644 --- a/.gitignore +++ b/.gitignore @@ -39,7 +39,6 @@ dbcon/ddlpackage/ddl-scan.h dbcon/dmlpackage/dml-scan.cpp dbcon/dmlpackage/dml-scan.h ddlproc/DDLProc -decomsvr/DecomSvr dmlproc/DMLProc exemgr/ExeMgr oamapps/calpontDB/calpontDBWrite diff --git a/CMakeLists.txt b/CMakeLists.txt index 44aed9bc8..4dd1f6ede 100755 --- a/CMakeLists.txt +++ b/CMakeLists.txt @@ -240,7 +240,6 @@ ADD_SUBDIRECTORY(dmlproc) ADD_SUBDIRECTORY(procmon) ADD_SUBDIRECTORY(procmgr) ADD_SUBDIRECTORY(oamapps) -ADD_SUBDIRECTORY(decomsvr) ADD_SUBDIRECTORY(primitives) ADD_SUBDIRECTORY(tools) ADD_SUBDIRECTORY(writeengine/server) diff --git a/cpackEngineRPM.cmake b/cpackEngineRPM.cmake index be5df3462..71d5e1f26 100644 --- a/cpackEngineRPM.cmake +++ b/cpackEngineRPM.cmake @@ -132,7 +132,6 @@ SET(CPACK_RPM_platform_USER_FILELIST "/usr/local/mariadb/columnstore/bin/post-mysqld-install" "/usr/local/mariadb/columnstore/bin/pre-uninstall" "/usr/local/mariadb/columnstore/bin/PrimProc" -"/usr/local/mariadb/columnstore/bin/DecomSvr" "/usr/local/mariadb/columnstore/bin/upgrade-columnstore.sh" "/usr/local/mariadb/columnstore/bin/run.sh" "/usr/local/mariadb/columnstore/bin/columnstore" diff --git a/dbcon/ddlpackage/ddl-gram-win.cpp b/dbcon/ddlpackage/ddl-gram-win.cpp index a3213b6e4..cf16912bd 100644 --- a/dbcon/ddlpackage/ddl-gram-win.cpp +++ b/dbcon/ddlpackage/ddl-gram-win.cpp @@ -850,14 +850,14 @@ static const yytype_uint8 yydefact[] = static const yytype_int16 yydefgoto[] = { -1, 5, 6, 7, 8, 33, 9, 10, 320, 11, - 29, 12, 96, 97, 98, 66, 104, 105, 141, 106, - 107, 371, 372, 375, 376, 382, 385, 377, 378, 391, - 321, 276, 13, 43, 44, 45, 46, 47, 48, 49, - 25, 26, 50, 142, 68, 252, 99, 194, 253, 129, - 254, 255, 167, 168, 219, 220, 169, 256, 257, 152, - 130, 131, 132, 133, 178, 172, 134, 190, 304, 135, - 185, 237, 51, 145, 52, 71 -}; + 29, 12, 96, 97, 98, 66, 104, 105, 141, 106, + 107, 371, 372, 375, 376, 382, 385, 377, 378, 391, + 321, 276, 13, 43, 44, 45, 46, 47, 48, 49, + 25, 26, 50, 142, 68, 252, 99, 194, 253, 129, + 254, 255, 167, 168, 219, 220, 169, 256, 257, 152, + 130, 131, 132, 133, 178, 172, 134, 190, 304, 135, + 185, 237, 51, 145, 52, 71 + }; /* YYPACT[STATE-NUM] -- Index in YYTABLE of the portion describing STATE-NUM. */ @@ -910,14 +910,14 @@ static const yytype_int16 yypact[] = static const yytype_int16 yypgoto[] = { -270, -270, -270, 331, -270, -270, -270, -270, 26, -270, - -270, -270, 62, 218, 336, 314, 287, -270, -136, -270, - -270, -270, 11, -270, -270, -270, -270, 6, 1, 7, - 175, -269, -270, -270, -28, -270, -270, -270, -270, -270, - -29, 5, -270, -23, -70, -183, 79, -270, -175, -74, - -179, -223, -157, -270, -270, -270, 216, 90, -5, -202, - -270, -270, -270, -270, 161, 116, -270, 166, -270, -270, - -270, -270, -270, 119, -270, -270 -}; + -270, -270, 62, 218, 336, 314, 287, -270, -136, -270, + -270, -270, 11, -270, -270, -270, -270, 6, 1, 7, + 175, -269, -270, -270, -28, -270, -270, -270, -270, -270, + -29, 5, -270, -23, -70, -183, 79, -270, -175, -74, + -179, -223, -157, -270, -270, -270, 216, 90, -5, -202, + -270, -270, -270, -270, 161, 116, -270, 166, -270, -270, + -270, -270, -270, 119, -270, -270 + }; /* YYTABLE[YYPACT[STATE-NUM]]. What to do in state STATE-NUM. If positive, shift that token. If negative, reduce the rule which diff --git a/dbcon/ddlpackage/ddl.l b/dbcon/ddlpackage/ddl.l index 4812bbd0d..fd5ee6900 100644 --- a/dbcon/ddlpackage/ddl.l +++ b/dbcon/ddlpackage/ddl.l @@ -18,6 +18,7 @@ /* $Id: ddl.l 9341 2013-03-27 14:10:35Z chao $ */ %{ +#include #include #include #include @@ -31,7 +32,7 @@ #endif using namespace ddlpackage; -typedef enum { NOOP, STRIP_QUOTES } copy_action_t; +typedef enum { NOOP, STRIP_QUOTES, STRIP_QUOTES_FQ } copy_action_t; int lineno = 1; void ddlerror(struct pass_to_bison* x, char const *s); @@ -49,6 +50,7 @@ static char* scanner_copy(char *str, yyscan_t yyscanner, copy_action_t action = %x check2 %x inquote %x endquote +%x c_comment space [ \t\n\r\f] horiz_space [ \t\f] @@ -60,6 +62,7 @@ double_quote \" grave_accent ` comment ("--"{non_newline}*) +extended_ident_cont [A-Za-z\200-\377_0-9\$#,()\[\].;\:\+\-\*\/\%\^\<\>\=!&|@\\] self [,()\[\].;\:\+\-\*\/\%\^\<\>\=] whitespace ({space}+|{comment}) @@ -67,10 +70,13 @@ digit [0-9] ident_start [A-Za-z\200-\377_0-9] ident_cont [A-Za-z\200-\377_0-9\$] identifier {ident_start}{ident_cont}* +extended_identifier {ident_start}{extended_ident_cont}* /* fully qualified names regexes */ fq_identifier {identifier}\.{identifier} -identifer_quoted {grave_accent}{identifier}{grave_accent} -identifer_double_quoted {double_quote}{identifier}{double_quote} +identifier_quoted {grave_accent}{extended_identifier}{grave_accent} +identifier_double_quoted {double_quote}{extended_identifier}{double_quote} +fq_quoted ({identifier_quoted}|{extended_identifier})\.({identifier_quoted}|{identifier}) +fq_double_quoted ({identifier_double_quoted}|{extended_identifier})\.({identifier_double_quoted}|{identifier}) integer [-+]?{digit}+ decimal ([-+]?({digit}*\.{digit}+)|({digit}+\.{digit}*)) @@ -79,12 +85,15 @@ realfail1 ({integer}|{decimal})[Ee] realfail2 ({integer}|{decimal})[Ee][-+] + %% -{identifer_quoted} { ddlget_lval(yyscanner)->str = scanner_copy( ddlget_text(yyscanner), yyscanner, STRIP_QUOTES ); return IDENT; } -{identifer_double_quoted} { ddlget_lval(yyscanner)->str = scanner_copy( ddlget_text(yyscanner), yyscanner, STRIP_QUOTES ); return IDENT; } +{identifier_quoted} { ddlget_lval(yyscanner)->str = scanner_copy( ddlget_text(yyscanner), yyscanner, STRIP_QUOTES ); return IDENT; } +{identifier_double_quoted} { ddlget_lval(yyscanner)->str = scanner_copy( ddlget_text(yyscanner), yyscanner, STRIP_QUOTES ); return IDENT; } {fq_identifier} { ddlget_lval(yyscanner)->str = scanner_copy(ddlget_text(yyscanner), yyscanner); return FQ_IDENT; } +{fq_quoted} { ddlget_lval(yyscanner)->str = scanner_copy(ddlget_text(yyscanner), yyscanner, STRIP_QUOTES_FQ); return FQ_IDENT; } +{fq_double_quoted} { ddlget_lval(yyscanner)->str = scanner_copy(ddlget_text(yyscanner), yyscanner, STRIP_QUOTES_FQ); return FQ_IDENT; } ACTION {return ACTION;} ADD {return ADD;} @@ -116,6 +125,7 @@ CREATE {return CREATE;} CURRENT_USER {return CURRENT_USER;} DATE {ddlget_lval(yyscanner)->str=strdup("date"); return DATE;} DATETIME {return DATETIME;} +TIME {return TIME;} DECIMAL {return DECIMAL;} DEC {return DECIMAL;} DEFAULT {return DEFAULT;} @@ -193,6 +203,10 @@ LONGTEXT {return LONGTEXT;} /* ignore */ } +"/*" { BEGIN(c_comment); } +"*/" { BEGIN(0); } +\n { } +. { } %% void ddlerror(struct pass_to_bison* x, char const *s) @@ -278,6 +292,57 @@ char* scanner_copy (char *str, yyscan_t yyscanner, copy_action_t action) nv[strlen(str) - 1] = '\0'; result = nv + 1; } + else if (action == STRIP_QUOTES_FQ) + { + bool move_left = false; + bool move_right = false; + char* left = nv; + char* tmp_first = nv; + // MCOL-1384 Loop through all comas in this quoted fq id + // looking for $quote_sign.$quote_sign sequence. + char* fq_delimiter; + int tmp_pos = 0; + while((fq_delimiter = strchr(tmp_first, '.')) != NULL) + { + if( (*(fq_delimiter -1) == '`' && *(fq_delimiter + 1) == '`') || + (*(fq_delimiter -1) == '"' && *(fq_delimiter + 1) == '"') ) + { + tmp_pos += fq_delimiter - tmp_first; + break; + } + tmp_first = fq_delimiter; + } + + char* fq_delimiter_orig = str + tmp_pos; + char* right = fq_delimiter + 1; + char* right_orig = fq_delimiter_orig + 1; + // MCOL-1384 Strip quotes from the left part. + if(*left == '"' || *left == '`') + { + result = left + 1; + *(fq_delimiter - 1) = '.'; + move_left = true; + } + else + { + fq_delimiter += 1; + } + + int right_length = strlen(right); + // MCOL-1384 Strip quotes from the right part. + if(*right == '`' || *right == '"') + { + right += 1; right_orig += 1; + right_length -= 2; + move_right = true; + *(fq_delimiter + right_length) = '\0'; + } + + if(move_left || move_right) + { + strncpy(fq_delimiter, right_orig, right_length); + } + } return result; } diff --git a/dbcon/ddlpackage/ddl.y b/dbcon/ddlpackage/ddl.y index 8f6e713fc..68747dfc9 100644 --- a/dbcon/ddlpackage/ddl.y +++ b/dbcon/ddlpackage/ddl.y @@ -177,7 +177,6 @@ VARYING WITH ZONE DOUBLE IDB_FLOAT REAL CHARSET IDB_IF EXISTS CHANGE TRUNCATE %type opt_precision_scale %type opt_referential_triggered_action %type opt_time_precision -%type opt_with_time_zone %type qualified_name %type referential_action %type referential_triggered_action @@ -1114,10 +1113,11 @@ literal: ; datetime_type: - DATETIME + DATETIME opt_time_precision { $$ = new ColumnType(DDL_DATETIME); $$->fLength = DDLDatatypeLength[DDL_DATETIME]; + $$->fPrecision = $2; } | DATE @@ -1126,12 +1126,11 @@ datetime_type: $$->fLength = DDLDatatypeLength[DDL_DATE]; } | - TIME opt_time_precision opt_with_time_zone + TIME opt_time_precision { - $$ = new ColumnType(DDL_DATETIME); - $$->fLength = DDLDatatypeLength[DDL_DATETIME]; + $$ = new ColumnType(DDL_TIME); + $$->fLength = DDLDatatypeLength[DDL_TIME]; $$->fPrecision = $2; - $$->fWithTimezone = $3; } opt_time_precision: @@ -1139,11 +1138,6 @@ opt_time_precision: | {$$ = -1;} ; -opt_with_time_zone: - WITH TIME ZONE {$$ = true;} - | {$$ = false;} - ; - drop_column_def: DROP column_name drop_behavior {$$ = new AtaDropColumn($2, $3);} | DROP COLUMN column_name drop_behavior {$$ = new AtaDropColumn($3, $4);} diff --git a/dbcon/ddlpackage/ddlpkg.h b/dbcon/ddlpackage/ddlpkg.h index 97e971b6b..779577f83 100644 --- a/dbcon/ddlpackage/ddlpkg.h +++ b/dbcon/ddlpackage/ddlpkg.h @@ -236,6 +236,7 @@ enum DDL_DATATYPES DDL_UNSIGNED_DOUBLE, DDL_UNSIGNED_NUMERIC, DDL_TEXT, + DDL_TIME, DDL_INVALID_DATATYPE }; @@ -273,6 +274,7 @@ const std::string DDLDatatypeString[] = "unsigned-double", "unsigned-numeric", "text", + "time" "" }; @@ -328,6 +330,7 @@ const int DDLDatatypeLength[] = 8, // UNSIGNED_DOUBLE, 2, // UNSIGNED_NUMERIC, 8, // TEXT + 8, // TIME -1 // INVALID LENGTH }; diff --git a/dbcon/ddlpackageproc/altertableprocessor.cpp b/dbcon/ddlpackageproc/altertableprocessor.cpp index c6edff71c..a398e656f 100644 --- a/dbcon/ddlpackageproc/altertableprocessor.cpp +++ b/dbcon/ddlpackageproc/altertableprocessor.cpp @@ -209,6 +209,11 @@ bool typesAreSame(const CalpontSystemCatalog::ColType& colType, const ColumnType break; + case (CalpontSystemCatalog::TIME): + if (newType.fType == DDL_TIME) return true; + + break; + case (CalpontSystemCatalog::VARCHAR): if (newType.fType == DDL_VARCHAR && colType.colWidth == newType.fLength) return true; diff --git a/dbcon/ddlpackageproc/ddlindexpopulator.cpp b/dbcon/ddlpackageproc/ddlindexpopulator.cpp index 12d6c9854..cb58df5bc 100644 --- a/dbcon/ddlpackageproc/ddlindexpopulator.cpp +++ b/dbcon/ddlpackageproc/ddlindexpopulator.cpp @@ -330,6 +330,7 @@ boost::any DDLIndexPopulator::convertData(const CalpontSystemCatalog::ColType& return *reinterpret_cast(&data); case execplan::CalpontSystemCatalog::DATETIME: // @bug 375 + case execplan::CalpontSystemCatalog::TIME: case execplan::CalpontSystemCatalog::BIGINT: return *reinterpret_cast(&data); @@ -524,6 +525,7 @@ bool DDLIndexPopulator::checkNotNull(const IdxTuple& data, const CalpontSystemCa break; case execplan::CalpontSystemCatalog::DATETIME: + case execplan::CalpontSystemCatalog::TIME: isNull = any_cast(data.data) == any_cast(nullvalue); break; diff --git a/dbcon/ddlpackageproc/ddlpackageprocessor.cpp b/dbcon/ddlpackageproc/ddlpackageprocessor.cpp index 9b8c66658..2a64dd1f7 100644 --- a/dbcon/ddlpackageproc/ddlpackageprocessor.cpp +++ b/dbcon/ddlpackageproc/ddlpackageprocessor.cpp @@ -232,6 +232,10 @@ execplan::CalpontSystemCatalog::ColDataType DDLPackageProcessor::convertDataType colDataType = CalpontSystemCatalog::DATETIME; break; + case ddlpackage::DDL_TIME: + colDataType = CalpontSystemCatalog::TIME; + break; + case ddlpackage::DDL_CLOB: colDataType = CalpontSystemCatalog::CLOB; break; @@ -476,6 +480,13 @@ DDLPackageProcessor::getNullValueForType(const execplan::CalpontSystemCatalog::C } break; + case execplan::CalpontSystemCatalog::TIME: + { + long long d = joblist::TIMENULL; + value = d; + } + break; + case execplan::CalpontSystemCatalog::CHAR: { std::string charnull; diff --git a/dbcon/dmlpackage/dml-gram-win.cpp b/dbcon/dmlpackage/dml-gram-win.cpp index d278d9340..d224d89b1 100644 --- a/dbcon/dmlpackage/dml-gram-win.cpp +++ b/dbcon/dmlpackage/dml-gram-win.cpp @@ -866,16 +866,16 @@ static const yytype_uint8 yydefact[] = static const yytype_int16 yydefgoto[] = { -1, 13, 14, 15, 128, 129, 130, 131, 295, 296, - 297, 358, 378, 298, 155, 132, 360, 105, 133, 363, - 180, 181, 182, 338, 339, 16, 188, 268, 269, 304, - 17, 18, 19, 20, 21, 22, 23, 158, 253, 254, - 24, 25, 26, 31, 27, 108, 109, 28, 101, 102, - 99, 136, 137, 138, 61, 170, 171, 214, 215, 100, - 261, 318, 290, 143, 144, 145, 146, 147, 282, 148, - 149, 278, 150, 243, 151, 192, 152, 63, 64, 65, - 66, 67, 216, 44, 68, 335, 156, 33, 69, 259, - 340, 80 -}; + 297, 358, 378, 298, 155, 132, 360, 105, 133, 363, + 180, 181, 182, 338, 339, 16, 188, 268, 269, 304, + 17, 18, 19, 20, 21, 22, 23, 158, 253, 254, + 24, 25, 26, 31, 27, 108, 109, 28, 101, 102, + 99, 136, 137, 138, 61, 170, 171, 214, 215, 100, + 261, 318, 290, 143, 144, 145, 146, 147, 282, 148, + 149, 278, 150, 243, 151, 192, 152, 63, 64, 65, + 66, 67, 216, 44, 68, 335, 156, 33, 69, 259, + 340, 80 + }; /* YYPACT[STATE-NUM] -- Index in YYTABLE of the portion describing STATE-NUM. */ @@ -930,16 +930,16 @@ static const yytype_int16 yypact[] = static const yytype_int16 yypgoto[] = { -270, -270, 489, -270, -270, -270, 375, -270, -270, 179, - -270, -270, -270, -270, -269, -270, -270, -163, -270, -270, - -270, -270, 281, -270, 145, -270, -270, -270, 204, 240, - -270, -270, -270, -270, -270, -270, -270, -270, -270, 226, - -270, -270, -270, -131, -270, -270, 354, -270, 427, 359, - -56, 386, -136, -103, -170, -216, -270, -270, 255, -270, - -270, -270, -270, -134, -270, -270, -270, -270, 242, -270, - -270, 244, -270, -270, -270, 8, -24, -270, -189, 61, - -270, 140, -11, 484, -97, -270, -72, 4, 47, -270, - 449, 443 -}; + -270, -270, -270, -270, -269, -270, -270, -163, -270, -270, + -270, -270, 281, -270, 145, -270, -270, -270, 204, 240, + -270, -270, -270, -270, -270, -270, -270, -270, -270, 226, + -270, -270, -270, -131, -270, -270, 354, -270, 427, 359, + -56, 386, -136, -103, -170, -216, -270, -270, 255, -270, + -270, -270, -270, -134, -270, -270, -270, -270, 242, -270, + -270, 244, -270, -270, -270, 8, -24, -270, -189, 61, + -270, 140, -11, 484, -97, -270, -72, 4, 47, -270, + 449, 443 + }; /* YYTABLE[YYPACT[STATE-NUM]]. What to do in state STATE-NUM. If positive, shift that token. If negative, reduce the rule which diff --git a/dbcon/execplan/aggregatecolumn.cpp b/dbcon/execplan/aggregatecolumn.cpp index e0914beb3..18cba2607 100644 --- a/dbcon/execplan/aggregatecolumn.cpp +++ b/dbcon/execplan/aggregatecolumn.cpp @@ -353,6 +353,14 @@ void AggregateColumn::evaluate(Row& row, bool& isNull) break; + case CalpontSystemCatalog::TIME: + if (row.equals<8>(TIMENULL, fInputIndex)) + isNull = true; + else + fResult.intVal = row.getIntField<8>(fInputIndex); + + break; + case CalpontSystemCatalog::CHAR: case CalpontSystemCatalog::VARCHAR: case CalpontSystemCatalog::STRINT: diff --git a/dbcon/execplan/aggregatecolumn.h b/dbcon/execplan/aggregatecolumn.h index a70189e50..d1db7e5a4 100644 --- a/dbcon/execplan/aggregatecolumn.h +++ b/dbcon/execplan/aggregatecolumn.h @@ -411,6 +411,14 @@ public: /** * F&E */ + virtual int64_t getTimeIntVal(rowgroup::Row& row, bool& isNull) + { + evaluate(row, isNull); + return TreeNode::getTimeIntVal(); + } + /** + * F&E + */ virtual int64_t getDatetimeIntVal(rowgroup::Row& row, bool& isNull) { evaluate(row, isNull); diff --git a/dbcon/execplan/arithmeticcolumn.h b/dbcon/execplan/arithmeticcolumn.h index b6ca823c4..191416fbf 100644 --- a/dbcon/execplan/arithmeticcolumn.h +++ b/dbcon/execplan/arithmeticcolumn.h @@ -248,6 +248,11 @@ public: return fExpression->getDatetimeIntVal(row, isNull); } + virtual int64_t getTimeIntVal(rowgroup::Row& row, bool& isNull) + { + return fExpression->getTimeIntVal(row, isNull); + } + virtual bool getBoolVal(rowgroup::Row& row, bool& isNull) { return fExpression->getBoolVal(row, isNull); diff --git a/dbcon/execplan/arithmeticoperator.h b/dbcon/execplan/arithmeticoperator.h index ef7b538fa..ca254075b 100644 --- a/dbcon/execplan/arithmeticoperator.h +++ b/dbcon/execplan/arithmeticoperator.h @@ -151,6 +151,11 @@ public: evaluate(row, isNull, lop, rop); return TreeNode::getDatetimeIntVal(); } + virtual int64_t getTimeIntVal(rowgroup::Row& row, bool& isNull, ParseTree* lop, ParseTree* rop) + { + evaluate(row, isNull, lop, rop); + return TreeNode::getTimeIntVal(); + } virtual bool getBoolVal(rowgroup::Row& row, bool& isNull, ParseTree* lop, ParseTree* rop) { evaluate(row, isNull, lop, rop); diff --git a/dbcon/execplan/calpontsystemcatalog.cpp b/dbcon/execplan/calpontsystemcatalog.cpp index 501f488ce..15ccd73aa 100644 --- a/dbcon/execplan/calpontsystemcatalog.cpp +++ b/dbcon/execplan/calpontsystemcatalog.cpp @@ -156,6 +156,10 @@ const string colDataTypeToString(CalpontSystemCatalog::ColDataType cdt) return "datetime"; break; + case CalpontSystemCatalog::TIME: + return "time"; + break; + case CalpontSystemCatalog::VARCHAR: return "varchar"; break; diff --git a/dbcon/execplan/calpontsystemcatalog.h b/dbcon/execplan/calpontsystemcatalog.h index 5918f7f8c..7b828a297 100644 --- a/dbcon/execplan/calpontsystemcatalog.h +++ b/dbcon/execplan/calpontsystemcatalog.h @@ -154,6 +154,7 @@ public: UBIGINT, /*!< Unsigned BIGINT type */ UDOUBLE, /*!< Unsigned DOUBLE type */ TEXT, /*!< TEXT type */ + TIME, /*!< TIME type */ NUM_OF_COL_DATA_TYPE, /* NEW TYPES ABOVE HERE */ LONGDOUBLE, /* @bug3241, dev and variance calculation only */ STRINT, /* @bug3532, string as int for fast comparison */ diff --git a/dbcon/execplan/constantcolumn.h b/dbcon/execplan/constantcolumn.h index 8e7f178aa..04098faae 100644 --- a/dbcon/execplan/constantcolumn.h +++ b/dbcon/execplan/constantcolumn.h @@ -308,6 +308,21 @@ public: return fResult.intVal; } + /** + * F&E + */ + virtual int64_t getTimeIntVal(rowgroup::Row& row, bool& isNull) + { + isNull = isNull || (fType == NULLDATA); + + if (!fResult.valueConverted) + { + fResult.intVal = dataconvert::DataConvert::stringToTime(fResult.strVal); + fResult.valueConverted = true; + } + + return fResult.intVal; + } /** * F&E */ diff --git a/dbcon/execplan/functioncolumn.h b/dbcon/execplan/functioncolumn.h index 8f27cad75..5a099d1d2 100644 --- a/dbcon/execplan/functioncolumn.h +++ b/dbcon/execplan/functioncolumn.h @@ -255,7 +255,10 @@ public: { return fFunctor->getDatetimeIntVal(row, fFunctionParms, isNull, fOperationType); } - + virtual int64_t getTimeIntVal(rowgroup::Row& row, bool& isNull) + { + return fFunctor->getTimeIntVal(row, fFunctionParms, isNull, fOperationType); + } private: funcexp::FunctionParm fFunctionParms; diff --git a/dbcon/execplan/operator.h b/dbcon/execplan/operator.h index e0e16be2f..1a33dcb94 100644 --- a/dbcon/execplan/operator.h +++ b/dbcon/execplan/operator.h @@ -190,6 +190,10 @@ public: { return fResult.intVal; } + virtual int64_t getTimeIntVal(rowgroup::Row& row, bool& isNull, ParseTree* lop, ParseTree* rop) + { + return fResult.intVal; + } virtual bool getBoolVal(rowgroup::Row& row, bool& isNull, ParseTree* lop, ParseTree* rop) { return fResult.boolVal; diff --git a/dbcon/execplan/parsetree.h b/dbcon/execplan/parsetree.h index 313f2906a..2ad41b6bb 100644 --- a/dbcon/execplan/parsetree.h +++ b/dbcon/execplan/parsetree.h @@ -282,6 +282,14 @@ public: return fData->getDatetimeIntVal(row, isNull); } + inline int64_t getTimeIntVal(rowgroup::Row& row, bool& isNull) + { + if (fLeft && fRight) + return (reinterpret_cast(fData))->getTimeIntVal(row, isNull, fLeft, fRight); + else + return fData->getTimeIntVal(row, isNull); + } + private: /** draw the tree * diff --git a/dbcon/execplan/predicateoperator.cpp b/dbcon/execplan/predicateoperator.cpp index 84579df22..166a8edc0 100644 --- a/dbcon/execplan/predicateoperator.cpp +++ b/dbcon/execplan/predicateoperator.cpp @@ -196,6 +196,7 @@ bool PredicateOperator::operator!=(const TreeNode* t) const void PredicateOperator::setOpType(Type& l, Type& r) { if ( l.colDataType == execplan::CalpontSystemCatalog::DATETIME || + l.colDataType == execplan::CalpontSystemCatalog::TIME || l.colDataType == execplan::CalpontSystemCatalog::DATE ) { switch (r.colDataType) @@ -210,6 +211,11 @@ void PredicateOperator::setOpType(Type& l, Type& r) fOperationType.colWidth = 8; break; + case execplan::CalpontSystemCatalog::TIME: + fOperationType.colDataType = execplan::CalpontSystemCatalog::TIME; + fOperationType.colWidth = 8; + break; + case execplan::CalpontSystemCatalog::DATE: fOperationType = l; break; @@ -221,6 +227,7 @@ void PredicateOperator::setOpType(Type& l, Type& r) } } else if ( r.colDataType == execplan::CalpontSystemCatalog::DATETIME || + r.colDataType == execplan::CalpontSystemCatalog::TIME || r.colDataType == execplan::CalpontSystemCatalog::DATE ) { switch (l.colDataType) @@ -236,6 +243,11 @@ void PredicateOperator::setOpType(Type& l, Type& r) fOperationType.colWidth = 8; break; + case execplan::CalpontSystemCatalog::TIME: + fOperationType.colDataType = execplan::CalpontSystemCatalog::TIME; + fOperationType.colWidth = 8; + break; + case execplan::CalpontSystemCatalog::DATE: fOperationType = r; break; diff --git a/dbcon/execplan/predicateoperator.h b/dbcon/execplan/predicateoperator.h index 2b63548d5..6253d9389 100644 --- a/dbcon/execplan/predicateoperator.h +++ b/dbcon/execplan/predicateoperator.h @@ -344,6 +344,37 @@ inline bool PredicateOperator::getBoolVal(rowgroup::Row& row, bool& isNull, Retu return numericCompare(val1, rop->getDatetimeIntVal(row, isNull)) && !isNull; } + case execplan::CalpontSystemCatalog::TIME: + { + if (fOp == OP_ISNULL) + { + lop->getTimeIntVal(row, isNull); + bool ret = isNull; + isNull = false; + return ret; + } + + if (fOp == OP_ISNOTNULL) + { + lop->getTimeIntVal(row, isNull); + bool ret = isNull; + isNull = false; + return !ret; + } + + if (isNull) + return false; + + int64_t val1 = lop->getTimeIntVal(row, isNull); + + if (isNull) + return false; + + return numericCompare(val1, rop->getTimeIntVal(row, isNull)) && !isNull; + } + + + case execplan::CalpontSystemCatalog::VARCHAR: case execplan::CalpontSystemCatalog::CHAR: case execplan::CalpontSystemCatalog::TEXT: diff --git a/dbcon/execplan/simplecolumn.cpp b/dbcon/execplan/simplecolumn.cpp index 429e4f883..64955401e 100644 --- a/dbcon/execplan/simplecolumn.cpp +++ b/dbcon/execplan/simplecolumn.cpp @@ -502,6 +502,7 @@ void SimpleColumn::evaluate(Row& row, bool& isNull) } case CalpontSystemCatalog::DATETIME: + case CalpontSystemCatalog::TIME: { fResult.intVal = row.getUintField<8>(fInputIndex); break; diff --git a/dbcon/execplan/simplecolumn.h b/dbcon/execplan/simplecolumn.h index 81571e057..60eff939b 100644 --- a/dbcon/execplan/simplecolumn.h +++ b/dbcon/execplan/simplecolumn.h @@ -331,6 +331,12 @@ public: return TreeNode::getDatetimeIntVal(); } + inline int64_t getTimeIntVal(rowgroup::Row& row, bool& isNull) + { + evaluate(row, isNull); + return TreeNode::getTimeIntVal(); + } + }; typedef boost::shared_ptr SSC; diff --git a/dbcon/execplan/simplefilter.cpp b/dbcon/execplan/simplefilter.cpp index 1df673dcd..639a04945 100644 --- a/dbcon/execplan/simplefilter.cpp +++ b/dbcon/execplan/simplefilter.cpp @@ -209,7 +209,8 @@ const string SimpleFilter::data() const fRhs->resultType().colDataType == CalpontSystemCatalog::TEXT || fRhs->resultType().colDataType == CalpontSystemCatalog::VARBINARY || fRhs->resultType().colDataType == CalpontSystemCatalog::DATE || - fRhs->resultType().colDataType == CalpontSystemCatalog::DATETIME)) + fRhs->resultType().colDataType == CalpontSystemCatalog::DATETIME || + fRhs->resultType().colDataType == CalpontSystemCatalog::TIME)) rhs = "'" + SimpleFilter::escapeString(fRhs->data()) + "'"; else rhs = fRhs->data(); @@ -221,6 +222,7 @@ const string SimpleFilter::data() const fLhs->resultType().colDataType == CalpontSystemCatalog::TEXT || fLhs->resultType().colDataType == CalpontSystemCatalog::VARBINARY || fLhs->resultType().colDataType == CalpontSystemCatalog::DATE || + fLhs->resultType().colDataType == CalpontSystemCatalog::TIME || fLhs->resultType().colDataType == CalpontSystemCatalog::DATETIME)) lhs = "'" + SimpleFilter::escapeString(fLhs->data()) + "'"; else @@ -544,6 +546,19 @@ void SimpleFilter::convertConstant() result.intVal = dataconvert::DataConvert::datetimeToInt(result.strVal); } } + else if (fRhs->resultType().colDataType == CalpontSystemCatalog::TIME) + { + if (lcc->constval().empty()) + { + lcc->constval("00:00:00"); + result.intVal = 0; + result.strVal = lcc->constval(); + } + else + { + result.intVal = dataconvert::DataConvert::timeToInt(result.strVal); + } + } lcc->result(result); } @@ -578,6 +593,19 @@ void SimpleFilter::convertConstant() result.intVal = dataconvert::DataConvert::datetimeToInt(result.strVal); } } + else if (fLhs->resultType().colDataType == CalpontSystemCatalog::TIME) + { + if (rcc->constval().empty()) + { + rcc->constval("00:00:00"); + result.intVal = 0; + result.strVal = rcc->constval(); + } + else + { + result.intVal = dataconvert::DataConvert::timeToInt(result.strVal); + } + } rcc->result(result); } diff --git a/dbcon/execplan/treenode.h b/dbcon/execplan/treenode.h index 92420da3f..cef9579e9 100644 --- a/dbcon/execplan/treenode.h +++ b/dbcon/execplan/treenode.h @@ -388,6 +388,10 @@ public: { return fResult.intVal; } + virtual int64_t getTimeIntVal(rowgroup::Row& row, bool& isNull) + { + return fResult.intVal; + } virtual void evaluate(rowgroup::Row& row, bool& isNull) {} inline bool getBoolVal(); @@ -399,6 +403,7 @@ public: inline IDB_Decimal getDecimalVal(); inline int32_t getDateIntVal(); inline int64_t getDatetimeIntVal(); + inline int64_t getTimeIntVal(); virtual const execplan::CalpontSystemCatalog::ColType& resultType() const { @@ -490,6 +495,7 @@ inline bool TreeNode::getBoolVal() case CalpontSystemCatalog::INT: case CalpontSystemCatalog::DATE: case CalpontSystemCatalog::DATETIME: + case CalpontSystemCatalog::TIME: return (fResult.intVal != 0); case CalpontSystemCatalog::UBIGINT: @@ -660,7 +666,14 @@ inline const std::string& TreeNode::getStrVal() case CalpontSystemCatalog::DATETIME: { - dataconvert::DataConvert::datetimeToString(fResult.intVal, tmp, 255); + dataconvert::DataConvert::datetimeToString(fResult.intVal, tmp, 255, fResultType.precision); + fResult.strVal = std::string(tmp); + break; + } + + case CalpontSystemCatalog::TIME: + { + dataconvert::DataConvert::timeToString(fResult.intVal, tmp, 255, fResultType.precision); fResult.strVal = std::string(tmp); break; } @@ -727,6 +740,7 @@ inline int64_t TreeNode::getIntVal() case CalpontSystemCatalog::DATE: case CalpontSystemCatalog::DATETIME: + case CalpontSystemCatalog::TIME: return fResult.intVal; default: @@ -769,6 +783,7 @@ inline uint64_t TreeNode::getUintVal() case CalpontSystemCatalog::DATE: case CalpontSystemCatalog::DATETIME: + case CalpontSystemCatalog::TIME: return fResult.intVal; default: @@ -831,6 +846,7 @@ inline float TreeNode::getFloatVal() case CalpontSystemCatalog::DATE: case CalpontSystemCatalog::DATETIME: + case CalpontSystemCatalog::TIME: return (float)fResult.intVal; default: @@ -895,6 +911,7 @@ inline double TreeNode::getDoubleVal() case CalpontSystemCatalog::DATE: case CalpontSystemCatalog::DATETIME: + case CalpontSystemCatalog::TIME: return (double)fResult.intVal; default: @@ -937,9 +954,14 @@ inline IDB_Decimal TreeNode::getDecimalVal() break; case CalpontSystemCatalog::DATE: + throw logging::InvalidConversionExcept("TreeNode::getDecimalVal: Invalid conversion from date."); + case CalpontSystemCatalog::DATETIME: throw logging::InvalidConversionExcept("TreeNode::getDecimalVal: Invalid conversion from datetime."); + case CalpontSystemCatalog::TIME: + throw logging::InvalidConversionExcept("TreeNode::getDecimalVal: Invalid conversion from time."); + case CalpontSystemCatalog::FLOAT: throw logging::InvalidConversionExcept("TreeNode::getDecimalVal: non-support conversion from float"); @@ -967,6 +989,28 @@ inline int64_t TreeNode::getDatetimeIntVal() { if (fResultType.colDataType == execplan::CalpontSystemCatalog::DATE) return (fResult.intVal & 0x00000000FFFFFFC0LL) << 32; + else if (fResultType.colDataType == execplan::CalpontSystemCatalog::TIME) + { + dataconvert::Time tt; + int day = 0; + + memcpy(&tt, &fResult.intVal, 8); + + // Note, this should probably be current date +/- time + if ((tt.hour > 23) && (!tt.is_neg)) + { + day = tt.hour / 24; + tt.hour = tt.hour % 24; + } + else if ((tt.hour < 0) || (tt.is_neg)) + { + tt.hour = 0; + } + + dataconvert::DateTime dt(0, 0, day, tt.hour, tt.minute, tt.second, tt.msecond); + memcpy(&fResult.intVal, &dt, 8); + return fResult.intVal; + } else if (fResultType.colDataType == execplan::CalpontSystemCatalog::DATETIME) //return (fResult.intVal & 0xFFFFFFFFFFF00000LL); return (fResult.intVal); @@ -974,6 +1018,23 @@ inline int64_t TreeNode::getDatetimeIntVal() return getIntVal(); } +inline int64_t TreeNode::getTimeIntVal() +{ + if (fResultType.colDataType == execplan::CalpontSystemCatalog::DATETIME) + { + dataconvert::DateTime dt; + + memcpy(&dt, &fResult.intVal, 8); + dataconvert::Time tt(0, dt.hour, dt.minute, dt.second, dt.msecond, false); + memcpy(&fResult.intVal, &tt, 8); + return fResult.intVal; + } + else if (fResultType.colDataType == execplan::CalpontSystemCatalog::TIME) + return (fResult.intVal); + else + return getIntVal(); +} + inline int32_t TreeNode::getDateIntVal() { if (fResultType.colDataType == execplan::CalpontSystemCatalog::DATETIME) diff --git a/dbcon/execplan/windowfunctioncolumn.cpp b/dbcon/execplan/windowfunctioncolumn.cpp index f81a6a5d4..5c84ff2d1 100644 --- a/dbcon/execplan/windowfunctioncolumn.cpp +++ b/dbcon/execplan/windowfunctioncolumn.cpp @@ -403,6 +403,16 @@ void WindowFunctionColumn::evaluate(Row& row, bool& isNull) break; } + case CalpontSystemCatalog::TIME: + { + if (row.equals<8>(TIMENULL, fInputIndex)) + isNull = true; + else + fResult.intVal = row.getIntField<8>(fInputIndex); + + break; + } + case CalpontSystemCatalog::CHAR: case CalpontSystemCatalog::VARCHAR: case CalpontSystemCatalog::STRINT: diff --git a/dbcon/execplan/windowfunctioncolumn.h b/dbcon/execplan/windowfunctioncolumn.h index 8b427947c..3fc983e55 100644 --- a/dbcon/execplan/windowfunctioncolumn.h +++ b/dbcon/execplan/windowfunctioncolumn.h @@ -213,7 +213,11 @@ public: evaluate(row, isNull); return TreeNode::getDatetimeIntVal(); } - + virtual int64_t getTimeIntVal(rowgroup::Row& row, bool& isNull) + { + evaluate(row, isNull); + return TreeNode::getTimeIntVal(); + } private: void evaluate(rowgroup::Row& row, bool& isNull); }; diff --git a/dbcon/joblist/crossenginestep.cpp b/dbcon/joblist/crossenginestep.cpp index 483249067..789e58cd4 100644 --- a/dbcon/joblist/crossenginestep.cpp +++ b/dbcon/joblist/crossenginestep.cpp @@ -351,6 +351,10 @@ int64_t CrossEngineStep::convertValueNum( rv = boost::any_cast(anyVal); break; + case CalpontSystemCatalog::TIME: + rv = boost::any_cast(anyVal); + break; + case CalpontSystemCatalog::DECIMAL: case CalpontSystemCatalog::UDECIMAL: if (ct.colWidth == CalpontSystemCatalog::ONE_BYTE) diff --git a/dbcon/joblist/expressionstep.cpp b/dbcon/joblist/expressionstep.cpp index 4a169e081..0e064c359 100644 --- a/dbcon/joblist/expressionstep.cpp +++ b/dbcon/joblist/expressionstep.cpp @@ -403,8 +403,9 @@ void ExpressionStep::populateColumnInfo(SimpleColumn* sc, JobInfo& jobInfo) TupleInfo ti(setTupleInfo(ct, sc->oid(), jobInfo, tblOid, sc, alias)); fColumnKeys.push_back(ti.key); - // @bug 2990, MySQL date/datetime type is different from IDB type - if (ti.dtype == CalpontSystemCatalog::DATE || ti.dtype == CalpontSystemCatalog::DATETIME) + // @bug 2990, MySQL time/date/datetime type is different from IDB type + if (ti.dtype == CalpontSystemCatalog::DATE || ti.dtype == CalpontSystemCatalog::DATETIME || + ti.dtype == CalpontSystemCatalog::TIME) { if (ti.dtype != ct.colDataType) { diff --git a/dbcon/joblist/groupconcat.cpp b/dbcon/joblist/groupconcat.cpp index fd122fe8d..234fc0a8e 100644 --- a/dbcon/joblist/groupconcat.cpp +++ b/dbcon/joblist/groupconcat.cpp @@ -507,6 +507,12 @@ void GroupConcator::outputRow(std::ostringstream& oss, const rowgroup::Row& row) break; } + case CalpontSystemCatalog::TIME: + { + oss << DataConvert::timeToString(row.getUintField(*i)); + break; + } + default: { break; @@ -640,6 +646,28 @@ int64_t GroupConcator::lengthEstimate(const rowgroup::Row& row) case CalpontSystemCatalog::DATETIME: { fieldLen = 19; // YYYY-MM-DD HH24:MI:SS + // Decimal point and milliseconds + uint64_t colPrecision = row.getPrecision(*i); + + if (colPrecision > 0 && colPrecision < 7) + { + fieldLen += colPrecision + 1; + } + + break; + } + + case CalpontSystemCatalog::TIME: + { + fieldLen = 10; // -HHH:MI:SS + // Decimal point and milliseconds + uint64_t colPrecision = row.getPrecision(*i); + + if (colPrecision > 0 && colPrecision < 7) + { + fieldLen += colPrecision + 1; + } + break; } diff --git a/dbcon/joblist/jlf_common.cpp b/dbcon/joblist/jlf_common.cpp index 063e24dfc..f5dbeee17 100644 --- a/dbcon/joblist/jlf_common.cpp +++ b/dbcon/joblist/jlf_common.cpp @@ -792,6 +792,11 @@ bool compatibleColumnTypes(const CalpontSystemCatalog::ColDataType& dt1, uint32_ break; + case CalpontSystemCatalog::TIME: + if (dt2 != CalpontSystemCatalog::TIME) return false; + + break; + case CalpontSystemCatalog::CHAR: case CalpontSystemCatalog::VARCHAR: case CalpontSystemCatalog::TEXT: diff --git a/dbcon/joblist/jlf_execplantojoblist.cpp b/dbcon/joblist/jlf_execplantojoblist.cpp index f4badbe57..b81a2ec8d 100644 --- a/dbcon/joblist/jlf_execplantojoblist.cpp +++ b/dbcon/joblist/jlf_execplantojoblist.cpp @@ -282,6 +282,10 @@ int64_t valueNullNum(const CalpontSystemCatalog::ColType& ct) n = boost::any_cast(anyVal); break; + case CalpontSystemCatalog::TIME: + n = boost::any_cast(anyVal); + break; + case CalpontSystemCatalog::DECIMAL: case CalpontSystemCatalog::UDECIMAL: if (ct.colWidth == execplan::CalpontSystemCatalog::ONE_BYTE) @@ -421,6 +425,10 @@ int64_t convertValueNum(const string& str, const CalpontSystemCatalog::ColType& v = boost::any_cast(anyVal); break; + case CalpontSystemCatalog::TIME: + v = boost::any_cast(anyVal); + break; + case CalpontSystemCatalog::DECIMAL: case CalpontSystemCatalog::UDECIMAL: if (ct.colWidth == execplan::CalpontSystemCatalog::ONE_BYTE) @@ -2099,6 +2107,7 @@ const JobStepVector doOuterJoinOnFilter(OuterJoinOnFilter* oj, JobInfo& jobInfo) set doneNodes; // solved joins and simple filters map cpMap; // link for node removal JobStepVector join; // join step with its projection steps + bool keepFilters = false; // keep filters for cross engine step // To compromise the front end difficulty on setting outer attributes. set tablesInOuter; @@ -2323,6 +2332,14 @@ const JobStepVector doOuterJoinOnFilter(OuterJoinOnFilter* oj, JobInfo& jobInfo) jsv.insert(jsv.end(), sfv.begin(), sfv.end()); + // MCOL-1182 if we are doing a join between a cross engine + // step and a constant then keep the filter for the cross + // engine step instead of deleting it further down. + if (!sc->isInfiniDB()) + { + keepFilters = true; + } + doneNodes.insert(cn); } } @@ -2364,7 +2381,11 @@ const JobStepVector doOuterJoinOnFilter(OuterJoinOnFilter* oj, JobInfo& jobInfo) if (p == NULL) { filters = NULL; - delete c; + + if (!keepFilters) + { + delete c; + } } else { @@ -2411,8 +2432,12 @@ const JobStepVector doOuterJoinOnFilter(OuterJoinOnFilter* oj, JobInfo& jobInfo) p->left(nullTree); p->right(nullTree); - delete p; - delete c; + + if (!keepFilters) + { + delete p; + delete c; + } } } diff --git a/dbcon/joblist/jlf_subquery.cpp b/dbcon/joblist/jlf_subquery.cpp index e1537d577..add3dc533 100644 --- a/dbcon/joblist/jlf_subquery.cpp +++ b/dbcon/joblist/jlf_subquery.cpp @@ -135,6 +135,11 @@ void getColumnValue(ConstantColumn** cc, uint64_t i, const Row& row) *cc = new ConstantColumn(oss.str()); break; + case CalpontSystemCatalog::TIME: + oss << dataconvert::DataConvert::timeToString(row.getUintField<8>(i)); + *cc = new ConstantColumn(oss.str()); + break; + case CalpontSystemCatalog::CHAR: case CalpontSystemCatalog::VARCHAR: case CalpontSystemCatalog::TEXT: @@ -751,8 +756,8 @@ int doFromSubquery(CalpontExecutionPlan* ep, const string& alias, const string& void addOrderByAndLimit(CalpontSelectExecutionPlan* csep, JobInfo& jobInfo) { // make sure there is a LIMIT - if (csep->orderByCols().size() > 0 && csep->limitNum() == (uint64_t) - 1) - return; +// if (csep->orderByCols().size() > 0 csep->limitNum() == (uint64_t) - 1) +// return; jobInfo.limitStart = csep->limitStart(); jobInfo.limitCount = csep->limitNum(); diff --git a/dbcon/joblist/jlf_tuplejoblist.cpp b/dbcon/joblist/jlf_tuplejoblist.cpp index aabc81486..b006eb90c 100644 --- a/dbcon/joblist/jlf_tuplejoblist.cpp +++ b/dbcon/joblist/jlf_tuplejoblist.cpp @@ -483,28 +483,30 @@ void adjustLastStep(JobStepVector& querySteps, DeliveredTableMap& deliverySteps, deliverySteps[CNX_VTABLE_ID] = ws; } - if ((jobInfo.limitCount != (uint64_t) - 1) || - (jobInfo.constantCol == CONST_COL_EXIST) || - (jobInfo.hasDistinct)) - { - if (jobInfo.annexStep.get() == NULL) - jobInfo.annexStep.reset(new TupleAnnexStep(jobInfo)); +// if ((jobInfo.limitCount != (uint64_t) - 1) || +// (jobInfo.constantCol == CONST_COL_EXIST) || +// (jobInfo.hasDistinct)) +// { + if (jobInfo.annexStep.get() == NULL) + jobInfo.annexStep.reset(new TupleAnnexStep(jobInfo)); - TupleAnnexStep* tas = dynamic_cast(jobInfo.annexStep.get()); - tas->setLimit(jobInfo.limitStart, jobInfo.limitCount); + TupleAnnexStep* tas = dynamic_cast(jobInfo.annexStep.get()); + tas->setLimit(jobInfo.limitStart, jobInfo.limitCount); - if (jobInfo.limitCount != (uint64_t) - 1) - { - if (jobInfo.orderByColVec.size() > 0) - tas->addOrderBy(new LimitedOrderBy()); - } +// if (jobInfo.limitCount != (uint64_t) - 1) +// { + if (jobInfo.orderByColVec.size() > 0) + tas->addOrderBy(new LimitedOrderBy()); - if (jobInfo.constantCol == CONST_COL_EXIST) - tas->addConstant(new TupleConstantStep(jobInfo)); +// } - if (jobInfo.hasDistinct) - tas->setDistinct(); - } + if (jobInfo.constantCol == CONST_COL_EXIST) + tas->addConstant(new TupleConstantStep(jobInfo)); + + if (jobInfo.hasDistinct) + tas->setDistinct(); + +// } if (jobInfo.annexStep) { diff --git a/dbcon/joblist/joblistfactory.cpp b/dbcon/joblist/joblistfactory.cpp index 812256869..a48ecd13a 100644 --- a/dbcon/joblist/joblistfactory.cpp +++ b/dbcon/joblist/joblistfactory.cpp @@ -1622,7 +1622,7 @@ void makeVtableModeSteps(CalpontSelectExecutionPlan* csep, JobInfo& jobInfo, JobStepVector& querySteps, JobStepVector& projectSteps, DeliveredTableMap& deliverySteps) { // @bug4848, enhance and unify limit handling. - if (csep->limitNum() != (uint64_t) - 1) +// if (csep->limitNum() != (uint64_t) - 1) { // special case for outer query order by limit -- return all if (jobInfo.subId == 0 && csep->hasOrderBy()) diff --git a/dbcon/joblist/joblisttypes.h b/dbcon/joblist/joblisttypes.h index 3f3755a71..37d52f9f4 100644 --- a/dbcon/joblist/joblisttypes.h +++ b/dbcon/joblist/joblisttypes.h @@ -56,6 +56,8 @@ const uint32_t DATENULL = 0xFFFFFFFE; const uint32_t DATEEMPTYROW = 0xFFFFFFFF; const uint64_t DATETIMENULL = 0xFFFFFFFFFFFFFFFEULL; const uint64_t DATETIMEEMPTYROW = 0xFFFFFFFFFFFFFFFFULL; +const uint64_t TIMENULL = 0xFFFFFFFFFFFFFFFEULL; +const uint64_t TIMEEMPTYROW = 0xFFFFFFFFFFFFFFFFULL; const uint8_t CHAR1NULL = 0xFE; const uint8_t CHAR1EMPTYROW = 0xFF; diff --git a/dbcon/joblist/lbidlist.cpp b/dbcon/joblist/lbidlist.cpp index bd012941e..c317defc9 100644 --- a/dbcon/joblist/lbidlist.cpp +++ b/dbcon/joblist/lbidlist.cpp @@ -510,6 +510,7 @@ bool LBIDList::CasualPartitionDataType(const CalpontSystemCatalog::ColDataType t case CalpontSystemCatalog::DECIMAL: case CalpontSystemCatalog::DATE: case CalpontSystemCatalog::DATETIME: + case CalpontSystemCatalog::TIME: case CalpontSystemCatalog::UTINYINT: case CalpontSystemCatalog::USMALLINT: case CalpontSystemCatalog::UDECIMAL: diff --git a/dbcon/joblist/limitedorderby.cpp b/dbcon/joblist/limitedorderby.cpp index e9bcaf95e..82d6041a8 100644 --- a/dbcon/joblist/limitedorderby.cpp +++ b/dbcon/joblist/limitedorderby.cpp @@ -76,7 +76,15 @@ void LimitedOrderBy::initialize(const RowGroup& rg, const JobInfo& jobInfo) { map::iterator j = keyToIndexMap.find(i->first); idbassert(j != keyToIndexMap.end()); - fOrderByCond.push_back(IdbSortSpec(j->second, i->second)); + + // MCOL-1052 Ordering direction in CSEP differs from + // internal direction representation. + if (i->second) + fOrderByCond.push_back(IdbSortSpec(j->second, false)); + else + fOrderByCond.push_back(IdbSortSpec(j->second, true)); + + //fOrderByCond.push_back(IdbSortSpec(j->second, i->second)); } // limit row count info @@ -174,7 +182,9 @@ void LimitedOrderBy::finalize() if (fRowGroup.getRowCount() > 0) fDataQueue.push(fData); - if (fStart != 0) + // MCOL-1052 The removed check effectivly disables sorting to happen, + // since fStart = 0; + if (true) { uint64_t newSize = fRowsPerRG * fRowGroup.getRowSize(); fMemSize += newSize; diff --git a/dbcon/joblist/pcolscan.cpp b/dbcon/joblist/pcolscan.cpp index cd19ce34c..6cea4fc03 100644 --- a/dbcon/joblist/pcolscan.cpp +++ b/dbcon/joblist/pcolscan.cpp @@ -1167,6 +1167,7 @@ bool pColScanStep::isEmptyVal(const uint8_t* val8) const case CalpontSystemCatalog::VARCHAR: case CalpontSystemCatalog::DATE: case CalpontSystemCatalog::DATETIME: + case CalpontSystemCatalog::TIME: if (width == 1) { return (*val8 == joblist::CHAR1EMPTYROW); diff --git a/dbcon/joblist/subquerytransformer.cpp b/dbcon/joblist/subquerytransformer.cpp index a5e0759fa..c212c701d 100644 --- a/dbcon/joblist/subquerytransformer.cpp +++ b/dbcon/joblist/subquerytransformer.cpp @@ -237,9 +237,10 @@ SJSTEP& SubQueryTransformer::makeSubQueryStep(execplan::CalpontSelectExecutionPl fVtable.columnType(ct, i); } } - // MySQL date/datetime type is different from IDB type + // MySQL time/date/datetime type is different from IDB type else if (colDataTypeInRg == CalpontSystemCatalog::DATE || - colDataTypeInRg == CalpontSystemCatalog::DATETIME) + colDataTypeInRg == CalpontSystemCatalog::DATETIME || + colDataTypeInRg == CalpontSystemCatalog::TIME) { ct.colWidth = row.getColumnWidth(i); ct.colDataType = row.getColTypes()[i]; diff --git a/dbcon/joblist/tupleaggregatestep.cpp b/dbcon/joblist/tupleaggregatestep.cpp index a77cbcdd1..9e23ac17b 100644 --- a/dbcon/joblist/tupleaggregatestep.cpp +++ b/dbcon/joblist/tupleaggregatestep.cpp @@ -232,6 +232,9 @@ inline string colTypeIdString(CalpontSystemCatalog::ColDataType type) case CalpontSystemCatalog::DATETIME: return string("DATETIME"); + case CalpontSystemCatalog::TIME: + return string("TIME"); + case CalpontSystemCatalog::VARCHAR: return string("VARCHAR"); @@ -1091,6 +1094,7 @@ void TupleAggregateStep::prep1PhaseAggregate( vector functionVec; uint32_t bigIntWidth = sizeof(int64_t); uint32_t bigUintWidth = sizeof(uint64_t); + uint32_t projColsUDAFIndex = 0; mcsv1sdk::mcsv1_UDAF* pUDAFFunc = NULL; // for count column of average function @@ -1279,16 +1283,26 @@ void TupleAggregateStep::prep1PhaseAggregate( if (aggOp == ROWAGG_UDAF) { - UDAFColumn* udafc = dynamic_cast(jobInfo.projectionCols[i].get()); + std::vector::iterator it = jobInfo.projectionCols.begin() + projColsUDAFIndex; - if (udafc) + for (; it != jobInfo.projectionCols.end(); it++) { - // Create a RowAggFunctionCol (UDAF subtype) with the context. - funct.reset(new RowUDAFFunctionCol(udafc->getContext(), colProj, i)); + UDAFColumn* udafc = dynamic_cast((*it).get()); + projColsUDAFIndex++; + + if (udafc) + { + pUDAFFunc = udafc->getContext().getFunction(); + // Create a RowAggFunctionCol (UDAF subtype) with the context. + funct.reset(new RowUDAFFunctionCol(udafc->getContext(), colProj, i)); + break; + } + } - else + + if (it == jobInfo.projectionCols.end()) { - throw logic_error("prep1PhasesAggregate: A UDAF function is called but there's no UDAFColumn"); + throw logic_error("prep1PhaseAggregate: A UDAF function is called but there's no/not enough UDAFColumn/-s"); } } else @@ -1322,7 +1336,8 @@ void TupleAggregateStep::prep1PhaseAggregate( typeProj[colProj] == CalpontSystemCatalog::BLOB || typeProj[colProj] == CalpontSystemCatalog::TEXT || typeProj[colProj] == CalpontSystemCatalog::DATE || - typeProj[colProj] == CalpontSystemCatalog::DATETIME) + typeProj[colProj] == CalpontSystemCatalog::DATETIME || + typeProj[colProj] == CalpontSystemCatalog::TIME) { Message::Args args; args.add("sum/average"); @@ -1404,7 +1419,8 @@ void TupleAggregateStep::prep1PhaseAggregate( typeProj[colProj] == CalpontSystemCatalog::TEXT || typeProj[colProj] == CalpontSystemCatalog::BLOB || typeProj[colProj] == CalpontSystemCatalog::DATE || - typeProj[colProj] == CalpontSystemCatalog::DATETIME) + typeProj[colProj] == CalpontSystemCatalog::DATETIME || + typeProj[colProj] == CalpontSystemCatalog::TIME) { Message::Args args; args.add("variance/standard deviation"); @@ -1649,6 +1665,7 @@ void TupleAggregateStep::prep1PhaseDistinctAggregate( AGG_MAP aggFuncMap; mcsv1sdk::mcsv1_UDAF* pUDAFFunc = NULL; set avgSet; + uint32_t projColsUDAFIndex = 0; // for count column of average function map avgFuncMap, avgDistFuncMap; @@ -1812,17 +1829,26 @@ void TupleAggregateStep::prep1PhaseDistinctAggregate( if (aggOp == ROWAGG_UDAF) { - UDAFColumn* udafc = dynamic_cast(jobInfo.projectionCols[i].get()); + std::vector::iterator it = jobInfo.projectionCols.begin() + projColsUDAFIndex; - if (udafc) + for (; it != jobInfo.projectionCols.end(); it++) { - pUDAFFunc = udafc->getContext().getFunction(); - // Create a RowAggFunctionCol (UDAF subtype) with the context. - funct.reset(new RowUDAFFunctionCol(udafc->getContext(), colProj, colAgg)); + UDAFColumn* udafc = dynamic_cast((*it).get()); + projColsUDAFIndex++; + + if (udafc) + { + pUDAFFunc = udafc->getContext().getFunction(); + // Create a RowAggFunctionCol (UDAF subtype) with the context. + funct.reset(new RowUDAFFunctionCol(udafc->getContext(), colProj, colAgg)); + break; + } + } - else + + if (it == jobInfo.projectionCols.end()) { - throw logic_error("prep1PhaseDistinctAggregate: A UDAF function is called but there's no UDAFColumn"); + throw logic_error("prep1PhaseDistinctAggregate: A UDAF function is called but there's no/not enough UDAFColumn/-s"); } } else @@ -1860,7 +1886,8 @@ void TupleAggregateStep::prep1PhaseDistinctAggregate( typeProj[colProj] == CalpontSystemCatalog::BLOB || typeProj[colProj] == CalpontSystemCatalog::TEXT || typeProj[colProj] == CalpontSystemCatalog::DATE || - typeProj[colProj] == CalpontSystemCatalog::DATETIME) + typeProj[colProj] == CalpontSystemCatalog::DATETIME || + typeProj[colProj] == CalpontSystemCatalog::TIME) { Message::Args args; args.add("sum/average"); @@ -1945,7 +1972,8 @@ void TupleAggregateStep::prep1PhaseDistinctAggregate( typeProj[colProj] == CalpontSystemCatalog::BLOB || typeProj[colProj] == CalpontSystemCatalog::TEXT || typeProj[colProj] == CalpontSystemCatalog::DATE || - typeProj[colProj] == CalpontSystemCatalog::DATETIME) + typeProj[colProj] == CalpontSystemCatalog::DATETIME || + typeProj[colProj] == CalpontSystemCatalog::TIME) { Message::Args args; args.add("variance/standard deviation"); @@ -2128,7 +2156,8 @@ void TupleAggregateStep::prep1PhaseDistinctAggregate( typeAgg[colAgg] == CalpontSystemCatalog::BLOB || typeAgg[colAgg] == CalpontSystemCatalog::TEXT || typeAgg[colAgg] == CalpontSystemCatalog::DATE || - typeAgg[colAgg] == CalpontSystemCatalog::DATETIME) + typeAgg[colAgg] == CalpontSystemCatalog::DATETIME || + typeAgg[colAgg] == CalpontSystemCatalog::TIME) { Message::Args args; args.add("sum/average"); @@ -2814,6 +2843,7 @@ void TupleAggregateStep::prep2PhasesAggregate( vector > aggColVec; set avgSet; vector >& returnedColVec = jobInfo.returnedColVec; + uint32_t projColsUDAFIndex = 0; for (uint64_t i = 0; i < returnedColVec.size(); i++) { @@ -2992,17 +3022,26 @@ void TupleAggregateStep::prep2PhasesAggregate( if (aggOp == ROWAGG_UDAF) { - UDAFColumn* udafc = dynamic_cast(jobInfo.projectionCols[i].get()); + std::vector::iterator it = jobInfo.projectionCols.begin() + projColsUDAFIndex; - if (udafc) + for (; it != jobInfo.projectionCols.end(); it++) { - pUDAFFunc = udafc->getContext().getFunction(); - // Create a RowAggFunctionCol (UDAF subtype) with the context. - funct.reset(new RowUDAFFunctionCol(udafc->getContext(), colProj, colAggPm)); + UDAFColumn* udafc = dynamic_cast((*it).get()); + projColsUDAFIndex++; + + if (udafc) + { + pUDAFFunc = udafc->getContext().getFunction(); + // Create a RowAggFunctionCol (UDAF subtype) with the context. + funct.reset(new RowUDAFFunctionCol(udafc->getContext(), colProj, colAggPm)); + break; + } + } - else + + if (it == jobInfo.projectionCols.end()) { - throw logic_error("prep2PhasesAggregate: A UDAF function is called but there's no UDAFColumn"); + throw logic_error("prep2PhasesAggregate: A UDAF function is called but there's no/not enough UDAFColumn/-s"); } } else @@ -3040,7 +3079,8 @@ void TupleAggregateStep::prep2PhasesAggregate( typeProj[colProj] == CalpontSystemCatalog::BLOB || typeProj[colProj] == CalpontSystemCatalog::TEXT || typeProj[colProj] == CalpontSystemCatalog::DATE || - typeProj[colProj] == CalpontSystemCatalog::DATETIME) + typeProj[colProj] == CalpontSystemCatalog::DATETIME || + typeProj[colProj] == CalpontSystemCatalog::TIME) { Message::Args args; args.add("sum/average"); @@ -3129,7 +3169,8 @@ void TupleAggregateStep::prep2PhasesAggregate( typeProj[colProj] == CalpontSystemCatalog::BLOB || typeProj[colProj] == CalpontSystemCatalog::TEXT || typeProj[colProj] == CalpontSystemCatalog::DATE || - typeProj[colProj] == CalpontSystemCatalog::DATETIME) + typeProj[colProj] == CalpontSystemCatalog::DATETIME || + typeProj[colProj] == CalpontSystemCatalog::TIME) { Message::Args args; args.add("variance/standard deviation"); @@ -3583,6 +3624,7 @@ void TupleAggregateStep::prep2PhasesDistinctAggregate( vector > aggColVec, aggNoDistColVec; set avgSet, avgDistSet; vector >& returnedColVec = jobInfo.returnedColVec; + uint32_t projColsUDAFIndex = 0; for (uint64_t i = 0; i < returnedColVec.size(); i++) { @@ -3796,17 +3838,25 @@ void TupleAggregateStep::prep2PhasesDistinctAggregate( if (aggOp == ROWAGG_UDAF) { - UDAFColumn* udafc = dynamic_cast(jobInfo.projectionCols[i].get()); + std::vector::iterator it = jobInfo.projectionCols.begin() + projColsUDAFIndex; - if (udafc) + for (; it != jobInfo.projectionCols.end(); it++) { - pUDAFFunc = udafc->getContext().getFunction(); - // Create a RowAggFunctionCol (UDAF subtype) with the context. - funct.reset(new RowUDAFFunctionCol(udafc->getContext(), colProj, colAggPm)); + UDAFColumn* udafc = dynamic_cast((*it).get()); + projColsUDAFIndex++; + + if (udafc) + { + pUDAFFunc = udafc->getContext().getFunction(); + // Create a RowAggFunctionCol (UDAF subtype) with the context. + funct.reset(new RowUDAFFunctionCol(udafc->getContext(), colProj, colAggPm)); + break; + } } - else + + if (it == jobInfo.projectionCols.end()) { - throw logic_error("prep2PhasesDistinctAggregate: A UDAF function is called but there's no UDAFColumn"); + throw logic_error("prep2PhasesDistinctAggregate: A UDAF function is called but there's no/not enough UDAFColumn/-s"); } } else @@ -3844,7 +3894,8 @@ void TupleAggregateStep::prep2PhasesDistinctAggregate( typeProj[colProj] == CalpontSystemCatalog::BLOB || typeProj[colProj] == CalpontSystemCatalog::TEXT || typeProj[colProj] == CalpontSystemCatalog::DATE || - typeProj[colProj] == CalpontSystemCatalog::DATETIME) + typeProj[colProj] == CalpontSystemCatalog::DATETIME || + typeProj[colProj] == CalpontSystemCatalog::TIME) { Message::Args args; args.add("sum/average"); @@ -3929,7 +3980,8 @@ void TupleAggregateStep::prep2PhasesDistinctAggregate( typeProj[colProj] == CalpontSystemCatalog::BLOB || typeProj[colProj] == CalpontSystemCatalog::TEXT || typeProj[colProj] == CalpontSystemCatalog::DATE || - typeProj[colProj] == CalpontSystemCatalog::DATETIME) + typeProj[colProj] == CalpontSystemCatalog::DATETIME || + typeProj[colProj] == CalpontSystemCatalog::TIME) { Message::Args args; args.add("variance/standard deviation"); @@ -4144,7 +4196,8 @@ void TupleAggregateStep::prep2PhasesDistinctAggregate( typeAggUm[colUm] == CalpontSystemCatalog::BLOB || typeAggUm[colUm] == CalpontSystemCatalog::TEXT || typeAggUm[colUm] == CalpontSystemCatalog::DATE || - typeAggUm[colUm] == CalpontSystemCatalog::DATETIME) + typeAggUm[colUm] == CalpontSystemCatalog::DATETIME || + typeAggUm[colUm] == CalpontSystemCatalog::TIME) { Message::Args args; args.add("sum/average"); diff --git a/dbcon/joblist/tupleconstantstep.cpp b/dbcon/joblist/tupleconstantstep.cpp index 6e75ad350..309bb0058 100644 --- a/dbcon/joblist/tupleconstantstep.cpp +++ b/dbcon/joblist/tupleconstantstep.cpp @@ -223,6 +223,7 @@ void TupleConstantStep::constructContanstRow(const JobInfo& jobInfo) case CalpontSystemCatalog::BIGINT: case CalpontSystemCatalog::DATE: case CalpontSystemCatalog::DATETIME: + case CalpontSystemCatalog::TIME: { fRowConst.setIntField(c.intVal, *i); break; diff --git a/dbcon/joblist/tupleunion.cpp b/dbcon/joblist/tupleunion.cpp index 51b5cde97..d0892e45f 100644 --- a/dbcon/joblist/tupleunion.cpp +++ b/dbcon/joblist/tupleunion.cpp @@ -48,7 +48,7 @@ using namespace dataconvert; namespace { //returns the value of 10 raised to the power x. -inline double pow10(double x) +inline double exp10(double x) { return exp(x * M_LN10); } @@ -460,7 +460,7 @@ void TupleUnion::normalize(const Row& in, Row* out) if (in.getScale(i)) { double d = in.getIntField(i); - d /= pow10(in.getScale(i)); + d /= exp10(in.getScale(i)); os.precision(15); os << d; } @@ -473,7 +473,8 @@ void TupleUnion::normalize(const Row& in, Row* out) case CalpontSystemCatalog::DATE: case CalpontSystemCatalog::DATETIME: - throw logic_error("TupleUnion::normalize(): tried to normalize an int to a date or datetime"); + case CalpontSystemCatalog::TIME: + throw logic_error("TupleUnion::normalize(): tried to normalize an int to a time, date or datetime"); case CalpontSystemCatalog::FLOAT: case CalpontSystemCatalog::UFLOAT: @@ -569,7 +570,7 @@ dec1: if (in.getScale(i)) { double d = in.getUintField(i); - d /= pow10(in.getScale(i)); + d /= exp10(in.getScale(i)); os.precision(15); os << d; } @@ -582,7 +583,8 @@ dec1: case CalpontSystemCatalog::DATE: case CalpontSystemCatalog::DATETIME: - throw logic_error("TupleUnion::normalize(): tried to normalize an int to a date or datetime"); + case CalpontSystemCatalog::TIME: + throw logic_error("TupleUnion::normalize(): tried to normalize an int to a time, date or datetime"); case CalpontSystemCatalog::FLOAT: case CalpontSystemCatalog::UFLOAT: @@ -736,6 +738,33 @@ dec2: break; + case CalpontSystemCatalog::TIME: + switch (out->getColTypes()[i]) + { + case CalpontSystemCatalog::TIME: + out->setIntField(in.getIntField(i), i); + break; + + case CalpontSystemCatalog::CHAR: + case CalpontSystemCatalog::TEXT: + case CalpontSystemCatalog::VARCHAR: + { + string d = DataConvert::timeToString(in.getIntField(i)); + out->setStringField(d, i); + break; + } + + default: + { + ostringstream os; + os << "TupleUnion::normalize(): tried an illegal conversion: time to " + << out->getColTypes()[i]; + throw logic_error(os.str()); + } + } + + break; + case CalpontSystemCatalog::FLOAT: case CalpontSystemCatalog::UFLOAT: case CalpontSystemCatalog::DOUBLE: @@ -1069,6 +1098,10 @@ void TupleUnion::writeNull(Row* out, uint32_t col) out->setUintField<8>(joblist::DATETIMENULL, col); break; + case CalpontSystemCatalog::TIME: + out->setUintField<8>(joblist::TIMENULL, col); + break; + case CalpontSystemCatalog::CHAR: case CalpontSystemCatalog::TEXT: case CalpontSystemCatalog::VARCHAR: diff --git a/dbcon/joblist/windowfunctionstep.cpp b/dbcon/joblist/windowfunctionstep.cpp index ed382ee4a..4d24f0b4b 100644 --- a/dbcon/joblist/windowfunctionstep.cpp +++ b/dbcon/joblist/windowfunctionstep.cpp @@ -1201,6 +1201,7 @@ boost::shared_ptr WindowFunctionStep::parseFrameBoundRows( case execplan::CalpontSystemCatalog::UDECIMAL: case execplan::CalpontSystemCatalog::DATE: case execplan::CalpontSystemCatalog::DATETIME: + case execplan::CalpontSystemCatalog::TIME: { fb.reset(new FrameBoundExpressionRow(type, id, idx)); break; @@ -1351,6 +1352,7 @@ boost::shared_ptr WindowFunctionStep::parseFrameBoundRange(const exe case execplan::CalpontSystemCatalog::UDECIMAL: case execplan::CalpontSystemCatalog::DATE: case execplan::CalpontSystemCatalog::DATETIME: + case execplan::CalpontSystemCatalog::TIME: { if (isConstant) { diff --git a/dbcon/mysql/ha_calpont.cpp b/dbcon/mysql/ha_calpont.cpp index 1a1b0b5ec..35953fc34 100644 --- a/dbcon/mysql/ha_calpont.cpp +++ b/dbcon/mysql/ha_calpont.cpp @@ -123,6 +123,9 @@ static int calpont_rollback(handlerton* hton, THD* thd, bool all); static int calpont_close_connection ( handlerton* hton, THD* thd ); handlerton* calpont_hton; +static group_by_handler* +create_calpont_group_by_handler(THD* thd, Query* query); + /* Variables for example share methods */ /* @@ -218,6 +221,7 @@ static int columnstore_init_func(void* p) calpont_hton->commit = calpont_commit; calpont_hton->rollback = calpont_rollback; calpont_hton->close_connection = calpont_close_connection; + calpont_hton->create_group_by = create_calpont_group_by_handler; DBUG_RETURN(0); } @@ -1135,6 +1139,86 @@ static MYSQL_SYSVAR_ULONG( 0); #endif +/*@brief create_calpont_group_by_handler- Creates handler*/ +/*********************************************************** + * DESCRIPTION: + * Creates a group_by pushdown handler. + * Details are in server/sql/group_by_handler.h + * PARAMETERS: + * thd - THD pointer. + * query - Query structure, that describes the pushdowned query. + * RETURN: + * group_by_handler if success + * NULL in other case + ***********************************************************/ +static group_by_handler* +create_calpont_group_by_handler(THD* thd, Query* query) +{ + ha_calpont_group_by_handler* handler = NULL; + + if ( thd->infinidb_vtable.vtable_state == THD::INFINIDB_DISABLE_VTABLE ) + { + handler = new ha_calpont_group_by_handler(thd, query); + + // Notify the server, that CS handles GROUP BY, ORDER BY and HAVING clauses. + query->group_by = NULL; + query->order_by = NULL; + query->having = NULL; + } + + return handler; +} + +/*********************************************************** + * DESCRIPTION: + * Makes the plan and prepares the data + * RETURN: + * int rc + ***********************************************************/ +int ha_calpont_group_by_handler::init_scan() +{ + DBUG_ENTER("ha_calpont_group_by_handler::init_scan"); + + // Save vtable_state to restore the after we inited. + THD::infinidb_state oldState = thd->infinidb_vtable.vtable_state; + // MCOL-1052 Should be removed after cleaning the code up. + thd->infinidb_vtable.vtable_state = THD::INFINIDB_CREATE_VTABLE; + int rc = ha_calpont_impl_group_by_init(this, table); + thd->infinidb_vtable.vtable_state = oldState; + + DBUG_RETURN(rc); +} + +/*********************************************************** + * DESCRIPTION: + * Fetches a row and saves it to a temporary table. + * RETURN: + * int rc + ***********************************************************/ +int ha_calpont_group_by_handler::next_row() +{ + DBUG_ENTER("ha_calpont_group_by_handler::next_row"); + int rc = ha_calpont_impl_group_by_next(this, table); + + DBUG_RETURN(rc); +} + +/*********************************************************** + * DESCRIPTION: + * Shuts the scan down. + * RETURN: + * int rc + ***********************************************************/ +int ha_calpont_group_by_handler::end_scan() +{ + DBUG_ENTER("ha_calpont_group_by_handler::end_scan"); + + int rc = ha_calpont_impl_group_by_end(this, table); + + DBUG_RETURN(rc); +} + + static struct st_mysql_sys_var* calpont_system_variables[] = { // MYSQL_SYSVAR(enum_var), diff --git a/dbcon/mysql/ha_calpont.h b/dbcon/mysql/ha_calpont.h index 6abaf51ed..bcbcdc5da 100644 --- a/dbcon/mysql/ha_calpont.h +++ b/dbcon/mysql/ha_calpont.h @@ -40,6 +40,8 @@ #include #include "idb_mysql.h" +extern handlerton* calpont_hton; + /** @brief EXAMPLE_SHARE is a structure that will be shared among all open handlers. This example implements the minimum of what you will probably need. @@ -245,5 +247,56 @@ public: } }; + +/*@brief group_by_handler class*/ +/*********************************************************** + * DESCRIPTION: + * Provides server with group_by_handler API methods. + * One should read comments in server/sql/group_by_handler.h + * Attributes: + * select - attribute contains all GROUP BY, HAVING, ORDER items and calls it + * an extended SELECT list accordin to comments in + * server/sql/group_handler.cc. + * So the temporary table for + * select count(*) from b group by a having a > 3 order by a + * will have 4 columns not 1. + * However server ignores all NULLs used in GROUP BY, HAVING, ORDER. + * table_list - contains all tables involved. Must be CS tables only. + * distinct - looks like a useless thing for now. Couldn't get it set by server. + * where - where items. + * group_by - group by ORDER items. + * order_by - order by ORDER items. + * having - having Item. + * Methods: + * init_scan - get plan and send it to ExeMgr. Get the execution result. + * next_row - get a row back from sm. + * end_scan - finish and clean the things up. + ***********************************************************/ +class ha_calpont_group_by_handler: public group_by_handler +{ +public: + ha_calpont_group_by_handler(THD* thd_arg, Query* query) + : group_by_handler(thd_arg, calpont_hton), + select(query->select), + table_list(query->from), + distinct(query->distinct), + where(query->where), + group_by(query->group_by), + order_by(query->order_by), + having(query->having) + { } + ~ha_calpont_group_by_handler() { } + int init_scan(); + int next_row(); + int end_scan(); + + List* select; + TABLE_LIST* table_list; + bool distinct; + Item* where; + ORDER* group_by; + ORDER* order_by; + Item* having; +}; #endif //HA_CALPONT_H__ diff --git a/dbcon/mysql/ha_calpont_ddl.cpp b/dbcon/mysql/ha_calpont_ddl.cpp index 221155592..fa8ef79dd 100644 --- a/dbcon/mysql/ha_calpont_ddl.cpp +++ b/dbcon/mysql/ha_calpont_ddl.cpp @@ -174,6 +174,10 @@ uint32_t convertDataType(int dataType) calpontDataType = CalpontSystemCatalog::DATETIME; break; + case ddlpackage::DDL_TIME: + calpontDataType = CalpontSystemCatalog::TIME; + break; + case ddlpackage::DDL_CLOB: calpontDataType = CalpontSystemCatalog::CLOB; break; @@ -1868,20 +1872,16 @@ int ProcessDDLStatement(string& ddlStatement, string& schema, const string& tabl if (ddlStatement.find("AUTO_INCREMENT") != string::npos) { thd->raise_error_printf(ER_CHECK_NOT_IMPLEMENTED, "Use of the MySQL auto_increment syntax is not supported in Columnstore. If you wish to create an auto increment column in Columnstore, please consult the Columnstore SQL Syntax Guide for the correct usage."); - } - // MCOL-867. MariaDB RENAME TABLE statement supports WAIT|NOWAIT options since 10.3.0 but Columnstore isn't yet. - else if (ddlStatement.find("WAIT") != string::npos || ddlStatement.find("NOWAIT") != string::npos) - { - thd->raise_error_printf(ER_CHECK_NOT_IMPLEMENTED, "WAIT and NOWAIT options are not supported in Columnstore. Please consult the Columnstore SQL Syntax Guide for the correct usage."); + ci->alterTableState = cal_connection_info::NOT_ALTER; + ci->isAlter = false; } else { //@Bug 1888,1885. update error message thd->raise_error_printf(ER_CHECK_NOT_IMPLEMENTED, "The syntax or the data type(s) is not supported by Columnstore. Please check the Columnstore syntax guide for supported syntax or data types."); + ci->alterTableState = cal_connection_info::NOT_ALTER; + ci->isAlter = false; } - - ci->alterTableState = cal_connection_info::NOT_ALTER; - ci->isAlter = false; } return rc; @@ -2183,7 +2183,6 @@ int ha_calpont_impl_rename_table_(const char* from, const char* to, cal_connecti THD* thd = current_thd; string emsg; - ostringstream stmt1; pair fromPair; pair toPair; string stmt; @@ -2211,20 +2210,21 @@ int ha_calpont_impl_rename_table_(const char* from, const char* to, cal_connecti return -1; } - stmt1 << "alter table " << fromPair.second << " rename to " << toPair.second << ";"; - - stmt = stmt1.str(); + // This explicitely shields both db objects with quotes that the lexer strips down later. + stmt = "alter table `" + fromPair.second + "` rename to `" + toPair.second + "`;"; string db; - if ( fromPair.first.length() != 0 ) - db = fromPair.first; - else if ( thd->db ) + if ( thd->db ) db = thd->db; + else if ( fromPair.first.length() != 0 ) + db = fromPair.first; + else + db = toPair.first; int rc = ProcessDDLStatement(stmt, db, "", tid2sid(thd->thread_id), emsg); if (rc != 0) - push_warning(thd, Sql_condition::WARN_LEVEL_WARN, 9999, emsg.c_str()); + push_warning(thd, Sql_condition::WARN_LEVEL_ERROR, 9999, emsg.c_str()); return rc; } diff --git a/dbcon/mysql/ha_calpont_dml.cpp b/dbcon/mysql/ha_calpont_dml.cpp index 1cef4d4da..cf103a801 100644 --- a/dbcon/mysql/ha_calpont_dml.cpp +++ b/dbcon/mysql/ha_calpont_dml.cpp @@ -136,7 +136,8 @@ int buildBuffer(uchar* buf, string& buffer, int& columns, TABLE* table) (*field)->type() == MYSQL_TYPE_STRING || (*field)->type() == MYSQL_TYPE_DATE || (*field)->type() == MYSQL_TYPE_DATETIME || - (*field)->type() == MYSQL_TYPE_DATETIME2 ) + (*field)->type() == MYSQL_TYPE_DATETIME2 || + (*field)->type() == MYSQL_TYPE_TIME ) vals.append("'"); while (ptr < end_ptr) @@ -166,7 +167,8 @@ int buildBuffer(uchar* buf, string& buffer, int& columns, TABLE* table) (*field)->type() == MYSQL_TYPE_STRING || (*field)->type() == MYSQL_TYPE_DATE || (*field)->type() == MYSQL_TYPE_DATETIME || - (*field)->type() == MYSQL_TYPE_DATETIME2 ) + (*field)->type() == MYSQL_TYPE_DATETIME2 || + (*field)->type() == MYSQL_TYPE_TIME ) vals.append("'"); } } @@ -838,11 +840,23 @@ int ha_calpont_impl_write_batch_row_(uchar* buf, TABLE* table, cal_impl_if::cal_ // mariadb 10.1 compatibility -- MYSQL_TYPE_DATETIME2 introduced in mysql 5.6 MYSQL_TIME ltime; const uchar* pos = buf; - longlong tmp = my_datetime_packed_from_binary(pos, 0); + longlong tmp = my_datetime_packed_from_binary(pos, table->field[colpos]->decimals()); TIME_from_longlong_datetime_packed(<ime, tmp); - fprintf(ci.filePtr, "%04d-%02d-%02d %02d:%02d:%02d%c", - ltime.year, ltime.month, ltime.day, - ltime.hour, ltime.minute, ltime.second, ci.delimiter); + + if (!ltime.second_part) + { + fprintf(ci.filePtr, "%04d-%02d-%02d %02d:%02d:%02d%c", + ltime.year, ltime.month, ltime.day, + ltime.hour, ltime.minute, ltime.second, ci.delimiter); + } + else + { + fprintf(ci.filePtr, "%04d-%02d-%02d %02d:%02d:%02d.%ld%c", + ltime.year, ltime.month, ltime.day, + ltime.hour, ltime.minute, ltime.second, + ltime.second_part, ci.delimiter); + } + buf += table->field[colpos]->pack_length(); } else @@ -866,6 +880,39 @@ int ha_calpont_impl_write_batch_row_(uchar* buf, TABLE* table, cal_impl_if::cal_ break; } + case CalpontSystemCatalog::TIME: + { + if (nullVal && (ci.columnTypes[colpos].constraintType != CalpontSystemCatalog::NOTNULL_CONSTRAINT)) + { + fprintf(ci.filePtr, "%c", ci.delimiter); + + buf += table->field[colpos]->pack_length(); + } + else + { + MYSQL_TIME ltime; + const uchar* pos = buf; + longlong tmp = my_time_packed_from_binary(pos, table->field[colpos]->decimals()); + TIME_from_longlong_time_packed(<ime, tmp); + + if (!ltime.second_part) + { + fprintf(ci.filePtr, "%02d:%02d:%02d%c", + ltime.hour, ltime.minute, ltime.second, ci.delimiter); + } + else + { + fprintf(ci.filePtr, "%02d:%02d:%02d.%ld%c", + ltime.hour, ltime.minute, ltime.second, + ltime.second_part, ci.delimiter); + } + + buf += table->field[colpos]->pack_length(); + } + + break; + } + case CalpontSystemCatalog::CHAR: { if (nullVal && (ci.columnTypes[colpos].constraintType != CalpontSystemCatalog::NOTNULL_CONSTRAINT)) diff --git a/dbcon/mysql/ha_calpont_execplan.cpp b/dbcon/mysql/ha_calpont_execplan.cpp index 9192f54ea..fac0cd032 100644 --- a/dbcon/mysql/ha_calpont_execplan.cpp +++ b/dbcon/mysql/ha_calpont_execplan.cpp @@ -221,7 +221,7 @@ void debug_walk(const Item* item, void* arg) case Item::FIELD_ITEM: { Item_field* ifp = (Item_field*)item; - cout << "FIELD_ITEM: " << (ifp->db_name ? ifp->db_name : "") << '.' << bestTableName(ifp) << + cerr << "FIELD_ITEM: " << (ifp->db_name ? ifp->db_name : "") << '.' << bestTableName(ifp) << '.' << ifp->field_name << endl; break; } @@ -229,10 +229,10 @@ void debug_walk(const Item* item, void* arg) case Item::INT_ITEM: { Item_int* iip = (Item_int*)item; - cout << "INT_ITEM: "; + cerr << "INT_ITEM: "; - if (iip->name) cout << iip->name << " (from name string)" << endl; - else cout << iip->val_int() << endl; + if (iip->name) cerr << iip->name << " (from name string)" << endl; + else cerr << iip->val_int() << endl; break; } @@ -243,19 +243,19 @@ void debug_walk(const Item* item, void* arg) String val, *str = isp->val_str(&val); string valStr; valStr.assign(str->ptr(), str->length()); - cout << "STRING_ITEM: >" << valStr << '<' << endl; + cerr << "STRING_ITEM: >" << valStr << '<' << endl; break; } case Item::REAL_ITEM: { - cout << "REAL_ITEM" << endl; + cerr << "REAL_ITEM" << endl; break; } case Item::DECIMAL_ITEM: { - cout << "DECIMAL_ITEM" << endl; + cerr << "DECIMAL_ITEM" << endl; break; } @@ -263,85 +263,85 @@ void debug_walk(const Item* item, void* arg) { Item_func* ifp = (Item_func*)item; Item_func_opt_neg* inp; - cout << "FUNC_ITEM: "; + cerr << "FUNC_ITEM: "; switch (ifp->functype()) { case Item_func::UNKNOWN_FUNC: // 0 - cout << ifp->func_name() << " (" << ifp->functype() << ")" << endl; + cerr << ifp->func_name() << " (" << ifp->functype() << ")" << endl; break; case Item_func::GT_FUNC: // 7 - cout << '>' << " (" << ifp->functype() << ")" << endl; + cerr << '>' << " (" << ifp->functype() << ")" << endl; break; case Item_func::EQ_FUNC: // 1 - cout << '=' << " (" << ifp->functype() << ")" << endl; + cerr << '=' << " (" << ifp->functype() << ")" << endl; break; case Item_func::GE_FUNC: - cout << ">=" << " (" << ifp->functype() << ")" << endl; + cerr << ">=" << " (" << ifp->functype() << ")" << endl; break; case Item_func::LE_FUNC: - cout << "<=" << " (" << ifp->functype() << ")" << endl; + cerr << "<=" << " (" << ifp->functype() << ")" << endl; break; case Item_func::LT_FUNC: - cout << '<' << " (" << ifp->functype() << ")" << endl; + cerr << '<' << " (" << ifp->functype() << ")" << endl; break; case Item_func::NE_FUNC: - cout << "<>" << " (" << ifp->functype() << ")" << endl; + cerr << "<>" << " (" << ifp->functype() << ")" << endl; break; case Item_func::NEG_FUNC: // 45 - cout << "unary minus" << " (" << ifp->functype() << ")" << endl; + cerr << "unary minus" << " (" << ifp->functype() << ")" << endl; break; case Item_func::IN_FUNC: // 16 inp = (Item_func_opt_neg*)ifp; - if (inp->negated) cout << "not "; + if (inp->negated) cerr << "not "; - cout << "in" << " (" << ifp->functype() << ")" << endl; + cerr << "in" << " (" << ifp->functype() << ")" << endl; break; case Item_func::BETWEEN: inp = (Item_func_opt_neg*)ifp; - if (inp->negated) cout << "not "; + if (inp->negated) cerr << "not "; - cout << "between" << " (" << ifp->functype() << ")" << endl; + cerr << "between" << " (" << ifp->functype() << ")" << endl; break; case Item_func::ISNULL_FUNC: // 10 - cout << "is null" << " (" << ifp->functype() << ")" << endl; + cerr << "is null" << " (" << ifp->functype() << ")" << endl; break; case Item_func::ISNOTNULL_FUNC: // 11 - cout << "is not null" << " (" << ifp->functype() << ")" << endl; + cerr << "is not null" << " (" << ifp->functype() << ")" << endl; break; case Item_func::NOT_ALL_FUNC: - cout << "not_all" << " (" << ifp->functype() << ")" << endl; + cerr << "not_all" << " (" << ifp->functype() << ")" << endl; break; case Item_func::NOT_FUNC: - cout << "not_func" << " (" << ifp->functype() << ")" << endl; + cerr << "not_func" << " (" << ifp->functype() << ")" << endl; break; case Item_func::TRIG_COND_FUNC: - cout << "trig_cond_func" << " (" << ifp->functype() << ")" << endl; + cerr << "trig_cond_func" << " (" << ifp->functype() << ")" << endl; break; case Item_func::ISNOTNULLTEST_FUNC: - cout << "isnotnulltest_func" << " (" << ifp->functype() << ")" << endl; + cerr << "isnotnulltest_func" << " (" << ifp->functype() << ")" << endl; break; case Item_func::MULT_EQUAL_FUNC: { - cout << "mult_equal_func:" << " (" << ifp->functype() << ")" << endl; + cerr << "mult_equal_func:" << " (" << ifp->functype() << ")" << endl; Item_equal* item_eq = (Item_equal*)ifp; Item_equal_fields_iterator it(*item_eq); Item* item; @@ -349,142 +349,142 @@ void debug_walk(const Item* item, void* arg) while ((item = it++)) { Field* equal_field = it.get_curr_field(); - cout << equal_field->field_name << endl; + cerr << equal_field->field_name << endl; } break; } case Item_func::EQUAL_FUNC: - cout << "equal func" << " (" << ifp->functype() << ")" << endl; + cerr << "equal func" << " (" << ifp->functype() << ")" << endl; break; case Item_func::FT_FUNC: - cout << "ft func" << " (" << ifp->functype() << ")" << endl; + cerr << "ft func" << " (" << ifp->functype() << ")" << endl; break; case Item_func::LIKE_FUNC: - cout << "like func" << " (" << ifp->functype() << ")" << endl; + cerr << "like func" << " (" << ifp->functype() << ")" << endl; break; case Item_func::COND_AND_FUNC: - cout << "cond and func" << " (" << ifp->functype() << ")" << endl; + cerr << "cond and func" << " (" << ifp->functype() << ")" << endl; break; case Item_func::COND_OR_FUNC: - cout << "cond or func" << " (" << ifp->functype() << ")" << endl; + cerr << "cond or func" << " (" << ifp->functype() << ")" << endl; break; case Item_func::XOR_FUNC: - cout << "xor func" << " (" << ifp->functype() << ")" << endl; + cerr << "xor func" << " (" << ifp->functype() << ")" << endl; break; case Item_func::INTERVAL_FUNC: - cout << "interval func" << " (" << ifp->functype() << ")" << endl; + cerr << "interval func" << " (" << ifp->functype() << ")" << endl; break; case Item_func::SP_EQUALS_FUNC: - cout << "sp equals func" << " (" << ifp->functype() << ")" << endl; + cerr << "sp equals func" << " (" << ifp->functype() << ")" << endl; break; case Item_func::SP_DISJOINT_FUNC: - cout << "sp disjoint func" << " (" << ifp->functype() << ")" << endl; + cerr << "sp disjoint func" << " (" << ifp->functype() << ")" << endl; break; case Item_func::SP_INTERSECTS_FUNC: - cout << "sp intersects func" << " (" << ifp->functype() << ")" << endl; + cerr << "sp intersects func" << " (" << ifp->functype() << ")" << endl; break; case Item_func::SP_TOUCHES_FUNC: - cout << "sp touches func" << " (" << ifp->functype() << ")" << endl; + cerr << "sp touches func" << " (" << ifp->functype() << ")" << endl; break; case Item_func::SP_CROSSES_FUNC: - cout << "sp crosses func" << " (" << ifp->functype() << ")" << endl; + cerr << "sp crosses func" << " (" << ifp->functype() << ")" << endl; break; case Item_func::SP_WITHIN_FUNC: - cout << "sp within func" << " (" << ifp->functype() << ")" << endl; + cerr << "sp within func" << " (" << ifp->functype() << ")" << endl; break; case Item_func::SP_CONTAINS_FUNC: - cout << "sp contains func" << " (" << ifp->functype() << ")" << endl; + cerr << "sp contains func" << " (" << ifp->functype() << ")" << endl; break; case Item_func::SP_OVERLAPS_FUNC: - cout << "sp overlaps func" << " (" << ifp->functype() << ")" << endl; + cerr << "sp overlaps func" << " (" << ifp->functype() << ")" << endl; break; case Item_func::SP_STARTPOINT: - cout << "sp startpoint func" << " (" << ifp->functype() << ")" << endl; + cerr << "sp startpoint func" << " (" << ifp->functype() << ")" << endl; break; case Item_func::SP_ENDPOINT: - cout << "sp endpoint func" << " (" << ifp->functype() << ")" << endl; + cerr << "sp endpoint func" << " (" << ifp->functype() << ")" << endl; break; case Item_func::SP_EXTERIORRING: - cout << "sp exteriorring func" << " (" << ifp->functype() << ")" << endl; + cerr << "sp exteriorring func" << " (" << ifp->functype() << ")" << endl; break; case Item_func::SP_POINTN: - cout << "sp pointn func" << " (" << ifp->functype() << ")" << endl; + cerr << "sp pointn func" << " (" << ifp->functype() << ")" << endl; break; case Item_func::SP_GEOMETRYN: - cout << "sp geometryn func" << " (" << ifp->functype() << ")" << endl; + cerr << "sp geometryn func" << " (" << ifp->functype() << ")" << endl; break; case Item_func::SP_INTERIORRINGN: - cout << "sp exteriorringn func" << " (" << ifp->functype() << ")" << endl; + cerr << "sp exteriorringn func" << " (" << ifp->functype() << ")" << endl; break; case Item_func::SP_RELATE_FUNC: - cout << "sp relate func" << " (" << ifp->functype() << ")" << endl; + cerr << "sp relate func" << " (" << ifp->functype() << ")" << endl; break; case Item_func::NOW_FUNC: - cout << "now func" << " (" << ifp->functype() << ")" << endl; + cerr << "now func" << " (" << ifp->functype() << ")" << endl; break; case Item_func::SUSERVAR_FUNC: - cout << "suservar func" << " (" << ifp->functype() << ")" << endl; + cerr << "suservar func" << " (" << ifp->functype() << ")" << endl; break; case Item_func::GUSERVAR_FUNC: - cout << "guservar func" << " (" << ifp->functype() << ")" << endl; + cerr << "guservar func" << " (" << ifp->functype() << ")" << endl; break; case Item_func::COLLATE_FUNC: - cout << "collate func" << " (" << ifp->functype() << ")" << endl; + cerr << "collate func" << " (" << ifp->functype() << ")" << endl; break; case Item_func::EXTRACT_FUNC: - cout << "extract func" << " (" << ifp->functype() << ")" << endl; + cerr << "extract func" << " (" << ifp->functype() << ")" << endl; break; case Item_func::CHAR_TYPECAST_FUNC: - cout << "char typecast func" << " (" << ifp->functype() << ")" << endl; + cerr << "char typecast func" << " (" << ifp->functype() << ")" << endl; break; case Item_func::FUNC_SP: - cout << "func sp func" << " (" << ifp->functype() << ")" << endl; + cerr << "func sp func" << " (" << ifp->functype() << ")" << endl; break; case Item_func::UDF_FUNC: - cout << "udf func" << " (" << ifp->functype() << ")" << endl; + cerr << "udf func" << " (" << ifp->functype() << ")" << endl; break; case Item_func::GSYSVAR_FUNC: - cout << "gsysvar func" << " (" << ifp->functype() << ")" << endl; + cerr << "gsysvar func" << " (" << ifp->functype() << ")" << endl; break; case Item_func::DYNCOL_FUNC: - cout << "dyncol func" << " (" << ifp->functype() << ")" << endl; + cerr << "dyncol func" << " (" << ifp->functype() << ")" << endl; break; default: - cout << "type=" << ifp->functype() << endl; + cerr << "type=" << ifp->functype() << endl; break; } @@ -494,7 +494,7 @@ void debug_walk(const Item* item, void* arg) case Item::COND_ITEM: { Item_cond* icp = (Item_cond*)item; - cout << "COND_ITEM: " << icp->func_name() << endl; + cerr << "COND_ITEM: " << icp->func_name() << endl; break; } @@ -503,7 +503,17 @@ void debug_walk(const Item* item, void* arg) Item_sum* isp = (Item_sum*)item; char* item_name = item->name; - if (!item_name) + // MCOL-1052 This is an extended SELECT list item + if (!item_name && isp->get_arg_count() && isp->get_arg(0)->name) + { + item_name = isp->get_arg(0)->name; + } + else if (!item_name && isp->get_arg_count() + && isp->get_arg(0)->type() == Item::INT_ITEM) + { + item_name = (char*)"INT||*"; + } + else if (!item_name) { item_name = (char*)""; } @@ -511,39 +521,39 @@ void debug_walk(const Item* item, void* arg) switch (isp->sum_func()) { case Item_sum::SUM_FUNC: - cout << "SUM_FUNC: " << item_name << endl; + cerr << "SUM_FUNC: " << item_name << endl; break; case Item_sum::SUM_DISTINCT_FUNC: - cout << "SUM_DISTINCT_FUNC: " << item_name << endl; + cerr << "SUM_DISTINCT_FUNC: " << item_name << endl; break; case Item_sum::AVG_FUNC: - cout << "AVG_FUNC: " << item_name << endl; + cerr << "AVG_FUNC: " << item_name << endl; break; case Item_sum::COUNT_FUNC: - cout << "COUNT_FUNC: " << item_name << endl; + cerr << "COUNT_FUNC: " << item_name << endl; break; case Item_sum::COUNT_DISTINCT_FUNC: - cout << "COUNT_DISTINCT_FUNC: " << item_name << endl; + cerr << "COUNT_DISTINCT_FUNC: " << item_name << endl; break; case Item_sum::MIN_FUNC: - cout << "MIN_FUNC: " << item_name << endl; + cerr << "MIN_FUNC: " << item_name << endl; break; case Item_sum::MAX_FUNC: - cout << "MAX_FUNC: " << item_name << endl; + cerr << "MAX_FUNC: " << item_name << endl; break; case Item_sum::UDF_SUM_FUNC: - cout << "UDAF_FUNC: " << item_name << endl; + cerr << "UDAF_FUNC: " << item_name << endl; break; default: - cout << "SUM_FUNC_ITEM type=" << isp->sum_func() << endl; + cerr << "SUM_FUNC_ITEM type=" << isp->sum_func() << endl; break; } @@ -553,24 +563,24 @@ void debug_walk(const Item* item, void* arg) case Item::SUBSELECT_ITEM: { Item_subselect* sub = (Item_subselect*)item; - cout << "SUBSELECT Item: "; + cerr << "SUBSELECT Item: "; switch (sub->substype()) { case Item_subselect::EXISTS_SUBS: - cout << "EXISTS"; + cerr << "EXISTS"; break; case Item_subselect::IN_SUBS: - cout << "IN"; + cerr << "IN"; break; default: - cout << sub->substype(); + cerr << sub->substype(); break; } - cout << endl; + cerr << endl; JOIN* join = sub->get_select_lex()->join; if (join) @@ -581,7 +591,7 @@ void debug_walk(const Item* item, void* arg) cond->traverse_cond(debug_walk, arg, Item::POSTFIX); } - cout << "Finish subselect item traversing" << endl; + cerr << "Finish subselect item traversing" << endl; break; } @@ -599,14 +609,14 @@ void debug_walk(const Item* item, void* arg) //ifp->cached_table->select_lex->select_number gives the select level. // could be used on alias. // could also be used to tell correlated join (equal level). - cout << "CACHED REF FIELD_ITEM: " << ifp->db_name << '.' << bestTableName(ifp) << + cerr << "CACHED REF FIELD_ITEM: " << ifp->db_name << '.' << bestTableName(ifp) << '.' << ifp->field_name << endl; break; } else if (field->type() == Item::FUNC_ITEM) { Item_func* ifp = (Item_func*)field; - cout << "CACHED REF FUNC_ITEM " << ifp->func_name() << endl; + cerr << "CACHED REF FUNC_ITEM " << ifp->func_name() << endl; } else if (field->type() == Item::REF_ITEM) { @@ -686,34 +696,45 @@ void debug_walk(const Item* item, void* arg) } } - cout << "CACHED REF_ITEM: ref type " << refType.c_str() << " real type " << realType.c_str() << endl; + cerr << "CACHED REF_ITEM: ref type " << refType.c_str() << " real type " << realType.c_str() << endl; break; } else { - cout << "REF_ITEM with CACHE_ITEM type unknown " << field->type() << endl; + cerr << "REF_ITEM with CACHE_ITEM type unknown " << field->type() << endl; } } else if (ref->real_item()->type() == Item::FIELD_ITEM) { Item_field* ifp = (Item_field*)ref->real_item(); - cout << "REF FIELD_ITEM: " << ifp->db_name << '.' << bestTableName(ifp) << '.' << - ifp->field_name << endl; + + // MCOL-1052 The field referenced presumable came from + // extended SELECT list. + if ( !ifp->field_name ) + { + cerr << "REF extra FIELD_ITEM: " << ifp->name << endl; + } + else + { + cerr << "REF FIELD_ITEM: " << ifp->db_name << '.' << bestTableName(ifp) << '.' << + ifp->field_name << endl; + } + break; } else if (ref->real_item()->type() == Item::FUNC_ITEM) { Item_func* ifp = (Item_func*)ref->real_item(); - cout << "REF FUNC_ITEM " << ifp->func_name() << endl; + cerr << "REF FUNC_ITEM " << ifp->func_name() << endl; } else if (ref->real_item()->type() == Item::WINDOW_FUNC_ITEM) { Item_window_func* ifp = (Item_window_func*)ref->real_item(); - cout << "REF WINDOW_FUNC_ITEM " << ifp->window_func()->func_name() << endl; + cerr << "REF WINDOW_FUNC_ITEM " << ifp->window_func()->func_name() << endl; } else { - cout << "UNKNOWN REF ITEM type " << ref->real_item()->type() << endl; + cerr << "UNKNOWN REF ITEM type " << ref->real_item()->type() << endl; } break; @@ -722,7 +743,7 @@ void debug_walk(const Item* item, void* arg) case Item::ROW_ITEM: { Item_row* row = (Item_row*)item; - cout << "ROW_ITEM: " << endl; + cerr << "ROW_ITEM: " << endl; for (uint32_t i = 0; i < row->cols(); i++) debug_walk(row->element_index(i), 0); @@ -732,7 +753,7 @@ void debug_walk(const Item* item, void* arg) case Item::EXPR_CACHE_ITEM: { - cout << "Expr Cache Item" << endl; + cerr << "Expr Cache Item" << endl; ((Item_cache_wrapper*)item)->get_orig_item()->traverse_cond(debug_walk, arg, Item::POSTFIX); break; } @@ -747,27 +768,27 @@ void debug_walk(const Item* item, void* arg) switch (item->result_type()) { case STRING_RESULT: - cout << "CACHE_STRING_ITEM" << endl; + cerr << "CACHE_STRING_ITEM" << endl; break; case REAL_RESULT: - cout << "CACHE_REAL_ITEM " << isp->val_real() << endl; + cerr << "CACHE_REAL_ITEM " << isp->val_real() << endl; break; case INT_RESULT: - cout << "CACHE_INT_ITEM " << isp->val_int() << endl; + cerr << "CACHE_INT_ITEM " << isp->val_int() << endl; break; case ROW_RESULT: - cout << "CACHE_ROW_ITEM" << endl; + cerr << "CACHE_ROW_ITEM" << endl; break; case DECIMAL_RESULT: - cout << "CACHE_DECIMAL_ITEM " << isp->val_decimal() << endl; + cerr << "CACHE_DECIMAL_ITEM " << isp->val_decimal() << endl; break; default: - cout << "CACHE_UNKNOWN_ITEM" << endl; + cerr << "CACHE_UNKNOWN_ITEM" << endl; break; } @@ -780,7 +801,7 @@ void debug_walk(const Item* item, void* arg) //ifp->cached_table->select_lex->select_number gives the select level. // could be used on alias. // could also be used to tell correlated join (equal level). - cout << "CACHED FIELD_ITEM: " << ifp->db_name << '.' << bestTableName(ifp) << + cerr << "CACHED FIELD_ITEM: " << ifp->db_name << '.' << bestTableName(ifp) << '.' << ifp->field_name << endl; break; } @@ -862,18 +883,18 @@ void debug_walk(const Item* item, void* arg) } } - cout << "CACHE_ITEM ref type " << refType.c_str() << " real type " << realType.c_str() << endl; + cerr << "CACHE_ITEM ref type " << refType.c_str() << " real type " << realType.c_str() << endl; break; } else if (field->type() == Item::FUNC_ITEM) { Item_func* ifp = (Item_func*)field; - cout << "CACHE_ITEM FUNC_ITEM " << ifp->func_name() << endl; + cerr << "CACHE_ITEM FUNC_ITEM " << ifp->func_name() << endl; break; } else { - cout << "CACHE_ITEM type unknown " << field->type() << endl; + cerr << "CACHE_ITEM type unknown " << field->type() << endl; } break; @@ -884,12 +905,12 @@ void debug_walk(const Item* item, void* arg) String val, *str = NULL; Item_temporal_literal* itp = (Item_temporal_literal*)item; str = itp->val_str(&val); - cout << "DATE ITEM: "; + cerr << "DATE ITEM: "; if (str) - cout << ": (" << str->ptr() << ')' << endl; + cerr << ": (" << str->ptr() << ')' << endl; else - cout << ": " << endl; + cerr << ": " << endl; break; } @@ -897,13 +918,19 @@ void debug_walk(const Item* item, void* arg) case Item::WINDOW_FUNC_ITEM: { Item_window_func* ifp = (Item_window_func*)item; - cout << "Window Function Item " << ifp->window_func()->func_name() << endl; + cerr << "Window Function Item " << ifp->window_func()->func_name() << endl; + break; + } + + case Item::NULL_ITEM: + { + cerr << "NULL item" << endl; break; } default: { - cout << "UNKNOWN_ITEM type " << item->type() << endl; + cerr << "UNKNOWN_ITEM type " << item->type() << endl; break; } } @@ -982,8 +1009,8 @@ uint32_t buildOuterJoin(gp_walk_info& gwi, SELECT_LEX& select_lex) // View is already processed in view::transform // @bug5319. view is sometimes treated as derived table and // fromSub::transform does not build outer join filters. - //if (!table_ptr->derived && table_ptr->view) - // continue; + if (!table_ptr->derived && table_ptr->view) + continue; CalpontSystemCatalog:: TableAliasName tan = make_aliasview( (table_ptr->db ? table_ptr->db : ""), @@ -1015,11 +1042,11 @@ uint32_t buildOuterJoin(gp_walk_info& gwi, SELECT_LEX& select_lex) #ifdef DEBUG_WALK_COND if (table_ptr->alias) - cout << table_ptr->alias ; + cerr << table_ptr->alias ; else if (table_ptr->alias) - cout << table_ptr->alias; + cerr << table_ptr->alias; - cout << " outer table expression: " << endl; + cerr << " outer table expression: " << endl; expr->traverse_cond(debug_walk, &gwi_outer, Item::POSTFIX); #endif expr->traverse_cond(gp_walk, &gwi_outer, Item::POSTFIX); @@ -1053,13 +1080,13 @@ uint32_t buildOuterJoin(gp_walk_info& gwi, SELECT_LEX& select_lex) Item_cond* expr = reinterpret_cast(table_ptr->embedding->on_expr); #ifdef DEBUG_WALK_COND - cout << "inner tables: " << endl; + cerr << "inner tables: " << endl; set::const_iterator it; for (it = gwi_outer.innerTables.begin(); it != gwi_outer.innerTables.end(); ++it) - cout << (*it) << " "; + cerr << (*it) << " "; - cout << endl; + cerr << endl; expr->traverse_cond(debug_walk, &gwi_outer, Item::POSTFIX); #endif expr->traverse_cond(gp_walk, &gwi_outer, Item::POSTFIX); @@ -1485,7 +1512,7 @@ bool buildPredicateItem(Item_func* ifp, gp_walk_info* gwip) // @bug5811. This filter string is for cross engine to use. // Use real table name. ifp->print(&str, QT_INFINIDB_DERIVED); - IDEBUG(cout << str.ptr() << endl); + IDEBUG(cerr << str.ptr() << endl); if (str.ptr()) cf->data(str.c_ptr()); @@ -1811,7 +1838,7 @@ bool buildPredicateItem(Item_func* ifp, gp_walk_info* gwip) (current_thd->lex->sql_command == SQLCOM_DELETE) || (current_thd->lex->sql_command == SQLCOM_DELETE_MULTI))) { - IDEBUG( cout << "deleted func with 2 const columns" << endl ); + IDEBUG( cerr << "deleted func with 2 const columns" << endl ); delete rhs; delete lhs; return false; @@ -1865,7 +1892,7 @@ bool buildPredicateItem(Item_func* ifp, gp_walk_info* gwip) if (notInner) { lhs->returnAll(true); - IDEBUG( cout << "setting returnAll on " << tan_lhs << endl); + IDEBUG( cerr << "setting returnAll on " << tan_lhs << endl); } } @@ -1882,7 +1909,7 @@ bool buildPredicateItem(Item_func* ifp, gp_walk_info* gwip) if (notInner) { rhs->returnAll(true); - IDEBUG( cout << "setting returnAll on " << tan_rhs << endl ); + IDEBUG( cerr << "setting returnAll on " << tan_rhs << endl ); } } @@ -2591,7 +2618,7 @@ CalpontSystemCatalog::ColType fieldType_MysqlToIDB (const Field* field) break; default: - IDEBUG( cout << "fieldType_MysqlToIDB:: Unknown result type of MySQL " + IDEBUG( cerr << "fieldType_MysqlToIDB:: Unknown result type of MySQL " << field->result_type() << endl ); break; } @@ -2657,6 +2684,11 @@ CalpontSystemCatalog::ColType colType_MysqlToIDB (const Item* item) ct.colDataType = CalpontSystemCatalog::DATETIME; ct.colWidth = 8; } + else if (item->field_type() == MYSQL_TYPE_TIME) + { + ct.colDataType = CalpontSystemCatalog::TIME; + ct.colWidth = 8; + } if (item->field_type() == MYSQL_TYPE_BLOB) { @@ -2699,7 +2731,7 @@ CalpontSystemCatalog::ColType colType_MysqlToIDB (const Item* item) break; default: - IDEBUG( cout << "colType_MysqlToIDB:: Unknown result type of MySQL " + IDEBUG( cerr << "colType_MysqlToIDB:: Unknown result type of MySQL " << item->result_type() << endl ); break; } @@ -2713,7 +2745,7 @@ ReturnedColumn* buildReturnedColumn(Item* item, gp_walk_info& gwi, bool& nonSupp if ( gwi.thd) { - if ( ((gwi.thd->lex)->sql_command == SQLCOM_UPDATE ) || ((gwi.thd->lex)->sql_command == SQLCOM_UPDATE_MULTI )) + //if ( ((gwi.thd->lex)->sql_command == SQLCOM_UPDATE ) || ((gwi.thd->lex)->sql_command == SQLCOM_UPDATE_MULTI )) { if ( !item->fixed) { @@ -2903,7 +2935,7 @@ ReturnedColumn* buildReturnedColumn(Item* item, gp_walk_info& gwi, bool& nonSupp { // TODO: item is a Item_cache_wrapper printf("EXPR_CACHE_ITEM in buildReturnedColumn\n"); - cout << "EXPR_CACHE_ITEM in buildReturnedColumn" << endl; + cerr << "EXPR_CACHE_ITEM in buildReturnedColumn" << endl; break; } @@ -3513,8 +3545,7 @@ ReturnedColumn* buildFunctionColumn(Item_func* ifp, gp_walk_info& gwi, bool& non if (ifp->field_type() == MYSQL_TYPE_DATETIME || ifp->field_type() == MYSQL_TYPE_DATETIME2 || ifp->field_type() == MYSQL_TYPE_TIMESTAMP || - ifp->field_type() == MYSQL_TYPE_TIMESTAMP2 || - funcName == "add_time") + ifp->field_type() == MYSQL_TYPE_TIMESTAMP2) { CalpontSystemCatalog::ColType ct; ct.colDataType = CalpontSystemCatalog::DATETIME; @@ -3528,6 +3559,13 @@ ReturnedColumn* buildFunctionColumn(Item_func* ifp, gp_walk_info& gwi, bool& non ct.colWidth = 4; fc->resultType(ct); } + else if (ifp->field_type() == MYSQL_TYPE_TIME) + { + CalpontSystemCatalog::ColType ct; + ct.colDataType = CalpontSystemCatalog::TIME; + ct.colWidth = 8; + fc->resultType(ct); + } #if 0 @@ -3648,6 +3686,22 @@ FunctionColumn* buildCaseFunction(Item_func* item, gp_walk_info& gwi, bool& nonS // and position. We can't tell which at this point, so we // rebuild the item from the arguments directly and then try to // figure what to pop, if anything, in order to sync the stacks. + // + // MCOL-1341 - With MariaDB 10.2.14 onwards CASE is now in the order: + // [case,]when1,when2,...,then1,then2,...[,else] + // See server commit bf1ca14ff3f3faa9f7a018097b25aa0f66d068cd for more + // information. + int32_t arg_offset = 0; + + if ((item->argument_count() - 1) % 2) + { + arg_offset = (item->argument_count() - 1) / 2; + } + else + { + arg_offset = item->argument_count() / 2; + } + for (int32_t i = item->argument_count() - 1; i >= 0; i--) { // For case_searched, we know the items for the WHEN clause will @@ -3656,7 +3710,7 @@ FunctionColumn* buildCaseFunction(Item_func* item, gp_walk_info& gwi, bool& nonS // Every even numbered arg is a WHEN. In between are the THEN. // An odd number of args indicates an ELSE residing in the last spot. if (funcName == "case_searched" && - i % 2 == 0 && uint(i) != item->argument_count() - 1) + (i < arg_offset)) { sptp.reset(buildParseTree((Item_func*)(item->arguments()[i]), gwi, nonSupport)); @@ -3955,7 +4009,7 @@ ParseTree* buildParseTree(Item_func* item, gp_walk_info& gwi, bool& nonSupport) Item_cond* icp = (Item_cond*)item; #ifdef DEBUG_WALK_COND // debug - cout << "Build Parsetree: " << endl; + cerr << "Build Parsetree: " << endl; icp->traverse_cond(debug_walk, &gwi, Item::POSTFIX); #endif //@bug5044. PPSTFIX walking should always be treated as WHERE clause filter @@ -4802,7 +4856,7 @@ void gp_walk(const Item* item, void* arg) gwip->rcWorkStack.push(cc); if (str) - IDEBUG( cout << "Const F&E " << item->full_name() << " evaluate: " << valStr << endl ); + IDEBUG( cerr << "Const F&E " << item->full_name() << " evaluate: " << valStr << endl ); break; } @@ -5005,8 +5059,13 @@ void gp_walk(const Item* item, void* arg) gwip->clauseType = SELECT; if (col->type() != Item::COND_ITEM) + { rc = buildReturnedColumn(col, *gwip, gwip->fatalParseError); + if ( col->type() == Item::FIELD_ITEM ) + gwip->fatalParseError = false; + } + SimpleColumn* sc = dynamic_cast(rc); if (sc) @@ -5077,6 +5136,31 @@ void gp_walk(const Item* item, void* arg) Item_func* ifp = (Item_func*)col; gwip->ptWorkStack.push(buildParseTree(ifp, *gwip, gwip->fatalParseError)); } + else if (col->type() == Item::FIELD_ITEM && gwip->clauseType == HAVING) + { + Item_field* ifip = static_cast(col); + std::vector::iterator iter = gwip->havingAggColsItems.begin(); + Item_func_or_sum* isfp = NULL; + + for ( ; iter != gwip->havingAggColsItems.end(); iter++ ) + { + Item* temp_isfp = *iter; + isfp = reinterpret_cast(temp_isfp); + + if ( isfp->type() == Item::SUM_FUNC_ITEM && + isfp->result_field == ifip->field ) + { + ReturnedColumn* rc = buildAggregateColumn(isfp, *gwip); + + if (rc) + gwip->rcWorkStack.push(rc); + + break; + } + } + + break; + } else cando = false; @@ -5362,7 +5446,7 @@ void parse_item (Item* item, vector& field_vec, bool& hasNonSupport } else { - cout << "UNKNOWN REF Item" << endl; + cerr << "UNKNOWN REF Item" << endl; break; } } @@ -5434,7 +5518,7 @@ bool isInfiniDB(TABLE* table_ptr) int getSelectPlan(gp_walk_info& gwi, SELECT_LEX& select_lex, SCSEP& csep, bool isUnion) { #ifdef DEBUG_WALK_COND - cout << "getSelectPlan()" << endl; + cerr << "getSelectPlan()" << endl; #endif // by pass the derived table resolve phase of mysql @@ -5521,7 +5605,7 @@ int getSelectPlan(gp_walk_info& gwi, SELECT_LEX& select_lex, SCSEP& csep, bool i while ((sj_nest = sj_list_it++)) { - cout << sj_nest->db << "." << sj_nest->table_name << endl; + cerr << sj_nest->db << "." << sj_nest->table_name << endl; } #endif @@ -5605,7 +5689,7 @@ int getSelectPlan(gp_walk_info& gwi, SELECT_LEX& select_lex, SCSEP& csep, bool i CalpontSystemCatalog::TableAliasName tan = make_aliastable(table_ptr->db, table_name, table_ptr->alias, infiniDB); gwi.tableMap[tan] = make_pair(0, table_ptr); #ifdef DEBUG_WALK_COND - cout << tn << endl; + cerr << tn << endl; #endif } } @@ -5685,15 +5769,15 @@ int getSelectPlan(gp_walk_info& gwi, SELECT_LEX& select_lex, SCSEP& csep, bool i distUnionNum = unionVec.size(); /*#ifdef DEBUG_WALK_COND - IDEBUG( cout << ">>>> UNION DEBUG" << endl ); + IDEBUG( cerr << ">>>> UNION DEBUG" << endl ); JOIN* join = sl->join; Item_cond* icp = 0; if (join != 0) icp = reinterpret_cast(join->conds); if (icp) icp->traverse_cond(debug_walk, &gwi, Item::POSTFIX); - IDEBUG ( cout << *plan << endl ); - IDEBUG ( cout << "<<<traverse_cond(debug_walk, &gwi, Item::POSTFIX); - cout << "------------------------------------------------\n" << endl; + cerr << "------------------------------------------------\n" << endl; #endif icp->traverse_cond(gp_walk, &gwi, Item::POSTFIX); @@ -5770,16 +5854,16 @@ int getSelectPlan(gp_walk_info& gwi, SELECT_LEX& select_lex, SCSEP& csep, bool i TABLE_LIST* curr = *tbl; if (curr->table_name) - cout << curr->table_name << " "; + cerr << curr->table_name << " "; else - cout << curr->alias << endl; + cerr << curr->alias << endl; if (curr->outer_join) - cout << " is inner table" << endl; + cerr << " is inner table" << endl; else if (curr->straight) - cout << "straight_join" << endl; + cerr << "straight_join" << endl; else - cout << "join" << endl; + cerr << "join" << endl; if (curr->nested_join) { @@ -5795,7 +5879,7 @@ int getSelectPlan(gp_walk_info& gwi, SELECT_LEX& select_lex, SCSEP& csep, bool i for (TABLE_LIST** tb = inner; tb < end1; tb++) { TABLE_LIST* curr1 = *tb; - cout << curr1->alias << endl; + cerr << curr1->alias << endl; if (curr1->sj_on_expr) { @@ -5870,7 +5954,7 @@ int getSelectPlan(gp_walk_info& gwi, SELECT_LEX& select_lex, SCSEP& csep, bool i gwi.clauseType = SELECT; #ifdef DEBUG_WALK_COND { - cout << "------------------- SELECT --------------------" << endl; + cerr << "------------------- SELECT --------------------" << endl; List_iterator_fast it(select_lex.item_list); Item* item; @@ -5879,7 +5963,7 @@ int getSelectPlan(gp_walk_info& gwi, SELECT_LEX& select_lex, SCSEP& csep, bool i debug_walk(item, 0); } - cout << "-----------------------------------------------\n" << endl; + cerr << "-----------------------------------------------\n" << endl; } #endif @@ -6337,7 +6421,7 @@ int getSelectPlan(gp_walk_info& gwi, SELECT_LEX& select_lex, SCSEP& csep, bool i } #ifdef DEBUG_WALK_COND - cout << "SELECT clause SUBSELECT Item: " << sub->substype() << endl; + cerr << "SELECT clause SUBSELECT Item: " << sub->substype() << endl; JOIN* join = sub->get_select_lex()->join; if (join) @@ -6348,7 +6432,7 @@ int getSelectPlan(gp_walk_info& gwi, SELECT_LEX& select_lex, SCSEP& csep, bool i cond->traverse_cond(debug_walk, &gwi, Item::POSTFIX); } - cout << "Finish SELECT clause subselect item traversing" << endl; + cerr << "Finish SELECT clause subselect item traversing" << endl; #endif SelectSubQuery* selectSub = new SelectSubQuery(gwi, sub); //selectSub->gwip(&gwi); @@ -6467,9 +6551,9 @@ int getSelectPlan(gp_walk_info& gwi, SELECT_LEX& select_lex, SCSEP& csep, bool i { Item_cond* having = reinterpret_cast(select_lex.having); #ifdef DEBUG_WALK_COND - cout << "------------------- HAVING ---------------------" << endl; + cerr << "------------------- HAVING ---------------------" << endl; having->traverse_cond(debug_walk, &gwi, Item::POSTFIX); - cout << "------------------------------------------------\n" << endl; + cerr << "------------------------------------------------\n" << endl; #endif having->traverse_cond(gp_walk, &gwi, Item::POSTFIX); @@ -7809,6 +7893,10 @@ int cp_get_plan(THD* thd, SCSEP& csep) else if (status < 0) return status; + cerr << "---------------- cp_get_plan EXECUTION PLAN ----------------" << endl; + cerr << *csep << endl ; + cerr << "-------------- EXECUTION PLAN END --------------\n" << endl; + // Derived table projection and filter optimization. derivedTableOptimization(csep); @@ -7910,4 +7998,1965 @@ int cp_get_table_plan(THD* thd, SCSEP& csep, cal_table_info& ti) return 0; } + +int cp_get_group_plan(THD* thd, SCSEP& csep, cal_impl_if::cal_group_info& gi) +{ + LEX* lex = thd->lex; + idbassert(lex != 0); + + SELECT_LEX select_lex = lex->select_lex; + gp_walk_info gwi; + gwi.thd = thd; + int status = getGroupPlan(gwi, select_lex, csep, gi); + + if (status > 0) + return ER_INTERNAL_ERROR; + else if (status < 0) + return status; + + return 0; +} + +/*@brief buildConstColFromFilter- change SimpleColumn into ConstColumn*/ +/*********************************************************** + * DESCRIPTION: + * Server could optimize out fields from GROUP BY list, when certain + * filter predicate is used, e.g. + * field = 'AIR', field IN ('AIR'). This utility function tries to + * replace such fields with ConstantColumns using cond_pushed filters. + * PARAMETERS: + * originalSC SimpleColumn* removed field + * gwi main strucutre + * gi auxilary group_by handler structure + * RETURNS + * ConstantColumn* if originalSC equals with cond_pushed columns. + * NULL otherwise + ***********************************************************/ +ConstantColumn* buildConstColFromFilter(SimpleColumn* originalSC, + gp_walk_info& gwi, cal_group_info& gi) +{ + execplan::SimpleColumn* simpleCol; + execplan::ConstantColumn* constCol; + execplan::SOP op; + execplan::SimpleFilter* simpFilter; + execplan::ConstantColumn* result = NULL; + std::vector::iterator ptIt = gi.pushedPts.begin(); + + for (; ptIt != gi.pushedPts.end(); ptIt++) + { + simpFilter = dynamic_cast((*ptIt)->data()); + + if (simpFilter == NULL) + continue; + + simpleCol = dynamic_cast(simpFilter->lhs()); + constCol = dynamic_cast(simpFilter->rhs()); + + if (simpleCol == NULL || constCol == NULL) + continue; + + op = simpFilter->op(); + + if ( originalSC->sameColumn(dynamic_cast(simpleCol)) + && op.get()->op() == OP_EQ && constCol) + { +#ifdef DEBUG_WALK_COND + cerr << "buildConstColFromFilter() replaced " << endl; + cerr << simpleCol->toString() << endl; + cerr << " with " << endl; + cerr << constCol << endl; +#endif + result = constCol; + } + } + + return result; +} + +int getGroupPlan(gp_walk_info& gwi, SELECT_LEX& select_lex, SCSEP& csep, cal_group_info& gi, bool isUnion) +{ +#ifdef DEBUG_WALK_COND + cerr << "getGroupPlan()" << endl; +#endif + + // rollup is currently not supported + if (select_lex.olap == ROLLUP_TYPE) + { + gwi.fatalParseError = true; + gwi.parseErrorText = IDBErrorInfo::instance()->errorMsg(ERR_ROLLUP_NOT_SUPPORT); + setError(gwi.thd, ER_CHECK_NOT_IMPLEMENTED, gwi.parseErrorText, gwi); + return ER_CHECK_NOT_IMPLEMENTED; + } + + gwi.internalDecimalScale = (gwi.thd->variables.infinidb_use_decimal_scale ? gwi.thd->variables.infinidb_decimal_scale : -1); + gwi.subSelectType = csep->subType(); + + JOIN* join = select_lex.join; + Item_cond* icp = 0; + + if ( gi.groupByWhere ) + icp = reinterpret_cast(gi.groupByWhere); + + uint32_t sessionID = csep->sessionID(); + gwi.sessionid = sessionID; + boost::shared_ptr csc = CalpontSystemCatalog::makeCalpontSystemCatalog(sessionID); + csc->identity(CalpontSystemCatalog::FE); + gwi.csc = csc; + + // @bug 2123. Override large table estimate if infinidb_ordered hint was used. + // @bug 2404. Always override if the infinidb_ordered_only variable is turned on. + if (gwi.thd->infinidb_vtable.override_largeside_estimate || gwi.thd->variables.infinidb_ordered_only) + csep->overrideLargeSideEstimate(true); + + // @bug 5741. Set a flag when in Local PM only query mode + csep->localQuery(gwi.thd->variables.infinidb_local_query); + + // @bug 3321. Set max number of blocks in a dictionary file to be scanned for filtering + csep->stringScanThreshold(gwi.thd->variables.infinidb_string_scan_threshold); + + csep->stringTableThreshold(gwi.thd->variables.infinidb_stringtable_threshold); + + csep->djsSmallSideLimit(gwi.thd->variables.infinidb_diskjoin_smallsidelimit * 1024ULL * 1024); + csep->djsLargeSideLimit(gwi.thd->variables.infinidb_diskjoin_largesidelimit * 1024ULL * 1024); + csep->djsPartitionSize(gwi.thd->variables.infinidb_diskjoin_bucketsize * 1024ULL * 1024); + + if (gwi.thd->variables.infinidb_um_mem_limit == 0) + csep->umMemLimit(numeric_limits::max()); + else + csep->umMemLimit(gwi.thd->variables.infinidb_um_mem_limit * 1024ULL * 1024); + + // populate table map and trigger syscolumn cache for all the tables (@bug 1637). + // all tables on FROM list must have at least one col in colmap + TABLE_LIST* table_ptr = gi.groupByTables; + CalpontSelectExecutionPlan::SelectList derivedTbList; + +// DEBUG +#ifdef DEBUG_WALK_COND + List_iterator sj_list_it(select_lex.sj_nests); + TABLE_LIST* sj_nest; + + while ((sj_nest = sj_list_it++)) + { + cerr << sj_nest->db << "." << sj_nest->table_name << endl; + } + +#endif + + // @bug 1796. Remember table order on the FROM list. + gwi.clauseType = FROM; + + try + { + for (; table_ptr; table_ptr = table_ptr->next_local) + { + // mysql put vtable here for from sub. we ignore it + //if (string(table_ptr->table_name).find("$vtable") != string::npos) + // continue; + + // Until we handle recursive cte: + // Checking here ensures we catch all with clauses in the query. + if (table_ptr->is_recursive_with_table()) + { + gwi.fatalParseError = true; + gwi.parseErrorText = "Recursive CTE"; + setError(gwi.thd, ER_CHECK_NOT_IMPLEMENTED, gwi.parseErrorText, gwi); + return ER_CHECK_NOT_IMPLEMENTED; + } + + string viewName = getViewName(table_ptr); + + // @todo process from subquery + if (table_ptr->derived) + { + String str; + (table_ptr->derived->first_select())->print(gwi.thd, &str, QT_INFINIDB_DERIVED); + + SELECT_LEX* select_cursor = table_ptr->derived->first_select(); + FromSubQuery fromSub(gwi, select_cursor); + string alias(table_ptr->alias); + fromSub.alias(lower(alias)); + + CalpontSystemCatalog::TableAliasName tn = make_aliasview("", "", alias, viewName); + // @bug 3852. check return execplan + SCSEP plan = fromSub.transform(); + + if (!plan) + { + setError(gwi.thd, ER_INTERNAL_ERROR, fromSub.gwip().parseErrorText, gwi); + CalpontSystemCatalog::removeCalpontSystemCatalog(sessionID); + return ER_INTERNAL_ERROR; + } + + gwi.derivedTbList.push_back(plan); + gwi.tbList.push_back(tn); + CalpontSystemCatalog::TableAliasName tan = make_aliastable("", alias, alias); + gwi.tableMap[tan] = make_pair(0, table_ptr); +// gwi.thd->infinidb_vtable.isUnion = true; //by-pass the 2nd pass of rnd_init + } + else if (table_ptr->view) + { + View* view = new View(table_ptr->view->select_lex, &gwi); + CalpontSystemCatalog::TableAliasName tn = make_aliastable(table_ptr->db, table_ptr->table_name, table_ptr->alias); + view->viewName(tn); + gwi.viewList.push_back(view); + view->transform(); + } + else + { + // check foreign engine tables + bool infiniDB = (table_ptr->table ? isInfiniDB(table_ptr->table) : true); + + // trigger system catalog cache + if (infiniDB) + csc->columnRIDs(make_table(table_ptr->db, table_ptr->table_name), true); + + string table_name = table_ptr->table_name; + + // @bug5523 + if (table_ptr->db && strcmp(table_ptr->db, "information_schema") == 0) + table_name = (table_ptr->schema_table_name ? table_ptr->schema_table_name : table_ptr->alias); + + CalpontSystemCatalog::TableAliasName tn = make_aliasview(table_ptr->db, table_name, table_ptr->alias, viewName, infiniDB); + gwi.tbList.push_back(tn); + CalpontSystemCatalog::TableAliasName tan = make_aliastable(table_ptr->db, table_name, table_ptr->alias, infiniDB); + gwi.tableMap[tan] = make_pair(0, table_ptr); +#ifdef DEBUG_WALK_COND + cerr << tn << endl; +#endif + } + } + + if (gwi.fatalParseError) + { + setError(gwi.thd, ER_INTERNAL_ERROR, gwi.parseErrorText, gwi); + return ER_INTERNAL_ERROR; + } + } + catch (IDBExcept& ie) + { + setError(gwi.thd, ER_INTERNAL_ERROR, ie.what(), gwi); + CalpontSystemCatalog::removeCalpontSystemCatalog(sessionID); + // @bug 3852. set error status for gwi. + gwi.fatalParseError = true; + gwi.parseErrorText = ie.what(); + return ER_INTERNAL_ERROR; + } + catch (...) + { + string emsg = IDBErrorInfo::instance()->errorMsg(ERR_LOST_CONN_EXEMGR); + // @bug3852 set error status for gwi. + gwi.fatalParseError = true; + gwi.parseErrorText = emsg; + setError(gwi.thd, ER_INTERNAL_ERROR, emsg, gwi); + CalpontSystemCatalog::removeCalpontSystemCatalog(sessionID); + return ER_INTERNAL_ERROR; + } + + csep->tableList(gwi.tbList); + + bool unionSel = false; + + gwi.clauseType = WHERE; + + + if (icp) + { + // MCOL-1052 The condition could be useless. + // MariaDB bug 624 - without the fix_fields call, delete with join may error with "No query step". + //#if MYSQL_VERSION_ID < 50172 + //@bug 3039. fix fields for constants + if (!icp->fixed) + { + icp->fix_fields(gwi.thd, (Item**)&icp); + } + +//#endif + gwi.fatalParseError = false; +#ifdef DEBUG_WALK_COND + cerr << "------------------ WHERE -----------------------" << endl; + icp->traverse_cond(debug_walk, &gwi, Item::POSTFIX); + cerr << "------------------------------------------------\n" << endl; +#endif + + icp->traverse_cond(gp_walk, &gwi, Item::POSTFIX); + + if (gwi.fatalParseError) + { + // if this is dervied table process phase, mysql may have not developed the plan + // completely. Do not error and eventually mysql will call JOIN::exec() again. + // related to bug 2922. Need to find a way to skip calling rnd_init for derived table + // processing. + if (gwi.thd->derived_tables_processing) + { + gwi.thd->infinidb_vtable.isUnion = false; + gwi.thd->infinidb_vtable.isUpdateWithDerive = true; + return -1; + } + + setError(gwi.thd, ER_INTERNAL_ERROR, gwi.parseErrorText, gwi); + return ER_INTERNAL_ERROR; + } + } + else if (join && join->zero_result_cause) + { + gwi.rcWorkStack.push(new ConstantColumn((int64_t)0, ConstantColumn::NUM)); + } + + uint32_t failed = buildOuterJoin(gwi, select_lex); + + if (failed) return failed; + + // @bug5764. build outer join for view, make sure outerjoin filter is appended + // to the end of the filter list. + for (uint i = 0; i < gwi.viewList.size(); i++) + { + failed = gwi.viewList[i]->processOuterJoin(gwi); + + if (failed) + break; + } + + if (failed != 0) + return failed; + + ParseTree* filters = NULL; + ParseTree* ptp = NULL; + ParseTree* lhs = NULL; + + // @bug 2932. for "select * from region where r_name" case. if icp not null and + // ptWorkStack empty, the item is in rcWorkStack. + // MySQL 5.6 (MariaDB?). when icp is null and zero_result_cause is set, a constant 0 + // is pushed to rcWorkStack. + if (/*icp && */gwi.ptWorkStack.empty() && !gwi.rcWorkStack.empty()) + { + filters = new ParseTree(gwi.rcWorkStack.top()); + gwi.rcWorkStack.pop(); + } + + while (!gwi.ptWorkStack.empty()) + { + filters = gwi.ptWorkStack.top(); + gwi.ptWorkStack.pop(); + + if (gwi.ptWorkStack.empty()) + break; + + ptp = new ParseTree(new LogicOperator("and")); + //ptp->left(filters); + ptp->right(filters); + lhs = gwi.ptWorkStack.top(); + gwi.ptWorkStack.pop(); + //ptp->right(rhs); + ptp->left(lhs); + gwi.ptWorkStack.push(ptp); + } + + if (filters) + { + csep->filters(filters); +#ifdef DEBUG_WALK_COND + filters->drawTree("/tmp/filter1.dot"); +#endif + } + + gwi.clauseType = SELECT; +#ifdef DEBUG_WALK_COND + { + cerr << "------------------- SELECT --------------------" << endl; + List_iterator_fast it(*gi.groupByFields); + Item* item; + + while ((item = it++)) + { + debug_walk(item, 0); + } + + cerr << "-----------------------------------------------\n" << endl; + } +#endif + + // populate returnedcolumnlist and columnmap + List_iterator_fast it(*gi.groupByFields); + Item* item; + vector funcFieldVec; + string sel_cols_in_create; + string sel_cols_in_select; + bool redo = false; + + // empty rcWorkStack and ptWorkStack. They should all be empty by now. + clearStacks(gwi); + + // indicate the starting pos of scalar returned column, because some join column + // has been inserted to the returned column list. + if (gwi.subQuery) + { + ScalarSub* scalar = dynamic_cast(gwi.subQuery); + + if (scalar) + scalar->returnedColPos(gwi.additionalRetCols.size()); + } + + CalpontSelectExecutionPlan::SelectList selectSubList; + + while ((item = it++)) + { + string itemAlias = (item->name ? item->name : ""); + + // @bug 5916. Need to keep checking until getting concret item in case + // of nested view. + while (item->type() == Item::REF_ITEM) + { + Item_ref* ref = (Item_ref*)item; + item = (*(ref->ref)); + } + + Item::Type itype = item->type(); + + switch (itype) + { + case Item::FIELD_ITEM: + { + Item_field* ifp = (Item_field*)item; + SimpleColumn* sc = NULL; + ConstantColumn* constCol = NULL; + + if (ifp->field_name && string(ifp->field_name) == "*") + { + collectAllCols(gwi, ifp); + break; + } + + sc = buildSimpleColumn(ifp, gwi); + + if (sc) + { + constCol = buildConstColFromFilter(sc, gwi, gi); + boost::shared_ptr spcc(constCol); + boost::shared_ptr spsc(sc); + + if (sel_cols_in_create.length() != 0) + sel_cols_in_create += ", "; + + string fullname; + String str; + ifp->print(&str, QT_INFINIDB_NO_QUOTE); + fullname = str.c_ptr(); + + //sel_cols_in_create += fullname; + if (ifp->is_autogenerated_name) // no alias + { + sel_cols_in_create += fullname + " `" + escapeBackTick(str.c_ptr()) + "`"; + sc->alias(fullname); + } + else // alias + { + if (!itemAlias.empty()) + sc->alias(itemAlias); + + sel_cols_in_create += fullname + " `" + escapeBackTick(sc->alias().c_str()) + "`"; + } + + if (ifp->is_autogenerated_name) + gwi.selectCols.push_back("`" + escapeBackTick(fullname.c_str()) + "`" + " `" + + escapeBackTick(itemAlias.empty() ? ifp->name : itemAlias.c_str()) + "`"); + else + gwi.selectCols.push_back("`" + escapeBackTick((itemAlias.empty() ? ifp->name : itemAlias.c_str())) + "`"); + + // MCOL-1052 Replace SimpleColumn with ConstantColumn, + // since it must have a single value only. + if (constCol) + { + gwi.returnedCols.push_back(spcc); + gwi.columnMap.insert(CalpontSelectExecutionPlan::ColumnMap::value_type(string(ifp->field_name), spcc)); + } + else + { + gwi.returnedCols.push_back(spsc); + gwi.columnMap.insert(CalpontSelectExecutionPlan::ColumnMap::value_type(string(ifp->field_name), spsc)); + } + + TABLE_LIST* tmp = 0; + + if (ifp->cached_table) + tmp = ifp->cached_table; + + gwi.tableMap[make_aliastable(sc->schemaName(), sc->tableName(), sc->tableAlias(), sc->isInfiniDB())] = + make_pair(1, tmp); + } + else + { + setError(gwi.thd, ER_INTERNAL_ERROR, gwi.parseErrorText, gwi); + delete sc; + return ER_INTERNAL_ERROR; + } + + break; + } + + //aggregate column + case Item::SUM_FUNC_ITEM: + { + ReturnedColumn* ac = buildAggregateColumn(item, gwi); + + if (gwi.fatalParseError) + { + // e.g., non-support ref column + setError(gwi.thd, ER_CHECK_NOT_IMPLEMENTED, gwi.parseErrorText, gwi); + delete ac; + return ER_CHECK_NOT_IMPLEMENTED; + } + + // add this agg col to returnedColumnList + boost::shared_ptr spac(ac); + gwi.returnedCols.push_back(spac); + // This item will be used in HAVING later. + Item_func_or_sum* isfp = reinterpret_cast(item); + + if ( ! isfp->name_length ) + { + gwi.havingAggColsItems.push_back(item); + } + + gwi.selectCols.push_back('`' + escapeBackTick(spac->alias().c_str()) + '`'); + String str(256); + item->print(&str, QT_INFINIDB_NO_QUOTE); + + if (sel_cols_in_create.length() != 0) + sel_cols_in_create += ", "; + + sel_cols_in_create += string(str.c_ptr()) + " `" + escapeBackTick(spac->alias().c_str()) + "`"; + break; + } + + case Item::FUNC_ITEM: + { + Item_func* ifp = reinterpret_cast(item); + + // @bug4383. error out non-support stored function + if (ifp->functype() == Item_func::FUNC_SP) + { + gwi.fatalParseError = true; + gwi.parseErrorText = IDBErrorInfo::instance()->errorMsg(ERR_SP_FUNCTION_NOT_SUPPORT); + setError(gwi.thd, ER_CHECK_NOT_IMPLEMENTED, gwi.parseErrorText, gwi); + return ER_CHECK_NOT_IMPLEMENTED; + } + + if (string(ifp->func_name()) == "xor") + { + gwi.fatalParseError = true; + gwi.parseErrorText = IDBErrorInfo::instance()->errorMsg(ERR_FILTER_COND_EXP); + setError(gwi.thd, ER_CHECK_NOT_IMPLEMENTED, gwi.parseErrorText, gwi); + return ER_CHECK_NOT_IMPLEMENTED; + } + + uint16_t parseInfo = 0; + vector tmpVec; + bool hasNonSupportItem = false; + parse_item(ifp, tmpVec, hasNonSupportItem, parseInfo); + + if (ifp->has_subquery() || + string(ifp->func_name()) == string("") || + ifp->functype() == Item_func::NOT_ALL_FUNC || + parseInfo & SUB_BIT) + { + gwi.fatalParseError = true; + gwi.parseErrorText = IDBErrorInfo::instance()->errorMsg(ERR_NON_SUPPORT_SELECT_SUB); + setError(gwi.thd, ER_CHECK_NOT_IMPLEMENTED, gwi.parseErrorText, gwi); + return ER_CHECK_NOT_IMPLEMENTED; + } + + ReturnedColumn* rc = buildFunctionColumn(ifp, gwi, hasNonSupportItem); + SRCP srcp(rc); + + if (rc) + { + if (!hasNonSupportItem && !nonConstFunc(ifp) && !(parseInfo & AF_BIT) && tmpVec.size() == 0) + { + if (isUnion || unionSel || gwi.subSelectType != CalpontSelectExecutionPlan::MAIN_SELECT || + parseInfo & SUB_BIT ) //|| select_lex.group_list.elements != 0) + { + srcp.reset(buildReturnedColumn(item, gwi, gwi.fatalParseError)); + gwi.returnedCols.push_back(srcp); + + if (ifp->name) + srcp->alias(ifp->name); + + continue; + } + + if ( ((gwi.thd->lex)->sql_command == SQLCOM_UPDATE ) || + ((gwi.thd->lex)->sql_command == SQLCOM_DELETE ) || + ((gwi.thd->lex)->sql_command == SQLCOM_UPDATE_MULTI ) || + ((gwi.thd->lex)->sql_command == SQLCOM_DELETE_MULTI ) ) + { } + else + { + redo = true; + String str; + ifp->print(&str, QT_INFINIDB_NO_QUOTE); + gwi.selectCols.push_back(string(str.c_ptr()) + " " + "`" + escapeBackTick(item->name) + "`"); + } + + break; + } + + //SRCP srcp(rc); + gwi.returnedCols.push_back(srcp); + + if ( ((gwi.thd->lex)->sql_command == SQLCOM_UPDATE ) || ((gwi.thd->lex)->sql_command == SQLCOM_DELETE ) || ((gwi.thd->lex)->sql_command == SQLCOM_UPDATE_MULTI ) || ((gwi.thd->lex)->sql_command == SQLCOM_DELETE_MULTI )) + { } + else + { + String str(256); + ifp->print(&str, QT_INFINIDB_NO_QUOTE); + + if (sel_cols_in_create.length() != 0) + sel_cols_in_create += ", "; + + sel_cols_in_create += string(str.c_ptr()) + " `" + ifp->name + "`"; + gwi.selectCols.push_back("`" + escapeBackTick(ifp->name) + "`"); + } + } + else // InfiniDB Non support functions still go through post process for now + { + hasNonSupportItem = false; + uint32_t before_size = funcFieldVec.size(); + parse_item(ifp, funcFieldVec, hasNonSupportItem, parseInfo); + uint32_t after_size = funcFieldVec.size(); + + // group by func and func in subquery can not be post processed + // @bug3881. set_user_var can not be treated as constant function + // @bug5716. Try to avoid post process function for union query. + if ((gwi.subQuery /*|| select_lex.group_list.elements != 0 */ || + !csep->unionVec().empty() || isUnion) && + !hasNonSupportItem && (after_size - before_size) == 0 && + !(parseInfo & AGG_BIT) && !(parseInfo & SUB_BIT) && + string(ifp->func_name()) != "set_user_var") + { + String val, *str = ifp->val_str(&val); + string valStr; + + if (str) + valStr.assign(str->ptr(), str->length()); + + ConstantColumn* cc = NULL; + + if (!str) + { + cc = new ConstantColumn("", ConstantColumn::NULLDATA); + } + else if (ifp->result_type() == STRING_RESULT) + { + cc = new ConstantColumn(valStr, ConstantColumn::LITERAL); + } + else if (ifp->result_type() == DECIMAL_RESULT) + { + cc = buildDecimalColumn(ifp, gwi); + } + else + { + cc = new ConstantColumn(valStr, ConstantColumn::NUM); + cc->resultType(colType_MysqlToIDB(item)); + } + + SRCP srcp(cc); + + if (ifp->name) + cc->alias(ifp->name); + + gwi.returnedCols.push_back(srcp); + + // clear the error set by buildFunctionColumn + gwi.fatalParseError = false; + gwi.parseErrorText = ""; + break; + } + else if (hasNonSupportItem || parseInfo & AGG_BIT || parseInfo & SUB_BIT || + (gwi.fatalParseError && gwi.subQuery)) + { + if (gwi.parseErrorText.empty()) + { + Message::Args args; + args.add(ifp->func_name()); + gwi.parseErrorText = IDBErrorInfo::instance()->errorMsg(ERR_NON_SUPPORTED_FUNCTION, args); + } + + setError(gwi.thd, ER_CHECK_NOT_IMPLEMENTED, gwi.parseErrorText, gwi); + return ER_CHECK_NOT_IMPLEMENTED; + } + else if ( gwi.subQuery && (isPredicateFunction(ifp, &gwi) || ifp->type() == Item::COND_ITEM )) + { + gwi.fatalParseError = true; + gwi.parseErrorText = IDBErrorInfo::instance()->errorMsg(ERR_FILTER_COND_EXP); + setError(gwi.thd, ER_CHECK_NOT_IMPLEMENTED, gwi.parseErrorText, gwi); + return ER_CHECK_NOT_IMPLEMENTED; + } + + //@Bug 3030 Add error check for dml statement + if ( ((gwi.thd->lex)->sql_command == SQLCOM_UPDATE ) || ((gwi.thd->lex)->sql_command == SQLCOM_DELETE ) || ((gwi.thd->lex)->sql_command == SQLCOM_UPDATE_MULTI ) || ((gwi.thd->lex)->sql_command == SQLCOM_DELETE_MULTI ) ) + { + if ( after_size - before_size != 0 ) + { + gwi.parseErrorText = ifp->func_name(); + return -1; + } + } + + //@Bug 3021. Bypass postprocess for update and delete. + //if ( ((gwi.thd->lex)->sql_command == SQLCOM_UPDATE ) || ((gwi.thd->lex)->sql_command == SQLCOM_DELETE ) || ((gwi.thd->lex)->sql_command == SQLCOM_UPDATE_MULTI ) || ((gwi.thd->lex)->sql_command == SQLCOM_DELETE_MULTI )) + //{} + else + { + // @bug 3881. Here is the real redo part. + redo = true; + // @bug 1706 + String funcStr; + ifp->print(&funcStr, QT_INFINIDB); + gwi.selectCols.push_back(string(funcStr.c_ptr()) + " `" + escapeBackTick(ifp->name) + "`"); + // clear the error set by buildFunctionColumn + gwi.fatalParseError = false; + gwi.parseErrorText = ""; + } + } + + break; + } + + case Item::INT_ITEM: + { + if ( ((gwi.thd->lex)->sql_command == SQLCOM_UPDATE ) || ((gwi.thd->lex)->sql_command == SQLCOM_DELETE ) || ((gwi.thd->lex)->sql_command == SQLCOM_UPDATE_MULTI ) || ((gwi.thd->lex)->sql_command == SQLCOM_DELETE_MULTI )) + { } + else + { + // do not push the dummy column (mysql added) to returnedCol + if (item->name && string(item->name) == "Not_used") + continue; + + // @bug3509. Constant column is sent to ExeMgr now. + SRCP srcp(buildReturnedColumn(item, gwi, gwi.fatalParseError)); + + if (item->name) + srcp->alias(item->name); + + gwi.returnedCols.push_back(srcp); + + Item_int* isp = reinterpret_cast(item); + ostringstream oss; + oss << isp->value << " `" << escapeBackTick(srcp->alias().c_str()) << "`"; + + if (sel_cols_in_create.length() != 0) + sel_cols_in_create += ", "; + + sel_cols_in_create += oss.str(); + gwi.selectCols.push_back(oss.str()); + } + + break; + } + + case Item::STRING_ITEM: + { + if ( ((gwi.thd->lex)->sql_command == SQLCOM_UPDATE ) || ((gwi.thd->lex)->sql_command == SQLCOM_DELETE ) || ((gwi.thd->lex)->sql_command == SQLCOM_UPDATE_MULTI ) || ((gwi.thd->lex)->sql_command == SQLCOM_DELETE_MULTI )) + { } + else + { + SRCP srcp(buildReturnedColumn(item, gwi, gwi.fatalParseError)); + gwi.returnedCols.push_back(srcp); + + if (item->name) + srcp->alias(item->name); + + Item_string* isp = reinterpret_cast(item); + String val, *str = isp->val_str(&val); + string valStr; + valStr.assign(str->ptr(), str->length()); + string name = "'" + valStr + "'" + " " + "`" + escapeBackTick(srcp->alias().c_str()) + "`"; + + if (sel_cols_in_create.length() != 0) + sel_cols_in_create += ", "; + + sel_cols_in_create += name; + gwi.selectCols.push_back(name); + } + + break; + } + + case Item::DECIMAL_ITEM: + { + if ( ((gwi.thd->lex)->sql_command == SQLCOM_UPDATE ) || ((gwi.thd->lex)->sql_command == SQLCOM_DELETE ) || ((gwi.thd->lex)->sql_command == SQLCOM_UPDATE_MULTI ) || ((gwi.thd->lex)->sql_command == SQLCOM_DELETE_MULTI )) + { } + else + { + SRCP srcp(buildReturnedColumn(item, gwi, gwi.fatalParseError)); + gwi.returnedCols.push_back(srcp); + + if (item->name) + srcp->alias(item->name); + + Item_decimal* isp = reinterpret_cast(item); + String val, *str = isp->val_str(&val); + string valStr; + valStr.assign(str->ptr(), str->length()); + ostringstream oss; + oss << valStr.c_str() << " `" << escapeBackTick(srcp->alias().c_str()) << "`"; + + if (sel_cols_in_create.length() != 0) + sel_cols_in_create += ", "; + + sel_cols_in_create += oss.str(); + gwi.selectCols.push_back(oss.str()); + } + + break; + } + + case Item::NULL_ITEM: + { + /*if ( ((gwi.thd->lex)->sql_command == SQLCOM_UPDATE ) || ((gwi.thd->lex)->sql_command == SQLCOM_DELETE ) || ((gwi.thd->lex)->sql_command == SQLCOM_UPDATE_MULTI ) || ((gwi.thd->lex)->sql_command == SQLCOM_DELETE_MULTI ) ) + { } + else + { + SRCP srcp(buildReturnedColumn(item, gwi, gwi.fatalParseError)); + gwi.returnedCols.push_back(srcp); + + if (item->name) + srcp->alias(item->name); + + string name = string("null `") + escapeBackTick(srcp->alias().c_str()) + string("`") ; + + if (sel_cols_in_create.length() != 0) + sel_cols_in_create += ", "; + + sel_cols_in_create += name; + gwi.selectCols.push_back("null"); + }*/ + + break; + } + + case Item::SUBSELECT_ITEM: + { + Item_subselect* sub = (Item_subselect*)item; + + if (sub->substype() != Item_subselect::SINGLEROW_SUBS) + { + gwi.fatalParseError = true; + gwi.parseErrorText = IDBErrorInfo::instance()->errorMsg(ERR_NON_SUPPORT_SELECT_SUB); + setError(gwi.thd, ER_CHECK_NOT_IMPLEMENTED, gwi.parseErrorText, gwi); + return ER_CHECK_NOT_IMPLEMENTED; + } + +#ifdef DEBUG_WALK_COND + cerr << "SELECT clause SUBSELECT Item: " << sub->substype() << endl; + JOIN* join = sub->get_select_lex()->join; + + if (join) + { + Item_cond* cond = reinterpret_cast(join->conds); + + if (cond) + cond->traverse_cond(debug_walk, &gwi, Item::POSTFIX); + } + + cerr << "Finish SELECT clause subselect item traversing" << endl; +#endif + SelectSubQuery* selectSub = new SelectSubQuery(gwi, sub); + //selectSub->gwip(&gwi); + SCSEP ssub = selectSub->transform(); + + if (!ssub || gwi.fatalParseError) + { + if (gwi.parseErrorText.empty()) + gwi.parseErrorText = "Unsupported Item in SELECT subquery."; + + setError(gwi.thd, ER_CHECK_NOT_IMPLEMENTED, gwi.parseErrorText, gwi); + return ER_CHECK_NOT_IMPLEMENTED; + } + + selectSubList.push_back(ssub); + SimpleColumn* rc = new SimpleColumn(); + rc->colSource(rc->colSource() | SELECT_SUB); + + if (sub->get_select_lex()->get_table_list()) + rc->viewName(lower(getViewName(sub->get_select_lex()->get_table_list()))); + + if (sub->name) + rc->alias(sub->name); + + gwi.returnedCols.push_back(SRCP(rc)); + String str; + sub->get_select_lex()->print(gwi.thd, &str, QT_INFINIDB_NO_QUOTE); + sel_cols_in_create += "(" + string(str.c_ptr()) + ")"; + + if (sub->name) + { + sel_cols_in_create += "`" + escapeBackTick(sub->name) + "`"; + gwi.selectCols.push_back(sub->name); + } + else + { + sel_cols_in_create += "`" + escapeBackTick(str.c_ptr()) + "`"; + gwi.selectCols.push_back("`" + escapeBackTick(str.c_ptr()) + "`"); + } + + break; + } + + case Item::COND_ITEM: + { + gwi.fatalParseError = true; + gwi.parseErrorText = IDBErrorInfo::instance()->errorMsg(ERR_FILTER_COND_EXP); + setError(gwi.thd, ER_CHECK_NOT_IMPLEMENTED, gwi.parseErrorText, gwi); + return ER_CHECK_NOT_IMPLEMENTED; + } + + case Item::EXPR_CACHE_ITEM: + { + printf("EXPR_CACHE_ITEM in getSelectPlan\n"); + gwi.fatalParseError = true; + gwi.parseErrorText = IDBErrorInfo::instance()->errorMsg(ERR_UNKNOWN_COL); + setError(gwi.thd, ER_CHECK_NOT_IMPLEMENTED, gwi.parseErrorText, gwi); + return ER_CHECK_NOT_IMPLEMENTED; + } + + case Item::WINDOW_FUNC_ITEM: + { + SRCP srcp(buildWindowFunctionColumn(item, gwi, gwi.fatalParseError)); + + if (!srcp || gwi.fatalParseError) + { + if (gwi.parseErrorText.empty()) + gwi.parseErrorText = "Unsupported Item in SELECT subquery."; + + setError(gwi.thd, ER_CHECK_NOT_IMPLEMENTED, gwi.parseErrorText, gwi); + return ER_CHECK_NOT_IMPLEMENTED; + } + + gwi.returnedCols.push_back(srcp); + break; + } + + default: + { + break; + } + } + } + + // @bug4388 normalize the project coltypes for union main select list + if (!csep->unionVec().empty()) + { + for (uint32_t i = 0; i < gwi.returnedCols.size(); i++) + { + vector coltypes; + + for (uint32_t j = 0; j < csep->unionVec().size(); j++) + { + coltypes.push_back( + dynamic_cast(csep->unionVec()[j].get())->returnedCols()[i]->resultType()); + + // @bug5976. set hasAggregate true for the main column if + // one corresponding union column has aggregate + if (dynamic_cast(csep->unionVec()[j].get())->returnedCols()[i]->hasAggregate()) + gwi.returnedCols[i]->hasAggregate(true); + } + + gwi.returnedCols[i]->resultType(dataconvert::DataConvert::convertUnionColType(coltypes)); + } + } + + // Having clause handling + gwi.clauseType = HAVING; + clearStacks(gwi); + ParseTree* havingFilter = 0; + // clear fatalParseError that may be left from post process functions + gwi.fatalParseError = false; + gwi.parseErrorText = ""; + + if (gi.groupByHaving != 0) + { + Item_cond* having = reinterpret_cast(gi.groupByHaving); +#ifdef DEBUG_WALK_COND + cerr << "------------------- HAVING ---------------------" << endl; + having->traverse_cond(debug_walk, &gwi, Item::POSTFIX); + cerr << "------------------------------------------------\n" << endl; +#endif + having->traverse_cond(gp_walk, &gwi, Item::POSTFIX); + + if (gwi.fatalParseError) + { + setError(gwi.thd, ER_INTERNAL_ERROR, gwi.parseErrorText, gwi); + return ER_INTERNAL_ERROR; + } + + ParseTree* ptp = 0; + ParseTree* rhs = 0; + + // @bug 4215. some function filter will be in the rcWorkStack. + if (gwi.ptWorkStack.empty() && !gwi.rcWorkStack.empty()) + { + havingFilter = new ParseTree(gwi.rcWorkStack.top()); + gwi.rcWorkStack.pop(); + } + + while (!gwi.ptWorkStack.empty()) + { + havingFilter = gwi.ptWorkStack.top(); + gwi.ptWorkStack.pop(); + + if (gwi.ptWorkStack.empty()) + break; + + ptp = new ParseTree(new LogicOperator("and")); + ptp->left(havingFilter); + rhs = gwi.ptWorkStack.top(); + gwi.ptWorkStack.pop(); + ptp->right(rhs); + gwi.ptWorkStack.push(ptp); + } + } + + // for post process expressions on the select list + // error out post process for union and sub select unit + if (isUnion || gwi.subSelectType != CalpontSelectExecutionPlan::MAIN_SELECT) + { + if (funcFieldVec.size() != 0 && !gwi.fatalParseError) + { + string emsg("Fatal parse error in vtable mode: Unsupported Items in union or sub select unit"); + setError(gwi.thd, ER_CHECK_NOT_IMPLEMENTED, emsg); + return ER_CHECK_NOT_IMPLEMENTED; + } + } + + for (uint32_t i = 0; i < funcFieldVec.size(); i++) + { + SimpleColumn* sc = buildSimpleColumn(funcFieldVec[i], gwi); + + if (!sc || gwi.fatalParseError) + { + string emsg; + + if (gwi.parseErrorText.empty()) + { + emsg = "un-recognized column"; + + if (funcFieldVec[i]->name) + emsg += string(funcFieldVec[i]->name); + } + else + { + emsg = gwi.parseErrorText; + } + + setError(gwi.thd, ER_INTERNAL_ERROR, emsg, gwi); + return ER_INTERNAL_ERROR; + } + + String str; + funcFieldVec[i]->print(&str, QT_INFINIDB_NO_QUOTE); + sc->alias(string(str.c_ptr())); + //sc->tableAlias(funcFieldVec[i]->table_name); + sc->tableAlias(sc->tableAlias()); + SRCP srcp(sc); + uint32_t j = 0; + + for (; j < gwi.returnedCols.size(); j++) + { + if (sc->sameColumn(gwi.returnedCols[j].get())) + { + SimpleColumn* field = dynamic_cast(gwi.returnedCols[j].get()); + + if (field && field->alias() == sc->alias()) + break; + } + } + + if (j == gwi.returnedCols.size()) + { + gwi.returnedCols.push_back(srcp); + gwi.columnMap.insert(CalpontSelectExecutionPlan::ColumnMap::value_type(string(funcFieldVec[i]->field_name), srcp)); + + if (sel_cols_in_create.length() != 0) + sel_cols_in_create += ", "; + + string fullname; + fullname = str.c_ptr(); + sel_cols_in_create += fullname + " `" + escapeBackTick(fullname.c_str()) + "`"; + TABLE_LIST* tmp = (funcFieldVec[i]->cached_table ? funcFieldVec[i]->cached_table : 0); + gwi.tableMap[make_aliastable(sc->schemaName(), sc->tableName(), sc->tableAlias(), sc->isInfiniDB())] = + make_pair(1, tmp); + } + } + + // post-process Order by list and expressions on select by redo phase1. only for vtable + // ignore ORDER BY clause for union select unit + string ord_cols = ""; // for normal select phase + SRCP minSc; // min width projected column. for count(*) use + + // Group by list. not valid for union main query + if (gwi.thd->infinidb_vtable.vtable_state == THD::INFINIDB_CREATE_VTABLE && !unionSel) + { + gwi.clauseType = GROUP_BY; + Item* nonSupportItem = NULL; + ORDER* groupcol = reinterpret_cast(gi.groupByGroup); + + // check if window functions are in order by. InfiniDB process order by list if + // window functions are involved, either in order by or projection. + bool hasWindowFunc = gwi.hasWindowFunc; + gwi.hasWindowFunc = false; + + for (; groupcol; groupcol = groupcol->next) + { + if ((*(groupcol->item))->type() == Item::WINDOW_FUNC_ITEM) + gwi.hasWindowFunc = true; + } + + if (gwi.hasWindowFunc) + { + gwi.fatalParseError = true; + gwi.parseErrorText = IDBErrorInfo::instance()->errorMsg(ERR_WF_NOT_ALLOWED, "GROUP BY clause"); + setError(gwi.thd, ER_CHECK_NOT_IMPLEMENTED, gwi.parseErrorText, gwi); + return ER_CHECK_NOT_IMPLEMENTED; + } + + gwi.hasWindowFunc = hasWindowFunc; + groupcol = reinterpret_cast(gi.groupByGroup); + + for (; groupcol; groupcol = groupcol->next) + { + Item* groupItem = *(groupcol->item); + + // @bug5993. Could be nested ref. + while (groupItem->type() == Item::REF_ITEM) + groupItem = (*((Item_ref*)groupItem)->ref); + + if (groupItem->type() == Item::FUNC_ITEM) + { + Item_func* ifp = (Item_func*)groupItem; + + // call buildFunctionColumn here mostly for finding out + // non-support column on GB list. Should be simplified. + ReturnedColumn* fc = buildFunctionColumn(ifp, gwi, gwi.fatalParseError); + + if (!fc || gwi.fatalParseError) + { + nonSupportItem = ifp; + break; + } + + if (groupcol->in_field_list && groupcol->counter_used) + { + delete fc; + fc = gwi.returnedCols[groupcol->counter - 1].get(); + SRCP srcp(fc->clone()); + + // check if no column parm + for (uint32_t i = 0; i < gwi.no_parm_func_list.size(); i++) + { + if (gwi.no_parm_func_list[i]->expressionId() == fc->expressionId()) + { + gwi.no_parm_func_list.push_back(dynamic_cast(srcp.get())); + break; + } + } + + srcp->orderPos(groupcol->counter - 1); + gwi.groupByCols.push_back(srcp); + continue; + } + else if (!groupItem->is_autogenerated_name) // alias + { + uint32_t i = 0; + + for (; i < gwi.returnedCols.size(); i++) + { + if (string(groupItem->name) == gwi.returnedCols[i]->alias()) + { + ReturnedColumn* rc = gwi.returnedCols[i]->clone(); + rc->orderPos(i); + gwi.groupByCols.push_back(SRCP(rc)); + delete fc; + break; + } + } + + if (i == gwi.returnedCols.size()) + { + nonSupportItem = groupItem; + break; + } + } + else + { + uint32_t i = 0; + + for (; i < gwi.returnedCols.size(); i++) + { + if (fc->operator==(gwi.returnedCols[i].get())) + { + ReturnedColumn* rc = gwi.returnedCols[i]->clone(); + rc->orderPos(i); + gwi.groupByCols.push_back(SRCP(rc)); + delete fc; + break; + } + } + + if (i == gwi.returnedCols.size()) + { + gwi.groupByCols.push_back(SRCP(fc)); + break; + } + } + } + else if (groupItem->type() == Item::FIELD_ITEM) + { + Item_field* ifp = (Item_field*)groupItem; + // this GB col could be an alias of F&E on the SELECT clause, not necessarily a field. + ReturnedColumn* rc = buildSimpleColumn(ifp, gwi); + SimpleColumn* sc = dynamic_cast(rc); + + for (uint32_t j = 0; j < gwi.returnedCols.size(); j++) + { + if (sc) + { + if (sc->sameColumn(gwi.returnedCols[j].get())) + { + sc->orderPos(j); + break; + } + else if (strcasecmp(sc->alias().c_str(), gwi.returnedCols[j]->alias().c_str()) == 0) + { + rc = gwi.returnedCols[j].get()->clone(); + rc->orderPos(j); + break; + } + } + else + { + if (ifp->name && string(ifp->name) == gwi.returnedCols[j].get()->alias()) + { + rc = gwi.returnedCols[j].get()->clone(); + rc->orderPos(j); + break; + } + } + } + + if (!rc) + { + nonSupportItem = ifp; + break; + } + + SRCP srcp(rc); + + // bug 3151 + AggregateColumn* ac = dynamic_cast(rc); + + if (ac) + { + nonSupportItem = ifp; + break; + } + + gwi.groupByCols.push_back(srcp); + gwi.columnMap.insert(CalpontSelectExecutionPlan::ColumnMap::value_type(string(ifp->field_name), srcp)); + } + // @bug5638. The group by column is constant but not counter, alias has to match a column + // on the select list + else if (!groupcol->counter_used && + (groupItem->type() == Item::INT_ITEM || + groupItem->type() == Item::STRING_ITEM || + groupItem->type() == Item::REAL_ITEM || + groupItem->type() == Item::DECIMAL_ITEM)) + { + ReturnedColumn* rc = 0; + + for (uint32_t j = 0; j < gwi.returnedCols.size(); j++) + { + if (groupItem->name && string(groupItem->name) == gwi.returnedCols[j].get()->alias()) + { + rc = gwi.returnedCols[j].get()->clone(); + rc->orderPos(j); + break; + } + } + + if (!rc) + { + nonSupportItem = groupItem; + break; + } + + gwi.groupByCols.push_back(SRCP(rc)); + } + else if ((*(groupcol->item))->type() == Item::SUBSELECT_ITEM) + { + if (!groupcol->in_field_list || !groupItem->name) + { + nonSupportItem = groupItem; + } + else + { + uint32_t i = 0; + + for (; i < gwi.returnedCols.size(); i++) + { + if (string(groupItem->name) == gwi.returnedCols[i]->alias()) + { + ReturnedColumn* rc = gwi.returnedCols[i]->clone(); + rc->orderPos(i); + gwi.groupByCols.push_back(SRCP(rc)); + break; + } + } + + if (i == gwi.returnedCols.size()) + { + nonSupportItem = groupItem; + } + } + } + // @bug 3761. + else if (groupcol->counter_used) + { + if (gwi.returnedCols.size() <= (uint32_t)(groupcol->counter - 1)) + { + nonSupportItem = groupItem; + } + else + { + gwi.groupByCols.push_back(SRCP(gwi.returnedCols[groupcol->counter - 1]->clone())); + } + } + else + { + nonSupportItem = groupItem; + } + + } + + // @bug 4756. Add internal groupby column for correlated join to the groupby list + if (gwi.aggOnSelect && !gwi.subGroupByCols.empty()) + gwi.groupByCols.insert(gwi.groupByCols.end(), gwi.subGroupByCols.begin(), gwi.subGroupByCols.end()); + + // this is window func on SELECT becuase ORDER BY has not been processed + if (!gwi.windowFuncList.empty() && !gwi.subGroupByCols.empty()) + { + for (uint32_t i = 0; i < gwi.windowFuncList.size(); i++) + { + if (gwi.windowFuncList[i]->hasWindowFunc()) + { + vector windowFunctions = gwi.windowFuncList[i]->windowfunctionColumnList(); + + for (uint32_t j = 0; j < windowFunctions.size(); j++) + windowFunctions[j]->addToPartition(gwi.subGroupByCols); + } + } + } + + if (nonSupportItem) + { + Message::Args args; + + if (nonSupportItem->name) + args.add("'" + string(nonSupportItem->name) + "'"); + else + args.add(""); + + gwi.parseErrorText = IDBErrorInfo::instance()->errorMsg(ERR_NON_SUPPORT_GROUP_BY, args); + setError(gwi.thd, ER_CHECK_NOT_IMPLEMENTED, gwi.parseErrorText, gwi); + return ER_CHECK_NOT_IMPLEMENTED; + } + } // GROUP processing ends here + + + if (gwi.thd->infinidb_vtable.vtable_state == THD::INFINIDB_CREATE_VTABLE) + { + ORDER* ordercol = reinterpret_cast(gi.groupByOrder); + string create_query(gwi.thd->infinidb_vtable.create_vtable_query.c_ptr()); + string select_query(gwi.thd->infinidb_vtable.select_vtable_query.c_ptr()); + string lower_create_query(gwi.thd->infinidb_vtable.create_vtable_query.c_ptr()); + string lower_select_query(gwi.thd->infinidb_vtable.select_vtable_query.c_ptr()); + algorithm::to_lower(lower_create_query); + algorithm::to_lower(lower_select_query); + + + // check if window functions are in order by. InfiniDB process order by list if + // window functions are involved, either in order by or projection. + for (; ordercol; ordercol = ordercol->next) + { + if ((*(ordercol->item))->type() == Item::WINDOW_FUNC_ITEM) + gwi.hasWindowFunc = true; + } + + // re-visit the first of ordercol list + ordercol = reinterpret_cast(gi.groupByOrder); + + // for subquery, order+limit by will be supported in infinidb. build order by columns + // @todo union order by and limit support + //if (gwi.hasWindowFunc || gwi.subSelectType != CalpontSelectExecutionPlan::MAIN_SELECT) + + for (; ordercol; ordercol = ordercol->next) + { + ReturnedColumn* rc = NULL; + + if (ordercol->in_field_list && ordercol->counter_used) + { + rc = gwi.returnedCols[ordercol->counter - 1]->clone(); + rc->orderPos(ordercol->counter - 1); + // can not be optimized off if used in order by with counter. + // set with self derived table alias if it's derived table + gwi.returnedCols[ordercol->counter - 1]->incRefCount(); + } + else + { + Item* ord_item = *(ordercol->item); + + // ignore not_used column on order by. + if (ord_item->type() == Item::INT_ITEM && ord_item->full_name() && string(ord_item->full_name()) == "Not_used") + continue; + else if (ord_item->type() == Item::INT_ITEM) + rc = gwi.returnedCols[((Item_int*)ord_item)->val_int() - 1]->clone(); + else if (ord_item->type() == Item::SUBSELECT_ITEM) + gwi.fatalParseError = true; + else + rc = buildReturnedColumn(ord_item, gwi, gwi.fatalParseError); + + // Looking for a match for this item in GROUP BY list. + if ( rc && ord_item->type() == Item::FIELD_ITEM ) + { + execplan::CalpontSelectExecutionPlan::ReturnedColumnList::iterator iter = gwi.groupByCols.begin(); + + for ( ; iter != gwi.groupByCols.end(); iter++ ) + { + if ( rc->sameColumn((*iter).get()) ) + break; + } + + // MCOL-1052 Find and remove the optimized field + // from ORDER using cond_pushed filters. + if (buildConstColFromFilter( + dynamic_cast(rc), gwi, gi)) + { + break; + } + + // MCOL-1052 GROUP BY items list doesn't contain + // this ORDER BY item. + if ( iter == gwi.groupByCols.end() ) + { + Item_ident* iip = reinterpret_cast(ord_item); + std::ostringstream ostream; + ostream << "'"; + + if (iip->db_name) + ostream << iip->db_name << '.'; + else + ostream << "unknown db" << '.'; + + if (iip->table_name) + ostream << iip->table_name << '.'; + else + ostream << "unknown table" << '.'; + + if (iip->field_name) + ostream << iip->field_name; + else + ostream << "unknown field"; + + ostream << "'"; + Message::Args args; + args.add(ostream.str()); + string emsg = IDBErrorInfo::instance()->errorMsg(ERR_NOT_GROUPBY_EXPRESSION, args); + gwi.parseErrorText = emsg; + setError(gwi.thd, ER_INTERNAL_ERROR, emsg, gwi); + return ERR_NOT_GROUPBY_EXPRESSION; + } + } + + // @bug5501 try item_ptr if item can not be fixed. For some + // weird dml statement state, item can not be fixed but the + // infomation is available in item_ptr. + if (!rc || gwi.fatalParseError) + { + gwi.fatalParseError = false; + Item* item_ptr = ordercol->item_ptr; + + while (item_ptr->type() == Item::REF_ITEM) + item_ptr = *(((Item_ref*)item_ptr)->ref); + + rc = buildReturnedColumn(item_ptr, gwi, gwi.fatalParseError); + } + + // This ORDER BY item must be an agg function - + // the ordercol->item_ptr and exteded SELECT list + // must contain the corresponding item. + if (!rc) + { + Item* item_ptr = ordercol->item_ptr; + + if (item_ptr) + rc = buildReturnedColumn(item_ptr, gwi, gwi.fatalParseError); + } + + if (!rc) + { + string emsg = IDBErrorInfo::instance()->errorMsg(ERR_NON_SUPPORT_ORDER_BY); + gwi.parseErrorText = emsg; + setError(gwi.thd, ER_CHECK_NOT_IMPLEMENTED, emsg, gwi); + return ER_CHECK_NOT_IMPLEMENTED; + } + } + + if (ordercol->direction == ORDER::ORDER_ASC) + rc->asc(true); + else + rc->asc(false); + + gwi.orderByCols.push_back(SRCP(rc)); + } + + + // make sure columnmap, returnedcols and count(*) arg_list are not empty + TableMap::iterator tb_iter = gwi.tableMap.begin(); + + try + { + for (; tb_iter != gwi.tableMap.end(); tb_iter++) + { + if ((*tb_iter).second.first == 1) continue; + + CalpontSystemCatalog::TableAliasName tan = (*tb_iter).first; + CalpontSystemCatalog::TableName tn = make_table((*tb_iter).first.schema, (*tb_iter).first.table); + SimpleColumn* sc = getSmallestColumn(csc, tn, tan, (*tb_iter).second.second->table, gwi); + SRCP srcp(sc); + gwi.columnMap.insert(CalpontSelectExecutionPlan::ColumnMap::value_type(sc->columnName(), srcp)); + (*tb_iter).second.first = 1; + } + } + catch (runtime_error& e) + { + setError(gwi.thd, ER_INTERNAL_ERROR, e.what(), gwi); + CalpontSystemCatalog::removeCalpontSystemCatalog(sessionID); + return ER_INTERNAL_ERROR; + } + catch (...) + { + string emsg = IDBErrorInfo::instance()->errorMsg(ERR_LOST_CONN_EXEMGR); + setError(gwi.thd, ER_INTERNAL_ERROR, emsg, gwi); + CalpontSystemCatalog::removeCalpontSystemCatalog(sessionID); + return ER_INTERNAL_ERROR; + } + + if (!gwi.count_asterisk_list.empty() || !gwi.no_parm_func_list.empty() || + gwi.returnedCols.empty()) + { + // get the smallest column from colmap + CalpontSelectExecutionPlan::ColumnMap::const_iterator iter; + int minColWidth = 0; + CalpontSystemCatalog::ColType ct; + + try + { + for (iter = gwi.columnMap.begin(); iter != gwi.columnMap.end(); ++iter) + { + // should always not null + SimpleColumn* sc = dynamic_cast(iter->second.get()); + + if (sc && !(sc->joinInfo() & JOIN_CORRELATED)) + { + ct = csc->colType(sc->oid()); + + if (minColWidth == 0) + { + minColWidth = ct.colWidth; + minSc = iter->second; + } + else if (ct.colWidth < minColWidth) + { + minColWidth = ct.colWidth; + minSc = iter->second; + } + } + } + } + catch (...) + { + string emsg = IDBErrorInfo::instance()->errorMsg(ERR_LOST_CONN_EXEMGR); + setError(gwi.thd, ER_INTERNAL_ERROR, emsg, gwi); + CalpontSystemCatalog::removeCalpontSystemCatalog(sessionID); + return ER_INTERNAL_ERROR; + } + + if (gwi.returnedCols.empty() && gwi.additionalRetCols.empty()) + gwi.returnedCols.push_back(minSc); + } + + if (!isUnion && !gwi.hasWindowFunc && gwi.subSelectType == CalpontSelectExecutionPlan::MAIN_SELECT) + { + std::ostringstream vtb; + vtb << "infinidb_vtable.$vtable_" << gwi.thd->thread_id; + + //vtb << "$vtable_" << gwi.thd->thread_id; + // re-construct the select query and redo phase 1 + if (redo) + { + // select now() from region case. returnedCols should have minSc. + if (sel_cols_in_create.length() == 0) + { + SimpleColumn* sc = dynamic_cast(gwi.returnedCols[0].get()); + + if (sc) + sel_cols_in_create = dynamic_cast(gwi.returnedCols[0].get())->columnName(); + else + sel_cols_in_create = gwi.returnedCols[0]->alias(); + } + + // select * from derived table case + if (gwi.selectCols.empty()) + sel_cols_in_create = " * "; + + create_query = "create temporary table " + vtb.str() + " engine = aria as select " + sel_cols_in_create + " from "; + TABLE_LIST* table_ptr = gi.groupByTables; + + bool firstTb = true; + + // put all tables, derived tables and views on the list + //TABLE_LIST* table_ptr = select_lex.get_table_list(); + set aliasSet; // to avoid duplicate table alias + + for (; table_ptr; table_ptr = table_ptr->next_local) + { + if (string(table_ptr->table_name).find("$vtable") != string::npos) + continue; + + if (table_ptr->derived) + { + if (aliasSet.find(table_ptr->alias) != aliasSet.end()) + continue; + + String str; + (table_ptr->derived->first_select())->print(gwi.thd, &str, QT_INFINIDB_DERIVED); + + if (!firstTb) + create_query += ", "; + + create_query += "(" + string(str.c_ptr()) + ") " + string(table_ptr->alias); + firstTb = false; + aliasSet.insert(table_ptr->alias); + } + else if (table_ptr->view) + { + if (aliasSet.find(table_ptr->alias) != aliasSet.end()) + continue; + + if (!firstTb) + create_query += ", "; + + create_query += string(table_ptr->db) + "." + string(table_ptr->table_name) + + string(" `") + escapeBackTick(table_ptr->alias) + string("`"); + aliasSet.insert(table_ptr->alias); + firstTb = false; + } + else + { + // table referenced by view is represented by viewAlias_tableAlias. + // consistent with item.cc field print. + if (table_ptr->referencing_view) + { + if (aliasSet.find(string(table_ptr->referencing_view->alias) + "_" + + string(table_ptr->alias)) != aliasSet.end()) + continue; + + if (!firstTb) + create_query += ", "; + + create_query += string(table_ptr->db) + "." + string(table_ptr->table_name) + string(" "); + create_query += string(" `") + + escapeBackTick(table_ptr->referencing_view->alias) + "_" + + escapeBackTick(table_ptr->alias) + string("`"); + aliasSet.insert(string(table_ptr->referencing_view->alias) + "_" + + string(table_ptr->alias)); + } + else + { + if (aliasSet.find(table_ptr->alias) != aliasSet.end()) + continue; + + if (!firstTb) + create_query += ", "; + + create_query += string(table_ptr->db) + "." + string(table_ptr->table_name) + string(" "); + create_query += string("`") + escapeBackTick(table_ptr->alias) + string("`"); + aliasSet.insert(table_ptr->alias); + } + + firstTb = false; + } + } + + + gwi.thd->infinidb_vtable.create_vtable_query.free(); + gwi.thd->infinidb_vtable.create_vtable_query.append(create_query.c_str(), create_query.length()); + gwi.thd->infinidb_vtable.vtable_state = THD::INFINIDB_REDO_PHASE1; // redo phase 1 + + // turn off select distinct from post process unless there're post process functions + // on the select list. + string sel_query = "select "; + + if (gi.groupByDistinct && redo) + sel_query = "select distinct "; + else + sel_query = "select "; + + // select * from derived table... + if (gwi.selectCols.size() == 0) + sel_query += " * "; + + for (uint32_t i = 0; i < gwi.selectCols.size(); i++) + { + sel_query += gwi.selectCols[i]; + + if ( i + 1 != gwi.selectCols.size()) + sel_query += ", "; + } + + select_query.replace(lower_select_query.find("select *"), string("select *").length(), sel_query); + } + else + { + // remove order by clause in case this phase has been executed before. + // need a better fix later, like skip all the other non-optimized phase. + size_t pos = lower_select_query.find("order by"); + + if (pos != string::npos) + select_query.replace(pos, lower_select_query.length() - pos, ""); + + //select_query = "select * from " + vtb.str(); + // MCOL-1052 + if (unionSel) + { + ordercol = reinterpret_cast(gi.groupByOrder); + //order_list = gi.groupByOrder; + } + else + ordercol = 0; + + ord_cols = ""; + + for (; ordercol; ordercol = ordercol->next) + { + Item* ord_item = *(ordercol->item); + + // @bug 1706. re-construct the order by item one by one, because the ord_cols constucted so far + // is for REDO phase. + if (ord_cols.length() != 0) + ord_cols += ", "; + + if (ordercol->in_field_list && ordercol->counter_used) + { + ostringstream oss; + oss << ordercol->counter; + ord_cols += oss.str(); + } + else if (ord_item->type() == Item::NULL_ITEM) + { + // MCOL-793 Do nothing for an ORDER BY NULL + } + else if (ord_item->type() == Item::SUM_FUNC_ITEM) + { + Item_sum* ifp = (Item_sum*)(*(ordercol->item)); + ReturnedColumn* fc = buildAggregateColumn(ifp, gwi); + + for (uint32_t i = 0; i < gwi.returnedCols.size(); i++) + { + if (fc->operator==(gwi.returnedCols[i].get())) + { + ostringstream oss; + oss << i + 1; + ord_cols += oss.str(); + break; + } + } + + //continue; + } + // @bug 3518. if order by clause = selected column, use position. + else if (ord_item->name && ord_item->type() == Item::FIELD_ITEM) + { + Item_field* field = reinterpret_cast(ord_item); + string fullname; + + if (field->db_name) + fullname += string(field->db_name) + "."; + + if (field->table_name) + fullname += string(field->table_name) + "."; + + if (field->field_name) + fullname += string(field->field_name); + + uint32_t i = 0; + + for (i = 0; i < gwi.returnedCols.size(); i++) + { + SimpleColumn* sc = dynamic_cast(gwi.returnedCols[i].get()); + + if (sc && ((Item_field*)ord_item)->cached_table && + (strcasecmp(getViewName(((Item_field*)ord_item)->cached_table).c_str(), sc->viewName().c_str()) != 0)) + continue; + + if (strcasecmp(fullname.c_str(), gwi.returnedCols[i]->alias().c_str()) == 0 || + strcasecmp(ord_item->name, gwi.returnedCols[i]->alias().c_str()) == 0) + { + ostringstream oss; + oss << i + 1; + ord_cols += oss.str(); + break; + } + } + + if (i == gwi.returnedCols.size()) + ord_cols += string(" `") + escapeBackTick(ord_item->name) + '`'; + } + + else if (ord_item->name) + { + // for union order by 1 case. For unknown reason, it doesn't show in_field_list + if (ord_item->type() == Item::INT_ITEM) + { + ord_cols += ord_item->name; + } + else if (ord_item->type() == Item::SUBSELECT_ITEM) + { + string emsg = IDBErrorInfo::instance()->errorMsg(ERR_NON_SUPPORT_ORDER_BY); + setError(gwi.thd, ER_CHECK_NOT_IMPLEMENTED, emsg, gwi); + return ER_CHECK_NOT_IMPLEMENTED; + } + else + { + ord_cols += string(" `") + escapeBackTick(ord_item->name) + '`'; + } + } + else if (ord_item->type() == Item::FUNC_ITEM) + { + // @bug5636. check if this order by column is on the select list + ReturnedColumn* rc = buildFunctionColumn((Item_func*)(ord_item), gwi, gwi.fatalParseError); + + for (uint32_t i = 0; i < gwi.returnedCols.size(); i++) + { + if (rc && rc->operator==(gwi.returnedCols[i].get())) + { + ostringstream oss; + oss << i + 1; + ord_cols += oss.str(); + break; + } + } + } + else + { + String str; + ord_item->print(&str, QT_INFINIDB_NO_QUOTE); + ord_cols += string(str.c_ptr()); + } + + if (ordercol->direction != ORDER::ORDER_ASC) + ord_cols += " desc"; + } + } + + if (ord_cols.length() > 0) // has order by + { + gwi.thd->infinidb_vtable.has_order_by = true; + csep->hasOrderBy(true); + ord_cols = " order by " + ord_cols; + select_query += ord_cols; + } + } + } // ORDER BY processing ends here + + if ( gi.groupByDistinct ) + csep->distinct(true); + + // add the smallest column to count(*) parm. + // select constant in subquery case + std::vector::iterator coliter; + + if (!minSc) + { + if (!gwi.returnedCols.empty()) + minSc = gwi.returnedCols[0]; + else if (!gwi.additionalRetCols.empty()) + minSc = gwi.additionalRetCols[0]; + } + + // @bug3523, count(*) on subquery always pick column[0]. + SimpleColumn* sc = dynamic_cast(minSc.get()); + + if (sc && sc->schemaName().empty()) + { + if (gwi.derivedTbList.size() >= 1) + { + SimpleColumn* sc1 = new SimpleColumn(); + sc1->columnName(sc->columnName()); + sc1->tableName(sc->tableName()); + sc1->tableAlias(sc->tableAlias()); + sc1->viewName(lower(sc->viewName())); + sc1->colPosition(0); + minSc.reset(sc1); + } + } + + for (coliter = gwi.count_asterisk_list.begin(); coliter != gwi.count_asterisk_list.end(); ++coliter) + { + // @bug5977 @note should never throw this, but checking just in case. + // When ExeMgr fix is ready, this should not error out... + if (dynamic_cast(minSc.get())) + { + gwi.fatalParseError = true; + gwi.parseErrorText = "No project column found for aggregate function"; + setError(gwi.thd, ER_INTERNAL_ERROR, gwi.parseErrorText, gwi); + return ER_CHECK_NOT_IMPLEMENTED; + } + + (*coliter)->functionParms(minSc); + } + + std::vector::iterator funciter; + + SPTP sptp(new ParseTree(minSc.get()->clone())); + + for (funciter = gwi.no_parm_func_list.begin(); funciter != gwi.no_parm_func_list.end(); ++funciter) + { + FunctionParm funcParms = (*funciter)->functionParms(); + funcParms.push_back(sptp); + (*funciter)->functionParms(funcParms); + } + + // set sequence# for subquery localCols + for (uint32_t i = 0; i < gwi.localCols.size(); i++) + gwi.localCols[i]->sequence(i); + + // append additionalRetCols to returnedCols + gwi.returnedCols.insert(gwi.returnedCols.begin(), gwi.additionalRetCols.begin(), + gwi.additionalRetCols.end()); + + csep->groupByCols(gwi.groupByCols); + csep->orderByCols(gwi.orderByCols); + csep->returnedCols(gwi.returnedCols); + csep->columnMap(gwi.columnMap); + csep->having(havingFilter); + csep->derivedTableList(gwi.derivedTbList); + csep->selectSubList(selectSubList); + csep->subSelectList(gwi.subselectList); + gwi.thd->infinidb_vtable.duplicate_field_name = false; + clearStacks(gwi); + return 0; +} + + } diff --git a/dbcon/mysql/ha_calpont_impl.cpp b/dbcon/mysql/ha_calpont_impl.cpp index 8078d452e..5ca94562b 100644 --- a/dbcon/mysql/ha_calpont_impl.cpp +++ b/dbcon/mysql/ha_calpont_impl.cpp @@ -171,6 +171,7 @@ const unsigned NONSUPPORTED_ERR_THRESH = 2000; vector rmParms; ResourceManager* rm = ResourceManager::instance(); bool useHdfs = rm->useHdfs(); + //convenience fcn inline uint32_t tid2sid(const uint32_t tid) { @@ -275,6 +276,104 @@ void storeNumericField(Field** f, int64_t value, CalpontSystemCatalog::ColType& } } +void storeNumericFieldGroupBy(Field** f, int64_t value, CalpontSystemCatalog::ColType& ct) +{ + // unset null bit first + if ((*f)->null_ptr) + *(*f)->null_ptr &= ~(*f)->null_bit; + + // For unsigned, use the ColType returned in the row rather than the + // unsigned_flag set by mysql. This is because mysql gets it wrong for SUM() + // Hopefully, in all other cases we get it right. + switch ((*f)->type()) + { + case MYSQL_TYPE_NEWDECIMAL: + { + Field_new_decimal* f2 = (Field_new_decimal*)*f; + + // @bug4388 stick to InfiniDB's scale in case mysql gives wrong scale due + // to create vtable limitation. + if (f2->dec < ct.scale) + f2->dec = ct.scale; + + char buf[256]; + dataconvert::DataConvert::decimalToString(value, (unsigned)ct.scale, buf, 256, ct.colDataType); + f2->store(buf, strlen(buf), f2->charset()); + break; + } + + case MYSQL_TYPE_TINY: //TINYINT type + { + Field_tiny* f2 = (Field_tiny*)*f; + longlong int_val = (longlong)value; + f2->store(int_val, f2->unsigned_flag); + break; + } + + case MYSQL_TYPE_SHORT: //SMALLINT type + { + Field_short* f2 = (Field_short*)*f; + longlong int_val = (longlong)value; + f2->store(int_val, f2->unsigned_flag); + break; + } + + case MYSQL_TYPE_LONG: //INT type + { + Field_long* f2 = (Field_long*)*f; + longlong int_val = (longlong)value; + f2->store(int_val, f2->unsigned_flag); + break; + } + + case MYSQL_TYPE_LONGLONG: //BIGINT type + { + Field_longlong* f2 = (Field_longlong*)*f; + longlong int_val = (longlong)value; + f2->store(int_val, f2->unsigned_flag); + break; + } + + case MYSQL_TYPE_FLOAT: // FLOAT type + { + Field_float* f2 = (Field_float*)*f; + float float_val = *(float*)(&value); + f2->store(float_val); + break; + } + + case MYSQL_TYPE_DOUBLE: // DOUBLE type + { + Field_double* f2 = (Field_double*)*f; + double double_val = *(double*)(&value); + f2->store(double_val); + break; + } + + case MYSQL_TYPE_VARCHAR: + { + Field_varstring* f2 = (Field_varstring*)*f; + char tmp[25]; + + if (ct.colDataType == CalpontSystemCatalog::DECIMAL) + dataconvert::DataConvert::decimalToString(value, (unsigned)ct.scale, tmp, 25, ct.colDataType); + else + snprintf(tmp, 25, "%ld", value); + + f2->store(tmp, strlen(tmp), f2->charset()); + break; + } + + default: + { + Field_longlong* f2 = (Field_longlong*)*f; + longlong int_val = (longlong)value; + f2->store(int_val, f2->unsigned_flag); + break; + } + } +} + // // @bug 2244. Log exception related to lost connection to ExeMgr. // Log exception error from calls to sm::tpl_scan_fetch in fetchNextRow() @@ -328,7 +427,7 @@ int vbin2hex(const uint8_t* p, const unsigned l, char* o) return 0; } -int fetchNextRow(uchar* buf, cal_table_info& ti, cal_connection_info* ci) +int fetchNextRow(uchar* buf, cal_table_info& ti, cal_connection_info* ci, bool handler_flag = false) { int rc = HA_ERR_END_OF_FILE; int num_attr = ti.msTablePtr->s->fields; @@ -370,8 +469,15 @@ int fetchNextRow(uchar* buf, cal_table_info& ti, cal_connection_info* ci) { Field** f; f = ti.msTablePtr->field; + //set all fields to null in null col bitmap - memset(buf, -1, ti.msTablePtr->s->null_bytes); + if (!handler_flag) + memset(buf, -1, ti.msTablePtr->s->null_bytes); + else + { + memset(ti.msTablePtr->null_flags, -1, ti.msTablePtr->s->null_bytes); + } + std::vector& colTypes = ti.tpl_scan_ctx->ctp; int64_t intColVal = 0; uint64_t uintColVal = 0; @@ -486,8 +592,21 @@ int fetchNextRow(uchar* buf, cal_table_info& ti, cal_connection_info* ci) * based on the result set. */ /* MCOL-683: UTF-8 datetime no msecs is 57, this sometimes happens! */ - if (((*f)->field_length > 19) && ((*f)->field_length != 57)) - (*f)->field_length = strlen(tmp); +// if (((*f)->field_length > 19) && ((*f)->field_length != 57)) +// (*f)->field_length = strlen(tmp); + + Field_varstring* f2 = (Field_varstring*)*f; + f2->store(tmp, strlen(tmp), f2->charset()); + break; + } + + case CalpontSystemCatalog::TIME: + { + if ((*f)->null_ptr) + *(*f)->null_ptr &= ~(*f)->null_bit; + + intColVal = row.getUintField<8>(s); + DataConvert::timeToString(intColVal, tmp, 255); Field_varstring* f2 = (Field_varstring*)*f; f2->store(tmp, strlen(tmp), f2->charset()); @@ -4957,5 +5076,803 @@ int ha_calpont_impl_rnd_pos(uchar* buf, uchar* pos) return ER_INTERNAL_ERROR; } -// vim:sw=4 ts=4: +/*@brief ha_calpont_impl_group_by_init - Get data for MariaDB group_by + pushdown handler */ +/*********************************************************** + * DESCRIPTION: + * Prepares data for group_by_handler::next_row() calls. + * PARAMETERS: + * group_hand - group by handler, that preserves initial table and items lists. . + * table - TABLE pointer The table to save the result set into. + * RETURN: + * 0 if success + * others if something went wrong whilst getting the result set + ***********************************************************/ +int ha_calpont_impl_group_by_init(ha_calpont_group_by_handler* group_hand, TABLE* table) +{ + string tableName = group_hand->table_list->table->s->table_name.str; + IDEBUG( cout << "group_by_init for table " << tableName << endl ); + THD* thd = current_thd; + //check whether the system is ready to process statement. +#ifndef _MSC_VER + static DBRM dbrm(true); + bool bSystemQueryReady = dbrm.getSystemQueryReady(); + + if (bSystemQueryReady == 0) + { + // Still not ready + setError(thd, ER_INTERNAL_ERROR, "The system is not yet ready to accept queries"); + thd->infinidb_vtable.vtable_state = THD::INFINIDB_ERROR; + return ER_INTERNAL_ERROR; + } + else if (bSystemQueryReady < 0) + { + // Still not ready + setError(thd, ER_INTERNAL_ERROR, "DBRM is not responding. Cannot accept queries"); + thd->infinidb_vtable.vtable_state = THD::INFINIDB_ERROR; + return ER_INTERNAL_ERROR; + } + +#endif + // prevent "create table as select" from running on slave + thd->infinidb_vtable.hasInfiniDBTable = true; + + // return error if error status has been already set + if (thd->infinidb_vtable.vtable_state == THD::INFINIDB_ERROR) + return ER_INTERNAL_ERROR; + + // MCOL-1052 + // by pass the extra union trips. return 0 + //if (thd->infinidb_vtable.isUnion && thd->infinidb_vtable.vtable_state == THD::INFINIDB_CREATE_VTABLE) + // return 0; + + // @bug 2232. Basic SP support. Error out non support sp cases. + // @bug 3939. Only error out for sp with select. Let pass for alter table in sp. + if (thd->infinidb_vtable.call_sp && (thd->lex)->sql_command != SQLCOM_ALTER_TABLE) + { + setError(thd, ER_CHECK_NOT_IMPLEMENTED, "This stored procedure syntax is not supported by Columnstore in this version"); + thd->infinidb_vtable.vtable_state = THD::INFINIDB_ERROR; + return ER_INTERNAL_ERROR; + } + + uint32_t sessionID = tid2sid(thd->thread_id); + boost::shared_ptr csc = CalpontSystemCatalog::makeCalpontSystemCatalog(sessionID); + csc->identity(CalpontSystemCatalog::FE); + + if (!thd->infinidb_vtable.cal_conn_info) + thd->infinidb_vtable.cal_conn_info = (void*)(new cal_connection_info()); + + cal_connection_info* ci = reinterpret_cast(thd->infinidb_vtable.cal_conn_info); + + idbassert(ci != 0); + + + // MySQL sometimes calls rnd_init multiple times, plan should only be + // generated and sent once. + if (thd->infinidb_vtable.vtable_state == THD::INFINIDB_DISABLE_VTABLE && + !thd->infinidb_vtable.isNewQuery) + return 0; + + if (thd->killed == KILL_QUERY || thd->killed == KILL_QUERY_HARD) + { + if (ci->cal_conn_hndl) + { + // send ExeMgr a signal before closing the connection + ByteStream msg; + ByteStream::quadbyte qb = 0; + msg << qb; + + try + { + ci->cal_conn_hndl->exeMgr->write(msg); + } + catch (...) + { + // canceling query. ignore connection failure. + } + + sm::sm_cleanup(ci->cal_conn_hndl); + ci->cal_conn_hndl = 0; + } + + return 0; + } + + sm::tableid_t tableid = 0; + cal_table_info ti; + cal_group_info gi; + sm::cpsm_conhdl_t* hndl; + SCSEP csep; + + bool localQuery = (thd->variables.infinidb_local_query > 0 ? true : false); + + { + ci->stats.reset(); // reset query stats + ci->stats.setStartTime(); + ci->stats.fUser = thd->main_security_ctx.user; + + if (thd->main_security_ctx.host) + ci->stats.fHost = thd->main_security_ctx.host; + else if (thd->main_security_ctx.host_or_ip) + ci->stats.fHost = thd->main_security_ctx.host_or_ip; + else + ci->stats.fHost = "unknown"; + + try + { + ci->stats.userPriority(ci->stats.fHost, ci->stats.fUser); + } + catch (std::exception& e) + { + string msg = string("Columnstore User Priority - ") + e.what(); + ci->warningMsg = msg; + } + + // if the previous query has error, re-establish the connection + if (ci->queryState != 0) + { + sm::sm_cleanup(ci->cal_conn_hndl); + ci->cal_conn_hndl = 0; + } + + sm::sm_init(sessionID, &ci->cal_conn_hndl, localQuery); + idbassert(ci->cal_conn_hndl != 0); + ci->cal_conn_hndl->csc = csc; + idbassert(ci->cal_conn_hndl->exeMgr != 0); + + try + { + ci->cal_conn_hndl->connect(); + } + catch (...) + { + setError(thd, ER_INTERNAL_ERROR, IDBErrorInfo::instance()->errorMsg(ERR_LOST_CONN_EXEMGR)); + CalpontSystemCatalog::removeCalpontSystemCatalog(sessionID); + goto error; + } + + hndl = ci->cal_conn_hndl; + + if (!csep) + csep.reset(new CalpontSelectExecutionPlan()); + + SessionManager sm; + BRM::TxnID txnID; + txnID = sm.getTxnID(sessionID); + + if (!txnID.valid) + { + txnID.id = 0; + txnID.valid = true; + } + + QueryContext verID; + verID = sm.verID(); + + csep->txnID(txnID.id); + csep->verID(verID); + csep->sessionID(sessionID); + + if (group_hand->table_list->db_length) + csep->schemaName(group_hand->table_list->db); + + csep->traceFlags(ci->traceFlags); + + // MCOL-1052 Send Items lists down to the optimizer. + gi.groupByTables = group_hand->table_list; + gi.groupByFields = group_hand->select; + gi.groupByWhere = group_hand->where; + gi.groupByGroup = group_hand->group_by; + gi.groupByOrder = group_hand->order_by; + gi.groupByHaving = group_hand->having; + gi.groupByDistinct = group_hand->distinct; + + // MCOL-1052 Send pushed conditions here, since server could omit GROUP BY + // items in case of = or IN functions used on GROUP BY columns. + { + CalTableMap::iterator mapiter; + execplan::CalpontSelectExecutionPlan::ColumnMap::iterator colMapIter; + execplan::CalpontSelectExecutionPlan::ColumnMap::iterator condColMapIter; + execplan::ParseTree* ptIt; + execplan::ReturnedColumn* rcIt; + + for (TABLE_LIST* tl = gi.groupByTables; tl; tl = tl->next_local) + { + mapiter = ci->tableMap.find(tl->table); + + if (mapiter != ci->tableMap.end() && mapiter->second.condInfo != NULL + && mapiter->second.condInfo->condPush) + { + while (!mapiter->second.condInfo->ptWorkStack.empty()) + { + ptIt = mapiter->second.condInfo->ptWorkStack.top(); + mapiter->second.condInfo->ptWorkStack.pop(); + gi.pushedPts.push_back(ptIt); + } + } + } + } + // send plan whenever group_init is called + int status = cp_get_group_plan(thd, csep, gi); + + if (status > 0) + goto internal_error; + else if (status < 0) + return 0; + + // @bug 2547. don't need to send the plan if it's impossible where for all unions. + if (thd->infinidb_vtable.impossibleWhereOnUnion) + return 0; + + string query; + query.assign(thd->infinidb_vtable.original_query.ptr(), + thd->infinidb_vtable.original_query.length()); + csep->data(query); + + try + { + csep->priority( ci->stats.userPriority(ci->stats.fHost, ci->stats.fUser)); + } + catch (std::exception& e) + { + string msg = string("Columnstore User Priority - ") + e.what(); + push_warning(thd, Sql_condition::WARN_LEVEL_WARN, 9999, msg.c_str()); + } + +#ifdef PLAN_HEX_FILE + // plan serialization + ifstream ifs("/tmp/li1-plan.hex"); + ByteStream bs1; + ifs >> bs1; + ifs.close(); + csep->unserialize(bs1); +#endif + + if (ci->traceFlags & 1) + { + cerr << "---------------- EXECUTION PLAN ----------------" << endl; + cerr << *csep << endl ; + cerr << "-------------- EXECUTION PLAN END --------------\n" << endl; + } + else + { + IDEBUG( cout << "---------------- EXECUTION PLAN ----------------" << endl ); + IDEBUG( cerr << *csep << endl ); + IDEBUG( cout << "-------------- EXECUTION PLAN END --------------\n" << endl ); + } + }// end of execution plan generation + + { + ByteStream msg; + ByteStream emsgBs; + + while (true) + { + try + { + ByteStream::quadbyte qb = 4; + msg << qb; + hndl->exeMgr->write(msg); + msg.restart(); + csep->rmParms(rmParms); + + //send plan + csep->serialize(msg); + hndl->exeMgr->write(msg); + + //get ExeMgr status back to indicate a vtable joblist success or not + msg.restart(); + emsgBs.restart(); + msg = hndl->exeMgr->read(); + emsgBs = hndl->exeMgr->read(); + string emsg; + + if (msg.length() == 0 || emsgBs.length() == 0) + { + emsg = "Lost connection to ExeMgr. Please contact your administrator"; + setError(thd, ER_INTERNAL_ERROR, emsg); + return ER_INTERNAL_ERROR; + } + + string emsgStr; + emsgBs >> emsgStr; + bool err = false; + + if (msg.length() == 4) + { + msg >> qb; + + if (qb != 0) + { + err = true; + // for makejoblist error, stats contains only error code and insert from here + // because table fetch is not started + ci->stats.setEndTime(); + ci->stats.fQuery = csep->data(); + ci->stats.fQueryType = csep->queryType(); + ci->stats.fErrorNo = qb; + + try + { + ci->stats.insert(); + } + catch (std::exception& e) + { + string msg = string("Columnstore Query Stats - ") + e.what(); + push_warning(thd, Sql_condition::WARN_LEVEL_WARN, 9999, msg.c_str()); + } + } + } + else + { + err = true; + } + + if (err) + { + setError(thd, ER_INTERNAL_ERROR, emsgStr); + return ER_INTERNAL_ERROR; + } + + rmParms.clear(); + + if (thd->infinidb_vtable.vtable_state == THD::INFINIDB_DISABLE_VTABLE) + { + ci->tableMap[table] = ti; + } + else + { + ci->queryState = 1; + } + + break; + } + catch (...) + { + sm::sm_cleanup(hndl); + hndl = 0; + + sm::sm_init(sessionID, &hndl, localQuery); + idbassert(hndl != 0); + hndl->csc = csc; + + if (thd->infinidb_vtable.vtable_state == THD::INFINIDB_DISABLE_VTABLE) + ti.conn_hndl = hndl; + else + ci->cal_conn_hndl = hndl; + + try + { + hndl->connect(); + } + catch (...) + { + setError(thd, ER_INTERNAL_ERROR, IDBErrorInfo::instance()->errorMsg(ERR_LOST_CONN_EXEMGR)); + CalpontSystemCatalog::removeCalpontSystemCatalog(sessionID); + goto error; + } + + msg.restart(); + } + } + } + + // set query state to be in_process. Sometimes mysql calls rnd_init multiple + // times, this makes sure plan only being generated and sent once. It will be + // reset when query finishes in sm::end_query + thd->infinidb_vtable.isNewQuery = false; + + // common path for both vtable select phase and table mode -- open scan handle + ti = ci->tableMap[table]; + ti.msTablePtr = table; + + // MCOL-1052 + thd->infinidb_vtable.vtable_state = THD::INFINIDB_SELECT_VTABLE; + + if ((thd->infinidb_vtable.vtable_state == THD::INFINIDB_SELECT_VTABLE) || + (thd->infinidb_vtable.vtable_state == THD::INFINIDB_DISABLE_VTABLE) || + (thd->infinidb_vtable.vtable_state == THD::INFINIDB_REDO_QUERY)) + { + if (ti.tpl_ctx == 0) + { + ti.tpl_ctx = new sm::cpsm_tplh_t(); + ti.tpl_scan_ctx = sm::sp_cpsm_tplsch_t(new sm::cpsm_tplsch_t()); + } + + // make sure rowgroup is null so the new meta data can be taken. This is for some case mysql + // call rnd_init for a table more than once. + ti.tpl_scan_ctx->rowGroup = NULL; + + try + { + tableid = execplan::IDB_VTABLE_ID; + } + catch (...) + { + string emsg = "No table ID found for table " + string(table->s->table_name.str); + setError(thd, ER_INTERNAL_ERROR, emsg); + CalpontSystemCatalog::removeCalpontSystemCatalog(sessionID); + goto internal_error; + } + + try + { + sm::tpl_open(tableid, ti.tpl_ctx, hndl); + sm::tpl_scan_open(tableid, ti.tpl_scan_ctx, hndl); + } + catch (std::exception& e) + { + string emsg = "table can not be opened: " + string(e.what()); + setError(thd, ER_INTERNAL_ERROR, emsg); + CalpontSystemCatalog::removeCalpontSystemCatalog(sessionID); + goto internal_error; + } + catch (...) + { + string emsg = "table can not be opened"; + setError(thd, ER_INTERNAL_ERROR, emsg); + CalpontSystemCatalog::removeCalpontSystemCatalog(sessionID); + goto internal_error; + } + + ti.tpl_scan_ctx->traceFlags = ci->traceFlags; + + if ((ti.tpl_scan_ctx->ctp).size() == 0) + { + uint32_t num_attr = table->s->fields; + + for (uint32_t i = 0; i < num_attr; i++) + { + CalpontSystemCatalog::ColType ctype; + ti.tpl_scan_ctx->ctp.push_back(ctype); + } + + // populate coltypes here for table mode because tableband gives treeoid for dictionary column + if (thd->infinidb_vtable.vtable_state == THD::INFINIDB_DISABLE_VTABLE) + { + CalpontSystemCatalog::RIDList oidlist = csc->columnRIDs(make_table(table->s->db.str, table->s->table_name.str), true); + + if (oidlist.size() != num_attr) + { + string emsg = "Size mismatch probably caused by front end out of sync"; + setError(thd, ER_INTERNAL_ERROR, emsg); + CalpontSystemCatalog::removeCalpontSystemCatalog(sessionID); + goto internal_error; + } + + for (unsigned int j = 0; j < oidlist.size(); j++) + { + CalpontSystemCatalog::ColType ctype = csc->colType(oidlist[j].objnum); + ti.tpl_scan_ctx->ctp[ctype.colPosition] = ctype; + ti.tpl_scan_ctx->ctp[ctype.colPosition].colPosition = -1; + } + } + } + } + + ci->tableMap[table] = ti; + return 0; + +error: + + if (ci->cal_conn_hndl) + { + sm::sm_cleanup(ci->cal_conn_hndl); + ci->cal_conn_hndl = 0; + } + + // do we need to close all connection handle of the table map? + return ER_INTERNAL_ERROR; + +internal_error: + + if (ci->cal_conn_hndl) + { + sm::sm_cleanup(ci->cal_conn_hndl); + ci->cal_conn_hndl = 0; + } + + return ER_INTERNAL_ERROR; +} + +/*@brief ha_calpont_impl_group_by_next - Return result set for MariaDB group_by + pushdown handler +*/ +/*********************************************************** + * DESCRIPTION: + * Return a result record for each group_by_handler::next_row() call. + * PARAMETERS: + * group_hand - group by handler, that preserves initial table and items lists. . + * table - TABLE pointer The table to save the result set in. + * RETURN: + * 0 if success + * HA_ERR_END_OF_FILE if the record set has come to an end + * others if something went wrong whilst getting the result set + ***********************************************************/ +int ha_calpont_impl_group_by_next(ha_calpont_group_by_handler* group_hand, TABLE* table) +{ + THD* thd = current_thd; + + /* If this node is the slave, ignore DML to IDB tables */ + if (thd->slave_thread && ( + thd->lex->sql_command == SQLCOM_INSERT || + thd->lex->sql_command == SQLCOM_INSERT_SELECT || + thd->lex->sql_command == SQLCOM_UPDATE || + thd->lex->sql_command == SQLCOM_UPDATE_MULTI || + thd->lex->sql_command == SQLCOM_DELETE || + thd->lex->sql_command == SQLCOM_DELETE_MULTI || + thd->lex->sql_command == SQLCOM_TRUNCATE || + thd->lex->sql_command == SQLCOM_LOAD)) + return 0; + + + if (thd->infinidb_vtable.vtable_state == THD::INFINIDB_ERROR) + return ER_INTERNAL_ERROR; + + if (((thd->lex)->sql_command == SQLCOM_UPDATE) || ((thd->lex)->sql_command == SQLCOM_DELETE) || + ((thd->lex)->sql_command == SQLCOM_DELETE_MULTI) || ((thd->lex)->sql_command == SQLCOM_UPDATE_MULTI)) + return HA_ERR_END_OF_FILE; + + // @bug 2547 + if (thd->infinidb_vtable.impossibleWhereOnUnion) + return HA_ERR_END_OF_FILE; + + // @bug 2232. Basic SP support + // @bug 3939. Only error out for sp with select. Let pass for alter table in sp. + /*if (thd->infinidb_vtable.call_sp && (thd->lex)->sql_command != SQLCOM_ALTER_TABLE) + { + setError(thd, ER_CHECK_NOT_IMPLEMENTED, "This stored procedure syntax is not supported by Columnstore in this version"); + thd->infinidb_vtable.vtable_state = THD::INFINIDB_ERROR; + return ER_INTERNAL_ERROR; + } + */ + if (!thd->infinidb_vtable.cal_conn_info) + thd->infinidb_vtable.cal_conn_info = (void*)(new cal_connection_info()); + + cal_connection_info* ci = reinterpret_cast(thd->infinidb_vtable.cal_conn_info); + + // @bug 3078 + if (thd->killed == KILL_QUERY || thd->killed == KILL_QUERY_HARD) + { + if (ci->cal_conn_hndl) + { + // send ExeMgr a signal before cloing the connection + ByteStream msg; + ByteStream::quadbyte qb = 0; + msg << qb; + + try + { + ci->cal_conn_hndl->exeMgr->write(msg); + } + catch (...) + { + // cancel query. ignore connection failure. + } + + sm::sm_cleanup(ci->cal_conn_hndl); + ci->cal_conn_hndl = 0; + } + + return 0; + } + + if (ci->alterTableState > 0) return HA_ERR_END_OF_FILE; + + cal_table_info ti; + ti = ci->tableMap[table]; + int rc = HA_ERR_END_OF_FILE; + + if (!ti.tpl_ctx || !ti.tpl_scan_ctx) + { + CalpontSystemCatalog::removeCalpontSystemCatalog(tid2sid(thd->thread_id)); + return ER_INTERNAL_ERROR; + } + + idbassert(ti.msTablePtr == table); + + try + { + // fetchNextRow interface forces to use buf. + unsigned char buf; + rc = fetchNextRow(&buf, ti, ci, true); + } + catch (std::exception& e) + { + string emsg = string("Error while fetching from ExeMgr: ") + e.what(); + setError(thd, ER_INTERNAL_ERROR, emsg); + CalpontSystemCatalog::removeCalpontSystemCatalog(tid2sid(thd->thread_id)); + return ER_INTERNAL_ERROR; + } + + ci->tableMap[table] = ti; + + if (rc != 0 && rc != HA_ERR_END_OF_FILE) + { + string emsg; + + // remove this check when all error handling migrated to the new framework. + if (rc >= 1000) + emsg = ti.tpl_scan_ctx->errMsg; + else + { + logging::ErrorCodes errorcodes; + emsg = errorcodes.errorString(rc); + } + + setError(thd, ER_INTERNAL_ERROR, emsg); + ci->stats.fErrorNo = rc; + CalpontSystemCatalog::removeCalpontSystemCatalog(tid2sid(thd->thread_id)); + rc = ER_INTERNAL_ERROR; + } + + return rc; +} + +int ha_calpont_impl_group_by_end(ha_calpont_group_by_handler* group_hand, TABLE* table) +{ + int rc = 0; + THD* thd = current_thd; + cal_connection_info* ci = NULL; + + + if (thd->slave_thread && ( + thd->lex->sql_command == SQLCOM_INSERT || + thd->lex->sql_command == SQLCOM_INSERT_SELECT || + thd->lex->sql_command == SQLCOM_UPDATE || + thd->lex->sql_command == SQLCOM_UPDATE_MULTI || + thd->lex->sql_command == SQLCOM_DELETE || + thd->lex->sql_command == SQLCOM_DELETE_MULTI || + thd->lex->sql_command == SQLCOM_TRUNCATE || + thd->lex->sql_command == SQLCOM_LOAD)) + return 0; + + thd->infinidb_vtable.isNewQuery = true; + thd->infinidb_vtable.isUnion = false; + + if (thd->infinidb_vtable.cal_conn_info) + ci = reinterpret_cast(thd->infinidb_vtable.cal_conn_info); + + // MCOL-1052 + //if (thd->infinidb_vtable.vtable_state == THD::INFINIDB_ORDER_BY ) + //{ + // thd->infinidb_vtable.vtable_state = THD::INFINIDB_SELECT_VTABLE;// flip back to normal state + // return rc; + //} + + if (((thd->lex)->sql_command == SQLCOM_INSERT) || + ((thd->lex)->sql_command == SQLCOM_INSERT_SELECT) ) + { + // @bug 4022. error handling for select part of dml + if (ci->cal_conn_hndl && ci->rc) + { + // send ExeMgr a signal before closing the connection + ByteStream msg; + ByteStream::quadbyte qb = 0; + msg << qb; + + try + { + ci->cal_conn_hndl->exeMgr->write(msg); + } + catch (...) + { + // this is error handling, so ignore connection failure. + } + + sm::sm_cleanup(ci->cal_conn_hndl); + ci->cal_conn_hndl = 0; + return rc; + } + } + + if (!ci) + { + thd->infinidb_vtable.cal_conn_info = (void*)(new cal_connection_info()); + ci = reinterpret_cast(thd->infinidb_vtable.cal_conn_info); + } + + // @bug 3078. Also session limit variable works the same as ctrl+c + if (thd->killed == KILL_QUERY || thd->killed == KILL_QUERY_HARD || + ((thd->lex)->sql_command != SQLCOM_INSERT && + (thd->lex)->sql_command != SQLCOM_INSERT_SELECT && + thd->variables.select_limit != (uint64_t) - 1)) + { + if (ci->cal_conn_hndl) + { + // send ExeMgr a signal before closing the connection + ByteStream msg; + ByteStream::quadbyte qb = 0; + msg << qb; + + try + { + ci->cal_conn_hndl->exeMgr->write(msg); + } + catch (...) + { + // this is the end of query. Ignore the exception if exemgr connection failed + // for whatever reason. + } + + sm::sm_cleanup(ci->cal_conn_hndl); + ci->cal_conn_hndl = 0; + // clear querystats because no query stats available for cancelled query + ci->queryStats = ""; + } + + return 0; + } + + IDEBUG( cerr << "group_by_end for table " << table->s->table_name.str << endl ); + + cal_table_info ti = ci->tableMap[table]; + sm::cpsm_conhdl_t* hndl; + + hndl = ci->cal_conn_hndl; + + if (ti.tpl_ctx) + { + if (ti.tpl_scan_ctx.get()) + { + try + { + sm::tpl_scan_close(ti.tpl_scan_ctx); + } + catch (...) + { + rc = ER_INTERNAL_ERROR; + } + } + + ti.tpl_scan_ctx.reset(); + + try + { + sm::tpl_close(ti.tpl_ctx, &hndl, ci->stats); + + ci->cal_conn_hndl = hndl; + + ti.tpl_ctx = 0; + } + catch (IDBExcept& e) + { + if (e.errorCode() == ERR_CROSS_ENGINE_CONNECT || e.errorCode() == ERR_CROSS_ENGINE_CONFIG) + { + string msg = string("Columnstore Query Stats - ") + e.what(); + push_warning(thd, Sql_condition::WARN_LEVEL_WARN, 9999, msg.c_str()); + } + else + { + setError(thd, ER_INTERNAL_ERROR, e.what()); + rc = ER_INTERNAL_ERROR; + } + } + catch (std::exception& e) + { + setError(thd, ER_INTERNAL_ERROR, e.what()); + rc = ER_INTERNAL_ERROR; + } + catch (...) + { + setError(thd, ER_INTERNAL_ERROR, "Internal error throwed in group_by_end"); + rc = ER_INTERNAL_ERROR; + } + } + + ti.tpl_ctx = 0; + + ci->tableMap[table] = ti; + + // push warnings from CREATE phase + if (!ci->warningMsg.empty()) + push_warning(thd, Sql_condition::WARN_LEVEL_WARN, 9999, ci->warningMsg.c_str()); + + ci->warningMsg.clear(); + // reset expressionId just in case + ci->expressionId = 0; + return rc; +} + +// vim:sw=4 ts=4: diff --git a/dbcon/mysql/ha_calpont_impl.h b/dbcon/mysql/ha_calpont_impl.h index 1e7f7d5df..7f7c43aae 100644 --- a/dbcon/mysql/ha_calpont_impl.h +++ b/dbcon/mysql/ha_calpont_impl.h @@ -48,11 +48,16 @@ extern int ha_calpont_impl_external_lock(THD* thd, TABLE* table, int lock_type); extern int ha_calpont_impl_update_row(); extern int ha_calpont_impl_delete_row(); extern int ha_calpont_impl_rnd_pos(uchar* buf, uchar* pos); +extern int ha_calpont_impl_group_by_init(ha_calpont_group_by_handler* group_hand, TABLE* table); +extern int ha_calpont_impl_group_by_next(ha_calpont_group_by_handler* group_hand, TABLE* table); +extern int ha_calpont_impl_group_by_end(ha_calpont_group_by_handler* group_hand, TABLE* table); + #endif #ifdef NEED_CALPONT_INTERFACE #include "ha_calpont_impl_if.h" #include "calpontsystemcatalog.h" +#include "ha_calpont.h" extern int ha_calpont_impl_rename_table_(const char* from, const char* to, cal_impl_if::cal_connection_info& ci); extern int ha_calpont_impl_write_row_(uchar* buf, TABLE* table, cal_impl_if::cal_connection_info& ci, ha_rows& rowsInserted); extern int ha_calpont_impl_write_batch_row_(uchar* buf, TABLE* table, cal_impl_if::cal_connection_info& ci); @@ -69,6 +74,9 @@ extern std::string ha_calpont_impl_droppartition_ (execplan::CalpontSystemCatal extern std::string ha_calpont_impl_viewtablelock( cal_impl_if::cal_connection_info& ci, execplan::CalpontSystemCatalog::TableName& tablename); extern std::string ha_calpont_impl_cleartablelock( cal_impl_if::cal_connection_info& ci, uint64_t tableLockID); +extern int ha_calpont_impl_group_by_init(ha_calpont_group_by_handler* group_hand, TABLE* table); +extern int ha_calpont_impl_group_by_next(ha_calpont_group_by_handler* group_hand, TABLE* table); +extern int ha_calpont_impl_group_by_end(ha_calpont_group_by_handler* group_hand, TABLE* table); #endif #endif diff --git a/dbcon/mysql/ha_calpont_impl_if.h b/dbcon/mysql/ha_calpont_impl_if.h index 709e1b051..cb603ca49 100644 --- a/dbcon/mysql/ha_calpont_impl_if.h +++ b/dbcon/mysql/ha_calpont_impl_if.h @@ -99,6 +99,7 @@ struct gp_walk_info execplan::CalpontSelectExecutionPlan::ReturnedColumnList groupByCols; execplan::CalpontSelectExecutionPlan::ReturnedColumnList subGroupByCols; execplan::CalpontSelectExecutionPlan::ReturnedColumnList orderByCols; + std::vector havingAggColsItems; execplan::CalpontSelectExecutionPlan::ColumnMap columnMap; // This vector temporarily hold the projection columns to be added // to the returnedCols vector for subquery processing. It will be appended @@ -191,6 +192,28 @@ struct cal_table_info bool moreRows; //are there more rows to consume (b/c of limit) }; +struct cal_group_info +{ + cal_group_info() : groupByFields(0), + groupByTables(0), + groupByWhere(0), + groupByGroup(0), + groupByOrder(0), + groupByHaving(0), + groupByDistinct(false) + { } + ~cal_group_info() { } + + List* groupByFields; // MCOL-1052 SELECT + TABLE_LIST* groupByTables; // MCOL-1052 FROM + Item* groupByWhere; // MCOL-1052 WHERE + ORDER* groupByGroup; // MCOL-1052 GROUP BY + ORDER* groupByOrder; // MCOL-1052 ORDER BY + Item* groupByHaving; // MCOL-1052 HAVING + bool groupByDistinct; //MCOL-1052 DISTINCT + std::vector pushedPts; +}; + typedef std::tr1::unordered_map CalTableMap; typedef std::vector ColValuesList; typedef std::vector ColNameList; @@ -297,7 +320,9 @@ const std::string infinidb_err_msg = "\nThe query includes syntax that is not su int cp_get_plan(THD* thd, execplan::SCSEP& csep); int cp_get_table_plan(THD* thd, execplan::SCSEP& csep, cal_impl_if::cal_table_info& ti); +int cp_get_group_plan(THD* thd, execplan::SCSEP& csep, cal_impl_if::cal_group_info& gi); int getSelectPlan(gp_walk_info& gwi, SELECT_LEX& select_lex, execplan::SCSEP& csep, bool isUnion = false); +int getGroupPlan(gp_walk_info& gwi, SELECT_LEX& select_lex, execplan::SCSEP& csep, cal_group_info& gi, bool isUnion = false); void setError(THD* thd, uint32_t errcode, const std::string errmsg, gp_walk_info* gwi); void setError(THD* thd, uint32_t errcode, const std::string errmsg); void gp_walk(const Item* item, void* arg); diff --git a/dbcon/mysql/ha_calpont_partition.cpp b/dbcon/mysql/ha_calpont_partition.cpp index 84ea65d82..99940262b 100644 --- a/dbcon/mysql/ha_calpont_partition.cpp +++ b/dbcon/mysql/ha_calpont_partition.cpp @@ -125,6 +125,9 @@ string name(CalpontSystemCatalog::ColType& ct) case CalpontSystemCatalog::DATETIME: return "DATETIME"; + case CalpontSystemCatalog::TIME: + return "TIME"; + case CalpontSystemCatalog::DECIMAL: return "DECIMAL"; @@ -201,6 +204,7 @@ bool CP_type(CalpontSystemCatalog::ColType& ct) ct.colDataType == CalpontSystemCatalog::BIGINT || ct.colDataType == CalpontSystemCatalog::DATE || ct.colDataType == CalpontSystemCatalog::DATETIME || + ct.colDataType == CalpontSystemCatalog::TIME || ct.colDataType == CalpontSystemCatalog::DECIMAL || ct.colDataType == CalpontSystemCatalog::UTINYINT || ct.colDataType == CalpontSystemCatalog::USMALLINT || @@ -261,6 +265,9 @@ const string format(int64_t v, CalpontSystemCatalog::ColType& ct) oss << DataConvert::datetimeToString(v); break; + case CalpontSystemCatalog::TIME: + oss << DataConvert::timeToString(v); + case CalpontSystemCatalog::CHAR: case CalpontSystemCatalog::VARCHAR: { @@ -387,6 +394,10 @@ const int64_t IDB_format(char* str, CalpontSystemCatalog::ColType& ct, uint8_t& v = boost::any_cast(anyVal); break; + case CalpontSystemCatalog::TIME: + v = boost::any_cast(anyVal); + break; + case CalpontSystemCatalog::DECIMAL: case CalpontSystemCatalog::UDECIMAL: if (ct.colWidth == execplan::CalpontSystemCatalog::ONE_BYTE) diff --git a/dbcon/mysql/ha_window_function.cpp b/dbcon/mysql/ha_window_function.cpp index 7e30389c4..4b648cb15 100644 --- a/dbcon/mysql/ha_window_function.cpp +++ b/dbcon/mysql/ha_window_function.cpp @@ -513,6 +513,19 @@ ReturnedColumn* buildWindowFunctionColumn(Item* item, gp_walk_info& gwi, bool& n Item* orderItem = *(orderCol->item); srcp.reset(buildReturnedColumn(orderItem, gwi, nonSupport)); + // MCOL-1052 GROUP BY handler has all of query's agg Items + // as field and correlates them with its extended SELECT Items. + if (!srcp) + { + orderItem = orderCol->item_ptr; + + if (orderItem) + { + gwi.fatalParseError = false; + srcp.reset(buildReturnedColumn(orderItem, gwi, nonSupport)); + } + } + if (!srcp) return nullOnError(gwi); @@ -590,6 +603,7 @@ ReturnedColumn* buildWindowFunctionColumn(Item* item, gp_walk_info& gwi, bool& n case CalpontSystemCatalog::DATE: case CalpontSystemCatalog::DATETIME: + case CalpontSystemCatalog::TIME: if (!frm.fIsRange) boundTypeErr = true; else if (dynamic_cast(frm.fStart.fVal.get()) == NULL) @@ -641,6 +655,7 @@ ReturnedColumn* buildWindowFunctionColumn(Item* item, gp_walk_info& gwi, bool& n case CalpontSystemCatalog::DATE: case CalpontSystemCatalog::DATETIME: + case CalpontSystemCatalog::TIME: if (!frm.fIsRange) boundTypeErr = true; else if (dynamic_cast(frm.fEnd.fVal.get()) == NULL) diff --git a/dbcon/mysql/is_columnstore_columns.cpp b/dbcon/mysql/is_columnstore_columns.cpp index 31cdb5578..53f67fbbf 100644 --- a/dbcon/mysql/is_columnstore_columns.cpp +++ b/dbcon/mysql/is_columnstore_columns.cpp @@ -27,6 +27,8 @@ #include #include "calpontsystemcatalog.h" #include "dataconvert.h" +#include "exceptclasses.h" +using namespace logging; // Required declaration as it isn't in a MairaDB include @@ -70,7 +72,25 @@ static int is_columnstore_columns_fill(THD* thd, TABLE_LIST* tables, COND* cond) for (std::vector >::const_iterator it = catalog_tables.begin(); it != catalog_tables.end(); ++it) { - execplan::CalpontSystemCatalog::RIDList column_rid_list = systemCatalogPtr->columnRIDs((*it).second, true); + execplan::CalpontSystemCatalog::RIDList column_rid_list; + + // Note a table may get dropped as you iterate over the list of tables. + // So simply ignore the dropped table. + try + { + column_rid_list = systemCatalogPtr->columnRIDs((*it).second, true); + } + catch (IDBExcept& ex) + { + if (ex.errorCode() == ERR_TABLE_NOT_IN_CATALOG) + { + continue; + } + else + { + return 1; + } + } for (size_t col_num = 0; col_num < column_rid_list.size(); col_num++) { diff --git a/decomsvr/CMakeLists.txt b/decomsvr/CMakeLists.txt deleted file mode 100644 index 5c4e5e6a7..000000000 --- a/decomsvr/CMakeLists.txt +++ /dev/null @@ -1,14 +0,0 @@ - -include_directories( ${ENGINE_COMMON_INCLUDES} ) - - -########### next target ############### - -set(DecomSvr_SRCS quicklz.c server.cpp) - -add_executable(DecomSvr ${DecomSvr_SRCS}) - -target_link_libraries(DecomSvr ${ENGINE_LDFLAGS} ${Boost_LIBRARIES} pthread rt) - -install(TARGETS DecomSvr DESTINATION ${ENGINE_BINDIR} COMPONENT platform) - diff --git a/decomsvr/DecomSvr.rc b/decomsvr/DecomSvr.rc deleted file mode 100644 index 38e974faa..000000000 --- a/decomsvr/DecomSvr.rc +++ /dev/null @@ -1,102 +0,0 @@ -// Microsoft Visual C++ generated resource script. -// -#include "resource.h" - -#define APSTUDIO_READONLY_SYMBOLS -///////////////////////////////////////////////////////////////////////////// -// -// Generated from the TEXTINCLUDE 2 resource. -// -#include "afxres.h" - -///////////////////////////////////////////////////////////////////////////// -#undef APSTUDIO_READONLY_SYMBOLS - -///////////////////////////////////////////////////////////////////////////// -// English (U.S.) resources - -#if !defined(AFX_RESOURCE_DLL) || defined(AFX_TARG_ENU) -#ifdef _WIN32 -LANGUAGE LANG_ENGLISH, SUBLANG_ENGLISH_US -#pragma code_page(1252) -#endif //_WIN32 - -#ifdef APSTUDIO_INVOKED -///////////////////////////////////////////////////////////////////////////// -// -// TEXTINCLUDE -// - -1 TEXTINCLUDE -BEGIN - "resource.h\0" -END - -2 TEXTINCLUDE -BEGIN - "#include ""afxres.h""\r\n" - "\0" -END - -3 TEXTINCLUDE -BEGIN - "\r\n" - "\0" -END - -#endif // APSTUDIO_INVOKED - - -///////////////////////////////////////////////////////////////////////////// -// -// Version -// - -VS_VERSION_INFO VERSIONINFO - FILEVERSION 4,6,0,0 - PRODUCTVERSION 4,6,0,0 - FILEFLAGSMASK 0x17L -#ifdef _DEBUG - FILEFLAGS 0x1L -#else - FILEFLAGS 0x0L -#endif - FILEOS 0x4L - FILETYPE 0x1L - FILESUBTYPE 0x0L -BEGIN - BLOCK "StringFileInfo" - BEGIN - BLOCK "040904b0" - BEGIN - VALUE "CompanyName", "InfiniDB, Inc." - VALUE "FileDescription", "InfiniDB V1 Decompression Server" - VALUE "FileVersion", "4.6.0-0" - VALUE "InternalName", "DecomSvr" - VALUE "LegalCopyright", "Copyright (C) 2014" - VALUE "OriginalFilename", "DecomSvr.exe" - VALUE "ProductName", "InfiniDB" - VALUE "ProductVersion", "4.6.0.0 Beta" - END - END - BLOCK "VarFileInfo" - BEGIN - VALUE "Translation", 0x409, 1200 - END -END - -#endif // English (U.S.) resources -///////////////////////////////////////////////////////////////////////////// - - - -#ifndef APSTUDIO_INVOKED -///////////////////////////////////////////////////////////////////////////// -// -// Generated from the TEXTINCLUDE 3 resource. -// - - -///////////////////////////////////////////////////////////////////////////// -#endif // not APSTUDIO_INVOKED - diff --git a/decomsvr/DecomSvr.vcxproj b/decomsvr/DecomSvr.vcxproj deleted file mode 100644 index 8e3bb55a4..000000000 --- a/decomsvr/DecomSvr.vcxproj +++ /dev/null @@ -1,290 +0,0 @@ - - - - - Debug - Win32 - - - Debug - x64 - - - EnterpriseRelease - Win32 - - - EnterpriseRelease - x64 - - - Release - Win32 - - - Release - x64 - - - - {E7F6F8A8-9DDE-4FC4-8F80-25AB98922E6F} - DecomSvr - Win32Proj - - - - Application - v110 - MultiByte - true - - - Application - v110 - MultiByte - true - - - Application - v110 - MultiByte - - - Application - v110 - MultiByte - true - - - Application - v110 - MultiByte - true - - - Application - v110 - MultiByte - - - - - - - - - - - - - - - - - - - - - - - - - <_ProjectFileVersion>11.0.50727.1 - - - $(SolutionDir)..\..\$(PlatformName)\$(ConfigurationName)\ - $(SolutionDir)..\..\obj\$(ProjectName)\$(PlatformName)\$(ConfigurationName)\ - true - - - $(SolutionDir)..\..\$(Platform)\$(Configuration)\ - $(SolutionDir)..\..\obj\$(ProjectName)\$(Platform)\$(Configuration)\ - true - - - $(SolutionDir)..\..\$(PlatformName)\$(ConfigurationName)\ - $(SolutionDir)..\..\obj\$(ProjectName)\$(PlatformName)\$(ConfigurationName)\ - false - - - $(SolutionDir)..\..\$(Platform)\$(Configuration)\ - $(SolutionDir)..\..\obj\$(ProjectName)\$(Platform)\$(Configuration)\ - false - - - $(SolutionDir)..\..\$(PlatformName)\$(ConfigurationName)\ - $(SolutionDir)..\..\obj\$(ProjectName)\$(PlatformName)\$(ConfigurationName)\ - false - - - $(SolutionDir)..\..\$(Platform)\$(Configuration)\ - $(SolutionDir)..\..\obj\$(ProjectName)\$(Platform)\$(Configuration)\ - false - - - - Disabled - $(SolutionDir)..\..\boost_1_54_0;$(SolutionDir)..\utils\winport;%(AdditionalIncludeDirectories) - WIN32;_DEBUG;_CONSOLE;%(PreprocessorDefinitions) - true - EnableFastChecks - MultiThreadedDebugDLL - - Level3 - EditAndContinue - - - ws2_32.lib;%(AdditionalDependencies) - $(SolutionDir)..\..\boost_1_54_0;%(AdditionalLibraryDirectories) - true - Console - MachineX86 - - - - - X64 - - - Disabled - $(SolutionDir)..\..\boost_1_54_0;$(SolutionDir)..\utils\winport;%(AdditionalIncludeDirectories) - WIN32;_DEBUG;_CONSOLE;%(PreprocessorDefinitions) - true - EnableFastChecks - MultiThreadedDebugDLL - - Level3 - ProgramDatabase - - - ws2_32.lib;%(AdditionalDependencies) - $(SolutionDir)..\..\boost_1_54_0;$(SolutionDir)..\..\x64\Debug;%(AdditionalLibraryDirectories) - true - Console - MachineX64 - - - - - MaxSpeed - true - $(SolutionDir)..\..\boost_1_54_0;$(SolutionDir)..\utils\winport;%(AdditionalIncludeDirectories) - WIN32;NDEBUG;_CONSOLE;%(PreprocessorDefinitions) - MultiThreadedDLL - true - - Level3 - ProgramDatabase - 4996;4244;4267;%(DisableSpecificWarnings) - - - ws2_32.lib;%(AdditionalDependencies) - $(SolutionDir)..\..\boost_1_54_0;%(AdditionalLibraryDirectories) - true - Console - true - true - MachineX86 - - - - - X64 - - - MaxSpeed - true - $(SolutionDir)..\..\boost_1_54_0;$(SolutionDir)..\utils\winport;%(AdditionalIncludeDirectories) - WIN32;NDEBUG;_CONSOLE;%(PreprocessorDefinitions) - MultiThreadedDLL - true - - Level3 - ProgramDatabase - 4996;4244;4267;%(DisableSpecificWarnings) - - - ws2_32.lib;%(AdditionalDependencies) - $(SolutionDir)..\..\boost_1_54_0;C:$(SolutionDir)..\..x64\Release;%(AdditionalLibraryDirectories) - true - Console - true - true - MachineX64 - - - $(SolutionDir)..\..\signit "InfiniDB V1 Decompression Server" $(SolutionDir)..\..x64\Release\DecomSvr.exe - - - - - MaxSpeed - true - $(SolutionDir)..\..\boost_1_54_0;$(SolutionDir)..\utils\winport;%(AdditionalIncludeDirectories) - WIN32;NDEBUG;_CONSOLE;%(PreprocessorDefinitions) - MultiThreadedDLL - true - - Level3 - ProgramDatabase - 4996;4244;4267;%(DisableSpecificWarnings) - - - ws2_32.lib;%(AdditionalDependencies) - $(SolutionDir)..\..\boost_1_54_0;%(AdditionalLibraryDirectories) - true - Console - true - true - MachineX86 - - - - - X64 - - - MaxSpeed - true - $(SolutionDir)..\..\boost_1_54_0;$(SolutionDir)..\utils\winport;%(AdditionalIncludeDirectories) - WIN32;NDEBUG;_CONSOLE;%(PreprocessorDefinitions) - MultiThreadedDLL - true - - Level3 - ProgramDatabase - 4996;4244;4267;%(DisableSpecificWarnings) - false - - - ws2_32.lib;%(AdditionalDependencies) - $(SolutionDir)..\..\boost_1_54_0;$(SolutionDir)..\..\x64\EnterpriseRelease;%(AdditionalLibraryDirectories) - false - Console - true - true - MachineX64 - - - $(SolutionDir)..\..\signit "InfiniDB V1 Decompression Server" $(SolutionDir)..\..\x64\EnterpriseRelease\DecomSvr.exe - - - - - - - - - - - - - - - {4f0851d3-b782-4f12-b748-73efa2da586b} - - - - - - \ No newline at end of file diff --git a/decomsvr/DecomSvr.vcxproj.filters b/decomsvr/DecomSvr.vcxproj.filters deleted file mode 100644 index 3166cce9d..000000000 --- a/decomsvr/DecomSvr.vcxproj.filters +++ /dev/null @@ -1,35 +0,0 @@ - - - - - {4FC737F1-C7A5-4376-A066-2A32D752A2FF} - cpp;c;cc;cxx;def;odl;idl;hpj;bat;asm;asmx - - - {93995380-89BD-4b04-88EB-625FBE52EBFB} - h;hpp;hxx;hm;inl;inc;xsd - - - {67DA6AB6-F800-4c08-8B7A-83BB121AAD01} - rc;ico;cur;bmp;dlg;rc2;rct;bin;rgs;gif;jpg;jpeg;jpe;resx;tiff;tif;png;wav - - - - - Source Files - - - Source Files - - - - - Header Files - - - - - Resource Files - - - \ No newline at end of file diff --git a/decomsvr/cli.cpp b/decomsvr/cli.cpp deleted file mode 100644 index e7b257a1c..000000000 --- a/decomsvr/cli.cpp +++ /dev/null @@ -1,112 +0,0 @@ -#include -#include -#include -#include -#include -#include -#include -#include -//#define NDEBUG -#include -#include -using namespace std; - -/* - Protocol definition: - On the control fifo: - This server waits for the other to send it: - 1. The name of the data fifo to open and read/write (string) - 2. The number of bytes of compressed data to read (number) - - On the data fifo: - The server then reads the compressed data from the data fifo: - 1. The compressed data - then it decompresses it, and sends back to the client: - 1. The number of bytes in the uncompressed stream (number) - 2. The uncompressed data - - strings are sent like this: - uint32_t string len - len bytes of the string - numbers are sent like this: - uint64_t the number - - This server expects numeric values to be in its native byte order, so - the sender needs to do it that way. -*/ - -namespace -{ -const string MessageFifo("/tmp/idbdsfifo"); -} - -int main(int argc, char** argv) -{ -again: - int fd = open(MessageFifo.c_str(), O_WRONLY | O_NONBLOCK); - - if (fd < 0) - { - if (errno == ENXIO) - { - cerr << "waiting for DS to startup..." << endl; - sleep(1); - goto again; - } - - throw runtime_error("while opening fifo for write"); - } - - uint32_t u32; - uint64_t u64; - string s; - ssize_t wrc; - - s = "/tmp/cdatafifo"; - mknod(s.c_str(), S_IFIFO | 0666, 0); - u32 = s.length(); - wrc = write(fd, &u32, 4); - assert(wrc == 4); - wrc = write(fd, s.c_str(), u32); - assert(wrc == u32); - - u64 = 707070; - write(fd, &u64, 8); - - close(fd); - - fd = open(s.c_str(), O_WRONLY); - assert(fd >= 0); - - char* b = new char[u64]; - assert(b); - - wrc = write (fd, b, u64); - assert(wrc == u64); - - delete [] b; - - close(fd); - fd = open(s.c_str(), O_RDONLY); - assert(fd >= 0); - - wrc = read(fd, &u64, 8); - assert(wrc == 8); - - b = new char[u64]; - assert(b); - - cout << "going to read " << u64 << " bytes of uncompressed data" << endl << flush; - wrc = read(fd, b, u64); - assert(wrc == u64); - cout << "read " << u64 << " bytes of uncompressed data" << endl; - - delete [] b; - - close(fd); - - unlink(s.c_str()); - - return 0; -} - diff --git a/decomsvr/quicklz.c b/decomsvr/quicklz.c deleted file mode 100644 index 374212902..000000000 --- a/decomsvr/quicklz.c +++ /dev/null @@ -1,848 +0,0 @@ -// Fast data compression library -// Copyright (C) 2006-2011 Lasse Mikkel Reinhold -// lar@quicklz.com -// -// QuickLZ can be used for free under the GPL 1, 2 or 3 license (where anything -// released into public must be open source) or under a commercial license if such -// has been acquired (see http://www.quicklz.com/order.html). The commercial license -// does not cover derived or ported versions created by third parties under GPL. - -// 1.5.0 final - -#include "quicklz.h" - -#if QLZ_VERSION_MAJOR != 1 || QLZ_VERSION_MINOR != 5 || QLZ_VERSION_REVISION != 0 - #error quicklz.c and quicklz.h have different versions -#endif - -#if (defined(__X86__) || defined(__i386__) || defined(i386) || defined(_M_IX86) || defined(__386__) || defined(__x86_64__) || defined(_M_X64)) - #define X86X64 -#endif - -#define MINOFFSET 2 -#define UNCONDITIONAL_MATCHLEN 6 -#define UNCOMPRESSED_END 4 -#define CWORD_LEN 4 - -#if QLZ_COMPRESSION_LEVEL == 1 && defined QLZ_PTR_64 && QLZ_STREAMING_BUFFER == 0 - #define OFFSET_BASE source - #define CAST (ui32)(size_t) -#else - #define OFFSET_BASE 0 - #define CAST -#endif - -int qlz_get_setting(int setting) -{ - switch (setting) - { - case 0: return QLZ_COMPRESSION_LEVEL; - case 1: return sizeof(qlz_state_compress); - case 2: return sizeof(qlz_state_decompress); - case 3: return QLZ_STREAMING_BUFFER; -#ifdef QLZ_MEMORY_SAFE - case 6: return 1; -#else - case 6: return 0; -#endif - case 7: return QLZ_VERSION_MAJOR; - case 8: return QLZ_VERSION_MINOR; - case 9: return QLZ_VERSION_REVISION; - } - return -1; -} - -#if QLZ_COMPRESSION_LEVEL == 1 -static int same(const unsigned char *src, size_t n) -{ - while(n > 0 && *(src + n) == *src) - n--; - return n == 0 ? 1 : 0; -} -#endif - -static void reset_table_compress(qlz_state_compress *state) -{ - int i; - for(i = 0; i < QLZ_HASH_VALUES; i++) - { -#if QLZ_COMPRESSION_LEVEL == 1 - state->hash[i].offset = 0; -#else - state->hash_counter[i] = 0; -#endif - } -} - -static void reset_table_decompress(qlz_state_decompress *state) -{ - int i; - (void)state; - (void)i; -#if QLZ_COMPRESSION_LEVEL == 2 - for(i = 0; i < QLZ_HASH_VALUES; i++) - { - state->hash_counter[i] = 0; - } -#endif -} - -static __inline ui32 hash_func(ui32 i) -{ -#if QLZ_COMPRESSION_LEVEL == 2 - return ((i >> 9) ^ (i >> 13) ^ i) & (QLZ_HASH_VALUES - 1); -#else - return ((i >> 12) ^ i) & (QLZ_HASH_VALUES - 1); -#endif -} - -static __inline ui32 fast_read(void const *src, ui32 bytes) -{ -#ifndef X86X64 - unsigned char *p = (unsigned char*)src; - switch (bytes) - { - case 4: - return(*p | *(p + 1) << 8 | *(p + 2) << 16 | *(p + 3) << 24); - case 3: - return(*p | *(p + 1) << 8 | *(p + 2) << 16); - case 2: - return(*p | *(p + 1) << 8); - case 1: - return(*p); - } - return 0; -#else - if (bytes >= 1 && bytes <= 4) - return *((ui32*)src); - else - return 0; -#endif -} - -static __inline ui32 hashat(const unsigned char *src) -{ - ui32 fetch, hash; - fetch = fast_read(src, 3); - hash = hash_func(fetch); - return hash; -} - -static __inline void fast_write(ui32 f, void *dst, size_t bytes) -{ -#ifndef X86X64 - unsigned char *p = (unsigned char*)dst; - - switch (bytes) - { - case 4: - *p = (unsigned char)f; - *(p + 1) = (unsigned char)(f >> 8); - *(p + 2) = (unsigned char)(f >> 16); - *(p + 3) = (unsigned char)(f >> 24); - return; - case 3: - *p = (unsigned char)f; - *(p + 1) = (unsigned char)(f >> 8); - *(p + 2) = (unsigned char)(f >> 16); - return; - case 2: - *p = (unsigned char)f; - *(p + 1) = (unsigned char)(f >> 8); - return; - case 1: - *p = (unsigned char)f; - return; - } -#else - switch (bytes) - { - case 4: - *((ui32*)dst) = f; - return; - case 3: - *((ui32*)dst) = f; - return; - case 2: - *((ui16 *)dst) = (ui16)f; - return; - case 1: - *((unsigned char*)dst) = (unsigned char)f; - return; - } -#endif -} - - -size_t qlz_size_decompressed(const char *source) -{ - ui32 n, r; - n = (((*source) & 2) == 2) ? 4 : 1; - r = fast_read(source + 1 + n, n); - r = r & (0xffffffff >> ((4 - n)*8)); - return r; -} - -size_t qlz_size_compressed(const char *source) -{ - ui32 n, r; - n = (((*source) & 2) == 2) ? 4 : 1; - r = fast_read(source + 1, n); - r = r & (0xffffffff >> ((4 - n)*8)); - return r; -} - -size_t qlz_size_header(const char *source) -{ - size_t n = 2*((((*source) & 2) == 2) ? 4 : 1) + 1; - return n; -} - - -static __inline void memcpy_up(unsigned char *dst, const unsigned char *src, ui32 n) -{ - // Caution if modifying memcpy_up! Overlap of dst and src must be special handled. -#ifndef X86X64 - unsigned char *end = dst + n; - while(dst < end) - { - *dst = *src; - dst++; - src++; - } -#else - ui32 f = 0; - do - { - *(ui32 *)(dst + f) = *(ui32 *)(src + f); - f += MINOFFSET + 1; - } - while (f < n); -#endif -} - -static __inline void update_hash(qlz_state_decompress *state, const unsigned char *s) -{ -#if QLZ_COMPRESSION_LEVEL == 1 - ui32 hash; - hash = hashat(s); - state->hash[hash].offset = s; - state->hash_counter[hash] = 1; -#elif QLZ_COMPRESSION_LEVEL == 2 - ui32 hash; - unsigned char c; - hash = hashat(s); - c = state->hash_counter[hash]; - state->hash[hash].offset[c & (QLZ_POINTERS - 1)] = s; - c++; - state->hash_counter[hash] = c; -#endif - (void)state; - (void)s; -} - -#if QLZ_COMPRESSION_LEVEL <= 2 -static void update_hash_upto(qlz_state_decompress *state, unsigned char **lh, const unsigned char *max) -{ - while(*lh < max) - { - (*lh)++; - update_hash(state, *lh); - } -} -#endif - -static size_t qlz_compress_core(const unsigned char *source, unsigned char *destination, size_t size, qlz_state_compress *state) -{ - const unsigned char *last_byte = source + size - 1; - const unsigned char *src = source; - unsigned char *cword_ptr = destination; - unsigned char *dst = destination + CWORD_LEN; - ui32 cword_val = 1U << 31; - const unsigned char *last_matchstart = last_byte - UNCONDITIONAL_MATCHLEN - UNCOMPRESSED_END; - ui32 fetch = 0; - unsigned int lits = 0; - - (void) lits; - - if(src <= last_matchstart) - fetch = fast_read(src, 3); - - while(src <= last_matchstart) - { - if ((cword_val & 1) == 1) - { - // store uncompressed if compression ratio is too low - if (src > source + (size >> 1) && dst - destination > src - source - ((src - source) >> 5)) - return 0; - - fast_write((cword_val >> 1) | (1U << 31), cword_ptr, CWORD_LEN); - - cword_ptr = dst; - dst += CWORD_LEN; - cword_val = 1U << 31; - fetch = fast_read(src, 3); - } -#if QLZ_COMPRESSION_LEVEL == 1 - { - const unsigned char *o; - ui32 hash, cached; - - hash = hash_func(fetch); - cached = fetch ^ state->hash[hash].cache; - state->hash[hash].cache = fetch; - - o = state->hash[hash].offset + OFFSET_BASE; - state->hash[hash].offset = CAST(src - OFFSET_BASE); - -#ifdef X86X64 - if ((cached & 0xffffff) == 0 && o != OFFSET_BASE && (src - o > MINOFFSET || (src == o + 1 && lits >= 3 && src > source + 3 && same(src - 3, 6)))) - { - if(cached != 0) - { -#else - if (cached == 0 && o != OFFSET_BASE && (src - o > MINOFFSET || (src == o + 1 && lits >= 3 && src > source + 3 && same(src - 3, 6)))) - { - if (*(o + 3) != *(src + 3)) - { -#endif - hash <<= 4; - cword_val = (cword_val >> 1) | (1U << 31); - fast_write((3 - 2) | hash, dst, 2); - src += 3; - dst += 2; - } - else - { - const unsigned char *old_src = src; - size_t matchlen; - hash <<= 4; - - cword_val = (cword_val >> 1) | (1U << 31); - src += 4; - - if(*(o + (src - old_src)) == *src) - { - src++; - if(*(o + (src - old_src)) == *src) - { - size_t q = last_byte - UNCOMPRESSED_END - (src - 5) + 1; - size_t remaining = q > 255 ? 255 : q; - src++; - while(*(o + (src - old_src)) == *src && (size_t)(src - old_src) < remaining) - src++; - } - } - - matchlen = src - old_src; - if (matchlen < 18) - { - fast_write((ui32)(matchlen - 2) | hash, dst, 2); - dst += 2; - } - else - { - fast_write((ui32)(matchlen << 16) | hash, dst, 3); - dst += 3; - } - } - fetch = fast_read(src, 3); - lits = 0; - } - else - { - lits++; - *dst = *src; - src++; - dst++; - cword_val = (cword_val >> 1); -#ifdef X86X64 - fetch = fast_read(src, 3); -#else - fetch = (fetch >> 8 & 0xffff) | (*(src + 2) << 16); -#endif - } - } -#elif QLZ_COMPRESSION_LEVEL >= 2 - { - const unsigned char *o, *offset2; - ui32 hash, matchlen, k, m, best_k = 0; - unsigned char c; - size_t remaining = (last_byte - UNCOMPRESSED_END - src + 1) > 255 ? 255 : (last_byte - UNCOMPRESSED_END - src + 1); - (void)best_k; - - - //hash = hashat(src); - fetch = fast_read(src, 3); - hash = hash_func(fetch); - - c = state->hash_counter[hash]; - - offset2 = state->hash[hash].offset[0]; - if(offset2 < src - MINOFFSET && c > 0 && ((fast_read(offset2, 3) ^ fetch) & 0xffffff) == 0) - { - matchlen = 3; - if(*(offset2 + matchlen) == *(src + matchlen)) - { - matchlen = 4; - while(*(offset2 + matchlen) == *(src + matchlen) && matchlen < remaining) - matchlen++; - } - } - else - matchlen = 0; - for(k = 1; k < QLZ_POINTERS && c > k; k++) - { - o = state->hash[hash].offset[k]; -#if QLZ_COMPRESSION_LEVEL == 3 - if(((fast_read(o, 3) ^ fetch) & 0xffffff) == 0 && o < src - MINOFFSET) -#elif QLZ_COMPRESSION_LEVEL == 2 - if(*(src + matchlen) == *(o + matchlen) && ((fast_read(o, 3) ^ fetch) & 0xffffff) == 0 && o < src - MINOFFSET) -#endif - { - m = 3; - while(*(o + m) == *(src + m) && m < remaining) - m++; -#if QLZ_COMPRESSION_LEVEL == 3 - if ((m > matchlen) || (m == matchlen && o > offset2)) -#elif QLZ_COMPRESSION_LEVEL == 2 - if (m > matchlen) -#endif - { - offset2 = o; - matchlen = m; - best_k = k; - } - } - } - o = offset2; - state->hash[hash].offset[c & (QLZ_POINTERS - 1)] = src; - c++; - state->hash_counter[hash] = c; - -#if QLZ_COMPRESSION_LEVEL == 3 - if(matchlen > 2 && src - o < 131071) - { - ui32 u; - size_t offset = src - o; - - for(u = 1; u < matchlen; u++) - { - hash = hashat(src + u); - c = state->hash_counter[hash]++; - state->hash[hash].offset[c & (QLZ_POINTERS - 1)] = src + u; - } - - cword_val = (cword_val >> 1) | (1U << 31); - src += matchlen; - - if(matchlen == 3 && offset <= 63) - { - *dst = (unsigned char)(offset << 2); - dst++; - } - else if (matchlen == 3 && offset <= 16383) - { - ui32 f = (ui32)((offset << 2) | 1); - fast_write(f, dst, 2); - dst += 2; - } - else if (matchlen <= 18 && offset <= 1023) - { - ui32 f = ((matchlen - 3) << 2) | ((ui32)offset << 6) | 2; - fast_write(f, dst, 2); - dst += 2; - } - - else if(matchlen <= 33) - { - ui32 f = ((matchlen - 2) << 2) | ((ui32)offset << 7) | 3; - fast_write(f, dst, 3); - dst += 3; - } - else - { - ui32 f = ((matchlen - 3) << 7) | ((ui32)offset << 15) | 3; - fast_write(f, dst, 4); - dst += 4; - } - } - else - { - *dst = *src; - src++; - dst++; - cword_val = (cword_val >> 1); - } -#elif QLZ_COMPRESSION_LEVEL == 2 - - if(matchlen > 2) - { - cword_val = (cword_val >> 1) | (1U << 31); - src += matchlen; - - if (matchlen < 10) - { - ui32 f = best_k | ((matchlen - 2) << 2) | (hash << 5); - fast_write(f, dst, 2); - dst += 2; - } - else - { - ui32 f = best_k | (matchlen << 16) | (hash << 5); - fast_write(f, dst, 3); - dst += 3; - } - } - else - { - *dst = *src; - src++; - dst++; - cword_val = (cword_val >> 1); - } -#endif - } -#endif - } - while (src <= last_byte) - { - if ((cword_val & 1) == 1) - { - fast_write((cword_val >> 1) | (1U << 31), cword_ptr, CWORD_LEN); - cword_ptr = dst; - dst += CWORD_LEN; - cword_val = 1U << 31; - } -#if QLZ_COMPRESSION_LEVEL < 3 - if (src <= last_byte - 3) - { -#if QLZ_COMPRESSION_LEVEL == 1 - ui32 hash, fetch; - fetch = fast_read(src, 3); - hash = hash_func(fetch); - state->hash[hash].offset = CAST(src - OFFSET_BASE); - state->hash[hash].cache = fetch; -#elif QLZ_COMPRESSION_LEVEL == 2 - ui32 hash; - unsigned char c; - hash = hashat(src); - c = state->hash_counter[hash]; - state->hash[hash].offset[c & (QLZ_POINTERS - 1)] = src; - c++; - state->hash_counter[hash] = c; -#endif - } -#endif - *dst = *src; - src++; - dst++; - cword_val = (cword_val >> 1); - } - - while((cword_val & 1) != 1) - cword_val = (cword_val >> 1); - - fast_write((cword_val >> 1) | (1U << 31), cword_ptr, CWORD_LEN); - - // min. size must be 9 bytes so that the qlz_size functions can take 9 bytes as argument - return dst - destination < 9 ? 9 : dst - destination; -} - -static size_t qlz_decompress_core(const unsigned char *source, unsigned char *destination, size_t size, qlz_state_decompress *state, const unsigned char *history) -{ - const unsigned char *src = source + qlz_size_header((const char *)source); - unsigned char *dst = destination; - const unsigned char *last_destination_byte = destination + size - 1; - ui32 cword_val = 1; - const unsigned char *last_matchstart = last_destination_byte - UNCONDITIONAL_MATCHLEN - UNCOMPRESSED_END; - unsigned char *last_hashed = destination - 1; - const unsigned char *last_source_byte = source + qlz_size_compressed((const char *)source) - 1; - static const ui32 bitlut[16] = {4, 0, 1, 0, 2, 0, 1, 0, 3, 0, 1, 0, 2, 0, 1, 0}; - - (void) last_source_byte; - (void) last_hashed; - (void) state; - (void) history; - - for(;;) - { - ui32 fetch; - - if (cword_val == 1) - { -#ifdef QLZ_MEMORY_SAFE - if(src + CWORD_LEN - 1 > last_source_byte) - return 0; -#endif - cword_val = fast_read(src, CWORD_LEN); - src += CWORD_LEN; - } - -#ifdef QLZ_MEMORY_SAFE - if(src + 4 - 1 > last_source_byte) - return 0; -#endif - - fetch = fast_read(src, 4); - - if ((cword_val & 1) == 1) - { - ui32 matchlen; - const unsigned char *offset2; - -#if QLZ_COMPRESSION_LEVEL == 1 - ui32 hash; - cword_val = cword_val >> 1; - hash = (fetch >> 4) & 0xfff; - offset2 = (const unsigned char *)(size_t)state->hash[hash].offset; - - if((fetch & 0xf) != 0) - { - matchlen = (fetch & 0xf) + 2; - src += 2; - } - else - { - matchlen = *(src + 2); - src += 3; - } - -#elif QLZ_COMPRESSION_LEVEL == 2 - ui32 hash; - unsigned char c; - cword_val = cword_val >> 1; - hash = (fetch >> 5) & 0x7ff; - c = (unsigned char)(fetch & 0x3); - offset2 = state->hash[hash].offset[c]; - - if((fetch & (28)) != 0) - { - matchlen = ((fetch >> 2) & 0x7) + 2; - src += 2; - } - else - { - matchlen = *(src + 2); - src += 3; - } - -#elif QLZ_COMPRESSION_LEVEL == 3 - ui32 offset; - cword_val = cword_val >> 1; - if ((fetch & 3) == 0) - { - offset = (fetch & 0xff) >> 2; - matchlen = 3; - src++; - } - else if ((fetch & 2) == 0) - { - offset = (fetch & 0xffff) >> 2; - matchlen = 3; - src += 2; - } - else if ((fetch & 1) == 0) - { - offset = (fetch & 0xffff) >> 6; - matchlen = ((fetch >> 2) & 15) + 3; - src += 2; - } - else if ((fetch & 127) != 3) - { - offset = (fetch >> 7) & 0x1ffff; - matchlen = ((fetch >> 2) & 0x1f) + 2; - src += 3; - } - else - { - offset = (fetch >> 15); - matchlen = ((fetch >> 7) & 255) + 3; - src += 4; - } - - offset2 = dst - offset; -#endif - -#ifdef QLZ_MEMORY_SAFE - if(offset2 < history || offset2 > dst - MINOFFSET - 1) - return 0; - - if(matchlen > (ui32)(last_destination_byte - dst - UNCOMPRESSED_END + 1)) - return 0; -#endif - - memcpy_up(dst, offset2, matchlen); - dst += matchlen; - -#if QLZ_COMPRESSION_LEVEL <= 2 - update_hash_upto(state, &last_hashed, dst - matchlen); - last_hashed = dst - 1; -#endif - } - else - { - if (dst < last_matchstart) - { - unsigned int n = bitlut[cword_val & 0xf]; -#ifdef X86X64 - *(ui32 *)dst = *(ui32 *)src; -#else - memcpy_up(dst, src, 4); -#endif - cword_val = cword_val >> n; - dst += n; - src += n; -#if QLZ_COMPRESSION_LEVEL <= 2 - update_hash_upto(state, &last_hashed, dst - 3); -#endif - } - else - { - while(dst <= last_destination_byte) - { - if (cword_val == 1) - { - src += CWORD_LEN; - cword_val = 1U << 31; - } -#ifdef QLZ_MEMORY_SAFE - if(src >= last_source_byte + 1) - return 0; -#endif - *dst = *src; - dst++; - src++; - cword_val = cword_val >> 1; - } - -#if QLZ_COMPRESSION_LEVEL <= 2 - update_hash_upto(state, &last_hashed, last_destination_byte - 3); // todo, use constant -#endif - return size; - } - - } - } -} - -size_t qlz_compress(const void *source, char *destination, size_t size, qlz_state_compress *state) -{ - size_t r; - ui32 compressed; - size_t base; - - if(size == 0 || size > 0xffffffff - 400) - return 0; - - if(size < 216) - base = 3; - else - base = 9; - -#if QLZ_STREAMING_BUFFER > 0 - if (state->stream_counter + size - 1 >= QLZ_STREAMING_BUFFER) -#endif - { - reset_table_compress(state); - r = base + qlz_compress_core((const unsigned char *)source, (unsigned char*)destination + base, size, state); -#if QLZ_STREAMING_BUFFER > 0 - reset_table_compress(state); -#endif - if(r == base) - { - memcpy(destination + base, source, size); - r = size + base; - compressed = 0; - } - else - { - compressed = 1; - } - state->stream_counter = 0; - } -#if QLZ_STREAMING_BUFFER > 0 - else - { - unsigned char *src = state->stream_buffer + state->stream_counter; - - memcpy(src, source, size); - r = base + qlz_compress_core(src, (unsigned char*)destination + base, size, state); - - if(r == base) - { - memcpy(destination + base, src, size); - r = size + base; - compressed = 0; - reset_table_compress(state); - } - else - { - compressed = 1; - } - state->stream_counter += size; - } -#endif - if(base == 3) - { - *destination = (unsigned char)(0 | compressed); - *(destination + 1) = (unsigned char)r; - *(destination + 2) = (unsigned char)size; - } - else - { - *destination = (unsigned char)(2 | compressed); - fast_write((ui32)r, destination + 1, 4); - fast_write((ui32)size, destination + 5, 4); - } - - *destination |= (QLZ_COMPRESSION_LEVEL << 2); - *destination |= (1 << 6); - *destination |= ((QLZ_STREAMING_BUFFER == 0 ? 0 : (QLZ_STREAMING_BUFFER == 100000 ? 1 : (QLZ_STREAMING_BUFFER == 1000000 ? 2 : 3))) << 4); - -// 76543210 -// 01SSLLHC - - return r; -} - -size_t qlz_decompress(const char *source, void *destination, qlz_state_decompress *state) -{ - size_t dsiz = qlz_size_decompressed(source); - -#if QLZ_STREAMING_BUFFER > 0 - if (state->stream_counter + qlz_size_decompressed(source) - 1 >= QLZ_STREAMING_BUFFER) -#endif - { - if((*source & 1) == 1) - { - reset_table_decompress(state); - dsiz = qlz_decompress_core((const unsigned char *)source, (unsigned char *)destination, dsiz, state, (const unsigned char *)destination); - } - else - { - memcpy(destination, source + qlz_size_header(source), dsiz); - } - state->stream_counter = 0; - reset_table_decompress(state); - } -#if QLZ_STREAMING_BUFFER > 0 - else - { - unsigned char *dst = state->stream_buffer + state->stream_counter; - if((*source & 1) == 1) - { - dsiz = qlz_decompress_core((const unsigned char *)source, dst, dsiz, state, (const unsigned char *)state->stream_buffer); - } - else - { - memcpy(dst, source + qlz_size_header(source), dsiz); - reset_table_decompress(state); - } - memcpy(destination, dst, dsiz); - state->stream_counter += dsiz; - } -#endif - return dsiz; -} - diff --git a/decomsvr/quicklz.h b/decomsvr/quicklz.h deleted file mode 100644 index 56915b56d..000000000 --- a/decomsvr/quicklz.h +++ /dev/null @@ -1,150 +0,0 @@ -#ifndef QLZ_HEADER -#define QLZ_HEADER - -// Fast data compression library -// Copyright (C) 2006-2011 Lasse Mikkel Reinhold -// lar@quicklz.com -// -// QuickLZ can be used for free under the GPL 1, 2 or 3 license (where anything -// released into public must be open source) or under a commercial license if such -// has been acquired (see http://www.quicklz.com/order.html). The commercial license -// does not cover derived or ported versions created by third parties under GPL. - -// You can edit following user settings. Data must be decompressed with the same -// setting of QLZ_COMPRESSION_LEVEL and QLZ_STREAMING_BUFFER as it was compressed -// (see manual). If QLZ_STREAMING_BUFFER > 0, scratch buffers must be initially -// zeroed out (see manual). First #ifndef makes it possible to define settings from -// the outside like the compiler command line. - -// 1.5.0 final - -#ifndef QLZ_COMPRESSION_LEVEL - -// 1 gives fastest compression speed. 3 gives fastest decompression speed and best -// compression ratio. -//#define QLZ_COMPRESSION_LEVEL 1 -//#define QLZ_COMPRESSION_LEVEL 2 -#define QLZ_COMPRESSION_LEVEL 3 - -// If > 0, zero out both states prior to first call to qlz_compress() or qlz_decompress() -// and decompress packets in the same order as they were compressed -#define QLZ_STREAMING_BUFFER 0 -//#define QLZ_STREAMING_BUFFER 100000 -//#define QLZ_STREAMING_BUFFER 1000000 - -// Guarantees that decompression of corrupted data cannot crash. Decreases decompression -// speed 10-20%. Compression speed not affected. -#define QLZ_MEMORY_SAFE -#endif - -#define QLZ_VERSION_MAJOR 1 -#define QLZ_VERSION_MINOR 5 -#define QLZ_VERSION_REVISION 0 - -// Using size_t, memset() and memcpy() -#include - -// Verify compression level -#if QLZ_COMPRESSION_LEVEL != 1 && QLZ_COMPRESSION_LEVEL != 2 && QLZ_COMPRESSION_LEVEL != 3 -#error QLZ_COMPRESSION_LEVEL must be 1, 2 or 3 -#endif - -typedef unsigned int ui32; -typedef unsigned short int ui16; - -// Decrease QLZ_POINTERS for level 3 to increase compression speed. Do not touch any other values! -#if QLZ_COMPRESSION_LEVEL == 1 -#define QLZ_POINTERS 1 -#define QLZ_HASH_VALUES 4096 -#elif QLZ_COMPRESSION_LEVEL == 2 -#define QLZ_POINTERS 4 -#define QLZ_HASH_VALUES 2048 -#elif QLZ_COMPRESSION_LEVEL == 3 -#define QLZ_POINTERS 16 -#define QLZ_HASH_VALUES 4096 -#endif - -// Detect if pointer size is 64-bit. It's not fatal if some 64-bit target is not detected because this is only for adding an optional 64-bit optimization. -#if defined _LP64 || defined __LP64__ || defined __64BIT__ || _ADDR64 || defined _WIN64 || defined __arch64__ || __WORDSIZE == 64 || (defined __sparc && defined __sparcv9) || defined __x86_64 || defined __amd64 || defined __x86_64__ || defined _M_X64 || defined _M_IA64 || defined __ia64 || defined __IA64__ -#define QLZ_PTR_64 -#endif - -// hash entry -typedef struct -{ -#if QLZ_COMPRESSION_LEVEL == 1 - ui32 cache; -#if defined QLZ_PTR_64 && QLZ_STREAMING_BUFFER == 0 - unsigned int offset; -#else - const unsigned char* offset; -#endif -#else - const unsigned char* offset[QLZ_POINTERS]; -#endif - -} qlz_hash_compress; - -typedef struct -{ -#if QLZ_COMPRESSION_LEVEL == 1 - const unsigned char* offset; -#else - const unsigned char* offset[QLZ_POINTERS]; -#endif -} qlz_hash_decompress; - - -// states -typedef struct -{ -#if QLZ_STREAMING_BUFFER > 0 - unsigned char stream_buffer[QLZ_STREAMING_BUFFER]; -#endif - size_t stream_counter; - qlz_hash_compress hash[QLZ_HASH_VALUES]; - unsigned char hash_counter[QLZ_HASH_VALUES]; -} qlz_state_compress; - - -#if QLZ_COMPRESSION_LEVEL == 1 || QLZ_COMPRESSION_LEVEL == 2 -typedef struct -{ -#if QLZ_STREAMING_BUFFER > 0 - unsigned char stream_buffer[QLZ_STREAMING_BUFFER]; -#endif - qlz_hash_decompress hash[QLZ_HASH_VALUES]; - unsigned char hash_counter[QLZ_HASH_VALUES]; - size_t stream_counter; -} qlz_state_decompress; -#elif QLZ_COMPRESSION_LEVEL == 3 -typedef struct -{ -#if QLZ_STREAMING_BUFFER > 0 - unsigned char stream_buffer[QLZ_STREAMING_BUFFER]; -#endif -#if QLZ_COMPRESSION_LEVEL <= 2 - qlz_hash_decompress hash[QLZ_HASH_VALUES]; -#endif - size_t stream_counter; -} qlz_state_decompress; -#endif - - -#if defined (__cplusplus) -extern "C" { -#endif - -// Public functions of QuickLZ -size_t qlz_size_decompressed(const char* source); -size_t qlz_size_compressed(const char* source); -size_t qlz_compress(const void* source, char* destination, size_t size, qlz_state_compress* state); -size_t qlz_decompress(const char* source, void* destination, qlz_state_decompress* state); -int qlz_get_setting(int setting); - -#if defined (__cplusplus) -} -#endif - -#endif - diff --git a/decomsvr/resource.h b/decomsvr/resource.h deleted file mode 100644 index 17a793ca1..000000000 --- a/decomsvr/resource.h +++ /dev/null @@ -1,14 +0,0 @@ -//{{NO_DEPENDENCIES}} -// Microsoft Visual C++ generated include file. -// Used by DecomSvr.rc - -// Next default values for new objects -// -#ifdef APSTUDIO_INVOKED -#ifndef APSTUDIO_READONLY_SYMBOLS -#define _APS_NEXT_RESOURCE_VALUE 101 -#define _APS_NEXT_COMMAND_VALUE 40001 -#define _APS_NEXT_CONTROL_VALUE 1001 -#define _APS_NEXT_SYMED_VALUE 101 -#endif -#endif diff --git a/decomsvr/server.cpp b/decomsvr/server.cpp deleted file mode 100644 index f49dc35bc..000000000 --- a/decomsvr/server.cpp +++ /dev/null @@ -1,978 +0,0 @@ -/* Copyright (C) 2014 InfiniDB, Inc. - - This program is free software; you can redistribute it and/or - modify it under the terms of the GNU General Public License - as published by the Free Software Foundation; version 2 of - the License. - - This program is distributed in the hope that it will be useful, - but WITHOUT ANY WARRANTY; without even the implied warranty of - MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the - GNU General Public License for more details. - - You should have received a copy of the GNU General Public License - along with this program; if not, write to the Free Software - Foundation, Inc., 51 Franklin Street, Fifth Floor, Boston, - MA 02110-1301, USA. */ - -/* - Protocol definition: - On the control socket: - This server waits for the other end to send it: - 1. The prefix name of the data fifos to open and read/write (string). The incoming, - compressed data is on "name".c and this server will write uncompressed - data on "name".u. - - On the data fifos: - The server then reads the compressed data from the cdata fifo: - 1. The number of bytes of compressed data to read (number) - 2. The compressed data - then it decompresses it, and writes to the udata fifo: - 1. The number of bytes in the uncompressed stream (number) - 2. The uncompressed data - - strings are sent like this: - uint32_t string len - len bytes of the string - numbers are sent like this: - uint64_t the number - - This server expects numeric values to be in its native byte order, so - the sender needs to send them that way. -*/ - -//#define _FILE_OFFSET_BITS 64 -//#define _LARGEFILE64_SOURCE - -#ifdef _MSC_VER -#define WIN32_LEAN_AND_MEAN -#define NOMINMAX -#include -#include -#include -#include -#include -#include -#else -#include -#include -#include -#include -#endif -#include -#include -#include -#include -#include -#include -#include -#include -#include -#include -#include -#ifndef _MSC_VER -#include -#endif -using namespace std; - -#include -#include -#include -using namespace boost; - -#include "quicklz.h" -#ifndef _MSC_VER -#include "config.h" -#endif - -//#define SINGLE_THREADED - -namespace -{ -#ifdef _MSC_VER -typedef SOCKET SockType; -#define SockReadFcn reads -#else -typedef int SockType; -#define SockReadFcn readn -#endif - -short PortNo; -SockType ListenSock; - -// version 1.1 of the chunk data has a short header -const uint8_t CHUNK_MAGIC1 = 0xff; -const int SIG_OFFSET = 0; -const int CHECKSUM_OFFSET = 1; -const int LEN_OFFSET = 5; -const uint32_t HEADER_SIZE = 9; - -const int ERR_OK = 0; -const int ERR_CHECKSUM = -1; -const int ERR_DECOMPRESS = -2; -const int ERR_BADINPUT = -3; - -/* version 1.2 of the chunk data changes the hash function used to calculate - * checksums. We can no longer use the algorithm used in ver 1.1. Everything - * else is the same - */ -const uint8_t CHUNK_MAGIC2 = 0xfe; - -// A version of idbassert_s & log() that doesn't require linking the logging lib. -// Things printed to stderr go to /tmp/decomsvr.err -#ifndef __STRING -#define __STRING(x) #x -#endif -#define idbassert_s(x, s) do { \ - if (!(x)) { \ - std::ostringstream os; \ -\ - os << __FILE__ << "@" << __LINE__ << ": assertion \'" << __STRING(x) << "\' failed. Error msg \'" << s << "\'"; \ - std::cerr << os.str() << std::endl; \ - throw runtime_error(os.str()); \ - } \ -} while (0) - -void log(const string& s) -{ - cerr << s << endl; -} - -struct DecomMessage -{ - DecomMessage() : isValid(false) { } - ~DecomMessage() { } - - string toString() const; - - bool isValid; - string pipeName; -}; - -string DecomMessage::toString() const -{ - ostringstream oss; - oss << "valid: " << boolalpha << isValid << ", " << - "pipepfx: " << pipeName; - return oss.str(); -} - -ostream& operator<<(ostream& os, const DecomMessage& rhs) -{ - os << rhs.toString(); - return os; -} - -class ThreadFunc -{ -public: - ThreadFunc(const DecomMessage& dm) : fDm(dm) { } - ~ThreadFunc() { } - - void operator()(); - -private: - //Defaults okay - //ThreadFunc(const ThreadFunc& rhs); - //ThreadFunc& operator=(const ThreadFunc& rhs); - - DecomMessage fDm; -}; - -bool serverInit() -{ -#ifndef _MSC_VER - - //Set parent PID to init - setsid(); - - //Handle certain signals (we want these to return EINTR so we can throw) - //SIGPIPE - //I don't think we'll get any of these from init (except possibly HUP, but that's an indication - // of bad things anyway) - //SIGHUP? - //SIGUSR1? - //SIGUSR2? - //SIGPOLL? - struct sigaction sa; - memset(&sa, 0, sizeof(struct sigaction)); - sa.sa_handler = SIG_IGN; - sigaction(SIGPIPE, &sa, 0); - sigaction(SIGHUP, &sa, 0); - sigaction(SIGUSR1, &sa, 0); - sigaction(SIGUSR2, &sa, 0); -#ifndef __FreeBSD__ - sigaction(SIGPOLL, &sa, 0); -#endif - int fd; - close(2); - fd = open("/tmp/decomsrv.err", O_CREAT | O_TRUNC | O_WRONLY, 0644); - - if (fd >= 0 && fd != 2) - { - dup2(fd, 2); - close(fd); - } - -#endif - return true; -} - -bool initCtlFifo() -{ -#ifdef _MSC_VER - WSAData wsadata; - const WORD minVersion = MAKEWORD(2, 2); - WSAStartup(minVersion, &wsadata); -#endif - ListenSock = socket(PF_INET, SOCK_STREAM, IPPROTO_TCP); - idbassert_s(ListenSock >= 0, string("socket create error: ") + strerror(errno)); - //if (ListenSock < 0) throw runtime_error(string("socket create error: ") + strerror(errno)); -#ifndef _MSC_VER - int optval = 1; - setsockopt(ListenSock, SOL_SOCKET, SO_REUSEADDR, reinterpret_cast(&optval), sizeof(optval)); -#endif - int rc = 0; - struct sockaddr_in serv_addr; - struct in_addr la; - inet_aton("127.0.0.1", &la); - memset(&serv_addr, 0, sizeof(serv_addr)); - serv_addr.sin_family = AF_INET; - serv_addr.sin_addr.s_addr = la.s_addr; - serv_addr.sin_port = htons(PortNo); - const int MaxTries = 5 * 60 / 10; - int tries = 0; -again: - rc = ::bind(ListenSock, (sockaddr*)&serv_addr, sizeof(serv_addr)); - - if (rc < 0) - { -#ifdef _MSC_VER - int x = WSAGetLastError(); - - if (x == WSAEADDRINUSE) -#else - if (errno == EADDRINUSE) -#endif - { - //cerr << "Addr in use..." << endl; - if (++tries >= MaxTries) - { - log("Waited too long for socket to bind...giving up"); - //cerr << "Waited too long for socket to bind...giving up" << endl; - exit(1); - } - - sleep(10); - goto again; - } - - idbassert_s(0, string("socket bind error: ") + strerror(errno)); - //throw runtime_error(string("socket bind error: ") + strerror(errno)); - } - - rc = listen(ListenSock, 16); - idbassert_s(rc >= 0, string("socket listen error") + strerror(errno)); - //if (rc < 0) throw runtime_error(string("socket listen error") + strerror(errno)); - - return true; -} - -#ifndef _MSC_VER -void readn(int fd, void* buf, const size_t wanted) -{ - size_t needed = wanted; - size_t sofar = 0; - char* p = reinterpret_cast(buf); - ssize_t rrc = -1; - pollfd fds[1]; - int en = 0; - int prc = 0; - ostringstream oss; - unsigned zerocount = 0; - - fds[0].fd = fd; - fds[0].events = POLLIN; - - while (wanted > sofar) - { - fds[0].revents = 0; - errno = 0; - prc = poll(fds, 1, -1); - en = errno; - - if (prc <= 0) - { - if (en == EAGAIN || en == EINTR || en == 512) - continue; - - oss << "decomsvr::readn: poll() returned " << prc << " (" << strerror(en) << ")"; - idbassert_s(0, oss.str()); - } - - //no data on fd - if ((fds[0].revents & POLLIN) == 0) - { - oss << "decomsvr::readn: revents for fd " << fds[0].fd << " was " << fds[0].revents; - idbassert_s(0, oss.str()); - } - - errno = 0; - rrc = read(fd, (p + sofar), needed); - en = errno; - - if (rrc < 0) - { - if (en == EAGAIN || en == EINTR || en == 512) - continue; - - oss << "decomsvr::readn: read() returned " << rrc << " (" << strerror(en) << ")"; - idbassert_s(0, oss.str()); - } - - if (rrc == 0) - { - ostringstream os; - zerocount++; - - if (zerocount >= 10) - { - os << "decomsvr::readn(): too many zero-length reads!"; - idbassert_s(0, oss.str()); - } - - os << "decomsvr::readn(): zero-length read on fd " << fd; - log(os.str()); - sleep(1); - } - else - zerocount = 0; - - needed -= rrc; - sofar += rrc; - } -} - -size_t writen(int fd, const void* data, size_t nbytes) -{ - size_t nleft; - ssize_t nwritten; - const char* bufp = (const char*) data; - nleft = nbytes; - - while (nleft > 0) - { - // the O_NONBLOCK flag is not set, this is a blocking I/O. - if ((nwritten = ::write(fd, bufp, nleft)) < 0) - { - if (errno == EINTR) - nwritten = 0; - else - { - // save the error no first - int e = errno; - string errorMsg = "decomsvr: write() error: "; - scoped_array buf(new char[80]); -#if STRERROR_R_CHAR_P - const char* p; - - if ((p = strerror_r(e, buf.get(), 80)) != 0) - errorMsg += p; - -#else - int p; - - if ((p = strerror_r(e, buf.get(), 80)) == 0) - errorMsg += buf.get(); - -#endif - idbassert_s(0, errorMsg); - } - } - - nleft -= nwritten; - bufp += nwritten; - } - - return nbytes; -} -#else -void reads(SOCKET fd, void* buf, const size_t wanted) -{ - size_t needed = wanted; - size_t sofar = 0; - char* p = reinterpret_cast(buf); - ssize_t rrc = -1; - pollfd fds[1]; - int en = 0; - - fds[0].fd = fd; - fds[0].events = POLLIN; - - while (wanted > sofar) - { - fds[0].revents = 0; - poll(fds, 1, -1); - errno = 0; - rrc = recv(fd, (p + sofar), (int)needed, 0); - en = errno; - - if (rrc < 0) - { - if (en == EAGAIN || en == EINTR) - continue; - - ostringstream oss; - oss << "read() returned " << rrc << " (" << strerror(en) << ")"; - idbassert_s(0, oss.str()); - } - - needed -= rrc; - sofar += rrc; - } -} -#endif - -uint32_t readNumber32(SockType fd) -{ - uint32_t np; - SockReadFcn(fd, &np, 4); - return np; -} - -string readString(SockType fd) -{ - string s; - uint32_t len = readNumber32(fd); - idbassert_s(len <= 64, "while reading a string len (>64)"); - //if (len > 64) - // throw runtime_error("while reading a string len (>64)"); - char* buf = (char*)alloca(len + 1); //this should be at most 65 bytes and should always succeed - SockReadFcn(fd, buf, len); - buf[len] = 0; - s = buf; - return s; -} - -DecomMessage getNextMsg(SockType fd) -{ - DecomMessage dm; - - try - { - dm.pipeName = readString(fd); - dm.isValid = true; - } - catch (runtime_error& rex) - { - cerr << "re reading ctl msg: " << rex.what() << endl; - dm.pipeName = ""; - } - catch (...) - { - cerr << "ex reading ctl msg" << endl; - dm.pipeName = ""; - } - - return dm; -} - -// Murmur3 from code.google.com - -uint64_t fmix(uint64_t k) -{ - k ^= k >> 33; - k *= 0xff51afd7ed558ccdULL; - k ^= k >> 33; - k *= 0xc4ceb9fe1a85ec53ULL; - k ^= k >> 33; - - return k; -} - -uint64_t rotl64(uint64_t x, int8_t r) -{ - return (x << r) | (x >> (64 - r)); -} - -class Hasher128 -{ -public: - inline uint64_t operator()(const char* data, uint64_t len) const - { - const int nblocks = len / 16; - - uint64_t h1 = 0; - uint64_t h2 = 0; - - const uint64_t c1 = 0x87c37b91114253d5ULL; - const uint64_t c2 = 0x4cf5ad432745937fULL; - - //---------- - // body - - const uint64_t* blocks = (const uint64_t*) (data); - - for (int i = 0; i < nblocks; i++) - { - uint64_t k1 = blocks[i * 2 + 0]; - uint64_t k2 = blocks[i * 2 + 1]; - - k1 *= c1; - k1 = rotl64(k1, 31); - k1 *= c2; - h1 ^= k1; - - h1 = rotl64(h1, 27); - h1 += h2; - h1 = h1 * 5 + 0x52dce729; - - k2 *= c2; - k2 = rotl64(k2, 33); - k2 *= c1; - h2 ^= k2; - - h2 = rotl64(h2, 31); - h2 += h1; - h2 = h2 * 5 + 0x38495ab5; - } - - //---------- - // tail - - const uint8_t* tail = (const uint8_t*) (data + nblocks * 16); - - uint64_t k1 = 0; - uint64_t k2 = 0; - - switch (len & 15) - { - case 15: - k2 ^= uint64_t(tail[14]) << 48; - - case 14: - k2 ^= uint64_t(tail[13]) << 40; - - case 13: - k2 ^= uint64_t(tail[12]) << 32; - - case 12: - k2 ^= uint64_t(tail[11]) << 24; - - case 11: - k2 ^= uint64_t(tail[10]) << 16; - - case 10: - k2 ^= uint64_t(tail[9]) << 8; - - case 9: - k2 ^= uint64_t(tail[8]) << 0; - k2 *= c2; - k2 = rotl64(k2, 33); - k2 *= c1; - h2 ^= k2; - - case 8: - k1 ^= uint64_t(tail[7]) << 56; - - case 7: - k1 ^= uint64_t(tail[6]) << 48; - - case 6: - k1 ^= uint64_t(tail[5]) << 40; - - case 5: - k1 ^= uint64_t(tail[4]) << 32; - - case 4: - k1 ^= uint64_t(tail[3]) << 24; - - case 3: - k1 ^= uint64_t(tail[2]) << 16; - - case 2: - k1 ^= uint64_t(tail[1]) << 8; - - case 1: - k1 ^= uint64_t(tail[0]) << 0; - k1 *= c1; - k1 = rotl64(k1, 31); - k1 *= c2; - h1 ^= k1; - }; - - //---------- - // finalization - - h1 ^= len; - - h2 ^= len; - - h1 += h2; - - h2 += h1; - - h1 = fmix(h1); - - h2 = fmix(h2); - - h1 += h2; - - h2 += h1; - - return h1; - } -}; - -int uncompressBlock(const char* in, const size_t inLen, unsigned char* out, - unsigned int& outLen) -{ - int rc = ERR_OK; - int qlzrc = 0; - - boost::scoped_ptr scratch(new qlz_state_decompress()); - - uint32_t realChecksum; - uint32_t storedChecksum; - uint32_t storedLen; - uint8_t storedMagic; - Hasher128 hasher; - - outLen = 0; - - if (inLen < 1) - { - return ERR_BADINPUT; - } - - storedMagic = *((uint8_t*) &in[SIG_OFFSET]); - - if (storedMagic == CHUNK_MAGIC1 || storedMagic == CHUNK_MAGIC2) - { - if (inLen < HEADER_SIZE) - { - return ERR_BADINPUT; - } - - storedChecksum = *((uint32_t*) &in[CHECKSUM_OFFSET]); - storedLen = *((uint32_t*) (&in[LEN_OFFSET])); - - if (inLen < storedLen + HEADER_SIZE) - { - return ERR_BADINPUT; - } - - /* We can no longer verify the checksum on ver 1.1 */ - if (storedMagic == CHUNK_MAGIC2) - { - realChecksum = hasher(&in[HEADER_SIZE], storedLen); - - if (storedChecksum != realChecksum) - { - return ERR_CHECKSUM; - } - } - - qlzrc = qlz_decompress(&in[HEADER_SIZE], out, scratch.get()); - } - else - qlzrc = qlz_decompress(in, out, scratch.get()); - - if (qlzrc == 0) - rc = ERR_DECOMPRESS; - else - outLen = qlzrc; - - return rc; -} - -struct ScopedCleaner -{ -#ifdef _MSC_VER - ScopedCleaner() : handle(INVALID_HANDLE_VALUE) { } - ~ScopedCleaner() - { - if (handle != INVALID_HANDLE_VALUE) CloseHandle(handle); - } - - HANDLE handle; -#else - ScopedCleaner() : fd(-1) { } - ~ScopedCleaner() - { - if (fd >= 0) close(fd); - } - - int fd; -#endif -}; - -void ThreadFunc::operator()() -{ - string cfifo = fDm.pipeName + ".c"; - string ufifo = fDm.pipeName + ".u"; - uint64_t ccount = 0; - ssize_t rrc = -1; - ScopedCleaner cleaner; -#ifdef _MSC_VER - HANDLE h; - h = CreateFile(cfifo.c_str(), GENERIC_READ, 0, 0, OPEN_EXISTING, FILE_ATTRIBUTE_NORMAL, 0); - idbassert_s(h != INVALID_HANDLE_VALUE, "while opening cpipe"); - //if (!(h != INVALID_HANDLE_VALUE)) - // throw runtime_error("while opening cpipe"); - cleaner.handle = h; - - DWORD nread; - BOOL drrc; - drrc = ReadFile(h, &ccount, 8, &nread, 0); - idbassert_s(drrc != 0 && nread == 8 && ccount < 8 * 1024 * 1024, "while reading from cpipe"); - //if (!(drrc != 0 && nread == 8 && ccount < 8 * 1024 * 1024)) - // throw runtime_error("while reading from cpipe"); - - scoped_array in(new char[ccount]); - - drrc = ReadFile(h, in.get(), (DWORD)ccount, &nread, 0); - idbassert_s(drrc != 0 && nread == ccount, "while reading from cpipe"); - //if (!(drrc != 0 && nread == ccount)) - // throw runtime_error("while reading from cpipe"); - - CloseHandle(h); - cleaner.handle = INVALID_HANDLE_VALUE; -#else - scoped_array in; - int fd = -1; - - try - { - fd = open(cfifo.c_str(), O_RDONLY); - idbassert_s(fd >= 0, "when opening data fifo for input"); - - cleaner.fd = fd; - - readn(fd, &ccount, 8); - - in.reset(new char[ccount]); - - readn(fd, in.get(), ccount); - - close(fd); - cleaner.fd = -1; - } - catch (std::exception& ) - { - //This is a protocol error and returning here will clean up resources on - //the stack unwind. - return; - } - -#endif -#ifdef _MSC_VER - h = CreateFile(ufifo.c_str(), GENERIC_WRITE, 0, 0, OPEN_EXISTING, FILE_ATTRIBUTE_NORMAL, 0); - idbassert_s(h != INVALID_HANDLE_VALUE, "while opening upipe"); - //if (!(h != INVALID_HANDLE_VALUE)) - // throw runtime_error("while opening upipe"); - cleaner.handle = h; - - uint64_t outlen = 512 * 1024 * 8; - unsigned int ol = static_cast(outlen); - - scoped_array out(new char[outlen]); - - int crc = uncompressBlock(in.get(), ccount, reinterpret_cast(out.get()), ol); - - if (crc != ERR_OK) - outlen = 0; - else - outlen = ol; - - BOOL dwrc; - DWORD nwritten; - dwrc = WriteFile(h, &outlen, 8, &nwritten, 0); - idbassert_s(dwrc != 0 && nwritten == 8, "while writing to upipe"); - //if (!(dwrc != 0 && nwritten == 8)) - // throw runtime_error("while writing to upipe"); - - dwrc = WriteFile(h, out.get(), (DWORD)outlen, &nwritten, 0); - idbassert_s(dwrc != 0 && nwritten == outlen, "while writing to upipe"); - //if (!(dwrc != 0 && nwritten == outlen)) - // throw runtime_error("while writing to upipe"); - - FlushFileBuffers(h); - CloseHandle(h); - cleaner.handle = INVALID_HANDLE_VALUE; -#else - scoped_array out; - - try - { - fd = open(ufifo.c_str(), O_WRONLY); - idbassert_s(fd >= 0, "when opening data fifo for output"); - //if (fd < 0) - // throw runtime_error("when opening data fifo for output"); - - cleaner.fd = fd; - - uint64_t outlen = 512 * 1024 * 8; - unsigned int ol = outlen; - - out.reset(new char[outlen]); - - int crc = uncompressBlock(in.get(), ccount, reinterpret_cast(out.get()), ol); - - if (crc != ERR_OK) - outlen = 0; - else - outlen = ol; - - rrc = writen(fd, &outlen, 8); - idbassert_s(rrc == 8, "when writing len to data fifo"); - - rrc = writen(fd, out.get(), outlen); - idbassert_s(rrc == (ssize_t)outlen, "when writing data to data fifo"); - - close(fd); - cleaner.fd = -1; - } - catch (std::exception& ) - { - //There was an error writing the uncompressed data back to PrimProc. Cleanup by - //unwinding the stack - return; - } - -#endif -} - -#ifndef _MSC_VER -void cleanupFifos() -{ - //find all existing fifos and try get rid of them.... - DIR* dirp = 0; - struct dirent* direntp = 0; - char fifoname[PATH_MAX]; - int fd = -1; - - dirp = opendir("/tmp"); - strcpy(fifoname, "/tmp/"); - - direntp = readdir(dirp); - - while (direntp != 0) - { - if (memcmp(direntp->d_name, "cdatafifo", 9) == 0) - { - strcpy(&fifoname[5], direntp->d_name); - fd = open(fifoname, O_RDONLY | O_NONBLOCK); - - //opening and closing this fifo will cause PP to unblock and retry - if (fd >= 0) - { - close(fd); - } - else - { - fd = open(fifoname, O_WRONLY | O_NONBLOCK); - - if (fd >= 0) - close(fd); - } - } - - direntp = readdir(dirp); - } - - closedir(dirp); -} -#endif - -} - -int main(int argc, char** argv) -{ - int c; - - PortNo = 0; - char* p = getenv("IDB_DECOMSVR_PORT"); - - if (p && *p) - PortNo = atoi(p); - - if (PortNo <= 0) - PortNo = 9199; - -#ifdef _MSC_VER - ListenSock = INVALID_SOCKET; -#else - ListenSock = -1; -#endif - opterr = 0; - - while ((c = getopt(argc, argv, "p:")) != -1) - switch (c) - { - case 'p': - PortNo = atoi(optarg); - break; - - case '?': - default: - break; - } - - if (!serverInit()) - { - log("Could not initialize the Decompression Server!"); - //cerr << "Could not initialize the Decompression Server!" << endl; - return 1; - } - - initCtlFifo(); - -#ifndef _MSC_VER - cleanupFifos(); -#endif - - DecomMessage m; - - for (;;) - { -#ifdef _MSC_VER - SOCKET dataSock = INVALID_SOCKET; -#else - int dataSock = -1; -#endif - dataSock = accept(ListenSock, 0, 0); -#ifdef _MSC_VER - idbassert_s(dataSock != INVALID_SOCKET, string("socket accept error: ") + strerror(errno)); - //if (dataSock == INVALID_SOCKET) - // throw runtime_error(string("socket accept error: ") + strerror(errno)); -#else - //if (dataSock < 0) - idbassert_s(dataSock >= 0, string("socket accept error: ") + strerror(errno)); -#endif - m = getNextMsg(dataSock); - shutdown(dataSock, SHUT_RDWR); -#ifdef _MSC_VER - closesocket(dataSock); -#else - close(dataSock); -#endif - - if (m.isValid) - { - ThreadFunc tf(m); -#ifdef SINGLE_THREADED - tf(); -#else - thread t(tf); -#endif - } - else - idbassert_s(0, "Invalid msg"); - - //cerr << "Invalid msg" << endl; - } - - return 0; -} - diff --git a/oam/etc/ProcessConfig.xml b/oam/etc/ProcessConfig.xml index 8a0c3618f..8088135bb 100644 --- a/oam/etc/ProcessConfig.xml +++ b/oam/etc/ProcessConfig.xml @@ -58,16 +58,6 @@ LOADSHARE off - - DecomSvr - pm - /$INSTALLDIR/bin/DecomSvr - 2 - 15 - - LOADSHARE - off - PrimProc pm diff --git a/oam/etc/ProcessConfig.xml.singleserver b/oam/etc/ProcessConfig.xml.singleserver index b01ca5585..9c543ae38 100644 --- a/oam/etc/ProcessConfig.xml.singleserver +++ b/oam/etc/ProcessConfig.xml.singleserver @@ -58,16 +58,6 @@ LOADSHARE off - - DecomSvr - pm - /usr/local/mariadb/columnstore/bin/DecomSvr - 2 - 15 - - LOADSHARE - off - PrimProc pm diff --git a/oam/install_scripts/syslogSetup.sh b/oam/install_scripts/syslogSetup.sh index f75efbd42..0d8c3550a 100755 --- a/oam/install_scripts/syslogSetup.sh +++ b/oam/install_scripts/syslogSetup.sh @@ -13,6 +13,7 @@ syslog_conf=nofile rsyslog7=0 user=`whoami 2>/dev/null` + groupname=adm username=syslog @@ -172,6 +173,21 @@ if [ ! -z "$syslog_conf" ] ; then # remove older version incase it was installed by previous build rm -rf /etc/rsyslog.d/columnstore.conf + // determine username/groupname + + if [ -f /var/log/messages ]; then + user=`stat -c "%U %G" /var/log/messages | awk '{print $1}'` + group=`stat -c "%U %G" /var/log/messages | awk '{print $2}'` + fi + + if [ -f /var/log/syslog ]; then + user=`stat -c "%U %G" /var/log/syslog | awk '{print $1}'` + group=`stat -c "%U %G" /var/log/syslog | awk '{print $2}'` + fi + + //set permissions + $SUDO chown $user:$group -R /var/log/mariadb > /dev/null 2>&1 + if [ $rsyslog7 == 1 ]; then rm -f /etc/rsyslog.d/49-columnstore.conf cp ${columnstoreSyslogFile7} ${syslog_conf} diff --git a/oamapps/postConfigure/helpers.cpp b/oamapps/postConfigure/helpers.cpp index 4378cb8df..f7d418144 100644 --- a/oamapps/postConfigure/helpers.cpp +++ b/oamapps/postConfigure/helpers.cpp @@ -77,7 +77,6 @@ void callFree(const char* ) } - bool waitForActive() { Oam oam; @@ -536,7 +535,6 @@ int sendReplicationRequest(int IserverTypeInstall, std::string password, bool pm } else { - cout << endl << "ERROR: Module not Active, replication not done on " << (*pt).DeviceName << endl; pt++; } } diff --git a/primitives/linux-port/column.cpp b/primitives/linux-port/column.cpp index 879859384..f31273b1c 100644 --- a/primitives/linux-port/column.cpp +++ b/primitives/linux-port/column.cpp @@ -290,6 +290,7 @@ inline bool isEmptyVal<8>(uint8_t type, const uint8_t* ival) case CalpontSystemCatalog::VARCHAR: case CalpontSystemCatalog::DATE: case CalpontSystemCatalog::DATETIME: + case CalpontSystemCatalog::TIME: case CalpontSystemCatalog::VARBINARY: case CalpontSystemCatalog::BLOB: case CalpontSystemCatalog::TEXT: @@ -322,6 +323,7 @@ inline bool isEmptyVal<4>(uint8_t type, const uint8_t* ival) case CalpontSystemCatalog::TEXT: case CalpontSystemCatalog::DATE: case CalpontSystemCatalog::DATETIME: + case CalpontSystemCatalog::TIME: return (joblist::CHAR4EMPTYROW == *val); case CalpontSystemCatalog::UINT: @@ -347,6 +349,7 @@ inline bool isEmptyVal<2>(uint8_t type, const uint8_t* ival) case CalpontSystemCatalog::TEXT: case CalpontSystemCatalog::DATE: case CalpontSystemCatalog::DATETIME: + case CalpontSystemCatalog::TIME: return (joblist::CHAR2EMPTYROW == *val); case CalpontSystemCatalog::USMALLINT: @@ -372,6 +375,7 @@ inline bool isEmptyVal<1>(uint8_t type, const uint8_t* ival) case CalpontSystemCatalog::TEXT: case CalpontSystemCatalog::DATE: case CalpontSystemCatalog::DATETIME: + case CalpontSystemCatalog::TIME: return (*val == joblist::CHAR1EMPTYROW); case CalpontSystemCatalog::UTINYINT: @@ -402,6 +406,7 @@ inline bool isNullVal<8>(uint8_t type, const uint8_t* ival) case CalpontSystemCatalog::VARCHAR: case CalpontSystemCatalog::DATE: case CalpontSystemCatalog::DATETIME: + case CalpontSystemCatalog::TIME: case CalpontSystemCatalog::VARBINARY: case CalpontSystemCatalog::BLOB: case CalpontSystemCatalog::TEXT: @@ -438,6 +443,7 @@ inline bool isNullVal<4>(uint8_t type, const uint8_t* ival) case CalpontSystemCatalog::DATE: case CalpontSystemCatalog::DATETIME: + case CalpontSystemCatalog::TIME: return (joblist::DATENULL == *val); case CalpontSystemCatalog::UINT: @@ -463,6 +469,7 @@ inline bool isNullVal<2>(uint8_t type, const uint8_t* ival) case CalpontSystemCatalog::TEXT: case CalpontSystemCatalog::DATE: case CalpontSystemCatalog::DATETIME: + case CalpontSystemCatalog::TIME: return (joblist::CHAR2NULL == *val); case CalpontSystemCatalog::USMALLINT: @@ -488,6 +495,7 @@ inline bool isNullVal<1>(uint8_t type, const uint8_t* ival) case CalpontSystemCatalog::TEXT: case CalpontSystemCatalog::DATE: case CalpontSystemCatalog::DATETIME: + case CalpontSystemCatalog::TIME: return (*val == joblist::CHAR1NULL); case CalpontSystemCatalog::UTINYINT: @@ -548,6 +556,7 @@ inline bool isMinMaxValid(const NewColRequestHeader* in) case CalpontSystemCatalog::DATE: case CalpontSystemCatalog::BIGINT: case CalpontSystemCatalog::DATETIME: + case CalpontSystemCatalog::TIME: case CalpontSystemCatalog::UTINYINT: case CalpontSystemCatalog::USMALLINT: case CalpontSystemCatalog::UINT: diff --git a/primitives/primproc/columncommand.cpp b/primitives/primproc/columncommand.cpp index 19e5e9a1f..ed4b26a44 100644 --- a/primitives/primproc/columncommand.cpp +++ b/primitives/primproc/columncommand.cpp @@ -980,6 +980,7 @@ const uint64_t ColumnCommand::getEmptyRowValue( const execplan::CalpontSystemCat case execplan::CalpontSystemCatalog::VARCHAR : case execplan::CalpontSystemCatalog::DATE : case execplan::CalpontSystemCatalog::DATETIME : + case execplan::CalpontSystemCatalog::TIME : case execplan::CalpontSystemCatalog::VARBINARY : case execplan::CalpontSystemCatalog::BLOB : case execplan::CalpontSystemCatalog::TEXT : diff --git a/procmgr/main.cpp b/procmgr/main.cpp index a4dc6879f..44f39ad9d 100644 --- a/procmgr/main.cpp +++ b/procmgr/main.cpp @@ -1961,7 +1961,7 @@ void pingDeviceThread() DeviceNetworkConfig devicenetworkconfig; devicenetworkconfig.DeviceName = moduleName; devicenetworklist.push_back(devicenetworkconfig); - processManager.setMySQLReplication(devicenetworklist, oam::UnassignedName, false, true); + processManager.setMySQLReplication(devicenetworklist, oam::UnassignedName, true); } } diff --git a/procmgr/processmanager.cpp b/procmgr/processmanager.cpp index 1fb6fb9f4..bdb8c5941 100644 --- a/procmgr/processmanager.cpp +++ b/procmgr/processmanager.cpp @@ -2817,7 +2817,7 @@ void processMSG(messageqcpp::IOSocket* cfIos) // target = root password oam::DeviceNetworkList devicenetworklist; - status = processManager.setMySQLReplication(devicenetworklist, oam::UnassignedName, false, true, target); + status = processManager.setMySQLReplication(devicenetworklist, oam::UnassignedName, true, target); log.writeLog(__LINE__, "Enable MySQL Replication status: " + oam.itoa(status) ); @@ -2841,7 +2841,7 @@ void processMSG(messageqcpp::IOSocket* cfIos) // target = root password oam::DeviceNetworkList devicenetworklist; - status = processManager.setMySQLReplication(devicenetworklist, oam::UnassignedName, false, false, target, false); + status = processManager.setMySQLReplication(devicenetworklist, oam::UnassignedName, false, target, false); log.writeLog(__LINE__, "Disable MySQL Replication status: " + oam.itoa(status) ); @@ -3726,12 +3726,12 @@ int ProcessManager::disableModule(string target, bool manualFlag) * purpose: recyle process, generally after some disable module is run * ******************************************************************************************/ -void ProcessManager::recycleProcess(string module) +void ProcessManager::recycleProcess(string module, bool enableModule) { Oam oam; ModuleConfig moduleconfig; - log.writeLog(__LINE__, "recycleProcess request after module was disabled: " + module, LOG_TYPE_DEBUG); + log.writeLog(__LINE__, "recycleProcess request after module status update: " + module, LOG_TYPE_DEBUG); string moduleType = module.substr(0, MAX_MODULE_TYPE_SIZE); @@ -3743,6 +3743,17 @@ void ProcessManager::recycleProcess(string module) } catch (...) {} + // restart DBRM Process and DMLProc and return if enable module is being done + if (enableModule) + { + //recycle DBRM processes in all cases + restartProcessType("DBRMControllerNode"); + restartProcessType("DBRMWorkerNode"); + + restartProcessType("DMLProc"); + return; + } + //recycle DBRM processes in all cases restartProcessType("DBRMControllerNode", module); restartProcessType("DBRMWorkerNode"); @@ -3827,6 +3838,9 @@ int ProcessManager::enableModule(string target, int state) if ( newStandbyModule == target) setStandbyModule(newStandbyModule); + //set recycle process + recycleProcess(target); + log.writeLog(__LINE__, "enableModule request for " + target + " completed", LOG_TYPE_DEBUG); return API_SUCCESS; @@ -3971,20 +3985,23 @@ int ProcessManager::startProcess(string moduleName, string processName, { Oam oam; - //skip if module is DISABLED - int opState; - bool degraded; - - try + if ( actionIndicator != oam::STATUS_UPDATE ) { - oam.getModuleStatus(moduleName, opState, degraded); - } - catch (...) - {} + //skip if module is DISABLED + int opState; + bool degraded; - //check if disabled - if (opState == oam::MAN_DISABLED || opState == oam::AUTO_DISABLED) - return API_SUCCESS; + try + { + oam.getModuleStatus(moduleName, opState, degraded); + } + catch (...) + {} + + //check if disabled + if (opState == oam::MAN_DISABLED || opState == oam::AUTO_DISABLED) + return API_SUCCESS; + } ByteStream msg; ByteStream::byte requestID = START; @@ -4942,9 +4959,9 @@ int ProcessManager::addModule(oam::DeviceNetworkList devicenetworklist, std::str } if ( packageType == "rpm") - calpontPackage = homedir + "/mariadb-columnstore*" + columnstore_version + "-" + columnstore_release + "*.rpm.tar.gz"; + calpontPackage = homedir + "/mariadb-columnstore*" + columnstore_version + "-" + columnstore_release + "*.rpm"; else if ( packageType == "deb") - calpontPackage = homedir + "/mariadb-columnstore*" + columnstore_version + "-" + columnstore_release + "*.deb.tar.gz"; + calpontPackage = homedir + "/mariadb-columnstore*" + columnstore_version + "-" + columnstore_release + "*.deb"; else calpontPackage = homedir + "/mariadb-columnstore*" + columnstore_version + "-" + columnstore_release + "*.bin.tar.gz"; @@ -5937,8 +5954,24 @@ int ProcessManager::addModule(oam::DeviceNetworkList devicenetworklist, std::str log.writeLog(__LINE__, "addModule - sleep 60 - give ProcMon time to CONFIGURE and restart", LOG_TYPE_DEBUG); sleep(60); + //start mysqld on the new modules so mysql replication can be setup + listPT = devicenetworklist.begin(); + + for ( ; listPT != devicenetworklist.end() ; listPT++) + { + processManager.startProcess((*listPT).DeviceName, "mysqld", oam::STATUS_UPDATE); + } + log.writeLog(__LINE__, "Setup MySQL Replication for new Modules being Added", LOG_TYPE_DEBUG); - processManager.setMySQLReplication(devicenetworklist, oam::UnassignedName, false, true, password ); + processManager.setMySQLReplication(devicenetworklist, oam::UnassignedName, true, password, true, true ); + + //stop mysqld + listPT = devicenetworklist.begin(); + + for ( ; listPT != devicenetworklist.end() ; listPT++) + { + processManager.stopProcess((*listPT).DeviceName, "mysqld", oam::FORCEFUL, true ); + } return API_SUCCESS; } @@ -9515,7 +9548,7 @@ int ProcessManager::switchParentOAMModule(std::string newActiveModuleName) //change master MySQL Replication setup log.writeLog(__LINE__, "Setup MySQL Replication for new Parent Module during switch-over", LOG_TYPE_DEBUG); oam::DeviceNetworkList devicenetworklist; - processManager.setMySQLReplication(devicenetworklist, newActiveModuleName, false, false, oam::UnassignedName); + processManager.setMySQLReplication(devicenetworklist, newActiveModuleName, false, oam::UnassignedName); } catch (exception& ex) @@ -10396,7 +10429,7 @@ int ProcessManager::OAMParentModuleChange() //change master MySQL Replication setup log.writeLog(__LINE__, "Setup this node as MySQL Replication Master", LOG_TYPE_DEBUG); oam::DeviceNetworkList devicenetworklist; - processManager.setMySQLReplication(devicenetworklist, config.moduleName(), true); + processManager.setMySQLReplication(devicenetworklist, config.moduleName()); } //set query system state not ready @@ -11112,7 +11145,7 @@ void ProcessManager::flushInodeCache() * * ******************************************************************************************/ -int ProcessManager::setMySQLReplication(oam::DeviceNetworkList devicenetworklist, std::string masterModule, bool failover, bool distributeDB, std::string password, bool enable) +int ProcessManager::setMySQLReplication(oam::DeviceNetworkList devicenetworklist, std::string masterModule, bool distributeDB, std::string password, bool enable, bool addModule) { Oam oam; @@ -11135,57 +11168,6 @@ int ProcessManager::setMySQLReplication(oam::DeviceNetworkList devicenetworklist return oam::API_SUCCESS; } - //also skip if single-server, multi-node seperate with 1 UM, multi-node combo with 1 PM - - /* string SingleServerInstall = "n"; - try { - oam.getSystemConfig("SingleServerInstall", SingleServerInstall); - } - catch(...) { - SingleServerInstall = "n"; - } - - //single server check - if ( SingleServerInstall == "y" ) - { - log.writeLog(__LINE__, "setMySQLReplication: Single-Server, exiting", LOG_TYPE_DEBUG); - return oam::API_SUCCESS; - } - - //combined system check - if ( config.ServerInstallType() == oam::INSTALL_COMBINE_DM_UM_PM && !failover ) { - try { - Config* sysConfig = Config::makeConfig(); - if ( sysConfig->getConfig("DBRM_Controller", "NumWorkers") == "1" ) { - log.writeLog(__LINE__, "setMySQLReplication: 1 configured module, exiting", LOG_TYPE_DEBUG); - return oam::API_SUCCESS; - } - } - catch(...) - { - log.writeLog(__LINE__, "setMySQLReplication: makeConfig exception, exiting", LOG_TYPE_DEBUG); - return oam::API_SUCCESS; - } - } - - //seperate system check - if ( ( config.ServerInstallType() != oam::INSTALL_COMBINE_DM_UM_PM ) && - (PMwithUM == "n" ) ) - { - ModuleTypeConfig moduletypeconfig; - try{ - oam.getSystemConfig("um", moduletypeconfig); - } - catch(...) - {} - - if ( moduletypeconfig.ModuleCount < 2 ) - { - log.writeLog(__LINE__, "setMySQLReplication: moduleCount = 1, exiting", LOG_TYPE_DEBUG); - return oam::API_SUCCESS; - } - } - */ log.writeLog(__LINE__, "Setup MySQL Replication", LOG_TYPE_DEBUG); //get master info @@ -11244,19 +11226,22 @@ int ProcessManager::setMySQLReplication(oam::DeviceNetworkList devicenetworklist if ( remoteModuleName == masterModule ) continue; - // skip disabled modules - int opState = oam::ACTIVE; - bool degraded; - - try + if ( !addModule ) { - oam.getModuleStatus(remoteModuleName, opState, degraded); - } - catch (...) - {} + // skip disabled modules + int opState = oam::ACTIVE; + bool degraded; - if (opState == oam::MAN_DISABLED || opState == oam::AUTO_DISABLED) - continue; + try + { + oam.getModuleStatus(remoteModuleName, opState, degraded); + } + catch (...) + {} + + if (opState == oam::MAN_DISABLED || opState == oam::AUTO_DISABLED) + continue; + } // don't do PMs unless PMwithUM flag is set if ( config.ServerInstallType() != oam::INSTALL_COMBINE_DM_UM_PM ) @@ -11344,19 +11329,22 @@ int ProcessManager::setMySQLReplication(oam::DeviceNetworkList devicenetworklist if ( remoteModuleName == masterModule ) continue; - // skip disabled modules - int opState = oam::ACTIVE; - bool degraded; - - try + if ( !addModule ) { - oam.getModuleStatus(remoteModuleName, opState, degraded); - } - catch (...) - {} + // skip disabled modules + int opState = oam::ACTIVE; + bool degraded; - if (opState == oam::MAN_DISABLED || opState == oam::AUTO_DISABLED) - continue; + try + { + oam.getModuleStatus(remoteModuleName, opState, degraded); + } + catch (...) + {} + + if (opState == oam::MAN_DISABLED || opState == oam::AUTO_DISABLED) + continue; + } // don't do PMs unless PMwithUM flag is set if ( config.ServerInstallType() != oam::INSTALL_COMBINE_DM_UM_PM ) @@ -11412,19 +11400,22 @@ int ProcessManager::setMySQLReplication(oam::DeviceNetworkList devicenetworklist if ( remoteModuleName == masterModule ) continue; - // skip disabled modules - int opState = oam::ACTIVE; - bool degraded; - - try + if ( !addModule ) { - oam.getModuleStatus(remoteModuleName, opState, degraded); - } - catch (...) - {} + // skip disabled modules + int opState = oam::ACTIVE; + bool degraded; - if (opState == oam::MAN_DISABLED || opState == oam::AUTO_DISABLED) - continue; + try + { + oam.getModuleStatus(remoteModuleName, opState, degraded); + } + catch (...) + {} + + if (opState == oam::MAN_DISABLED || opState == oam::AUTO_DISABLED) + continue; + } log.writeLog(__LINE__, "Setup Slave MySQL Replication on " + remoteModuleName, LOG_TYPE_DEBUG); diff --git a/procmgr/processmanager.h b/procmgr/processmanager.h index 61233ce28..fd2fe834e 100644 --- a/procmgr/processmanager.h +++ b/procmgr/processmanager.h @@ -304,7 +304,7 @@ public: /** *@brief recycle Processes */ - void recycleProcess(std::string module); + void recycleProcess(std::string module, bool enableModule = false); /** *@brief Enable a specified module @@ -547,7 +547,7 @@ public: /** @brief Set MySQL Replication */ - int setMySQLReplication(oam::DeviceNetworkList devicenetworklist, std::string masterModule = oam::UnassignedName, bool failover = false, bool distributeDB = false, std::string password = oam::UnassignedName, bool enable = true); + int setMySQLReplication(oam::DeviceNetworkList devicenetworklist, std::string masterModule = oam::UnassignedName, bool distributeDB = false, std::string password = oam::UnassignedName, bool enable = true, bool addModule = false); /** @brief Gluster Assign dbroot to a module */ diff --git a/procmon/processmonitor.cpp b/procmon/processmonitor.cpp index 16ca6b6e9..6efb0105c 100644 --- a/procmon/processmonitor.cpp +++ b/procmon/processmonitor.cpp @@ -484,6 +484,26 @@ void ProcessMonitor::processMessage(messageqcpp::ByteStream msg, messageqcpp::IO log.writeLog(__LINE__, "MSG RECEIVED: Stop process request on " + processName); int requestStatus = API_SUCCESS; + // check for mysql + if ( processName == "mysqld" ) + { + try + { + oam.actionMysqlCalpont(MYSQL_STOP); + } + catch (...) + {} + + ackMsg << (ByteStream::byte) ACK; + ackMsg << (ByteStream::byte) STOP; + ackMsg << (ByteStream::byte) API_SUCCESS; + mq.write(ackMsg); + + log.writeLog(__LINE__, "STOP: ACK back to ProcMgr, return status = " + oam.itoa((int) API_SUCCESS)); + + break; + } + processList::iterator listPtr; processList* aPtr = config.monitoredListPtr(); listPtr = aPtr->begin(); @@ -533,6 +553,26 @@ void ProcessMonitor::processMessage(messageqcpp::ByteStream msg, messageqcpp::IO msg >> manualFlag; log.writeLog(__LINE__, "MSG RECEIVED: Start process request on: " + processName); + // check for mysql + if ( processName == "mysqld" ) + { + try + { + oam.actionMysqlCalpont(MYSQL_START); + } + catch (...) + {} + + ackMsg << (ByteStream::byte) ACK; + ackMsg << (ByteStream::byte) START; + ackMsg << (ByteStream::byte) API_SUCCESS; + mq.write(ackMsg); + + log.writeLog(__LINE__, "START: ACK back to ProcMgr, return status = " + oam.itoa((int) API_SUCCESS)); + + break; + } + ProcessConfig processconfig; ProcessStatus processstatus; @@ -645,7 +685,7 @@ void ProcessMonitor::processMessage(messageqcpp::ByteStream msg, messageqcpp::IO int requestStatus = API_SUCCESS; // check for mysql restart - if ( processName == "mysql" ) + if ( processName == "mysqld" ) { try { @@ -2658,20 +2698,6 @@ pid_t ProcessMonitor::startProcess(string processModuleType, string processName, return oam::API_MINOR_FAILURE; } - if (processLocation.find("DecomSvr") != string::npos) - { - // DecomSvr app is special - - sleep(1); - //record the process information into processList - config.buildList(processModuleType, processName, processLocation, arg_list, - launchID, newProcessID, oam::ACTIVE, BootLaunch, RunType, - DepProcessName, DepModuleName, LogFile); - - //Update Process Status: Mark Process oam::ACTIVE state - updateProcessInfo(processName, oam::ACTIVE, newProcessID); - } - //FYI - NEEDS TO STAY HERE TO HAVE newProcessID //record the process information into processList @@ -2725,14 +2751,6 @@ pid_t ProcessMonitor::startProcess(string processModuleType, string processName, close(i); } - // open STDIN, STDOUT & STDERR for trapDaemon and DecomSvr - if (processName == "DecomSvr" ) - { - open("/dev/null", O_RDONLY); //Should be fd 0 - open("/dev/null", O_WRONLY); //Should be fd 1 - open("/dev/null", O_WRONLY); //Should be fd 2 - } - else { int fd; fd = open("/dev/null", O_RDONLY); @@ -5312,12 +5330,11 @@ int ProcessMonitor::runMasterRep(std::string& masterLogFile, std::string& master //skip if module is not ACTIVE - int opState = oam::ACTIVE; - bool degraded; - oam.getModuleStatus(moduleName, opState, degraded); - - if (opState != oam::ACTIVE) - continue; +// int opState = oam::ACTIVE; +// bool degraded; +// oam.getModuleStatus(moduleName, opState, degraded); +// if (opState != oam::ACTIVE) +// continue; bool passwordError = false; diff --git a/tools/pingproc/pingproc.cpp b/tools/pingproc/pingproc.cpp index cb1a7ab11..b40eec905 100644 --- a/tools/pingproc/pingproc.cpp +++ b/tools/pingproc/pingproc.cpp @@ -308,6 +308,7 @@ bool OidOperation::isIntegralDataType() DataType() == CalpontSystemCatalog::DATE || DataType() == CalpontSystemCatalog::BIGINT || DataType() == CalpontSystemCatalog::DATETIME || + DataType() == CalpontSystemCatalog::TIME || DataType() == CalpontSystemCatalog::UTINYINT || DataType() == CalpontSystemCatalog::USMALLINT || DataType() == CalpontSystemCatalog::UMEDINT || diff --git a/tools/qfe/bison-win.cpp b/tools/qfe/bison-win.cpp index 7291f6eca..2c1e94a73 100644 --- a/tools/qfe/bison-win.cpp +++ b/tools/qfe/bison-win.cpp @@ -554,9 +554,9 @@ static const yytype_uint8 yydefact[] = static const yytype_int8 yydefgoto[] = { -1, 2, 6, 7, 8, 9, 18, 19, 23, 24, - 25, 33, 36, 10, 11, 52, 39, 45, 46, 48, - 61, 56 -}; + 25, 33, 36, 10, 11, 52, 39, 45, 46, 48, + 61, 56 + }; /* YYPACT[STATE-NUM] -- Index in YYTABLE of the portion describing STATE-NUM. */ @@ -576,9 +576,9 @@ static const yytype_int8 yypact[] = static const yytype_int8 yypgoto[] = { -20, -20, -20, 37, -20, 32, 36, -20, -20, -20, - -20, -20, -20, -15, -19, -10, -20, 6, -1, -20, - -20, -20 -}; + -20, -20, -20, -15, -19, -10, -20, 6, -1, -20, + -20, -20 + }; /* YYTABLE[YYPACT[STATE-NUM]]. What to do in state STATE-NUM. If positive, shift that token. If negative, reduce the rule which diff --git a/utils/clusterTester/columnstoreClusterTester.sh b/utils/clusterTester/columnstoreClusterTester.sh index 9ee4797ec..de092ca0e 100755 --- a/utils/clusterTester/columnstoreClusterTester.sh +++ b/utils/clusterTester/columnstoreClusterTester.sh @@ -736,7 +736,8 @@ checkPackages() echo "** Run MariaDB ColumnStore Dependent Package Check" echo "" - declare -a CENTOS_PKG=("expect" "perl" "perl-DBI" "openssl" "zlib" "file" "libaio" "rsync" "snappy" "net-tools") + + declare -a CENTOS_PKG=("expect" "perl" "perl-DBI" "openssl" "zlib" "file" "sudo" "libaio" "rsync" "snappy" "net-tools" "numactl-libs") declare -a CENTOS_PKG_NOT=("mariadb-libs") if [ "$OS" == "centos6" ] || [ "$OS" == "centos7" ]; then @@ -855,7 +856,8 @@ checkPackages() fi fi - declare -a SUSE_PKG=("boost-devel" "expect" "perl" "perl-DBI" "openssl" "file" "libaio1" "rsync" "libsnappy1" "net-tools") + + declare -a SUSE_PKG=("boost-devel" "expect" "perl" "perl-DBI" "openssl" "file" "sudo" "libaio1" "rsync" "libsnappy1" "net-tools" "libnuma1") declare -a SUSE_PKG_NOT=("mariadb" , "libmariadb18") if [ "$OS" == "suse12" ]; then @@ -946,7 +948,7 @@ checkPackages() fi fi - declare -a UBUNTU_PKG=("libboost-all-dev" "expect" "libdbi-perl" "perl" "openssl" "file" "libreadline-dev" "rsync" "libsnappy1V5" "net-tools") + declare -a UBUNTU_PKG=("libboost-all-dev" "expect" "libdbi-perl" "perl" "openssl" "file" "sudo" "libreadline-dev" "rsync" "libsnappy1V5" "net-tools" "libnuma1" ) declare -a UBUNTU_PKG_NOT=("mariadb-server" "libmariadb18") if [ "$OS" == "ubuntu16" ] ; then @@ -1063,7 +1065,8 @@ checkPackages() fi fi - declare -a DEBIAN_PKG=("libboost-all-dev" "expect" "libdbi-perl" "perl" "openssl" "file" "libreadline-dev" "rsync" "libsnappy1" "net-tools") + + declare -a DEBIAN_PKG=("libboost-all-dev" "expect" "libdbi-perl" "perl" "openssl" "file" "sudo" "libreadline-dev" "rsync" "libsnappy1" "net-tools" "libnuma1") declare -a DEBIAN_PKG_NOT=("libmariadb18" "mariadb-server") if [ "$OS" == "debian8" ]; then @@ -1180,7 +1183,8 @@ checkPackages() fi fi - declare -a DEBIAN9_PKG=("libboost-all-dev" "expect" "libdbi-perl" "perl" "openssl" "file" "libreadline5" "rsync" "libsnappy1V5" "net-tools" "libaio1") + + declare -a DEBIAN9_PKG=("libboost-all-dev" "expect" "libdbi-perl" "perl" "openssl" "file" "sudo" "libreadline5" "rsync" "libsnappy1V5" "net-tools" "libaio1" "libnuma1") declare -a DEBIAN9_PKG_NOT=("libmariadb18" "mariadb-server") if [ "$OS" == "debian9" ]; then diff --git a/utils/common/any.hpp b/utils/common/any.hpp index 5265015f1..be0ca679b 100755 --- a/utils/common/any.hpp +++ b/utils/common/any.hpp @@ -54,15 +54,25 @@ namespace anyimpl template struct big_any_policy : typed_base_any_policy { - virtual void static_delete(void** x) { if (*x) - delete(*reinterpret_cast(x)); *x = NULL; } - virtual void copy_from_value(void const* src, void** dest) { - *dest = new T(*reinterpret_cast(src)); } - virtual void clone(void* const* src, void** dest) { - *dest = new T(**reinterpret_cast(src)); } - virtual void move(void* const* src, void** dest) { - (*reinterpret_cast(dest))->~T(); - **reinterpret_cast(dest) = **reinterpret_cast(src); } + virtual void static_delete(void** x) + { + if (*x) + delete(*reinterpret_cast(x)); + *x = NULL; + } + virtual void copy_from_value(void const* src, void** dest) + { + *dest = new T(*reinterpret_cast(src)); + } + virtual void clone(void* const* src, void** dest) + { + *dest = new T(**reinterpret_cast(src)); + } + virtual void move(void* const* src, void** dest) + { + (*reinterpret_cast(dest))->~T(); + **reinterpret_cast(dest) = **reinterpret_cast(src); + } virtual void* get_value(void** src) { return *src; } }; diff --git a/utils/common/cgroupconfigurator.cpp b/utils/common/cgroupconfigurator.cpp index a5c736027..2be36b3a8 100644 --- a/utils/common/cgroupconfigurator.cpp +++ b/utils/common/cgroupconfigurator.cpp @@ -19,6 +19,7 @@ #include "configcpp.h" #include "logger.h" #include +#include #include #ifdef _MSC_VER #include "unistd.h" diff --git a/utils/common/nullvaluemanip.cpp b/utils/common/nullvaluemanip.cpp index 4eb96f77c..475f495cc 100644 --- a/utils/common/nullvaluemanip.cpp +++ b/utils/common/nullvaluemanip.cpp @@ -56,6 +56,9 @@ uint64_t getNullValue(CalpontSystemCatalog::ColDataType t, uint32_t colWidth) case CalpontSystemCatalog::DATETIME: return joblist::DATETIMENULL; + case CalpontSystemCatalog::TIME: + return joblist::TIMENULL; + case CalpontSystemCatalog::CHAR: case CalpontSystemCatalog::VARCHAR: case CalpontSystemCatalog::STRINT: @@ -163,6 +166,9 @@ int64_t getSignedNullValue(CalpontSystemCatalog::ColDataType t, uint32_t colWidt case CalpontSystemCatalog::DATETIME: return joblist::DATETIMENULL; + case CalpontSystemCatalog::TIME: + return joblist::TIMENULL; + case CalpontSystemCatalog::CHAR: case CalpontSystemCatalog::VARCHAR: case CalpontSystemCatalog::STRINT: diff --git a/utils/dataconvert/dataconvert.cpp b/utils/dataconvert/dataconvert.cpp index d0689fb0d..657a8d0b1 100644 --- a/utils/dataconvert/dataconvert.cpp +++ b/utils/dataconvert/dataconvert.cpp @@ -859,6 +859,180 @@ bool mysql_str_to_datetime( const string& input, DateTime& output, bool& isDate return true; } +bool mysql_str_to_time( const string& input, Time& output ) +{ + int32_t datesepct = 0; + uint32_t dtend = 0; + bool isNeg = false; + + /** + * We need to deal with the time portion. + * The rules are: + * - Time portion always ends with '\0' + * - Time portion always starts with hour + * - Without time separators (':'): + * HHMMSS + * - All Times can end with option .[microseconds] + * - With time separators there are no specific field length + * requirements + */ + while ( input[dtend] == ' ' && dtend < input.length() ) + { + ++dtend; + } + + if ( dtend == input.length() ) + { + return false; + } + + uint32_t timesep_ct = 0; + bool has_usec = false; + uint32_t len_before_msec = 0; + uint32_t tmstart = dtend; + uint32_t tmend = tmstart; + + for ( ; tmend < input.length(); ++tmend ) + { + char c = input[tmend]; + + if ( isdigit( c ) ) + { + // digits always ok + continue; + } +// else if( c == ':' ) +// { +// timesep_ct++; +// } +// else if( c == '.' ) +// { +// len_before_msec = ( tmend - tmstart ); +// has_usec = true; +// } + else if ( ispunct(c) ) + { + if ( c == '.' && timesep_ct == 2 ) + { + len_before_msec = ( tmend - tmstart ); + has_usec = true; + } + else if (c == '-' && (tmend == tmstart)) + { + isNeg = true; + ++tmstart; + } + else + { + timesep_ct++; + } + } + else + { + // some other character showed up + output.reset(); + return false; + } + } + + if ( !len_before_msec ) + len_before_msec = ( tmend - tmstart ); + + int32_t hour = -1; + int32_t min = 0; + int32_t sec = 0; + int32_t usec = 0; + const char* tstart = input.c_str() + tmstart; + + if ( timesep_ct == 2 ) + { + readDecimal(tstart, hour); + ++tstart; // skip one separator + readDecimal(tstart, min); + ++tstart; // skip one separator + readDecimal(tstart, sec); + } + else if ( timesep_ct == 1 ) + { + readDecimal(tstart, hour); + ++tstart; // skip one separator + readDecimal(tstart, min); + } + else if ( timesep_ct == 0 && len_before_msec == 6 ) + { + readDecimal(tstart, hour, 2); + readDecimal(tstart, min, 2); + readDecimal(tstart, sec, 2); + } + else if ( timesep_ct == 0 && len_before_msec == 4 ) + { + readDecimal(tstart, hour, 2); + readDecimal(tstart, min, 2); + } + else if ( timesep_ct == 0 && len_before_msec == 2 ) + { + readDecimal(tstart, hour, 2); + } + else + { + output.reset(); + return false; + } + + if ( has_usec ) + { + ++tstart; // skip '.' character. We could error check if we wanted to + uint32_t numread = readDecimal(tstart, usec); + + if ( numread > 6 || numread < 1 ) + { + // don't allow more than 6 digits when specifying microseconds + output.reset(); + return false; + } + + // usec have to be scaled up so that it always represents microseconds + for ( int i = numread; i < 6; i++ ) + usec *= 10; + } + + if ( !isTimeValid( hour, min, sec, usec ) ) + { + // Emulate MariaDB's time saturation + if (hour > 838) + { + output.hour = 838; + output.minute = 59; + output.second = 59; + output.msecond = 999999; + output.is_neg = 0; + } + else if (hour < -838) + { + output.hour = -838; + output.minute = 59; + output.second = 59; + output.msecond = 999999; + output.is_neg = 1; + } + // If neither of the above match then we return a 0 time + else + { + output.reset(); + } + + return false; + } + + output.hour = isNeg ? 0 - hour : hour; + output.minute = min; + output.second = sec; + output.msecond = usec; + output.is_neg = isNeg; + return true; +} + + bool stringToDateStruct( const string& data, Date& date ) { bool isDate; @@ -894,6 +1068,14 @@ bool stringToDatetimeStruct(const string& data, DateTime& dtime, bool* date) return true; } +bool stringToTimeStruct(const string& data, Time& dtime) +{ + if ( !mysql_str_to_time( data, dtime ) ) + return false; + + return true; +} + boost::any DataConvert::convertColumnData(const CalpontSystemCatalog::ColType& colType, const std::string& dataOrig, bool& pushWarning, bool nulFlag, bool noRoundup, bool isUpdate ) @@ -1229,6 +1411,23 @@ DataConvert::convertColumnData(const CalpontSystemCatalog::ColType& colType, } break; + case CalpontSystemCatalog::TIME: + { + Time aTime; + + if (stringToTimeStruct(data, aTime)) + { + value = (int64_t) * (reinterpret_cast(&aTime)); + } + else + { + value = (int64_t) 0; + pushWarning = true; + } + } + break; + + case CalpontSystemCatalog::BLOB: case CalpontSystemCatalog::CLOB: value = data; @@ -1345,6 +1544,13 @@ DataConvert::convertColumnData(const CalpontSystemCatalog::ColType& colType, } break; + case CalpontSystemCatalog::TIME: + { + uint64_t d = joblist::TIMENULL; + value = d; + } + break; + case CalpontSystemCatalog::CHAR: { std::string charnull; @@ -1692,6 +1898,140 @@ int64_t DataConvert::convertColumnDatetime( return value; } +//------------------------------------------------------------------------------ +// Convert time string to binary time. Used by BulkLoad. +// Most of this is taken from str_to_time in sql-common/my_time.c +//------------------------------------------------------------------------------ +int64_t DataConvert::convertColumnTime( + const char* dataOrg, + CalpontDateTimeFormat datetimeFormat, + int& status, + unsigned int dataOrgLen ) +{ + status = 0; + char* p; + char* savePoint = NULL; + p = const_cast(dataOrg); + int64_t value = 0; + int inHour, inMinute, inSecond, inMicrosecond; + inHour = 0; + inMinute = 0; + inSecond = 0; + inMicrosecond = 0; + bool isNeg = false; + + if ( datetimeFormat != CALPONTTIME_ENUM ) + { + status = -1; + return value; + } + + if (p[0] == '-') + { + isNeg = true; + } + + errno = 0; + + p = strtok_r(p, ":.", &savePoint); + inHour = strtol(p, 0, 10); + + if (errno) + { + status = -1; + return value; + } + + p = strtok_r(NULL, ":.", &savePoint); + + if (p == NULL) + { + status = -1; + return value; + } + + inMinute = strtol(p, 0, 10); + + if (errno) + { + status = -1; + return value; + } + + p = strtok_r(NULL, ":.", &savePoint); + + if (p == NULL) + { + status = -1; + return value; + } + + inSecond = strtol(p, 0, 10); + + if (errno) + { + status = -1; + return value; + } + + p = strtok_r(NULL, ":.", &savePoint); + + if (p != NULL) + { + inMicrosecond = strtol(p, 0, 10); + + if (errno) + { + status = -1; + return value; + } + } + + if ( isTimeValid (inHour, inMinute, inSecond, inMicrosecond) ) + { + Time atime; + atime.hour = inHour; + atime.minute = inMinute; + atime.second = inSecond; + atime.msecond = inMicrosecond; + atime.is_neg = isNeg; + + memcpy( &value, &atime, 8); + } + else + { + // Emulate MariaDB's time saturation + if (inHour > 838) + { + Time atime; + atime.hour = 838; + atime.minute = 59; + atime.second = 59; + atime.msecond = 999999; + atime.is_neg = false; + memcpy( &value, &atime, 8); + } + else if (inHour < -838) + { + Time atime; + atime.hour = -838; + atime.minute = 59; + atime.second = 59; + atime.msecond = 999999; + atime.is_neg = false; + memcpy( &value, &atime, 8); + } + + // If neither of the above match then we return a 0 time + + status = -1; + } + + return value; + +} + + //------------------------------------------------------------------------------ // Verify that specified datetime is valid //------------------------------------------------------------------------------ @@ -1706,6 +2046,14 @@ bool DataConvert::isColumnDateTimeValid( int64_t dateTime ) return false; } +bool DataConvert::isColumnTimeValid( int64_t time ) +{ + Time dt; + memcpy(&dt, &time, sizeof(uint64_t)); + + return isTimeValid(dt.hour, dt.minute, dt.second, dt.msecond); +} + std::string DataConvert::dateToString( int datevalue ) { // @bug 4703 abandon multiple ostringstream's for conversion @@ -1717,14 +2065,69 @@ std::string DataConvert::dateToString( int datevalue ) return buf; } -std::string DataConvert::datetimeToString( long long datetimevalue ) +std::string DataConvert::datetimeToString( long long datetimevalue, long decimals ) { + // 10 is default which means we don't need microseconds + if (decimals > 6 || decimals < 0) + { + decimals = 0; + } + // @bug 4703 abandon multiple ostringstream's for conversion DateTime dt(datetimevalue); - const int DATETIMETOSTRING_LEN = 21; // YYYY-MM-DD HH:MM:SS\0 + const int DATETIMETOSTRING_LEN = 28; // YYYY-MM-DD HH:MM:SS.mmmmmm\0 char buf[DATETIMETOSTRING_LEN]; sprintf(buf, "%04d-%02d-%02d %02d:%02d:%02d", dt.year, dt.month, dt.day, dt.hour, dt.minute, dt.second); + + if (dt.msecond && decimals) + { + snprintf(buf + strlen(buf), 21 + decimals, ".%d", dt.msecond); + + // Pad end with zeros + if (strlen(buf) < (size_t)(21 + decimals)) + { + sprintf(buf + strlen(buf), "%0*d", (int)(21 + decimals - strlen(buf)), 0); + } + } + + return buf; +} + +std::string DataConvert::timeToString( long long timevalue, long decimals ) +{ + // 10 is default which means we don't need microseconds + if (decimals > 6 || decimals < 0) + { + decimals = 0; + } + + // @bug 4703 abandon multiple ostringstream's for conversion + Time dt(timevalue); + const int TIMETOSTRING_LEN = 19; // (-H)HH:MM:SS.mmmmmm\0 + char buf[TIMETOSTRING_LEN]; + char* outbuf = buf; + + if ((dt.hour >= 0) && dt.is_neg) + { + outbuf[0] = '-'; + outbuf++; + } + + sprintf(outbuf, "%02d:%02d:%02d", dt.hour, dt.minute, dt.second); + + if (dt.msecond && decimals) + { + size_t start = strlen(buf); + snprintf(buf + strlen(buf), 12 + decimals, ".%d", dt.msecond); + + // Pad end with zeros + if (strlen(buf) - start < (size_t)decimals) + { + sprintf(buf + strlen(buf), "%0*d", (int)(decimals - (strlen(buf) - start) + 1), 0); + } + } + return buf; } @@ -1749,6 +2152,20 @@ std::string DataConvert::datetimeToString1( long long datetimevalue ) sprintf(buf, "%04d%02d%02d%02d%02d%02d%06d", dt.year, dt.month, dt.day, dt.hour, dt.minute, dt.second, dt.msecond); return buf; } + +std::string DataConvert::timeToString1( long long datetimevalue ) +{ + // @bug 4703 abandon multiple ostringstream's for conversion + DateTime dt(datetimevalue); + const int TIMETOSTRING1_LEN = 14; // HHMMSSmmmmmm\0 + char buf[TIMETOSTRING1_LEN]; + + char* outbuf = buf; + + sprintf(outbuf, "%02d%02d%02d%06d", dt.hour, dt.minute, dt.second, dt.msecond); + return buf; +} + #if 0 bool DataConvert::isNullData(ColumnResult* cr, int rownum, CalpontSystemCatalog::ColType colType) { @@ -1969,6 +2386,11 @@ int64_t DataConvert::datetimeToInt(const string& datetime) return stringToDatetime(datetime); } +int64_t DataConvert::timeToInt(const string& time) +{ + return stringToTime(time); +} + int64_t DataConvert::stringToDate(const string& data) { Date aDay; @@ -2242,18 +2664,104 @@ int64_t DataConvert::intToDatetime(int64_t data, bool* date) return *(reinterpret_cast(&adaytime)); } +int64_t DataConvert::intToTime(int64_t data) +{ + char buf[21] = {0}; + char* bufread = buf; + Time atime; + bool isNeg = false; + + if (data == 0) + { + atime.hour = 0; + atime.minute = 0; + atime.second = 0; + atime.msecond = 0; + atime.is_neg = 0; + + return *(reinterpret_cast(&atime)); + } + + snprintf( buf, 15, "%llu", (long long unsigned int)data); + //string date = buf; + string hour, min, sec, msec; + int64_t h = 0, minute = 0, s = 0, ms = 0; + + if (bufread[0] == '-') + { + isNeg = true; + bufread++; + } + + switch (strlen(bufread)) + { + case 7: + hour = string(bufread, 3); + min = string(bufread + 2, 2); + sec = string(bufread + 4, 2); + msec = string(bufread + 6, 6); + break; + + case 6: + hour = string(bufread, 2); + min = string(bufread + 2, 2); + sec = string(bufread + 4, 2); + msec = string(bufread + 6, 6); + break; + + case 4: + min = string(bufread, 2); + sec = string(bufread + 2, 2); + msec = string(bufread + 4, 6); + break; + + case 2: + sec = string(bufread, 2); + msec = string(bufread + 2, 6); + break; + + default: + return -1; + } + + h = atoi(hour.c_str()); + minute = atoi(min.c_str()); + s = atoi(sec.c_str()); + ms = atoi(msec.c_str()); + + if (!isTimeValid(h, minute, s, ms)) + return -1; + + atime.hour = h; + atime.minute = minute; + atime.second = s; + atime.msecond = ms; + atime.is_neg = isNeg; + + return *(reinterpret_cast(&atime)); +} + int64_t DataConvert::stringToTime(const string& data) { // MySQL supported time value format 'D HHH:MM:SS.fraction' // -34 <= D <= 34 // -838 <= H <= 838 uint64_t min = 0, sec = 0, msec = 0; - int64_t day = 0, hour = 0; + int64_t day = -1, hour = 0; + bool isNeg = false; string time, hms, ms; char* end = NULL; + + size_t pos = data.find("-"); + + if (pos != string::npos) + { + isNeg = true; + } + // Day - size_t pos = data.find(" "); + pos = data.find(" "); if (pos != string::npos) { @@ -2261,7 +2769,8 @@ int64_t DataConvert::stringToTime(const string& data) if (*end != '\0') return -1; - + hour = day * 24; + day = -1; time = data.substr(pos + 1, data.length() - pos - 1); } else @@ -2287,11 +2796,11 @@ int64_t DataConvert::stringToTime(const string& data) if (pos == string::npos) { - hour = atoi(hms.c_str()); + hour += atoi(hms.c_str()); } else { - hour = atoi(hms.substr(0, pos).c_str()); + hour += atoi(hms.substr(0, pos).c_str()); ms = hms.substr(pos + 1, hms.length() - pos - 1); } @@ -2314,6 +2823,7 @@ int64_t DataConvert::stringToTime(const string& data) atime.minute = min; atime.second = sec; atime.msecond = msec; + atime.is_neg = isNeg; return *(reinterpret_cast(&atime)); } @@ -2388,6 +2898,7 @@ CalpontSystemCatalog::ColType DataConvert::convertUnionColType(vector> 24) & 0xff), minute((val >> 32) & 0xff), hour((val >> 40) & 0xfff), - day((val >> 52) & 0xfff) + day((val >> 52) & 0x7ff), + is_neg(val >> 63) {} + + Time(signed d, signed h, signed min, signed sec, signed msec, bool neg) : + msecond(msec), second(sec), minute(min), hour(h), day(d), is_neg(neg) + { + if (h < 0) + is_neg = 0b1; + } + + int64_t convertToMySQLint() const; + void reset(); }; -static uint32_t daysInMonth[13] = {31, 28, 31, 30, 31, 30, 31, 31, 30, 31, 30, 31, 0}; - -inline uint32_t getDaysInMonth(uint32_t month) +inline +void Time::reset() { - return ( (month < 1 || month > 12) ? 0 : daysInMonth[month - 1]); + msecond = 0xFFFFFE; + second = 0xFF; + minute = 0xFF; + hour = 0xFFF; + is_neg = 0b1; + day = 0x7FF; } +inline +int64_t Time::convertToMySQLint() const +{ + if ((hour >= 0) && is_neg) + { + return (int64_t) ((hour * 10000) + (minute * 100) + second) * -1; + } + else if (hour >= 0) + { + return (int64_t) (hour * 10000) + (minute * 100) + second; + } + else + { + return (int64_t) (hour * 10000) - (minute * 100) - second; + } +} + +static uint32_t daysInMonth[13] = {31, 28, 31, 30, 31, 30, 31, 31, 30, 31, 30, 31, 0}; + inline bool isLeapYear ( int year) { if ( year % 400 == 0 ) @@ -233,6 +271,19 @@ inline bool isLeapYear ( int year) return false; } +inline uint32_t getDaysInMonth(uint32_t month, int year) +{ + if (month < 1 || month > 12) + return 0; + + uint32_t days = daysInMonth[month - 1]; + + if ((month == 2) && isLeapYear(year)) + days++; + + return days; +} + inline bool isDateValid ( int day, int month, int year) { @@ -243,11 +294,7 @@ bool isDateValid ( int day, int month, int year) return true; } - int daycheck = getDaysInMonth( month ); - - if ( month == 2 && isLeapYear( year ) ) - // 29 days in February in a leap year - daycheck = 29; + int daycheck = getDaysInMonth( month, year ); if ( ( year < 1000 ) || ( year > 9999 ) ) valid = false; @@ -281,6 +328,28 @@ bool isDateTimeValid ( int hour, int minute, int second, int microSecond) return valid; } +inline +bool isTimeValid ( int hour, int minute, int second, int microSecond) +{ + bool valid = false; + + if ( hour >= -838 && hour <= 838 ) + { + if ( minute >= 0 && minute < 60 ) + { + if ( second >= 0 && second < 60 ) + { + if ( microSecond >= 0 && microSecond <= 999999 ) + { + valid = true; + } + } + } + } + + return valid; +} + inline int64_t string_to_ll( const std::string& data, bool& bSaturate ) { @@ -361,8 +430,17 @@ public: * @param type the columns database type * @param data the columns string representation of it's data */ - EXPORT static std::string datetimeToString( long long datetimevalue ); - static inline void datetimeToString( long long datetimevalue, char* buf, unsigned int buflen ); + EXPORT static std::string datetimeToString( long long datetimevalue, long decimals = 0 ); + static inline void datetimeToString( long long datetimevalue, char* buf, unsigned int buflen, long decimals = 0 ); + + /** + * @brief convert a columns data from native format to a string + * + * @param type the columns database type + * @param data the columns string representation of it's data + */ + EXPORT static std::string timeToString( long long timevalue, long decimals = 0 ); + static inline void timeToString( long long timevalue, char* buf, unsigned int buflen, long decimals = 0); /** * @brief convert a columns data from native format to a string @@ -382,6 +460,15 @@ public: EXPORT static std::string datetimeToString1( long long datetimevalue ); static inline void datetimeToString1( long long datetimevalue, char* buf, unsigned int buflen ); + /** + * @brief convert a columns data from native format to a string + * + * @param type the columns database type + * @param data the columns string representation of it's data + */ + EXPORT static std::string timeToString1( long long timevalue ); + static inline void timeToString1( long long timevalue, char* buf, unsigned int buflen ); + /** * @brief convert a date column data, represnted as a string, to it's native * format. This function is for bulkload to use. @@ -415,10 +502,25 @@ public: CalpontDateTimeFormat datetimeFormat, int& status, unsigned int dataOrgLen ); + /** + * @brief convert a time column data, represented as a string, + * to it's native format. This function is for bulkload to use. + * + * @param type the columns data type + * @param dataOrig the columns string representation of it's data + * @param timeFormat the format the time value in + * @param status 0 - success, -1 - fail + * @param dataOrgLen length specification of dataOrg + */ + EXPORT static int64_t convertColumnTime( const char* dataOrg, + CalpontDateTimeFormat datetimeFormat, + int& status, unsigned int dataOrgLen ); + /** * @brief Is specified datetime valid; used by binary bulk load */ EXPORT static bool isColumnDateTimeValid( int64_t dateTime ); + EXPORT static bool isColumnTimeValid( int64_t time ); EXPORT static bool isNullData(execplan::ColumnResult* cr, int rownum, execplan::CalpontSystemCatalog::ColType colType); static inline std::string decimalToString(int64_t value, uint8_t scale, execplan::CalpontSystemCatalog::ColDataType colDataType); @@ -438,11 +540,13 @@ public: EXPORT static int64_t intToDate(int64_t data); // convert integer to datetime EXPORT static int64_t intToDatetime(int64_t data, bool* isDate = NULL); - + // convert integer to date + EXPORT static int64_t intToTime(int64_t data); // convert string to date. alias to stringToDate EXPORT static int64_t dateToInt(const std::string& date); // convert string to datetime. alias to datetimeToInt EXPORT static int64_t datetimeToInt(const std::string& datetime); + EXPORT static int64_t timeToInt(const std::string& time); EXPORT static int64_t stringToTime (const std::string& data); // bug4388, union type conversion EXPORT static execplan::CalpontSystemCatalog::ColType convertUnionColType(std::vector&); @@ -457,8 +561,21 @@ inline void DataConvert::dateToString( int datevalue, char* buf, unsigned int bu ); } -inline void DataConvert::datetimeToString( long long datetimevalue, char* buf, unsigned int buflen ) +inline void DataConvert::datetimeToString( long long datetimevalue, char* buf, unsigned int buflen, long decimals ) { + // 10 is default which means we don't need microseconds + if (decimals > 6 || decimals < 0) + { + decimals = 0; + } + + int msec = 0; + + if ((datetimevalue & 0xfffff) > 0) + { + msec = (unsigned)((datetimevalue) & 0xfffff); + } + snprintf( buf, buflen, "%04d-%02d-%02d %02d:%02d:%02d", (unsigned)((datetimevalue >> 48) & 0xffff), (unsigned)((datetimevalue >> 44) & 0xf), @@ -467,6 +584,67 @@ inline void DataConvert::datetimeToString( long long datetimevalue, char* buf, u (unsigned)((datetimevalue >> 26) & 0x3f), (unsigned)((datetimevalue >> 20) & 0x3f) ); + + if (msec || decimals) + { + size_t start = strlen(buf); + snprintf(buf + strlen(buf), buflen - start, ".%d", msec); + + // Pad end with zeros + if (strlen(buf) - start < (size_t)decimals) + { + snprintf(buf + strlen(buf), buflen - strlen(buf), "%0*d", (int)(decimals - (strlen(buf) - start) + 1), 0); + } + } +} + +inline void DataConvert::timeToString( long long timevalue, char* buf, unsigned int buflen, long decimals ) +{ + // 10 is default which means we don't need microseconds + if (decimals > 6 || decimals < 0) + { + decimals = 0; + } + + // Handle negative correctly + int hour = 0, msec = 0; + + if ((timevalue >> 40) & 0x800) + { + hour = 0xfffff000; + } + + hour |= ((timevalue >> 40) & 0xfff); + + if ((timevalue & 0xffffff) > 0) + { + msec = (unsigned)((timevalue) & 0xffffff); + } + + if ((hour >= 0) && (timevalue >> 63)) + { + buf[0] = '-'; + buf++; + buflen--; + } + + snprintf( buf, buflen, "%02d:%02d:%02d", + hour, + (unsigned)((timevalue >> 32) & 0xff), + (unsigned)((timevalue >> 24) & 0xff) + ); + + if (msec || decimals) + { + size_t start = strlen(buf); + snprintf(buf + strlen(buf), buflen - start, ".%d", msec); + + // Pad end with zeros + if (strlen(buf) - start < (size_t)decimals) + { + snprintf(buf + strlen(buf), buflen - strlen(buf), "%0*d", (int)(decimals - (strlen(buf) - start) + 1), 0); + } + } } inline void DataConvert::dateToString1( int datevalue, char* buf, unsigned int buflen) @@ -490,6 +668,32 @@ inline void DataConvert::datetimeToString1( long long datetimevalue, char* buf, ); } +inline void DataConvert::timeToString1( long long timevalue, char* buf, unsigned int buflen ) +{ + // Handle negative correctly + int hour = 0; + + if ((timevalue >> 40) & 0x800) + { + hour = 0xfffff000; + } + + hour |= ((timevalue >> 40) & 0xfff); + + if ((hour >= 0) && (timevalue >> 63)) + { + buf[0] = '-'; + buf++; + buflen--; + } + + snprintf( buf, buflen, "%02d%02d%02d", + hour, + (unsigned)((timevalue >> 32) & 0xff), + (unsigned)((timevalue >> 14) & 0xff) + ); +} + inline std::string DataConvert::decimalToString(int64_t value, uint8_t scale, execplan::CalpontSystemCatalog::ColDataType colDataType) { char buf[80]; diff --git a/utils/funcexp/func_add_time.cpp b/utils/funcexp/func_add_time.cpp index fb4469eec..61eb9ec6c 100644 --- a/utils/funcexp/func_add_time.cpp +++ b/utils/funcexp/func_add_time.cpp @@ -80,16 +80,24 @@ int64_t addTime(DateTime& dt1, Time& dt2) } hour = (signed)(dt1.hour + dt2.hour + min / 60); - dt.hour = tmp = hour % 24; -// if (tmp < -1) - if (tmp < 0) // fix for subtime dlh + if ((hour < 0) || (hour > 23)) { - dt.hour = tmp + 24; + dt2.day = hour / 24; + hour = hour % 24; + } + if (hour < 0) + { + dt.hour = hour + 24; dt2.day--; } + else + { + dt.hour = hour; + } + + day = (signed)(dt1.day + dt2.day); - day = (signed)(dt1.day + dt2.day + hour / 24); if (isLeapYear(dt1.year) && dt1.month == 2) day--; @@ -97,7 +105,7 @@ int64_t addTime(DateTime& dt1, Time& dt2) month = dt1.month; int addyear = 0; - if (dt2.day < 0 || dt2.hour < 0) + if (day < 0) { int monthSave = month; @@ -106,7 +114,7 @@ int64_t addTime(DateTime& dt1, Time& dt2) month = (month == 1 ? 12 : month - 1); for (; day <= 0 && month > 0; month--) - day += getDaysInMonth(month); + day += getDaysInMonth(month, dt1.year); month++; // month=12; @@ -119,10 +127,10 @@ int64_t addTime(DateTime& dt1, Time& dt2) { int monthSave = month; - while (day > getDaysInMonth(month)) + while (day > getDaysInMonth(month, dt1.year)) { - for (; day > getDaysInMonth(month) && month <= 12; month++) - day -= getDaysInMonth(month); + for (; day > getDaysInMonth(month, dt1.year) && month <= 12; month++) + day -= getDaysInMonth(month, dt1.year); if (month > 12) month = 1; @@ -138,6 +146,67 @@ int64_t addTime(DateTime& dt1, Time& dt2) return *(reinterpret_cast(&dt)); } + +int64_t addTime(Time& dt1, Time& dt2) +{ + Time dt; + dt.is_neg = false; + dt.hour = 0; + dt.minute = 0; + dt.second = 0; + dt.msecond = 0; + + int64_t min, sec, msec, tmp; + msec = (signed)(dt1.msecond + dt2.msecond); + dt.msecond = tmp = msec % 1000000; + + if (tmp < 0) + { + dt.msecond = tmp + 1000000; + dt2.second--; + } + + sec = (signed)(dt1.second + dt2.second + msec / 1000000); + dt.second = tmp = sec % 60; + + if (tmp < 0) + { + dt.second = tmp + 60; + dt2.minute--; + } + + min = (signed)(dt1.minute + dt2.minute + sec / 60); + dt.minute = tmp = min % 60; + + if (tmp < 0) + { + dt.minute = tmp + 60; + dt2.hour--; + } + + dt.hour = tmp = (signed)(dt1.hour + dt2.hour + min / 60); + + // Saturation + if (tmp > 838) + { + dt.hour = 838; + dt.minute = 59; + dt.second = 59; + dt.msecond = 999999; + } + else if (tmp < -838) + { + dt.is_neg = true; + dt.hour = -838; + dt.minute = 59; + dt.second = 59; + dt.msecond = 999999; + } + + return *(reinterpret_cast(&dt)); +} + + } namespace funcexp @@ -182,6 +251,13 @@ int64_t Func_add_time::getDatetimeIntVal(rowgroup::Row& row, if (isNull) return -1; + // Adding a zero date to a time is always NULL + if (val1 == 0) + { + isNull = true; + return -1; + } + const string& val2 = parm[1]->data()->getStrVal(row, isNull); int sign = parm[2]->data()->getIntVal(row, isNull); DateTime dt1; @@ -212,26 +288,89 @@ int64_t Func_add_time::getDatetimeIntVal(rowgroup::Row& row, int val_sign = 1; - if (t2.day != 0 && t2.hour < 0) - { - isNull = true; - return -1; - } - else if (t2.day < 0 || t2.hour < 0) + if (t2.hour < 0) { val_sign = -1; } - if ((abs(t2.day) * 24 + abs(t2.hour)) > 838) + if (abs(t2.hour) > 838) { t2.hour = 838; t2.minute = 59; t2.second = 59; t2.msecond = 999999; } + + if (val_sign * sign < 0) + { + t2.hour = -abs(t2.hour); + t2.minute = -abs(t2.minute); + t2.second = -abs(t2.second); + t2.msecond = -abs(t2.msecond); + } else { - t2.hour = abs(t2.day) * 24 + t2.hour; + t2.hour = abs(t2.hour); + t2.minute = abs(t2.minute); + t2.second = abs(t2.second); + t2.msecond = abs(t2.msecond); + } + + t2.day = 0; + + return addTime(dt1, t2); +} + +int64_t Func_add_time::getTimeIntVal(rowgroup::Row& row, + FunctionParm& parm, + bool& isNull, + CalpontSystemCatalog::ColType& ct) +{ + int64_t val1 = parm[0]->data()->getTimeIntVal(row, isNull); + + if (isNull) + return -1; + + const string& val2 = parm[1]->data()->getStrVal(row, isNull); + int sign = parm[2]->data()->getIntVal(row, isNull); + Time dt1; + dt1.day = 0; + dt1.is_neg = val1 >> 63; + dt1.hour = (val1 >> 40) & 0xfff; + dt1.minute = (val1 >> 32) & 0xff; + dt1.second = (val1 >> 24) & 0xff; + dt1.msecond = val1 & 0xffffff; + + int64_t time = DataConvert::stringToTime(val2); + + if (time == -1) + { + isNull = true; + return -1; + } + + Time t2 = *(reinterpret_cast(&time)); + + // MySQL TIME type range '-838:59:59' and '838:59:59' + if (t2.minute > 59 || t2.second > 59 || t2.msecond > 999999) + { + isNull = true; + return -1; + } + + int val_sign = 1; + + if (t2.hour < 0) + { + val_sign = -1; + } + + if (abs(t2.hour) > 838) + { + t2.hour = 838; + t2.minute = 59; + t2.second = 59; + t2.msecond = 999999; } t2.day = 0; @@ -255,5 +394,6 @@ int64_t Func_add_time::getDatetimeIntVal(rowgroup::Row& row, } + } // namespace funcexp // vim:ts=4 sw=4: diff --git a/utils/funcexp/func_between.cpp b/utils/funcexp/func_between.cpp index af950cc95..f23d67826 100644 --- a/utils/funcexp/func_between.cpp +++ b/utils/funcexp/func_between.cpp @@ -156,6 +156,25 @@ inline bool getBool(rowgroup::Row& row, numericLE(val, pm[2]->data()->getDatetimeIntVal(row, isNull)); } + case execplan::CalpontSystemCatalog::TIME: + { + // Shift out unused day for compare + int64_t val = pm[0]->data()->getTimeIntVal(row, isNull) << 12; + + if (notBetween) + { + if (!numericGE(val, pm[1]->data()->getTimeIntVal(row, isNull) << 12) && !isNull) + return true; + + isNull = false; + return (!numericLE(val, pm[2]->data()->getTimeIntVal(row, isNull) << 12) && !isNull); + } + + return !isNull && + numericGE(val, pm[1]->data()->getTimeIntVal(row, isNull) << 12) && + numericLE(val, pm[2]->data()->getTimeIntVal(row, isNull) << 12); + } + case execplan::CalpontSystemCatalog::DOUBLE: case execplan::CalpontSystemCatalog::UDOUBLE: case execplan::CalpontSystemCatalog::FLOAT: @@ -247,7 +266,8 @@ CalpontSystemCatalog::ColType Func_between::operationType( FunctionParm& fp, Cal fp[i]->data()->resultType().colDataType != CalpontSystemCatalog::TEXT && fp[i]->data()->resultType().colDataType != CalpontSystemCatalog::VARCHAR) || ct.colDataType == CalpontSystemCatalog::DATE || - ct.colDataType == CalpontSystemCatalog::DATETIME) + ct.colDataType == CalpontSystemCatalog::DATETIME || + ct.colDataType == CalpontSystemCatalog::TIME) { allString = false; } @@ -275,6 +295,23 @@ CalpontSystemCatalog::ColType Func_between::operationType( FunctionParm& fp, Cal } } } + else if (op.operationType().colDataType == CalpontSystemCatalog::TIME) + { + ConstantColumn* cc = NULL; + + for (uint32_t i = 1; i < fp.size(); i++) + { + cc = dynamic_cast(fp[i]->data()); + + if (cc) + { + Result result = cc->result(); + result.intVal = dataconvert::DataConvert::timeToInt(result.strVal); + cc->result(result); + } + } + } + return ct; } diff --git a/utils/funcexp/func_bitand.cpp b/utils/funcexp/func_bitand.cpp index 89aadf39b..69429bfb8 100644 --- a/utils/funcexp/func_bitand.cpp +++ b/utils/funcexp/func_bitand.cpp @@ -141,7 +141,8 @@ int64_t Func_bitand::getIntVal(Row& row, day = 0, hour = 0, min = 0, - sec = 0; + sec = 0, + msec = 0; year = (uint32_t)((time >> 48) & 0xffff); month = (uint32_t)((time >> 44) & 0xf); @@ -149,9 +150,39 @@ int64_t Func_bitand::getIntVal(Row& row, hour = (uint32_t)((time >> 32) & 0x3f); min = (uint32_t)((time >> 26) & 0x3f); sec = (uint32_t)((time >> 20) & 0x3f); + msec = (uint32_t)(time & 0xfffff); // return (int64_t) (year*1000000000000)+(month*100000000)+(day*1000000)+(hour*10000)+(min*100)+sec; - values.push_back((month * 100000000) + (day * 1000000) + (hour * 10000) + (min * 100) + sec); + values.push_back((month * 100000000000000) + (day * 1000000000000) + (hour * 10000000000) + (min * 100000000) + (sec * 1000000) + msec); + } + break; + + case execplan::CalpontSystemCatalog::TIME: + { + int64_t time = parm[i]->data()->getTimeIntVal(row, isNull); + + int32_t hour = 0, + min = 0, + sec = 0, + msec = 0; + + // Handle negative correctly + if ((time >> 40) & 0x800) + { + hour = 0xfffff000; + } + + hour |= ((time >> 40) & 0xfff); + + if ((hour >= 0) && (time >> 63)) + hour *= -1; + + min = (uint32_t)((time >> 32) & 0xff); + sec = (uint32_t)((time >> 24) & 0xff); + msec = (uint32_t)(time & 0xffffff); + + // return (int64_t) (year*1000000000000)+(month*100000000)+(day*1000000)+(hour*10000)+(min*100)+sec; + values.push_back((hour * 10000000000) + (min * 100000000) + (sec * 1000000) + msec); } break; diff --git a/utils/funcexp/func_bitwise.cpp b/utils/funcexp/func_bitwise.cpp index 84c83f231..752c28561 100644 --- a/utils/funcexp/func_bitwise.cpp +++ b/utils/funcexp/func_bitwise.cpp @@ -143,6 +143,15 @@ bool getUIntValFromParm( } break; + case execplan::CalpontSystemCatalog::TIME: + { + int64_t time = parm->data()->getTimeIntVal(row, isNull); + + Time dt(time); + value = dt.convertToMySQLint(); + } + break; + default: { return false; diff --git a/utils/funcexp/func_case.cpp b/utils/funcexp/func_case.cpp index 98c108bcf..4c416360c 100644 --- a/utils/funcexp/func_case.cpp +++ b/utils/funcexp/func_case.cpp @@ -52,9 +52,10 @@ inline uint64_t simple_case_cmp(Row& row, CalpontSystemCatalog::ColType& operationColType) { uint64_t i = 0; // index to the parm list - uint64_t n = parm.size() - 1; // remove expression from count of expression_i + result_i - uint64_t hasElse = n % 2; // if 1, then ELSE exist - n -= hasElse; // index to expression + uint64_t n = 0; // remove expression from count of expression_i + result_i + uint64_t hasElse = (parm.size() - 1) % 2; // if 1, then ELSE exist + uint64_t whereCount = hasElse ? (parm.size() - 2) / 2 : (parm.size() - 1) / 2; + bool foundIt = false; switch (operationColType.colDataType) { @@ -64,17 +65,19 @@ inline uint64_t simple_case_cmp(Row& row, case execplan::CalpontSystemCatalog::INT: case execplan::CalpontSystemCatalog::BIGINT: case execplan::CalpontSystemCatalog::DATE: - case execplan::CalpontSystemCatalog::DATETIME: { int64_t ev = parm[n]->data()->getIntVal(row, isNull); if (isNull) break; - for (; i < n; i += 2) + for (i = 1; i <= whereCount; i++) { if (ev == parm[i]->data()->getIntVal(row, isNull) && !isNull) + { + foundIt = true; break; + } else isNull = false; } @@ -82,6 +85,49 @@ inline uint64_t simple_case_cmp(Row& row, break; } + case execplan::CalpontSystemCatalog::DATETIME: + { + int64_t ev = parm[n]->data()->getDatetimeIntVal(row, isNull); + + if (isNull) + break; + + for (i = 1; i <= whereCount; i++) + { + if (ev == parm[i]->data()->getDatetimeIntVal(row, isNull) && !isNull) + { + foundIt = true; + break; + } + else + isNull = false; + } + + break; + } + + case execplan::CalpontSystemCatalog::TIME: + { + int64_t ev = parm[n]->data()->getTimeIntVal(row, isNull); + + if (isNull) + break; + + for (i = 1; i <= whereCount; i++) + { + if (ev == parm[i]->data()->getTimeIntVal(row, isNull) && !isNull) + { + foundIt = true; + break; + } + else + isNull = false; + } + + break; + } + + case execplan::CalpontSystemCatalog::UBIGINT: case execplan::CalpontSystemCatalog::UINT: case execplan::CalpontSystemCatalog::UMEDINT: @@ -93,10 +139,13 @@ inline uint64_t simple_case_cmp(Row& row, if (isNull) break; - for (; i < n; i += 2) + for (i = 1; i <= whereCount; i++) { if (ev == parm[i]->data()->getUintVal(row, isNull) && !isNull) + { + foundIt = true; break; + } else isNull = false; } @@ -113,11 +162,14 @@ inline uint64_t simple_case_cmp(Row& row, if (isNull) break; - for (; i < n; i += 2) + for (i = 1; i <= whereCount; i++) { //BUG 5362 if (utf8::idb_strcoll(ev.c_str(), parm[i]->data()->getStrVal(row, isNull).c_str()) == 0 && !isNull) + { + foundIt = true; break; + } else isNull = false; } @@ -133,10 +185,13 @@ inline uint64_t simple_case_cmp(Row& row, if (isNull) break; - for (; i < n; i += 2) + for (i = 1; i <= whereCount; i++) { if (ev == parm[i]->data()->getDecimalVal(row, isNull) && !isNull) + { + foundIt = true; break; + } else isNull = false; } @@ -152,10 +207,13 @@ inline uint64_t simple_case_cmp(Row& row, if (isNull) break; - for (; i < n; i += 2) + for (i = 1; i <= whereCount; i++) { if (ev == parm[i]->data()->getDoubleVal(row, isNull) && !isNull) + { + foundIt = true; break; + } else isNull = false; } @@ -171,10 +229,13 @@ inline uint64_t simple_case_cmp(Row& row, if (isNull) break; - for (; i < n; i += 2) + for (i = 1; i <= whereCount; i++) { if (ev == parm[i]->data()->getFloatVal(row, isNull) && !isNull) + { + foundIt = true; break; + } else isNull = false; } @@ -190,16 +251,24 @@ inline uint64_t simple_case_cmp(Row& row, } } - if (i == n && !hasElse) + if (!foundIt && !hasElse) isNull = true; + else if (!foundIt && hasElse && !isNull) + { + i = parm.size() - 1; + } else if (isNull && hasElse) // BUG 5110. Only way we can exit above with isNull == true is when ev is NULL - // if so and we have else condition we need to use it by setting i = n + // if so and we have else condition we need to use it by setting i = else { - i = n; + i = parm.size() - 1; isNull = false; } + if (foundIt) + { + i += whereCount; + } return i; } @@ -210,22 +279,35 @@ inline uint64_t searched_case_cmp(Row& row, bool& isNull) { uint64_t i = 0; // index to the parm list - uint64_t n = parm.size(); // count of boolean_expression_i + result_i - uint64_t hasElse = n % 2; // if 1, then ELSE exist - n -= hasElse; // index to expression + uint64_t hasElse = parm.size() % 2; // if 1, then ELSE exist + uint64_t whereCount = hasElse ? (parm.size() - 1) / 2 : parm.size() / 2; + bool foundIt = false; - for (; i < n; i += 2) + + for (i = 0; i < whereCount; i++) { if (parm[i]->getBoolVal(row, isNull)) + { + foundIt = true; break; + } } isNull = false; - if (i == n && !hasElse) + if (!foundIt && !hasElse) isNull = true; + else if (!foundIt && hasElse) + { + i = parm.size() - 1; + } - return (i == n ? i - 1 : i); + if (foundIt) + { + i += whereCount; + } + + return i; } @@ -233,7 +315,6 @@ CalpontSystemCatalog::ColType caseOperationType(FunctionParm& fp, CalpontSystemCatalog::ColType& resultType, bool simpleCase) { - // ... expression_i + result_i + ... [[expression] + result_N] FunctionParm::size_type n = fp.size(); if (simpleCase) // simple case has an expression @@ -244,6 +325,9 @@ CalpontSystemCatalog::ColType caseOperationType(FunctionParm& fp, if (hasElse) --n; // n now is an even number + uint64_t parmCount = hasElse ? (fp.size() - 2) : (fp.size() - 1); + uint64_t whereCount = hasElse ? (fp.size() - 2 + simpleCase) / 2 : (fp.size() - 1) / 2 + simpleCase; + idbassert((n % 2) == 0); bool allStringO = true; @@ -255,10 +339,10 @@ CalpontSystemCatalog::ColType caseOperationType(FunctionParm& fp, CalpontSystemCatalog::ColType rct = resultType; bool operation = true; - for (uint64_t i = 0; i <= n; i++) + for (uint64_t i = 0; i <= parmCount; i++) { // operation or result type - operation = ((i % 2) == 0); + operation = ((i > 0) && (i <= whereCount)); // the result type of ELSE, if exists. if (i == n) @@ -353,8 +437,9 @@ namespace funcexp // END // // simple CASE parm order: -// expression1 result1 expression2 result2 ... expression [resultN] +// expression condition1 condition2 ... result1 result2 ... [resultN] // +// Note that this order changed in 10.2.14, see MCOL-1341 CalpontSystemCatalog::ColType Func_simple_case::operationType(FunctionParm& fp, CalpontSystemCatalog::ColType& resultType) { @@ -372,7 +457,7 @@ bool Func_simple_case::getBoolVal(Row& row, if (isNull) return joblist::BIGINTNULL; - return parm[i + 1]->data()->getBoolVal(row, isNull); + return parm[i]->data()->getBoolVal(row, isNull); } @@ -386,7 +471,7 @@ int64_t Func_simple_case::getIntVal(Row& row, if (isNull) return joblist::BIGINTNULL; - return parm[i + 1]->data()->getIntVal(row, isNull); + return parm[i]->data()->getIntVal(row, isNull); } @@ -400,7 +485,7 @@ string Func_simple_case::getStrVal(Row& row, if (isNull) return string(""); - return parm[i + 1]->data()->getStrVal(row, isNull); + return parm[i]->data()->getStrVal(row, isNull); } @@ -414,7 +499,7 @@ IDB_Decimal Func_simple_case::getDecimalVal(Row& row, if (isNull) return IDB_Decimal(); // need a null value for IDB_Decimal?? - return parm[i + 1]->data()->getDecimalVal(row, isNull); + return parm[i]->data()->getDecimalVal(row, isNull); } @@ -428,7 +513,7 @@ double Func_simple_case::getDoubleVal(Row& row, if (isNull) return doubleNullVal(); - return parm[i + 1]->data()->getDoubleVal(row, isNull); + return parm[i]->data()->getDoubleVal(row, isNull); } @@ -442,7 +527,7 @@ int32_t Func_simple_case::getDateIntVal(rowgroup::Row& row, if (isNull) return joblist::DATENULL; - return parm[i + 1]->data()->getDateIntVal(row, isNull); + return parm[i]->data()->getDateIntVal(row, isNull); } @@ -456,10 +541,23 @@ int64_t Func_simple_case::getDatetimeIntVal(rowgroup::Row& row, if (isNull) return joblist::DATETIMENULL; - return parm[i + 1]->data()->getDatetimeIntVal(row, isNull); + return parm[i]->data()->getDatetimeIntVal(row, isNull); } +int64_t Func_simple_case::getTimeIntVal(rowgroup::Row& row, + FunctionParm& parm, + bool& isNull, + execplan::CalpontSystemCatalog::ColType& op_ct) +{ + uint64_t i = simple_case_cmp(row, parm, isNull, op_ct); + + if (isNull) + return joblist::TIMENULL; + + return parm[i]->data()->getTimeIntVal(row, isNull); +} + // searched CASE: // SELECT CASE @@ -470,8 +568,10 @@ int64_t Func_simple_case::getDatetimeIntVal(rowgroup::Row& row, // END // // searched CASE parm order: -// boolean_expression1 result1 boolean_expression2 result2 ... [resultN] +// boolean_expression1 boolean_expression2 ... result1 result2 ... [resultN] // +// Note that this order changed in 10.2.14, see MCOL-1341 + CalpontSystemCatalog::ColType Func_searched_case::operationType(FunctionParm& fp, CalpontSystemCatalog::ColType& resultType) { // operation type not used by this functor. @@ -489,15 +589,15 @@ bool Func_searched_case::getBoolVal(Row& row, if (isNull) return joblist::BIGINTNULL; - ParseTree* lop = parm[i + 1]->left(); - ParseTree* rop = parm[i + 1]->right(); + ParseTree* lop = parm[i]->left(); + ParseTree* rop = parm[i]->right(); if (lop && rop) { - return (reinterpret_cast(parm[i + 1]->data()))->getBoolVal(row, isNull, lop, rop); + return (reinterpret_cast(parm[i]->data()))->getBoolVal(row, isNull, lop, rop); } - return parm[i + 1]->data()->getBoolVal(row, isNull); + return parm[i]->data()->getBoolVal(row, isNull); } int64_t Func_searched_case::getIntVal(Row& row, @@ -510,7 +610,7 @@ int64_t Func_searched_case::getIntVal(Row& row, if (isNull) return joblist::BIGINTNULL; - return parm[i + 1]->data()->getIntVal(row, isNull); + return parm[i]->data()->getIntVal(row, isNull); } @@ -524,7 +624,7 @@ string Func_searched_case::getStrVal(Row& row, if (isNull) return string(""); - return parm[i + 1]->data()->getStrVal(row, isNull); + return parm[i]->data()->getStrVal(row, isNull); } @@ -538,7 +638,7 @@ IDB_Decimal Func_searched_case::getDecimalVal(Row& row, if (isNull) return IDB_Decimal(); // need a null value for IDB_Decimal?? - return parm[i + 1]->data()->getDecimalVal(row, isNull); + return parm[i]->data()->getDecimalVal(row, isNull); } @@ -552,7 +652,7 @@ double Func_searched_case::getDoubleVal(Row& row, if (isNull) return doubleNullVal(); - return parm[i + 1]->data()->getDoubleVal(row, isNull); + return parm[i]->data()->getDoubleVal(row, isNull); } @@ -566,7 +666,7 @@ int32_t Func_searched_case::getDateIntVal(rowgroup::Row& row, if (isNull) return joblist::DATENULL; - return parm[i + 1]->data()->getDateIntVal(row, isNull); + return parm[i]->data()->getDateIntVal(row, isNull); } @@ -580,9 +680,22 @@ int64_t Func_searched_case::getDatetimeIntVal(rowgroup::Row& row, if (isNull) return joblist::DATETIMENULL; - return parm[i + 1]->data()->getDatetimeIntVal(row, isNull); + return parm[i]->data()->getDatetimeIntVal(row, isNull); } +int64_t Func_searched_case::getTimeIntVal(rowgroup::Row& row, + FunctionParm& parm, + bool& isNull, + execplan::CalpontSystemCatalog::ColType& op_ct) +{ + uint64_t i = simple_case_cmp(row, parm, isNull, op_ct); + + if (isNull) + return joblist::TIMENULL; + + return parm[i]->data()->getTimeIntVal(row, isNull); +} + } // namespace funcexp // vim:ts=4 sw=4: diff --git a/utils/funcexp/func_cast.cpp b/utils/funcexp/func_cast.cpp index 9d2423293..3542e341d 100644 --- a/utils/funcexp/func_cast.cpp +++ b/utils/funcexp/func_cast.cpp @@ -191,6 +191,15 @@ int64_t Func_cast_signed::getIntVal(Row& row, } break; + case execplan::CalpontSystemCatalog::TIME: + { + int64_t time = parm[0]->data()->getTimeIntVal(row, isNull); + + Time dt(time); + return dt.convertToMySQLint(); + } + break; + default: { std::ostringstream oss; @@ -313,6 +322,15 @@ uint64_t Func_cast_unsigned::getUintVal(Row& row, } break; + case execplan::CalpontSystemCatalog::TIME: + { + int64_t time = parm[0]->data()->getTimeIntVal(row, isNull); + + Time dt(time); + return dt.convertToMySQLint(); + } + break; + default: { std::ostringstream oss; @@ -805,6 +823,85 @@ int64_t Func_cast_datetime::getDatetimeIntVal(rowgroup::Row& row, return -1; } + +int64_t Func_cast_datetime::getTimeIntVal(rowgroup::Row& row, + FunctionParm& parm, + bool& isNull, + execplan::CalpontSystemCatalog::ColType& operationColType) +{ + int64_t val; + + switch (parm[0]->data()->resultType().colDataType) + { + case execplan::CalpontSystemCatalog::BIGINT: + case execplan::CalpontSystemCatalog::INT: + case execplan::CalpontSystemCatalog::MEDINT: + case execplan::CalpontSystemCatalog::TINYINT: + case execplan::CalpontSystemCatalog::SMALLINT: + case execplan::CalpontSystemCatalog::UBIGINT: + case execplan::CalpontSystemCatalog::UINT: + case execplan::CalpontSystemCatalog::UMEDINT: + case execplan::CalpontSystemCatalog::UTINYINT: + case execplan::CalpontSystemCatalog::USMALLINT: + { + val = dataconvert::DataConvert::intToTime(parm[0]->data()->getIntVal(row, isNull)); + + if (val == -1) + isNull = true; + else + return val; + + break; + } + + case execplan::CalpontSystemCatalog::DECIMAL: + case execplan::CalpontSystemCatalog::UDECIMAL: + { + val = dataconvert::DataConvert::intToTime(parm[0]->data()->getIntVal(row, isNull)); + + if (val == -1) + isNull = true; + else + return val; + + break; + } + + case execplan::CalpontSystemCatalog::VARCHAR: + case execplan::CalpontSystemCatalog::CHAR: + case execplan::CalpontSystemCatalog::TEXT: + { + val = dataconvert::DataConvert::stringToTime(parm[0]->data()->getStrVal(row, isNull)); + + if (val == -1) + isNull = true; + else + return val; + + break; + } + + case execplan::CalpontSystemCatalog::DATE: + { + return parm[0]->data()->getTimeIntVal(row, isNull); + } + + case execplan::CalpontSystemCatalog::DATETIME: + { + return parm[0]->data()->getTimeIntVal(row, isNull); + } + + default: + { + isNull = true; + } + } + + return -1; +} + + + // // Func_cast_decimal // @@ -1138,6 +1235,25 @@ IDB_Decimal Func_cast_decimal::getDecimalVal(Row& row, } break; + case execplan::CalpontSystemCatalog::TIME: + { + int32_t s = 0; + + string value = dataconvert::DataConvert::timeToString1(parm[0]->data()->getTimeIntVal(row, isNull)); + + //strip off micro seconds + string date = value.substr(0, 14); + + int64_t x = atoll(date.c_str()); + + if (!isNull) + { + decimal.value = x; + decimal.scale = s; + } + } + break; + default: { std::ostringstream oss; diff --git a/utils/funcexp/func_ceil.cpp b/utils/funcexp/func_ceil.cpp index 7c1cafdfb..7c2f7f571 100644 --- a/utils/funcexp/func_ceil.cpp +++ b/utils/funcexp/func_ceil.cpp @@ -152,6 +152,15 @@ int64_t Func_ceil::getIntVal(Row& row, } break; + case CalpontSystemCatalog::TIME: + { + Time dt(parm[0]->data()->getTimeIntVal(row, isNull)); + + if (!isNull) + ret = dt.convertToMySQLint(); + } + break; + default: { std::ostringstream oss; @@ -233,6 +242,15 @@ uint64_t Func_ceil::getUintVal(Row& row, } break; + case CalpontSystemCatalog::TIME: + { + Time dt(parm[0]->data()->getTimeIntVal(row, isNull)); + + if (!isNull) + ret = dt.convertToMySQLint(); + } + break; + default: { std::ostringstream oss; diff --git a/utils/funcexp/func_char_length.cpp b/utils/funcexp/func_char_length.cpp index 9421554f8..b47afd58a 100644 --- a/utils/funcexp/func_char_length.cpp +++ b/utils/funcexp/func_char_length.cpp @@ -101,6 +101,12 @@ int64_t Func_char_length::getIntVal(rowgroup::Row& row, return (int64_t)date.size(); } + case execplan::CalpontSystemCatalog::TIME: + { + string date = dataconvert::DataConvert::timeToString(parm[0]->data()->getTimeIntVal(row, isNull)); + return (int64_t)date.size(); + } + default: { std::ostringstream oss; diff --git a/utils/funcexp/func_coalesce.cpp b/utils/funcexp/func_coalesce.cpp index 33a1312bc..e03040350 100644 --- a/utils/funcexp/func_coalesce.cpp +++ b/utils/funcexp/func_coalesce.cpp @@ -139,6 +139,30 @@ int64_t Func_coalesce::getDatetimeIntVal(rowgroup::Row& row, return val; } +int64_t Func_coalesce::getTimeIntVal(rowgroup::Row& row, + FunctionParm& parm, + bool& isNull, + CalpontSystemCatalog::ColType& ct) +{ + int64_t val = 0; + + for (uint32_t i = 0; i < parm.size(); i++) + { + val = parm[i]->data()->getTimeIntVal(row, isNull); + + if (isNull) + { + isNull = false; + continue; + } + + return val; + } + + isNull = true; + return val; +} + double Func_coalesce::getDoubleVal(rowgroup::Row& row, FunctionParm& parm, bool& isNull, diff --git a/utils/funcexp/func_extract.cpp b/utils/funcexp/func_extract.cpp index 08a545fde..332c05441 100644 --- a/utils/funcexp/func_extract.cpp +++ b/utils/funcexp/func_extract.cpp @@ -131,6 +131,95 @@ long long dateGet( uint64_t time, IntervalColumn::interval_type unit, bool dateT throw runtime_error("unit type is not supported: " + unit); }; } + +long long timeGet( uint64_t time, IntervalColumn::interval_type unit ) +{ + int32_t hour = 0, + min = 0, + sec = 0, + msec = 0, + day = 0; + + min = (int32_t)((time >> 32) & 0xff); + sec = (int32_t)((time >> 24) & 0xff); + msec = (int32_t)((time & 0xfffff)); + + // If negative, mask so it doesn't turn positive + int64_t mask = 0; + + if ((time >> 40) & 0x800) + mask = 0xfffffffffffff000; + + hour = mask | ((time >> 40) & 0xfff); + + if ((hour >= 0) && (time >> 63)) + hour *= -1; + + // Always positive! + day = abs(hour / 24); + + switch ( unit ) + { + case IntervalColumn::INTERVAL_YEAR: + case IntervalColumn::INTERVAL_MONTH: + return 0; + + case IntervalColumn::INTERVAL_DAY: + return day; + + case IntervalColumn::INTERVAL_HOUR: + return hour; + + case IntervalColumn::INTERVAL_MINUTE: + return min; + + case IntervalColumn::INTERVAL_SECOND: + return sec; + + case IntervalColumn::INTERVAL_MICROSECOND: + return msec; + + case IntervalColumn::INTERVAL_QUARTER: + case IntervalColumn::INTERVAL_WEEK: + case IntervalColumn::INTERVAL_YEAR_MONTH: + return 0; + + case IntervalColumn::INTERVAL_DAY_HOUR: + return (day * 100) + hour; + + case IntervalColumn::INTERVAL_DAY_MINUTE: + return (day * 10000) + (hour * 100) + min; + + case IntervalColumn::INTERVAL_DAY_SECOND: + return (day * 1000000) + (hour * 10000) + (min * 100) + sec; + + case IntervalColumn::INTERVAL_HOUR_MINUTE: + return (hour * 100) + min; + + case IntervalColumn::INTERVAL_HOUR_SECOND: + return (hour * 10000) + (min * 100) + sec; + + case IntervalColumn::INTERVAL_MINUTE_SECOND: + return (min * 100) + sec; + + case IntervalColumn::INTERVAL_DAY_MICROSECOND: + return (((day * 1000000) + (hour * 10000) + (min * 100) + sec) * 1000000) + msec; + + case IntervalColumn::INTERVAL_HOUR_MICROSECOND: + return (((hour * 10000) + (min * 100) + sec) * 1000000) + msec; + + case IntervalColumn::INTERVAL_MINUTE_MICROSECOND: + return (((min * 100) + sec) * 1000000) + msec; + + case IntervalColumn::INTERVAL_SECOND_MICROSECOND: + return (sec * 1000000) + msec; + + default: + throw runtime_error("unit type is not supported: " + unit); + }; +} + + } namespace funcexp @@ -148,6 +237,7 @@ int64_t Func_extract::getIntVal(rowgroup::Row& row, { IntervalColumn::interval_type unit = static_cast(parm[1]->data()->getIntVal(row, isNull)); uint64_t time; + bool isTime = false; //@bug4678 handle conversion from non date/datetime datatype switch (parm[0]->data()->resultType().colDataType) @@ -157,6 +247,11 @@ int64_t Func_extract::getIntVal(rowgroup::Row& row, time = parm[0]->data()->getDatetimeIntVal(row, isNull); break; + case CalpontSystemCatalog::TIME: + time = parm[0]->data()->getTimeIntVal(row, isNull); + isTime = true; + break; + case CalpontSystemCatalog::VARCHAR: case CalpontSystemCatalog::CHAR: case CalpontSystemCatalog::TEXT: @@ -181,7 +276,12 @@ int64_t Func_extract::getIntVal(rowgroup::Row& row, time = parm[0]->data()->getIntVal(row, isNull); } - long long value = dateGet( time, unit, false ); + long long value; + + if (isTime) + value = timeGet( time, unit ); + else + value = dateGet( time, unit, false ); return value; } diff --git a/utils/funcexp/func_floor.cpp b/utils/funcexp/func_floor.cpp index ad47dc988..eb05b405f 100644 --- a/utils/funcexp/func_floor.cpp +++ b/utils/funcexp/func_floor.cpp @@ -151,6 +151,19 @@ int64_t Func_floor::getIntVal(Row& row, } break; + case execplan::CalpontSystemCatalog::TIME: + { + string str = + DataConvert::timeToString1(parm[0]->data()->getTimeIntVal(row, isNull)); + + // strip off micro seconds + str = str.substr(0, 14); + + if (!isNull) + ret = atoll(str.c_str()); + } + break; + default: { std::ostringstream oss; @@ -236,6 +249,19 @@ uint64_t Func_floor::getUintVal(Row& row, } break; + case execplan::CalpontSystemCatalog::TIME: + { + string str = + DataConvert::timeToString1(parm[0]->data()->getTimeIntVal(row, isNull)); + + // strip off micro seconds + str = str.substr(0, 14); + + if (!isNull) + ret = strtoull(str.c_str(), NULL, 10); + } + break; + default: { std::ostringstream oss; diff --git a/utils/funcexp/func_from_unixtime.cpp b/utils/funcexp/func_from_unixtime.cpp index 183a75a56..9968ba1f0 100644 --- a/utils/funcexp/func_from_unixtime.cpp +++ b/utils/funcexp/func_from_unixtime.cpp @@ -148,6 +148,22 @@ int64_t Func_from_unixtime::getDatetimeIntVal(rowgroup::Row& row, return *reinterpret_cast(&dt); } +int64_t Func_from_unixtime::getTimeIntVal(rowgroup::Row& row, + FunctionParm& parm, + bool& isNull, + CalpontSystemCatalog::ColType& ct) +{ + DateTime dt = getDateTime(row, parm, isNull); + + if (*reinterpret_cast(&dt) == 0) + { + isNull = true; + return 0; + } + + return *reinterpret_cast(&dt); +} + int64_t Func_from_unixtime::getIntVal(rowgroup::Row& row, FunctionParm& parm, bool& isNull, diff --git a/utils/funcexp/func_greatest.cpp b/utils/funcexp/func_greatest.cpp index 8309456ae..de63c283e 100644 --- a/utils/funcexp/func_greatest.cpp +++ b/utils/funcexp/func_greatest.cpp @@ -206,6 +206,31 @@ int64_t Func_greatest::getDatetimeIntVal(rowgroup::Row& row, return greatestStr; } +int64_t Func_greatest::getTimeIntVal(rowgroup::Row& row, + FunctionParm& fp, + bool& isNull, + execplan::CalpontSystemCatalog::ColType& ct) +{ + // Strip off unused day + int64_t greatestStr = fp[0]->data()->getTimeIntVal(row, isNull); + + int64_t str = greatestStr << 12; + + for (uint32_t i = 1; i < fp.size(); i++) + { + int64_t str1 = fp[i]->data()->getTimeIntVal(row, isNull); + int64_t str2 = str1 << 12; + + if ( str < str2 ) + { + greatestStr = str1; + str = str2; + } + } + + return greatestStr; +} + } // namespace funcexp // vim:ts=4 sw=4: diff --git a/utils/funcexp/func_hour.cpp b/utils/funcexp/func_hour.cpp index f5058a084..685a264db 100644 --- a/utils/funcexp/func_hour.cpp +++ b/utils/funcexp/func_hour.cpp @@ -48,6 +48,7 @@ int64_t Func_hour::getIntVal(rowgroup::Row& row, CalpontSystemCatalog::ColType& op_ct) { int64_t val = 0; + bool isTime = false; switch (parm[0]->data()->resultType().colDataType) { @@ -108,6 +109,13 @@ int64_t Func_hour::getIntVal(rowgroup::Row& row, break; } + case execplan::CalpontSystemCatalog::TIME: + { + isTime = true; + val = parm[0]->data()->getTimeIntVal(row, isNull); + break; + } + default: { isNull = true; @@ -117,10 +125,31 @@ int64_t Func_hour::getIntVal(rowgroup::Row& row, if (isNull) return -1; - if ( val < 1000000000 ) - return 0; + if (isTime) + { + // If negative, mask so it doesn't turn positive + bool isNeg = false; + int64_t mask = 0; - return (uint32_t)((val >> 32) & 0x3f); + if ((val >> 40) & 0x800) + mask = 0xfffffffffffff000; + + if (!mask && (val >> 63)) + { + isNeg = true; + } + + val = mask | ((val >> 40) & 0xfff); + + if (isNeg) + val *= -1; + } + else + { + val = (val >> 32) & 0x3f; + } + + return val; } diff --git a/utils/funcexp/func_if.cpp b/utils/funcexp/func_if.cpp index 378c6d74a..ad65d10c9 100644 --- a/utils/funcexp/func_if.cpp +++ b/utils/funcexp/func_if.cpp @@ -75,6 +75,7 @@ bool boolVal(SPTP& parm, Row& row) case CalpontSystemCatalog::UINT: case CalpontSystemCatalog::DATE: case CalpontSystemCatalog::DATETIME: + case CalpontSystemCatalog::TIME: default: ret = (parm->data()->getIntVal(row, isNull) != 0); } @@ -219,6 +220,19 @@ int64_t Func_if::getDatetimeIntVal(Row& row, } } - +int64_t Func_if::getTimeIntVal(Row& row, + FunctionParm& parm, + bool& isNull, + CalpontSystemCatalog::ColType&) +{ + if (boolVal(parm[0], row)) + { + return parm[1]->data()->getTimeIntVal(row, isNull); + } + else + { + return parm[2]->data()->getTimeIntVal(row, isNull); + } +} } // namespace funcexp // vim:ts=4 sw=4: diff --git a/utils/funcexp/func_ifnull.cpp b/utils/funcexp/func_ifnull.cpp index 41db7e3b4..4b8606d4e 100644 --- a/utils/funcexp/func_ifnull.cpp +++ b/utils/funcexp/func_ifnull.cpp @@ -168,6 +168,25 @@ int64_t Func_ifnull::getDatetimeIntVal(Row& row, return r; } +int64_t Func_ifnull::getTimeIntVal(Row& row, + FunctionParm& parm, + bool& isNull, + CalpontSystemCatalog::ColType&) +{ + if (isNull) + return 0; + + int64_t r = parm[0]->data()->getTimeIntVal(row, isNull); + + if (isNull) + { + isNull = false; + return parm[1]->data()->getTimeIntVal(row, isNull); + } + + return r; +} + bool Func_ifnull::getBoolVal(Row& row, FunctionParm& parm, bool& isNull, diff --git a/utils/funcexp/func_in.cpp b/utils/funcexp/func_in.cpp index 404c0f39f..e271fa0f7 100644 --- a/utils/funcexp/func_in.cpp +++ b/utils/funcexp/func_in.cpp @@ -158,6 +158,27 @@ inline bool getBoolForIn(rowgroup::Row& row, return false; } + case execplan::CalpontSystemCatalog::TIME: + { + int64_t val = pm[0]->data()->getTimeIntVal(row, isNull); + + if (isNull) + return false; + + for (uint32_t i = 1; i < pm.size(); i++) + { + isNull = false; + + if ( val == pm[i]->data()->getTimeIntVal(row, isNull) && !isNull ) + return true; + + if (isNull && isNotIn) + return true; // will be reversed to false by the caller + } + + return false; + } + case execplan::CalpontSystemCatalog::DOUBLE: case execplan::CalpontSystemCatalog::UDOUBLE: case execplan::CalpontSystemCatalog::FLOAT: diff --git a/utils/funcexp/func_inet_aton.cpp b/utils/funcexp/func_inet_aton.cpp index bd2d2a2d7..9692cf850 100644 --- a/utils/funcexp/func_inet_aton.cpp +++ b/utils/funcexp/func_inet_aton.cpp @@ -222,6 +222,26 @@ int64_t Func_inet_aton::getDatetimeIntVal(rowgroup::Row& row, return iValue; } +int64_t Func_inet_aton::getTimeIntVal(rowgroup::Row& row, + FunctionParm& fp, + bool& isNull, + execplan::CalpontSystemCatalog::ColType& op_ct) +{ + int64_t iValue = joblist::TIMENULL; + + const std::string& sValue = fp[0]->data()->getStrVal(row, isNull); + + if (!isNull) + { + int64_t iVal = convertAton( sValue, isNull ); + + if (!isNull) + iValue = iVal; + } + + return iValue; +} + //------------------------------------------------------------------------------ // Convert an ascii IP address string to it's integer equivalent. // isNull is set to true if the IP address string has invalid content. diff --git a/utils/funcexp/func_inet_ntoa.cpp b/utils/funcexp/func_inet_ntoa.cpp index 84c476eb6..ff8c889ed 100644 --- a/utils/funcexp/func_inet_ntoa.cpp +++ b/utils/funcexp/func_inet_ntoa.cpp @@ -256,6 +256,20 @@ int64_t Func_inet_ntoa::getDatetimeIntVal(rowgroup::Row& row, return iValue; } +int64_t Func_inet_ntoa::getTimeIntVal(rowgroup::Row& row, + FunctionParm& fp, + bool& isNull, + execplan::CalpontSystemCatalog::ColType& op_ct) +{ +// std::cout << "In Func_inet_ntoa::getTimeVal" << std::endl; + +// int64t iValue = fp[0]->data()->getTimeIntVal(row, isNull); + int64_t iValue = joblist::TIMENULL; + isNull = true; + + return iValue; +} + //------------------------------------------------------------------------------ // Convert an integer IP address to its equivalent IP address string. // Source code based on MySQL source (Item_func_inet_ntoa() in item_strfunc.cc). diff --git a/utils/funcexp/func_least.cpp b/utils/funcexp/func_least.cpp index e273a4618..9ba5c98de 100644 --- a/utils/funcexp/func_least.cpp +++ b/utils/funcexp/func_least.cpp @@ -182,6 +182,30 @@ int64_t Func_least::getDatetimeIntVal(rowgroup::Row& row, return leastStr; } +int64_t Func_least::getTimeIntVal(rowgroup::Row& row, + FunctionParm& fp, + bool& isNull, + execplan::CalpontSystemCatalog::ColType& op_ct) +{ + // Strip off unused day + int64_t leastStr = fp[0]->data()->getTimeIntVal(row, isNull); + + int64_t str = leastStr << 12; + + for (uint32_t i = 1; i < fp.size(); i++) + { + int64_t str1 = fp[i]->data()->getTimeIntVal(row, isNull); + int64_t str2 = str1 << 12; + + if ( str > str2 ) + { + leastStr = str1; + str = str2; + } + } + + return leastStr; +} } // namespace funcexp diff --git a/utils/funcexp/func_math.cpp b/utils/funcexp/func_math.cpp index 8e5b84def..8592ec8b0 100644 --- a/utils/funcexp/func_math.cpp +++ b/utils/funcexp/func_math.cpp @@ -154,6 +154,20 @@ double Func_acos::getDoubleVal(Row& row, } break; + case execplan::CalpontSystemCatalog::TIME: + { + int64_t value = parm[0]->data()->getTimeIntVal(row, isNull); + + if (isNull || (value < -1.0 || value > 1.0)) + { + isNull = true; + return doubleNullVal(); + } + + return acos((double)value); + } + break; + default: { std::ostringstream oss; @@ -244,6 +258,20 @@ double Func_asin::getDoubleVal(Row& row, } break; + case execplan::CalpontSystemCatalog::TIME: + { + int64_t value = parm[0]->data()->getTimeIntVal(row, isNull); + + if (isNull || (value < -1.0 || value > 1.0)) + { + isNull = true; + return doubleNullVal(); + } + + return asin((double)value); + } + break; + default: { std::ostringstream oss; @@ -373,6 +401,34 @@ double Func_atan::getDoubleVal(Row& row, } break; + + case execplan::CalpontSystemCatalog::TIME: + { + int64_t value = parm[0]->data()->getTimeIntVal(row, isNull); + + if (isNull) + { + isNull = true; + return doubleNullVal(); + } + + if (parm.size() > 1 ) + { + double value2 = parm[1]->data()->getDoubleVal(row, isNull); + + if (isNull) + { + isNull = true; + return doubleNullVal(); + } + + return atan2(value, value2); + } + + return atan((double)value); + } + break; + default: { std::ostringstream oss; @@ -462,6 +518,20 @@ double Func_cos::getDoubleVal(Row& row, } break; + case execplan::CalpontSystemCatalog::TIME: + { + int64_t value = parm[0]->data()->getTimeIntVal(row, isNull); + + if (isNull) + { + isNull = true; + return doubleNullVal(); + } + + return cos((double)value); + } + break; + default: { std::ostringstream oss; @@ -578,6 +648,30 @@ double Func_cot::getDoubleVal(Row& row, } break; + case execplan::CalpontSystemCatalog::TIME: + { + int64_t value = parm[0]->data()->getTimeIntVal(row, isNull); + + if (value == 0) + { + Message::Args args; + args.add("cot"); + args.add((uint64_t)value); + unsigned errcode = ERR_FUNC_OUT_OF_RANGE_RESULT; + throw IDBExcept(IDBErrorInfo::instance()->errorMsg(errcode, args), errcode); + } + + if (isNull) + { + isNull = true; + return doubleNullVal(); + } + + return 1.0 / tan((double)value); + } + break; + + default: { std::ostringstream oss; @@ -703,6 +797,33 @@ double Func_log::getDoubleVal(Row& row, } break; + case execplan::CalpontSystemCatalog::TIME: + { + int64_t value = parm[0]->data()->getTimeIntVal(row, isNull); + + if (isNull || value <= 0.0) + { + isNull = true; + return doubleNullVal(); + } + + if (parm.size() > 1 ) + { + double value2 = parm[1]->data()->getDoubleVal(row, isNull); + + if (isNull || (value2 <= 0.0 || value == 1.0) ) + { + isNull = true; + return doubleNullVal(); + } + + return log(value2) / log((double)value); + } + + return log((double)value); + } + break; + case execplan::CalpontSystemCatalog::VARCHAR: case execplan::CalpontSystemCatalog::CHAR: case execplan::CalpontSystemCatalog::TEXT: @@ -797,6 +918,20 @@ double Func_log2::getDoubleVal(Row& row, } break; + case execplan::CalpontSystemCatalog::TIME: + { + int64_t value = parm[0]->data()->getTimeIntVal(row, isNull); + + if (isNull || value <= 0.0) + { + isNull = true; + return doubleNullVal(); + } + + return log2(value); + } + break; + case execplan::CalpontSystemCatalog::VARCHAR: case execplan::CalpontSystemCatalog::CHAR: case execplan::CalpontSystemCatalog::TEXT: @@ -891,6 +1026,20 @@ double Func_log10::getDoubleVal(Row& row, } break; + case execplan::CalpontSystemCatalog::TIME: + { + int64_t value = parm[0]->data()->getTimeIntVal(row, isNull); + + if (isNull || value <= 0.0) + { + isNull = true; + return doubleNullVal(); + } + + return log10((double)value); + } + break; + case execplan::CalpontSystemCatalog::VARCHAR: case execplan::CalpontSystemCatalog::CHAR: case execplan::CalpontSystemCatalog::TEXT: @@ -988,6 +1137,20 @@ double Func_sin::getDoubleVal(Row& row, } break; + case execplan::CalpontSystemCatalog::TIME: + { + int64_t value = parm[0]->data()->getTimeIntVal(row, isNull); + + if (isNull) + { + isNull = true; + return doubleNullVal(); + } + + return sin((double)value); + } + break; + default: { std::ostringstream oss; @@ -1077,6 +1240,20 @@ double Func_sqrt::getDoubleVal(Row& row, } break; + case execplan::CalpontSystemCatalog::TIME: + { + int64_t value = parm[0]->data()->getTimeIntVal(row, isNull); + + if (isNull || value < 0) + { + isNull = true; + return doubleNullVal(); + } + + return sqrt((double)value); + } + break; + default: { std::ostringstream oss; @@ -1166,6 +1343,20 @@ double Func_tan::getDoubleVal(Row& row, } break; + case execplan::CalpontSystemCatalog::TIME: + { + int64_t value = parm[0]->data()->getTimeIntVal(row, isNull); + + if (isNull) + { + isNull = true; + return doubleNullVal(); + } + + return tan((double)value); + } + break; + default: { std::ostringstream oss; @@ -1253,6 +1444,12 @@ string Func_format::getStrVal(Row& row, } break; + case execplan::CalpontSystemCatalog::TIME: + { + value = dataconvert::DataConvert::timeToString1(parm[0]->data()->getTimeIntVal(row, isNull)); + } + break; + case execplan::CalpontSystemCatalog::DECIMAL: case execplan::CalpontSystemCatalog::UDECIMAL: { @@ -1474,6 +1671,20 @@ double Func_radians::getDoubleVal(Row& row, } break; + case execplan::CalpontSystemCatalog::TIME: + { + int64_t value = parm[0]->data()->getTimeIntVal(row, isNull); + + if (isNull) + { + isNull = true; + return doubleNullVal(); + } + + return radians((double)value); + } + break; + default: { std::ostringstream oss; @@ -1563,6 +1774,20 @@ double Func_degrees::getDoubleVal(Row& row, } break; + case execplan::CalpontSystemCatalog::TIME: + { + int64_t value = parm[0]->data()->getTimeIntVal(row, isNull); + + if (isNull) + { + isNull = true; + return doubleNullVal(); + } + + return degrees((double)value); + } + break; + default: { std::ostringstream oss; diff --git a/utils/funcexp/func_microsecond.cpp b/utils/funcexp/func_microsecond.cpp index e0a8e6d11..f292e26a5 100644 --- a/utils/funcexp/func_microsecond.cpp +++ b/utils/funcexp/func_microsecond.cpp @@ -63,6 +63,11 @@ int64_t Func_microsecond::getIntVal(rowgroup::Row& row, microSecond = (uint32_t)((val & 0xfffff)); break; + case CalpontSystemCatalog::TIME: + val = parm[0]->data()->getIntVal(row, isNull); + microSecond = (uint32_t)((val & 0xffffff)); + break; + case CalpontSystemCatalog::CHAR: case CalpontSystemCatalog::TEXT: case CalpontSystemCatalog::VARCHAR: diff --git a/utils/funcexp/func_minute.cpp b/utils/funcexp/func_minute.cpp index f70af29e3..299dd6ae2 100644 --- a/utils/funcexp/func_minute.cpp +++ b/utils/funcexp/func_minute.cpp @@ -102,6 +102,7 @@ int64_t Func_minute::getIntVal(rowgroup::Row& row, break; } + case execplan::CalpontSystemCatalog::TIME: case execplan::CalpontSystemCatalog::DATETIME: { val = parm[0]->data()->getDatetimeIntVal(row, isNull); diff --git a/utils/funcexp/func_nullif.cpp b/utils/funcexp/func_nullif.cpp index 7d1647d1d..a268b0ea1 100644 --- a/utils/funcexp/func_nullif.cpp +++ b/utils/funcexp/func_nullif.cpp @@ -127,8 +127,11 @@ int64_t Func_nullif::getIntVal(rowgroup::Row& row, { exp2 = parm[1]->data()->getDatetimeIntVal(row, isNull); - if (parm[0]->data()->resultType().colDataType == - execplan::CalpontSystemCatalog::DATE) + if ((parm[0]->data()->resultType().colDataType == + execplan::CalpontSystemCatalog::DATE) || + (parm[0]->data()->resultType().colDataType == + execplan::CalpontSystemCatalog::DATETIME)) + { // NULLIF arg0 is DATE, arg1 is DATETIME, // Upgrade arg1 to datetime @@ -155,6 +158,40 @@ int64_t Func_nullif::getIntVal(rowgroup::Row& row, break; } + case execplan::CalpontSystemCatalog::TIME: + { + exp2 = parm[1]->data()->getTimeIntVal(row, isNull); + + if ((parm[0]->data()->resultType().colDataType == + execplan::CalpontSystemCatalog::DATETIME) || + (parm[0]->data()->resultType().colDataType == + execplan::CalpontSystemCatalog::TIME)) + { + // NULLIF arg0 is DATETIME, arg1 is TIME, + // Upgrade arg1 to time + // When comparing exp1 as a Date, we can't simply promote. We have + // to be careful of the return value in case of not null return. + int64_t exp1 = parm[0]->data()->getTimeIntVal(row, isNull); + + if ( exp1 == exp2 ) + { + isNull = true; + return 0; + } + + // since exp1 here is inside the block, when we leave the block, the + // original (Date) value is restored. + } + + if (isNull) + { + isNull = false; + return exp1; + } + + break; + } + default: { isNull = true; @@ -244,6 +281,19 @@ uint64_t Func_nullif::getUintVal(rowgroup::Row& row, break; } + case execplan::CalpontSystemCatalog::TIME: + { + exp2 = parm[1]->data()->getTimeIntVal(row, isNull); + + if (isNull) + { + isNull = false; + return exp1; + } + + break; + } + default: { isNull = true; @@ -400,18 +450,6 @@ int64_t Func_nullif::getDatetimeIntVal(rowgroup::Row& row, case execplan::CalpontSystemCatalog::VARCHAR: case execplan::CalpontSystemCatalog::CHAR: case execplan::CalpontSystemCatalog::TEXT: - { - exp2 = parm[1]->data()->getIntVal(row, isNull); - - if (isNull) - { - isNull = false; - return exp1; - } - - break; - } - case execplan::CalpontSystemCatalog::DATE: { // Upgrade to datetime for proper comparison @@ -427,6 +465,7 @@ int64_t Func_nullif::getDatetimeIntVal(rowgroup::Row& row, } break; + case execplan::CalpontSystemCatalog::TIME: case execplan::CalpontSystemCatalog::DATETIME: { exp2 = parm[1]->data()->getDatetimeIntVal(row, isNull); @@ -455,6 +494,57 @@ int64_t Func_nullif::getDatetimeIntVal(rowgroup::Row& row, return exp1; } +int64_t Func_nullif::getTimeIntVal(rowgroup::Row& row, + FunctionParm& parm, + bool& isNull, + CalpontSystemCatalog::ColType& ct) +{ + int64_t exp1 = parm[0]->data()->getTimeIntVal(row, isNull); + int64_t exp2 = 0; + + switch (parm[1]->data()->resultType().colDataType) + { + case execplan::CalpontSystemCatalog::BIGINT: + case execplan::CalpontSystemCatalog::INT: + case execplan::CalpontSystemCatalog::MEDINT: + case execplan::CalpontSystemCatalog::TINYINT: + case execplan::CalpontSystemCatalog::SMALLINT: + case execplan::CalpontSystemCatalog::DOUBLE: + case execplan::CalpontSystemCatalog::FLOAT: + case execplan::CalpontSystemCatalog::DECIMAL: + case execplan::CalpontSystemCatalog::VARCHAR: + case execplan::CalpontSystemCatalog::CHAR: + case execplan::CalpontSystemCatalog::TEXT: + case execplan::CalpontSystemCatalog::TIME: + case execplan::CalpontSystemCatalog::DATETIME: + { + exp2 = parm[1]->data()->getTimeIntVal(row, isNull); + + if (isNull) + { + isNull = false; + return exp1; + } + + break; + } + + default: + { + isNull = true; + } + } + + if ( exp1 == exp2 ) + { + isNull = true; + return 0; + } + + return exp1; +} + + double Func_nullif::getDoubleVal(rowgroup::Row& row, FunctionParm& parm, bool& isNull, @@ -502,6 +592,7 @@ double Func_nullif::getDoubleVal(rowgroup::Row& row, } break; + case execplan::CalpontSystemCatalog::TIME: case execplan::CalpontSystemCatalog::DATETIME: { exp2 = parm[1]->data()->getDatetimeIntVal(row, isNull); @@ -644,6 +735,7 @@ execplan::IDB_Decimal Func_nullif::getDecimalVal(rowgroup::Row& row, } break; + case execplan::CalpontSystemCatalog::TIME: case execplan::CalpontSystemCatalog::DATETIME: { int32_t s = 0; diff --git a/utils/funcexp/func_regexp.cpp b/utils/funcexp/func_regexp.cpp index 11f702a60..420c792cf 100644 --- a/utils/funcexp/func_regexp.cpp +++ b/utils/funcexp/func_regexp.cpp @@ -94,6 +94,14 @@ inline bool getBool(rowgroup::Row& row, break; } + case execplan::CalpontSystemCatalog::TIME: + { + expr = dataconvert::DataConvert::timeToString(pm[0]->data()->getTimeIntVal(row, isNull)); + //strip off micro seconds + expr = expr.substr(0, 19); + break; + } + case execplan::CalpontSystemCatalog::DECIMAL: case execplan::CalpontSystemCatalog::UDECIMAL: { @@ -153,6 +161,14 @@ inline bool getBool(rowgroup::Row& row, break; } + case execplan::CalpontSystemCatalog::TIME: + { + pattern = dataconvert::DataConvert::timeToString(pm[1]->data()->getTimeIntVal(row, isNull)); + //strip off micro seconds + pattern = pattern.substr(0, 19); + break; + } + case execplan::CalpontSystemCatalog::DECIMAL: case execplan::CalpontSystemCatalog::UDECIMAL: { diff --git a/utils/funcexp/func_round.cpp b/utils/funcexp/func_round.cpp index 18b888b2c..c107fc0fc 100644 --- a/utils/funcexp/func_round.cpp +++ b/utils/funcexp/func_round.cpp @@ -390,6 +390,7 @@ IDB_Decimal Func_round::getDecimalVal(Row& row, } break; + case execplan::CalpontSystemCatalog::TIME: case execplan::CalpontSystemCatalog::DATETIME: { int32_t s = 0; diff --git a/utils/funcexp/func_second.cpp b/utils/funcexp/func_second.cpp index 509633c46..ae12b157c 100644 --- a/utils/funcexp/func_second.cpp +++ b/utils/funcexp/func_second.cpp @@ -108,6 +108,13 @@ int64_t Func_second::getIntVal(rowgroup::Row& row, break; } + case execplan::CalpontSystemCatalog::TIME: + { + val = parm[0]->data()->getTimeIntVal(row, isNull); + return (uint32_t)((val >> 24) & 0xff); + break; + } + default: { isNull = true; diff --git a/utils/funcexp/func_sysdate.cpp b/utils/funcexp/func_sysdate.cpp index f8a5a98bd..bf71e0415 100644 --- a/utils/funcexp/func_sysdate.cpp +++ b/utils/funcexp/func_sysdate.cpp @@ -100,6 +100,15 @@ int64_t Func_sysdate::getDatetimeIntVal(rowgroup::Row& row, return getIntVal(row, parm, isNull, operationColType); } +int64_t Func_sysdate::getTimeIntVal(rowgroup::Row& row, + FunctionParm& parm, + bool& isNull, + CalpontSystemCatalog::ColType& operationColType) +{ + return getIntVal(row, parm, isNull, operationColType); +} + + } // namespace funcexp // vim:ts=4 sw=4: diff --git a/utils/funcexp/func_time.cpp b/utils/funcexp/func_time.cpp index 509caa777..4a90e0fe2 100644 --- a/utils/funcexp/func_time.cpp +++ b/utils/funcexp/func_time.cpp @@ -59,7 +59,7 @@ string Func_time::getStrVal(rowgroup::Row& row, case execplan::CalpontSystemCatalog::TINYINT: case execplan::CalpontSystemCatalog::SMALLINT: { - val = dataconvert::DataConvert::intToDatetime(parm[0]->data()->getIntVal(row, isNull)); + val = dataconvert::DataConvert::intToTime(parm[0]->data()->getIntVal(row, isNull)); if (val == -1) isNull = true; @@ -73,7 +73,7 @@ string Func_time::getStrVal(rowgroup::Row& row, { if (parm[0]->data()->resultType().scale) { - val = dataconvert::DataConvert::intToDatetime(parm[0]->data()->getIntVal(row, isNull)); + val = dataconvert::DataConvert::intToTime(parm[0]->data()->getIntVal(row, isNull)); if (val == -1) isNull = true; @@ -96,7 +96,7 @@ string Func_time::getStrVal(rowgroup::Row& row, case execplan::CalpontSystemCatalog::CHAR: case execplan::CalpontSystemCatalog::TEXT: { - val = dataconvert::DataConvert::stringToDatetime(parm[0]->data()->getStrVal(row, isNull)); + val = dataconvert::DataConvert::stringToTime(parm[0]->data()->getStrVal(row, isNull)); if (val == -1) isNull = true; @@ -106,15 +106,15 @@ string Func_time::getStrVal(rowgroup::Row& row, break; } - case execplan::CalpontSystemCatalog::DATE: + case execplan::CalpontSystemCatalog::TIME: { - val = parm[0]->data()->getDatetimeIntVal(row, isNull); + val = parm[0]->data()->getTimeIntVal(row, isNull); break; } case execplan::CalpontSystemCatalog::DATETIME: { - val = parm[0]->data()->getDatetimeIntVal(row, isNull); + val = parm[0]->data()->getTimeIntVal(row, isNull); break; } @@ -128,9 +128,9 @@ string Func_time::getStrVal(rowgroup::Row& row, return ""; char buf[30] = {'\0'}; - dataconvert::DataConvert::datetimeToString(val, buf, sizeof(buf)); + dataconvert::DataConvert::timeToString(val, buf, sizeof(buf)); string time(buf); - return time.substr(11, 80); + return time; } int64_t Func_time::getIntVal(rowgroup::Row& row, @@ -138,7 +138,7 @@ int64_t Func_time::getIntVal(rowgroup::Row& row, bool& isNull, execplan::CalpontSystemCatalog::ColType& op_ct) { - return dataconvert::DataConvert::datetimeToInt(getStrVal(row, fp, isNull, op_ct)); + return dataconvert::DataConvert::timeToInt(getStrVal(row, fp, isNull, op_ct)); } double Func_time::getDoubleVal(rowgroup::Row& row, diff --git a/utils/funcexp/func_time_to_sec.cpp b/utils/funcexp/func_time_to_sec.cpp index b87c2995a..ee5c4b669 100644 --- a/utils/funcexp/func_time_to_sec.cpp +++ b/utils/funcexp/func_time_to_sec.cpp @@ -49,12 +49,13 @@ int64_t Func_time_to_sec::getIntVal(rowgroup::Row& row, CalpontSystemCatalog::ColType& op_ct) { // assume 256 is enough. assume not allowing incomplete date - uint32_t hour = 0, - min = 0, - sec = 0; + int32_t hour = 0, + min = 0, + sec = 0; bool bIsNegative = false; // Only set to true if CHAR or VARCHAR with a '-' int64_t val = 0; + int64_t mask = 0; dataconvert::Time tval; switch (parm[0]->data()->resultType().colDataType) @@ -64,9 +65,28 @@ int64_t Func_time_to_sec::getIntVal(rowgroup::Row& row, case CalpontSystemCatalog::DATETIME: val = parm[0]->data()->getIntVal(row, isNull); - hour = (uint32_t)((val >> 32) & 0x3f); - min = (uint32_t)((val >> 26) & 0x3f); - sec = (uint32_t)((val >> 20) & 0x3f); + hour = (int32_t)((val >> 32) & 0x3f); + min = (int32_t)((val >> 26) & 0x3f); + sec = (int32_t)((val >> 20) & 0x3f); + break; + + case CalpontSystemCatalog::TIME: + val = parm[0]->data()->getTimeIntVal(row, isNull); + + // If negative, mask so it doesn't turn positive + if ((val >> 40) & 0x800) + mask = 0xfffffffffffff000; + + bIsNegative = val >> 63; + hour = (int32_t)(mask | ((val >> 40) & 0xfff)); + + if ((hour >= 0) && bIsNegative) + hour *= -1; + else + bIsNegative = false; + + min = (int32_t)((val >> 32) & 0xff); + sec = (int32_t)((val >> 24) & 0xff); break; case CalpontSystemCatalog::CHAR: @@ -112,9 +132,9 @@ int64_t Func_time_to_sec::getIntVal(rowgroup::Row& row, } else { - hour = (uint32_t)((val >> 32) & 0x3f); - min = (uint32_t)((val >> 26) & 0x3f); - sec = (uint32_t)((val >> 20) & 0x3f); + hour = (int32_t)((val >> 32) & 0x3f); + min = (int32_t)((val >> 26) & 0x3f); + sec = (int32_t)((val >> 20) & 0x3f); } break; @@ -131,9 +151,9 @@ int64_t Func_time_to_sec::getIntVal(rowgroup::Row& row, } else { - hour = (uint32_t)((val >> 32) & 0x3f); - min = (uint32_t)((val >> 26) & 0x3f); - sec = (uint32_t)((val >> 20) & 0x3f); + hour = (int32_t)((val >> 32) & 0x3f); + min = (int32_t)((val >> 26) & 0x3f); + sec = (int32_t)((val >> 20) & 0x3f); } } @@ -144,7 +164,16 @@ int64_t Func_time_to_sec::getIntVal(rowgroup::Row& row, return -1; } - int64_t rtn = (int64_t)(hour * 60 * 60) + (min * 60) + sec; + int64_t rtn; + + if (hour < 0) + { + rtn = (int64_t)(hour * 60 * 60) - (min * 60) - sec; + } + else + { + rtn = (int64_t)(hour * 60 * 60) + (min * 60) + sec; + } if (bIsNegative) { diff --git a/utils/funcexp/func_timediff.cpp b/utils/funcexp/func_timediff.cpp index c87e99b64..742e8faf7 100644 --- a/utils/funcexp/func_timediff.cpp +++ b/utils/funcexp/func_timediff.cpp @@ -116,6 +116,7 @@ string Func_timediff::getStrVal(rowgroup::Row& row, isDate1 = true; break; + case execplan::CalpontSystemCatalog::TIME: case execplan::CalpontSystemCatalog::DATETIME: val1 = parm[0]->data()->getDatetimeIntVal(row, isNull); break; @@ -157,6 +158,7 @@ string Func_timediff::getStrVal(rowgroup::Row& row, isDate2 = true; break; + case execplan::CalpontSystemCatalog::TIME: case execplan::CalpontSystemCatalog::DATETIME: val2 = parm[1]->data()->getDatetimeIntVal(row, isNull); break; @@ -213,6 +215,14 @@ int64_t Func_timediff::getDatetimeIntVal(rowgroup::Row& row, return dataconvert::DataConvert::datetimeToInt(getStrVal(row, parm, isNull, ct)); } +int64_t Func_timediff::getTimeIntVal(rowgroup::Row& row, + FunctionParm& parm, + bool& isNull, + CalpontSystemCatalog::ColType& ct) +{ + return dataconvert::DataConvert::timeToInt(getStrVal(row, parm, isNull, ct)); +} + int64_t Func_timediff::getIntVal(rowgroup::Row& row, FunctionParm& parm, bool& isNull, diff --git a/utils/funcexp/func_timestampdiff.cpp b/utils/funcexp/func_timestampdiff.cpp index f8d1fc7ac..9394203d7 100644 --- a/utils/funcexp/func_timestampdiff.cpp +++ b/utils/funcexp/func_timestampdiff.cpp @@ -152,6 +152,13 @@ int64_t Func_timestampdiff::getDatetimeIntVal(rowgroup::Row& row, return getIntVal(row, parm, isNull, ct); } +int64_t Func_timestampdiff::getTimeIntVal(rowgroup::Row& row, + FunctionParm& parm, + bool& isNull, + CalpontSystemCatalog::ColType& ct) +{ + return getIntVal(row, parm, isNull, ct); +} } // namespace funcexp // vim:ts=4 sw=4: diff --git a/utils/funcexp/func_truncate.cpp b/utils/funcexp/func_truncate.cpp index c8be27de0..b822caacd 100644 --- a/utils/funcexp/func_truncate.cpp +++ b/utils/funcexp/func_truncate.cpp @@ -387,6 +387,51 @@ IDB_Decimal Func_truncate::getDecimalVal(Row& row, } break; + case execplan::CalpontSystemCatalog::TIME: + { + int32_t s = 0; + int64_t x = 0; + + string value = + DataConvert::timeToString1(parm[0]->data()->getTimeIntVal(row, isNull)); + + s = parm[1]->data()->getIntVal(row, isNull); + + if (!isNull) + { + //strip off micro seconds + value = value.substr(0, 14); + int64_t x = atoll(value.c_str()); + + if ( s > 5 ) + s = 0; + + if ( s > 0 ) + { + x *= helpers::powerOf10_c[s]; + } + else if (s < 0) + { + s = -s; + + if ( s >= (int32_t) value.size() ) + { + x = 0; + } + else + { + x /= helpers::powerOf10_c[s]; + x *= helpers::powerOf10_c[s]; + } + + s = 0; + } + } + + decimal.value = x; + decimal.scale = s; + } + break; default: { diff --git a/utils/funcexp/funcexp.cpp b/utils/funcexp/funcexp.cpp index a59be9a7c..3c530f381 100644 --- a/utils/funcexp/funcexp.cpp +++ b/utils/funcexp/funcexp.cpp @@ -280,6 +280,18 @@ void FuncExp::evaluate(rowgroup::Row& row, std::vector& expressi break; } + case CalpontSystemCatalog::TIME: + { + int64_t val = expression[i]->getTimeIntVal(row, isNull); + + if (isNull) + row.setIntField<8>(TIMENULL, expression[i]->outputIndex()); + else + row.setIntField<8>(val, expression[i]->outputIndex()); + + break; + } + case CalpontSystemCatalog::CHAR: case CalpontSystemCatalog::VARCHAR: diff --git a/utils/funcexp/functor.cpp b/utils/funcexp/functor.cpp index 662db6448..b74812ee9 100644 --- a/utils/funcexp/functor.cpp +++ b/utils/funcexp/functor.cpp @@ -106,6 +106,21 @@ uint64_t Func::stringToDatetime(const string str) return ret; } +int64_t Func::stringToTime(const string str) +{ + int64_t ret = DataConvert::stringToTime(str); + + if (ret == -1) + { + Message::Args args; + args.add("time"); + args.add(str); + unsigned errcode = ERR_INCORRECT_VALUE; + throw IDBExcept(IDBErrorInfo::instance()->errorMsg(errcode, args), errcode); + } + + return ret; +} uint32_t Func::intToDate(int64_t i) { @@ -124,6 +139,11 @@ uint64_t Func::intToDatetime(int64_t i) return i; } +int64_t Func::intToTime(int64_t i) +{ + // Don't think we need to do anything here? + return i; +} string Func::intToString(int64_t i) { diff --git a/utils/funcexp/functor.h b/utils/funcexp/functor.h index 9e03e4af2..9edb9bf62 100644 --- a/utils/funcexp/functor.h +++ b/utils/funcexp/functor.h @@ -116,6 +116,14 @@ public: return intToDatetime(getIntVal(row, fp, isNull, op_ct)); } + virtual int64_t getTimeIntVal(rowgroup::Row& row, + FunctionParm& fp, + bool& isNull, + execplan::CalpontSystemCatalog::ColType& op_ct) + { + return intToTime(getIntVal(row, fp, isNull, op_ct)); + } + virtual bool getBoolVal(rowgroup::Row& row, FunctionParm& fp, bool& isNull, @@ -145,9 +153,11 @@ public: protected: virtual uint32_t stringToDate(std::string); virtual uint64_t stringToDatetime(std::string); + virtual int64_t stringToTime(std::string); virtual uint32_t intToDate(int64_t); virtual uint64_t intToDatetime(int64_t); + virtual int64_t intToTime(int64_t); virtual std::string intToString(int64_t); virtual std::string doubleToString(double); diff --git a/utils/funcexp/functor_all.h b/utils/funcexp/functor_all.h index 6300084cd..553abc40f 100644 --- a/utils/funcexp/functor_all.h +++ b/utils/funcexp/functor_all.h @@ -104,6 +104,11 @@ public: FunctionParm& fp, bool& isNull, execplan::CalpontSystemCatalog::ColType& op_ct); + + int64_t getTimeIntVal(rowgroup::Row& row, + FunctionParm& fp, + bool& isNull, + execplan::CalpontSystemCatalog::ColType& op_ct); }; @@ -151,6 +156,12 @@ public: FunctionParm& fp, bool& isNull, execplan::CalpontSystemCatalog::ColType& op_ct); + + int64_t getTimeIntVal(rowgroup::Row& row, + FunctionParm& fp, + bool& isNull, + execplan::CalpontSystemCatalog::ColType& op_ct); + }; @@ -193,6 +204,11 @@ public: FunctionParm& fp, bool& isNull, execplan::CalpontSystemCatalog::ColType& op_ct); + + int64_t getTimeIntVal(rowgroup::Row& row, + FunctionParm& fp, + bool& isNull, + execplan::CalpontSystemCatalog::ColType& op_ct); }; @@ -236,6 +252,11 @@ public: bool& isNull, execplan::CalpontSystemCatalog::ColType& op_ct); + int64_t getTimeIntVal(rowgroup::Row& row, + FunctionParm& fp, + bool& isNull, + execplan::CalpontSystemCatalog::ColType& op_ct); + bool getBoolVal(rowgroup::Row& row, FunctionParm& fp, bool& isNull, @@ -287,6 +308,12 @@ public: FunctionParm& fp, bool& isNull, execplan::CalpontSystemCatalog::ColType& op_ct); + + int64_t getTimeIntVal(rowgroup::Row& row, + FunctionParm& fp, + bool& isNull, + execplan::CalpontSystemCatalog::ColType& op_ct); + }; @@ -329,6 +356,11 @@ public: FunctionParm& fp, bool& isNull, execplan::CalpontSystemCatalog::ColType& op_ct); + + int64_t getTimeIntVal(rowgroup::Row& row, + FunctionParm& fp, + bool& isNull, + execplan::CalpontSystemCatalog::ColType& op_ct); }; @@ -371,6 +403,11 @@ public: FunctionParm& fp, bool& isNull, execplan::CalpontSystemCatalog::ColType& op_ct); + + int64_t getTimeIntVal(rowgroup::Row& row, + FunctionParm& fp, + bool& isNull, + execplan::CalpontSystemCatalog::ColType& op_ct); }; @@ -399,6 +436,11 @@ public: bool& isNull, execplan::CalpontSystemCatalog::ColType& op_ct); + int64_t getTimeIntVal(rowgroup::Row& row, + FunctionParm& fp, + bool& isNull, + execplan::CalpontSystemCatalog::ColType& op_ct); + int64_t getIntVal(rowgroup::Row& row, FunctionParm& fp, bool& isNull, diff --git a/utils/funcexp/functor_bool.h b/utils/funcexp/functor_bool.h index a20f925b3..b7b85106b 100644 --- a/utils/funcexp/functor_bool.h +++ b/utils/funcexp/functor_bool.h @@ -97,6 +97,15 @@ public: isNull = true; return 0; } + + int64_t getTimeIntVal(rowgroup::Row& row, + FunctionParm& fp, + bool& isNull, + execplan::CalpontSystemCatalog::ColType& op_ct) + { + isNull = true; + return 0; + } }; diff --git a/utils/funcexp/functor_dtm.h b/utils/funcexp/functor_dtm.h index dc662a176..d7837a4fe 100644 --- a/utils/funcexp/functor_dtm.h +++ b/utils/funcexp/functor_dtm.h @@ -137,6 +137,11 @@ public: bool& isNull, execplan::CalpontSystemCatalog::ColType& op_ct); + int64_t getTimeIntVal(rowgroup::Row& row, + FunctionParm& fp, + bool& isNull, + execplan::CalpontSystemCatalog::ColType& op_ct); + int64_t getIntVal(rowgroup::Row& row, FunctionParm& fp, bool& isNull, @@ -247,6 +252,12 @@ public: FunctionParm& fp, bool& isNull, execplan::CalpontSystemCatalog::ColType& op_ct); + + int64_t getTimeIntVal(rowgroup::Row& row, + FunctionParm& fp, + bool& isNull, + execplan::CalpontSystemCatalog::ColType& op_ct); + }; @@ -311,6 +322,11 @@ public: FunctionParm& fp, bool& isNull, execplan::CalpontSystemCatalog::ColType& op_ct); + + int64_t getTimeIntVal(rowgroup::Row& row, + FunctionParm& fp, + bool& isNull, + execplan::CalpontSystemCatalog::ColType& op_ct); }; @@ -343,6 +359,11 @@ public: FunctionParm& fp, bool& isNull, execplan::CalpontSystemCatalog::ColType& op_ct); + + int64_t getTimeIntVal(rowgroup::Row& row, + FunctionParm& fp, + bool& isNull, + execplan::CalpontSystemCatalog::ColType& op_ct); }; @@ -371,6 +392,11 @@ public: bool& isNull, execplan::CalpontSystemCatalog::ColType& op_ct); + int64_t getTimeIntVal(rowgroup::Row& row, + FunctionParm& fp, + bool& isNull, + execplan::CalpontSystemCatalog::ColType& op_ct); + int64_t getIntVal(rowgroup::Row& row, FunctionParm& fp, bool& isNull, @@ -411,6 +437,11 @@ public: FunctionParm& fp, bool& isNull, execplan::CalpontSystemCatalog::ColType& op_ct); + + int64_t getTimeIntVal(rowgroup::Row& row, + FunctionParm& fp, + bool& isNull, + execplan::CalpontSystemCatalog::ColType& op_ct); }; /** @brief Func_str_to_date class diff --git a/utils/funcexp/functor_real.h b/utils/funcexp/functor_real.h index 936401f80..5d40ce589 100644 --- a/utils/funcexp/functor_real.h +++ b/utils/funcexp/functor_real.h @@ -654,6 +654,12 @@ public: bool& isNull, execplan::CalpontSystemCatalog::ColType& op_ct); + int64_t getTimeIntVal(rowgroup::Row& row, + FunctionParm& fp, + bool& isNull, + execplan::CalpontSystemCatalog::ColType& op_ct); + + private: int64_t convertAton(const std::string& ipString, bool& isNull); }; diff --git a/utils/funcexp/functor_str.h b/utils/funcexp/functor_str.h index 0b40685f6..c71cdec91 100644 --- a/utils/funcexp/functor_str.h +++ b/utils/funcexp/functor_str.h @@ -89,6 +89,14 @@ public: return (isNull ? 0 : stringToDatetime(str)); } + int64_t getTimeIntVal(rowgroup::Row& row, + FunctionParm& fp, + bool& isNull, + execplan::CalpontSystemCatalog::ColType& op_ct) + { + std::string str = getStrVal(row, fp, isNull, op_ct); + return (isNull ? 0 : stringToTime(str)); + } protected: const std::string& stringValue(execplan::SPTP& fp, rowgroup::Row& row, bool& isNull) @@ -676,6 +684,10 @@ public: bool& isNull, execplan::CalpontSystemCatalog::ColType& op_ct); + int64_t getTimeIntVal(rowgroup::Row& row, + FunctionParm& fp, + bool& isNull, + execplan::CalpontSystemCatalog::ColType& op_ct); private: void convertNtoa(int64_t ipNum, std::string& ipString); }; diff --git a/utils/rowgroup/rowaggregation.cpp b/utils/rowgroup/rowaggregation.cpp index 0a98f6870..8d110cfc8 100644 --- a/utils/rowgroup/rowaggregation.cpp +++ b/utils/rowgroup/rowaggregation.cpp @@ -125,6 +125,11 @@ inline uint64_t getUintNullValue(int colType, int colWidth = 0) return joblist::DATETIMENULL; } + case execplan::CalpontSystemCatalog::TIME: + { + return joblist::TIMENULL; + } + case execplan::CalpontSystemCatalog::DECIMAL: case execplan::CalpontSystemCatalog::UDECIMAL: { @@ -640,6 +645,12 @@ inline bool RowAggregation::isNull(const RowGroup* pRowGroup, const Row& row, in break; } + case execplan::CalpontSystemCatalog::TIME: + { + ret = ((uint64_t)row.getUintField(col) == joblist::TIMENULL); + break; + } + case execplan::CalpontSystemCatalog::VARBINARY: case execplan::CalpontSystemCatalog::BLOB: { @@ -1117,6 +1128,7 @@ void RowAggregation::initMapData(const Row& rowIn) case execplan::CalpontSystemCatalog::DATE: case execplan::CalpontSystemCatalog::DATETIME: + case execplan::CalpontSystemCatalog::TIME: { fRow.setUintField(rowIn.getUintField(colIn), colOut); break; @@ -1247,6 +1259,7 @@ void RowAggregation::makeAggFieldsNull(Row& row) case execplan::CalpontSystemCatalog::DATE: case execplan::CalpontSystemCatalog::DATETIME: + case execplan::CalpontSystemCatalog::TIME: { row.setUintField(getUintNullValue(colDataType), colOut); break; @@ -1375,6 +1388,7 @@ void RowAggregation::doMinMaxSum(const Row& rowIn, int64_t colIn, int64_t colOut case execplan::CalpontSystemCatalog::DATE: case execplan::CalpontSystemCatalog::DATETIME: + case execplan::CalpontSystemCatalog::TIME: { if (funcType == ROWAGG_SUM) { @@ -1518,9 +1532,26 @@ void RowAggregation::doBitOp(const Row& rowIn, int64_t colIn, int64_t colOut, in case execplan::CalpontSystemCatalog::DATETIME: { uint64_t dtm = rowIn.getUintField(colIn); - valIn = ((dtm >> 48) * 10000000000LL) + (((dtm >> 44) & 0xF) * 100000000) + - (((dtm >> 38) & 077) * 1000000) + (((dtm >> 32) & 077) * 10000) + - (((dtm >> 26) & 077) * 100) + ((dtm >> 20) & 077); + valIn = ((dtm >> 48) * 10000000000000000LL) + (((dtm >> 44) & 0xF) * 100000000000000) + + (((dtm >> 38) & 077) * 1000000000000) + (((dtm >> 32) & 077) * 10000000000) + + (((dtm >> 26) & 077) * 100000000) + (((dtm >> 20) & 077) * 1000000) + (dtm & 0xfffff); + break; + } + + case execplan::CalpontSystemCatalog::TIME: + { + int64_t dtm = rowIn.getUintField(colIn); + // Handle negative correctly + int hour = 0; + + if ((dtm >> 40) & 0x800) + { + hour = 0xfffff000; + } + + hour |= ((dtm >> 40) & 0xfff); + valIn = (hour * 10000000000) + + (((dtm >> 32) & 0xff) * 100000000) + (((dtm >> 24) & 0xff) * 1000000) + (dtm & 0xffffff); break; } @@ -2048,6 +2079,13 @@ void RowAggregation::doUDAF(const Row& rowIn, int64_t colIn, int64_t colOut, int break; } + case execplan::CalpontSystemCatalog::TIME: + { + datum.dataType = execplan::CalpontSystemCatalog::BIGINT; + datum.columnData = rowIn.getIntField(colIn); + break; + } + case execplan::CalpontSystemCatalog::CHAR: case execplan::CalpontSystemCatalog::VARCHAR: case execplan::CalpontSystemCatalog::TEXT: @@ -2723,6 +2761,11 @@ void RowAggregationUM::SetUDAFValue(static_any::any& valOut, int64_t colOut) fRow.setUintField<8>(uintOut, colOut); break; + case execplan::CalpontSystemCatalog::TIME: + + fRow.setIntField<8>(intOut, colOut); + break; + case execplan::CalpontSystemCatalog::FLOAT: case execplan::CalpontSystemCatalog::UFLOAT: fRow.setFloatField(floatOut, colOut); @@ -3031,6 +3074,12 @@ void RowAggregationUM::doNullConstantAggregate(const ConstantAggData& aggData, u } break; + case execplan::CalpontSystemCatalog::TIME: + { + fRow.setIntField(getIntNullValue(colDataType), colOut); + } + break; + case execplan::CalpontSystemCatalog::CHAR: case execplan::CalpontSystemCatalog::VARCHAR: case execplan::CalpontSystemCatalog::TEXT: @@ -3207,6 +3256,12 @@ void RowAggregationUM::doNotNullConstantAggregate(const ConstantAggData& aggData } break; + case execplan::CalpontSystemCatalog::TIME: + { + fRow.setIntField(DataConvert::stringToTime(aggData.fConstValue), colOut); + } + break; + case execplan::CalpontSystemCatalog::CHAR: case execplan::CalpontSystemCatalog::VARCHAR: case execplan::CalpontSystemCatalog::TEXT: @@ -3295,6 +3350,7 @@ void RowAggregationUM::doNotNullConstantAggregate(const ConstantAggData& aggData case execplan::CalpontSystemCatalog::DATE: case execplan::CalpontSystemCatalog::DATETIME: + case execplan::CalpontSystemCatalog::TIME: case execplan::CalpontSystemCatalog::CHAR: case execplan::CalpontSystemCatalog::VARCHAR: case execplan::CalpontSystemCatalog::TEXT: @@ -3355,6 +3411,7 @@ void RowAggregationUM::doNotNullConstantAggregate(const ConstantAggData& aggData break; case execplan::CalpontSystemCatalog::DATETIME: + case execplan::CalpontSystemCatalog::TIME: { fRow.setUintField(0, colOut); } @@ -3492,6 +3549,12 @@ void RowAggregationUM::doNotNullConstantAggregate(const ConstantAggData& aggData } break; + case execplan::CalpontSystemCatalog::TIME: + { + datum.columnData = DataConvert::stringToTime(aggData.fConstValue); + } + break; + case execplan::CalpontSystemCatalog::CHAR: case execplan::CalpontSystemCatalog::VARCHAR: case execplan::CalpontSystemCatalog::TEXT: diff --git a/utils/rowgroup/rowgroup.cpp b/utils/rowgroup/rowgroup.cpp index c35129d03..3411f6fa6 100644 --- a/utils/rowgroup/rowgroup.cpp +++ b/utils/rowgroup/rowgroup.cpp @@ -82,10 +82,10 @@ StringStore::~StringStore() #endif } -uint32_t StringStore::storeString(const uint8_t* data, uint32_t len) +uint64_t StringStore::storeString(const uint8_t* data, uint32_t len) { MemChunk* lastMC = NULL; - uint32_t ret = 0; + uint64_t ret = 0; empty = false; // At least a NULL is being stored. @@ -95,7 +95,7 @@ uint32_t StringStore::storeString(const uint8_t* data, uint32_t len) if ((len == 8 || len == 9) && *((uint64_t*) data) == *((uint64_t*) joblist::CPNULLSTRMARK.c_str())) - return numeric_limits::max(); + return numeric_limits::max(); //@bug6065, make StringStore::storeString() thread safe boost::mutex::scoped_lock lk(fMutex, defer_lock); @@ -106,20 +106,21 @@ uint32_t StringStore::storeString(const uint8_t* data, uint32_t len) if (mem.size() > 0) lastMC = (MemChunk*) mem.back().get(); - if (len >= CHUNK_SIZE) + if ((len + 4) >= CHUNK_SIZE) { - shared_array newOne(new uint8_t[len + sizeof(MemChunk)]); + shared_array newOne(new uint8_t[len + sizeof(MemChunk) + 4]); longStrings.push_back(newOne); lastMC = (MemChunk*) longStrings.back().get(); - lastMC->capacity = lastMC->currentSize = len; - memcpy(lastMC->data, data, len); + lastMC->capacity = lastMC->currentSize = len + 4; + memcpy(lastMC->data, &len, 4); + memcpy(lastMC->data + 4, data, len); // High bit to mark a long string - ret = 0x80000000; + ret = 0x8000000000000000; ret += longStrings.size() - 1; } else { - if ((lastMC == NULL) || (lastMC->capacity - lastMC->currentSize < len)) + if ((lastMC == NULL) || (lastMC->capacity - lastMC->currentSize < (len + 4))) { // mem usage debugging //if (lastMC) @@ -134,7 +135,13 @@ uint32_t StringStore::storeString(const uint8_t* data, uint32_t len) ret = ((mem.size() - 1) * CHUNK_SIZE) + lastMC->currentSize; - memcpy(&(lastMC->data[lastMC->currentSize]), data, len); + + // If this ever happens then we have big problems + if (ret & 0x8000000000000000) + throw logic_error("StringStore memory exceeded."); + + memcpy(&(lastMC->data[lastMC->currentSize]), &len, 4); + memcpy(&(lastMC->data[lastMC->currentSize]) + 4, data, len); /* cout << "stored: '" << hex; for (uint32_t i = 0; i < len ; i++) { @@ -142,7 +149,7 @@ uint32_t StringStore::storeString(const uint8_t* data, uint32_t len) } cout << "' at position " << lastMC->currentSize << " len " << len << dec << endl; */ - lastMC->currentSize += len; + lastMC->currentSize += len + 4; } return ret; @@ -150,35 +157,35 @@ uint32_t StringStore::storeString(const uint8_t* data, uint32_t len) void StringStore::serialize(ByteStream& bs) const { - uint32_t i; + uint64_t i; MemChunk* mc; - bs << (uint32_t) mem.size(); + bs << (uint64_t) mem.size(); bs << (uint8_t) empty; for (i = 0; i < mem.size(); i++) { mc = (MemChunk*) mem[i].get(); - bs << (uint32_t) mc->currentSize; + bs << (uint64_t) mc->currentSize; //cout << "serialized " << mc->currentSize << " bytes\n"; bs.append(mc->data, mc->currentSize); } - bs << (uint32_t) longStrings.size(); + bs << (uint64_t) longStrings.size(); for (i = 0; i < longStrings.size(); i++) { mc = (MemChunk*) longStrings[i].get(); - bs << (uint32_t) mc->currentSize; + bs << (uint64_t) mc->currentSize; bs.append(mc->data, mc->currentSize); } } void StringStore::deserialize(ByteStream& bs) { - uint32_t i; - uint32_t count; - uint32_t size; + uint64_t i; + uint64_t count; + uint64_t size; uint8_t* buf; MemChunk* mc; uint8_t tmp8; @@ -699,6 +706,10 @@ void Row::initToNull() *((uint64_t*) &data[offsets[i]]) = joblist::DATETIMENULL; break; + case CalpontSystemCatalog::TIME: + *((uint64_t*) &data[offsets[i]]) = joblist::TIMENULL; + break; + case CalpontSystemCatalog::CHAR: case CalpontSystemCatalog::VARCHAR: case CalpontSystemCatalog::TEXT: @@ -841,6 +852,9 @@ bool Row::isNullValue(uint32_t colIndex) const case CalpontSystemCatalog::DATETIME: return (*((uint64_t*) &data[offsets[colIndex]]) == joblist::DATETIMENULL); + case CalpontSystemCatalog::TIME: + return (*((uint64_t*) &data[offsets[colIndex]]) == joblist::TIMENULL); + case CalpontSystemCatalog::CHAR: case CalpontSystemCatalog::VARCHAR: case CalpontSystemCatalog::STRINT: @@ -849,10 +863,9 @@ bool Row::isNullValue(uint32_t colIndex) const if (inStringTable(colIndex)) { - uint32_t offset, length; - offset = *((uint32_t*) &data[offsets[colIndex]]); - length = *((uint32_t*) &data[offsets[colIndex] + 4]); - return strings->isNullValue(offset, length); + uint64_t offset; + offset = *((uint64_t*) &data[offsets[colIndex]]); + return strings->isNullValue(offset); } if (data[offsets[colIndex]] == 0) // empty string @@ -915,10 +928,9 @@ bool Row::isNullValue(uint32_t colIndex) const if (inStringTable(colIndex)) { - uint32_t offset, length; - offset = *((uint32_t*) &data[pos]); - length = *((uint32_t*) &data[pos + 4]); - return strings->isNullValue(offset, length); + uint64_t offset; + offset = *((uint64_t*) &data[pos]); + return strings->isNullValue(offset); } if (*((uint16_t*) &data[pos]) == 0) @@ -1713,8 +1725,8 @@ RGData RowGroup::duplicate() void Row::setStringField(const std::string& val, uint32_t colIndex) { - uint32_t length; - uint32_t offset; + uint64_t offset; + uint64_t length; //length = strlen(val.c_str()) + 1; length = val.length(); @@ -1725,8 +1737,7 @@ void Row::setStringField(const std::string& val, uint32_t colIndex) if (inStringTable(colIndex)) { offset = strings->storeString((const uint8_t*) val.data(), length); - *((uint32_t*) &data[offsets[colIndex]]) = offset; - *((uint32_t*) &data[offsets[colIndex] + 4]) = length; + *((uint64_t*) &data[offsets[colIndex]]) = offset; // cout << " -- stored offset " << *((uint32_t *) &data[offsets[colIndex]]) // << " length " << *((uint32_t *) &data[offsets[colIndex] + 4]) // << endl; diff --git a/utils/rowgroup/rowgroup.h b/utils/rowgroup/rowgroup.h index 135479918..896da1f4a 100644 --- a/utils/rowgroup/rowgroup.h +++ b/utils/rowgroup/rowgroup.h @@ -92,13 +92,14 @@ public: StringStore(); virtual ~StringStore(); - inline std::string getString(uint32_t offset, uint32_t length) const; - uint32_t storeString(const uint8_t* data, uint32_t length); //returns the offset - inline const uint8_t* getPointer(uint32_t offset) const; + inline std::string getString(uint64_t offset) const; + uint64_t storeString(const uint8_t* data, uint32_t length); //returns the offset + inline const uint8_t* getPointer(uint64_t offset) const; + inline uint32_t getStringLength(uint64_t offset); inline bool isEmpty() const; inline uint64_t getSize() const; - inline bool isNullValue(uint32_t offset, uint32_t length) const; - inline bool equals(const std::string& str, uint32_t offset, uint32_t length) const; + inline bool isNullValue(uint64_t offset) const; + inline bool equals(const std::string& str, uint64_t offset) const; void clear(); @@ -615,9 +616,8 @@ inline bool Row::equals(const std::string& val, uint32_t colIndex) const { if (inStringTable(colIndex)) { - uint32_t offset = *((uint32_t*) &data[offsets[colIndex]]); - uint32_t length = *((uint32_t*) &data[offsets[colIndex] + 4]); - return strings->equals(val, offset, length); + uint64_t offset = *((uint64_t*) &data[offsets[colIndex]]); + return strings->equals(val, offset); } else return (strncmp(val.c_str(), (char*) &data[offsets[colIndex]], getColumnWidth(colIndex)) == 0); @@ -719,7 +719,7 @@ inline int64_t Row::getIntField(uint32_t colIndex) const inline const uint8_t* Row::getStringPointer(uint32_t colIndex) const { if (inStringTable(colIndex)) - return strings->getPointer(*((uint32_t*) &data[offsets[colIndex]])); + return strings->getPointer(*((uint64_t*) &data[offsets[colIndex]])); return &data[offsets[colIndex]]; } @@ -727,14 +727,14 @@ inline const uint8_t* Row::getStringPointer(uint32_t colIndex) const inline uint32_t Row::getStringLength(uint32_t colIndex) const { if (inStringTable(colIndex)) - return *((uint32_t*) &data[offsets[colIndex] + 4]); + return strings->getStringLength(*((uint64_t*) &data[offsets[colIndex]])); return strnlen((char*) &data[offsets[colIndex]], getColumnWidth(colIndex)); } inline void Row::setStringField(const uint8_t* strdata, uint32_t length, uint32_t colIndex) { - uint32_t offset; + uint64_t offset; if (length > getColumnWidth(colIndex)) length = getColumnWidth(colIndex); @@ -742,8 +742,7 @@ inline void Row::setStringField(const uint8_t* strdata, uint32_t length, uint32_ if (inStringTable(colIndex)) { offset = strings->storeString(strdata, length); - *((uint32_t*) &data[offsets[colIndex]]) = offset; - *((uint32_t*) &data[offsets[colIndex] + 4]) = length; + *((uint64_t*) &data[offsets[colIndex]]) = offset; // cout << " -- stored offset " << *((uint32_t *) &data[offsets[colIndex]]) // << " length " << *((uint32_t *) &data[offsets[colIndex] + 4]) // << endl; @@ -759,8 +758,7 @@ inline void Row::setStringField(const uint8_t* strdata, uint32_t length, uint32_ inline std::string Row::getStringField(uint32_t colIndex) const { if (inStringTable(colIndex)) - return strings->getString(*((uint32_t*) &data[offsets[colIndex]]), - *((uint32_t*) &data[offsets[colIndex] + 4])); + return strings->getString(*((uint64_t*) &data[offsets[colIndex]])); // Not all CHAR/VARCHAR are NUL terminated so use length return std::string((char*) &data[offsets[colIndex]], @@ -778,7 +776,7 @@ inline std::string Row::getVarBinaryStringField(uint32_t colIndex) const inline uint32_t Row::getVarBinaryLength(uint32_t colIndex) const { if (inStringTable(colIndex)) - return *((uint32_t*) &data[offsets[colIndex] + 4]); + return strings->getStringLength(*((uint64_t*) &data[offsets[colIndex]]));; return *((uint16_t*) &data[offsets[colIndex]]); } @@ -786,7 +784,7 @@ inline uint32_t Row::getVarBinaryLength(uint32_t colIndex) const inline const uint8_t* Row::getVarBinaryField(uint32_t colIndex) const { if (inStringTable(colIndex)) - return strings->getPointer(*((uint32_t*) &data[offsets[colIndex]])); + return strings->getPointer(*((uint64_t*) &data[offsets[colIndex]])); return &data[offsets[colIndex] + 2]; } @@ -795,7 +793,7 @@ inline const uint8_t* Row::getVarBinaryField(uint32_t& len, uint32_t colIndex) c { if (inStringTable(colIndex)) { - len = *((uint32_t*) &data[offsets[colIndex] + 4]); + len = strings->getStringLength(*((uint64_t*) &data[offsets[colIndex]])); return getVarBinaryField(colIndex); } else @@ -1043,9 +1041,8 @@ inline void Row::setVarBinaryField(const uint8_t* val, uint32_t len, uint32_t co if (inStringTable(colIndex)) { - uint32_t offset = strings->storeString(val, len); - *((uint32_t*) &data[offsets[colIndex]]) = offset; - *((uint32_t*) &data[offsets[colIndex] + 4]) = len; + uint64_t offset = strings->storeString(val, len); + *((uint64_t*) &data[offsets[colIndex]]) = offset; } else { @@ -1762,26 +1759,29 @@ inline void copyRow(const Row& in, Row* out) copyRow(in, out, std::min(in.getColumnCount(), out->getColumnCount())); } -inline std::string StringStore::getString(uint32_t off, uint32_t len) const +inline std::string StringStore::getString(uint64_t off) const { - if (off == std::numeric_limits::max()) + uint32_t length; + + if (off == std::numeric_limits::max()) return joblist::CPNULLSTRMARK; MemChunk* mc; - if (off & 0x80000000) + if (off & 0x8000000000000000) { - off = off - 0x80000000; + off = off - 0x8000000000000000; if (longStrings.size() <= off) return joblist::CPNULLSTRMARK; mc = (MemChunk*) longStrings[off].get(); - return std::string((char*) mc->data, len); + memcpy(&length, mc->data, 4); + return std::string((char*) mc->data + 4, length); } - uint32_t chunk = off / CHUNK_SIZE; - uint32_t offset = off % CHUNK_SIZE; + uint64_t chunk = off / CHUNK_SIZE; + uint64_t offset = off % CHUNK_SIZE; // this has to handle uninitialized data as well. If it's uninitialized it doesn't matter // what gets returned, it just can't go out of bounds. @@ -1790,30 +1790,32 @@ inline std::string StringStore::getString(uint32_t off, uint32_t len) const mc = (MemChunk*) mem[chunk].get(); - if ((offset + len) > mc->currentSize) + memcpy(&length, &mc->data[offset], 4); + + if ((offset + length) > mc->currentSize) return joblist::CPNULLSTRMARK; - return std::string((char*) & (mc->data[offset]), len); + return std::string((char*) & (mc->data[offset]) + 4, length); } -inline const uint8_t* StringStore::getPointer(uint32_t off) const +inline const uint8_t* StringStore::getPointer(uint64_t off) const { - if (off == std::numeric_limits::max()) + if (off == std::numeric_limits::max()) return (const uint8_t*) joblist::CPNULLSTRMARK.c_str(); - uint32_t chunk = off / CHUNK_SIZE; - uint32_t offset = off % CHUNK_SIZE; + uint64_t chunk = off / CHUNK_SIZE; + uint64_t offset = off % CHUNK_SIZE; MemChunk* mc; - if (off & 0x80000000) + if (off & 0x8000000000000000) { - off = off - 0x80000000; + off = off - 0x8000000000000000; if (longStrings.size() <= off) return (const uint8_t*) joblist::CPNULLSTRMARK.c_str(); mc = (MemChunk*) longStrings[off].get(); - return mc->data; + return mc->data + 4; } // this has to handle uninitialized data as well. If it's uninitialized it doesn't matter @@ -1826,19 +1828,18 @@ inline const uint8_t* StringStore::getPointer(uint32_t off) const if (offset > mc->currentSize) return (const uint8_t*) joblist::CPNULLSTRMARK.c_str(); - return &(mc->data[offset]); + return &(mc->data[offset]) + 4; } -inline bool StringStore::isNullValue(uint32_t off, uint32_t len) const +inline bool StringStore::isNullValue(uint64_t off) const { - if (off == std::numeric_limits::max() || len == 0) + uint32_t length; + + if (off == std::numeric_limits::max()) return true; - if (len < 8) - return false; - // Long strings won't be NULL - if (off & 0x80000000) + if (off & 0x8000000000000000) return false; uint32_t chunk = off / CHUNK_SIZE; @@ -1849,35 +1850,46 @@ inline bool StringStore::isNullValue(uint32_t off, uint32_t len) const return true; mc = (MemChunk*) mem[chunk].get(); + memcpy(&length, &mc->data[offset], 4); - if ((offset + len) > mc->currentSize) + if (length == 0) return true; - if (mc->data[offset] == 0) // "" = NULL string for some reason... + if (length < 8) + return false; + + if ((offset + length) > mc->currentSize) return true; - return (*((uint64_t*) &mc->data[offset]) == *((uint64_t*) joblist::CPNULLSTRMARK.c_str())); + if (mc->data[offset + 4] == 0) // "" = NULL string for some reason... + return true; + + return (*((uint64_t*) &mc->data[offset] + 4) == *((uint64_t*) joblist::CPNULLSTRMARK.c_str())); } -inline bool StringStore::equals(const std::string& str, uint32_t off, uint32_t len) const +inline bool StringStore::equals(const std::string& str, uint64_t off) const { - if (off == std::numeric_limits::max() || len == 0) + uint32_t length; + + if (off == std::numeric_limits::max()) return str == joblist::CPNULLSTRMARK; MemChunk* mc; - if (off & 0x80000000) + if (off & 0x8000000000000000) { - if (longStrings.size() <= (off - 0x80000000)) + if (longStrings.size() <= (off - 0x8000000000000000)) return false; - mc = (MemChunk*) longStrings[off - 0x80000000].get(); + mc = (MemChunk*) longStrings[off - 0x8000000000000000].get(); + + memcpy(&length, mc->data, 4); // Not sure if this check it needed, but adds safety - if (len > mc->currentSize) + if (length > mc->currentSize) return false; - return (strncmp(str.c_str(), (const char*) mc->data, len) == 0); + return (strncmp(str.c_str(), (const char*) mc->data + 4, length) == 0); } uint32_t chunk = off / CHUNK_SIZE; @@ -1887,11 +1899,44 @@ inline bool StringStore::equals(const std::string& str, uint32_t off, uint32_t l return false; mc = (MemChunk*) mem[chunk].get(); + memcpy(&length, &mc->data[offset], 4); - if ((offset + len) > mc->currentSize) + if ((offset + length) > mc->currentSize) return false; - return (strncmp(str.c_str(), (const char*) &mc->data[offset], len) == 0); + return (strncmp(str.c_str(), (const char*) &mc->data[offset] + 4, length) == 0); +} +inline uint32_t StringStore::getStringLength(uint64_t off) +{ + uint32_t length; + MemChunk* mc; + + if (off == std::numeric_limits::max()) + return 0; + + if (off & 0x8000000000000000) + { + off = off - 0x8000000000000000; + + if (longStrings.size() <= off) + return 0; + + mc = (MemChunk*) longStrings[off].get(); + memcpy(&length, mc->data, 4); + } + else + { + uint64_t chunk = off / CHUNK_SIZE; + uint64_t offset = off % CHUNK_SIZE; + + if (mem.size() <= chunk) + return 0; + + mc = (MemChunk*) mem[chunk].get(); + memcpy(&length, &mc->data[offset], 4); + } + + return length; } inline bool StringStore::isEmpty() const diff --git a/utils/threadpool/threadpool.cpp b/utils/threadpool/threadpool.cpp index 8b3b40ecd..f1aa4ec19 100644 --- a/utils/threadpool/threadpool.cpp +++ b/utils/threadpool/threadpool.cpp @@ -21,6 +21,7 @@ * ***********************************************************************/ #include +#include using namespace std; #include "messageobj.h" diff --git a/utils/udfsdk/docs/source/reference/ColumnDatum.rst b/utils/udfsdk/docs/source/reference/ColumnDatum.rst index 564b24e44..dd1006363 100644 --- a/utils/udfsdk/docs/source/reference/ColumnDatum.rst +++ b/utils/udfsdk/docs/source/reference/ColumnDatum.rst @@ -74,6 +74,8 @@ The provided values are: - A floating point number. Represented as a C++ double type. * - DATETIME - A Columnstore date-time stored as an eight byte unsigned integer. + * - TIME + - A Columnstore time stored as an eight byte unsigned integer. * - VARCHAR - A mariadb variable length string. Represented a std::string * - VARBINARY diff --git a/utils/udfsdk/mcsv1_udaf.cpp b/utils/udfsdk/mcsv1_udaf.cpp index a31ae71c0..349a642ec 100644 --- a/utils/udfsdk/mcsv1_udaf.cpp +++ b/utils/udfsdk/mcsv1_udaf.cpp @@ -95,6 +95,7 @@ int32_t mcsv1Context::getColWidth() case CalpontSystemCatalog::DOUBLE: case CalpontSystemCatalog::UDOUBLE: case CalpontSystemCatalog::DATETIME: + case CalpontSystemCatalog::TIME: case CalpontSystemCatalog::STRINT: fColWidth = 8; break; diff --git a/utils/udfsdk/udfsdk.cpp b/utils/udfsdk/udfsdk.cpp index ea206a167..9a8973232 100644 --- a/utils/udfsdk/udfsdk.cpp +++ b/utils/udfsdk/udfsdk.cpp @@ -101,7 +101,9 @@ CalpontSystemCatalog::ColType MCS_add::operationType (FunctionParm& fp, else if (fp[0]->data()->resultType().colDataType == CalpontSystemCatalog::DATE || fp[1]->data()->resultType().colDataType == CalpontSystemCatalog::DATE || fp[0]->data()->resultType().colDataType == CalpontSystemCatalog::DATETIME || - fp[1]->data()->resultType().colDataType == CalpontSystemCatalog::DATETIME) + fp[1]->data()->resultType().colDataType == CalpontSystemCatalog::DATETIME || + fp[0]->data()->resultType().colDataType == CalpontSystemCatalog::TIME || + fp[1]->data()->resultType().colDataType == CalpontSystemCatalog::TIME) { rt.colDataType = CalpontSystemCatalog::BIGINT; rt.colWidth = 8; diff --git a/utils/windowfunction/idborderby.cpp b/utils/windowfunction/idborderby.cpp index 5764afba5..8a021f8d8 100644 --- a/utils/windowfunction/idborderby.cpp +++ b/utils/windowfunction/idborderby.cpp @@ -281,6 +281,7 @@ void CompareRule::compileRules(const std::vector& spec, const rowgr case CalpontSystemCatalog::DATE: case CalpontSystemCatalog::DATETIME: + case CalpontSystemCatalog::TIME: { Compare* c = new UintCompare(*i); fCompares.push_back(c); @@ -413,6 +414,7 @@ bool EqualCompData::operator()(Row::Pointer a, Row::Pointer b) case CalpontSystemCatalog::UBIGINT: case CalpontSystemCatalog::DATE: case CalpontSystemCatalog::DATETIME: + case CalpontSystemCatalog::TIME: { // equal compare. ignore sign and null eq = (fRow1.getUintField(*i) == fRow2.getUintField(*i)); diff --git a/utils/windowfunction/wf_lead_lag.cpp b/utils/windowfunction/wf_lead_lag.cpp index dddfb91c6..5160b75bf 100644 --- a/utils/windowfunction/wf_lead_lag.cpp +++ b/utils/windowfunction/wf_lead_lag.cpp @@ -77,6 +77,7 @@ boost::shared_ptr WF_lead_lag::makeFunction(int id, const case CalpontSystemCatalog::UDECIMAL: case CalpontSystemCatalog::DATE: case CalpontSystemCatalog::DATETIME: + case CalpontSystemCatalog::TIME: { func.reset(new WF_lead_lag(id, name)); break; diff --git a/utils/windowfunction/wf_min_max.cpp b/utils/windowfunction/wf_min_max.cpp index 99a8fef84..b0c1fe033 100644 --- a/utils/windowfunction/wf_min_max.cpp +++ b/utils/windowfunction/wf_min_max.cpp @@ -76,6 +76,7 @@ boost::shared_ptr WF_min_max::makeFunction(int id, const case CalpontSystemCatalog::UDECIMAL: case CalpontSystemCatalog::DATE: case CalpontSystemCatalog::DATETIME: + case CalpontSystemCatalog::TIME: { func.reset(new WF_min_max(id, name)); break; diff --git a/utils/windowfunction/wf_nth_value.cpp b/utils/windowfunction/wf_nth_value.cpp index 5362ee8ec..eacd1202a 100644 --- a/utils/windowfunction/wf_nth_value.cpp +++ b/utils/windowfunction/wf_nth_value.cpp @@ -77,6 +77,7 @@ boost::shared_ptr WF_nth_value::makeFunction(int id, cons case CalpontSystemCatalog::UDECIMAL: case CalpontSystemCatalog::DATE: case CalpontSystemCatalog::DATETIME: + case CalpontSystemCatalog::TIME: { func.reset(new WF_nth_value(id, name)); break; diff --git a/utils/windowfunction/wf_percentile.cpp b/utils/windowfunction/wf_percentile.cpp index 9ccc07b9c..83d6f57d8 100644 --- a/utils/windowfunction/wf_percentile.cpp +++ b/utils/windowfunction/wf_percentile.cpp @@ -82,6 +82,7 @@ boost::shared_ptr WF_percentile::makeFunction(int id, con case CalpontSystemCatalog::UDECIMAL: case CalpontSystemCatalog::DATE: case CalpontSystemCatalog::DATETIME: + case CalpontSystemCatalog::TIME: { func.reset(new WF_percentile(id, name)); break; diff --git a/utils/windowfunction/wf_udaf.cpp b/utils/windowfunction/wf_udaf.cpp index fe394c028..f302c49cd 100644 --- a/utils/windowfunction/wf_udaf.cpp +++ b/utils/windowfunction/wf_udaf.cpp @@ -392,6 +392,7 @@ void WF_udaf::SetUDAFValue(static_any::any& valOut, int64_t colOut, case execplan::CalpontSystemCatalog::UBIGINT: case execplan::CalpontSystemCatalog::DATE: case execplan::CalpontSystemCatalog::DATETIME: + case execplan::CalpontSystemCatalog::TIME: setValue(colDataType, b, e, c, &uintOut); break; diff --git a/utils/windowfunction/windowfunctiontype.cpp b/utils/windowfunction/windowfunctiontype.cpp index 5a101f870..950045899 100644 --- a/utils/windowfunction/windowfunctiontype.cpp +++ b/utils/windowfunction/windowfunctiontype.cpp @@ -91,6 +91,7 @@ map colType2String = assign::map_list_of (CalpontSystemCatalog::LONGDOUBLE, "INTERNAL LONG DOUBLE") (CalpontSystemCatalog::STRINT, "INTERNAL SHORT STRING") (CalpontSystemCatalog::TEXT, "TEXT") + (CalpontSystemCatalog::TIME, "TIME") ; @@ -490,6 +491,7 @@ void* WindowFunctionType::getNullValueByType(int ct, int pos) static uint64_t doubleNull = joblist::DOUBLENULL; static uint64_t dateNull = joblist::DATENULL; static uint64_t datetimeNull = joblist::DATETIMENULL; + static uint64_t timeNull = joblist::TIMENULL; static uint64_t char1Null = joblist::CHAR1NULL; static uint64_t char2Null = joblist::CHAR2NULL; static uint64_t char4Null = joblist::CHAR4NULL; @@ -525,6 +527,10 @@ void* WindowFunctionType::getNullValueByType(int ct, int pos) v = &datetimeNull; break; + case CalpontSystemCatalog::TIME: + v = &timeNull; + break; + case CalpontSystemCatalog::FLOAT: case CalpontSystemCatalog::UFLOAT: v = &floatNull; diff --git a/writeengine/bulk/we_bulkloadbuffer.cpp b/writeengine/bulk/we_bulkloadbuffer.cpp index 68bf71027..093aec751 100644 --- a/writeengine/bulk/we_bulkloadbuffer.cpp +++ b/writeengine/bulk/we_bulkloadbuffer.cpp @@ -902,7 +902,8 @@ void BulkLoadBuffer::convert(char* field, int fieldLength, { bool bSatVal = false; - if ( column.dataType != CalpontSystemCatalog::DATETIME ) + if ( column.dataType != CalpontSystemCatalog::DATETIME && + column.dataType != CalpontSystemCatalog::TIME ) { if (nullFlag) { @@ -976,6 +977,58 @@ void BulkLoadBuffer::convert(char* field, int fieldLength, pVal = &llVal; } + else if (column.dataType == CalpontSystemCatalog::TIME) + { + // time conversion + int rc = 0; + + if (nullFlag) + { + if (column.fWithDefault) + { + llDate = column.fDefaultInt; + // fall through to update saturation and min/max + } + else + { + llDate = joblist::TIMENULL; + pVal = &llDate; + break; + } + } + else + { + if (fImportDataMode != IMPORT_DATA_TEXT) + { + memcpy(&llDate, field, sizeof(llDate)); + + if (!dataconvert::DataConvert::isColumnTimeValid( + llDate)) + rc = -1; + } + else + { + llDate = dataconvert::DataConvert::convertColumnTime( + field, dataconvert::CALPONTTIME_ENUM, + rc, fieldLength ); + } + } + + if (rc == 0) + { + if (llDate < bufStats.minBufferVal) + bufStats.minBufferVal = llDate; + + if (llDate > bufStats.maxBufferVal) + bufStats.maxBufferVal = llDate; + } + else + { + bufStats.satCount++; + } + + pVal = &llDate; + } else { // datetime conversion @@ -1437,8 +1490,8 @@ int BulkLoadBuffer::parseCol(ColumnInfo& columnInfo) } // create a buffer for the size of the rows being written. - unsigned char* buf = new unsigned char[fTotalReadRowsParser * - columnInfo.column.width]; + unsigned char* buf = new unsigned char[fTotalReadRowsParser* + columnInfo.column.width]; char* field = new char[MAX_FIELD_SIZE + 1]; // Initialize min/max buffer values. We initialize to a sufficient @@ -2973,6 +3026,11 @@ bool BulkLoadBuffer::isBinaryFieldNull(void* val, if ((*(uint64_t*)val) == joblist::DATETIMENULL) isNullFlag = true; } + else if (dt == execplan::CalpontSystemCatalog::TIME) + { + if ((*(uint64_t*)val) == joblist::TIMENULL) + isNullFlag = true; + } else { if ((*(uint64_t*)val) == joblist::BIGINTNULL) diff --git a/writeengine/bulk/we_tableinfo.cpp b/writeengine/bulk/we_tableinfo.cpp index 8da9ce968..da28204d2 100644 --- a/writeengine/bulk/we_tableinfo.cpp +++ b/writeengine/bulk/we_tableinfo.cpp @@ -978,6 +978,11 @@ void TableInfo::reportTotals(double elapsedTime) ossSatCnt << "invalid date/times replaced with zero value : "; } + else if (fColumns[i].column.dataType == CalpontSystemCatalog::TIME) + { + ossSatCnt << + "invalid times replaced with zero value : "; + } else if (fColumns[i].column.dataType == CalpontSystemCatalog::CHAR) ossSatCnt << "character strings truncated: "; diff --git a/writeengine/server/we_ddlcommon.h b/writeengine/server/we_ddlcommon.h index aca25dc39..1d482b442 100644 --- a/writeengine/server/we_ddlcommon.h +++ b/writeengine/server/we_ddlcommon.h @@ -274,6 +274,13 @@ inline boost::any getNullValueForType(const execplan::CalpontSystemCatalog::ColT } break; + case execplan::CalpontSystemCatalog::TIME: + { + long long d = joblist::TIMENULL; + value = d; + } + break; + case execplan::CalpontSystemCatalog::CHAR: { std::string charnull; @@ -433,6 +440,10 @@ inline int convertDataType(int dataType) calpontDataType = CalpontSystemCatalog::DATETIME; break; + case ddlpackage::DDL_TIME: + calpontDataType = CalpontSystemCatalog::TIME; + break; + case ddlpackage::DDL_CLOB: calpontDataType = CalpontSystemCatalog::CLOB; break; diff --git a/writeengine/server/we_dmlcommandproc.cpp b/writeengine/server/we_dmlcommandproc.cpp index bf5adf906..86625d013 100644 --- a/writeengine/server/we_dmlcommandproc.cpp +++ b/writeengine/server/we_dmlcommandproc.cpp @@ -1781,6 +1781,7 @@ uint8_t WE_DMLCommandProc::processBatchInsertBinary(messageqcpp::ByteStream& bs, case execplan::CalpontSystemCatalog::BIGINT: case execplan::CalpontSystemCatalog::DATETIME: + case execplan::CalpontSystemCatalog::TIME: case execplan::CalpontSystemCatalog::UBIGINT: bs >> val64; @@ -2977,7 +2978,14 @@ uint8_t WE_DMLCommandProc::processUpdate(messageqcpp::ByteStream& bs, case CalpontSystemCatalog::DATETIME: { intColVal = row.getUintField<8>(fetchColPos); - value = DataConvert::datetimeToString(intColVal); + value = DataConvert::datetimeToString(intColVal, colType.precision); + break; + } + + case CalpontSystemCatalog::TIME: + { + intColVal = row.getIntField<8>(fetchColPos); + value = DataConvert::timeToString(intColVal, colType.precision); break; } @@ -3305,7 +3313,14 @@ uint8_t WE_DMLCommandProc::processUpdate(messageqcpp::ByteStream& bs, case CalpontSystemCatalog::DATETIME: { intColVal = row.getUintField<8>(fetchColPos); - value = DataConvert::datetimeToString(intColVal); + value = DataConvert::datetimeToString(intColVal, colType.precision); + break; + } + + case CalpontSystemCatalog::TIME: + { + intColVal = row.getIntField<8>(fetchColPos); + value = DataConvert::timeToString(intColVal, colType.precision); break; } diff --git a/writeengine/shared/we_convertor.cpp b/writeengine/shared/we_convertor.cpp index dc9754355..8050426f3 100644 --- a/writeengine/shared/we_convertor.cpp +++ b/writeengine/shared/we_convertor.cpp @@ -420,6 +420,7 @@ void Convertor::convertColType(CalpontSystemCatalog::ColDataType dataType, // Map BIGINT and DATETIME to WR_LONGLONG case CalpontSystemCatalog::BIGINT : case CalpontSystemCatalog::DATETIME : + case CalpontSystemCatalog::TIME : internalType = WriteEngine::WR_LONGLONG; break; @@ -629,6 +630,7 @@ void Convertor::convertColType(ColStruct* curStruct) // Map BIGINT and DATETIME to WR_LONGLONG case CalpontSystemCatalog::BIGINT : case CalpontSystemCatalog::DATETIME : + case CalpontSystemCatalog::TIME : *internalType = WriteEngine::WR_LONGLONG; break; @@ -792,6 +794,7 @@ int Convertor::getCorrectRowWidth(CalpontSystemCatalog::ColDataType dataType, in break; case CalpontSystemCatalog::DATETIME: + case CalpontSystemCatalog::TIME: newWidth = 8; break; diff --git a/writeengine/shared/we_type.h b/writeengine/shared/we_type.h index 21fad3e08..06a367ecb 100644 --- a/writeengine/shared/we_type.h +++ b/writeengine/shared/we_type.h @@ -161,7 +161,8 @@ const char ColDataTypeStr[execplan::CalpontSystemCatalog::NUM_OF_COL_DATA_TYPE] "unsigned-float", "unsigned-bigint", "unsigned-double", - "text" + "text", + "time" }; enum FuncType { FUNC_WRITE_ENGINE, FUNC_INDEX, FUNC_DICTIONARY }; diff --git a/writeengine/splitter/we_sdhandler.cpp b/writeengine/splitter/we_sdhandler.cpp index 68d5bf784..802fd108b 100644 --- a/writeengine/splitter/we_sdhandler.cpp +++ b/writeengine/splitter/we_sdhandler.cpp @@ -1907,6 +1907,10 @@ void WESDHandler::onCleanupResult(int PmId, messageqcpp::SBS& Sbs) ossSatCnt << "invalid date/times replaced with zero value: "; break; + case CalpontSystemCatalog::TIME: + ossSatCnt << "invalid times replaced with zero value: "; + break; + case CalpontSystemCatalog::CHAR: ossSatCnt << "character strings truncated: "; break; diff --git a/writeengine/splitter/we_splitterapp.cpp b/writeengine/splitter/we_splitterapp.cpp index 862eb03c0..0804baf1b 100644 --- a/writeengine/splitter/we_splitterapp.cpp +++ b/writeengine/splitter/we_splitterapp.cpp @@ -165,7 +165,7 @@ void WESplitterApp::setupSignalHandlers() sa.sa_handler = SIG_IGN; sigaction(SIGPIPE, &sa, 0); sa.sa_handler = WESplitterApp::onSigHup; - sigaction(SIGPIPE, &sa, 0); + sigaction(SIGHUP, &sa, 0); sa.sa_handler = WESplitterApp::onSigInterrupt; sigaction(SIGUSR1, &sa, 0); /* diff --git a/writeengine/wrapper/writeengine.cpp b/writeengine/wrapper/writeengine.cpp index cb6eb3745..5d3dfec85 100644 --- a/writeengine/wrapper/writeengine.cpp +++ b/writeengine/wrapper/writeengine.cpp @@ -443,6 +443,8 @@ void WriteEngineWrapper::convertValue(const ColType colType, void* valArray, con case WriteEngine::WR_LONGLONG: if (data.type() == typeid(long long)) ((long long*)valArray)[pos] = boost::any_cast(data); + else if (data.type() == typeid(long)) + ((long long*)valArray)[pos] = (long long)boost::any_cast(data); else ((long long*)valArray)[pos] = boost::any_cast(data); diff --git a/writeengine/xml/we_xmljob.cpp b/writeengine/xml/we_xmljob.cpp index 135754ac1..8d755b824 100644 --- a/writeengine/xml/we_xmljob.cpp +++ b/writeengine/xml/we_xmljob.cpp @@ -1116,6 +1116,22 @@ void XMLJob::fillInXMLDataNotNullDefault( break; } + case execplan::CalpontSystemCatalog::TIME: + { + int convertStatus; + int64_t dt = + dataconvert::DataConvert::convertColumnTime( + col_defaultValue.c_str(), + dataconvert::CALPONTTIME_ENUM, convertStatus, + col_defaultValue.length() ); + + if (convertStatus != 0) + bDefaultConvertError = true; + + col.fDefaultInt = dt; + break; + } + case execplan::CalpontSystemCatalog::FLOAT: case execplan::CalpontSystemCatalog::DOUBLE: case execplan::CalpontSystemCatalog::UFLOAT: