From 12acd033fba86f99dd6b135a281f49fbcf5e5746 Mon Sep 17 00:00:00 2001 From: David Hall Date: Fri, 3 Mar 2017 15:57:41 -0600 Subject: [PATCH 01/27] MCOL-597 Take Window Functions just to get a compile. No other changes were needed. --- dbcon/mysql/ha_calpont.cpp | 8 ---- dbcon/mysql/ha_calpont_execplan.cpp | 61 +++++++++++++++++------------ dbcon/mysql/ha_calpont_impl.cpp | 12 +----- dbcon/mysql/ha_calpont_impl.h | 1 - dbcon/mysql/ha_window_function.cpp | 13 +++--- 5 files changed, 46 insertions(+), 49 deletions(-) diff --git a/dbcon/mysql/ha_calpont.cpp b/dbcon/mysql/ha_calpont.cpp index 31b08451c..16e1e0def 100755 --- a/dbcon/mysql/ha_calpont.cpp +++ b/dbcon/mysql/ha_calpont.cpp @@ -121,7 +121,6 @@ static int calpont_commit(handlerton *hton, THD* thd, bool all); static int calpont_rollback(handlerton *hton, THD* thd, bool all); static int calpont_close_connection ( handlerton *hton, THD* thd ); -static void calpont_set_error( THD*, uint64_t, LEX_STRING*, uint32_t); handlerton *calpont_hton; /* Variables for example share methods */ @@ -219,7 +218,6 @@ static int columnstore_init_func(void *p) calpont_hton->commit= calpont_commit; calpont_hton->rollback= calpont_rollback; calpont_hton->close_connection = calpont_close_connection; - calpont_hton->set_error= calpont_set_error; DBUG_RETURN(0); } @@ -248,7 +246,6 @@ static int infinidb_init_func(void *p) calpont_hton->commit= calpont_commit; calpont_hton->rollback= calpont_rollback; calpont_hton->close_connection = calpont_close_connection; - calpont_hton->set_error= calpont_set_error; DBUG_RETURN(0); } @@ -372,11 +369,6 @@ static int calpont_close_connection ( handlerton *hton, THD* thd ) return rc; } -static void calpont_set_error(THD* thd, uint64_t errCode, LEX_STRING* args, uint32_t argCount) -{ - return ha_calpont_impl_set_error(thd, errCode, args, argCount); -} - ha_calpont::ha_calpont(handlerton *hton, TABLE_SHARE *table_arg) : handler(hton, table_arg), int_table_flags(HA_BINLOG_STMT_CAPABLE | HA_BINLOG_ROW_CAPABLE | diff --git a/dbcon/mysql/ha_calpont_execplan.cpp b/dbcon/mysql/ha_calpont_execplan.cpp index 4c2f7959b..56df504dd 100755 --- a/dbcon/mysql/ha_calpont_execplan.cpp +++ b/dbcon/mysql/ha_calpont_execplan.cpp @@ -699,11 +699,13 @@ void debug_walk(const Item *item, void *arg) cout << ": " << endl; break; } +#if 0 case Item::WINDOW_FUNC_ITEM: { cout << "Window Function Item" << endl; break; } +#endif default: { cout << "UNKNOWN_ITEM type " << item->type() << endl; @@ -2453,6 +2455,7 @@ ReturnedColumn* buildReturnedColumn(Item* item, gp_walk_info& gwi, bool& nonSupp rc = new ConstantColumn(valStr); break; } +#if 0 case Item::WINDOW_FUNC_ITEM: { return buildWindowFunctionColumn(item, gwi, nonSupport); @@ -2469,6 +2472,7 @@ ReturnedColumn* buildReturnedColumn(Item* item, gp_walk_info& gwi, bool& nonSupp rc->resultType(srcp->resultType()); break; } +#endif case Item::SUBSELECT_ITEM: { gwi.hasSubSelect = true; @@ -3500,23 +3504,24 @@ ReturnedColumn* buildAggregateColumn(Item* item, gp_walk_info& gwi) end=order_item + gc->order_field();order_item < end; order_item++) { - Item *ord_col= *(*order_item)->item; - if (ord_col->type() == Item::INT_ITEM) - { - Item_int* id = (Item_int*)ord_col; - if (id->val_int() > (int)selCols.size()) - { - gwi.fatalParseError = true; - return NULL; - } - rc = selCols[id->val_int()-1]->clone(); - rc->orderPos(id->val_int()-1); - } - else - { - rc = buildReturnedColumn(ord_col, gwi, gwi.fatalParseError); - } - rc->asc((*order_item)->asc); + Item *ord_col= *(*order_item)->item; + if (ord_col->type() == Item::INT_ITEM) + { + Item_int* id = (Item_int*)ord_col; + if (id->val_int() > (int)selCols.size()) + { + gwi.fatalParseError = true; + return NULL; + } + rc = selCols[id->val_int()-1]->clone(); + rc->orderPos(id->val_int()-1); + } + else + { + rc = buildReturnedColumn(ord_col, gwi, gwi.fatalParseError); + } + // 10.2 TODO: direction is now a tri-state flag + rc->asc((*order_item)->direction == ORDER::ORDER_ASC ? true : false); orderCols.push_back(SRCP(rc)); } @@ -4423,6 +4428,7 @@ void gp_walk(const Item *item, void *arg) gwip->rcWorkStack.push(buildReturnedColumn(itp, *gwip, gwip->fatalParseError)); break; } +#if 0 case Item::WINDOW_FUNC_ITEM: { gwip->hasWindowFunc = true; @@ -4432,6 +4438,7 @@ void gp_walk(const Item *item, void *arg) gwip->rcWorkStack.push(af); break; } +#endif case Item::COPY_STR_ITEM: printf("********** received COPY_STR_ITEM *********\n"); break; @@ -4615,9 +4622,11 @@ void parse_item (Item *item, vector& field_vec, bool& hasNonSupport setError(item->thd(), ER_CHECK_NOT_IMPLEMENTED, parseErrorText); break; } +#if 0 case Item::WINDOW_FUNC_ITEM: parseInfo |= AF_BIT; break; +#endif default: break; } @@ -5507,6 +5516,7 @@ int getSelectPlan(gp_walk_info& gwi, SELECT_LEX& select_lex, SCSEP& csep, bool i setError(gwi.thd, ER_CHECK_NOT_IMPLEMENTED, gwi.parseErrorText, gwi); return ER_CHECK_NOT_IMPLEMENTED; } +#if 0 case Item::WINDOW_FUNC_ITEM: { SRCP srcp(buildWindowFunctionColumn(item, gwi, gwi.fatalParseError)); @@ -5520,6 +5530,7 @@ int getSelectPlan(gp_walk_info& gwi, SELECT_LEX& select_lex, SCSEP& csep, bool i gwi.returnedCols.push_back(srcp); break; } +#endif default: { break; @@ -5672,6 +5683,7 @@ int getSelectPlan(gp_walk_info& gwi, SELECT_LEX& select_lex, SCSEP& csep, bool i // check if window functions are in order by. InfiniDB process order by list if // window functions are involved, either in order by or projection. +#if 0 bool hasWindowFunc = gwi.hasWindowFunc; gwi.hasWindowFunc = false; for (; groupcol; groupcol= groupcol->next) @@ -5687,7 +5699,7 @@ int getSelectPlan(gp_walk_info& gwi, SELECT_LEX& select_lex, SCSEP& csep, bool i return ER_CHECK_NOT_IMPLEMENTED; } gwi.hasWindowFunc = hasWindowFunc; - +#endif groupcol = reinterpret_cast(select_lex.group_list.first); for (; groupcol; groupcol= groupcol->next) @@ -5933,12 +5945,13 @@ int getSelectPlan(gp_walk_info& gwi, SELECT_LEX& select_lex, SCSEP& csep, bool i // check if window functions are in order by. InfiniDB process order by list if // window functions are involved, either in order by or projection. +#if 0 for (; ordercol; ordercol= ordercol->next) { if ((*(ordercol->item))->type() == Item::WINDOW_FUNC_ITEM) gwi.hasWindowFunc = true; } - +#endif // re-visit the first of ordercol list ordercol = reinterpret_cast(order_list.first); @@ -5989,7 +6002,7 @@ int getSelectPlan(gp_walk_info& gwi, SELECT_LEX& select_lex, SCSEP& csep, bool i return ER_CHECK_NOT_IMPLEMENTED; } } - if (ordercol->asc) + if (ordercol->direction == ORDER::ORDER_ASC) rc->asc(true); else rc->asc(false); @@ -6022,7 +6035,7 @@ int getSelectPlan(gp_walk_info& gwi, SELECT_LEX& select_lex, SCSEP& csep, bool i ostringstream oss; oss << ordercol->counter; ord_cols += oss.str(); - if (!ordercol->asc) + if (!ordercol->direction == ORDER::ORDER_ASC) ord_cols += " desc"; continue; } @@ -6147,7 +6160,7 @@ int getSelectPlan(gp_walk_info& gwi, SELECT_LEX& select_lex, SCSEP& csep, bool i gwi.returnedCols.push_back(srcp); ord_cols += " `" + escapeBackTick(str.c_ptr()) + "`"; } - if (!ordercol->asc) + if (!ordercol->direction == ORDER::ORDER_ASC) ord_cols += " desc"; continue; } @@ -6211,7 +6224,7 @@ int getSelectPlan(gp_walk_info& gwi, SELECT_LEX& select_lex, SCSEP& csep, bool i ord_item->print(&str, QT_INFINIDB); ord_cols += str.c_ptr(); } - if (!ordercol->asc) + if (!ordercol->direction == ORDER::ORDER_ASC) ord_cols += " desc"; } } @@ -6551,7 +6564,7 @@ int getSelectPlan(gp_walk_info& gwi, SELECT_LEX& select_lex, SCSEP& csep, bool i ord_item->print(&str, QT_INFINIDB_NO_QUOTE); ord_cols += string(str.c_ptr()); } - if (!ordercol->asc) + if (!ordercol->direction == ORDER::ORDER_ASC) ord_cols += " desc"; } } diff --git a/dbcon/mysql/ha_calpont_impl.cpp b/dbcon/mysql/ha_calpont_impl.cpp index 71300fae0..3d5f1e381 100755 --- a/dbcon/mysql/ha_calpont_impl.cpp +++ b/dbcon/mysql/ha_calpont_impl.cpp @@ -1170,7 +1170,7 @@ uint32_t doUpdateDelete(THD *thd) { multi_delete* deleteTable = (multi_delete*)((thd->lex->select_lex.join)->result); first_table= (TABLE_LIST*) deleteTable->get_tables(); - if (deleteTable->num_of_tables == 1) + if (deleteTable->get_num_of_tables() == 1) { schemaName = first_table->db; tableName = first_table->table_name; @@ -4444,15 +4444,5 @@ int ha_calpont_impl_rnd_pos(uchar *buf, uchar *pos) return ER_INTERNAL_ERROR; } -// Called from mysql parser to set IDB error for window functions -void ha_calpont_impl_set_error(THD* thd, uint64_t errCode, LEX_STRING* args, uint32_t argCount) -{ - IDEBUG( cout << "ha_calpont_impl_ser_error" << endl); - Message::Args arguments; - for (uint32_t i = 0; i < argCount; i++) - arguments.add(args[i].str); - string emsg = logging::IDBErrorInfo::instance()->errorMsg(errCode, arguments); - setError(thd, ER_INTERNAL_ERROR, emsg); -} // vim:sw=4 ts=4: diff --git a/dbcon/mysql/ha_calpont_impl.h b/dbcon/mysql/ha_calpont_impl.h index 58f680a41..79fc6933b 100644 --- a/dbcon/mysql/ha_calpont_impl.h +++ b/dbcon/mysql/ha_calpont_impl.h @@ -48,7 +48,6 @@ extern int ha_calpont_impl_external_lock(THD *thd, TABLE* table, int lock_type); extern int ha_calpont_impl_update_row(); extern int ha_calpont_impl_delete_row(); extern int ha_calpont_impl_rnd_pos(uchar *buf, uchar *pos); -extern void ha_calpont_impl_set_error(THD* thd, uint64_t errCode, LEX_STRING* args, uint32_t argCount); #endif #ifdef NEED_CALPONT_INTERFACE diff --git a/dbcon/mysql/ha_window_function.cpp b/dbcon/mysql/ha_window_function.cpp index ee91eed26..c7bed4f15 100755 --- a/dbcon/mysql/ha_window_function.cpp +++ b/dbcon/mysql/ha_window_function.cpp @@ -63,6 +63,7 @@ ReturnedColumn* nullOnError(gp_walk_info& gwi) return NULL; } +#if 0 WF_FRAME frame(BOUND& bound) { switch (bound) @@ -81,7 +82,7 @@ WF_FRAME frame(BOUND& bound) return WF_UNKNOWN; } } - +#endif ReturnedColumn* buildBoundExp(WF_Boundary& bound, SRCP& order, gp_walk_info& gwi) { if (!(gwi.thd->infinidb_vtable.cal_conn_info)) @@ -188,12 +189,13 @@ ReturnedColumn* buildWindowFunctionColumn(Item* item, gp_walk_info& gwi, bool& n cal_connection_info* ci = reinterpret_cast(gwi.thd->infinidb_vtable.cal_conn_info); gwi.hasWindowFunc = true; - Item_func_window* wf = (Item_func_window*)item; - string funcName = wf->func_name(); +// Item_func_window* wf = (Item_func_window*)item; + string funcName /*= wf->func_name()*/; WindowFunctionColumn* ac = new WindowFunctionColumn(funcName); - ac->distinct(wf->isDistinct()); - Window_context *wf_ctx = wf->window_ctx(); +// ac->distinct(wf->isDistinct()); +// Window_context *wf_ctx = wf->window_ctx(); SRCP srcp; +#if 0 // arguments vector funcParms; @@ -436,6 +438,7 @@ ReturnedColumn* buildWindowFunctionColumn(Item* item, gp_walk_info& gwi, bool& n // put ac on windowFuncList gwi.windowFuncList.push_back(ac); +#endif return ac; } From 3055964ddc2607a00d656aefff26f3392357b679 Mon Sep 17 00:00:00 2001 From: David Hall Date: Mon, 13 Mar 2017 13:20:57 -0500 Subject: [PATCH 02/27] MCOL-597 stabilization --- dbcon/mysql/CMakeLists.txt | 10 +- dbcon/mysql/ha_calpont_execplan.cpp | 19 +- dbcon/mysql/ha_window_function.cpp | 2 +- dbcon/mysql/install_calpont_mysql.sh | 2 +- dbcon/mysql/my.cnf | 1 + dbcon/mysql/mysql.vpj | 582 ++++++++++++++++++--------- utils/funcexp/func_div.cpp | 4 +- utils/loggingcpp/ErrorMessage.txt | 2 +- 8 files changed, 415 insertions(+), 207 deletions(-) diff --git a/dbcon/mysql/CMakeLists.txt b/dbcon/mysql/CMakeLists.txt index d478843b0..4c423ed16 100644 --- a/dbcon/mysql/CMakeLists.txt +++ b/dbcon/mysql/CMakeLists.txt @@ -25,7 +25,7 @@ set_source_files_properties(ha_calpont.cpp PROPERTIES COMPILE_FLAGS "-fno-rtti - add_library(calmysql SHARED ${libcalmysql_SRCS}) -target_link_libraries(calmysql ${ENGINE_LDFLAGS} ${ENGINE_WRITE_LIBS} ${NETSNMP_LIBRARIES} threadpool) +target_link_libraries(calmysql ${ENGINE_LDFLAGS} ${ENGINE_WRITE_LIBS} ${NETSNMP_LIBRARIES} mysqlservices threadpool) set_target_properties(calmysql PROPERTIES VERSION 1.0.0 SOVERSION 1) @@ -34,7 +34,7 @@ SET ( is_columnstore_tables_SRCS ) add_library(is_columnstore_tables SHARED ${is_columnstore_tables_SRCS}) -target_link_libraries(is_columnstore_tables ${ENGINE_LDFLAGS} ${ENGINE_WRITE_LIBS} threadpool) +target_link_libraries(is_columnstore_tables ${ENGINE_LDFLAGS} ${ENGINE_WRITE_LIBS} mysqlservices threadpool) # Don't prepend .so file with 'lib' set_target_properties(is_columnstore_tables PROPERTIES PREFIX "") @@ -45,7 +45,7 @@ SET ( is_columnstore_columns_SRCS ) add_library(is_columnstore_columns SHARED ${is_columnstore_columns_SRCS}) -target_link_libraries(is_columnstore_columns ${ENGINE_LDFLAGS} ${ENGINE_WRITE_LIBS} threadpool) +target_link_libraries(is_columnstore_columns ${ENGINE_LDFLAGS} ${ENGINE_WRITE_LIBS} mysqlservices threadpool) # Don't prepend .so file with 'lib' set_target_properties(is_columnstore_columns PROPERTIES PREFIX "") @@ -56,7 +56,7 @@ SET ( is_columnstore_extents_SRCS ) add_library(is_columnstore_extents SHARED ${is_columnstore_extents_SRCS}) -target_link_libraries(is_columnstore_extents ${ENGINE_LDFLAGS} ${ENGINE_WRITE_LIBS} threadpool) +target_link_libraries(is_columnstore_extents ${ENGINE_LDFLAGS} ${ENGINE_WRITE_LIBS} mysqlservices threadpool) # Don't prepend .so file with 'lib' set_target_properties(is_columnstore_extents PROPERTIES PREFIX "") @@ -67,7 +67,7 @@ SET ( is_columnstore_files_SRCS ) add_library(is_columnstore_files SHARED ${is_columnstore_files_SRCS}) -target_link_libraries(is_columnstore_files ${ENGINE_LDFLAGS} ${ENGINE_WRITE_LIBS} threadpool) +target_link_libraries(is_columnstore_files ${ENGINE_LDFLAGS} ${ENGINE_WRITE_LIBS} mysqlservices threadpool) # Don't prepend .so file with 'lib' set_target_properties(is_columnstore_files PROPERTIES PREFIX "") diff --git a/dbcon/mysql/ha_calpont_execplan.cpp b/dbcon/mysql/ha_calpont_execplan.cpp index 56df504dd..85fe0b45f 100755 --- a/dbcon/mysql/ha_calpont_execplan.cpp +++ b/dbcon/mysql/ha_calpont_execplan.cpp @@ -423,7 +423,7 @@ void debug_walk(const Item *item, void *arg) char* item_name = item->name; if (!item_name) { - item_name = ""; + item_name = (char*)""; } switch (isp->sum_func()) { @@ -1111,6 +1111,15 @@ bool buildRowColumnFilter(gp_walk_info* gwip, RowColumn* rhs, RowColumn* lhs, It bool buildPredicateItem(Item_func* ifp, gp_walk_info* gwip) { boost::shared_ptr sop(new PredicateOperator(ifp->func_name())); + if (ifp->functype() == Item_func::LIKE_FUNC) + { + // Starting with MariaDB 10.2, LIKE uses a negated flag instead of FUNC_NOT + // Further processing is done below as before for LIKE + if (((Item_func_like*)ifp)->negated) + { + sop->reverseOp(); + } + } if (!(gwip->thd->infinidb_vtable.cal_conn_info)) gwip->thd->infinidb_vtable.cal_conn_info = (void*)(new cal_connection_info()); cal_connection_info* ci = reinterpret_cast(gwip->thd->infinidb_vtable.cal_conn_info); @@ -6035,7 +6044,7 @@ int getSelectPlan(gp_walk_info& gwi, SELECT_LEX& select_lex, SCSEP& csep, bool i ostringstream oss; oss << ordercol->counter; ord_cols += oss.str(); - if (!ordercol->direction == ORDER::ORDER_ASC) + if (ordercol->direction != ORDER::ORDER_ASC) ord_cols += " desc"; continue; } @@ -6160,7 +6169,7 @@ int getSelectPlan(gp_walk_info& gwi, SELECT_LEX& select_lex, SCSEP& csep, bool i gwi.returnedCols.push_back(srcp); ord_cols += " `" + escapeBackTick(str.c_ptr()) + "`"; } - if (!ordercol->direction == ORDER::ORDER_ASC) + if (ordercol->direction != ORDER::ORDER_ASC) ord_cols += " desc"; continue; } @@ -6224,7 +6233,7 @@ int getSelectPlan(gp_walk_info& gwi, SELECT_LEX& select_lex, SCSEP& csep, bool i ord_item->print(&str, QT_INFINIDB); ord_cols += str.c_ptr(); } - if (!ordercol->direction == ORDER::ORDER_ASC) + if (ordercol->direction != ORDER::ORDER_ASC) ord_cols += " desc"; } } @@ -6564,7 +6573,7 @@ int getSelectPlan(gp_walk_info& gwi, SELECT_LEX& select_lex, SCSEP& csep, bool i ord_item->print(&str, QT_INFINIDB_NO_QUOTE); ord_cols += string(str.c_ptr()); } - if (!ordercol->direction == ORDER::ORDER_ASC) + if (ordercol->direction != ORDER::ORDER_ASC) ord_cols += " desc"; } } diff --git a/dbcon/mysql/ha_window_function.cpp b/dbcon/mysql/ha_window_function.cpp index c7bed4f15..4106329da 100755 --- a/dbcon/mysql/ha_window_function.cpp +++ b/dbcon/mysql/ha_window_function.cpp @@ -186,7 +186,7 @@ ReturnedColumn* buildWindowFunctionColumn(Item* item, gp_walk_info& gwi, bool& n //cout << str.c_ptr() << endl; if (!(gwi.thd->infinidb_vtable.cal_conn_info)) gwi.thd->infinidb_vtable.cal_conn_info = (void*)(new cal_connection_info()); - cal_connection_info* ci = reinterpret_cast(gwi.thd->infinidb_vtable.cal_conn_info); +// cal_connection_info* ci = reinterpret_cast(gwi.thd->infinidb_vtable.cal_conn_info); gwi.hasWindowFunc = true; // Item_func_window* wf = (Item_func_window*)item; diff --git a/dbcon/mysql/install_calpont_mysql.sh b/dbcon/mysql/install_calpont_mysql.sh index 9b55f9448..611b50ef3 100755 --- a/dbcon/mysql/install_calpont_mysql.sh +++ b/dbcon/mysql/install_calpont_mysql.sh @@ -99,7 +99,7 @@ CREATE TABLE IF NOT EXISTS infinidb_querystats.querystats query varchar(8000), startTime timestamp NOT NULL, endTime timestamp NOT NULL, - rows bigint, + `rows` bigint, errno int, phyIO bigint, cacheIO bigint, diff --git a/dbcon/mysql/my.cnf b/dbcon/mysql/my.cnf index 9d9907daa..d4f595062 100644 --- a/dbcon/mysql/my.cnf +++ b/dbcon/mysql/my.cnf @@ -44,6 +44,7 @@ query_cache_size = 0 thread_stack = 512K lower_case_table_names=1 group_concat_max_len=512 +sql_mode="ERROR_FOR_DIVISION_BY_ZERO,NO_AUTO_CREATE_USER,NO_ENGINE_SUBSTITUTION" # Enable compression by default on create, set to 0 to turn off infinidb_compression_type=2 diff --git a/dbcon/mysql/mysql.vpj b/dbcon/mysql/mysql.vpj index cdb8f4f12..fd92a4562 100644 --- a/dbcon/mysql/mysql.vpj +++ b/dbcon/mysql/mysql.vpj @@ -197,6 +197,198 @@ + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + @@ -218,204 +410,210 @@ - - - - - - - - - - - - - - - - - - - - - - - - - - + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - 0 ? val1 + 0.5 : val1 - 0.5); // MCOL-176 If abs(int_val2) is small enough (like -1), then, this may cause overflow. @@ -78,7 +78,7 @@ uint64_t Func_div::getUintVal(rowgroup::Row& row, if (val2 == 0) { isNull = true; - return NULL; + return 0; } return val1 / val2; } diff --git a/utils/loggingcpp/ErrorMessage.txt b/utils/loggingcpp/ErrorMessage.txt index 1266a21e4..c4d134899 100644 --- a/utils/loggingcpp/ErrorMessage.txt +++ b/utils/loggingcpp/ErrorMessage.txt @@ -27,7 +27,7 @@ 1008 ERR_CREATE_DATATYPE_NOT_SUPPORT The syntax or the data type(s) is not supported by Columnstore. Please check the Columnstore syntax guide for supported syntax or data types. 1010 ERR_AGGREGATE_TYPE_NOT_SUPPORT '%1%' for column type '%2%' isn't supported. 1011 ERR_DML_VIEW %1% on VIEW is currently not supported. -#1012 ERR_UPDATE_NOT_SUPPORT_FEATURE This version of Columstore supports update of only one table at a time. +1012 ERR_UPDATE_NOT_SUPPORT_FEATURE This version of Columnstore supports update of only one table at a time. 1014 ERR_ROLLUP_NOT_SUPPORT Rollup is currently not supported. 1015 ERR_OUTER_JOIN_SUBSELECT Subquery on OUTER JOIN ON clause is currently not supported. 1016 ERR_PARTITION_BY_RANGE The column type %1% is currently not supported in %2% function. From 324a9fd215aa4a480b33e0a3709d9f6416ccbaa6 Mon Sep 17 00:00:00 2001 From: David Hall Date: Tue, 21 Mar 2017 16:27:09 -0500 Subject: [PATCH 03/27] MCOL-597 Fix up Windows Functions --- dbcon/execplan/execplan.vpj | 622 +++++++++++++------------- dbcon/execplan/windowfunctioncolumn.h | 2 +- dbcon/mysql/ha_calpont_execplan.cpp | 21 +- dbcon/mysql/ha_calpont_impl.cpp | 2 +- dbcon/mysql/ha_window_function.cpp | 293 +++++++++--- dbcon/mysql/idb_mysql.h | 1 + utils/windowfunction/wf_lead_lag.cpp | 31 ++ 7 files changed, 576 insertions(+), 396 deletions(-) diff --git a/dbcon/execplan/execplan.vpj b/dbcon/execplan/execplan.vpj index 956fe3dac..6f964bc80 100644 --- a/dbcon/execplan/execplan.vpj +++ b/dbcon/execplan/execplan.vpj @@ -1,326 +1,302 @@ - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - + Version="10.0" + VendorName="SlickEdit" + TemplateName="GNU C/C++" + WorkingDir="."> + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + diff --git a/dbcon/execplan/windowfunctioncolumn.h b/dbcon/execplan/windowfunctioncolumn.h index d8b9fec75..477660b2f 100644 --- a/dbcon/execplan/windowfunctioncolumn.h +++ b/dbcon/execplan/windowfunctioncolumn.h @@ -41,7 +41,7 @@ class ByteStream; */ namespace execplan { -// This enum is made consistant with mysql item_func_window +// This enum is made consistant with mysql Item_window_func enum WF_FRAME { WF_PRECEDING = 0, diff --git a/dbcon/mysql/ha_calpont_execplan.cpp b/dbcon/mysql/ha_calpont_execplan.cpp index 85fe0b45f..93499aa4d 100755 --- a/dbcon/mysql/ha_calpont_execplan.cpp +++ b/dbcon/mysql/ha_calpont_execplan.cpp @@ -21,7 +21,7 @@ */ /** @file */ -//#define DEBUG_WALK_COND +#define DEBUG_WALK_COND #include #include #include @@ -38,7 +38,6 @@ #include #include #include -//#define NDEBUG #include #include #include @@ -699,13 +698,11 @@ void debug_walk(const Item *item, void *arg) cout << ": " << endl; break; } -#if 0 case Item::WINDOW_FUNC_ITEM: { cout << "Window Function Item" << endl; break; } -#endif default: { cout << "UNKNOWN_ITEM type " << item->type() << endl; @@ -2464,11 +2461,11 @@ ReturnedColumn* buildReturnedColumn(Item* item, gp_walk_info& gwi, bool& nonSupp rc = new ConstantColumn(valStr); break; } -#if 0 case Item::WINDOW_FUNC_ITEM: { return buildWindowFunctionColumn(item, gwi, nonSupport); } +#if INTERVAL_ITEM case Item::INTERVAL_ITEM: { Item_interval* interval = (Item_interval*)item; @@ -2481,7 +2478,7 @@ ReturnedColumn* buildReturnedColumn(Item* item, gp_walk_info& gwi, bool& nonSupp rc->resultType(srcp->resultType()); break; } -#endif +#endif case Item::SUBSELECT_ITEM: { gwi.hasSubSelect = true; @@ -4437,17 +4434,15 @@ void gp_walk(const Item *item, void *arg) gwip->rcWorkStack.push(buildReturnedColumn(itp, *gwip, gwip->fatalParseError)); break; } -#if 0 case Item::WINDOW_FUNC_ITEM: { gwip->hasWindowFunc = true; - Item_func_window* ifa = (Item_func_window*)item; + Item_window_func* ifa = (Item_window_func*)item; ReturnedColumn* af = buildWindowFunctionColumn(ifa, *gwip, gwip->fatalParseError); if (af) gwip->rcWorkStack.push(af); break; } -#endif case Item::COPY_STR_ITEM: printf("********** received COPY_STR_ITEM *********\n"); break; @@ -4631,11 +4626,9 @@ void parse_item (Item *item, vector& field_vec, bool& hasNonSupport setError(item->thd(), ER_CHECK_NOT_IMPLEMENTED, parseErrorText); break; } -#if 0 case Item::WINDOW_FUNC_ITEM: parseInfo |= AF_BIT; break; -#endif default: break; } @@ -5525,7 +5518,6 @@ int getSelectPlan(gp_walk_info& gwi, SELECT_LEX& select_lex, SCSEP& csep, bool i setError(gwi.thd, ER_CHECK_NOT_IMPLEMENTED, gwi.parseErrorText, gwi); return ER_CHECK_NOT_IMPLEMENTED; } -#if 0 case Item::WINDOW_FUNC_ITEM: { SRCP srcp(buildWindowFunctionColumn(item, gwi, gwi.fatalParseError)); @@ -5539,7 +5531,6 @@ int getSelectPlan(gp_walk_info& gwi, SELECT_LEX& select_lex, SCSEP& csep, bool i gwi.returnedCols.push_back(srcp); break; } -#endif default: { break; @@ -5692,7 +5683,6 @@ int getSelectPlan(gp_walk_info& gwi, SELECT_LEX& select_lex, SCSEP& csep, bool i // check if window functions are in order by. InfiniDB process order by list if // window functions are involved, either in order by or projection. -#if 0 bool hasWindowFunc = gwi.hasWindowFunc; gwi.hasWindowFunc = false; for (; groupcol; groupcol= groupcol->next) @@ -5708,7 +5698,6 @@ int getSelectPlan(gp_walk_info& gwi, SELECT_LEX& select_lex, SCSEP& csep, bool i return ER_CHECK_NOT_IMPLEMENTED; } gwi.hasWindowFunc = hasWindowFunc; -#endif groupcol = reinterpret_cast(select_lex.group_list.first); for (; groupcol; groupcol= groupcol->next) @@ -5954,13 +5943,11 @@ int getSelectPlan(gp_walk_info& gwi, SELECT_LEX& select_lex, SCSEP& csep, bool i // check if window functions are in order by. InfiniDB process order by list if // window functions are involved, either in order by or projection. -#if 0 for (; ordercol; ordercol= ordercol->next) { if ((*(ordercol->item))->type() == Item::WINDOW_FUNC_ITEM) gwi.hasWindowFunc = true; } -#endif // re-visit the first of ordercol list ordercol = reinterpret_cast(order_list.first); diff --git a/dbcon/mysql/ha_calpont_impl.cpp b/dbcon/mysql/ha_calpont_impl.cpp index 3d5f1e381..93ce00949 100755 --- a/dbcon/mysql/ha_calpont_impl.cpp +++ b/dbcon/mysql/ha_calpont_impl.cpp @@ -3017,7 +3017,7 @@ int ha_calpont_impl_rnd_next(uchar *buf, TABLE* table) rc = fetchNextRow(buf, ti, ci); } catch (std::exception& e) { - string emsg = string("Lost connection to ExeMgr while fetching: ") + e.what(); + string emsg = string("Error while fetching from ExeMgr: ") + e.what(); setError(thd, ER_INTERNAL_ERROR, emsg); CalpontSystemCatalog::removeCalpontSystemCatalog(tid2sid(thd->thread_id)); return ER_INTERNAL_ERROR; diff --git a/dbcon/mysql/ha_window_function.cpp b/dbcon/mysql/ha_window_function.cpp index 4106329da..33e58fff2 100755 --- a/dbcon/mysql/ha_window_function.cpp +++ b/dbcon/mysql/ha_window_function.cpp @@ -63,26 +63,26 @@ ReturnedColumn* nullOnError(gp_walk_info& gwi) return NULL; } -#if 0 -WF_FRAME frame(BOUND& bound) +WF_FRAME frame(Window_frame_bound::Bound_precedence_type bound, Item* offset) { switch (bound) { - case PRECEDING: - return WF_PRECEDING; - case FOLLOWING: - return WF_FOLLOWING; - case UNBOUNDED_PRECEDING: - return WF_UNBOUNDED_PRECEDING; - case UNBOUNDED_FOLLOWING: - return WF_UNBOUNDED_FOLLOWING; - case CURRENT_ROW: + case Window_frame_bound::PRECEDING: + if (offset) + return WF_PRECEDING; + else + return WF_UNBOUNDED_PRECEDING; + case Window_frame_bound::FOLLOWING: + if (offset) + return WF_FOLLOWING; + else + return WF_UNBOUNDED_FOLLOWING; + case Window_frame_bound::CURRENT: // Offset is meaningless return WF_CURRENT_ROW; default: return WF_UNKNOWN; } } -#endif ReturnedColumn* buildBoundExp(WF_Boundary& bound, SRCP& order, gp_walk_info& gwi) { if (!(gwi.thd->infinidb_vtable.cal_conn_info)) @@ -178,6 +178,96 @@ ReturnedColumn* buildBoundExp(WF_Boundary& bound, SRCP& order, gp_walk_info& gwi return rc; } +// Since columnstore implemented Windows Functions before MariaDB, we need +// map from the enum MariaDB uses to the string that columnstore uses to +// identify the function type. +string ConvertFuncName(Item_sum* item) +{ + switch (item->sum_func()) + { + case Item_sum::COUNT_FUNC: + if (!item->arguments()[0]->name) + return "COUNT(*)"; + return "COUNT"; + break; + case Item_sum::COUNT_DISTINCT_FUNC: + return "COUNT_DISTINCT"; + break; + case Item_sum::SUM_FUNC: + return "SUM"; + break; + case Item_sum::SUM_DISTINCT_FUNC: + return "SUM_DISTINCT"; + break; + case Item_sum::AVG_FUNC: + return "AVG"; + break; + case Item_sum::AVG_DISTINCT_FUNC: + return "AVG_DISTINCT"; + break; + case Item_sum::MIN_FUNC: + return "MIN"; + break; + case Item_sum::MAX_FUNC: + return "MAX"; + break; + case Item_sum::STD_FUNC: + return "STDDEV_POP"; + break; + case Item_sum::VARIANCE_FUNC: + return "VAR_POP"; + break; + case Item_sum::SUM_BIT_FUNC: + if (strcmp(item->func_name(), "bit_or(") == 0) + return "BIT_OR"; + if (strcmp(item->func_name(), "bit_and(") == 0) + return "BIT_AND"; + if (strcmp(item->func_name(), "bit_xor(") == 0) + return "BIT_XOR"; + break; + case Item_sum::UDF_SUM_FUNC: + return "UDF_SUM_FUNC"; // Not supported + break; + case Item_sum::GROUP_CONCAT_FUNC: + return "GROUP_CONCAT"; // Not supported + break; + case Item_sum::ROW_NUMBER_FUNC: + return "ROW_NUMBER"; + break; + case Item_sum::RANK_FUNC: + return "RANK"; + break; + case Item_sum::DENSE_RANK_FUNC: + return "DENSE_RANK"; + break; + case Item_sum::PERCENT_RANK_FUNC: + return "PERCENT_RANK"; + break; + case Item_sum::CUME_DIST_FUNC: + return "CUME_DIST"; + break; + case Item_sum::NTILE_FUNC: + return "NTILE"; + break; + case Item_sum::FIRST_VALUE_FUNC: + return "FIRST_VALUE"; + break; + case Item_sum::LAST_VALUE_FUNC: + return "LAST_VALUE"; + break; + case Item_sum::NTH_VALUE_FUNC: + return "NTH_VALUE"; + break; + case Item_sum::LEAD_FUNC: + return "LEAD"; + break; + case Item_sum::LAG_FUNC: + return "LAG"; + break; + }; + return ""; +} + ReturnedColumn* buildWindowFunctionColumn(Item* item, gp_walk_info& gwi, bool& nonSupport) { //@todo fix print for create view @@ -186,37 +276,82 @@ ReturnedColumn* buildWindowFunctionColumn(Item* item, gp_walk_info& gwi, bool& n //cout << str.c_ptr() << endl; if (!(gwi.thd->infinidb_vtable.cal_conn_info)) gwi.thd->infinidb_vtable.cal_conn_info = (void*)(new cal_connection_info()); -// cal_connection_info* ci = reinterpret_cast(gwi.thd->infinidb_vtable.cal_conn_info); + cal_connection_info* ci = reinterpret_cast(gwi.thd->infinidb_vtable.cal_conn_info); gwi.hasWindowFunc = true; -// Item_func_window* wf = (Item_func_window*)item; - string funcName /*= wf->func_name()*/; + Item_window_func* wf = (Item_window_func*)item; + string funcName = ConvertFuncName(wf->window_func()); WindowFunctionColumn* ac = new WindowFunctionColumn(funcName); -// ac->distinct(wf->isDistinct()); -// Window_context *wf_ctx = wf->window_ctx(); + ac->distinct(wf->window_func()->has_with_distinct()); + Window_spec *win_spec = wf->window_spec; SRCP srcp; -#if 0 + // arguments + vector funcParms; + Item_sum* item_sum = (Item_sum*)wf->arguments()[0]; + for (uint32_t i = 0; i < item_sum->argument_count(); i++) + { + srcp.reset(buildReturnedColumn((item_sum->arguments()[i]), gwi, nonSupport)); + if (!srcp) + return nullOnError(gwi); + funcParms.push_back(srcp); + if (gwi.clauseType == WHERE && !gwi.rcWorkStack.empty()) + gwi.rcWorkStack.pop(); + } + // Some functions, such as LEAD/LAG don't have all parameters implemented in the + // front end. Add dummies here to make the backend use defaults. + // Some of these will be temporary until they are implemented in the front end. + // Others need to stay because the back end expects them, but the front end + // no longer sends them. + // This case is kept in enum order in hopes the compiler can optimize + switch (wf->window_func()->sum_func()) + { + case Item_sum::COUNT_FUNC: + case Item_sum::COUNT_DISTINCT_FUNC: + break; + case Item_sum::FIRST_VALUE_FUNC: + srcp.reset(new ConstantColumn("1", (uint64_t)1, ConstantColumn::NUM)); // OFFSET (always one) + funcParms.push_back(srcp); + srcp.reset(new ConstantColumn("1", (uint64_t)1, ConstantColumn::NUM)); // FROM_FIRST + funcParms.push_back(srcp); + srcp.reset(new ConstantColumn("1", (uint64_t)1, ConstantColumn::NUM)); // IGNORE/RESPECT NULLS. 1 => RESPECT + funcParms.push_back(srcp); + break; + case Item_sum::LAST_VALUE_FUNC: + srcp.reset(new ConstantColumn("1", (uint64_t)1, ConstantColumn::NUM)); // OFFSET (always one) + funcParms.push_back(srcp); + srcp.reset(new ConstantColumn("0", (uint64_t)0, ConstantColumn::NUM)); // FROM_LAST + funcParms.push_back(srcp); + srcp.reset(new ConstantColumn("1", (uint64_t)1, ConstantColumn::NUM)); // IGNORE/RESPECT NULLS. 1 => RESPECT + funcParms.push_back(srcp); + break; + case Item_sum::NTH_VALUE_FUNC: + // When the front end supports these paramters, this needs modification + srcp.reset(new ConstantColumn("1", (uint64_t)1, ConstantColumn::NUM)); // FROM FIRST/LAST 1 => FIRST + funcParms.push_back(srcp); + srcp.reset(new ConstantColumn("1", (uint64_t)1, ConstantColumn::NUM)); // IGNORE/RESPECT NULLS. 1 => RESPECT + funcParms.push_back(srcp); + break; + case Item_sum::LEAD_FUNC: + case Item_sum::LAG_FUNC: + // When the front end supports these paramters, this needs modification + srcp.reset(new ConstantColumn("", ConstantColumn::NULLDATA)); // Default to fill in for NULL values + funcParms.push_back(srcp); + srcp.reset(new ConstantColumn("1", (uint64_t)1, ConstantColumn::NUM)); // IGNORE/RESPECT NULLS. 1 => RESPECT + funcParms.push_back(srcp); + break; + default: + break; + }; - // arguments - vector funcParms; - for (uint32_t i = 0; i < wf->argument_count(); i++) - { - srcp.reset(buildReturnedColumn(wf->arguments()[i], gwi, nonSupport)); - if (!srcp) - return nullOnError(gwi); - funcParms.push_back(srcp); - if (gwi.clauseType == WHERE && !gwi.rcWorkStack.empty()) - gwi.rcWorkStack.pop(); - } - ac->functionParms(funcParms); + ac->functionParms(funcParms); // Partition by - if (wf_ctx) + if (win_spec) { vector partitions; - for (uint32_t i = 0; i < wf_ctx->partition_count; i++) + for (ORDER *ord= win_spec->partition_list->first; ord; ord=ord->next) { - srcp.reset(buildReturnedColumn(wf_ctx->partitions[i], gwi, nonSupport)); + srcp.reset(buildReturnedColumn(*ord->item, gwi, nonSupport)); if (!srcp) return nullOnError(gwi); partitions.push_back(srcp); @@ -224,40 +359,42 @@ ReturnedColumn* buildWindowFunctionColumn(Item* item, gp_walk_info& gwi, bool& n ac->partitions(partitions); // Order by - if (wf_ctx->ordering) + if (win_spec->order_list) { WF_OrderBy orderBy; // order columns - if (wf_ctx->ordering->orders) + if (win_spec->order_list) { vector orders; - ORDER* orderCol = reinterpret_cast(wf_ctx->ordering->orders->first); + ORDER* orderCol = reinterpret_cast(win_spec->order_list->first); for (; orderCol; orderCol= orderCol->next) { Item* orderItem = *(orderCol->item); srcp.reset(buildReturnedColumn(orderItem, gwi, nonSupport)); if (!srcp) return nullOnError(gwi); - srcp->asc(orderCol->asc); - srcp->nullsFirst(orderCol->nulls); // nulls 1-nulls first 0-nulls last - orders.push_back(srcp); + srcp->asc(orderCol->direction == ORDER::ORDER_ASC ? true : false); +// srcp->nullsFirst(orderCol->nulls); // nulls 2-default, 1-nulls first, 0-nulls last + srcp->nullsFirst(1); // WINDOWS TODO: implement NULLS FIRST/LAST in 10.2 front end + orders.push_back(srcp); } orderBy.fOrders = orders; } // window frame WF_Frame frm; - if (wf_ctx->ordering->frame) + if (win_spec->window_frame) { - frm.fIsRange = wf_ctx->ordering->frame->isRange; + frm.fIsRange = win_spec->window_frame->units == Window_frame::UNITS_RANGE; // start - if (wf_ctx->ordering->frame->start) + if (win_spec->window_frame->top_bound) { - frm.fStart.fFrame = frame(wf_ctx->ordering->frame->start->bound); + frm.fStart.fFrame = frame(win_spec->window_frame->top_bound->precedence_type, + win_spec->window_frame->top_bound->offset); // offset NULL means UNBOUNDED - if (wf_ctx->ordering->frame->start->item) + if (win_spec->window_frame->top_bound->offset) { - frm.fStart.fVal.reset(buildReturnedColumn(wf_ctx->ordering->frame->start->item, gwi, nonSupport)); + frm.fStart.fVal.reset(buildReturnedColumn(win_spec->window_frame->top_bound->offset, gwi, nonSupport)); if (!frm.fStart.fVal) return nullOnError(gwi); @@ -295,12 +432,13 @@ ReturnedColumn* buildWindowFunctionColumn(Item* item, gp_walk_info& gwi, bool& n } // end - if (wf_ctx->ordering->frame->end) + if (win_spec->window_frame->bottom_bound) { - frm.fEnd.fFrame = frame(wf_ctx->ordering->frame->end->bound); - if (wf_ctx->ordering->frame->end->item) + frm.fEnd.fFrame = frame(win_spec->window_frame->bottom_bound->precedence_type, + win_spec->window_frame->bottom_bound->offset); + if (win_spec->window_frame->bottom_bound->offset) { - frm.fEnd.fVal.reset(buildReturnedColumn(wf_ctx->ordering->frame->end->item, gwi, nonSupport)); + frm.fEnd.fVal.reset(buildReturnedColumn(win_spec->window_frame->bottom_bound->offset, gwi, nonSupport)); if (!frm.fEnd.fVal) return nullOnError(gwi); @@ -409,8 +547,57 @@ ReturnedColumn* buildWindowFunctionColumn(Item* item, gp_walk_info& gwi, bool& n } else { - frm.fStart.fFrame = WF_UNBOUNDED_PRECEDING; - frm.fEnd.fFrame = WF_CURRENT_ROW; + // Certain function types have different default boundaries + // This case is kept in enum order in hopes the compiler can optimize + switch (wf->window_func()->sum_func()) + { + case Item_sum::COUNT_FUNC: + case Item_sum::COUNT_DISTINCT_FUNC: + case Item_sum::SUM_FUNC: + case Item_sum::SUM_DISTINCT_FUNC: + case Item_sum::AVG_FUNC: + case Item_sum::AVG_DISTINCT_FUNC: + case Item_sum::MIN_FUNC: + case Item_sum::MAX_FUNC: + case Item_sum::STD_FUNC: + case Item_sum::VARIANCE_FUNC: + case Item_sum::SUM_BIT_FUNC: + case Item_sum::UDF_SUM_FUNC: + case Item_sum::GROUP_CONCAT_FUNC: + frm.fStart.fFrame = WF_UNBOUNDED_PRECEDING; + frm.fEnd.fFrame = WF_CURRENT_ROW; + break; + case Item_sum::ROW_NUMBER_FUNC: + case Item_sum::RANK_FUNC: + frm.fStart.fFrame = WF_UNBOUNDED_PRECEDING; + frm.fEnd.fFrame = WF_UNBOUNDED_FOLLOWING; + break; + case Item_sum::DENSE_RANK_FUNC: + case Item_sum::PERCENT_RANK_FUNC: + case Item_sum::CUME_DIST_FUNC: + frm.fStart.fFrame = WF_UNBOUNDED_PRECEDING; + frm.fEnd.fFrame = WF_CURRENT_ROW; + break; + case Item_sum::NTILE_FUNC: + frm.fStart.fFrame = WF_UNBOUNDED_PRECEDING; + frm.fEnd.fFrame = WF_UNBOUNDED_FOLLOWING; + break; + case Item_sum::FIRST_VALUE_FUNC: + case Item_sum::LAST_VALUE_FUNC: + case Item_sum::NTH_VALUE_FUNC: + frm.fStart.fFrame = WF_UNBOUNDED_PRECEDING; + frm.fEnd.fFrame = WF_CURRENT_ROW; + break; + case Item_sum::LEAD_FUNC: + case Item_sum::LAG_FUNC: + frm.fStart.fFrame = WF_UNBOUNDED_PRECEDING; + frm.fEnd.fFrame = WF_UNBOUNDED_FOLLOWING; + break; + default: + frm.fStart.fFrame = WF_UNBOUNDED_PRECEDING; + frm.fEnd.fFrame = WF_CURRENT_ROW; + break; + }; } orderBy.fFrame = frm; @@ -426,8 +613,7 @@ ReturnedColumn* buildWindowFunctionColumn(Item* item, gp_walk_info& gwi, bool& n return NULL; } - ac->resultType(colType_MysqlToIDB(wf)); - + ac->resultType(colType_MysqlToIDB(wf->arguments()[0])); // bug5736. Make the result type double for some window functions when // infinidb_double_for_decimal_math is set. ac->adjustResultType(); @@ -438,7 +624,6 @@ ReturnedColumn* buildWindowFunctionColumn(Item* item, gp_walk_info& gwi, bool& n // put ac on windowFuncList gwi.windowFuncList.push_back(ac); -#endif return ac; } diff --git a/dbcon/mysql/idb_mysql.h b/dbcon/mysql/idb_mysql.h index f8eb5d5c0..670f0e0e0 100644 --- a/dbcon/mysql/idb_mysql.h +++ b/dbcon/mysql/idb_mysql.h @@ -65,6 +65,7 @@ template bool isnan(T); #include "sql_table.h" #include "sql_select.h" +#include "item_windowfunc.h" // Now clean up the pollution as best we can... #undef min diff --git a/utils/windowfunction/wf_lead_lag.cpp b/utils/windowfunction/wf_lead_lag.cpp index 1ab9818f3..8d3b6169b 100644 --- a/utils/windowfunction/wf_lead_lag.cpp +++ b/utils/windowfunction/wf_lead_lag.cpp @@ -121,6 +121,11 @@ void WF_lead_lag::parseParms(const std::vector& parms) { // lead | lag fLead = 1; + fRespectNulls = true; + fDefNull = false; + fDefault = (T)0; + fOffsetNull = false; + fOffset = 0; if (fFunctionId == WF__LAG) fLead = -1; @@ -148,6 +153,32 @@ void WF_lead_lag::parseParms(const std::vector& parms) idbassert(cc != NULL); bool isNull = false; // dummy, harded coded fRespectNulls = (cc->getIntVal(fRow, isNull) > 0); +#if 0 + // parms[1]: offset + for (std::vector::size_type i = 1/*ignore 0th element*/; i < parms.size(); ++i) + { + ConstantColumn* cc = dynamic_cast(parms[i].get()); + if (cc != NULL) + { + fOffset = cc->getIntVal(fRow, fOffsetNull) * fLead; // row not used, no need to setData. + continue; + } + + cc = dynamic_cast(parms[i].get()); + if (cc != NULL) + { + getConstValue(cc, fDefault, fDefNull); + continue; + } + // IGNORE/RESPECT nulls is currently broken in the front end + cc = dynamic_cast(parms[i].get()); + if (cc != NULL) + { + bool isNull = false; // dummy, harded coded + fRespectNulls = (cc->getIntVal(fRow, isNull) > 0); + } + } +#endif } From b15f79e8ce41811073f8585ac96ed377a1e40b7d Mon Sep 17 00:00:00 2001 From: David Hall Date: Tue, 28 Mar 2017 13:03:28 -0500 Subject: [PATCH 04/27] MCOL-597 Fix up Windows Functions --- dbcon/joblist/CMakeLists.txt | 2 +- dbcon/mysql/ha_calpont_execplan.cpp | 48 ++++++++++++++++++------- dbcon/mysql/ha_window_function.cpp | 27 ++++++++------ oam/install_scripts/post-mysqld-install | 10 +++--- utils/querystats/CMakeLists.txt | 2 -- versioning/BRM/CMakeLists.txt | 8 ++--- 6 files changed, 61 insertions(+), 36 deletions(-) diff --git a/dbcon/joblist/CMakeLists.txt b/dbcon/joblist/CMakeLists.txt index 7fac3a859..0a7880413 100644 --- a/dbcon/joblist/CMakeLists.txt +++ b/dbcon/joblist/CMakeLists.txt @@ -61,7 +61,7 @@ set(joblist_LIB_SRCS add_library(joblist SHARED ${joblist_LIB_SRCS}) -target_link_libraries(joblist ${NETSNMP_LIBRARIES} -L${SERVER_BUILD_INCLUDE_DIR}/../libmysql/ libmysqlclient_r.so) +target_link_libraries(joblist -L${SERVER_BUILD_INCLUDE_DIR}/../libmariadb/libmariadb/ libmariadb.so) set_target_properties(joblist PROPERTIES VERSION 1.0.0 SOVERSION 1) diff --git a/dbcon/mysql/ha_calpont_execplan.cpp b/dbcon/mysql/ha_calpont_execplan.cpp index 93499aa4d..5180e03ae 100755 --- a/dbcon/mysql/ha_calpont_execplan.cpp +++ b/dbcon/mysql/ha_calpont_execplan.cpp @@ -496,6 +496,11 @@ void debug_walk(const Item *item, void *arg) '.' << ifp->field_name << endl; break; } + else if (field->type() == Item::FUNC_ITEM) + { + Item_func* ifp = (Item_func*)field; + cout << "CACHED REF FUNC_ITEM " << ifp->func_name() << endl; + } else if (field->type() == Item::REF_ITEM) { Item_ref* ifr = (Item_ref*)field; @@ -563,7 +568,20 @@ void debug_walk(const Item *item, void *arg) ifp->field_name << endl; break; } - cout << "UNKNOWN REF ITEM type " << ref->real_item()->type() << endl; + else if (ref->real_item()->type() == Item::FUNC_ITEM) + { + Item_func* ifp = (Item_func*)ref->real_item(); + cout << "REF FUNC_ITEM " << ifp->func_name() << endl; + } + else if (ref->real_item()->type() == Item::WINDOW_FUNC_ITEM) + { + Item_window_func* ifp = (Item_window_func*)ref->real_item(); + cout << "REF WINDOW_FUNC_ITEM " << ifp->window_func()->func_name() << endl; + } + else + { + cout << "UNKNOWN REF ITEM type " << ref->real_item()->type() << endl; + } break; } case Item::ROW_ITEM: @@ -700,7 +718,8 @@ void debug_walk(const Item *item, void *arg) } case Item::WINDOW_FUNC_ITEM: { - cout << "Window Function Item" << endl; + Item_window_func* ifp = (Item_window_func*)item; + cout << "Window Function Item " << ifp->window_func()->func_name() << endl; break; } default: @@ -2407,18 +2426,19 @@ ReturnedColumn* buildReturnedColumn(Item* item, gp_walk_info& gwi, bool& nonSupp case Item::REF_ITEM: { Item_ref* ref = (Item_ref*)item; - if ((*(ref->ref))->type() == Item::SUM_FUNC_ITEM) - { + switch ((*(ref->ref))->type()) + { + case Item::SUM_FUNC_ITEM: return buildAggregateColumn(*(ref->ref), gwi); - } - else if ((*(ref->ref))->type() == Item::FIELD_ITEM) + case Item::FIELD_ITEM: return buildReturnedColumn(*(ref->ref), gwi, nonSupport); - else if ((*(ref->ref))->type() == Item::REF_ITEM) + case Item::REF_ITEM: return buildReturnedColumn(*(((Item_ref*)(*(ref->ref)))->ref), gwi, nonSupport); - else if ((*(ref->ref))->type() == Item::FUNC_ITEM) + case Item::FUNC_ITEM: return buildFunctionColumn((Item_func*)(*(ref->ref)), gwi, nonSupport); - else - { + case Item::WINDOW_FUNC_ITEM: + return buildWindowFunctionColumn(*(ref->ref), gwi, nonSupport); + default: gwi.fatalParseError = true; gwi.parseErrorText = "Unknown REF item"; break; @@ -4390,8 +4410,6 @@ void gp_walk(const Item *item, void *arg) gwip->subQuery = orig; gwip->lastSub = existsSub; } - -#if MYSQL_VERSION_ID >= 50172 else if (sub->substype() == Item_subselect::IN_SUBS) { if (!((Item_in_subselect*)sub)->getOptimizer() && gwip->thd->derived_tables_processing) @@ -4403,7 +4421,6 @@ void gp_walk(const Item *item, void *arg) break; } } -#endif // store a dummy subselect object. the transform is handled in item_func. SubSelect *subselect = new SubSelect(); gwip->rcWorkStack.push(subselect); @@ -4595,6 +4612,11 @@ void parse_item (Item *item, vector& field_vec, bool& hasNonSupport item = (*(ref->ref)); continue; } + else if ((*(ref->ref))->type() == Item::WINDOW_FUNC_ITEM) + { + parseInfo |= AF_BIT; + break; + } else { cout << "UNKNOWN REF Item" << endl; diff --git a/dbcon/mysql/ha_window_function.cpp b/dbcon/mysql/ha_window_function.cpp index 33e58fff2..7361dfd51 100755 --- a/dbcon/mysql/ha_window_function.cpp +++ b/dbcon/mysql/ha_window_function.cpp @@ -212,10 +212,16 @@ string ConvertFuncName(Item_sum* item) return "MAX"; break; case Item_sum::STD_FUNC: - return "STDDEV_POP"; + if (((Item_sum_variance*)item)->sample) + return "STDDEV_SAMP"; + else + return "STDDEV_POP"; break; - case Item_sum::VARIANCE_FUNC: - return "VAR_POP"; + case Item_sum::VARIANCE_FUNC: + if (((Item_sum_variance*)item)->sample) + return "VAR_SAMP"; + else + return "VAR_POP"; break; case Item_sum::SUM_BIT_FUNC: if (strcmp(item->func_name(), "bit_or(") == 0) @@ -375,7 +381,7 @@ ReturnedColumn* buildWindowFunctionColumn(Item* item, gp_walk_info& gwi, bool& n return nullOnError(gwi); srcp->asc(orderCol->direction == ORDER::ORDER_ASC ? true : false); // srcp->nullsFirst(orderCol->nulls); // nulls 2-default, 1-nulls first, 0-nulls last - srcp->nullsFirst(1); // WINDOWS TODO: implement NULLS FIRST/LAST in 10.2 front end + srcp->nullsFirst(orderCol->direction == ORDER::ORDER_ASC ? 1 : 0); // WINDOWS TODO: implement NULLS FIRST/LAST in 10.2 front end orders.push_back(srcp); } orderBy.fOrders = orders; @@ -557,8 +563,15 @@ ReturnedColumn* buildWindowFunctionColumn(Item* item, gp_walk_info& gwi, bool& n case Item_sum::SUM_DISTINCT_FUNC: case Item_sum::AVG_FUNC: case Item_sum::AVG_DISTINCT_FUNC: + frm.fStart.fFrame = WF_UNBOUNDED_PRECEDING; + frm.fEnd.fFrame = WF_CURRENT_ROW; + break; case Item_sum::MIN_FUNC: case Item_sum::MAX_FUNC: + frm.fStart.fFrame = WF_UNBOUNDED_PRECEDING; +// frm.fEnd.fFrame = WF_UNBOUNDED_FOLLOWING; + frm.fEnd.fFrame = WF_CURRENT_ROW; + break; case Item_sum::STD_FUNC: case Item_sum::VARIANCE_FUNC: case Item_sum::SUM_BIT_FUNC: @@ -569,15 +582,9 @@ ReturnedColumn* buildWindowFunctionColumn(Item* item, gp_walk_info& gwi, bool& n break; case Item_sum::ROW_NUMBER_FUNC: case Item_sum::RANK_FUNC: - frm.fStart.fFrame = WF_UNBOUNDED_PRECEDING; - frm.fEnd.fFrame = WF_UNBOUNDED_FOLLOWING; - break; case Item_sum::DENSE_RANK_FUNC: case Item_sum::PERCENT_RANK_FUNC: case Item_sum::CUME_DIST_FUNC: - frm.fStart.fFrame = WF_UNBOUNDED_PRECEDING; - frm.fEnd.fFrame = WF_CURRENT_ROW; - break; case Item_sum::NTILE_FUNC: frm.fStart.fFrame = WF_UNBOUNDED_PRECEDING; frm.fEnd.fFrame = WF_UNBOUNDED_FOLLOWING; diff --git a/oam/install_scripts/post-mysqld-install b/oam/install_scripts/post-mysqld-install index 546945b22..7fa634c54 100755 --- a/oam/install_scripts/post-mysqld-install +++ b/oam/install_scripts/post-mysqld-install @@ -28,11 +28,11 @@ for arg in "$@"; do fi done -cd $installdir/mysql/lib -for file in libmysqlclient libmysqlclient_r; do - ln -sf $file.so.18.0.0 $file.so - ln -sf $file.so.18.0.0 $file.so.18 -done +#cd $installdir/mysql/lib +#for file in libmysqlclient libmysqlclient_r; do +# ln -sf $file.so.18.0.0 $file.so +# ln -sf $file.so.18.0.0 $file.so.18 +#done cd $installdir/mysql/lib/plugin for file in ha_archive ha_blackhole ha_federated ha_innodb; do ln -sf $file.so $file.so.0 diff --git a/utils/querystats/CMakeLists.txt b/utils/querystats/CMakeLists.txt index 935898818..940749a6b 100644 --- a/utils/querystats/CMakeLists.txt +++ b/utils/querystats/CMakeLists.txt @@ -8,8 +8,6 @@ set(querystats_LIB_SRCS querystats.cpp) add_library(querystats SHARED ${querystats_LIB_SRCS}) -target_link_libraries(querystats -L${SERVER_BUILD_INCLUDE_DIR}/../libmysql/ libmysqlclient_r.so) - set_target_properties(querystats PROPERTIES VERSION 1.0.0 SOVERSION 1) install(TARGETS querystats DESTINATION ${ENGINE_LIBDIR} COMPONENT libs) diff --git a/versioning/BRM/CMakeLists.txt b/versioning/BRM/CMakeLists.txt index 2ce60254d..de392d6b1 100644 --- a/versioning/BRM/CMakeLists.txt +++ b/versioning/BRM/CMakeLists.txt @@ -32,8 +32,6 @@ set(brm_LIB_SRCS add_library(brm SHARED ${brm_LIB_SRCS}) -target_link_libraries(brm ${NETSNMP_LIBRARIES}) - set_target_properties(brm PROPERTIES VERSION 1.0.0 SOVERSION 1) install(TARGETS brm DESTINATION ${ENGINE_LIBDIR} COMPONENT libs) @@ -45,7 +43,7 @@ set(controllernode_SRCS masternode.cpp masterdbrmnode.cpp) add_executable(controllernode ${controllernode_SRCS}) -target_link_libraries(controllernode ${ENGINE_LDFLAGS} ${ENGINE_OAM_LIBS} ${ENGINE_EXEC_LIBS} ${NETSNMP_LIBRARIES}) +target_link_libraries(controllernode ${ENGINE_LDFLAGS} ${ENGINE_OAM_LIBS} ${ENGINE_EXEC_LIBS} ) install(TARGETS controllernode DESTINATION ${ENGINE_BINDIR} COMPONENT platform) @@ -56,7 +54,7 @@ set(workernode_SRCS slavenode.cpp) add_executable(workernode ${workernode_SRCS}) -target_link_libraries(workernode ${ENGINE_LDFLAGS} ${ENGINE_OAM_LIBS} ${ENGINE_EXEC_LIBS} ${NETSNMP_LIBRARIES}) +target_link_libraries(workernode ${ENGINE_LDFLAGS} ${ENGINE_OAM_LIBS} ${ENGINE_EXEC_LIBS}) install(TARGETS workernode DESTINATION ${ENGINE_BINDIR} COMPONENT platform) @@ -67,7 +65,7 @@ set(dbrmctl_SRCS dbrmctl.cpp) add_executable(dbrmctl ${dbrmctl_SRCS}) -target_link_libraries(dbrmctl ${ENGINE_LDFLAGS} ${ENGINE_OAM_LIBS} ${ENGINE_EXEC_LIBS} ${NETSNMP_LIBRARIES}) +target_link_libraries(dbrmctl ${ENGINE_LDFLAGS} ${ENGINE_OAM_LIBS} ${ENGINE_EXEC_LIBS}) install(TARGETS dbrmctl DESTINATION ${ENGINE_BINDIR} COMPONENT platform) From d233b2c7a8d4b8e8e703792257182597ef8dbc09 Mon Sep 17 00:00:00 2001 From: david hill Date: Tue, 23 Aug 2016 15:59:13 -0500 Subject: [PATCH 05/27] change readme --- README.md | 16 +++++++++------- 1 file changed, 9 insertions(+), 7 deletions(-) diff --git a/README.md b/README.md index 577ac33c1..5df193e89 100644 --- a/README.md +++ b/README.md @@ -1,16 +1,18 @@ -#MariaDB ColumnStore Storage/Execution engine 1.1.0 -MariaDB ColumnStore 1.1.0 is the development version of MariaDB ColumnStore. -It is built by porting InfiniDB 4.6.7 on MariaDB 10.1.21 and adding entirely +#MariaDB ColumnStore Storage/Execution engine 1.0.1 +MariaDB ColumnStore 1.0.2 is the development version of MariaDB ColumnStore. +It is built by porting InfiniDB 4.6.2 on MariaDB 10.1.14 and adding entirely new features not found anywhere else. -#MariaDB ColumnStore 1.1.0 is an development release. +#MariaDB ColumnStore 1.0.2 is an Alpha release. +This is the first MariaDB ColumnStore release, not all features planned for the MariaDB ColumnStore 1.0 +series are included in this release. -- Do not use development releases on production systems. +Additional features will be pushed in future releases. +A few things to notice: +- Do not use alpha releases on production systems. #Building This repository is not meant to be built independently outside of the server. This repository is integrated into http://mariadb-corporation/mariadb-columnstore-server (ie, the *server*) as a git submodule. As such, you can find complete build instructions on *the server* page. https://github.com/mariadb-corporation/mariadb-columnstore-server -#Issue tracking -Issue tracking of MariaDB ColumnStore happens in JIRA, https://jira.mariadb.org/browse/MCOL From b3a344d6e535670b2fcfe666c7c9161ab8eec403 Mon Sep 17 00:00:00 2001 From: David Hall Date: Fri, 20 Jan 2017 10:10:24 -0600 Subject: [PATCH 06/27] MCOL-455: fix crash caused by rewind of unopened redistribute plan file during display when no partitions have been planned to be moved. --- writeengine/redistribute/we_redistributecontrolthread.cpp | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/writeengine/redistribute/we_redistributecontrolthread.cpp b/writeengine/redistribute/we_redistributecontrolthread.cpp index 652b48565..c43f83c3b 100644 --- a/writeengine/redistribute/we_redistributecontrolthread.cpp +++ b/writeengine/redistribute/we_redistributecontrolthread.cpp @@ -501,7 +501,7 @@ void RedistributeControlThread::displayPlan() if (!fControl->fPlanFilePtr) { ostringstream oss; - oss << "No data is scheduled to be moved" << endl; + oss << "No data is schefuled to be moved" << endl; fControl->logMessage(oss.str()); return; } From 6b30c20b2fd730f739e52870761699ccf87c557b Mon Sep 17 00:00:00 2001 From: david hill Date: Fri, 20 Jan 2017 23:15:00 -0600 Subject: [PATCH 07/27] MCOL-527 - add code for non-root --- procmon/processmonitor.cpp | 86 +++++--------------------------------- 1 file changed, 10 insertions(+), 76 deletions(-) diff --git a/procmon/processmonitor.cpp b/procmon/processmonitor.cpp index af7f4dd2d..b45e5be84 100644 --- a/procmon/processmonitor.cpp +++ b/procmon/processmonitor.cpp @@ -4835,6 +4835,11 @@ int ProcessMonitor::changeMyCnf(std::string type) } } + pos = buf.find("slave-skip-errors=all",0); + if ( pos != string::npos ) { + buf = "# slave-skip-errors=all"; + } + //output to temp file lines.push_back(buf); } @@ -5266,8 +5271,6 @@ int ProcessMonitor::runMasterDist(std::string& password, std::string& slaveModul string cmd = startup::StartUp::installDir() + "/bin/rsync.sh " + ipAddr + " " + password + " " + startup::StartUp::installDir() + " 1 > /tmp/master-dist_" + slaveModule + ".log"; system(cmd.c_str()); - - log.writeLog(__LINE__, "cmd = " + cmd, LOG_TYPE_DEBUG); string logFile = "/tmp/master-dist_" + slaveModule + ".log"; if (!oam.checkLogStatus(logFile, "FAILED")) @@ -5500,47 +5503,15 @@ bool ProcessMonitor::amazonVolumeCheck(int dbrootID) if ( status == "attached" ) { string cmd; if ( rootUser) - cmd = "mount " + deviceName + " " + startup::StartUp::installDir() + "/mysql/db -t ext2 -o noatime,nodiratime,noauto > /tmp/um_mount.log"; + cmd = "mount " + deviceName + " " + startup::StartUp::installDir() + "/mysql/db -t ext2 -o defaults > /tmp/um_mount.log"; else - cmd = "sudo mount " + deviceName + " " + startup::StartUp::installDir() + "/mysql/db -t ext2 -o noatime,nodiratime,noauto,user > /tmp/um_mount.log"; + cmd = "sudo mount " + deviceName + " " + startup::StartUp::installDir() + "/mysql/db -t ext2 -o defaults > /tmp/um_mount.log"; system(cmd.c_str()); log.writeLog(__LINE__, "mount cmd: " + cmd, LOG_TYPE_DEBUG); - if ( rootUser) - cmd = "chown -R mysql:mysql " + startup::StartUp::installDir() + "/mysql/db"; - else - cmd = "sudo chown -R " + USER + ":" + USER + " " + startup::StartUp::installDir() + "/mysql/db"; - + cmd = "chown mysql:mysql -R " + startup::StartUp::installDir() + "/mysql/db"; system(cmd.c_str()); - log.writeLog(__LINE__, "chown cmd: " + cmd, LOG_TYPE_DEBUG); - - //check for setup files in mysq/db, if not, create them for a new install - string file = startup::StartUp::installDir() + "/mysql/db/mysql"; - ifstream new_file (file.c_str()); - if (!new_file) { - string cmd; - cmd = startup::StartUp::installDir() + "/bin/post-mysqld-install --installdir=" + startup::StartUp::installDir() + " > /tmp/post-mysqld-install.log 2>&1"; - log.writeLog(__LINE__, "cmd: " + cmd, LOG_TYPE_DEBUG); - - int rtnCode = system(cmd.c_str()); - if (WEXITSTATUS(rtnCode) != 0) { - log.writeLog(__LINE__, "amazonVolumeCheck function failed, post-mysqld-install error" , LOG_TYPE_ERROR); - return false; - } - else - log.writeLog(__LINE__, "amazonVolumeCheck function, post-mysqld-install passed" , LOG_TYPE_DEBUG); - - cmd = startup::StartUp::installDir() + "/bin/post-mysql-install --installdir=" + startup::StartUp::installDir() + " > /tmp/post-mysql-install.log";; - log.writeLog(__LINE__, "cmd: " + cmd, LOG_TYPE_DEBUG); - rtnCode = system(cmd.c_str()); - if (WEXITSTATUS(rtnCode) != 0) { - log.writeLog(__LINE__, "amazonVolumeCheck function failed, post-mysql-install error" , LOG_TYPE_ERROR); - return false; - } - else - log.writeLog(__LINE__, "amazonVolumeCheck function, post-mysql-install passed" , LOG_TYPE_DEBUG); - } log.writeLog(__LINE__, "amazonVolumeCheck function successfully completed, volume attached: " + volumeName, LOG_TYPE_DEBUG); return true; @@ -5565,49 +5536,12 @@ bool ProcessMonitor::amazonVolumeCheck(int dbrootID) {} if (oam.attachEC2Volume(volumeName, deviceName, instanceName)) { - string cmd; - if ( rootUser) - cmd = "mount " + deviceName + " " + startup::StartUp::installDir() + "/mysql/db -t ext2 -o noatime,nodiratime,noauto > /tmp/um_mount.log"; - else - cmd = "sudo mount " + deviceName + " " + startup::StartUp::installDir() + "/mysql/db -t ext2 -o noatime,nodiratime,noauto,user > /tmp/um_mount.log"; - + string cmd = "mount " + deviceName + " " + startup::StartUp::installDir() + "/mysql/db -t ext2 -o defaults > /dev/null"; system(cmd.c_str()); log.writeLog(__LINE__, "mount cmd: " + cmd, LOG_TYPE_DEBUG); - if ( rootUser) - cmd = "chown -R mysql:mysql " + startup::StartUp::installDir() + "/mysql/db"; - else - cmd = "sudo chown -R " + USER + ":" + USER + " " + startup::StartUp::installDir() + "/mysql/db"; - + cmd = "chown mysql:mysql -R " + startup::StartUp::installDir() + "/mysql/db"; system(cmd.c_str()); - log.writeLog(__LINE__, "chown cmd: " + cmd, LOG_TYPE_DEBUG); - - //check for setup files in mysq/db, if not, create them for a new install - string file = startup::StartUp::installDir() + "/mysql/db/mysql"; - ifstream new_file (file.c_str()); - if (!new_file) { - string cmd; - cmd = startup::StartUp::installDir() + "/bin/post-mysqld-install --installdir=" + startup::StartUp::installDir() + " > /tmp/post-mysqld-install.log 2>&1"; - log.writeLog(__LINE__, "cmd: " + cmd, LOG_TYPE_DEBUG); - int rtnCode = system(cmd.c_str()); - if (WEXITSTATUS(rtnCode) != 0) { - log.writeLog(__LINE__, "amazonVolumeCheck function failed, post-mysqld-install error" , LOG_TYPE_ERROR); - return false; - } - else - log.writeLog(__LINE__, "amazonVolumeCheck function, post-mysqld-install passed" , LOG_TYPE_DEBUG); - - cmd = startup::StartUp::installDir() + "/bin/post-mysql-install --installdir=" + startup::StartUp::installDir() + " > /tmp/post-mysql-install.log";; - log.writeLog(__LINE__, "cmd: " + cmd, LOG_TYPE_DEBUG); - - rtnCode = system(cmd.c_str()); - if (WEXITSTATUS(rtnCode) != 0) { - log.writeLog(__LINE__, "amazonVolumeCheck function failed, post-mysql-install error" , LOG_TYPE_ERROR); - return false; - } - else - log.writeLog(__LINE__, "amazonVolumeCheck function, post-mysql-install passed" , LOG_TYPE_DEBUG); - } return true; } From b9d06e2bf23f02266b7859601414ec0e2ae5f1f0 Mon Sep 17 00:00:00 2001 From: David Hall Date: Fri, 21 Apr 2017 09:37:07 -0500 Subject: [PATCH 08/27] MCOL-597 some clean up and make Release builds work --- dbcon/mysql/CMakeLists.txt | 10 +++++----- dbcon/mysql/ha_calpont_execplan.cpp | 2 +- dbcon/mysql/ha_calpont_impl.cpp | 10 ++++++---- dbcon/mysql/idb_mysql.h | 1 + utils/windowfunction/wf_lead_lag.cpp | 26 -------------------------- 5 files changed, 13 insertions(+), 36 deletions(-) diff --git a/dbcon/mysql/CMakeLists.txt b/dbcon/mysql/CMakeLists.txt index 4c423ed16..015ad58ea 100644 --- a/dbcon/mysql/CMakeLists.txt +++ b/dbcon/mysql/CMakeLists.txt @@ -25,7 +25,7 @@ set_source_files_properties(ha_calpont.cpp PROPERTIES COMPILE_FLAGS "-fno-rtti - add_library(calmysql SHARED ${libcalmysql_SRCS}) -target_link_libraries(calmysql ${ENGINE_LDFLAGS} ${ENGINE_WRITE_LIBS} ${NETSNMP_LIBRARIES} mysqlservices threadpool) +target_link_libraries(calmysql ${ENGINE_LDFLAGS} ${ENGINE_WRITE_LIBS} ${NETSNMP_LIBRARIES} ${SERVER_BUILD_INCLUDE_DIR}/../libservices/libmysqlservices.a threadpool) set_target_properties(calmysql PROPERTIES VERSION 1.0.0 SOVERSION 1) @@ -34,7 +34,7 @@ SET ( is_columnstore_tables_SRCS ) add_library(is_columnstore_tables SHARED ${is_columnstore_tables_SRCS}) -target_link_libraries(is_columnstore_tables ${ENGINE_LDFLAGS} ${ENGINE_WRITE_LIBS} mysqlservices threadpool) +target_link_libraries(is_columnstore_tables ${ENGINE_LDFLAGS} ${ENGINE_WRITE_LIBS} ${SERVER_BUILD_INCLUDE_DIR}/../libservices/libmysqlservices.a threadpool) # Don't prepend .so file with 'lib' set_target_properties(is_columnstore_tables PROPERTIES PREFIX "") @@ -45,7 +45,7 @@ SET ( is_columnstore_columns_SRCS ) add_library(is_columnstore_columns SHARED ${is_columnstore_columns_SRCS}) -target_link_libraries(is_columnstore_columns ${ENGINE_LDFLAGS} ${ENGINE_WRITE_LIBS} mysqlservices threadpool) +target_link_libraries(is_columnstore_columns ${ENGINE_LDFLAGS} ${ENGINE_WRITE_LIBS} ${SERVER_BUILD_INCLUDE_DIR}/../libservices/libmysqlservices.a threadpool) # Don't prepend .so file with 'lib' set_target_properties(is_columnstore_columns PROPERTIES PREFIX "") @@ -56,7 +56,7 @@ SET ( is_columnstore_extents_SRCS ) add_library(is_columnstore_extents SHARED ${is_columnstore_extents_SRCS}) -target_link_libraries(is_columnstore_extents ${ENGINE_LDFLAGS} ${ENGINE_WRITE_LIBS} mysqlservices threadpool) +target_link_libraries(is_columnstore_extents ${ENGINE_LDFLAGS} ${ENGINE_WRITE_LIBS} ${SERVER_BUILD_INCLUDE_DIR}/../libservices/libmysqlservices.a threadpool) # Don't prepend .so file with 'lib' set_target_properties(is_columnstore_extents PROPERTIES PREFIX "") @@ -67,7 +67,7 @@ SET ( is_columnstore_files_SRCS ) add_library(is_columnstore_files SHARED ${is_columnstore_files_SRCS}) -target_link_libraries(is_columnstore_files ${ENGINE_LDFLAGS} ${ENGINE_WRITE_LIBS} mysqlservices threadpool) +target_link_libraries(is_columnstore_files ${ENGINE_LDFLAGS} ${ENGINE_WRITE_LIBS} ${SERVER_BUILD_INCLUDE_DIR}/../libservices/libmysqlservices.a threadpool) # Don't prepend .so file with 'lib' set_target_properties(is_columnstore_files PROPERTIES PREFIX "") diff --git a/dbcon/mysql/ha_calpont_execplan.cpp b/dbcon/mysql/ha_calpont_execplan.cpp index 5180e03ae..d3c0c4e01 100755 --- a/dbcon/mysql/ha_calpont_execplan.cpp +++ b/dbcon/mysql/ha_calpont_execplan.cpp @@ -21,7 +21,7 @@ */ /** @file */ -#define DEBUG_WALK_COND +//#define DEBUG_WALK_COND #include #include #include diff --git a/dbcon/mysql/ha_calpont_impl.cpp b/dbcon/mysql/ha_calpont_impl.cpp index 93ce00949..c0dab9b14 100755 --- a/dbcon/mysql/ha_calpont_impl.cpp +++ b/dbcon/mysql/ha_calpont_impl.cpp @@ -4390,10 +4390,11 @@ int ha_calpont_impl_external_lock(THD *thd, TABLE* table, int lock_type) // table mode if (thd->infinidb_vtable.vtable_state == THD::INFINIDB_DISABLE_VTABLE) { - if (ci->traceFlags & 1) - push_warning(thd, Sql_condition::WARN_LEVEL_NOTE, 9999, mapiter->second.conn_hndl->queryStats.c_str()); if (mapiter->second.conn_hndl) { + if (ci->traceFlags & 1) + push_warning(thd, Sql_condition::WARN_LEVEL_NOTE, 9999, mapiter->second.conn_hndl->queryStats.c_str()); + ci->queryStats = mapiter->second.conn_hndl->queryStats; ci->extendedStats = mapiter->second.conn_hndl->extendedStats; ci->miniStats = mapiter->second.conn_hndl->miniStats; @@ -4417,11 +4418,12 @@ int ha_calpont_impl_external_lock(THD *thd, TABLE* table, int lock_type) { if (thd->infinidb_vtable.vtable_state == THD::INFINIDB_SELECT_VTABLE) { + if (!ci->cal_conn_hndl) + return 0; + if (ci->traceFlags & 1) push_warning(thd, Sql_condition::WARN_LEVEL_NOTE, 9999, ci->cal_conn_hndl->queryStats.c_str()); - if (!ci->cal_conn_hndl) - return 0; ci->queryStats = ci->cal_conn_hndl->queryStats; ci->extendedStats = ci->cal_conn_hndl->extendedStats; ci->miniStats = ci->cal_conn_hndl->miniStats; diff --git a/dbcon/mysql/idb_mysql.h b/dbcon/mysql/idb_mysql.h index 670f0e0e0..637f517d7 100644 --- a/dbcon/mysql/idb_mysql.h +++ b/dbcon/mysql/idb_mysql.h @@ -65,6 +65,7 @@ template bool isnan(T); #include "sql_table.h" #include "sql_select.h" +#include "mysqld_error.h" #include "item_windowfunc.h" // Now clean up the pollution as best we can... diff --git a/utils/windowfunction/wf_lead_lag.cpp b/utils/windowfunction/wf_lead_lag.cpp index 8d3b6169b..85ad54f70 100644 --- a/utils/windowfunction/wf_lead_lag.cpp +++ b/utils/windowfunction/wf_lead_lag.cpp @@ -153,32 +153,6 @@ void WF_lead_lag::parseParms(const std::vector& parms) idbassert(cc != NULL); bool isNull = false; // dummy, harded coded fRespectNulls = (cc->getIntVal(fRow, isNull) > 0); -#if 0 - // parms[1]: offset - for (std::vector::size_type i = 1/*ignore 0th element*/; i < parms.size(); ++i) - { - ConstantColumn* cc = dynamic_cast(parms[i].get()); - if (cc != NULL) - { - fOffset = cc->getIntVal(fRow, fOffsetNull) * fLead; // row not used, no need to setData. - continue; - } - - cc = dynamic_cast(parms[i].get()); - if (cc != NULL) - { - getConstValue(cc, fDefault, fDefNull); - continue; - } - // IGNORE/RESPECT nulls is currently broken in the front end - cc = dynamic_cast(parms[i].get()); - if (cc != NULL) - { - bool isNull = false; // dummy, harded coded - fRespectNulls = (cc->getIntVal(fRow, isNull) > 0); - } - } -#endif } From eb7c05dce7ded78d2710e05a8b8303666c5d2281 Mon Sep 17 00:00:00 2001 From: david hill Date: Wed, 24 Aug 2016 08:42:56 -0500 Subject: [PATCH 09/27] update readme --- README.md | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/README.md b/README.md index 5df193e89..86a9168be 100644 --- a/README.md +++ b/README.md @@ -1,4 +1,4 @@ -#MariaDB ColumnStore Storage/Execution engine 1.0.1 +#MariaDB ColumnStore Storage/Execution engine 1.0.2 MariaDB ColumnStore 1.0.2 is the development version of MariaDB ColumnStore. It is built by porting InfiniDB 4.6.2 on MariaDB 10.1.14 and adding entirely new features not found anywhere else. From 4e9d1dfb6e8cff640b651f9cc500c9815e8d7d83 Mon Sep 17 00:00:00 2001 From: david hill Date: Wed, 24 Aug 2016 08:44:57 -0500 Subject: [PATCH 10/27] update readme --- README | 8 ++++---- 1 file changed, 4 insertions(+), 4 deletions(-) diff --git a/README b/README index 9ebf34303..70b9f790f 100644 --- a/README +++ b/README @@ -1,9 +1,9 @@ -This is MariaDB ColumnStore 1.0.6 -MariaDB ColumnStore 1.0.6 is the development version of MariaDB ColumnStore. -It is built by porting InfiniDB 4.6.7 on MariaDB 10.1.19 and adding entirely +This is MariaDB ColumnStore 1.0.2 +MariaDB ColumnStore 1.0.2 is the development version of MariaDB ColumnStore. +It is built by porting InfiniDB 4.6.2 on MariaDB 10.1.14 and adding entirely new features not found anywhere else. -MariaDB ColumnStore 1.0.6 is an GA release. This is the first MariaDB +MariaDB ColumnStore 1.0.2 is an Alpha release. This is the first MariaDB ColumnStore release, not all features planned for the MariaDB ColumnStore 1.0 series are included in this release. From 370afdc346bca32b9217140fc14511305ba67761 Mon Sep 17 00:00:00 2001 From: david hill Date: Thu, 24 Nov 2016 09:14:05 -0600 Subject: [PATCH 11/27] Update README.md --- README.md | 10 ++++++---- 1 file changed, 6 insertions(+), 4 deletions(-) diff --git a/README.md b/README.md index 86a9168be..efeca3c89 100644 --- a/README.md +++ b/README.md @@ -1,9 +1,9 @@ -#MariaDB ColumnStore Storage/Execution engine 1.0.2 -MariaDB ColumnStore 1.0.2 is the development version of MariaDB ColumnStore. -It is built by porting InfiniDB 4.6.2 on MariaDB 10.1.14 and adding entirely +#MariaDB ColumnStore Storage/Execution engine 1.0.5 +MariaDB ColumnStore 1.0.5 is the development version of MariaDB ColumnStore. +It is built by porting InfiniDB 4.6.2 on MariaDB 10.1.19 and adding entirely new features not found anywhere else. -#MariaDB ColumnStore 1.0.2 is an Alpha release. +#MariaDB ColumnStore 1.0.5 is an RC release. This is the first MariaDB ColumnStore release, not all features planned for the MariaDB ColumnStore 1.0 series are included in this release. @@ -16,3 +16,5 @@ This repository is not meant to be built independently outside of the server. T https://github.com/mariadb-corporation/mariadb-columnstore-server +#Issue tracking +Issue tracking of MariaDB ColumnStore happens in JIRA, https://jira.mariadb.org/browse/MCOL From b37cee79de57f195696f9f51eaf84454f6f2a405 Mon Sep 17 00:00:00 2001 From: david hill Date: Fri, 13 Jan 2017 10:02:59 -0600 Subject: [PATCH 12/27] change release number --- README.md | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) diff --git a/README.md b/README.md index efeca3c89..62a8a218d 100644 --- a/README.md +++ b/README.md @@ -1,9 +1,9 @@ -#MariaDB ColumnStore Storage/Execution engine 1.0.5 -MariaDB ColumnStore 1.0.5 is the development version of MariaDB ColumnStore. +#MariaDB ColumnStore Storage/Execution engine 1.0.6 +MariaDB ColumnStore 1.0.6 is the development version of MariaDB ColumnStore. It is built by porting InfiniDB 4.6.2 on MariaDB 10.1.19 and adding entirely new features not found anywhere else. -#MariaDB ColumnStore 1.0.5 is an RC release. +#MariaDB ColumnStore 1.0.6 is an GA release. This is the first MariaDB ColumnStore release, not all features planned for the MariaDB ColumnStore 1.0 series are included in this release. From aa71ca0dcdf3dcd95b31f6d606dd5a25a211dce8 Mon Sep 17 00:00:00 2001 From: david hill Date: Fri, 13 Jan 2017 10:04:42 -0600 Subject: [PATCH 13/27] change release number --- README | 8 ++++---- 1 file changed, 4 insertions(+), 4 deletions(-) diff --git a/README b/README index 70b9f790f..ac59b3816 100644 --- a/README +++ b/README @@ -1,9 +1,9 @@ -This is MariaDB ColumnStore 1.0.2 -MariaDB ColumnStore 1.0.2 is the development version of MariaDB ColumnStore. -It is built by porting InfiniDB 4.6.2 on MariaDB 10.1.14 and adding entirely +This is MariaDB ColumnStore 1.0.6 +MariaDB ColumnStore 1.0.6 is the development version of MariaDB ColumnStore. +It is built by porting InfiniDB 4.6.2 on MariaDB 10.1.19 and adding entirely new features not found anywhere else. -MariaDB ColumnStore 1.0.2 is an Alpha release. This is the first MariaDB +MariaDB ColumnStore 1.0.6 is an GA release. This is the first MariaDB ColumnStore release, not all features planned for the MariaDB ColumnStore 1.0 series are included in this release. From 0d933297880b67d0d67d7323d3f41cb7028986c4 Mon Sep 17 00:00:00 2001 From: david hill Date: Wed, 18 Jan 2017 14:32:54 -0600 Subject: [PATCH 14/27] mcol-481 - just changed the note --- oamapps/postConfigure/postConfigure.cpp | 371 +++++++++++------------- 1 file changed, 175 insertions(+), 196 deletions(-) diff --git a/oamapps/postConfigure/postConfigure.cpp b/oamapps/postConfigure/postConfigure.cpp index 01115e8b6..f8e66311a 100644 --- a/oamapps/postConfigure/postConfigure.cpp +++ b/oamapps/postConfigure/postConfigure.cpp @@ -52,17 +52,6 @@ #include #include -#include -#include -#include -#include - -#include /* for strncpy */ -#include -#include -#include -#include - #include #include #include "boost/filesystem/operations.hpp" @@ -109,7 +98,7 @@ bool updateProcessConfig(int serverTypeInstall); bool uncommentCalpontXml( string entry); bool makeRClocal(string moduleType, string moduleName, int IserverTypeInstall); bool createDbrootDirs(string DBRootStorageType); -bool pkgCheck(std::string columnstorePackage); +bool pkgCheck(); bool storageSetup(bool amazonInstall); void setSystemName(); bool singleServerDBrootSetup(); @@ -126,7 +115,7 @@ typedef struct ModuleIP_struct std::string launchInstance(ModuleIP moduleip); -string columnstorePackage; +string calpontPackage1; //string calpontPackage2; //string calpontPackage3; //string mysqlPackage; @@ -408,7 +397,7 @@ int main(int argc, char *argv[]) //check if MariaDB ColumnStore is up and running if (oam.checkSystemRunning()) { cout << "MariaDB ColumnStore is running, can't run postConfigure while MariaDB ColumnStore is running. Exiting.." << endl; - exit (1); + exit (0); } //if InitialInstallFlag is set, then an install has been done @@ -434,60 +423,6 @@ int main(int argc, char *argv[]) exit(1); } - //check for local ip address as pm1 - ModuleConfig moduleconfig; - - try - { - oam.getSystemConfig("pm1", moduleconfig); - if (moduleconfig.hostConfigList.size() > 0 ) - { - HostConfigList::iterator pt1 = moduleconfig.hostConfigList.begin(); - string PM1ipAdd = (*pt1).IPAddr; - //cout << PM1ipAdd << endl; - - if ( PM1ipAdd != "127.0.0.1" && PM1ipAdd != "0.0.0.0") - { - struct ifaddrs *ifap, *ifa; - struct sockaddr_in *sa; - char *addr; - bool found = false; - - getifaddrs (&ifap); - for (ifa = ifap; ifa; ifa = ifa->ifa_next) { - if (ifa->ifa_addr->sa_family==AF_INET) { - sa = (struct sockaddr_in *) ifa->ifa_addr; - addr = inet_ntoa(sa->sin_addr); - //printf("Interface: %s\tAddress: %s\n", ifa->ifa_name, addr); - - if ( PM1ipAdd == addr ) - { - //match - found = true; - } - } - - if (found) - break; - } - - freeifaddrs(ifap); - - if (!found) - { - cout << endl; - cout << "ERROR: postConfigure install can only be done on the PM1" << endl; - cout << "designated node. The configured PM1 IP address doesn't match the local" << endl; - cout << "IP Address. exiting..." << endl; - exit(1); - } - } - } - } - catch(...) - {} - - // run my.cnf upgrade script if ( reuseConfig == "y" ) { @@ -731,6 +666,75 @@ int main(int argc, char *argv[]) exit(1); } + cout << "NOTE: The MariaDB ColumnStore Schema Sync feature will replicate all of the" << endl; + cout << " schemas and InnoDB tables across the User Module nodes. This feature can be enabled" << endl; + cout << " or disabled, for example, if you wish to configure your own replication post installation." << endl << endl; + + try { + MySQLRep = sysConfig->getConfig(InstallSection, "MySQLRep"); + } + catch(...) + {} + + if ( MySQLRep == "y" ) + mysqlRep = true; + + string answer = "y"; + + while(true) { + if ( mysqlRep ) + prompt = "MariaDB ColumnStore Schema Sync feature is Enabled, do you want to leave enabled? [y,n] (y) > "; + else + prompt = "MariaDB ColumnStore Schema Sync feature, do you want to enable? [y,n] (y) > "; + + pcommand = callReadline(prompt.c_str()); + if (pcommand) { + if (strlen(pcommand) > 0) answer = pcommand; + callFree(pcommand); + } + + if ( answer == "y" || answer == "n" ) { + cout << endl; + break; + } + else + cout << "Invalid Entry, please enter 'y' for yes or 'n' for no" << endl; + if ( noPrompting ) + exit(1); + } + + if ( mysqlRep ) + { // current enabled + if ( answer == "y" ) { + mysqlRep = true; + MySQLRep = "y"; + } + else + { + mysqlRep = false; + MySQLRep = "n"; + } + } + else + { // currently disabled + if ( answer == "y" ) { + mysqlRep = false; + MySQLRep = "n"; + } + else + { + mysqlRep = true; + MySQLRep = "y"; + } + } + + try { + sysConfig->setConfig(InstallSection, "MySQLRep", MySQLRep); + } + catch(...) + {} + + switch ( IserverTypeInstall ) { case (oam::INSTALL_COMBINE_DM_UM_PM): // combined #1 - dm/um/pm on a single server { @@ -820,75 +824,6 @@ int main(int argc, char *argv[]) break; } - // check for Schema Schema is Local Query wasnt selected - if (!pmwithum) - { - cout << "NOTE: The MariaDB ColumnStore Schema Sync feature will replicate all of the" << endl; - cout << " schemas and InnoDB tables across the User Module nodes. This feature can be enabled" << endl; - cout << " or disabled, for example, if you wish to configure your own replication post installation." << endl << endl; - - try { - MySQLRep = sysConfig->getConfig(InstallSection, "MySQLRep"); - } - catch(...) - {} - - if ( MySQLRep == "y" ) - mysqlRep = true; - - string answer = "y"; - - while(true) { - if ( mysqlRep ) - prompt = "MariaDB ColumnStore Schema Sync feature is Enabled, do you want to leave enabled? [y,n] (y) > "; - else - prompt = "MariaDB ColumnStore Schema Sync feature, do you want to enable? [y,n] (y) > "; - - pcommand = callReadline(prompt.c_str()); - if (pcommand) { - if (strlen(pcommand) > 0) answer = pcommand; - callFree(pcommand); - } - - if ( answer == "y" || answer == "n" ) { - cout << endl; - break; - } - else - cout << "Invalid Entry, please enter 'y' for yes or 'n' for no" << endl; - - if ( noPrompting ) - exit(1); - } - - if ( answer == "y" ) { - mysqlRep = true; - MySQLRep = "y"; - } - else - { - mysqlRep = false; - MySQLRep = "n"; - } - - try { - sysConfig->setConfig(InstallSection, "MySQLRep", MySQLRep); - } - catch(...) - {} - } - else - { //Schema Sync is default as on when Local Query is Selected - mysqlRep = true; - MySQLRep = "y"; - - try { - sysConfig->setConfig(InstallSection, "MySQLRep", MySQLRep); - } - catch(...) - {} - } - if ( !writeConfig(sysConfig) ) { cout << "ERROR: Failed trying to update MariaDB ColumnStore System Configuration file" << endl; exit(1); @@ -2676,42 +2611,7 @@ int main(int argc, char *argv[]) /* create a thread_data_t argument array */ thread_data_t thr_data[childmodulelist.size()]; - // determine package type - string EEPackageType; - - if (!rootUser) - EEPackageType = "binary"; - else - { - int rtnCode = system("rpm -qi mariadb-columnstore-platform > /tmp/columnstore.txt 2>&1"); - if (WEXITSTATUS(rtnCode) == 0) - EEPackageType = "rpm"; - else { - rtnCode = system("dpkg -s mariadb-columnstore-platform > /tmp/columnstore.txt 2>&1"); - if (WEXITSTATUS(rtnCode) == 0) - EEPackageType = "deb"; - else - EEPackageType = "binary"; - } - } - - try { - sysConfig->setConfig(InstallSection, "EEPackageType", EEPackageType); - } - catch(...) - { - cout << "ERROR: Problem setting EEPackageType from the MariaDB ColumnStore System Configuration file" << endl; - exit(1); - } - - if ( !writeConfig(sysConfig) ) { - cout << "ERROR: Failed trying to update MariaDB ColumnStore System Configuration file" << endl; - exit(1); - } - - string install = "y"; - if ( IserverTypeInstall != oam::INSTALL_COMBINE_DM_UM_PM || pmNumber > 1 ) { // @@ -2754,6 +2654,39 @@ int main(int argc, char *argv[]) cout << endl; + //Write out Updated System Configuration File + string EEPackageType; + if ( rootUser ) + { + try { + EEPackageType = sysConfig->getConfig(InstallSection, "EEPackageType"); + } + catch(...) + { + cout << "ERROR: Problem getting EEPackageType from the MariaDB ColumnStore System Configuration file" << endl; + exit(1); + } + } + else //nonroot, default to binary + EEPackageType = "binary"; + + while(true) { + prompt = "Enter the Package Type being installed to other servers [rpm,deb,binary] (" + EEPackageType + ") > "; + pcommand = callReadline(prompt); + if (pcommand) { + if (strlen(pcommand) > 0) EEPackageType = pcommand; + callFree(pcommand); + } + + if ( EEPackageType == "rpm" || EEPackageType == "deb" || EEPackageType == "binary" ) { + break; + } + cout << "Invalid Package Type, please re-enter" << endl; + EEPackageType = "rpm"; + if ( noPrompting ) + exit(1); + } + if ( EEPackageType == "rpm" ) { cout << "Performing an MariaDB ColumnStore System install using RPM packages" << endl; @@ -2773,20 +2706,76 @@ int main(int argc, char *argv[]) } } + //Write out Updated System Configuration File + try { + sysConfig->setConfig(InstallSection, "EEPackageType", EEPackageType); + } + catch(...) + { + cout << "ERROR: Problem setting EEPackageType from the MariaDB ColumnStore System Configuration file" << endl; + exit(1); + } + + if ( !writeConfig(sysConfig) ) { + cout << "ERROR: Failed trying to update MariaDB ColumnStore System Configuration file" << endl; + exit(1); + } + //check if pkgs are located in $HOME directory string version = systemsoftware.Version + "-" + systemsoftware.Release; - if ( EEPackageType == "rpm") - columnstorePackage = HOME + "/" + "mariadb-columnstore-" + version + "*.rpm.tar.gz"; - else - if ( EEPackageType == "deb") - columnstorePackage = HOME + "/" + "mariadb-columnstore-" + version + "*.deb.tar.gz"; + if ( EEPackageType != "binary") { + string separator = "-"; + calpontPackage1 = "mariadb-columnstore-*" + separator + version; + //calpontPackage2 = "mariadb-columnstore-libs" + separator + version; + //calpontPackage3 = "mariadb-columnstore-enterprise" + separator + version; + //mysqlPackage = "mariadb-columnstore-storage-engine" + separator + version; + //mysqldPackage = "mariadb-columnstore-mysql" + separator + version; + + if( !pkgCheck() ) { + exit(1); + } else - columnstorePackage = HOME + "/" + "mariadb-columnstore-" + version + "*.bin.tar.gz"; + { + //mariadb + calpontPackage1 = "mariadb-columnstore-*" + separator + version; + calpontPackage1 = HOME + "/" + calpontPackage1 + "*." + EEPackageType; + //calpontPackage2 = HOME + "/" + calpontPackage2 + "*." + EEPackageType; + //calpontPackage3 = HOME + "/" + calpontPackage3 + "*." + EEPackageType; + //mysqlPackage = HOME + "/" + mysqlPackage + "*." + EEPackageType; + //mysqldPackage = HOME + "/" + mysqldPackage + "*." + EEPackageType; + } + } + else + { + // binary + //string fileName = installDir + "/bin/healthcheck"; + //ifstream file (fileName.c_str()); + //if (!file) // CE + calpontPackage1 = "mariadb-columnstore-" + version; + //else // EE + //calpontPackage1 = "mariadb-columnstore-ent-" + version; + //calpontPackage2 = "dummy"; + //calpontPackage3 = "dummy"; + //mysqlPackage = calpontPackage1; + //mysqldPackage = calpontPackage1; + + if( !pkgCheck() ) + exit(1); + calpontPackage1 = HOME + "/" + calpontPackage1 + "*.bin.tar.gz"; + //calpontPackage2 = "dummy"; + //calpontPackage3 = "dummy"; + } + + //If ent pkg is not there, mark it as such + //{ + // glob_t gt; + // memset(>, 0, sizeof(gt)); + // if (glob(calpontPackage3.c_str(), 0, 0, >) != 0) + // calpontPackage3 = "dummy.rpm"; + // globfree(>); + //} - if( !pkgCheck(columnstorePackage) ) - exit(1); - if ( password.empty() ) { cout << endl; @@ -2996,7 +2985,7 @@ int main(int argc, char *argv[]) // checkRemoteMysqlPort(remoteModuleIP, remoteModuleName, USER, password, mysqlPort, sysConfig); cmd = installDir + "/bin/binary_installer.sh " + remoteModuleName + " " + - remoteModuleIP + " " + password + " " + columnstorePackage + " " + remoteModuleType + + remoteModuleIP + " " + password + " " + calpontPackage1 + " " + remoteModuleType + " initial " + binservertype + " " + mysqlPort + " " + remote_installer_debug + " " + installDir + " " + debug_logfile; @@ -3066,7 +3055,7 @@ int main(int argc, char *argv[]) if ( pmwithum ) binservertype = "pmwithum"; cmd = installDir + "/bin/binary_installer.sh " + remoteModuleName + " " + remoteModuleIP + - " " + password + " " + columnstorePackage + " " + remoteModuleType + " initial " + + " " + password + " " + calpontPackage1 + " " + remoteModuleType + " initial " + binservertype + " " + mysqlPort + " " + remote_installer_debug + " " + installDir + " " + debug_logfile; @@ -3683,14 +3672,6 @@ bool setOSFiles(string parentOAMModuleName, int serverTypeInstall) system(cmd.c_str()); } - //check and do the amazon credentials file - string fileName = HOME + "/.aws/credentials"; - ifstream oldFile (fileName.c_str()); - if (!oldFile) - return allfound; - - string cmd = "cp " + fileName + " " + installDir + "/local/etc/. > /dev/null 2>&1"; - system(cmd.c_str()); return allfound; } @@ -3987,14 +3968,14 @@ bool createDbrootDirs(string DBRootStorageType) /* * pkgCheck */ -bool pkgCheck(string columnstorePackage) +bool pkgCheck() { while(true) { - string cmd = "ls " + columnstorePackage + " > /tmp/calpontpkgs"; + string cmd = "ls " + HOME + " | grep " + calpontPackage1 + " > /tmp/calpontpkgs"; system(cmd.c_str()); - string pkg = columnstorePackage; + string pkg = calpontPackage1; string fileName = "/tmp/calpontpkgs"; ifstream oldFile (fileName.c_str()); if (oldFile) { @@ -4858,7 +4839,6 @@ bool storageSetup(bool amazonInstall) void setSystemName() { - Oam oam; //setup System Name try { systemName = sysConfig->getConfig(SystemSection, "SystemName"); @@ -4881,7 +4861,6 @@ void setSystemName() try { sysConfig->setConfig(SystemSection, "SystemName", systemName); - oam.changeMyCnf( "server_audit_syslog_info", systemName ); } catch(...) { From 344a6afd5b02718bd8131edd42c8d79d9abe736f Mon Sep 17 00:00:00 2001 From: david hill Date: Thu, 19 Jan 2017 15:04:00 -0600 Subject: [PATCH 15/27] MCOL-481 fix --- oamapps/postConfigure/postConfigure.cpp | 47 ++++++++++--------------- 1 file changed, 18 insertions(+), 29 deletions(-) diff --git a/oamapps/postConfigure/postConfigure.cpp b/oamapps/postConfigure/postConfigure.cpp index f8e66311a..1679a07d1 100644 --- a/oamapps/postConfigure/postConfigure.cpp +++ b/oamapps/postConfigure/postConfigure.cpp @@ -703,37 +703,26 @@ int main(int argc, char *argv[]) exit(1); } - if ( mysqlRep ) - { // current enabled - if ( answer == "y" ) { - mysqlRep = true; - MySQLRep = "y"; - } - else - { - mysqlRep = false; - MySQLRep = "n"; - } - } - else - { // currently disabled - if ( answer == "y" ) { - mysqlRep = false; - MySQLRep = "n"; - } - else - { - mysqlRep = true; - MySQLRep = "y"; - } - } + if ( answer == "y" ) { + mysqlRep = true; + MySQLRep = "y"; + } + else + { + mysqlRep = false; + MySQLRep = "n"; + } - try { - sysConfig->setConfig(InstallSection, "MySQLRep", MySQLRep); - } - catch(...) - {} + try { + sysConfig->setConfig(InstallSection, "MySQLRep", MySQLRep); + } + catch(...) + {} + if ( !writeConfig(sysConfig) ) { + cout << "ERROR: Failed trying to update MariaDB ColumnStore System Configuration file" << endl; + exit(1); + } switch ( IserverTypeInstall ) { case (oam::INSTALL_COMBINE_DM_UM_PM): // combined #1 - dm/um/pm on a single server From 558970fc9021fa97f1af48d80b405a6d869dc90b Mon Sep 17 00:00:00 2001 From: David Hill Date: Fri, 20 Jan 2017 19:16:54 -0600 Subject: [PATCH 16/27] MCOL-455 - add start remove to help --- oam/etc/ConsoleCmds.xml | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) diff --git a/oam/etc/ConsoleCmds.xml b/oam/etc/ConsoleCmds.xml index 01f6001cc..4bf12d36b 100644 --- a/oam/etc/ConsoleCmds.xml +++ b/oam/etc/ConsoleCmds.xml @@ -29,9 +29,9 @@ redistributeData Redistribute table data accross all dbroots to balance disk usage START to begin a redistribution - START REMOVE n to being a redistribution where data is removed from dbroot 'n' - STOP to stop redistribution before completion - STATUS to to view statistics and progress + STOP to stop redistribution before completion + STATUS to to view statistics and progress + START REMOVE to redistribute and move data off the given dbroot to other dbroots findObjectFile From 81b37c8f937a70343cd08ba45f849f07f9f692bd Mon Sep 17 00:00:00 2001 From: David Hill Date: Fri, 20 Jan 2017 19:25:48 -0600 Subject: [PATCH 17/27] MCOL-455 - add start remove to help --- oam/etc/ConsoleCmds.xml | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) diff --git a/oam/etc/ConsoleCmds.xml b/oam/etc/ConsoleCmds.xml index 4bf12d36b..01f6001cc 100644 --- a/oam/etc/ConsoleCmds.xml +++ b/oam/etc/ConsoleCmds.xml @@ -29,9 +29,9 @@ redistributeData Redistribute table data accross all dbroots to balance disk usage START to begin a redistribution - STOP to stop redistribution before completion - STATUS to to view statistics and progress - START REMOVE to redistribute and move data off the given dbroot to other dbroots + START REMOVE n to being a redistribution where data is removed from dbroot 'n' + STOP to stop redistribution before completion + STATUS to to view statistics and progress findObjectFile From 0b5a5f921680d7c86d89a74dc0feb2f5713ddda3 Mon Sep 17 00:00:00 2001 From: david hill Date: Fri, 20 Jan 2017 19:49:26 -0600 Subject: [PATCH 18/27] MCOL-528 - removed schema sync prompt and default to on when local query is selected --- oamapps/postConfigure/postConfigure.cpp | 127 +++++++++++++----------- 1 file changed, 69 insertions(+), 58 deletions(-) diff --git a/oamapps/postConfigure/postConfigure.cpp b/oamapps/postConfigure/postConfigure.cpp index 1679a07d1..1facf1a7c 100644 --- a/oamapps/postConfigure/postConfigure.cpp +++ b/oamapps/postConfigure/postConfigure.cpp @@ -666,64 +666,6 @@ int main(int argc, char *argv[]) exit(1); } - cout << "NOTE: The MariaDB ColumnStore Schema Sync feature will replicate all of the" << endl; - cout << " schemas and InnoDB tables across the User Module nodes. This feature can be enabled" << endl; - cout << " or disabled, for example, if you wish to configure your own replication post installation." << endl << endl; - - try { - MySQLRep = sysConfig->getConfig(InstallSection, "MySQLRep"); - } - catch(...) - {} - - if ( MySQLRep == "y" ) - mysqlRep = true; - - string answer = "y"; - - while(true) { - if ( mysqlRep ) - prompt = "MariaDB ColumnStore Schema Sync feature is Enabled, do you want to leave enabled? [y,n] (y) > "; - else - prompt = "MariaDB ColumnStore Schema Sync feature, do you want to enable? [y,n] (y) > "; - - pcommand = callReadline(prompt.c_str()); - if (pcommand) { - if (strlen(pcommand) > 0) answer = pcommand; - callFree(pcommand); - } - - if ( answer == "y" || answer == "n" ) { - cout << endl; - break; - } - else - cout << "Invalid Entry, please enter 'y' for yes or 'n' for no" << endl; - if ( noPrompting ) - exit(1); - } - - if ( answer == "y" ) { - mysqlRep = true; - MySQLRep = "y"; - } - else - { - mysqlRep = false; - MySQLRep = "n"; - } - - try { - sysConfig->setConfig(InstallSection, "MySQLRep", MySQLRep); - } - catch(...) - {} - - if ( !writeConfig(sysConfig) ) { - cout << "ERROR: Failed trying to update MariaDB ColumnStore System Configuration file" << endl; - exit(1); - } - switch ( IserverTypeInstall ) { case (oam::INSTALL_COMBINE_DM_UM_PM): // combined #1 - dm/um/pm on a single server { @@ -813,6 +755,75 @@ int main(int argc, char *argv[]) break; } + // check for Schema Schema is Local Query wasnt selected + if (!pmwithum) + { + cout << "NOTE: The MariaDB ColumnStore Schema Sync feature will replicate all of the" << endl; + cout << " schemas and InnoDB tables across the User Module nodes. This feature can be enabled" << endl; + cout << " or disabled, for example, if you wish to configure your own replication post installation." << endl << endl; + + try { + MySQLRep = sysConfig->getConfig(InstallSection, "MySQLRep"); + } + catch(...) + {} + + if ( MySQLRep == "y" ) + mysqlRep = true; + + string answer = "y"; + + while(true) { + if ( mysqlRep ) + prompt = "MariaDB ColumnStore Schema Sync feature is Enabled, do you want to leave enabled? [y,n] (y) > "; + else + prompt = "MariaDB ColumnStore Schema Sync feature, do you want to enable? [y,n] (y) > "; + + pcommand = callReadline(prompt.c_str()); + if (pcommand) { + if (strlen(pcommand) > 0) answer = pcommand; + callFree(pcommand); + } + + if ( answer == "y" || answer == "n" ) { + cout << endl; + break; + } + else + cout << "Invalid Entry, please enter 'y' for yes or 'n' for no" << endl; + + if ( noPrompting ) + exit(1); + } + + if ( answer == "y" ) { + mysqlRep = true; + MySQLRep = "y"; + } + else + { + mysqlRep = false; + MySQLRep = "n"; + } + + try { + sysConfig->setConfig(InstallSection, "MySQLRep", MySQLRep); + } + catch(...) + {} + } + else + { //Schema Sync is default as on when Local Query is Selected + mysqlRep = true; + MySQLRep = "y"; + + try { + sysConfig->setConfig(InstallSection, "MySQLRep", MySQLRep); + } + catch(...) + {} + } + if ( !writeConfig(sysConfig) ) { cout << "ERROR: Failed trying to update MariaDB ColumnStore System Configuration file" << endl; exit(1); From 4c9fc54e5a3249ac1ea915f66b9086e8311119c5 Mon Sep 17 00:00:00 2001 From: David Hill Date: Sat, 21 Jan 2017 15:51:25 -0600 Subject: [PATCH 19/27] MCOL-527 - change to run mysql scripts when um disk is mounted --- procmon/processmonitor.cpp | 81 +++++++++++++++++++++++++++++++++++--- 1 file changed, 76 insertions(+), 5 deletions(-) diff --git a/procmon/processmonitor.cpp b/procmon/processmonitor.cpp index b45e5be84..e474c69d5 100644 --- a/procmon/processmonitor.cpp +++ b/procmon/processmonitor.cpp @@ -5271,6 +5271,8 @@ int ProcessMonitor::runMasterDist(std::string& password, std::string& slaveModul string cmd = startup::StartUp::installDir() + "/bin/rsync.sh " + ipAddr + " " + password + " " + startup::StartUp::installDir() + " 1 > /tmp/master-dist_" + slaveModule + ".log"; system(cmd.c_str()); + + log.writeLog(__LINE__, "cmd = " + cmd, LOG_TYPE_DEBUG); string logFile = "/tmp/master-dist_" + slaveModule + ".log"; if (!oam.checkLogStatus(logFile, "FAILED")) @@ -5503,15 +5505,47 @@ bool ProcessMonitor::amazonVolumeCheck(int dbrootID) if ( status == "attached" ) { string cmd; if ( rootUser) - cmd = "mount " + deviceName + " " + startup::StartUp::installDir() + "/mysql/db -t ext2 -o defaults > /tmp/um_mount.log"; + cmd = "mount " + deviceName + " " + startup::StartUp::installDir() + "/mysql/db -t ext2 -o noatime,nodiratime,noauto > /tmp/um_mount.log"; else - cmd = "sudo mount " + deviceName + " " + startup::StartUp::installDir() + "/mysql/db -t ext2 -o defaults > /tmp/um_mount.log"; + cmd = "sudo mount " + deviceName + " " + startup::StartUp::installDir() + "/mysql/db -t ext2 -o noatime,nodiratime,noauto,user > /tmp/um_mount.log"; system(cmd.c_str()); log.writeLog(__LINE__, "mount cmd: " + cmd, LOG_TYPE_DEBUG); - cmd = "chown mysql:mysql -R " + startup::StartUp::installDir() + "/mysql/db"; + if ( rootUser) + cmd = "chown -R mysql:mysql " + startup::StartUp::installDir() + "/mysql/db"; + else + cmd = "sudo chown -R " + USER + ":" + USER + " " + startup::StartUp::installDir() + "/mysql/db"; + system(cmd.c_str()); + log.writeLog(__LINE__, "chown cmd: " + cmd, LOG_TYPE_DEBUG); + + //check for setup files in mysq/db, if not, create them for a new install + string file = startup::StartUp::installDir() + "/mysql/db/mysql"; + ifstream new_file (file.c_str()); + if (!new_file) { + string cmd; + cmd = startup::StartUp::installDir() + "/bin/post-mysqld-install --installdir=" + startup::StartUp::installDir() + " > /tmp/post-mysqld-install.log 2>&1"; + log.writeLog(__LINE__, "cmd: " + cmd, LOG_TYPE_DEBUG); + + int rtnCode = system(cmd.c_str()); + if (WEXITSTATUS(rtnCode) != 0) { + log.writeLog(__LINE__, "amazonVolumeCheck function failed, post-mysqld-install error" , LOG_TYPE_ERROR); + return false; + } + else + log.writeLog(__LINE__, "amazonVolumeCheck function, post-mysqld-install passed" , LOG_TYPE_DEBUG); + + cmd = startup::StartUp::installDir() + "/bin/post-mysql-install --installdir=" + startup::StartUp::installDir() + " > /tmp/post-mysql-install.log";; + log.writeLog(__LINE__, "cmd: " + cmd, LOG_TYPE_DEBUG); + rtnCode = system(cmd.c_str()); + if (WEXITSTATUS(rtnCode) != 0) { + log.writeLog(__LINE__, "amazonVolumeCheck function failed, post-mysql-install error" , LOG_TYPE_ERROR); + return false; + } + else + log.writeLog(__LINE__, "amazonVolumeCheck function, post-mysql-install passed" , LOG_TYPE_DEBUG); + } log.writeLog(__LINE__, "amazonVolumeCheck function successfully completed, volume attached: " + volumeName, LOG_TYPE_DEBUG); return true; @@ -5536,12 +5570,49 @@ bool ProcessMonitor::amazonVolumeCheck(int dbrootID) {} if (oam.attachEC2Volume(volumeName, deviceName, instanceName)) { - string cmd = "mount " + deviceName + " " + startup::StartUp::installDir() + "/mysql/db -t ext2 -o defaults > /dev/null"; + string cmd; + if ( rootUser) + cmd = "mount " + deviceName + " " + startup::StartUp::installDir() + "/mysql/db -t ext2 -o noatime,nodiratime,noauto > /tmp/um_mount.log"; + else + cmd = "sudo mount " + deviceName + " " + startup::StartUp::installDir() + "/mysql/db -t ext2 -o noatime,nodiratime,noauto,user > /tmp/um_mount.log"; + system(cmd.c_str()); log.writeLog(__LINE__, "mount cmd: " + cmd, LOG_TYPE_DEBUG); - cmd = "chown mysql:mysql -R " + startup::StartUp::installDir() + "/mysql/db"; + if ( rootUser) + cmd = "chown -R mysql:mysql " + startup::StartUp::installDir() + "/mysql/db"; + else + cmd = "sudo chown -R " + USER + ":" + USER + " " + startup::StartUp::installDir() + "/mysql/db"; + system(cmd.c_str()); + log.writeLog(__LINE__, "chown cmd: " + cmd, LOG_TYPE_DEBUG); + + //check for setup files in mysq/db, if not, create them for a new install + string file = startup::StartUp::installDir() + "/mysql/db/mysql"; + ifstream new_file (file.c_str()); + if (!new_file) { + string cmd; + cmd = startup::StartUp::installDir() + "/bin/post-mysqld-install --installdir=" + startup::StartUp::installDir() + " > /tmp/post-mysqld-install.log 2>&1"; + log.writeLog(__LINE__, "cmd: " + cmd, LOG_TYPE_DEBUG); + int rtnCode = system(cmd.c_str()); + if (WEXITSTATUS(rtnCode) != 0) { + log.writeLog(__LINE__, "amazonVolumeCheck function failed, post-mysqld-install error" , LOG_TYPE_ERROR); + return false; + } + else + log.writeLog(__LINE__, "amazonVolumeCheck function, post-mysqld-install passed" , LOG_TYPE_DEBUG); + + cmd = startup::StartUp::installDir() + "/bin/post-mysql-install --installdir=" + startup::StartUp::installDir() + " > /tmp/post-mysql-install.log";; + log.writeLog(__LINE__, "cmd: " + cmd, LOG_TYPE_DEBUG); + + rtnCode = system(cmd.c_str()); + if (WEXITSTATUS(rtnCode) != 0) { + log.writeLog(__LINE__, "amazonVolumeCheck function failed, post-mysql-install error" , LOG_TYPE_ERROR); + return false; + } + else + log.writeLog(__LINE__, "amazonVolumeCheck function, post-mysql-install passed" , LOG_TYPE_DEBUG); + } return true; } From 7d2d94e0d34416f87f0c0ce6ff51451f6d1b752c Mon Sep 17 00:00:00 2001 From: david hill Date: Tue, 23 Aug 2016 15:59:13 -0500 Subject: [PATCH 20/27] change readme --- README.md | 10 ++++------ 1 file changed, 4 insertions(+), 6 deletions(-) diff --git a/README.md b/README.md index 62a8a218d..5df193e89 100644 --- a/README.md +++ b/README.md @@ -1,9 +1,9 @@ -#MariaDB ColumnStore Storage/Execution engine 1.0.6 -MariaDB ColumnStore 1.0.6 is the development version of MariaDB ColumnStore. -It is built by porting InfiniDB 4.6.2 on MariaDB 10.1.19 and adding entirely +#MariaDB ColumnStore Storage/Execution engine 1.0.1 +MariaDB ColumnStore 1.0.2 is the development version of MariaDB ColumnStore. +It is built by porting InfiniDB 4.6.2 on MariaDB 10.1.14 and adding entirely new features not found anywhere else. -#MariaDB ColumnStore 1.0.6 is an GA release. +#MariaDB ColumnStore 1.0.2 is an Alpha release. This is the first MariaDB ColumnStore release, not all features planned for the MariaDB ColumnStore 1.0 series are included in this release. @@ -16,5 +16,3 @@ This repository is not meant to be built independently outside of the server. T https://github.com/mariadb-corporation/mariadb-columnstore-server -#Issue tracking -Issue tracking of MariaDB ColumnStore happens in JIRA, https://jira.mariadb.org/browse/MCOL From 6111c1c7e19ebb9f38fdb40c850464adee9ecd61 Mon Sep 17 00:00:00 2001 From: david hill Date: Wed, 18 Jan 2017 14:32:54 -0600 Subject: [PATCH 21/27] mcol-481 - just changed the note --- oamapps/postConfigure/postConfigure.cpp | 138 ++++++++++++------------ 1 file changed, 69 insertions(+), 69 deletions(-) diff --git a/oamapps/postConfigure/postConfigure.cpp b/oamapps/postConfigure/postConfigure.cpp index 1facf1a7c..f8e66311a 100644 --- a/oamapps/postConfigure/postConfigure.cpp +++ b/oamapps/postConfigure/postConfigure.cpp @@ -666,6 +666,75 @@ int main(int argc, char *argv[]) exit(1); } + cout << "NOTE: The MariaDB ColumnStore Schema Sync feature will replicate all of the" << endl; + cout << " schemas and InnoDB tables across the User Module nodes. This feature can be enabled" << endl; + cout << " or disabled, for example, if you wish to configure your own replication post installation." << endl << endl; + + try { + MySQLRep = sysConfig->getConfig(InstallSection, "MySQLRep"); + } + catch(...) + {} + + if ( MySQLRep == "y" ) + mysqlRep = true; + + string answer = "y"; + + while(true) { + if ( mysqlRep ) + prompt = "MariaDB ColumnStore Schema Sync feature is Enabled, do you want to leave enabled? [y,n] (y) > "; + else + prompt = "MariaDB ColumnStore Schema Sync feature, do you want to enable? [y,n] (y) > "; + + pcommand = callReadline(prompt.c_str()); + if (pcommand) { + if (strlen(pcommand) > 0) answer = pcommand; + callFree(pcommand); + } + + if ( answer == "y" || answer == "n" ) { + cout << endl; + break; + } + else + cout << "Invalid Entry, please enter 'y' for yes or 'n' for no" << endl; + if ( noPrompting ) + exit(1); + } + + if ( mysqlRep ) + { // current enabled + if ( answer == "y" ) { + mysqlRep = true; + MySQLRep = "y"; + } + else + { + mysqlRep = false; + MySQLRep = "n"; + } + } + else + { // currently disabled + if ( answer == "y" ) { + mysqlRep = false; + MySQLRep = "n"; + } + else + { + mysqlRep = true; + MySQLRep = "y"; + } + } + + try { + sysConfig->setConfig(InstallSection, "MySQLRep", MySQLRep); + } + catch(...) + {} + + switch ( IserverTypeInstall ) { case (oam::INSTALL_COMBINE_DM_UM_PM): // combined #1 - dm/um/pm on a single server { @@ -755,75 +824,6 @@ int main(int argc, char *argv[]) break; } - // check for Schema Schema is Local Query wasnt selected - if (!pmwithum) - { - cout << "NOTE: The MariaDB ColumnStore Schema Sync feature will replicate all of the" << endl; - cout << " schemas and InnoDB tables across the User Module nodes. This feature can be enabled" << endl; - cout << " or disabled, for example, if you wish to configure your own replication post installation." << endl << endl; - - try { - MySQLRep = sysConfig->getConfig(InstallSection, "MySQLRep"); - } - catch(...) - {} - - if ( MySQLRep == "y" ) - mysqlRep = true; - - string answer = "y"; - - while(true) { - if ( mysqlRep ) - prompt = "MariaDB ColumnStore Schema Sync feature is Enabled, do you want to leave enabled? [y,n] (y) > "; - else - prompt = "MariaDB ColumnStore Schema Sync feature, do you want to enable? [y,n] (y) > "; - - pcommand = callReadline(prompt.c_str()); - if (pcommand) { - if (strlen(pcommand) > 0) answer = pcommand; - callFree(pcommand); - } - - if ( answer == "y" || answer == "n" ) { - cout << endl; - break; - } - else - cout << "Invalid Entry, please enter 'y' for yes or 'n' for no" << endl; - - if ( noPrompting ) - exit(1); - } - - if ( answer == "y" ) { - mysqlRep = true; - MySQLRep = "y"; - } - else - { - mysqlRep = false; - MySQLRep = "n"; - } - - try { - sysConfig->setConfig(InstallSection, "MySQLRep", MySQLRep); - } - catch(...) - {} - } - else - { //Schema Sync is default as on when Local Query is Selected - mysqlRep = true; - MySQLRep = "y"; - - try { - sysConfig->setConfig(InstallSection, "MySQLRep", MySQLRep); - } - catch(...) - {} - } - if ( !writeConfig(sysConfig) ) { cout << "ERROR: Failed trying to update MariaDB ColumnStore System Configuration file" << endl; exit(1); From f8e5f5d7ed6a2462e4fb3ee1da61fc8d6a97ebc3 Mon Sep 17 00:00:00 2001 From: david hill Date: Thu, 19 Jan 2017 15:04:00 -0600 Subject: [PATCH 22/27] MCOL-481 fix --- oamapps/postConfigure/postConfigure.cpp | 47 ++++++++++--------------- 1 file changed, 18 insertions(+), 29 deletions(-) diff --git a/oamapps/postConfigure/postConfigure.cpp b/oamapps/postConfigure/postConfigure.cpp index f8e66311a..1679a07d1 100644 --- a/oamapps/postConfigure/postConfigure.cpp +++ b/oamapps/postConfigure/postConfigure.cpp @@ -703,37 +703,26 @@ int main(int argc, char *argv[]) exit(1); } - if ( mysqlRep ) - { // current enabled - if ( answer == "y" ) { - mysqlRep = true; - MySQLRep = "y"; - } - else - { - mysqlRep = false; - MySQLRep = "n"; - } - } - else - { // currently disabled - if ( answer == "y" ) { - mysqlRep = false; - MySQLRep = "n"; - } - else - { - mysqlRep = true; - MySQLRep = "y"; - } - } + if ( answer == "y" ) { + mysqlRep = true; + MySQLRep = "y"; + } + else + { + mysqlRep = false; + MySQLRep = "n"; + } - try { - sysConfig->setConfig(InstallSection, "MySQLRep", MySQLRep); - } - catch(...) - {} + try { + sysConfig->setConfig(InstallSection, "MySQLRep", MySQLRep); + } + catch(...) + {} + if ( !writeConfig(sysConfig) ) { + cout << "ERROR: Failed trying to update MariaDB ColumnStore System Configuration file" << endl; + exit(1); + } switch ( IserverTypeInstall ) { case (oam::INSTALL_COMBINE_DM_UM_PM): // combined #1 - dm/um/pm on a single server From 18f28a959c639477a81100c70190c67762a8860b Mon Sep 17 00:00:00 2001 From: David Hill Date: Fri, 20 Jan 2017 19:16:54 -0600 Subject: [PATCH 23/27] MCOL-455 - add start remove to help --- oam/etc/ConsoleCmds.xml | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) diff --git a/oam/etc/ConsoleCmds.xml b/oam/etc/ConsoleCmds.xml index 01f6001cc..4bf12d36b 100644 --- a/oam/etc/ConsoleCmds.xml +++ b/oam/etc/ConsoleCmds.xml @@ -29,9 +29,9 @@ redistributeData Redistribute table data accross all dbroots to balance disk usage START to begin a redistribution - START REMOVE n to being a redistribution where data is removed from dbroot 'n' - STOP to stop redistribution before completion - STATUS to to view statistics and progress + STOP to stop redistribution before completion + STATUS to to view statistics and progress + START REMOVE to redistribute and move data off the given dbroot to other dbroots findObjectFile From c2b6d201d4fc9bfa4c3a77c18ae3bac9158b4740 Mon Sep 17 00:00:00 2001 From: david hill Date: Fri, 20 Jan 2017 23:15:00 -0600 Subject: [PATCH 24/27] MCOL-527 - add code for non-root --- procmon/processmonitor.cpp | 81 +++----------------------------------- 1 file changed, 5 insertions(+), 76 deletions(-) diff --git a/procmon/processmonitor.cpp b/procmon/processmonitor.cpp index e474c69d5..b45e5be84 100644 --- a/procmon/processmonitor.cpp +++ b/procmon/processmonitor.cpp @@ -5271,8 +5271,6 @@ int ProcessMonitor::runMasterDist(std::string& password, std::string& slaveModul string cmd = startup::StartUp::installDir() + "/bin/rsync.sh " + ipAddr + " " + password + " " + startup::StartUp::installDir() + " 1 > /tmp/master-dist_" + slaveModule + ".log"; system(cmd.c_str()); - - log.writeLog(__LINE__, "cmd = " + cmd, LOG_TYPE_DEBUG); string logFile = "/tmp/master-dist_" + slaveModule + ".log"; if (!oam.checkLogStatus(logFile, "FAILED")) @@ -5505,47 +5503,15 @@ bool ProcessMonitor::amazonVolumeCheck(int dbrootID) if ( status == "attached" ) { string cmd; if ( rootUser) - cmd = "mount " + deviceName + " " + startup::StartUp::installDir() + "/mysql/db -t ext2 -o noatime,nodiratime,noauto > /tmp/um_mount.log"; + cmd = "mount " + deviceName + " " + startup::StartUp::installDir() + "/mysql/db -t ext2 -o defaults > /tmp/um_mount.log"; else - cmd = "sudo mount " + deviceName + " " + startup::StartUp::installDir() + "/mysql/db -t ext2 -o noatime,nodiratime,noauto,user > /tmp/um_mount.log"; + cmd = "sudo mount " + deviceName + " " + startup::StartUp::installDir() + "/mysql/db -t ext2 -o defaults > /tmp/um_mount.log"; system(cmd.c_str()); log.writeLog(__LINE__, "mount cmd: " + cmd, LOG_TYPE_DEBUG); - if ( rootUser) - cmd = "chown -R mysql:mysql " + startup::StartUp::installDir() + "/mysql/db"; - else - cmd = "sudo chown -R " + USER + ":" + USER + " " + startup::StartUp::installDir() + "/mysql/db"; - + cmd = "chown mysql:mysql -R " + startup::StartUp::installDir() + "/mysql/db"; system(cmd.c_str()); - log.writeLog(__LINE__, "chown cmd: " + cmd, LOG_TYPE_DEBUG); - - //check for setup files in mysq/db, if not, create them for a new install - string file = startup::StartUp::installDir() + "/mysql/db/mysql"; - ifstream new_file (file.c_str()); - if (!new_file) { - string cmd; - cmd = startup::StartUp::installDir() + "/bin/post-mysqld-install --installdir=" + startup::StartUp::installDir() + " > /tmp/post-mysqld-install.log 2>&1"; - log.writeLog(__LINE__, "cmd: " + cmd, LOG_TYPE_DEBUG); - - int rtnCode = system(cmd.c_str()); - if (WEXITSTATUS(rtnCode) != 0) { - log.writeLog(__LINE__, "amazonVolumeCheck function failed, post-mysqld-install error" , LOG_TYPE_ERROR); - return false; - } - else - log.writeLog(__LINE__, "amazonVolumeCheck function, post-mysqld-install passed" , LOG_TYPE_DEBUG); - - cmd = startup::StartUp::installDir() + "/bin/post-mysql-install --installdir=" + startup::StartUp::installDir() + " > /tmp/post-mysql-install.log";; - log.writeLog(__LINE__, "cmd: " + cmd, LOG_TYPE_DEBUG); - rtnCode = system(cmd.c_str()); - if (WEXITSTATUS(rtnCode) != 0) { - log.writeLog(__LINE__, "amazonVolumeCheck function failed, post-mysql-install error" , LOG_TYPE_ERROR); - return false; - } - else - log.writeLog(__LINE__, "amazonVolumeCheck function, post-mysql-install passed" , LOG_TYPE_DEBUG); - } log.writeLog(__LINE__, "amazonVolumeCheck function successfully completed, volume attached: " + volumeName, LOG_TYPE_DEBUG); return true; @@ -5570,49 +5536,12 @@ bool ProcessMonitor::amazonVolumeCheck(int dbrootID) {} if (oam.attachEC2Volume(volumeName, deviceName, instanceName)) { - string cmd; - if ( rootUser) - cmd = "mount " + deviceName + " " + startup::StartUp::installDir() + "/mysql/db -t ext2 -o noatime,nodiratime,noauto > /tmp/um_mount.log"; - else - cmd = "sudo mount " + deviceName + " " + startup::StartUp::installDir() + "/mysql/db -t ext2 -o noatime,nodiratime,noauto,user > /tmp/um_mount.log"; - + string cmd = "mount " + deviceName + " " + startup::StartUp::installDir() + "/mysql/db -t ext2 -o defaults > /dev/null"; system(cmd.c_str()); log.writeLog(__LINE__, "mount cmd: " + cmd, LOG_TYPE_DEBUG); - if ( rootUser) - cmd = "chown -R mysql:mysql " + startup::StartUp::installDir() + "/mysql/db"; - else - cmd = "sudo chown -R " + USER + ":" + USER + " " + startup::StartUp::installDir() + "/mysql/db"; - + cmd = "chown mysql:mysql -R " + startup::StartUp::installDir() + "/mysql/db"; system(cmd.c_str()); - log.writeLog(__LINE__, "chown cmd: " + cmd, LOG_TYPE_DEBUG); - - //check for setup files in mysq/db, if not, create them for a new install - string file = startup::StartUp::installDir() + "/mysql/db/mysql"; - ifstream new_file (file.c_str()); - if (!new_file) { - string cmd; - cmd = startup::StartUp::installDir() + "/bin/post-mysqld-install --installdir=" + startup::StartUp::installDir() + " > /tmp/post-mysqld-install.log 2>&1"; - log.writeLog(__LINE__, "cmd: " + cmd, LOG_TYPE_DEBUG); - int rtnCode = system(cmd.c_str()); - if (WEXITSTATUS(rtnCode) != 0) { - log.writeLog(__LINE__, "amazonVolumeCheck function failed, post-mysqld-install error" , LOG_TYPE_ERROR); - return false; - } - else - log.writeLog(__LINE__, "amazonVolumeCheck function, post-mysqld-install passed" , LOG_TYPE_DEBUG); - - cmd = startup::StartUp::installDir() + "/bin/post-mysql-install --installdir=" + startup::StartUp::installDir() + " > /tmp/post-mysql-install.log";; - log.writeLog(__LINE__, "cmd: " + cmd, LOG_TYPE_DEBUG); - - rtnCode = system(cmd.c_str()); - if (WEXITSTATUS(rtnCode) != 0) { - log.writeLog(__LINE__, "amazonVolumeCheck function failed, post-mysql-install error" , LOG_TYPE_ERROR); - return false; - } - else - log.writeLog(__LINE__, "amazonVolumeCheck function, post-mysql-install passed" , LOG_TYPE_DEBUG); - } return true; } From 3fdae115a2c50560c13d14beb5ca81a4add10a07 Mon Sep 17 00:00:00 2001 From: David Hall Date: Fri, 21 Apr 2017 10:18:06 -0500 Subject: [PATCH 25/27] MCOL-597 Update README and README.md to avoid merge conflict --- README | 2 +- README.md | 16 +++++++--------- 2 files changed, 8 insertions(+), 10 deletions(-) diff --git a/README b/README index ac59b3816..9ebf34303 100644 --- a/README +++ b/README @@ -1,6 +1,6 @@ This is MariaDB ColumnStore 1.0.6 MariaDB ColumnStore 1.0.6 is the development version of MariaDB ColumnStore. -It is built by porting InfiniDB 4.6.2 on MariaDB 10.1.19 and adding entirely +It is built by porting InfiniDB 4.6.7 on MariaDB 10.1.19 and adding entirely new features not found anywhere else. MariaDB ColumnStore 1.0.6 is an GA release. This is the first MariaDB diff --git a/README.md b/README.md index 5df193e89..577ac33c1 100644 --- a/README.md +++ b/README.md @@ -1,18 +1,16 @@ -#MariaDB ColumnStore Storage/Execution engine 1.0.1 -MariaDB ColumnStore 1.0.2 is the development version of MariaDB ColumnStore. -It is built by porting InfiniDB 4.6.2 on MariaDB 10.1.14 and adding entirely +#MariaDB ColumnStore Storage/Execution engine 1.1.0 +MariaDB ColumnStore 1.1.0 is the development version of MariaDB ColumnStore. +It is built by porting InfiniDB 4.6.7 on MariaDB 10.1.21 and adding entirely new features not found anywhere else. -#MariaDB ColumnStore 1.0.2 is an Alpha release. -This is the first MariaDB ColumnStore release, not all features planned for the MariaDB ColumnStore 1.0 -series are included in this release. +#MariaDB ColumnStore 1.1.0 is an development release. -Additional features will be pushed in future releases. -A few things to notice: -- Do not use alpha releases on production systems. +- Do not use development releases on production systems. #Building This repository is not meant to be built independently outside of the server. This repository is integrated into http://mariadb-corporation/mariadb-columnstore-server (ie, the *server*) as a git submodule. As such, you can find complete build instructions on *the server* page. https://github.com/mariadb-corporation/mariadb-columnstore-server +#Issue tracking +Issue tracking of MariaDB ColumnStore happens in JIRA, https://jira.mariadb.org/browse/MCOL From 6a0362a3273f5b1426763144b2575d1bc5d6ba2b Mon Sep 17 00:00:00 2001 From: David Hall Date: Fri, 21 Apr 2017 10:29:49 -0500 Subject: [PATCH 26/27] MCOL-597 changes from develop --- oamapps/postConfigure/postConfigure.cpp | 360 +++++++++++++----------- 1 file changed, 196 insertions(+), 164 deletions(-) diff --git a/oamapps/postConfigure/postConfigure.cpp b/oamapps/postConfigure/postConfigure.cpp index 1679a07d1..01115e8b6 100644 --- a/oamapps/postConfigure/postConfigure.cpp +++ b/oamapps/postConfigure/postConfigure.cpp @@ -52,6 +52,17 @@ #include #include +#include +#include +#include +#include + +#include /* for strncpy */ +#include +#include +#include +#include + #include #include #include "boost/filesystem/operations.hpp" @@ -98,7 +109,7 @@ bool updateProcessConfig(int serverTypeInstall); bool uncommentCalpontXml( string entry); bool makeRClocal(string moduleType, string moduleName, int IserverTypeInstall); bool createDbrootDirs(string DBRootStorageType); -bool pkgCheck(); +bool pkgCheck(std::string columnstorePackage); bool storageSetup(bool amazonInstall); void setSystemName(); bool singleServerDBrootSetup(); @@ -115,7 +126,7 @@ typedef struct ModuleIP_struct std::string launchInstance(ModuleIP moduleip); -string calpontPackage1; +string columnstorePackage; //string calpontPackage2; //string calpontPackage3; //string mysqlPackage; @@ -397,7 +408,7 @@ int main(int argc, char *argv[]) //check if MariaDB ColumnStore is up and running if (oam.checkSystemRunning()) { cout << "MariaDB ColumnStore is running, can't run postConfigure while MariaDB ColumnStore is running. Exiting.." << endl; - exit (0); + exit (1); } //if InitialInstallFlag is set, then an install has been done @@ -423,6 +434,60 @@ int main(int argc, char *argv[]) exit(1); } + //check for local ip address as pm1 + ModuleConfig moduleconfig; + + try + { + oam.getSystemConfig("pm1", moduleconfig); + if (moduleconfig.hostConfigList.size() > 0 ) + { + HostConfigList::iterator pt1 = moduleconfig.hostConfigList.begin(); + string PM1ipAdd = (*pt1).IPAddr; + //cout << PM1ipAdd << endl; + + if ( PM1ipAdd != "127.0.0.1" && PM1ipAdd != "0.0.0.0") + { + struct ifaddrs *ifap, *ifa; + struct sockaddr_in *sa; + char *addr; + bool found = false; + + getifaddrs (&ifap); + for (ifa = ifap; ifa; ifa = ifa->ifa_next) { + if (ifa->ifa_addr->sa_family==AF_INET) { + sa = (struct sockaddr_in *) ifa->ifa_addr; + addr = inet_ntoa(sa->sin_addr); + //printf("Interface: %s\tAddress: %s\n", ifa->ifa_name, addr); + + if ( PM1ipAdd == addr ) + { + //match + found = true; + } + } + + if (found) + break; + } + + freeifaddrs(ifap); + + if (!found) + { + cout << endl; + cout << "ERROR: postConfigure install can only be done on the PM1" << endl; + cout << "designated node. The configured PM1 IP address doesn't match the local" << endl; + cout << "IP Address. exiting..." << endl; + exit(1); + } + } + } + } + catch(...) + {} + + // run my.cnf upgrade script if ( reuseConfig == "y" ) { @@ -666,64 +731,6 @@ int main(int argc, char *argv[]) exit(1); } - cout << "NOTE: The MariaDB ColumnStore Schema Sync feature will replicate all of the" << endl; - cout << " schemas and InnoDB tables across the User Module nodes. This feature can be enabled" << endl; - cout << " or disabled, for example, if you wish to configure your own replication post installation." << endl << endl; - - try { - MySQLRep = sysConfig->getConfig(InstallSection, "MySQLRep"); - } - catch(...) - {} - - if ( MySQLRep == "y" ) - mysqlRep = true; - - string answer = "y"; - - while(true) { - if ( mysqlRep ) - prompt = "MariaDB ColumnStore Schema Sync feature is Enabled, do you want to leave enabled? [y,n] (y) > "; - else - prompt = "MariaDB ColumnStore Schema Sync feature, do you want to enable? [y,n] (y) > "; - - pcommand = callReadline(prompt.c_str()); - if (pcommand) { - if (strlen(pcommand) > 0) answer = pcommand; - callFree(pcommand); - } - - if ( answer == "y" || answer == "n" ) { - cout << endl; - break; - } - else - cout << "Invalid Entry, please enter 'y' for yes or 'n' for no" << endl; - if ( noPrompting ) - exit(1); - } - - if ( answer == "y" ) { - mysqlRep = true; - MySQLRep = "y"; - } - else - { - mysqlRep = false; - MySQLRep = "n"; - } - - try { - sysConfig->setConfig(InstallSection, "MySQLRep", MySQLRep); - } - catch(...) - {} - - if ( !writeConfig(sysConfig) ) { - cout << "ERROR: Failed trying to update MariaDB ColumnStore System Configuration file" << endl; - exit(1); - } - switch ( IserverTypeInstall ) { case (oam::INSTALL_COMBINE_DM_UM_PM): // combined #1 - dm/um/pm on a single server { @@ -813,6 +820,75 @@ int main(int argc, char *argv[]) break; } + // check for Schema Schema is Local Query wasnt selected + if (!pmwithum) + { + cout << "NOTE: The MariaDB ColumnStore Schema Sync feature will replicate all of the" << endl; + cout << " schemas and InnoDB tables across the User Module nodes. This feature can be enabled" << endl; + cout << " or disabled, for example, if you wish to configure your own replication post installation." << endl << endl; + + try { + MySQLRep = sysConfig->getConfig(InstallSection, "MySQLRep"); + } + catch(...) + {} + + if ( MySQLRep == "y" ) + mysqlRep = true; + + string answer = "y"; + + while(true) { + if ( mysqlRep ) + prompt = "MariaDB ColumnStore Schema Sync feature is Enabled, do you want to leave enabled? [y,n] (y) > "; + else + prompt = "MariaDB ColumnStore Schema Sync feature, do you want to enable? [y,n] (y) > "; + + pcommand = callReadline(prompt.c_str()); + if (pcommand) { + if (strlen(pcommand) > 0) answer = pcommand; + callFree(pcommand); + } + + if ( answer == "y" || answer == "n" ) { + cout << endl; + break; + } + else + cout << "Invalid Entry, please enter 'y' for yes or 'n' for no" << endl; + + if ( noPrompting ) + exit(1); + } + + if ( answer == "y" ) { + mysqlRep = true; + MySQLRep = "y"; + } + else + { + mysqlRep = false; + MySQLRep = "n"; + } + + try { + sysConfig->setConfig(InstallSection, "MySQLRep", MySQLRep); + } + catch(...) + {} + } + else + { //Schema Sync is default as on when Local Query is Selected + mysqlRep = true; + MySQLRep = "y"; + + try { + sysConfig->setConfig(InstallSection, "MySQLRep", MySQLRep); + } + catch(...) + {} + } + if ( !writeConfig(sysConfig) ) { cout << "ERROR: Failed trying to update MariaDB ColumnStore System Configuration file" << endl; exit(1); @@ -2600,7 +2676,42 @@ int main(int argc, char *argv[]) /* create a thread_data_t argument array */ thread_data_t thr_data[childmodulelist.size()]; + // determine package type + string EEPackageType; + + if (!rootUser) + EEPackageType = "binary"; + else + { + int rtnCode = system("rpm -qi mariadb-columnstore-platform > /tmp/columnstore.txt 2>&1"); + if (WEXITSTATUS(rtnCode) == 0) + EEPackageType = "rpm"; + else { + rtnCode = system("dpkg -s mariadb-columnstore-platform > /tmp/columnstore.txt 2>&1"); + if (WEXITSTATUS(rtnCode) == 0) + EEPackageType = "deb"; + else + EEPackageType = "binary"; + } + } + + try { + sysConfig->setConfig(InstallSection, "EEPackageType", EEPackageType); + } + catch(...) + { + cout << "ERROR: Problem setting EEPackageType from the MariaDB ColumnStore System Configuration file" << endl; + exit(1); + } + + if ( !writeConfig(sysConfig) ) { + cout << "ERROR: Failed trying to update MariaDB ColumnStore System Configuration file" << endl; + exit(1); + } + + string install = "y"; + if ( IserverTypeInstall != oam::INSTALL_COMBINE_DM_UM_PM || pmNumber > 1 ) { // @@ -2643,39 +2754,6 @@ int main(int argc, char *argv[]) cout << endl; - //Write out Updated System Configuration File - string EEPackageType; - if ( rootUser ) - { - try { - EEPackageType = sysConfig->getConfig(InstallSection, "EEPackageType"); - } - catch(...) - { - cout << "ERROR: Problem getting EEPackageType from the MariaDB ColumnStore System Configuration file" << endl; - exit(1); - } - } - else //nonroot, default to binary - EEPackageType = "binary"; - - while(true) { - prompt = "Enter the Package Type being installed to other servers [rpm,deb,binary] (" + EEPackageType + ") > "; - pcommand = callReadline(prompt); - if (pcommand) { - if (strlen(pcommand) > 0) EEPackageType = pcommand; - callFree(pcommand); - } - - if ( EEPackageType == "rpm" || EEPackageType == "deb" || EEPackageType == "binary" ) { - break; - } - cout << "Invalid Package Type, please re-enter" << endl; - EEPackageType = "rpm"; - if ( noPrompting ) - exit(1); - } - if ( EEPackageType == "rpm" ) { cout << "Performing an MariaDB ColumnStore System install using RPM packages" << endl; @@ -2695,76 +2773,20 @@ int main(int argc, char *argv[]) } } - //Write out Updated System Configuration File - try { - sysConfig->setConfig(InstallSection, "EEPackageType", EEPackageType); - } - catch(...) - { - cout << "ERROR: Problem setting EEPackageType from the MariaDB ColumnStore System Configuration file" << endl; - exit(1); - } - - if ( !writeConfig(sysConfig) ) { - cout << "ERROR: Failed trying to update MariaDB ColumnStore System Configuration file" << endl; - exit(1); - } - //check if pkgs are located in $HOME directory string version = systemsoftware.Version + "-" + systemsoftware.Release; - if ( EEPackageType != "binary") { - string separator = "-"; - calpontPackage1 = "mariadb-columnstore-*" + separator + version; - //calpontPackage2 = "mariadb-columnstore-libs" + separator + version; - //calpontPackage3 = "mariadb-columnstore-enterprise" + separator + version; - //mysqlPackage = "mariadb-columnstore-storage-engine" + separator + version; - //mysqldPackage = "mariadb-columnstore-mysql" + separator + version; - - if( !pkgCheck() ) { - exit(1); - } - else - { - //mariadb - calpontPackage1 = "mariadb-columnstore-*" + separator + version; - - calpontPackage1 = HOME + "/" + calpontPackage1 + "*." + EEPackageType; - //calpontPackage2 = HOME + "/" + calpontPackage2 + "*." + EEPackageType; - //calpontPackage3 = HOME + "/" + calpontPackage3 + "*." + EEPackageType; - //mysqlPackage = HOME + "/" + mysqlPackage + "*." + EEPackageType; - //mysqldPackage = HOME + "/" + mysqldPackage + "*." + EEPackageType; - } - } + if ( EEPackageType == "rpm") + columnstorePackage = HOME + "/" + "mariadb-columnstore-" + version + "*.rpm.tar.gz"; else - { - // binary - //string fileName = installDir + "/bin/healthcheck"; - //ifstream file (fileName.c_str()); - //if (!file) // CE - calpontPackage1 = "mariadb-columnstore-" + version; - //else // EE - //calpontPackage1 = "mariadb-columnstore-ent-" + version; - //calpontPackage2 = "dummy"; - //calpontPackage3 = "dummy"; - //mysqlPackage = calpontPackage1; - //mysqldPackage = calpontPackage1; + if ( EEPackageType == "deb") + columnstorePackage = HOME + "/" + "mariadb-columnstore-" + version + "*.deb.tar.gz"; + else + columnstorePackage = HOME + "/" + "mariadb-columnstore-" + version + "*.bin.tar.gz"; - if( !pkgCheck() ) - exit(1); - calpontPackage1 = HOME + "/" + calpontPackage1 + "*.bin.tar.gz"; - //calpontPackage2 = "dummy"; - //calpontPackage3 = "dummy"; - } - - //If ent pkg is not there, mark it as such - //{ - // glob_t gt; - // memset(>, 0, sizeof(gt)); - // if (glob(calpontPackage3.c_str(), 0, 0, >) != 0) - // calpontPackage3 = "dummy.rpm"; - // globfree(>); - //} + if( !pkgCheck(columnstorePackage) ) + exit(1); + if ( password.empty() ) { cout << endl; @@ -2974,7 +2996,7 @@ int main(int argc, char *argv[]) // checkRemoteMysqlPort(remoteModuleIP, remoteModuleName, USER, password, mysqlPort, sysConfig); cmd = installDir + "/bin/binary_installer.sh " + remoteModuleName + " " + - remoteModuleIP + " " + password + " " + calpontPackage1 + " " + remoteModuleType + + remoteModuleIP + " " + password + " " + columnstorePackage + " " + remoteModuleType + " initial " + binservertype + " " + mysqlPort + " " + remote_installer_debug + " " + installDir + " " + debug_logfile; @@ -3044,7 +3066,7 @@ int main(int argc, char *argv[]) if ( pmwithum ) binservertype = "pmwithum"; cmd = installDir + "/bin/binary_installer.sh " + remoteModuleName + " " + remoteModuleIP + - " " + password + " " + calpontPackage1 + " " + remoteModuleType + " initial " + + " " + password + " " + columnstorePackage + " " + remoteModuleType + " initial " + binservertype + " " + mysqlPort + " " + remote_installer_debug + " " + installDir + " " + debug_logfile; @@ -3661,6 +3683,14 @@ bool setOSFiles(string parentOAMModuleName, int serverTypeInstall) system(cmd.c_str()); } + //check and do the amazon credentials file + string fileName = HOME + "/.aws/credentials"; + ifstream oldFile (fileName.c_str()); + if (!oldFile) + return allfound; + + string cmd = "cp " + fileName + " " + installDir + "/local/etc/. > /dev/null 2>&1"; + system(cmd.c_str()); return allfound; } @@ -3957,14 +3987,14 @@ bool createDbrootDirs(string DBRootStorageType) /* * pkgCheck */ -bool pkgCheck() +bool pkgCheck(string columnstorePackage) { while(true) { - string cmd = "ls " + HOME + " | grep " + calpontPackage1 + " > /tmp/calpontpkgs"; + string cmd = "ls " + columnstorePackage + " > /tmp/calpontpkgs"; system(cmd.c_str()); - string pkg = calpontPackage1; + string pkg = columnstorePackage; string fileName = "/tmp/calpontpkgs"; ifstream oldFile (fileName.c_str()); if (oldFile) { @@ -4828,6 +4858,7 @@ bool storageSetup(bool amazonInstall) void setSystemName() { + Oam oam; //setup System Name try { systemName = sysConfig->getConfig(SystemSection, "SystemName"); @@ -4850,6 +4881,7 @@ void setSystemName() try { sysConfig->setConfig(SystemSection, "SystemName", systemName); + oam.changeMyCnf( "server_audit_syslog_info", systemName ); } catch(...) { From f6caa88703c4b845dbaa20eda9fa06a17778bbaa Mon Sep 17 00:00:00 2001 From: David Hall Date: Tue, 25 Apr 2017 16:49:32 -0500 Subject: [PATCH 27/27] MCOL-597 merge with 10.2.5 --- dbcon/mysql/ha_calpont_execplan.cpp | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/dbcon/mysql/ha_calpont_execplan.cpp b/dbcon/mysql/ha_calpont_execplan.cpp index d3c0c4e01..4874adc11 100755 --- a/dbcon/mysql/ha_calpont_execplan.cpp +++ b/dbcon/mysql/ha_calpont_execplan.cpp @@ -5953,7 +5953,7 @@ int getSelectPlan(gp_walk_info& gwi, SELECT_LEX& select_lex, SCSEP& csep, bool i if (gwi.thd->infinidb_vtable.vtable_state == THD::INFINIDB_CREATE_VTABLE) { - SQL_LIST order_list = select_lex.order_list; + SQL_I_List order_list = select_lex.order_list; ORDER* ordercol = reinterpret_cast(order_list.first); string create_query(gwi.thd->infinidb_vtable.create_vtable_query.c_ptr()); string select_query(gwi.thd->infinidb_vtable.select_vtable_query.c_ptr());