diff --git a/dbcon/mysql/CMakeLists.txt b/dbcon/mysql/CMakeLists.txt index 965b138ed..d99c75d9f 100644 --- a/dbcon/mysql/CMakeLists.txt +++ b/dbcon/mysql/CMakeLists.txt @@ -8,6 +8,7 @@ include_directories( ${ENGINE_COMMON_INCLUDES} SET ( libcalmysql_SRCS ha_mcs_sysvars.cpp ha_mcs_client_udfs.cpp + ha_mcs_opt_rewrites.cpp ha_mcs_pushdown.cpp ha_mcs.cpp ha_mcs_impl.cpp diff --git a/dbcon/mysql/ha_from_sub.cpp b/dbcon/mysql/ha_from_sub.cpp index 014ded75e..12b2b24de 100644 --- a/dbcon/mysql/ha_from_sub.cpp +++ b/dbcon/mysql/ha_from_sub.cpp @@ -326,8 +326,7 @@ FromSubQuery::FromSubQuery(gp_walk_info& gwip, SELECT_LEX* sub, bool isPushdownHandler) : SubQuery(gwip), - fFromSub(sub), - fPushdownHand(isPushdownHandler) + fFromSub(sub) {} FromSubQuery::~FromSubQuery() @@ -349,9 +348,7 @@ SCSEP FromSubQuery::transform() csep->derivedTbAlias(fAlias); // always lower case csep->derivedTbView(fGwip.viewName.alias); - // DRRTUY isUnion - false. fPushdownHand could be safely set to true - // b/c only pushdowns get here. - if (getSelectPlan(gwi, *fFromSub, csep, false, fPushdownHand) != 0) + if (getSelectPlan(gwi, *fFromSub, csep, false, true) != 0) { fGwip.fatalParseError = true; diff --git a/dbcon/mysql/ha_mcs.h b/dbcon/mysql/ha_mcs.h index c2a9d6532..1a06b7449 100644 --- a/dbcon/mysql/ha_mcs.h +++ b/dbcon/mysql/ha_mcs.h @@ -1,5 +1,6 @@ /* Copyright (C) 2014 InfiniDB, Inc. Copyright (C) 2016 MariaDB Corporation + Copyright (C) 2019 MariaDB Corporation This program is free software; you can redistribute it and/or modify it under the terms of the GNU General Public License diff --git a/dbcon/mysql/ha_mcs_execplan.cpp b/dbcon/mysql/ha_mcs_execplan.cpp index df6bc4959..63dbb91a8 100755 --- a/dbcon/mysql/ha_mcs_execplan.cpp +++ b/dbcon/mysql/ha_mcs_execplan.cpp @@ -16,11 +16,6 @@ Foundation, Inc., 51 Franklin Street, Fifth Floor, Boston, MA 02110-1301, USA. */ -/* - * $Id: ha_mcs_execplan.cpp 9749 2013-08-15 04:00:39Z zzhu $ - */ - -/** @file */ //#define DEBUG_WALK_COND #include #include @@ -61,7 +56,6 @@ using namespace logging; #include "ha_mcs_impl_if.h" #include "ha_mcs_sysvars.h" #include "ha_subquery.h" -//#include "ha_view.h" using namespace cal_impl_if; #include "calpontselectexecutionplan.h" @@ -132,7 +126,6 @@ public: gp_walk_info* fgwip; }; -//#define OUTER_JOIN_DEBUG namespace { string lower(string str) @@ -1321,6 +1314,7 @@ uint32_t buildOuterJoin(gp_walk_info& gwi, SELECT_LEX& select_lex) if (table_ptr->outer_join && table_ptr->on_expr) { + // inner tables block Item_cond* expr = reinterpret_cast(table_ptr->on_expr); gwi_outer.innerTables.insert(tan); @@ -1392,8 +1386,10 @@ uint32_t buildOuterJoin(gp_walk_info& gwi, SELECT_LEX& select_lex) #endif expr->traverse_cond(gp_walk, &gwi_outer, Item::POSTFIX); } +// *DRRTUY This part is to be removed in 1.4.3 +#if 0 // @bug 2849 - else if (table_ptr->embedding && table_ptr->embedding->nested_join) + /*else if (table_ptr->embedding && table_ptr->embedding->nested_join) { // if this is dervied table process phase, mysql may have not developed the plan // completely. Return and let it finish. It will come to rnd_init again. @@ -1434,8 +1430,8 @@ uint32_t buildOuterJoin(gp_walk_info& gwi, SELECT_LEX& select_lex) } } } - } - + } */ +#endif // Error out subquery in outer join on filter for now if (gwi_outer.hasSubSelect) { @@ -1444,9 +1440,8 @@ uint32_t buildOuterJoin(gp_walk_info& gwi, SELECT_LEX& select_lex) setError(gwi.thd, ER_INTERNAL_ERROR, gwi.parseErrorText); return -1; } - // build outerjoinon filter - ParseTree* filters = NULL, *ptp = NULL, /**rhs = NULL*/*lhs = NULL; + ParseTree* filters = NULL, *ptp = NULL, *lhs = NULL; while (!gwi_outer.ptWorkStack.empty()) { @@ -6096,69 +6091,15 @@ bool isMCSTable(TABLE* table_ptr) return false; } -int getSelectPlan(gp_walk_info& gwi, SELECT_LEX& select_lex, - SCSEP& csep, - bool isUnion, - bool isPushdownHand) +/*@brief set some runtime params to run the query */ +/*********************************************************** + * DESCRIPTION: + * This function just sets a number of runtime params that + * limits resource consumed. + ***********************************************************/ +void setExecutionParams(gp_walk_info &gwi, SCSEP &csep) { -#ifdef DEBUG_WALK_COND - cerr << "getSelectPlan()" << endl; -#endif - - // by pass the derived table resolve phase of mysql - if ( !isPushdownHand && - !(((gwi.thd->lex)->sql_command == SQLCOM_UPDATE ) || - ((gwi.thd->lex)->sql_command == SQLCOM_DELETE ) || - ((gwi.thd->lex)->sql_command == SQLCOM_UPDATE_MULTI ) || - ((gwi.thd->lex)->sql_command == SQLCOM_DELETE_MULTI ) ) && gwi.thd->derived_tables_processing) - { - // MCOL-2178 isUnion member only assigned, never used - //MIGR::infinidb_vtable.isUnion = false; - return -1; - } - - // rollup is currently not supported - if (select_lex.olap == ROLLUP_TYPE) - { - gwi.fatalParseError = true; - gwi.parseErrorText = IDBErrorInfo::instance()->errorMsg(ERR_ROLLUP_NOT_SUPPORT); - setError(gwi.thd, ER_CHECK_NOT_IMPLEMENTED, gwi.parseErrorText, gwi); - return ER_CHECK_NOT_IMPLEMENTED; - } - gwi.internalDecimalScale = (get_use_decimal_scale(gwi.thd) ? get_decimal_scale(gwi.thd) : -1); - - gwi.subSelectType = csep->subType(); - - JOIN* join = select_lex.join; - Item_cond* icp = 0; - - if (join != 0) - icp = reinterpret_cast(join->conds); - - // if icp is null, try to find the where clause other where - if (!join && gwi.thd->lex->derived_tables) - { - if (select_lex.prep_where) - icp = (Item_cond*)(select_lex.prep_where); - else if (select_lex.where) - icp = (Item_cond*)(select_lex.where); - } - else if (!join && ( ((gwi.thd->lex)->sql_command == SQLCOM_UPDATE ) || - ((gwi.thd->lex)->sql_command == SQLCOM_DELETE ) || - ((gwi.thd->lex)->sql_command == SQLCOM_UPDATE_MULTI ) || - ((gwi.thd->lex)->sql_command == SQLCOM_DELETE_MULTI ))) - { - icp = reinterpret_cast(select_lex.where); - } - - uint32_t sessionID = csep->sessionID(); - gwi.sessionid = sessionID; - boost::shared_ptr csc = CalpontSystemCatalog::makeCalpontSystemCatalog(sessionID); - csc->identity(CalpontSystemCatalog::FE); - csep->timeZone(gwi.thd->variables.time_zone->get_name()->ptr()); - gwi.csc = csc; - // @bug 2123. Override large table estimate if infinidb_ordered hint was used. // @bug 2404. Always override if the infinidb_ordered_only variable is turned on. if (get_ordered_only(gwi.thd)) @@ -6180,13 +6121,30 @@ int getSelectPlan(gp_walk_info& gwi, SELECT_LEX& select_lex, csep->umMemLimit(numeric_limits::max()); else csep->umMemLimit(get_um_mem_limit(gwi.thd) * 1024ULL * 1024); +} +/*@brief Process FROM part of the query or sub-query */ +/*********************************************************** + * DESCRIPTION: + * This function processes elements of List in + * FROM part of the query. + * isUnion tells that CS processes FROM taken from UNION UNIT. + * The notion is described in MDB code. + * on_expr_list ON expressions used in OUTER JOINs. These are + * later used in processWhere() + * RETURNS + * error id as an int + ***********************************************************/ +int processFrom(bool &isUnion, + SELECT_LEX &select_lex, + gp_walk_info &gwi, + SCSEP &csep, + List &on_expr_list) +{ // populate table map and trigger syscolumn cache for all the tables (@bug 1637). // all tables on FROM list must have at least one col in colmap TABLE_LIST* table_ptr = select_lex.get_table_list(); - CalpontSelectExecutionPlan::SelectList derivedTbList; -// DEBUG #ifdef DEBUG_WALK_COND List_iterator sj_list_it(select_lex.sj_nests); TABLE_LIST* sj_nest; @@ -6195,12 +6153,8 @@ int getSelectPlan(gp_walk_info& gwi, SELECT_LEX& select_lex, { cerr << sj_nest->db.str << "." << sj_nest->table_name.str << endl; } - #endif - // @bug 1796. Remember table order on the FROM list. - gwi.clauseType = FROM; - try { for (; table_ptr; table_ptr = table_ptr->next_local) @@ -6214,17 +6168,20 @@ int getSelectPlan(gp_walk_info& gwi, SELECT_LEX& select_lex, setError(gwi.thd, ER_CHECK_NOT_IMPLEMENTED, gwi.parseErrorText, gwi); return ER_CHECK_NOT_IMPLEMENTED; } + + // Save on_expr to use it for WHERE processing + if (!table_ptr->outer_join && table_ptr->on_expr) + { + on_expr_list.push_back(table_ptr->on_expr); + } string viewName = getViewName(table_ptr); // @todo process from subquery if (table_ptr->derived) { - String str; - (table_ptr->derived->first_select())->print(gwi.thd, &str, QT_ORDINARY); - SELECT_LEX* select_cursor = table_ptr->derived->first_select(); - FromSubQuery fromSub(gwi, select_cursor, isPushdownHand); + FromSubQuery fromSub(gwi, select_cursor, true); string alias(table_ptr->alias.str); fromSub.alias(lower(alias)); @@ -6235,7 +6192,7 @@ int getSelectPlan(gp_walk_info& gwi, SELECT_LEX& select_lex, if (!plan) { setError(gwi.thd, ER_INTERNAL_ERROR, fromSub.gwip().parseErrorText, gwi); - CalpontSystemCatalog::removeCalpontSystemCatalog(sessionID); + CalpontSystemCatalog::removeCalpontSystemCatalog(gwi.sessionid); return ER_INTERNAL_ERROR; } @@ -6261,7 +6218,7 @@ int getSelectPlan(gp_walk_info& gwi, SELECT_LEX& select_lex, // trigger system catalog cache if (columnStore) - csc->columnRIDs(make_table(table_ptr->db.str, table_ptr->table_name.str), true); + gwi.csc->columnRIDs(make_table(table_ptr->db.str, table_ptr->table_name.str), true); string table_name = table_ptr->table_name.str; @@ -6288,7 +6245,7 @@ int getSelectPlan(gp_walk_info& gwi, SELECT_LEX& select_lex, catch (IDBExcept& ie) { setError(gwi.thd, ER_INTERNAL_ERROR, ie.what(), gwi); - CalpontSystemCatalog::removeCalpontSystemCatalog(sessionID); + CalpontSystemCatalog::removeCalpontSystemCatalog(gwi.sessionid); // @bug 3852. set error status for gwi. gwi.fatalParseError = true; gwi.parseErrorText = ie.what(); @@ -6301,14 +6258,14 @@ int getSelectPlan(gp_walk_info& gwi, SELECT_LEX& select_lex, gwi.fatalParseError = true; gwi.parseErrorText = emsg; setError(gwi.thd, ER_INTERNAL_ERROR, emsg, gwi); - CalpontSystemCatalog::removeCalpontSystemCatalog(sessionID); + CalpontSystemCatalog::removeCalpontSystemCatalog(gwi.sessionid); return ER_INTERNAL_ERROR; } csep->tableList(gwi.tbList); + // Send this recursively to getSelectPlan bool unionSel = false; - // UNION master unit check // Existed pushdown handlers won't get in this scope // except UNION pushdown that is to come. @@ -6331,25 +6288,12 @@ int getSelectPlan(gp_walk_info& gwi, SELECT_LEX& select_lex, plan->traceFlags(csep->traceFlags()); plan->data(csep->data()); - // @bug 3853. When one or more sides or union queries contain derived tables, - // sl->join->zero_result_cause is not trustable. Since we've already handled - // constant filter now (0/1), we can relax the following checking. - // @bug 2547. ignore union unit of zero result set case -// if (sl->join) -// { -// sl->join->optimize(); - // @bug 3067. not clear MySQL's behavior. when in subquery, this variable - // is not trustable. -// if (sl->join->zero_result_cause && !gwi.subQuery) -// continue; -// } - // gwi for the union unit gp_walk_info union_gwi; union_gwi.thd = gwi.thd; uint32_t err = 0; - if ((err = getSelectPlan(union_gwi, *sl, plan, unionSel, isPushdownHand)) != 0) + if ((err = getSelectPlan(union_gwi, *sl, plan, unionSel, true)) != 0) return err; unionVec.push_back(SCEP(plan)); @@ -6357,63 +6301,67 @@ int getSelectPlan(gp_walk_info& gwi, SELECT_LEX& select_lex, // distinct union num if (sl == select_lex.master_unit()->union_distinct) distUnionNum = unionVec.size(); - - /*#ifdef DEBUG_WALK_COND - IDEBUG( cerr << ">>>> UNION DEBUG" << endl ); - JOIN* join = sl->join; - Item_cond* icp = 0; - if (join != 0) - icp = reinterpret_cast(join->conds); - if (icp) - icp->traverse_cond(debug_walk, &gwi, Item::POSTFIX); - IDEBUG ( cerr << *plan << endl ); - IDEBUG ( cerr << "<<<unionVec(unionVec); csep->distinctUnionNum(distUnionNum); } - gwi.clauseType = WHERE; + + return 0; +} + +/*@brief Process WHERE part of the query or sub-query */ +/*********************************************************** + * DESCRIPTION: + * This function processes conditions from either JOIN->conds + * or SELECT_LEX->where|prep_where + * on_expr_list ON expressions used in OUTER JOINs. These are + * populated used in processFrom() + * RETURNS + * error id as an int + ***********************************************************/ +int processWhere(SELECT_LEX &select_lex, + gp_walk_info &gwi, + SCSEP &csep, + List &on_expr_list) +{ + JOIN* join = select_lex.join; + Item_cond* icp = 0; + + if (join != 0) + icp = reinterpret_cast(join->conds); + + // if icp is null, try to find the where clause other where + if (!join && gwi.thd->lex->derived_tables) + { + if (select_lex.prep_where) + icp = (Item_cond*)(select_lex.prep_where); + else if (select_lex.where) + icp = (Item_cond*)(select_lex.where); + } + else if (!join && ( ((gwi.thd->lex)->sql_command == SQLCOM_UPDATE ) || + ((gwi.thd->lex)->sql_command == SQLCOM_DELETE ) || + ((gwi.thd->lex)->sql_command == SQLCOM_UPDATE_MULTI ) || + ((gwi.thd->lex)->sql_command == SQLCOM_DELETE_MULTI ))) + { + icp = reinterpret_cast(select_lex.where); + } if (icp) { -// MariaDB bug 624 - without the fix_fields call, delete with join may error with "No query step". -//#if MYSQL_VERSION_ID < 50172 + // MariaDB bug 624 - without the fix_fields call, delete with join may error with "No query step". //@bug 3039. fix fields for constants if (!icp->is_fixed()) { icp->fix_fields(gwi.thd, (Item**)&icp); } -//#endif gwi.fatalParseError = false; #ifdef DEBUG_WALK_COND - cerr << "------------------ WHERE -----------------------" << endl; + std::cerr << "------------------ WHERE -----------------------" << std::endl; icp->traverse_cond(debug_walk, &gwi, Item::POSTFIX); - if (join && join->cond_equal) - { - List_iterator li(join->cond_equal->current_level); - Item_equal *cur_item_eq; - while ((cur_item_eq= li++)) - { - // DRRTUY TODO replace the block with - //cur_item_eq->traverse_cond(debug_walk, gwip, Item::POSTFIX); - std::cerr << "item_equal("; - Item *item; - Item_equal_fields_iterator it(*cur_item_eq); - while ((item= it++)) - { - std::ostringstream ostream; - std::ostringstream& osr = ostream; - getColNameFromItem(osr, item); - std::cerr << osr.str() << ","; - } - std::cerr << ")" << std::endl; - } - } - cerr << "------------------------------------------------\n" << endl; + std::cerr << "------------------------------------------------\n" << std::endl; #endif icp->traverse_cond(gp_walk, &gwi, Item::POSTFIX); @@ -6442,6 +6390,26 @@ int getSelectPlan(gp_walk_info& gwi, SELECT_LEX& select_lex, (dynamic_cast(gwi.rcWorkStack.top()))->timeZone(gwi.thd->variables.time_zone->get_name()->ptr()); } +#ifdef DEBUG_WALK_COND + std::cerr << "------------------ ON_EXPR -----------------------" << endl; +#endif + // MCOL-3593 MDB now doesn't rewrite and/or consolidate ON and WHERE expressions + // and CS handles INNER ON expressions here. + if (!on_expr_list.is_empty()) + { + List_iterator on_expr_it(on_expr_list); + Item_cond *on_expr = NULL; + while((on_expr = reinterpret_cast(on_expr_it++))) + { + on_expr->traverse_cond(gp_walk, &gwi, Item::POSTFIX); +#ifdef DEBUG_WALK_COND + on_expr->traverse_cond(debug_walk, &gwi, Item::POSTFIX); +#endif + } + } +#ifdef DEBUG_WALK_COND + std::cerr << "-------------------------------------------------\n" << std::endl; +#endif // ZZ - the followinig debug shows the structure of nested outer join. should // use a recursive function. @@ -6532,7 +6500,7 @@ int getSelectPlan(gp_walk_info& gwi, SELECT_LEX& select_lex, // ptWorkStack empty, the item is in rcWorkStack. // MySQL 5.6 (MariaDB?). when icp is null and zero_result_cause is set, a constant 0 // is pushed to rcWorkStack. - if (/*icp && */gwi.ptWorkStack.empty() && !gwi.rcWorkStack.empty()) + if (gwi.ptWorkStack.empty() && !gwi.rcWorkStack.empty()) { filters = new ParseTree(gwi.rcWorkStack.top()); gwi.rcWorkStack.pop(); @@ -6547,11 +6515,9 @@ int getSelectPlan(gp_walk_info& gwi, SELECT_LEX& select_lex, break; ptp = new ParseTree(new LogicOperator("and")); - //ptp->left(filters); ptp->right(filters); lhs = gwi.ptWorkStack.top(); gwi.ptWorkStack.pop(); - //ptp->right(rhs); ptp->left(lhs); gwi.ptWorkStack.push(ptp); } @@ -6564,6 +6530,68 @@ int getSelectPlan(gp_walk_info& gwi, SELECT_LEX& select_lex, filters->drawTree(aTmpDir); } + return 0; +} + +/*@brief Translates SELECT_LEX into CSEP */ +/*********************************************************** + * DESCRIPTION: + * This function takes SELECT_LEX and tries to produce + * a corresponding CSEP out of it. It is made of parts that + * process parts of the query, e.g. FROM, WHERE, SELECT, + * HAVING, GROUP BY, ORDER BY. FROM and WHERE are processed + * by processFrom(), processWhere(). CS calls getSelectPlan() + * recursively to process subqueries. + * ARGS + * isUnion if true CS processes UNION unit now + * isPushdownHand legacy to be removed + * RETURNS + * error id as an int + ***********************************************************/ +int getSelectPlan(gp_walk_info& gwi, SELECT_LEX& select_lex, + SCSEP& csep, + bool isUnion, + bool isPushdownHand) +{ +#ifdef DEBUG_WALK_COND + cerr << "getSelectPlan()" << endl; +#endif + int rc = 0; + // rollup is currently not supported + if (select_lex.olap == ROLLUP_TYPE) + { + gwi.fatalParseError = true; + gwi.parseErrorText = IDBErrorInfo::instance()->errorMsg(ERR_ROLLUP_NOT_SUPPORT); + setError(gwi.thd, ER_CHECK_NOT_IMPLEMENTED, gwi.parseErrorText, gwi); + return ER_CHECK_NOT_IMPLEMENTED; + } + + setExecutionParams(gwi, csep); + + gwi.subSelectType = csep->subType(); + uint32_t sessionID = csep->sessionID(); + gwi.sessionid = sessionID; + boost::shared_ptr csc = CalpontSystemCatalog::makeCalpontSystemCatalog(sessionID); + csc->identity(CalpontSystemCatalog::FE); + csep->timeZone(gwi.thd->variables.time_zone->get_name()->ptr()); + gwi.csc = csc; + + CalpontSelectExecutionPlan::SelectList derivedTbList; + // @bug 1796. Remember table order on the FROM list. + gwi.clauseType = FROM; + List on_expr_list; + if ((rc = processFrom(isUnion, select_lex, gwi, csep, on_expr_list))) + { + return rc; + } + bool unionSel = (!isUnion && select_lex.master_unit()->is_unit_op()) ? true : false; + + gwi.clauseType = WHERE; + if ((rc = processWhere(select_lex, gwi, csep, on_expr_list))) + { + return rc; + } + gwi.clauseType = SELECT; #ifdef DEBUG_WALK_COND { @@ -7116,7 +7144,6 @@ int getSelectPlan(gp_walk_info& gwi, SELECT_LEX& select_lex, for (uint32_t i = 0; i < funcFieldVec.size(); i++) { - //SimpleColumn *sc = new SimpleColumn(funcFieldVec[i]->db_name, bestTableName(funcFieldVec[i])/*funcFieldVec[i]->table_name*/, funcFieldVec[i]->field_name, sessionID); SimpleColumn* sc = buildSimpleColumn(funcFieldVec[i], gwi); if (!sc || gwi.fatalParseError) @@ -7142,7 +7169,6 @@ int getSelectPlan(gp_walk_info& gwi, SELECT_LEX& select_lex, String str; funcFieldVec[i]->print(&str, QT_ORDINARY); sc->alias(string(str.c_ptr())); - //sc->tableAlias(funcFieldVec[i]->table_name); sc->tableAlias(sc->tableAlias()); SRCP srcp(sc); uint32_t j = 0; @@ -7899,8 +7925,9 @@ int getSelectPlan(gp_walk_info& gwi, SELECT_LEX& select_lex, { uint32_t limitOffset = 0; - if (join) + if (select_lex.join) { + JOIN* join = select_lex.join; #if MYSQL_VERSION_ID >= 50172 // @bug5729. After upgrade, join->unit sometimes is uninitialized pointer diff --git a/dbcon/mysql/ha_mcs_impl.cpp b/dbcon/mysql/ha_mcs_impl.cpp index 57dbe8bdb..fef506c6f 100644 --- a/dbcon/mysql/ha_mcs_impl.cpp +++ b/dbcon/mysql/ha_mcs_impl.cpp @@ -413,6 +413,23 @@ int vbin2hex(const uint8_t* p, const unsigned l, char* o) return 0; } +// Table Map is used by both cond_push and table mode processing +// Entries made by cond_push don't have csep though. +// When +bool onlyOneTableinTM(cal_impl_if::cal_connection_info* ci) +{ + size_t counter = 0; + for (auto &tableMapEntry: ci->tableMap) + { + if (tableMapEntry.second.csep) + counter++; + if (counter >= 1) + return false; + } + + return true; +} + int fetchNextRow(uchar* buf, cal_table_info& ti, cal_connection_info* ci, bool handler_flag = false) { int rc = HA_ERR_END_OF_FILE; @@ -2441,7 +2458,9 @@ int ha_mcs_impl_rnd_init(TABLE* table) csep = ti.csep; // for ExeMgr logging sqltext. only log once for the query although multi plans may be sent - if (ci->tableMap.size() == 1) + // CS adds the ti into TM in the end of rnd_init thus we log the SQL + // only once when there is no ti with csep. + if (onlyOneTableinTM(ci)) { ti.csep->data(idb_mysql_query_str(thd)); } @@ -3251,7 +3270,6 @@ void ha_mcs_impl_start_bulk_insert(ha_rows rows, TABLE* table) aCmdLine = aCmdLine + table->s->db.str + " " + table->s->table_name.str ; - //cout << "aCmdLine = " << aCmdLine << endl; std::istringstream ss(aCmdLine); std::string arg; std::vector v2(20, ""); @@ -3278,19 +3296,6 @@ void ha_mcs_impl_start_bulk_insert(ha_rows rows, TABLE* table) saAttr.lpSecurityDescriptor = NULL; HANDLE handleList[2]; const char* pSectionMsg; - // Create a pipe for the child process's STDOUT. -#if 0 // We don't need stdout to come back right now. - pSectionMsg = "Create Stdout"; - bSuccess = CreatePipe(&ci->cpimport_stdout_Rd, &ci->cpimport_stdout_Wr, &saAttr, 0); - - // Ensure the read handle to the pipe for STDIN is not inherited. - if (bSuccess) - { - pSectionMsg = "SetHandleInformation(stdout)"; - bSuccess = SetHandleInformation(ci->cpimport_stdout_Rd, HANDLE_FLAG_INHERIT, 0); - } - -#endif bSuccess = true; // Create a pipe for the child process's STDIN. @@ -3340,10 +3345,8 @@ void ha_mcs_impl_start_bulk_insert(ha_rows rows, TABLE* table) pSectionMsg = "UpdateProcThreadAttribute"; bInitialized = true; handleList[0] = ci->cpimport_stdin_Rd; -// handleList[1] = ci->cpimport_stdout_Wr; bSuccess = UpdateProcThreadAttribute(lpAttributeList, 0, PROC_THREAD_ATTRIBUTE_HANDLE_LIST, -// handleList, 2*sizeof(HANDLE), NULL, NULL); handleList, sizeof(HANDLE), NULL, NULL); } @@ -3365,8 +3368,6 @@ void ha_mcs_impl_start_bulk_insert(ha_rows rows, TABLE* table) siStartInfo.lpAttributeList = lpAttributeList; siStartInfo.StartupInfo.hStdError = NULL; siStartInfo.StartupInfo.hStdOutput = NULL; -// siStartInfo.StartupInfo.hStdError = ci->cpimport_stdout_Wr; -// siStartInfo.StartupInfo.hStdOutput = ci->cpimport_stdout_Wr; siStartInfo.StartupInfo.hStdInput = ci->cpimport_stdin_Rd; siStartInfo.StartupInfo.dwFlags |= STARTF_USESTDHANDLES; // Create the child process. @@ -3485,14 +3486,11 @@ void ha_mcs_impl_start_bulk_insert(ha_rows rows, TABLE* table) { ci->filePtr = fdopen(ci->fdt[1], "w"); ci->cpimport_pid = aChPid; // This is the child PID - //cout << "Child PID is " << aChPid << endl; close(ci->fdt[0]); //close the READER of PARENT ci->fdt[0] = -1; // now we can send all the data thru FIFO[1], writer of PARENT } - //if(aChPid == 0) - //cout << "******** Child finished its work ********" << endl; #endif } else @@ -3500,7 +3498,6 @@ void ha_mcs_impl_start_bulk_insert(ha_rows rows, TABLE* table) if (!ci->dmlProc) { ci->dmlProc = new MessageQueueClient("DMLProc"); - //cout << "start_bulk_insert starts a client " << ci->dmlProc << " for session " << thd->thread_id << endl; } } } @@ -3616,22 +3613,6 @@ int ha_mcs_impl_end_bulk_insert(bool abort, TABLE* table) // @bug 2515. Check command intead of vtable state if ( ( ((thd->lex)->sql_command == SQLCOM_INSERT) || ((thd->lex)->sql_command == SQLCOM_LOAD) || (thd->lex)->sql_command == SQLCOM_INSERT_SELECT) && !ci->singleInsert ) { - - //@Bug 2438. Only load data infile calls last batch process - /* if ( ci->isLoaddataInfile && ((thd->variables.option_bits & (OPTION_NOT_AUTOCOMMIT | OPTION_BEGIN)) || (ci->useCpimport == 0))) { - //@Bug 2829 Handle ctrl-C - if ( thd->killed > 0 ) - abort = true; - - if ( !ci->dmlProc ) - { - ci->dmlProc = new MessageQueueClient("DMLProc"); - //cout << "end_bulk_insert starts a client " << ci->dmlProc << " for session " << thd->thread_id << endl; - } - rc = ha_mcs_impl_write_last_batch(table, *ci, abort); - } - else if ((ci->useCpimport > 0) && (!(thd->variables.option_bits & (OPTION_NOT_AUTOCOMMIT | OPTION_BEGIN))) && (!ci->singleInsert) && ((ci->isLoaddataInfile) || - } */ if ((ci->useCpimport > 0) && (!(thd->variables.option_bits & (OPTION_NOT_AUTOCOMMIT | OPTION_BEGIN))) && (!ci->singleInsert) && ((ci->isLoaddataInfile) || ((thd->lex)->sql_command == SQLCOM_INSERT) || ((thd->lex)->sql_command == SQLCOM_LOAD) || ((thd->lex)->sql_command == SQLCOM_INSERT_SELECT)) ) @@ -4076,9 +4057,7 @@ int ha_mcs_impl_external_lock(THD* thd, TABLE* table, int lock_type) CalTableMap::iterator mapiter = ci->tableMap.find(table); // make sure this is a release lock (2nd) call called in // the table mode. - if (mapiter != ci->tableMap.end() - && (mapiter->second.condInfo && mapiter->second.csep) - && lock_type == 2) + if (mapiter != ci->tableMap.end() && mapiter->second.csep && lock_type == 2) { // table mode if (mapiter->second.conn_hndl) diff --git a/dbcon/mysql/ha_mcs_impl_if.h b/dbcon/mysql/ha_mcs_impl_if.h index 8e2ec327a..e135c9c92 100644 --- a/dbcon/mysql/ha_mcs_impl_if.h +++ b/dbcon/mysql/ha_mcs_impl_if.h @@ -189,7 +189,6 @@ struct cal_table_info enum RowSources { FROM_ENGINE, FROM_FILE }; cal_table_info() : tpl_ctx(0), - //tpl_scan_ctx(0), c(0), msTablePtr(0), conn_hndl(0), diff --git a/dbcon/mysql/ha_mcs_opt_rewrites.cpp b/dbcon/mysql/ha_mcs_opt_rewrites.cpp new file mode 100644 index 000000000..00c08d9e0 --- /dev/null +++ b/dbcon/mysql/ha_mcs_opt_rewrites.cpp @@ -0,0 +1,245 @@ +/* Copyright (C) 2019 MariaDB Corporation + + This program is free software; you can redistribute it and/or + modify it under the terms of the GNU General Public License + as published by the Free Software Foundation; version 2 of + the License. + + This program is distributed in the hope that it will be useful, + but WITHOUT ANY WARRANTY; without even the implied warranty of + MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the + GNU General Public License for more details. + + You should have received a copy of the GNU General Public License + along with this program; if not, write to the Free Software + Foundation, Inc., 51 Franklin Street, Fifth Floor, Boston, + MA 02110-1301, USA. */ +#include "ha_mcs_opt_rewrites.h" + +// Search simplify_joins() function in the server's code for detail +COND * +simplify_joins_(JOIN *join, List *join_list, COND *conds, bool top, + bool in_sj) +{ + TABLE_LIST *table; + NESTED_JOIN *nested_join; + TABLE_LIST *prev_table= 0; + List_iterator li(*join_list); + bool straight_join= MY_TEST(join->select_options & SELECT_STRAIGHT_JOIN); + DBUG_ENTER("simplify_joins"); + + /* + Try to simplify join operations from join_list. + The most outer join operation is checked for conversion first. + */ + while ((table= li++)) + { + table_map used_tables; + table_map not_null_tables= (table_map) 0; + + if ((nested_join= table->nested_join)) + { + /* + If the element of join_list is a nested join apply + the procedure to its nested join list first. + */ + if (table->on_expr) + { + Item *expr= table->on_expr; + /* + If an on expression E is attached to the table, + check all null rejected predicates in this expression. + If such a predicate over an attribute belonging to + an inner table of an embedded outer join is found, + the outer join is converted to an inner join and + the corresponding on expression is added to E. + */ + expr= simplify_joins_(join, &nested_join->join_list, + expr, FALSE, in_sj || table->sj_on_expr); + + if (!table->prep_on_expr || expr != table->on_expr) + { + DBUG_ASSERT(expr); + + table->on_expr= expr; + table->prep_on_expr= expr->copy_andor_structure(join->thd); + } + } + nested_join->used_tables= (table_map) 0; + nested_join->not_null_tables=(table_map) 0; + conds= simplify_joins_(join, &nested_join->join_list, conds, top, + in_sj || table->sj_on_expr); + used_tables= nested_join->used_tables; + not_null_tables= nested_join->not_null_tables; + /* The following two might become unequal after table elimination: */ + nested_join->n_tables= nested_join->join_list.elements; + } + else + { + if (!table->prep_on_expr) + table->prep_on_expr= table->on_expr; + used_tables= table->get_map(); + if (conds) + not_null_tables= conds->not_null_tables(); + } + + if (table->embedding) + { + table->embedding->nested_join->used_tables|= used_tables; + table->embedding->nested_join->not_null_tables|= not_null_tables; + } + + if (!(table->outer_join & (JOIN_TYPE_LEFT | JOIN_TYPE_RIGHT)) || + (used_tables & not_null_tables)) + { + /* + For some of the inner tables there are conjunctive predicates + that reject nulls => the outer join can be replaced by an inner join. + */ + if (table->outer_join && !table->embedding && table->table) + table->table->maybe_null= FALSE; + table->outer_join= 0; + if (!(straight_join || table->straight)) + { + table->dep_tables= 0; + TABLE_LIST *embedding= table->embedding; + while (embedding) + { + if (embedding->nested_join->join_list.head()->outer_join) + { + if (!embedding->sj_subq_pred) + table->dep_tables= embedding->dep_tables; + break; + } + embedding= embedding->embedding; + } + } + if (table->on_expr) + { + /* Add ON expression to the WHERE or upper-level ON condition. */ + if (conds) + { + conds= and_conds(join->thd, conds, table->on_expr); + conds->top_level_item(); + /* conds is always a new item as both cond and on_expr existed */ + DBUG_ASSERT(!conds->is_fixed()); + conds->fix_fields(join->thd, &conds); + } + else + conds= table->on_expr; + table->prep_on_expr= table->on_expr= 0; + } + } + + /* + Only inner tables of non-convertible outer joins + remain with on_expr. + */ + if (table->on_expr) + { + table_map table_on_expr_used_tables= table->on_expr->used_tables(); + table->dep_tables|= table_on_expr_used_tables; + if (table->embedding) + { + table->dep_tables&= ~table->embedding->nested_join->used_tables; + /* + Embedding table depends on tables used + in embedded on expressions. + */ + table->embedding->on_expr_dep_tables|= table_on_expr_used_tables; + } + else + table->dep_tables&= ~table->get_map(); + } + + if (prev_table) + { + /* The order of tables is reverse: prev_table follows table */ + if (prev_table->straight || straight_join) + prev_table->dep_tables|= used_tables; + if (prev_table->on_expr) + { + prev_table->dep_tables|= table->on_expr_dep_tables; + table_map prev_used_tables= prev_table->nested_join ? + prev_table->nested_join->used_tables : + prev_table->get_map(); + /* + If on expression contains only references to inner tables + we still make the inner tables dependent on the outer tables. + It would be enough to set dependency only on one outer table + for them. Yet this is really a rare case. + Note: + RAND_TABLE_BIT mask should not be counted as it + prevents update of inner table dependences. + For example it might happen if RAND() function + is used in JOIN ON clause. + */ + if (!((prev_table->on_expr->used_tables() & + ~(OUTER_REF_TABLE_BIT | RAND_TABLE_BIT)) & + ~prev_used_tables)) + prev_table->dep_tables|= used_tables; + } + } + prev_table= table; + } + + /* + Flatten nested joins that can be flattened. + no ON expression and not a semi-join => can be flattened. + */ + li.rewind(); + while ((table= li++)) + { + nested_join= table->nested_join; + if (table->sj_on_expr && !in_sj) + { + /* + If this is a semi-join that is not contained within another semi-join + leave it intact (otherwise it is flattened) + */ + /* + Make sure that any semi-join appear in + the join->select_lex->sj_nests list only once + */ + List_iterator_fast sj_it(join->select_lex->sj_nests); + TABLE_LIST *sj_nest; + while ((sj_nest= sj_it++)) + { + if (table == sj_nest) + break; + } + if (sj_nest) + continue; + join->select_lex->sj_nests.push_back(table, join->thd->mem_root); + + /* + Also, walk through semi-join children and mark those that are now + top-level + */ + TABLE_LIST *tbl; + List_iterator it(nested_join->join_list); + while ((tbl= it++)) + { + if (!tbl->on_expr && tbl->table) + tbl->table->maybe_null= FALSE; + } + } + else if (nested_join && !table->on_expr) + { + TABLE_LIST *tbl; + List_iterator it(nested_join->join_list); + List repl_list; + while ((tbl= it++)) + { + tbl->embedding= table->embedding; + if (!tbl->embedding && !tbl->on_expr && tbl->table) + tbl->table->maybe_null= FALSE; + tbl->join_list= table->join_list; + repl_list.push_back(tbl, join->thd->mem_root); + tbl->dep_tables|= table->dep_tables; + } + li.replace(repl_list); + } + } + DBUG_RETURN(conds); +} diff --git a/dbcon/mysql/ha_mcs_opt_rewrites.h b/dbcon/mysql/ha_mcs_opt_rewrites.h new file mode 100644 index 000000000..d433c06a5 --- /dev/null +++ b/dbcon/mysql/ha_mcs_opt_rewrites.h @@ -0,0 +1,26 @@ +/* Copyright (C) 2019 MariaDB Corporation + + This program is free software; you can redistribute it and/or + modify it under the terms of the GNU General Public License + as published by the Free Software Foundation; version 2 of + the License. + + This program is distributed in the hope that it will be useful, + but WITHOUT ANY WARRANTY; without even the implied warranty of + MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the + GNU General Public License for more details. + + You should have received a copy of the GNU General Public License + along with this program; if not, write to the Free Software + Foundation, Inc., 51 Franklin Street, Fifth Floor, Boston, + MA 02110-1301, USA. */ + +#ifndef HA_MCS_REWRITES +#define HA_MCS_REWRITES + +#include "idb_mysql.h" + +COND *simplify_joins_(JOIN *join, List *join_list, COND *conds, bool top, bool in_sj); + +#endif + diff --git a/dbcon/mysql/ha_mcs_pushdown.cpp b/dbcon/mysql/ha_mcs_pushdown.cpp index d4adc4fea..bab93edd2 100644 --- a/dbcon/mysql/ha_mcs_pushdown.cpp +++ b/dbcon/mysql/ha_mcs_pushdown.cpp @@ -21,6 +21,27 @@ void check_walk(const Item* item, void* arg); + +void disable_indices_for_CEJ(THD *thd_) +{ + TABLE_LIST* global_list; + for (global_list = thd_->lex->query_tables; global_list; global_list = global_list->next_global) + { + // MCOL-652 - doing this with derived tables can cause bad things to happen + if (!global_list->derived) + { + global_list->index_hints= new (thd_->mem_root) List(); + + global_list->index_hints->push_front(new (thd_->mem_root) + Index_hint(INDEX_HINT_USE, + INDEX_HINT_MASK_JOIN, + NULL, + 0), thd_->mem_root); + + } + } +} + void mutate_optimizer_flags(THD *thd_) { // MCOL-2178 Disable all optimizer flags as it was in the fork. @@ -756,6 +777,7 @@ create_columnstore_select_handler(THD* thd, SELECT_LEX* select_lex) return handler; } + // Remove this in 1.4.3 // Save the original group_list as it can be mutated by the // optimizer which calls the remove_const() function Group_list_ptrs *group_list_ptrs = NULL; @@ -780,9 +802,24 @@ create_columnstore_select_handler(THD* thd, SELECT_LEX* select_lex) // if execution fails. if (!unsupported_feature) { - // Most of optimizer_switch flags disabled in external_lock - join->optimization_state= JOIN::OPTIMIZATION_IN_PROGRESS; - join->optimize_inner(); + disable_indices_for_CEJ(thd); + + if (select_lex->handle_derived(thd->lex, DT_MERGE)) + { + // early quit b/c of the error in handle_derived + return handler; + } + + COND *conds = simplify_joins_(join, select_lex->join_list, join->conds, TRUE, FALSE); + select_lex->optimize_unflattened_subqueries(false); + + if (conds) + { +#ifdef DEBUG_WALK_COND + conds->traverse_cond(cal_impl_if::debug_walk, NULL, Item::POSTFIX); +#endif + join->conds = conds; + } // Impossible HAVING or WHERE // TODO replace with function call @@ -818,43 +855,18 @@ create_columnstore_select_handler(THD* thd, SELECT_LEX* select_lex) } } - if (!unsupported_feature) { handler= new ha_columnstore_select_handler(thd, select_lex); - // This is an ugly hack to call simplify_joins() mcs_handler_info mhi= mcs_handler_info(reinterpret_cast(handler), SELECT); // this::table is the place for the result set int rc= ha_cs_impl_pushdown_init(&mhi, handler->table); - // Return SH if query execution is fine or fallback is disabled - if (!rc || !get_fallback_knob(thd)) - return handler; - - // Reset the DA and restore optimizer flags - // to allow query to fallback to other handlers - if (thd->get_stmt_da()->is_error()) - { - thd->get_stmt_da()->reset_diagnostics_area(); - restore_optimizer_flags(thd); + // Return SH even if init fails b/c CS changed SELECT_LEX structures + // with simplify_joins_() + if (rc) unsupported_feature = true; - } - } - - if (join->optimization_state != JOIN::NOT_OPTIMIZED) - { - if (!join->with_two_phase_optimization) - { - if (unsupported_feature && join->have_query_plan != JOIN::QEP_DELETED) - { - join->build_explain(); - } - join->optimization_state= JOIN::OPTIMIZATION_DONE; - } - else - { - join->optimization_state= JOIN::OPTIMIZATION_PHASE_1_DONE; - } + return handler; } return NULL; diff --git a/dbcon/mysql/ha_mcs_pushdown.h b/dbcon/mysql/ha_mcs_pushdown.h index 7e9d31696..268d6b5b6 100644 --- a/dbcon/mysql/ha_mcs_pushdown.h +++ b/dbcon/mysql/ha_mcs_pushdown.h @@ -23,6 +23,8 @@ #include "ha_mcs_sysvars.h" #define NEED_CALPONT_EXTERNS #include "ha_mcs_impl.h" +#include "ha_mcs_impl_if.h" +#include "ha_mcs_opt_rewrites.h" void mutate_optimizer_flags(THD *thd_); void restore_optimizer_flags(THD *thd_); diff --git a/dbcon/mysql/ha_mcs_sysvars.cpp b/dbcon/mysql/ha_mcs_sysvars.cpp index af4016139..597d50efa 100644 --- a/dbcon/mysql/ha_mcs_sysvars.cpp +++ b/dbcon/mysql/ha_mcs_sysvars.cpp @@ -98,15 +98,6 @@ static MYSQL_THDVAR_BOOL( 1 ); -static MYSQL_THDVAR_BOOL( - processing_handlers_fallback, - PLUGIN_VAR_NOCMDARG, - "Enable/Disable the unsupported features check in handlers.", - NULL, - NULL, - 0 -); - static MYSQL_THDVAR_UINT( orderby_threads, PLUGIN_VAR_RQCMDARG, @@ -304,7 +295,6 @@ st_mysql_sys_var* mcs_system_variables[] = MYSQL_SYSVAR(original_optimizer_flags), MYSQL_SYSVAR(select_handler), MYSQL_SYSVAR(derived_handler), - MYSQL_SYSVAR(processing_handlers_fallback), MYSQL_SYSVAR(group_by_handler), MYSQL_SYSVAR(orderby_threads), MYSQL_SYSVAR(decimal_scale), @@ -391,16 +381,7 @@ void set_group_by_handler(THD* thd, bool value) THDVAR(thd, group_by_handler) = value; } -bool get_fallback_knob(THD* thd) -{ - return ( thd == NULL ) ? false : THDVAR(thd, processing_handlers_fallback); -} -void set_fallback_knob(THD* thd, bool value) -{ - THDVAR(thd, processing_handlers_fallback) = value; -} - - void set_compression_type(THD* thd, ulong value) +void set_compression_type(THD* thd, ulong value) { THDVAR(thd, compression_type) = value; } diff --git a/dbcon/mysql/ha_mcs_sysvars.h b/dbcon/mysql/ha_mcs_sysvars.h index d9ec7f4d0..1efe29fcc 100644 --- a/dbcon/mysql/ha_mcs_sysvars.h +++ b/dbcon/mysql/ha_mcs_sysvars.h @@ -55,9 +55,6 @@ void set_derived_handler(THD* thd, bool value); bool get_group_by_handler(THD* thd); void set_group_by_handler(THD* thd, bool value); -bool get_fallback_knob(THD* thd); -void set_fallback_knob(THD* thd, bool value); - uint get_orderby_threads(THD* thd); void set_orderby_threads(THD* thd, uint value); diff --git a/dbcon/mysql/ha_subquery.h b/dbcon/mysql/ha_subquery.h index bd7f74d38..d14c95131 100644 --- a/dbcon/mysql/ha_subquery.h +++ b/dbcon/mysql/ha_subquery.h @@ -203,7 +203,6 @@ public: private: SELECT_LEX* fFromSub; std::string fAlias; - bool fPushdownHand; }; class SelectSubQuery : public SubQuery diff --git a/dbcon/mysql/sm.cpp b/dbcon/mysql/sm.cpp index a7e476f6f..9925c3c8b 100644 --- a/dbcon/mysql/sm.cpp +++ b/dbcon/mysql/sm.cpp @@ -400,7 +400,7 @@ tpl_close ( cpsm_tplh_t* ntplh, // MCOL-1601 Dispose of unused empty RowGroup if (clear_scan_ctx) { - std::cout << "tpl_close() clear_scan_ctx read" << std::endl; + SMDEBUGLOG << "tpl_close() clear_scan_ctx read" << std::endl; bs = hndl->exeMgr->read(); }