From ab2003baaa594eeec8acbdd0863cd9259217df3d Mon Sep 17 00:00:00 2001 From: Roman Nozdrin Date: Thu, 26 Mar 2020 10:19:27 +0000 Subject: [PATCH 1/4] MCOL-3903 Enable Select Handler to run query part of INSERT..SELECT. Original SH implementation sends the result set back to the client thus it can't be used in INSERT..SELECT, SELECT INTO OUTFILE,CREATE TABLE AS SELECT etc. CLX-77 feature has been backported into MDB to enable SH to run query part of the mentioned queries. --- dbcon/mysql/ha_exists_sub.cpp | 2 +- dbcon/mysql/ha_from_sub.cpp | 5 +- dbcon/mysql/ha_in_sub.cpp | 2 +- dbcon/mysql/ha_mcs_execplan.cpp | 493 +++++++++++------------------- dbcon/mysql/ha_mcs_impl.cpp | 2 +- dbcon/mysql/ha_mcs_impl_if.h | 8 +- dbcon/mysql/ha_mcs_pushdown.cpp | 6 +- dbcon/mysql/ha_scalar_sub.cpp | 2 +- dbcon/mysql/ha_select_sub.cpp | 2 +- dbcon/mysql/ha_subquery.h | 2 +- utils/loggingcpp/ErrorMessage.txt | 2 +- 11 files changed, 201 insertions(+), 325 deletions(-) diff --git a/dbcon/mysql/ha_exists_sub.cpp b/dbcon/mysql/ha_exists_sub.cpp index 9db9294c9..4c6ebd424 100644 --- a/dbcon/mysql/ha_exists_sub.cpp +++ b/dbcon/mysql/ha_exists_sub.cpp @@ -112,7 +112,7 @@ execplan::ParseTree* ExistsSub::transform() return NULL; } - if (getSelectPlan(gwi, *(fSub->get_select_lex()), csep, false, true) != 0) + if (getSelectPlan(gwi, *(fSub->get_select_lex()), csep, false) != 0) { fGwip.fatalParseError = true; diff --git a/dbcon/mysql/ha_from_sub.cpp b/dbcon/mysql/ha_from_sub.cpp index 12b2b24de..d34bc8c2e 100644 --- a/dbcon/mysql/ha_from_sub.cpp +++ b/dbcon/mysql/ha_from_sub.cpp @@ -323,8 +323,7 @@ FromSubQuery::FromSubQuery(gp_walk_info& gwip) : SubQuery(gwip) {} FromSubQuery::FromSubQuery(gp_walk_info& gwip, - SELECT_LEX* sub, - bool isPushdownHandler) : + SELECT_LEX* sub) : SubQuery(gwip), fFromSub(sub) {} @@ -348,7 +347,7 @@ SCSEP FromSubQuery::transform() csep->derivedTbAlias(fAlias); // always lower case csep->derivedTbView(fGwip.viewName.alias); - if (getSelectPlan(gwi, *fFromSub, csep, false, true) != 0) + if (getSelectPlan(gwi, *fFromSub, csep, false) != 0) { fGwip.fatalParseError = true; diff --git a/dbcon/mysql/ha_in_sub.cpp b/dbcon/mysql/ha_in_sub.cpp index 4d1e828ce..5ae5e18f2 100644 --- a/dbcon/mysql/ha_in_sub.cpp +++ b/dbcon/mysql/ha_in_sub.cpp @@ -148,7 +148,7 @@ execplan::ParseTree* InSub::transform() gwi.tbList.insert(gwi.tbList.begin(), fGwip.tbList.begin(), fGwip.tbList.end()); gwi.derivedTbList.insert(gwi.derivedTbList.begin(), fGwip.derivedTbList.begin(), fGwip.derivedTbList.end()); - if (getSelectPlan(gwi, *(fSub->get_select_lex()), csep, false, true) != 0) + if (getSelectPlan(gwi, *(fSub->get_select_lex()), csep, false) != 0) { fGwip.fatalParseError = true; diff --git a/dbcon/mysql/ha_mcs_execplan.cpp b/dbcon/mysql/ha_mcs_execplan.cpp index 6a40e4992..dc833c7c4 100755 --- a/dbcon/mysql/ha_mcs_execplan.cpp +++ b/dbcon/mysql/ha_mcs_execplan.cpp @@ -3117,7 +3117,10 @@ CalpontSystemCatalog::ColType colType_MysqlToIDB (const Item* item) return ct; } -ReturnedColumn* buildReturnedColumn(Item* item, gp_walk_info& gwi, bool& nonSupport, bool pushdownHand, bool isRefItem) +ReturnedColumn* buildReturnedColumn( + Item* item, gp_walk_info& gwi, + bool& nonSupport, + bool isRefItem) { ReturnedColumn* rc = NULL; @@ -3275,9 +3278,9 @@ ReturnedColumn* buildReturnedColumn(Item* item, gp_walk_info& gwi, bool& nonSupp } if (func_name == "+" || func_name == "-" || func_name == "*" || func_name == "/" ) - return buildArithmeticColumn(ifp, gwi, nonSupport, pushdownHand); + return buildArithmeticColumn(ifp, gwi, nonSupport); else - return buildFunctionColumn(ifp, gwi, nonSupport, pushdownHand); + return buildFunctionColumn(ifp, gwi, nonSupport); } case Item::SUM_FUNC_ITEM: @@ -3410,8 +3413,7 @@ ReturnedColumn* buildReturnedColumn(Item* item, gp_walk_info& gwi, bool& nonSupp ArithmeticColumn* buildArithmeticColumn( Item_func* item, gp_walk_info& gwi, - bool& nonSupport, - bool pushdownHand) + bool& nonSupport) { if (get_fe_conn_info_ptr() == NULL) set_fe_conn_info_ptr((void*)new cal_connection_info()); @@ -3435,7 +3437,7 @@ ArithmeticColumn* buildArithmeticColumn( { if (gwi.clauseType == SELECT || /*gwi.clauseType == HAVING || */gwi.clauseType == GROUP_BY || gwi.clauseType == FROM) // select list { - lhs = new ParseTree(buildReturnedColumn(sfitempp[0], gwi, nonSupport, pushdownHand)); + lhs = new ParseTree(buildReturnedColumn(sfitempp[0], gwi, nonSupport)); if (!lhs->data() && (sfitempp[0]->type() == Item::FUNC_ITEM)) { @@ -3443,7 +3445,7 @@ ArithmeticColumn* buildArithmeticColumn( Item_func* ifp = (Item_func*)sfitempp[0]; lhs = buildParseTree(ifp, gwi, nonSupport); } - else if(pushdownHand && !lhs->data() && (sfitempp[0]->type() == Item::REF_ITEM)) + else if(!lhs->data() && (sfitempp[0]->type() == Item::REF_ITEM)) { // There must be an aggregation column in extended SELECT // list so find the corresponding column. @@ -3454,7 +3456,7 @@ ArithmeticColumn* buildArithmeticColumn( if(rc) lhs = new ParseTree(rc); } - rhs = new ParseTree(buildReturnedColumn(sfitempp[1], gwi, nonSupport, pushdownHand)); + rhs = new ParseTree(buildReturnedColumn(sfitempp[1], gwi, nonSupport)); if (!rhs->data() && (sfitempp[1]->type() == Item::FUNC_ITEM)) { @@ -3462,7 +3464,7 @@ ArithmeticColumn* buildArithmeticColumn( Item_func* ifp = (Item_func*)sfitempp[1]; rhs = buildParseTree(ifp, gwi, nonSupport); } - else if(pushdownHand && !rhs->data() && (sfitempp[1]->type() == Item::REF_ITEM)) + else if(!rhs->data() && (sfitempp[1]->type() == Item::REF_ITEM)) { // There must be an aggregation column in extended SELECT // list so find the corresponding column. @@ -3638,7 +3640,6 @@ ReturnedColumn* buildFunctionColumn( Item_func* ifp, gp_walk_info& gwi, bool& nonSupport, - bool pushdownHand, bool selectBetweenIn) { if (get_fe_conn_info_ptr() == NULL) @@ -3680,7 +3681,7 @@ ReturnedColumn* buildFunctionColumn( // Arithmetic exp if (funcName == "+" || funcName == "-" || funcName == "*" || funcName == "/" ) { - ArithmeticColumn* ac = buildArithmeticColumn(ifp, gwi, nonSupport, pushdownHand); + ArithmeticColumn* ac = buildArithmeticColumn(ifp, gwi, nonSupport); return ac; } @@ -3834,11 +3835,10 @@ ReturnedColumn* buildFunctionColumn( return NULL; } - ReturnedColumn* rc = buildReturnedColumn(ifp->arguments()[i], gwi, nonSupport, pushdownHand); + ReturnedColumn* rc = buildReturnedColumn(ifp->arguments()[i], gwi, nonSupport); // MCOL-1510 It must be a temp table field, so find the corresponding column. - if (!rc && pushdownHand - && ifp->arguments()[i]->type() == Item::REF_ITEM) + if (!rc && ifp->arguments()[i]->type() == Item::REF_ITEM) { gwi.fatalParseError = false; rc = buildAggFrmTempField(ifp->arguments()[i], gwi); @@ -5653,7 +5653,7 @@ void gp_walk(const Item* item, void* arg) if (col->type() != Item::COND_ITEM) { - rc = buildReturnedColumn(col, *gwip, gwip->fatalParseError, false, true); + rc = buildReturnedColumn(col, *gwip, gwip->fatalParseError, false); if ( col->type() == Item::FIELD_ITEM ) gwip->fatalParseError = false; @@ -6192,7 +6192,7 @@ int processFrom(bool &isUnion, if (table_ptr->derived) { SELECT_LEX* select_cursor = table_ptr->derived->first_select(); - FromSubQuery fromSub(gwi, select_cursor, true); + FromSubQuery fromSub(gwi, select_cursor); string alias(table_ptr->alias.str); fromSub.alias(lower(alias)); @@ -6304,7 +6304,7 @@ int processFrom(bool &isUnion, union_gwi.thd = gwi.thd; uint32_t err = 0; - if ((err = getSelectPlan(union_gwi, *sl, plan, unionSel, true)) != 0) + if ((err = getSelectPlan(union_gwi, *sl, plan, unionSel)) != 0) return err; unionVec.push_back(SCEP(plan)); @@ -6544,25 +6544,176 @@ int processWhere(SELECT_LEX &select_lex, return 0; } +/*@brief Process LIMIT part of a query or sub-query */ +/*********************************************************** + * DESCRIPTION: + * Processes LIMIT and OFFSET parts + * RETURNS + * error id as an int + ***********************************************************/ +int processLimitAndOffset( + SELECT_LEX& select_lex, + gp_walk_info& gwi, + SCSEP& csep, + bool unionSel, + bool isUnion, + bool isSelectHandlerTop +) +{ + // LIMIT processing part + uint64_t limitNum = std::numeric_limits::max(); + + // non-MAIN union branch + if (unionSel || gwi.subSelectType != CalpontSelectExecutionPlan::MAIN_SELECT) + { + /* Consider the following query: + "select a from t1 where exists (select b from t2 where a=b);" + CS first builds a hash table for t2, then pushes down the hash to + PrimProc for a distributed hash join execution, with t1 being the + large-side table. However, the server applies an optimization in + Item_exists_subselect::fix_length_and_dec in sql/item_subselect.cc + (see server commit ae476868a5394041a00e75a29c7d45917e8dfae8) + where it sets explicit_limit to true, which causes csep->limitNum set to 1. + This causes the hash table for t2 to only contain a single record for the + hash join, giving less number of rows in the output result set than expected. + We therefore do not allow limit set to 1 here for such queries. + */ + if (gwi.subSelectType != CalpontSelectExecutionPlan::IN_SUBS + && gwi.subSelectType != CalpontSelectExecutionPlan::EXISTS_SUBS + && select_lex.master_unit()->global_parameters()->explicit_limit) + { + if (select_lex.master_unit()->global_parameters()->offset_limit) + { + Item_int* offset = (Item_int*)select_lex.master_unit()->global_parameters()->offset_limit; + csep->limitStart(offset->val_int()); + } + + if (select_lex.master_unit()->global_parameters()->select_limit) + { + Item_int* select = (Item_int*)select_lex.master_unit()->global_parameters()->select_limit; + csep->limitNum(select->val_int()); + // MCOL-894 Activate parallel ORDER BY + csep->orderByThreads(get_orderby_threads(gwi.thd)); + } + } + } + // union with explicit select at the top level + else if (isUnion && select_lex.explicit_limit) + { + if (select_lex.braces) + { + if (select_lex.offset_limit) + csep->limitStart(((Item_int*)select_lex.offset_limit)->val_int()); + + if (select_lex.select_limit) + csep->limitNum(((Item_int*)select_lex.select_limit)->val_int()); + } + } + // other types of queries that have explicit LIMIT + else if (select_lex.explicit_limit) + { + uint32_t limitOffset = 0; + + if (select_lex.join) + { + JOIN* join = select_lex.join; +#if MYSQL_VERSION_ID >= 50172 + + // @bug5729. After upgrade, join->unit sometimes is uninitialized pointer + // (not null though) and will cause seg fault. Prefer checking + // select_lex->offset_limit if not null. + if (join->select_lex && + join->select_lex->offset_limit && + join->select_lex->offset_limit->is_fixed() && + join->select_lex->select_limit && + join->select_lex->select_limit->is_fixed()) + { + limitOffset = join->select_lex->offset_limit->val_int(); + limitNum = join->select_lex->select_limit->val_int(); + } + else if (join->unit) + { + limitOffset = join->unit->offset_limit_cnt; + limitNum = join->unit->select_limit_cnt - limitOffset; + } + +#else + limitOffset = (join->unit)->offset_limit_cnt; + limitNum = (join->unit)->select_limit_cnt - (join->unit)->offset_limit_cnt; +#endif + } + else + { + if (select_lex.master_unit()->global_parameters()->offset_limit) + { + Item_int* offset = (Item_int*)select_lex.master_unit()->global_parameters()->offset_limit; + limitOffset = offset->val_int(); + } + + if (select_lex.master_unit()->global_parameters()->select_limit) + { + Item_int* select = (Item_int*)select_lex.master_unit()->global_parameters()->select_limit; + limitNum = select->val_int(); + } + } + + csep->limitStart(limitOffset); + csep->limitNum(limitNum); + } + // If an explicit limit is not specified, use the system variable value + else + { + csep->limitNum(gwi.thd->variables.select_limit); + } + + // We don't currently support limit with correlated subquery + if (gwi.subQuery && !gwi.correlatedTbNameVec.empty() && csep->hasOrderBy()) + { + gwi.fatalParseError = true; + gwi.parseErrorText = IDBErrorInfo::instance()->errorMsg(ERR_NON_SUPPORT_LIMIT_SUB); + setError(gwi.thd, ER_INTERNAL_ERROR, gwi.parseErrorText, gwi); + return ER_CHECK_NOT_IMPLEMENTED; + } + + // MDB applies OFFSET on its own for SH processing. + // See related MDEV-16327 for details. + if (isSelectHandlerTop && csep->limitStart()) + { + if (std::numeric_limits::max()-csep->limitStart() + < csep->limitNum()) + { + csep->limitNum(std::numeric_limits::max()); + csep->limitStart(0); + } + else + { + csep->limitNum(csep->limitNum()+csep->limitStart()); + csep->limitStart(0); + } + } + + return 0; +} + /*@brief Translates SELECT_LEX into CSEP */ /*********************************************************** * DESCRIPTION: * This function takes SELECT_LEX and tries to produce * a corresponding CSEP out of it. It is made of parts that * process parts of the query, e.g. FROM, WHERE, SELECT, - * HAVING, GROUP BY, ORDER BY. FROM and WHERE are processed - * by processFrom(), processWhere(). CS calls getSelectPlan() + * HAVING, GROUP BY, ORDER BY. FROM, WHERE, LIMIT are processed + * by corresponding methods. CS calls getSelectPlan() * recursively to process subqueries. * ARGS * isUnion if true CS processes UNION unit now - * isPushdownHand legacy to be removed + * isSelectHandlerTop removes offset at the top of SH query. * RETURNS * error id as an int ***********************************************************/ int getSelectPlan(gp_walk_info& gwi, SELECT_LEX& select_lex, SCSEP& csep, bool isUnion, - bool isPushdownHand) + bool isSelectHandlerTop) { #ifdef DEBUG_WALK_COND cerr << "getSelectPlan()" << endl; @@ -6770,7 +6921,7 @@ int getSelectPlan(gp_walk_info& gwi, SELECT_LEX& select_lex, ReturnedColumn* rc; if (funcName == "in" || funcName == " IN " || funcName == "between") { - rc = buildFunctionColumn(ifp, gwi, hasNonSupportItem, false, true); + rc = buildFunctionColumn(ifp, gwi, hasNonSupportItem, false); } else { @@ -6802,15 +6953,11 @@ int getSelectPlan(gp_walk_info& gwi, SELECT_LEX& select_lex, parse_item(ifp, funcFieldVec, hasNonSupportItem, parseInfo, &gwi); uint32_t after_size = funcFieldVec.size(); - // group by func and func in subquery can not be post processed // pushdown handler projection functions // @bug3881. set_user_var can not be treated as constant function // @bug5716. Try to avoid post process function for union query. - if ((gwi.subQuery || select_lex.group_list.elements != 0 || - !csep->unionVec().empty() || isUnion || isPushdownHand ) && - !hasNonSupportItem && (after_size - before_size) == 0 && - !(parseInfo & AGG_BIT) && !(parseInfo & SUB_BIT) - ) + if (!hasNonSupportItem && (after_size - before_size) == 0 && + !(parseInfo & AGG_BIT) && !(parseInfo & SUB_BIT)) { String val, *str = ifp->val_str(&val); string valStr; @@ -7491,6 +7638,7 @@ int getSelectPlan(gp_walk_info& gwi, SELECT_LEX& select_lex, } } + // ORDER BY processing { SQL_I_List order_list = select_lex.order_list; ORDER* ordercol = reinterpret_cast(order_list.first); @@ -7504,9 +7652,9 @@ int getSelectPlan(gp_walk_info& gwi, SELECT_LEX& select_lex, // MCOL-2166 Looking for this sorting item in GROUP_BY items list. // Shouldn't look into this if query doesn't have GROUP BY or // aggregations - if(isPushdownHand - && select_lex.agg_func_used() && select_lex.group_list.first - && !sortItemIsInGrouping(*ordercol->item, select_lex.group_list.first)) + if(select_lex.agg_func_used() && select_lex.group_list.first + && !sortItemIsInGrouping(*ordercol->item, + select_lex.group_list.first)) { std::ostringstream ostream; std::ostringstream& osr = ostream; @@ -7523,10 +7671,6 @@ int getSelectPlan(gp_walk_info& gwi, SELECT_LEX& select_lex, // re-visit the first of ordercol list ordercol = reinterpret_cast(order_list.first); - // for subquery or pushdown query, order+limit by will be supported in CS - // union order by and limit are supported - if (gwi.hasWindowFunc || isPushdownHand || ( isUnion && ordercol ) - || gwi.subSelectType != CalpontSelectExecutionPlan::MAIN_SELECT ) { for (; ordercol; ordercol = ordercol->next) { @@ -7602,158 +7746,6 @@ int getSelectPlan(gp_walk_info& gwi, SELECT_LEX& select_lex, gwi.orderByCols.push_back(SRCP(rc)); } } - // DRRTUY The whole block is marked for removal in 1.4.1 -#if 0 - else if (!isUnion) - { - vector fieldVec; - - // the following order by is just for redo phase - if (!unionSel) - { - for (; ordercol; ordercol = ordercol->next) - { - Item* ord_item = *(ordercol->item); - - // @bug5993. Could be nested ref. - while (ord_item->type() == Item::REF_ITEM) - ord_item = (*((Item_ref*)ord_item)->ref); - - //ReturnedColumn* rc = 0; - // check if this order by column is on the select list - //Item_func* ifp = (Item_func*)(*(ordercol->item)); - //rc = buildFunctionColumn(ifp, gwi, gwi.fatalParseError); - - if (ord_item->type() == Item::FUNC_ITEM) - { - //FunctionColumn* fc = dynamic_cast(rc); - } - else if (ord_item->type() == Item::SUBSELECT_ITEM) - { - string emsg = IDBErrorInfo::instance()->errorMsg(ERR_NON_SUPPORT_ORDER_BY); - setError(gwi.thd, ER_CHECK_NOT_IMPLEMENTED, emsg, gwi); - return ER_CHECK_NOT_IMPLEMENTED; - } - else if (ord_item->type() == Item::SUM_FUNC_ITEM) - { - ReturnedColumn* ac = 0; - - Item_sum* ifp = (Item_sum*)(*(ordercol->item)); - // @bug3477. add aggregate column to the select list of the create phase. - ac = buildAggregateColumn(ifp, gwi); - - if (!ac) - { - setError(gwi.thd, ER_CHECK_NOT_IMPLEMENTED, - IDBErrorInfo::instance()->errorMsg(ERR_NON_SUPPORT_ORDER_BY), gwi); - return ER_CHECK_NOT_IMPLEMENTED; - } - - // check if this order by column is on the select list - for (uint32_t i = 0; i < gwi.returnedCols.size(); i++) - { - AggregateColumn* ret = dynamic_cast(gwi.returnedCols[i].get()); - - if (!ret) - continue; - - } - - if (ac || !gwi.groupByCols.empty()) - { - SRCP srcp(ac); - gwi.returnedCols.push_back(srcp); - - continue; - } - } - else if (ord_item->name.length && ord_item->type() == Item::FIELD_ITEM) - { - Item_field* field = reinterpret_cast(ord_item); - ReturnedColumn* rc = buildSimpleColumn(field, gwi); - - for (uint32_t i = 0; i < gwi.returnedCols.size(); i++) - { - SimpleColumn* sc = dynamic_cast(gwi.returnedCols[i].get()); - - if (sc && ((Item_field*)ord_item)->cached_table && - (strcasecmp(getViewName(((Item_field*)ord_item)->cached_table).c_str(), sc->viewName().c_str()) != 0)) - { - continue; - } - - if (sc && sc->sameColumn(rc)) - { - break; - } - } - } - - { - // @bug 2719. Error out order by not on the distinct select list. - if (select_lex.options & SELECT_DISTINCT) - { - gwi.parseErrorText = IDBErrorInfo::instance()->errorMsg(ERR_ORDERBY_NOT_IN_DISTINCT); - setError(gwi.thd, ER_CHECK_NOT_IMPLEMENTED, gwi.parseErrorText, gwi); - return ER_CHECK_NOT_IMPLEMENTED; - } - - bool hasNonSupportItem = false; - uint16_t parseInfo = 0; - parse_item(ord_item, fieldVec, hasNonSupportItem, parseInfo, &gwi); - - if (hasNonSupportItem) - { - string emsg = IDBErrorInfo::instance()->errorMsg(ERR_NON_SUPPORT_ORDER_BY); - setError(gwi.thd, ER_CHECK_NOT_IMPLEMENTED, emsg, gwi); - return ER_CHECK_NOT_IMPLEMENTED; - } - - } - } - } - - // populate string to be added to the select list for order by - for (uint32_t i = 0; i < fieldVec.size(); i++) - { - SimpleColumn* sc = buildSimpleColumn(fieldVec[i], gwi); - - if (!sc) - { - string emsg = IDBErrorInfo::instance()->errorMsg(ERR_NON_SUPPORT_ORDER_BY); - setError(gwi.thd, ER_CHECK_NOT_IMPLEMENTED, emsg, gwi); - return ER_CHECK_NOT_IMPLEMENTED; - } - - String str; - fieldVec[i]->print(&str, QT_ORDINARY); - sc->alias(string(str.c_ptr())); - SRCP srcp(sc); - uint32_t j = 0; - - for (; j < gwi.returnedCols.size(); j++) - { - if (sc->sameColumn(gwi.returnedCols[j].get())) - { - SimpleColumn* field = dynamic_cast(gwi.returnedCols[j].get()); - - if (field && field->alias() == sc->alias()) - break; - } - } - - if (j == gwi.returnedCols.size()) - { - gwi.returnedCols.push_back(srcp); - gwi.columnMap.insert(CalpontSelectExecutionPlan::ColumnMap::value_type(string(fieldVec[i]->field_name.str), srcp)); - TABLE_LIST* tmp = (fieldVec[i]->cached_table ? fieldVec[i]->cached_table : 0); - gwi.tableMap[make_aliastable(sc->schemaName(), sc->tableName(), sc->tableAlias(), sc->isColumnStore())] = - make_pair(1, tmp); - } - } - } -#endif - // make sure columnmap, returnedcols and count(*) arg_list are not empty TableMap::iterator tb_iter = gwi.tableMap.begin(); @@ -7879,129 +7871,16 @@ int getSelectPlan(gp_walk_info& gwi, SELECT_LEX& select_lex, { csep->hasOrderBy(true); // To activate LimitedOrderBy - if(isPushdownHand) - { - csep->orderByThreads(get_orderby_threads(gwi.thd)); - csep->specHandlerProcessed(true); - } + csep->orderByThreads(get_orderby_threads(gwi.thd)); + csep->specHandlerProcessed(true); } } - // LIMIT processing part - uint64_t limitNum = std::numeric_limits::max(); - - // non-MAIN union branch - if (unionSel || gwi.subSelectType != CalpontSelectExecutionPlan::MAIN_SELECT) + if ((rc = processLimitAndOffset(select_lex, gwi, csep, unionSel, isUnion, isSelectHandlerTop))) { - /* Consider the following query: - "select a from t1 where exists (select b from t2 where a=b);" - CS first builds a hash table for t2, then pushes down the hash to - PrimProc for a distributed hash join execution, with t1 being the - large-side table. However, the server applies an optimization in - Item_exists_subselect::fix_length_and_dec in sql/item_subselect.cc - (see server commit ae476868a5394041a00e75a29c7d45917e8dfae8) - where it sets explicit_limit to true, which causes csep->limitNum set to 1. - This causes the hash table for t2 to only contain a single record for the - hash join, giving less number of rows in the output result set than expected. - We therefore do not allow limit set to 1 here for such queries. - */ - if (gwi.subSelectType != CalpontSelectExecutionPlan::IN_SUBS - && gwi.subSelectType != CalpontSelectExecutionPlan::EXISTS_SUBS - && select_lex.master_unit()->global_parameters()->explicit_limit) - { - if (select_lex.master_unit()->global_parameters()->offset_limit) - { - Item_int* offset = (Item_int*)select_lex.master_unit()->global_parameters()->offset_limit; - csep->limitStart(offset->val_int()); - } - - if (select_lex.master_unit()->global_parameters()->select_limit) - { - Item_int* select = (Item_int*)select_lex.master_unit()->global_parameters()->select_limit; - csep->limitNum(select->val_int()); - // MCOL-894 Activate parallel ORDER BY - csep->orderByThreads(get_orderby_threads(gwi.thd)); - } - } + return rc; } - // union with explicit select at the top level - else if (isUnion && select_lex.explicit_limit) - { - if (select_lex.braces) - { - if (select_lex.offset_limit) - csep->limitStart(((Item_int*)select_lex.offset_limit)->val_int()); - - if (select_lex.select_limit) - csep->limitNum(((Item_int*)select_lex.select_limit)->val_int()); - } - } - // other types of queries that have explicit LIMIT - else if (select_lex.explicit_limit) - { - uint32_t limitOffset = 0; - - if (select_lex.join) - { - JOIN* join = select_lex.join; -#if MYSQL_VERSION_ID >= 50172 - - // @bug5729. After upgrade, join->unit sometimes is uninitialized pointer - // (not null though) and will cause seg fault. Prefer checking - // select_lex->offset_limit if not null. - if (join->select_lex && - join->select_lex->offset_limit && - join->select_lex->offset_limit->is_fixed() && - join->select_lex->select_limit && - join->select_lex->select_limit->is_fixed()) - { - limitOffset = join->select_lex->offset_limit->val_int(); - limitNum = join->select_lex->select_limit->val_int(); - } - else if (join->unit) - { - limitOffset = join->unit->lim.get_offset_limit(); - limitNum = join->unit->lim.get_select_limit() - limitOffset; - } - -#else - limitOffset = (join->unit)->offset_limit_cnt; - limitNum = (join->unit)->select_limit_cnt - (join->unit)->offset_limit_cnt; -#endif - } - else - { - if (select_lex.master_unit()->global_parameters()->offset_limit) - { - Item_int* offset = (Item_int*)select_lex.master_unit()->global_parameters()->offset_limit; - limitOffset = offset->val_int(); - } - - if (select_lex.master_unit()->global_parameters()->select_limit) - { - Item_int* select = (Item_int*)select_lex.master_unit()->global_parameters()->select_limit; - limitNum = select->val_int(); - } - } - - csep->limitStart(limitOffset); - csep->limitNum(limitNum); - } - // If an explicit limit is not specified, use the system variable value - else - { - csep->limitNum(gwi.thd->variables.select_limit); - } - - // We don't currently support limit with correlated subquery - if (gwi.subQuery && !gwi.correlatedTbNameVec.empty() && csep->hasOrderBy()) - { - gwi.fatalParseError = true; - gwi.parseErrorText = IDBErrorInfo::instance()->errorMsg(ERR_NON_SUPPORT_LIMIT_SUB); - setError(gwi.thd, ER_INTERNAL_ERROR, gwi.parseErrorText, gwi); - return ER_CHECK_NOT_IMPLEMENTED; - } - } // LIMIT processing finishes here + } // ORDER BY end if (select_lex.options & SELECT_DISTINCT) csep->distinct(true); @@ -8215,7 +8094,7 @@ int cp_get_group_plan(THD* thd, SCSEP& csep, cal_impl_if::cal_group_info& gi) int cs_get_derived_plan(derived_handler* handler, THD* thd, SCSEP& csep, gp_walk_info& gwi) { SELECT_LEX select_lex = *handler->select; - int status = getSelectPlan(gwi, select_lex, csep, false, true); + int status = getSelectPlan(gwi, select_lex, csep, false); if (status > 0) return ER_INTERNAL_ERROR; @@ -8413,7 +8292,7 @@ int getGroupPlan(gp_walk_info& gwi, SELECT_LEX& select_lex, SCSEP& csep, cal_gro SELECT_LEX* select_cursor = table_ptr->derived->first_select(); // Use Pushdown handler for subquery processing - FromSubQuery fromSub(gwi, select_cursor, true); + FromSubQuery fromSub(gwi, select_cursor); string alias(table_ptr->alias.str); fromSub.alias(lower(alias)); diff --git a/dbcon/mysql/ha_mcs_impl.cpp b/dbcon/mysql/ha_mcs_impl.cpp index d04d5f186..deb4249ed 100644 --- a/dbcon/mysql/ha_mcs_impl.cpp +++ b/dbcon/mysql/ha_mcs_impl.cpp @@ -1780,7 +1780,7 @@ uint32_t doUpdateDelete(THD* thd, gp_walk_info& gwi) gwi.clauseType = WHERE; - if (getSelectPlan(gwi, select_lex, updateCP, false, true) != 0) //@Bug 3030 Modify the error message for unsupported functions + if (getSelectPlan(gwi, select_lex, updateCP, false) != 0) //@Bug 3030 Modify the error message for unsupported functions { if (gwi.cs_vtable_is_update_with_derive) { diff --git a/dbcon/mysql/ha_mcs_impl_if.h b/dbcon/mysql/ha_mcs_impl_if.h index 3c1d2ada6..9975605b7 100644 --- a/dbcon/mysql/ha_mcs_impl_if.h +++ b/dbcon/mysql/ha_mcs_impl_if.h @@ -342,7 +342,7 @@ int cp_get_table_plan(THD* thd, execplan::SCSEP& csep, cal_impl_if::cal_table_in int cp_get_group_plan(THD* thd, execplan::SCSEP& csep, cal_impl_if::cal_group_info& gi); int cs_get_derived_plan(derived_handler* handler, THD* thd, execplan::SCSEP& csep, gp_walk_info& gwi); int cs_get_select_plan(select_handler* handler, THD* thd, execplan::SCSEP& csep, gp_walk_info& gwi); -int getSelectPlan(gp_walk_info& gwi, SELECT_LEX& select_lex, execplan::SCSEP& csep, bool isUnion = false, bool isPushdownHand = false); +int getSelectPlan(gp_walk_info& gwi, SELECT_LEX& select_lex, execplan::SCSEP& csep, bool isUnion = false, bool isSelectHandlerTop = false); int getGroupPlan(gp_walk_info& gwi, SELECT_LEX& select_lex, execplan::SCSEP& csep, cal_group_info& gi, bool isUnion = false); void setError(THD* thd, uint32_t errcode, const std::string errmsg, gp_walk_info* gwi); void setError(THD* thd, uint32_t errcode, const std::string errmsg); @@ -352,9 +352,9 @@ const std::string bestTableName(const Item_field* ifp); bool isMCSTable(TABLE* table_ptr); // execution plan util functions prototypes -execplan::ReturnedColumn* buildReturnedColumn(Item* item, gp_walk_info& gwi, bool& nonSupport, bool pushdownHand = false, bool isRefItem = false); -execplan::ReturnedColumn* buildFunctionColumn(Item_func* item, gp_walk_info& gwi, bool& nonSupport, bool pushdownHand = false, bool selectBetweenIn = false); -execplan::ArithmeticColumn* buildArithmeticColumn(Item_func* item, gp_walk_info& gwi, bool& nonSupport, bool pushdownHand = false); +execplan::ReturnedColumn* buildReturnedColumn(Item* item, gp_walk_info& gwi, bool& nonSupport, bool isRefItem = false); +execplan::ReturnedColumn* buildFunctionColumn(Item_func* item, gp_walk_info& gwi, bool& nonSupport, bool selectBetweenIn = false); +execplan::ArithmeticColumn* buildArithmeticColumn(Item_func* item, gp_walk_info& gwi, bool& nonSupport); execplan::ConstantColumn* buildDecimalColumn(Item* item, gp_walk_info& gwi); execplan::SimpleColumn* buildSimpleColumn(Item_field* item, gp_walk_info& gwi); execplan::FunctionColumn* buildCaseFunction(Item_func* item, gp_walk_info& gwi, bool& nonSupport); diff --git a/dbcon/mysql/ha_mcs_pushdown.cpp b/dbcon/mysql/ha_mcs_pushdown.cpp index 5c5db65f8..80884e6bd 100644 --- a/dbcon/mysql/ha_mcs_pushdown.cpp +++ b/dbcon/mysql/ha_mcs_pushdown.cpp @@ -757,11 +757,9 @@ create_columnstore_select_handler(THD* thd, SELECT_LEX* select_lex) return handler; } - // Select_handler use the short-cut that effectively disables - // INSERT..SELECT, LDI, SELECT..INTO OUTFILE + // Select_handler couldn't properly process UPSERT..SELECT if ((thd->lex)->sql_command == SQLCOM_INSERT_SELECT - || (thd->lex)->sql_command == SQLCOM_CREATE_TABLE - || (thd->lex)->exchange) + && thd->lex->duplicates == DUP_UPDATE) { return handler; } diff --git a/dbcon/mysql/ha_scalar_sub.cpp b/dbcon/mysql/ha_scalar_sub.cpp index f41482ba2..7d6d2dd11 100644 --- a/dbcon/mysql/ha_scalar_sub.cpp +++ b/dbcon/mysql/ha_scalar_sub.cpp @@ -253,7 +253,7 @@ execplan::ParseTree* ScalarSub::buildParseTree(PredicateOperator* op) gwi.tbList.insert(gwi.tbList.begin(), fGwip.tbList.begin(), fGwip.tbList.end()); gwi.derivedTbList.insert(gwi.derivedTbList.begin(), fGwip.derivedTbList.begin(), fGwip.derivedTbList.end()); - if (getSelectPlan(gwi, *(fSub->get_select_lex()), csep, false, true) != 0) + if (getSelectPlan(gwi, *(fSub->get_select_lex()), csep, false) != 0) { //@todo more in error handling if (!gwi.fatalParseError) diff --git a/dbcon/mysql/ha_select_sub.cpp b/dbcon/mysql/ha_select_sub.cpp index 708d2cca8..6d5dfd611 100644 --- a/dbcon/mysql/ha_select_sub.cpp +++ b/dbcon/mysql/ha_select_sub.cpp @@ -77,7 +77,7 @@ SCSEP SelectSubQuery::transform() gwi.tbList.insert(gwi.tbList.begin(), fGwip.tbList.begin(), fGwip.tbList.end()); gwi.derivedTbList.insert(gwi.derivedTbList.begin(), fGwip.derivedTbList.begin(), fGwip.derivedTbList.end()); - if (getSelectPlan(gwi, *(fSelSub->get_select_lex()), csep, false, true) != 0) + if (getSelectPlan(gwi, *(fSelSub->get_select_lex()), csep, false) != 0) { if (!gwi.fatalParseError) { diff --git a/dbcon/mysql/ha_subquery.h b/dbcon/mysql/ha_subquery.h index d14c95131..506cdf0eb 100644 --- a/dbcon/mysql/ha_subquery.h +++ b/dbcon/mysql/ha_subquery.h @@ -181,7 +181,7 @@ class FromSubQuery : public SubQuery { public: FromSubQuery(gp_walk_info&); - FromSubQuery(gp_walk_info&, SELECT_LEX* fromSub, bool isPushdownHand=false); + FromSubQuery(gp_walk_info&, SELECT_LEX* fromSub); ~FromSubQuery(); const SELECT_LEX* fromSub() const { diff --git a/utils/loggingcpp/ErrorMessage.txt b/utils/loggingcpp/ErrorMessage.txt index 3b066fc0f..08233bbf6 100755 --- a/utils/loggingcpp/ErrorMessage.txt +++ b/utils/loggingcpp/ErrorMessage.txt @@ -17,7 +17,7 @@ # Non support errors 1000 ~ 2000. # The query will go through the optimizer again with some optimization turned off 1000 ERR_MISS_JOIN %1% not joined. -1001 ERR_NON_SUPPORTED_FUNCTION Function '%1%' can only be used in the outermost select or order by clause and cannot be used in conjunction with an aggregate function. +1001 ERR_NON_SUPPORTED_FUNCTION Function '%1%' isn't supported. 1002 ERR_INCOMPATIBLE_JOIN %1% incompatible column type specified for join condition. 1003 ERR_CIRCULAR_JOIN Circular joins are not supported. 1004 ERR_MIX_JOIN Mixed %1% JOIN is not supported. From 6ea30f64fda2149dbd08fa28ee31d41a1b2bb394 Mon Sep 17 00:00:00 2001 From: Roman Nozdrin Date: Wed, 1 Apr 2020 17:22:57 +0000 Subject: [PATCH 2/4] MCOL-3903 Fix the fail in expBetween(BETWEEN in projection wasn't processed as expected.) --- dbcon/mysql/ha_mcs_execplan.cpp | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/dbcon/mysql/ha_mcs_execplan.cpp b/dbcon/mysql/ha_mcs_execplan.cpp index dc833c7c4..eca370187 100755 --- a/dbcon/mysql/ha_mcs_execplan.cpp +++ b/dbcon/mysql/ha_mcs_execplan.cpp @@ -5653,7 +5653,7 @@ void gp_walk(const Item* item, void* arg) if (col->type() != Item::COND_ITEM) { - rc = buildReturnedColumn(col, *gwip, gwip->fatalParseError, false); + rc = buildReturnedColumn(col, *gwip, gwip->fatalParseError, true); if ( col->type() == Item::FIELD_ITEM ) gwip->fatalParseError = false; @@ -6921,7 +6921,7 @@ int getSelectPlan(gp_walk_info& gwi, SELECT_LEX& select_lex, ReturnedColumn* rc; if (funcName == "in" || funcName == " IN " || funcName == "between") { - rc = buildFunctionColumn(ifp, gwi, hasNonSupportItem, false); + rc = buildFunctionColumn(ifp, gwi, hasNonSupportItem, true); } else { From e6d1aa27f4a81da1a6725a4209bc09ad334d664b Mon Sep 17 00:00:00 2001 From: Roman Nozdrin Date: Wed, 8 Apr 2020 13:41:30 +0000 Subject: [PATCH 3/4] MCOL-3903 10.5 allows engines to process LIMIT and OFFSET on their own. --- dbcon/mysql/ha_mcs_execplan.cpp | 27 ++------------------------- 1 file changed, 2 insertions(+), 25 deletions(-) diff --git a/dbcon/mysql/ha_mcs_execplan.cpp b/dbcon/mysql/ha_mcs_execplan.cpp index eca370187..d75137728 100755 --- a/dbcon/mysql/ha_mcs_execplan.cpp +++ b/dbcon/mysql/ha_mcs_execplan.cpp @@ -6617,7 +6617,6 @@ int processLimitAndOffset( if (select_lex.join) { JOIN* join = select_lex.join; -#if MYSQL_VERSION_ID >= 50172 // @bug5729. After upgrade, join->unit sometimes is uninitialized pointer // (not null though) and will cause seg fault. Prefer checking @@ -6633,14 +6632,9 @@ int processLimitAndOffset( } else if (join->unit) { - limitOffset = join->unit->offset_limit_cnt; - limitNum = join->unit->select_limit_cnt - limitOffset; + limitOffset = join->unit->lim.get_offset_limit(); + limitNum = join->unit->lim.get_select_limit() - limitOffset; } - -#else - limitOffset = (join->unit)->offset_limit_cnt; - limitNum = (join->unit)->select_limit_cnt - (join->unit)->offset_limit_cnt; -#endif } else { @@ -6675,23 +6669,6 @@ int processLimitAndOffset( return ER_CHECK_NOT_IMPLEMENTED; } - // MDB applies OFFSET on its own for SH processing. - // See related MDEV-16327 for details. - if (isSelectHandlerTop && csep->limitStart()) - { - if (std::numeric_limits::max()-csep->limitStart() - < csep->limitNum()) - { - csep->limitNum(std::numeric_limits::max()); - csep->limitStart(0); - } - else - { - csep->limitNum(csep->limitNum()+csep->limitStart()); - csep->limitStart(0); - } - } - return 0; } From 6f3ae1b429c063b8fb668f3b89aac6b069d314a7 Mon Sep 17 00:00:00 2001 From: Roman Nozdrin Date: Mon, 20 Apr 2020 15:25:30 +0300 Subject: [PATCH 4/4] MCOL-3941 Ninja now builds MCS. Generated files in utils/loggingcpp/ are now moved into bin dir. --- CMakeLists.txt | 2 +- dbcon/ddlpackage/CMakeLists.txt | 20 ++++++++--------- dbcon/ddlpackage/ddl-gram.sh | 26 ---------------------- dbcon/ddlpackage/ddl-scan.sh | 16 -------------- dbcon/dmlpackage/CMakeLists.txt | 20 ++++++++--------- dbcon/dmlpackage/dml-gram.sh | 23 -------------------- dbcon/dmlpackage/dml-scan.sh | 14 ------------ utils/libmarias3/CMakeLists.txt | 1 + utils/loggingcpp/CMakeLists.txt | 29 ++++++++++++++----------- utils/loggingcpp/genErrId.pl | 7 +----- utils/loggingcpp/genMsgAndErrId.sh | 8 ------- utils/loggingcpp/genMsgId.pl | 7 +----- utils/udfsdk/CMakeLists.txt | 2 -- versioning/BRM/autoincrementmanager.cpp | 2 ++ versioning/BRM/autoincrementmanager.h | 2 +- 15 files changed, 41 insertions(+), 138 deletions(-) delete mode 100755 dbcon/ddlpackage/ddl-gram.sh delete mode 100755 dbcon/ddlpackage/ddl-scan.sh delete mode 100755 dbcon/dmlpackage/dml-gram.sh delete mode 100755 dbcon/dmlpackage/dml-scan.sh delete mode 100755 utils/loggingcpp/genMsgAndErrId.sh diff --git a/CMakeLists.txt b/CMakeLists.txt index aefcc420a..f478eed72 100644 --- a/CMakeLists.txt +++ b/CMakeLists.txt @@ -253,7 +253,7 @@ ENDIF() SET (ENGINE_UTILS_MESSAGEQCPP_INCLUDE "${CMAKE_CURRENT_SOURCE_DIR}/utils/messageqcpp") SET (ENGINE_WE_SHARED_INCLUDE "${CMAKE_CURRENT_SOURCE_DIR}/writeengine/shared") SET (ENGINE_UTILS_IDBDATAFILE_INCLUDE "${CMAKE_CURRENT_SOURCE_DIR}/utils/idbdatafile") -SET (ENGINE_UTILS_LOGGINGCPP_INCLUDE "${CMAKE_CURRENT_SOURCE_DIR}/utils/loggingcpp") +SET (ENGINE_UTILS_LOGGINGCPP_INCLUDE "${CMAKE_CURRENT_SOURCE_DIR}/utils/loggingcpp" "${CMAKE_CURRENT_BINARY_DIR}/utils/loggingcpp") SET (ENGINE_UTILS_CONFIGCPP_INCLUDE "${CMAKE_CURRENT_SOURCE_DIR}/utils/configcpp") SET (ENGINE_UTILS_COMPRESS_INCLUDE "${CMAKE_CURRENT_SOURCE_DIR}/utils/compress") SET (ENGINE_VERSIONING_BRM_INCLUDE "${CMAKE_CURRENT_SOURCE_DIR}/versioning/BRM") diff --git a/dbcon/ddlpackage/CMakeLists.txt b/dbcon/ddlpackage/CMakeLists.txt index 4e95ff073..7a5d2198b 100644 --- a/dbcon/ddlpackage/CMakeLists.txt +++ b/dbcon/ddlpackage/CMakeLists.txt @@ -1,13 +1,11 @@ INCLUDE_DIRECTORIES( ${ENGINE_COMMON_INCLUDES} ) -#TODO: put generated files in the binary directory -ADD_CUSTOM_COMMAND( - OUTPUT ${CMAKE_CURRENT_SOURCE_DIR}/ddl-gram.cpp ${CMAKE_CURRENT_SOURCE_DIR}/ddl-scan.cpp - COMMAND /bin/sh ./ddl-gram.sh ${BISON_EXECUTABLE} - COMMAND /bin/sh ./ddl-scan.sh ${LEX_EXECUTABLE} - WORKING_DIRECTORY ${CMAKE_CURRENT_SOURCE_DIR} - DEPENDS ddl.y ddl.l -) +FIND_PACKAGE(BISON REQUIRED) +BISON_TARGET(ddl_gram ddl.y ${CMAKE_CURRENT_BINARY_DIR}/ddl-gram.cpp DEFINES_FILE ${CMAKE_CURRENT_BINARY_DIR}/ddl-gram.h COMPILE_FLAGS "-l -d -p ddl") + +FIND_PACKAGE(FLEX REQUIRED) +FLEX_TARGET(ddl_scan ddl.l ${CMAKE_CURRENT_BINARY_DIR}/ddl-scan.cpp COMPILE_FLAGS "-i -L -Pddl") +ADD_FLEX_BISON_DEPENDENCY(ddl_scan ddl_gram) set_source_files_properties(ddl-scan.cpp PROPERTIES COMPILE_FLAGS -Wno-sign-compare) @@ -31,9 +29,9 @@ ADD_LIBRARY(ddlpackage SHARED markpartition.cpp restorepartition.cpp droppartition.cpp - ${CMAKE_CURRENT_SOURCE_DIR}/ddl-gram.cpp - ${CMAKE_CURRENT_SOURCE_DIR}/ddl-scan.cpp - ) + ${BISON_ddl_gram_OUTPUTS} + ${FLEX_ddl_scan_OUTPUTS} +) INSTALL(TARGETS ddlpackage DESTINATION ${ENGINE_LIBDIR} COMPONENT columnstore-libs) diff --git a/dbcon/ddlpackage/ddl-gram.sh b/dbcon/ddlpackage/ddl-gram.sh deleted file mode 100755 index 5c1b121c0..000000000 --- a/dbcon/ddlpackage/ddl-gram.sh +++ /dev/null @@ -1,26 +0,0 @@ -#!/bin/sh - -#ddl-gram.cpp: ddl.y -$1 -y -l -v -d -p ddl -o ddl-gram-temp.cpp ddl.y -set +e; \ -if [ -f ddl-gram.cpp ]; \ - then diff -abBq ddl-gram-temp.cpp ddl-gram.cpp >/dev/null 2>&1; \ - if [ $? -ne 0 ]; \ - then mv -f ddl-gram-temp.cpp ddl-gram.cpp; \ - else touch ddl-gram.cpp; \ - fi; \ - else mv -f ddl-gram-temp.cpp ddl-gram.cpp; \ -fi -set +e; \ -if [ -f ddl-gram.h ]; \ - then diff -abBq ddl-gram-temp.hpp ddl-gram.h >/dev/null 2>&1; \ - if [ $? -ne 0 ]; \ - then mv -f ddl-gram-temp.hpp ddl-gram.h; \ - else touch ddl-gram.h; \ - fi; \ - else mv -f ddl-gram-temp.hpp ddl-gram.h; \ -fi -rm -f ddl-gram-temp.cpp ddl-gram-temp.hpp ddl-gram-temp.output - - - diff --git a/dbcon/ddlpackage/ddl-scan.sh b/dbcon/ddlpackage/ddl-scan.sh deleted file mode 100755 index 4d62056ec..000000000 --- a/dbcon/ddlpackage/ddl-scan.sh +++ /dev/null @@ -1,16 +0,0 @@ -#!/bin/sh - -#ddl-scan.cpp: ddl.l -$1 -i -L -P ddl -o ddl-scan-temp.cpp ddl.l -set +e; -if [ -f ddl-scan.cpp ]; - then diff -abBq ddl-scan-temp.cpp ddl-scan.cpp >/dev/null 2>&1; - if [ "$?" -ne 0 ]; - then mv -f ddl-scan-temp.cpp ddl-scan.cpp; - else touch ddl-scan.cpp; - fi; - else mv -f ddl-scan-temp.cpp ddl-scan.cpp; -fi -rm -f ddl-scan-temp.cpp - - diff --git a/dbcon/dmlpackage/CMakeLists.txt b/dbcon/dmlpackage/CMakeLists.txt index b54bbc822..66e43621f 100644 --- a/dbcon/dmlpackage/CMakeLists.txt +++ b/dbcon/dmlpackage/CMakeLists.txt @@ -1,14 +1,12 @@ INCLUDE_DIRECTORIES( ${ENGINE_COMMON_INCLUDES} ) -#TODO: put generated files in binary folder -ADD_CUSTOM_COMMAND( - OUTPUT ${CMAKE_CURRENT_SOURCE_DIR}/dml-gram.cpp ${CMAKE_CURRENT_SOURCE_DIR}/dml-scan.cpp - COMMAND /bin/sh ./dml-gram.sh ${BISON_EXECUTABLE} - COMMAND /bin/sh ./dml-scan.sh ${LEX_EXECUTABLE} - WORKING_DIRECTORY ${CMAKE_CURRENT_SOURCE_DIR} - DEPENDS dml.y dml.l -) +FIND_PACKAGE(BISON REQUIRED) +BISON_TARGET(dml_gram dml.y ${CMAKE_CURRENT_BINARY_DIR}/dml-gram.cpp DEFINES_FILE ${CMAKE_CURRENT_BINARY_DIR}/dml-gram.h COMPILE_FLAGS "-l -d -p dml") + +FIND_PACKAGE(FLEX REQUIRED) +FLEX_TARGET(dml_scan dml.l ${CMAKE_CURRENT_BINARY_DIR}/dml-scan.cpp COMPILE_FLAGS "-i -L -Pdml") +ADD_FLEX_BISON_DEPENDENCY(dml_scan dml_gram) set_source_files_properties(dml-scan.cpp PROPERTIES COMPILE_FLAGS -Wno-sign-compare) @@ -32,9 +30,9 @@ ADD_LIBRARY(dmlpackage SHARED commanddmlpackage.cpp dmlpkg.cpp dmlparser.cpp - ${CMAKE_CURRENT_SOURCE_DIR}/dml-gram.cpp - ${CMAKE_CURRENT_SOURCE_DIR}/dml-scan.cpp - ) + ${BISON_dml_gram_OUTPUTS} + ${FLEX_dml_scan_OUTPUTS} +) INSTALL(TARGETS dmlpackage DESTINATION ${ENGINE_LIBDIR} COMPONENT columnstore-libs) diff --git a/dbcon/dmlpackage/dml-gram.sh b/dbcon/dmlpackage/dml-gram.sh deleted file mode 100755 index 665a80da3..000000000 --- a/dbcon/dmlpackage/dml-gram.sh +++ /dev/null @@ -1,23 +0,0 @@ -#!/bin/sh - -#dml-gram.cpp: dml.y -$1 -l -v -d -p dml -o dml-gram-temp.cpp dml.y - set +e; \ - if [ -f dml-gram.cpp ]; \ - then diff -abBq dml-gram-temp.cpp dml-gram.cpp >/dev/null 2>&1; \ - if [ $? -ne 0 ]; \ - then mv -f dml-gram-temp.cpp dml-gram.cpp; \ - else touch dml-gram.cpp; \ - fi; \ - else mv -f dml-gram-temp.cpp dml-gram.cpp; \ - fi - set +e; \ - if [ -f dml-gram.h ]; \ - then diff -abBq dml-gram-temp.hpp dml-gram.h >/dev/null 2>&1; \ - if [ $? -ne 0 ]; \ - then mv -f dml-gram-temp.hpp dml-gram.h; \ - else touch dml-gram.h; \ - fi; \ - else mv -f dml-gram-temp.hpp dml-gram.h; \ - fi - rm -f dml-gram-temp.cpp dml-gram-temp.hpp dml-gram-temp.output \ No newline at end of file diff --git a/dbcon/dmlpackage/dml-scan.sh b/dbcon/dmlpackage/dml-scan.sh deleted file mode 100755 index 1daa312cf..000000000 --- a/dbcon/dmlpackage/dml-scan.sh +++ /dev/null @@ -1,14 +0,0 @@ -#!/bin/sh - -#dml-scan.cpp: dml.l - $1 -i -L -Pdml -odml-scan-temp.cpp dml.l - set +e; \ - if [ -f dml-scan.cpp ]; \ - then diff -abBq dml-scan-temp.cpp dml-scan.cpp >/dev/null 2>&1; \ - if [ $? -ne 0 ]; \ - then mv -f dml-scan-temp.cpp dml-scan.cpp; \ - else touch dml-scan.cpp; \ - fi; \ - else mv -f dml-scan-temp.cpp dml-scan.cpp; \ - fi - rm -f dml-scan-temp.cpp diff --git a/utils/libmarias3/CMakeLists.txt b/utils/libmarias3/CMakeLists.txt index 8695dcefd..a74b547a6 100644 --- a/utils/libmarias3/CMakeLists.txt +++ b/utils/libmarias3/CMakeLists.txt @@ -23,6 +23,7 @@ ExternalProject_Add(ms3 BUILD_COMMAND make BUILD_IN_SOURCE 0 EXCLUDE_FROM_ALL TRUE + BUILD_BYPRODUCTS ${CMAKE_BINARY_DIR}/lib/libmarias3.so.3.1.2 ) set(S3API_DEPS marias3 curl xml2 CACHE INTERNAL "S3API_DEPS") diff --git a/utils/loggingcpp/CMakeLists.txt b/utils/loggingcpp/CMakeLists.txt index 491346529..0ce8f2925 100644 --- a/utils/loggingcpp/CMakeLists.txt +++ b/utils/loggingcpp/CMakeLists.txt @@ -1,17 +1,22 @@ include_directories( ${ENGINE_COMMON_INCLUDES} ) - ########### next target ############### -#TODO: put generated files in binary dir ADD_CUSTOM_COMMAND( - OUTPUT ${CMAKE_CURRENT_SOURCE_DIR}/messageids.h ${CMAKE_CURRENT_SOURCE_DIR}/errorids.h - COMMAND /bin/sh genMsgAndErrId.sh - WORKING_DIRECTORY ${CMAKE_CURRENT_SOURCE_DIR} - DEPENDS genMsgId.pl genErrId.pl + OUTPUT ${CMAKE_CURRENT_BINARY_DIR}/messageids.h + COMMAND perl ${CMAKE_CURRENT_SOURCE_DIR}/genMsgId.pl < ${CMAKE_CURRENT_SOURCE_DIR}/MessageFile.txt > messageids-temp.h + COMMAND ${CMAKE_COMMAND} -E copy_if_different messageids-temp.h messageids.h + DEPENDS genMsgId.pl ) - - +ADD_CUSTOM_COMMAND( + OUTPUT ${CMAKE_CURRENT_BINARY_DIR}/errorids.h + COMMAND perl ${CMAKE_CURRENT_SOURCE_DIR}/genErrId.pl < ${CMAKE_CURRENT_SOURCE_DIR}/ErrorMessage.txt > errorids-temp.h + COMMAND ${CMAKE_COMMAND} -E copy_if_different errorids-temp.h errorids.h + DEPENDS genErrId.pl + ) + +set_source_files_properties(${CMAKE_CURRENT_BINARY_DIR}/errorids.h PROPERTIES GENERATED TRUE) + add_library(loggingcpp SHARED message.cpp messagelog.cpp @@ -20,11 +25,9 @@ add_library(loggingcpp SHARED sqllogger.cpp stopwatch.cpp idberrorinfo.cpp - ${CMAKE_CURRENT_SOURCE_DIR}/messageids.h - ${CMAKE_CURRENT_SOURCE_DIR}/errorids.h - ) - - + ${CMAKE_CURRENT_BINARY_DIR}/messageids.h + ${CMAKE_CURRENT_BINARY_DIR}/errorids.h +) install(TARGETS loggingcpp DESTINATION ${ENGINE_LIBDIR} COMPONENT columnstore-libs) diff --git a/utils/loggingcpp/genErrId.pl b/utils/loggingcpp/genErrId.pl index a731236f4..91c953d07 100755 --- a/utils/loggingcpp/genErrId.pl +++ b/utils/loggingcpp/genErrId.pl @@ -3,8 +3,6 @@ # $Id: genErrId.pl 3048 2012-04-04 15:33:45Z rdempsey $ # -open FH, "< ./ErrorMessage.txt" or die; - $frontmatter = <<'EOD'; /* Copyright (C) 2014 InfiniDB, Inc. @@ -46,7 +44,7 @@ EOD print $frontmatter; -while () +while (<>) { chomp; next if (/^$/); @@ -56,6 +54,3 @@ while () } print $backmatter; - -close FH; - diff --git a/utils/loggingcpp/genMsgAndErrId.sh b/utils/loggingcpp/genMsgAndErrId.sh deleted file mode 100755 index 9182e78d6..000000000 --- a/utils/loggingcpp/genMsgAndErrId.sh +++ /dev/null @@ -1,8 +0,0 @@ -#!/bin/sh - -perl ./genMsgId.pl > messageids-temp.h -diff -abBq messageids-temp.h messageids.h >/dev/null 2>&1; if [ $? -ne 0 ]; then mv -f messageids-temp.h messageids.h; else touch -a messageids.h; fi; -rm -f messageids-temp.h -perl ./genErrId.pl > errorids-temp.h -diff -abBq errorids-temp.h errorids.h >/dev/null 2>&1; if [ $? -ne 0 ]; then mv -f errorids-temp.h errorids.h; else touch -a errorids.h; fi; -rm -f errorids-temp.h diff --git a/utils/loggingcpp/genMsgId.pl b/utils/loggingcpp/genMsgId.pl index 5426d44ac..3067122da 100755 --- a/utils/loggingcpp/genMsgId.pl +++ b/utils/loggingcpp/genMsgId.pl @@ -3,8 +3,6 @@ # $Id: genMsgId.pl 3048 2012-04-04 15:33:45Z rdempsey $ # -open FH, "< ./MessageFile.txt" or die; - $frontmatter = <<'EOD'; /* Copyright (C) 2014 InfiniDB, Inc. @@ -46,7 +44,7 @@ EOD print $frontmatter; -while () +while (<>) { chomp; next if (/^$/); @@ -56,6 +54,3 @@ while () } print $backmatter; - -close FH; - diff --git a/utils/udfsdk/CMakeLists.txt b/utils/udfsdk/CMakeLists.txt index 016e7d892..ade4c8415 100755 --- a/utils/udfsdk/CMakeLists.txt +++ b/utils/udfsdk/CMakeLists.txt @@ -12,8 +12,6 @@ add_library(udfsdk SHARED ${udfsdk_LIB_SRCS}) install(TARGETS udfsdk DESTINATION ${ENGINE_LIBDIR} COMPONENT columnstore-libs) - - set(udf_mysql_LIB_SRCS udfmysql.cpp) add_library(udf_mysql SHARED ${udf_mysql_LIB_SRCS}) diff --git a/versioning/BRM/autoincrementmanager.cpp b/versioning/BRM/autoincrementmanager.cpp index b62eaa2ab..9decda641 100644 --- a/versioning/BRM/autoincrementmanager.cpp +++ b/versioning/BRM/autoincrementmanager.cpp @@ -102,6 +102,8 @@ void AutoincrementManager::resetSequence(uint32_t oid, uint64_t value) it->second.value = value; } +const uint32_t AutoincrementManager::lockTime = 30; + void AutoincrementManager::getLock(uint32_t oid) { boost::mutex::scoped_lock lk(lock); diff --git a/versioning/BRM/autoincrementmanager.h b/versioning/BRM/autoincrementmanager.h index e1b04585f..133d99d51 100644 --- a/versioning/BRM/autoincrementmanager.h +++ b/versioning/BRM/autoincrementmanager.h @@ -54,7 +54,7 @@ public: EXPORT void deleteSequence(uint32_t OID); private: - static const uint32_t lockTime = 30; // 30 seconds + static const uint32_t lockTime; // 30 seconds struct sequence { sequence() : value(0), overflow(0) { }