diff --git a/dbcon/execplan/aggregatecolumn.cpp b/dbcon/execplan/aggregatecolumn.cpp index 5bce12d79..c996dad17 100644 --- a/dbcon/execplan/aggregatecolumn.cpp +++ b/dbcon/execplan/aggregatecolumn.cpp @@ -107,8 +107,8 @@ AggregateColumn::AggregateColumn(const string& functionName, const string& conte fData(functionName + "(" + content + ")") { // TODO: need to handle distinct - SRCP srcp(new ArithmeticColumn(content)); - fAggParms.push_back(srcp); + SRCP srcp(new ArithmeticColumn(content)); + fAggParms.push_back(srcp); } AggregateColumn::AggregateColumn( const AggregateColumn& rhs, const uint32_t sessionID ): @@ -144,6 +144,7 @@ const string AggregateColumn::toString() const { output << *(fAggParms[i]) << " "; } + output << endl; if (fConstCol) @@ -167,6 +168,7 @@ void AggregateColumn::serialize(messageqcpp::ByteStream& b) const b << static_cast(fAggOp); b << static_cast(fAggParms.size()); + for (uint32_t i = 0; i < fAggParms.size(); ++i) { fAggParms[i]->serialize(b); @@ -208,6 +210,7 @@ void AggregateColumn::unserialize(messageqcpp::ByteStream& b) b >> fAggOp; b >> size; + for (i = 0; i < size; i++) { rc = dynamic_cast(ObjectReader::createTreeNode(b)); @@ -264,9 +267,10 @@ bool AggregateColumn::operator==(const AggregateColumn& t) const { return false; } - for (it = fAggParms.begin(), it2 = t.fAggParms.begin(); - it != fAggParms.end(); - ++it, ++it2) + + for (it = fAggParms.begin(), it2 = t.fAggParms.begin(); + it != fAggParms.end(); + ++it, ++it2) { if (**it != **it2) return false; diff --git a/dbcon/execplan/aggregatecolumn.h b/dbcon/execplan/aggregatecolumn.h index b0884f179..07bbab0b6 100644 --- a/dbcon/execplan/aggregatecolumn.h +++ b/dbcon/execplan/aggregatecolumn.h @@ -163,7 +163,7 @@ public: fAggParms = parms; } - + /** return a copy of this pointer * * deep copy of this pointer and return the copy @@ -316,8 +316,8 @@ protected: uint8_t fAggOp; /** - * ReturnedColumn objects that are the arguments to this - * function + * ReturnedColumn objects that are the arguments to this + * function */ AggParms fAggParms; diff --git a/dbcon/joblist/joblistfactory.cpp b/dbcon/joblist/joblistfactory.cpp index 033bf2643..6fa0adbab 100644 --- a/dbcon/joblist/joblistfactory.cpp +++ b/dbcon/joblist/joblistfactory.cpp @@ -896,36 +896,44 @@ const JobStepVector doAggProject(const CalpontSelectExecutionPlan* csep, JobInfo continue; } + #if 0 // MCOL-1201 Add support for multi-parameter UDAnF - UDAFColumn* udafc = dynamic_cast(retCols[i].get()); - if (udafc != NULL) - { - srcp = udafc->aggParms()[0]; - const RowColumn* rcp = dynamic_cast(srcp.get()); + UDAFColumn* udafc = dynamic_cast(retCols[i].get()); - const vector& cols = rcp->columnVec(); - for (vector::const_iterator j = cols.begin(); j != cols.end(); j++) - { + if (udafc != NULL) + { + srcp = udafc->aggParms()[0]; + const RowColumn* rcp = dynamic_cast(srcp.get()); + + const vector& cols = rcp->columnVec(); + + for (vector::const_iterator j = cols.begin(); j != cols.end(); j++) + { srcp = *j; - if (dynamic_cast(srcp.get()) == NULL) - retCols.push_back(srcp); + + if (dynamic_cast(srcp.get()) == NULL) + retCols.push_back(srcp); // Do we need this? - const ArithmeticColumn* ac = dynamic_cast(srcp.get()); - const FunctionColumn* fc = dynamic_cast(srcp.get()); - if (ac != NULL || fc != NULL) - { - // bug 3728, make a dummy expression step for each expression. - scoped_ptr es(new ExpressionStep(jobInfo)); - es->expression(srcp, jobInfo); - } - } - continue; - } + const ArithmeticColumn* ac = dynamic_cast(srcp.get()); + const FunctionColumn* fc = dynamic_cast(srcp.get()); + + if (ac != NULL || fc != NULL) + { + // bug 3728, make a dummy expression step for each expression. + scoped_ptr es(new ExpressionStep(jobInfo)); + es->expression(srcp, jobInfo); + } + } + + continue; + } + #endif srcp = retCols[i]; const AggregateColumn* ag = dynamic_cast(retCols[i].get()); + // bug 3728 Make a dummy expression for srcp if it is an // expression. This is needed to fill in some stuff. // Note that es.expression does nothing if the item is not an expression. @@ -937,7 +945,7 @@ const JobStepVector doAggProject(const CalpontSelectExecutionPlan* csep, JobInfo } else { - // MCOL-1201 multi-argument aggregate. make a dummy expression + // MCOL-1201 multi-argument aggregate. make a dummy expression // step for each argument that is an expression. for (uint32_t i = 0; i < ag->aggParms().size(); ++i) { @@ -1017,7 +1025,7 @@ const JobStepVector doAggProject(const CalpontSelectExecutionPlan* csep, JobInfo // replace the aggregate on constant with a count(*) SRCP clone; UDAFColumn* udafc = dynamic_cast(aggc); - + if (udafc) { clone.reset(new UDAFColumn(*udafc, aggc->sessionID())); @@ -1026,14 +1034,15 @@ const JobStepVector doAggProject(const CalpontSelectExecutionPlan* csep, JobInfo { clone.reset(new AggregateColumn(*aggc, aggc->sessionID())); } - + jobInfo.constAggregate.insert(make_pair(i, clone)); aggc->aggOp(AggregateColumn::COUNT_ASTERISK); aggc->distinct(false); } - + srcp = aggParms[parm]; sc = dynamic_cast(srcp.get()); + if (parm == 0) { op = aggc->aggOp(); @@ -1042,7 +1051,9 @@ const JobStepVector doAggProject(const CalpontSelectExecutionPlan* csep, JobInfo { op = AggregateColumn::MULTI_PARM; } + doDistinct = aggc->distinct(); + if (aggParms.size() == 1) { // Set the col type based on the single parm. @@ -1050,6 +1061,7 @@ const JobStepVector doAggProject(const CalpontSelectExecutionPlan* csep, JobInfo // doesn't really make sense. updateAggregateColType(aggc, srcp, op, jobInfo); } + aggCt = aggc->resultType(); // As of bug3695, make sure varbinary is not used in aggregation. diff --git a/dbcon/joblist/tupleaggregatestep.cpp b/dbcon/joblist/tupleaggregatestep.cpp index 0f981e68f..da91919f0 100644 --- a/dbcon/joblist/tupleaggregatestep.cpp +++ b/dbcon/joblist/tupleaggregatestep.cpp @@ -852,6 +852,7 @@ SJSTEP TupleAggregateStep::prepAggregate(SJSTEP& step, JobInfo& jobInfo) if (ac->aggOp() == ROWAGG_UDAF) { UDAFColumn* udafc = dynamic_cast(ac); + if (udafc) { constAggDataVec.push_back( @@ -1295,10 +1296,12 @@ void TupleAggregateStep::prep1PhaseAggregate( if (aggOp == ROWAGG_UDAF) { std::vector::iterator it = jobInfo.projectionCols.begin() + projColsUDAFIdx; + for (; it != jobInfo.projectionCols.end(); it++) { udafc = dynamic_cast((*it).get()); projColsUDAFIdx++; + if (udafc) { pUDAFFunc = udafc->getContext().getFunction(); @@ -1307,6 +1310,7 @@ void TupleAggregateStep::prep1PhaseAggregate( break; } } + if (it == jobInfo.projectionCols.end()) { throw logic_error("(1)prep1PhaseAggregate: A UDAF function is called but there\'s not enough UDAFColumns"); @@ -1488,10 +1492,12 @@ void TupleAggregateStep::prep1PhaseAggregate( // If the first param is const udafcParamIdx = 0; ConstantColumn* cc = dynamic_cast(udafc->aggParms()[udafcParamIdx].get()); + if (cc) { funct->fpConstCol = udafc->aggParms()[udafcParamIdx]; } + ++udafcParamIdx; break; } @@ -1504,10 +1510,12 @@ void TupleAggregateStep::prep1PhaseAggregate( precisionAgg.push_back(precisionProj[colProj]); typeAgg.push_back(typeProj[colProj]); widthAgg.push_back(width[colProj]); + // If the param is const if (udafc) { ConstantColumn* cc = dynamic_cast(udafc->aggParms()[udafcParamIdx].get()); + if (cc) { funct->fpConstCol = udafc->aggParms()[udafcParamIdx]; @@ -1517,6 +1525,7 @@ void TupleAggregateStep::prep1PhaseAggregate( { throw QueryDataExcept("prep1PhaseAggregate: UDAF multi function with no parms", aggregateFuncErr); } + ++udafcParamIdx; } break; @@ -1892,6 +1901,7 @@ void TupleAggregateStep::prep1PhaseDistinctAggregate( break; } } + if (it == jobInfo.projectionCols.end()) { throw logic_error("(1)prep1PhaseDistinctAggregate: A UDAF function is called but there\'s not enough UDAFColumns"); @@ -2111,10 +2121,12 @@ void TupleAggregateStep::prep1PhaseDistinctAggregate( // If the first param is const udafcParamIdx = 0; ConstantColumn* cc = dynamic_cast(udafc->aggParms()[udafcParamIdx].get()); + if (cc) { funct->fpConstCol = udafc->aggParms()[udafcParamIdx]; } + ++udafcParamIdx; break; } @@ -2129,10 +2141,12 @@ void TupleAggregateStep::prep1PhaseDistinctAggregate( widthAgg.push_back(widthProj[colProj]); multiParmIndexes.push_back(colAgg); ++colAgg; + // If the param is const if (udafc) { ConstantColumn* cc = dynamic_cast(udafc->aggParms()[udafcParamIdx].get()); + if (cc) { funct->fpConstCol = udafc->aggParms()[udafcParamIdx]; @@ -2142,6 +2156,7 @@ void TupleAggregateStep::prep1PhaseDistinctAggregate( { throw QueryDataExcept("prep1PhaseDistinctAggregate: UDAF multi function with no parms", aggregateFuncErr); } + ++udafcParamIdx; } break; @@ -2193,9 +2208,10 @@ void TupleAggregateStep::prep1PhaseDistinctAggregate( groupByNoDist.push_back(groupby); aggFuncMap.insert(make_pair(boost::make_tuple(keysAgg[i], 0, pUDAFFunc), i)); } - + // locate the return column position in aggregated rowgroup uint64_t outIdx = 0; + for (uint64_t i = 0; i < returnedColVec.size(); i++) { udafc = NULL; @@ -2240,16 +2256,19 @@ void TupleAggregateStep::prep1PhaseDistinctAggregate( if (aggOp == ROWAGG_UDAF) { std::vector::iterator it = jobInfo.projectionCols.begin() + projColsUDAFIdx; + for (; it != jobInfo.projectionCols.end(); it++) { udafc = dynamic_cast((*it).get()); projColsUDAFIdx++; + if (udafc) { pUDAFFunc = udafc->getContext().getFunction(); break; } } + if (it == jobInfo.projectionCols.end()) { throw logic_error("(1)prep1PhaseDistinctAggregate: A UDAF function is called but there\'s not enough UDAFColumns"); @@ -2546,6 +2565,7 @@ void TupleAggregateStep::prep1PhaseDistinctAggregate( { // update the aggregate function vector SP_ROWAGG_FUNC_t funct; + if (aggOp == ROWAGG_UDAF) { funct.reset(new RowUDAFFunctionCol(udafc->getContext(), colAgg, outIdx)); @@ -2589,6 +2609,7 @@ void TupleAggregateStep::prep1PhaseDistinctAggregate( else if (returnedColVec[i].second == AggregateColumn::DISTINCT_AVG) avgDistFuncMap.insert(make_pair(returnedColVec[i].first, funct)); } + ++outIdx; } // for (i @@ -2839,6 +2860,7 @@ void TupleAggregateStep::prep1PhaseDistinctAggregate( ++multiParms; continue; } + if (returnedColVec[k].first != distinctColKey) continue; @@ -2859,7 +2881,7 @@ void TupleAggregateStep::prep1PhaseDistinctAggregate( f->fStatsFunction, groupBySub.size() - 1, f->fOutputColumnIndex, - f->fAuxColumnIndex-multiParms)); + f->fAuxColumnIndex - multiParms)); functionSub2.push_back(funct); } } @@ -2887,6 +2909,7 @@ void TupleAggregateStep::prep1PhaseDistinctAggregate( ++multiParms; continue; } + // search non-distinct functions in functionVec vector::iterator it = functionVec2.begin(); @@ -2902,7 +2925,7 @@ void TupleAggregateStep::prep1PhaseDistinctAggregate( udafFuncCol->fUDAFContext, udafFuncCol->fInputColumnIndex, udafFuncCol->fOutputColumnIndex, - udafFuncCol->fAuxColumnIndex-multiParms)); + udafFuncCol->fAuxColumnIndex - multiParms)); functionSub2.push_back(funct); } else if ((f->fOutputColumnIndex == k) && @@ -2924,7 +2947,7 @@ void TupleAggregateStep::prep1PhaseDistinctAggregate( f->fStatsFunction, f->fInputColumnIndex, f->fOutputColumnIndex, - f->fAuxColumnIndex-multiParms)); + f->fAuxColumnIndex - multiParms)); functionSub2.push_back(funct); } } @@ -3160,10 +3183,12 @@ void TupleAggregateStep::prep2PhasesAggregate( if (aggOp == ROWAGG_UDAF) { std::vector::iterator it = jobInfo.projectionCols.begin() + projColsUDAFIdx; + for (; it != jobInfo.projectionCols.end(); it++) { udafc = dynamic_cast((*it).get()); projColsUDAFIdx++; + if (udafc) { pUDAFFunc = udafc->getContext().getFunction(); @@ -3172,6 +3197,7 @@ void TupleAggregateStep::prep2PhasesAggregate( break; } } + if (it == jobInfo.projectionCols.end()) { throw logic_error("(1)prep2PhasesAggregate: A UDAF function is called but there\'s not enough UDAFColumns"); @@ -3394,10 +3420,12 @@ void TupleAggregateStep::prep2PhasesAggregate( // If the first param is const udafcParamIdx = 0; ConstantColumn* cc = dynamic_cast(udafc->aggParms()[udafcParamIdx].get()); + if (cc) { funct->fpConstCol = udafc->aggParms()[udafcParamIdx]; } + ++udafcParamIdx; break; } @@ -3411,10 +3439,12 @@ void TupleAggregateStep::prep2PhasesAggregate( typeAggPm.push_back(typeProj[colProj]); widthAggPm.push_back(width[colProj]); colAggPm++; + // If the param is const if (udafc) { ConstantColumn* cc = dynamic_cast(udafc->aggParms()[udafcParamIdx].get()); + if (cc) { funct->fpConstCol = udafc->aggParms()[udafcParamIdx]; @@ -3424,6 +3454,7 @@ void TupleAggregateStep::prep2PhasesAggregate( { throw QueryDataExcept("prep2PhasesAggregate: UDAF multi function with no parms", aggregateFuncErr); } + ++udafcParamIdx; } break; @@ -3451,6 +3482,7 @@ void TupleAggregateStep::prep2PhasesAggregate( AGG_MAP aggDupFuncMap; projColsUDAFIdx = 0; + // copy over the groupby vector // update the outputColumnIndex if returned for (uint64_t i = 0; i < groupByPm.size(); i++) @@ -3462,6 +3494,7 @@ void TupleAggregateStep::prep2PhasesAggregate( // locate the return column position in aggregated rowgroup from PM // outIdx is i without the multi-columns, uint64_t outIdx = 0; + for (uint64_t i = 0; i < returnedColVec.size(); i++) { uint32_t retKey = returnedColVec[i].first; @@ -3478,6 +3511,7 @@ void TupleAggregateStep::prep2PhasesAggregate( // Is this a UDAF? use the function as part of the key. pUDAFFunc = NULL; udafc = NULL; + if (aggOp == ROWAGG_UDAF) { std::vector::iterator it = jobInfo.projectionCols.begin() + projColsUDAFIdx; @@ -3486,12 +3520,14 @@ void TupleAggregateStep::prep2PhasesAggregate( { udafc = dynamic_cast((*it).get()); projColsUDAFIdx++; + if (udafc) { pUDAFFunc = udafc->getContext().getFunction(); break; } } + if (it == jobInfo.projectionCols.end()) { throw logic_error("(3)prep2PhasesAggregate: A UDAF function is called but there\'s not enough UDAFColumns"); @@ -3644,6 +3680,7 @@ void TupleAggregateStep::prep2PhasesAggregate( { // update the aggregate function vector SP_ROWAGG_FUNC_t funct; + if (aggOp == ROWAGG_UDAF) { funct.reset(new RowUDAFFunctionCol(udafc->getContext(), colPm, outIdx)); @@ -3685,6 +3722,7 @@ void TupleAggregateStep::prep2PhasesAggregate( if (returnedColVec[i].second == AggregateColumn::AVG) avgFuncMap.insert(make_pair(returnedColVec[i].first, funct)); } + ++outIdx; } @@ -4029,10 +4067,12 @@ void TupleAggregateStep::prep2PhasesDistinctAggregate( if (aggOp == ROWAGG_UDAF) { std::vector::iterator it = jobInfo.projectionCols.begin() + projColsUDAFIdx; + for (; it != jobInfo.projectionCols.end(); it++) { udafc = dynamic_cast((*it).get()); projColsUDAFIdx++; + if (udafc) { pUDAFFunc = udafc->getContext().getFunction(); @@ -4041,6 +4081,7 @@ void TupleAggregateStep::prep2PhasesDistinctAggregate( break; } } + if (it == jobInfo.projectionCols.end()) { throw logic_error("(1)prep2PhasesDistinctAggregate: A UDAF function is called but there\'s not enough UDAFColumns"); @@ -4259,10 +4300,12 @@ void TupleAggregateStep::prep2PhasesDistinctAggregate( // If the first param is const udafcParamIdx = 0; ConstantColumn* cc = dynamic_cast(udafc->aggParms()[udafcParamIdx].get()); + if (cc) { funct->fpConstCol = udafc->aggParms()[udafcParamIdx]; } + ++udafcParamIdx; break; } @@ -4277,10 +4320,12 @@ void TupleAggregateStep::prep2PhasesDistinctAggregate( widthAggPm.push_back(width[colProj]); multiParmIndexes.push_back(colAggPm); colAggPm++; + // If the param is const if (udafc) { ConstantColumn* cc = dynamic_cast(udafc->aggParms()[udafcParamIdx].get()); + if (cc) { funct->fpConstCol = udafc->aggParms()[udafcParamIdx]; @@ -4290,6 +4335,7 @@ void TupleAggregateStep::prep2PhasesDistinctAggregate( { throw QueryDataExcept("prep2PhasesDistinctAggregate: UDAF multi function with no parms", aggregateFuncErr); } + ++udafcParamIdx; } break; @@ -4332,15 +4378,17 @@ void TupleAggregateStep::prep2PhasesDistinctAggregate( if (funcPm->fAggFunction == ROWAGG_UDAF) { RowUDAFFunctionCol* udafFuncCol = dynamic_cast(funcPm.get()); + if (!udafFuncCol) { - throw logic_error("(3)prep2PhasesDistinctAggregate: A UDAF function is called but there's no RowUDAFFunctionCol"); + throw logic_error("(3)prep2PhasesDistinctAggregate: A UDAF function is called but there's no RowUDAFFunctionCol"); } + funct.reset(new RowUDAFFunctionCol( udafFuncCol->fUDAFContext, udafFuncCol->fOutputColumnIndex, - udafFuncCol->fOutputColumnIndex-multiParms, - udafFuncCol->fAuxColumnIndex-multiParms)); + udafFuncCol->fOutputColumnIndex - multiParms, + udafFuncCol->fAuxColumnIndex - multiParms)); functionNoDistVec.push_back(funct); pUDAFFunc = udafFuncCol->fUDAFContext.getFunction(); } @@ -4350,8 +4398,8 @@ void TupleAggregateStep::prep2PhasesDistinctAggregate( funcPm->fAggFunction, funcPm->fStatsFunction, funcPm->fOutputColumnIndex, - funcPm->fOutputColumnIndex-multiParms, - funcPm->fAuxColumnIndex-multiParms)); + funcPm->fOutputColumnIndex - multiParms, + funcPm->fAuxColumnIndex - multiParms)); functionNoDistVec.push_back(funct); pUDAFFunc = NULL; } @@ -4364,6 +4412,7 @@ void TupleAggregateStep::prep2PhasesDistinctAggregate( { continue; } + oidsAggUm.push_back(oidsAggPm[idx]); keysAggUm.push_back(keysAggPm[idx]); scaleAggUm.push_back(scaleAggPm[idx]); @@ -4400,6 +4449,7 @@ void TupleAggregateStep::prep2PhasesDistinctAggregate( // locate the return column position in aggregated rowgroup from PM // outIdx is i without the multi-columns, uint64_t outIdx = 0; + for (uint64_t i = 0; i < returnedColVec.size(); i++) { pUDAFFunc = NULL; @@ -4420,16 +4470,19 @@ void TupleAggregateStep::prep2PhasesDistinctAggregate( if (aggOp == ROWAGG_UDAF) { std::vector::iterator it = jobInfo.projectionCols.begin() + projColsUDAFIdx; + for (; it != jobInfo.projectionCols.end(); it++) { udafc = dynamic_cast((*it).get()); projColsUDAFIdx++; + if (udafc) { pUDAFFunc = udafc->getContext().getFunction(); break; } } + if (it == jobInfo.projectionCols.end()) { throw logic_error("(4)prep2PhasesDistinctAggregate: A UDAF function is called but there\'s not enough UDAFColumns"); @@ -4606,6 +4659,7 @@ void TupleAggregateStep::prep2PhasesDistinctAggregate( oidsAggDist.push_back(oidsAggUm[colUm]); keysAggDist.push_back(retKey); scaleAggDist.push_back(0); + if (isUnsigned(typeAggUm[colUm])) { precisionAggDist.push_back(20); @@ -4616,6 +4670,7 @@ void TupleAggregateStep::prep2PhasesDistinctAggregate( precisionAggDist.push_back(19); typeAggDist.push_back(CalpontSystemCatalog::BIGINT); } + widthAggDist.push_back(bigIntWidth); } } @@ -4702,6 +4757,7 @@ void TupleAggregateStep::prep2PhasesDistinctAggregate( { // update the aggregate function vector SP_ROWAGG_FUNC_t funct; + if (aggOp == ROWAGG_UDAF) { funct.reset(new RowUDAFFunctionCol(udafc->getContext(), colUm, outIdx)); @@ -4745,6 +4801,7 @@ void TupleAggregateStep::prep2PhasesDistinctAggregate( else if (returnedColVec[i].second == AggregateColumn::DISTINCT_AVG) avgDistFuncMap.insert(make_pair(returnedColVec[i].first, funct)); } + ++outIdx; } // for (i @@ -4987,6 +5044,7 @@ void TupleAggregateStep::prep2PhasesDistinctAggregate( ++multiParms; continue; } + if (returnedColVec[k].first != distinctColKey) continue; @@ -5008,7 +5066,7 @@ void TupleAggregateStep::prep2PhasesDistinctAggregate( f->fStatsFunction, groupBySub.size() - 1, f->fOutputColumnIndex, - f->fAuxColumnIndex-multiParms)); + f->fAuxColumnIndex - multiParms)); functionSub2.push_back(funct); } } @@ -5034,6 +5092,7 @@ void TupleAggregateStep::prep2PhasesDistinctAggregate( ++multiParms; continue; } + // search non-distinct functions in functionVec vector::iterator it = functionVecUm.begin(); @@ -5051,7 +5110,7 @@ void TupleAggregateStep::prep2PhasesDistinctAggregate( udafFuncCol->fUDAFContext, udafFuncCol->fInputColumnIndex, udafFuncCol->fOutputColumnIndex, - udafFuncCol->fAuxColumnIndex-multiParms)); + udafFuncCol->fAuxColumnIndex - multiParms)); functionSub2.push_back(funct); } else if (f->fAggFunction == ROWAGG_COUNT_ASTERISK || @@ -5072,7 +5131,7 @@ void TupleAggregateStep::prep2PhasesDistinctAggregate( f->fStatsFunction, f->fInputColumnIndex, f->fOutputColumnIndex, - f->fAuxColumnIndex-multiParms)); + f->fAuxColumnIndex - multiParms)); functionSub2.push_back(funct); } } diff --git a/dbcon/joblist/windowfunctionstep.cpp b/dbcon/joblist/windowfunctionstep.cpp index 2a93f680b..823b2bd04 100644 --- a/dbcon/joblist/windowfunctionstep.cpp +++ b/dbcon/joblist/windowfunctionstep.cpp @@ -655,6 +655,7 @@ void WindowFunctionStep::initialize(const RowGroup& rg, JobInfo& jobInfo) // make sure index is in range else if (fields.size() > 1 && fields[1] >= 0 && static_cast(fields[1]) < types.size()) ct = types[fields[1]]; + // workaround for functions using "within group (order by)" syntax string fn = boost::to_upper_copy(wc->functionName()); diff --git a/dbcon/mysql/ha_calpont_dml.cpp b/dbcon/mysql/ha_calpont_dml.cpp index a9b64b757..67aab9721 100644 --- a/dbcon/mysql/ha_calpont_dml.cpp +++ b/dbcon/mysql/ha_calpont_dml.cpp @@ -899,6 +899,7 @@ int ha_calpont_impl_write_batch_row_(uchar* buf, TABLE* table, cal_impl_if::cal_ { fprintf(ci.filePtr, "-"); } + if (!ltime.second_part) { fprintf(ci.filePtr, "%02d:%02d:%02d%c", diff --git a/dbcon/mysql/ha_calpont_execplan.cpp b/dbcon/mysql/ha_calpont_execplan.cpp index 8df06c6b4..4cedeecb6 100644 --- a/dbcon/mysql/ha_calpont_execplan.cpp +++ b/dbcon/mysql/ha_calpont_execplan.cpp @@ -4039,8 +4039,8 @@ ParseTree* buildParseTree(Item_func* item, gp_walk_info& gwi, bool& nonSupport) ReturnedColumn* buildAggregateColumn(Item* item, gp_walk_info& gwi) { // MCOL-1201 For UDAnF multiple parameters - vector selCols; - vector orderCols; + vector selCols; + vector orderCols; if (!(gwi.thd->infinidb_vtable.cal_conn_info)) gwi.thd->infinidb_vtable.cal_conn_info = (void*)(new cal_connection_info()); @@ -4059,6 +4059,7 @@ ReturnedColumn* buildAggregateColumn(Item* item, gp_walk_info& gwi) // N.B. argument_count() is the # of formal parms to the agg fcn. InifniDB only supports 1 argument // TODO: Support more than one parm #if 0 + if (isp->argument_count() != 1 && isp->sum_func() != Item_sum::GROUP_CONCAT_FUNC && isp->sum_func() != Item_sum::UDF_SUM_FUNC) { @@ -4066,6 +4067,7 @@ ReturnedColumn* buildAggregateColumn(Item* item, gp_walk_info& gwi) gwi.parseErrorText = IDBErrorInfo::instance()->errorMsg(ERR_MUL_ARG_AGG); return NULL; } + #endif AggregateColumn* ac = NULL; @@ -4089,446 +4091,459 @@ ReturnedColumn* buildAggregateColumn(Item* item, gp_walk_info& gwi) { gwi.fatalParseError = true; gwi.parseErrorText = "Non supported aggregate type on the select clause"; + if (ac) delete ac; + return NULL; } try { - // special parsing for group_concat - if (isp->sum_func() == Item_sum::GROUP_CONCAT_FUNC) - { - Item_func_group_concat* gc = (Item_func_group_concat*)isp; - vector orderCols; - RowColumn* rowCol = new RowColumn(); - vector selCols; - - uint32_t select_ctn = gc->count_field(); - ReturnedColumn* rc = NULL; - - for (uint32_t i = 0; i < select_ctn; i++) + // special parsing for group_concat + if (isp->sum_func() == Item_sum::GROUP_CONCAT_FUNC) { - rc = buildReturnedColumn(sfitempp[i], gwi, gwi.fatalParseError); + Item_func_group_concat* gc = (Item_func_group_concat*)isp; + vector orderCols; + RowColumn* rowCol = new RowColumn(); + vector selCols; - if (!rc || gwi.fatalParseError) - { - if (ac) - delete ac; - return NULL; - } + uint32_t select_ctn = gc->count_field(); + ReturnedColumn* rc = NULL; - selCols.push_back(SRCP(rc)); - } - - ORDER** order_item, **end; - - for (order_item = gc->get_order(), - end = order_item + gc->order_field(); order_item < end; - order_item++) - { - Item* ord_col = *(*order_item)->item; - - if (ord_col->type() == Item::INT_ITEM) + for (uint32_t i = 0; i < select_ctn; i++) { - Item_int* id = (Item_int*)ord_col; - - if (id->val_int() > (int)selCols.size()) - { - gwi.fatalParseError = true; - if (ac) - delete ac; - return NULL; - } - - rc = selCols[id->val_int() - 1]->clone(); - rc->orderPos(id->val_int() - 1); - } - else - { - rc = buildReturnedColumn(ord_col, gwi, gwi.fatalParseError); + rc = buildReturnedColumn(sfitempp[i], gwi, gwi.fatalParseError); if (!rc || gwi.fatalParseError) { - if (ac) - delete ac; + if (ac) + delete ac; + return NULL; } + + selCols.push_back(SRCP(rc)); } - // 10.2 TODO: direction is now a tri-state flag - rc->asc((*order_item)->direction == ORDER::ORDER_ASC ? true : false); - orderCols.push_back(SRCP(rc)); - } + ORDER** order_item, **end; - rowCol->columnVec(selCols); - (dynamic_cast(ac))->orderCols(orderCols); - parm.reset(rowCol); - ac->aggParms().push_back(parm); - - if (gc->str_separator()) - { - string separator; - separator.assign(gc->str_separator()->ptr(), gc->str_separator()->length()); - (dynamic_cast(ac))->separator(separator); - } - } - else - { - for (uint32_t i = 0; i < isp->argument_count(); i++) - { - Item* sfitemp = sfitempp[i]; - Item::Type sfitype = sfitemp->type(); - - switch (sfitype) + for (order_item = gc->get_order(), + end = order_item + gc->order_field(); order_item < end; + order_item++) { - case Item::FIELD_ITEM: - { - Item_field* ifp = reinterpret_cast(sfitemp); - SimpleColumn* sc = buildSimpleColumn(ifp, gwi); + Item* ord_col = *(*order_item)->item; - if (!sc) + if (ord_col->type() == Item::INT_ITEM) + { + Item_int* id = (Item_int*)ord_col; + + if (id->val_int() > (int)selCols.size()) { gwi.fatalParseError = true; + + if (ac) + delete ac; + + return NULL; + } + + rc = selCols[id->val_int() - 1]->clone(); + rc->orderPos(id->val_int() - 1); + } + else + { + rc = buildReturnedColumn(ord_col, gwi, gwi.fatalParseError); + + if (!rc || gwi.fatalParseError) + { + if (ac) + delete ac; + + return NULL; + } + } + + // 10.2 TODO: direction is now a tri-state flag + rc->asc((*order_item)->direction == ORDER::ORDER_ASC ? true : false); + orderCols.push_back(SRCP(rc)); + } + + rowCol->columnVec(selCols); + (dynamic_cast(ac))->orderCols(orderCols); + parm.reset(rowCol); + ac->aggParms().push_back(parm); + + if (gc->str_separator()) + { + string separator; + separator.assign(gc->str_separator()->ptr(), gc->str_separator()->length()); + (dynamic_cast(ac))->separator(separator); + } + } + else + { + for (uint32_t i = 0; i < isp->argument_count(); i++) + { + Item* sfitemp = sfitempp[i]; + Item::Type sfitype = sfitemp->type(); + + switch (sfitype) + { + case Item::FIELD_ITEM: + { + Item_field* ifp = reinterpret_cast(sfitemp); + SimpleColumn* sc = buildSimpleColumn(ifp, gwi); + + if (!sc) + { + gwi.fatalParseError = true; + break; + } + + parm.reset(sc); + gwi.columnMap.insert(CalpontSelectExecutionPlan::ColumnMap::value_type(string(ifp->field_name), parm)); + TABLE_LIST* tmp = (ifp->cached_table ? ifp->cached_table : 0); + gwi.tableMap[make_aliastable(sc->schemaName(), sc->tableName(), sc->tableAlias(), sc->isInfiniDB())] = make_pair(1, tmp); break; } - parm.reset(sc); - gwi.columnMap.insert(CalpontSelectExecutionPlan::ColumnMap::value_type(string(ifp->field_name), parm)); - TABLE_LIST* tmp = (ifp->cached_table ? ifp->cached_table : 0); - gwi.tableMap[make_aliastable(sc->schemaName(), sc->tableName(), sc->tableAlias(), sc->isInfiniDB())] = make_pair(1, tmp); - break; - } + case Item::INT_ITEM: + case Item::STRING_ITEM: + case Item::REAL_ITEM: + case Item::DECIMAL_ITEM: + { + // treat as count(*) + if (ac->aggOp() == AggregateColumn::COUNT) + ac->aggOp(AggregateColumn::COUNT_ASTERISK); - case Item::INT_ITEM: - case Item::STRING_ITEM: - case Item::REAL_ITEM: - case Item::DECIMAL_ITEM: - { - // treat as count(*) - if (ac->aggOp() == AggregateColumn::COUNT) - ac->aggOp(AggregateColumn::COUNT_ASTERISK); parm.reset(buildReturnedColumn(sfitemp, gwi, gwi.fatalParseError)); ac->constCol(parm); - break; - } - - case Item::NULL_ITEM: - { - parm.reset(new ConstantColumn("", ConstantColumn::NULLDATA)); - ac->constCol(SRCP(buildReturnedColumn(sfitemp, gwi, gwi.fatalParseError))); - break; - } - - case Item::FUNC_ITEM: - { - Item_func* ifp = (Item_func*)sfitemp; - ReturnedColumn* rc = 0; - - // check count(1+1) case - vector tmpVec; - uint16_t parseInfo = 0; - parse_item(ifp, tmpVec, gwi.fatalParseError, parseInfo); - - if (parseInfo & SUB_BIT) - { - gwi.fatalParseError = true; break; } - else if (!gwi.fatalParseError && - !(parseInfo & AGG_BIT) && - !(parseInfo & AF_BIT) && - tmpVec.size() == 0) + + case Item::NULL_ITEM: { - rc = buildFunctionColumn(ifp, gwi, gwi.fatalParseError); - FunctionColumn* fc = dynamic_cast(rc); + parm.reset(new ConstantColumn("", ConstantColumn::NULLDATA)); + ac->constCol(SRCP(buildReturnedColumn(sfitemp, gwi, gwi.fatalParseError))); + break; + } - if ((fc && fc->functionParms().empty()) || !fc) + case Item::FUNC_ITEM: + { + Item_func* ifp = (Item_func*)sfitemp; + ReturnedColumn* rc = 0; + + // check count(1+1) case + vector tmpVec; + uint16_t parseInfo = 0; + parse_item(ifp, tmpVec, gwi.fatalParseError, parseInfo); + + if (parseInfo & SUB_BIT) { - //ac->aggOp(AggregateColumn::COUNT_ASTERISK); - ReturnedColumn* rc = buildReturnedColumn(sfitemp, gwi, gwi.fatalParseError); + gwi.fatalParseError = true; + break; + } + else if (!gwi.fatalParseError && + !(parseInfo & AGG_BIT) && + !(parseInfo & AF_BIT) && + tmpVec.size() == 0) + { + rc = buildFunctionColumn(ifp, gwi, gwi.fatalParseError); + FunctionColumn* fc = dynamic_cast(rc); - if (dynamic_cast(rc)) + if ((fc && fc->functionParms().empty()) || !fc) { - //@bug5229. handle constant function on aggregate argument - ac->constCol(SRCP(rc)); - break; + //ac->aggOp(AggregateColumn::COUNT_ASTERISK); + ReturnedColumn* rc = buildReturnedColumn(sfitemp, gwi, gwi.fatalParseError); + + if (dynamic_cast(rc)) + { + //@bug5229. handle constant function on aggregate argument + ac->constCol(SRCP(rc)); + break; + } } } + + // MySQL carelessly allows correlated aggregate function on the WHERE clause. + // Here is the work around to deal with that inconsistence. + // e.g., SELECT (SELECT t.c FROM t1 AS t WHERE t.b=MAX(t1.b + 0)) FROM t1; + ClauseType clauseType = gwi.clauseType; + + if (gwi.clauseType == WHERE) + gwi.clauseType = HAVING; + + // @bug 3603. for cases like max(rand()). try to build function first. + if (!rc) + rc = buildFunctionColumn(ifp, gwi, gwi.fatalParseError); + + parm.reset(rc); + gwi.clauseType = clauseType; + + if (gwi.fatalParseError) + break; + + break; + } + + case Item::REF_ITEM: + { + ReturnedColumn* rc = buildReturnedColumn(sfitemp, gwi, gwi.fatalParseError); + + if (rc) + { + parm.reset(rc); + break; + } } - // MySQL carelessly allows correlated aggregate function on the WHERE clause. - // Here is the work around to deal with that inconsistence. - // e.g., SELECT (SELECT t.c FROM t1 AS t WHERE t.b=MAX(t1.b + 0)) FROM t1; - ClauseType clauseType = gwi.clauseType; - - if (gwi.clauseType == WHERE) - gwi.clauseType = HAVING; - - // @bug 3603. for cases like max(rand()). try to build function first. - if (!rc) - rc = buildFunctionColumn(ifp, gwi, gwi.fatalParseError); - - parm.reset(rc); - gwi.clauseType = clauseType; - - if (gwi.fatalParseError) - break; - - break; - } - - case Item::REF_ITEM: - { - ReturnedColumn* rc = buildReturnedColumn(sfitemp, gwi, gwi.fatalParseError); - - if (rc) + default: { - parm.reset(rc); - break; + gwi.fatalParseError = true; + //gwi.parseErrorText = "Non-supported Item in Aggregate function"; } } - default: + if (gwi.fatalParseError) { - gwi.fatalParseError = true; - //gwi.parseErrorText = "Non-supported Item in Aggregate function"; - } - } + if (gwi.parseErrorText.empty()) + { + Message::Args args; - if (gwi.fatalParseError) - { - if (gwi.parseErrorText.empty()) - { - Message::Args args; + if (item->name) + args.add(item->name); + else + args.add(""); - if (item->name) - args.add(item->name); - else - args.add(""); + gwi.parseErrorText = IDBErrorInfo::instance()->errorMsg(ERR_NON_SUPPORT_AGG_ARGS, args); + } - gwi.parseErrorText = IDBErrorInfo::instance()->errorMsg(ERR_NON_SUPPORT_AGG_ARGS, args); + if (ac) + delete ac; + + return NULL; } - if (ac) - delete ac; - return NULL; - } - if (parm) - { - // MCOL-1201 multi-argument aggregate - ac->aggParms().push_back(parm); + if (parm) + { + // MCOL-1201 multi-argument aggregate + ac->aggParms().push_back(parm); + } } } - } // Get result type // Modified for MCOL-1201 multi-argument aggregate if (ac->aggParms().size() > 0) - { + { // These are all one parm functions, so we can safely // use the first parm for result type. parm = ac->aggParms()[0]; - if (isp->sum_func() == Item_sum::AVG_FUNC || - isp->sum_func() == Item_sum::AVG_DISTINCT_FUNC) - { - CalpontSystemCatalog::ColType ct = parm->resultType(); - switch (ct.colDataType) + if (isp->sum_func() == Item_sum::AVG_FUNC || + isp->sum_func() == Item_sum::AVG_DISTINCT_FUNC) { - case CalpontSystemCatalog::TINYINT: - case CalpontSystemCatalog::SMALLINT: - case CalpontSystemCatalog::MEDINT: - case CalpontSystemCatalog::INT: - case CalpontSystemCatalog::BIGINT: - case CalpontSystemCatalog::DECIMAL: - case CalpontSystemCatalog::UDECIMAL: - case CalpontSystemCatalog::UTINYINT: - case CalpontSystemCatalog::USMALLINT: - case CalpontSystemCatalog::UMEDINT: - case CalpontSystemCatalog::UINT: - case CalpontSystemCatalog::UBIGINT: - ct.colDataType = CalpontSystemCatalog::DECIMAL; - ct.colWidth = 8; - ct.scale += 4; - break; + CalpontSystemCatalog::ColType ct = parm->resultType(); + + switch (ct.colDataType) + { + case CalpontSystemCatalog::TINYINT: + case CalpontSystemCatalog::SMALLINT: + case CalpontSystemCatalog::MEDINT: + case CalpontSystemCatalog::INT: + case CalpontSystemCatalog::BIGINT: + case CalpontSystemCatalog::DECIMAL: + case CalpontSystemCatalog::UDECIMAL: + case CalpontSystemCatalog::UTINYINT: + case CalpontSystemCatalog::USMALLINT: + case CalpontSystemCatalog::UMEDINT: + case CalpontSystemCatalog::UINT: + case CalpontSystemCatalog::UBIGINT: + ct.colDataType = CalpontSystemCatalog::DECIMAL; + ct.colWidth = 8; + ct.scale += 4; + break; #if PROMOTE_FLOAT_TO_DOUBLE_ON_SUM - case CalpontSystemCatalog::FLOAT: - case CalpontSystemCatalog::UFLOAT: - case CalpontSystemCatalog::DOUBLE: - case CalpontSystemCatalog::UDOUBLE: - ct.colDataType = CalpontSystemCatalog::DOUBLE; - ct.colWidth = 8; - break; + case CalpontSystemCatalog::FLOAT: + case CalpontSystemCatalog::UFLOAT: + case CalpontSystemCatalog::DOUBLE: + case CalpontSystemCatalog::UDOUBLE: + ct.colDataType = CalpontSystemCatalog::DOUBLE; + ct.colWidth = 8; + break; #endif - default: - break; + default: + break; + } + + ac->resultType(ct); } - - ac->resultType(ct); - } - else if (isp->sum_func() == Item_sum::COUNT_FUNC || - isp->sum_func() == Item_sum::COUNT_DISTINCT_FUNC) - { - CalpontSystemCatalog::ColType ct; - ct.colDataType = CalpontSystemCatalog::BIGINT; - ct.colWidth = 8; - ct.scale = parm->resultType().scale; - ac->resultType(ct); - } - else if (isp->sum_func() == Item_sum::SUM_FUNC || - isp->sum_func() == Item_sum::SUM_DISTINCT_FUNC) - { - CalpontSystemCatalog::ColType ct = parm->resultType(); - - switch (ct.colDataType) + else if (isp->sum_func() == Item_sum::COUNT_FUNC || + isp->sum_func() == Item_sum::COUNT_DISTINCT_FUNC) { - case CalpontSystemCatalog::TINYINT: - case CalpontSystemCatalog::SMALLINT: - case CalpontSystemCatalog::MEDINT: - case CalpontSystemCatalog::INT: - case CalpontSystemCatalog::BIGINT: - ct.colDataType = CalpontSystemCatalog::BIGINT; + CalpontSystemCatalog::ColType ct; + ct.colDataType = CalpontSystemCatalog::BIGINT; + ct.colWidth = 8; + ct.scale = parm->resultType().scale; + ac->resultType(ct); + } + else if (isp->sum_func() == Item_sum::SUM_FUNC || + isp->sum_func() == Item_sum::SUM_DISTINCT_FUNC) + { + CalpontSystemCatalog::ColType ct = parm->resultType(); - // no break, let fall through + switch (ct.colDataType) + { + case CalpontSystemCatalog::TINYINT: + case CalpontSystemCatalog::SMALLINT: + case CalpontSystemCatalog::MEDINT: + case CalpontSystemCatalog::INT: + case CalpontSystemCatalog::BIGINT: + ct.colDataType = CalpontSystemCatalog::BIGINT; - case CalpontSystemCatalog::DECIMAL: - case CalpontSystemCatalog::UDECIMAL: - ct.colWidth = 8; - break; + // no break, let fall through - case CalpontSystemCatalog::UTINYINT: - case CalpontSystemCatalog::USMALLINT: - case CalpontSystemCatalog::UMEDINT: - case CalpontSystemCatalog::UINT: - case CalpontSystemCatalog::UBIGINT: - ct.colDataType = CalpontSystemCatalog::UBIGINT; - ct.colWidth = 8; - break; + case CalpontSystemCatalog::DECIMAL: + case CalpontSystemCatalog::UDECIMAL: + ct.colWidth = 8; + break; + + case CalpontSystemCatalog::UTINYINT: + case CalpontSystemCatalog::USMALLINT: + case CalpontSystemCatalog::UMEDINT: + case CalpontSystemCatalog::UINT: + case CalpontSystemCatalog::UBIGINT: + ct.colDataType = CalpontSystemCatalog::UBIGINT; + ct.colWidth = 8; + break; #if PROMOTE_FLOAT_TO_DOUBLE_ON_SUM - case CalpontSystemCatalog::FLOAT: - case CalpontSystemCatalog::UFLOAT: - case CalpontSystemCatalog::DOUBLE: - case CalpontSystemCatalog::UDOUBLE: - ct.colDataType = CalpontSystemCatalog::DOUBLE; - ct.colWidth = 8; - break; + case CalpontSystemCatalog::FLOAT: + case CalpontSystemCatalog::UFLOAT: + case CalpontSystemCatalog::DOUBLE: + case CalpontSystemCatalog::UDOUBLE: + ct.colDataType = CalpontSystemCatalog::DOUBLE; + ct.colWidth = 8; + break; #endif - default: - break; - } + default: + break; + } - ac->resultType(ct); - } - else if (isp->sum_func() == Item_sum::STD_FUNC || - isp->sum_func() == Item_sum::VARIANCE_FUNC) - { - CalpontSystemCatalog::ColType ct; - ct.colDataType = CalpontSystemCatalog::DOUBLE; - ct.colWidth = 8; - ct.scale = 0; - ac->resultType(ct); - } - else if (isp->sum_func() == Item_sum::SUM_BIT_FUNC) - { - CalpontSystemCatalog::ColType ct; - ct.colDataType = CalpontSystemCatalog::BIGINT; - ct.colWidth = 8; - ct.scale = 0; - ct.precision = -16; // borrowed to indicate skip null value check on connector - ac->resultType(ct); - } - else if (isp->sum_func() == Item_sum::GROUP_CONCAT_FUNC) - { - //Item_func_group_concat* gc = (Item_func_group_concat*)isp; - CalpontSystemCatalog::ColType ct; - ct.colDataType = CalpontSystemCatalog::VARCHAR; - ct.colWidth = isp->max_length; - ct.precision = 0; - ac->resultType(ct); + ac->resultType(ct); + } + else if (isp->sum_func() == Item_sum::STD_FUNC || + isp->sum_func() == Item_sum::VARIANCE_FUNC) + { + CalpontSystemCatalog::ColType ct; + ct.colDataType = CalpontSystemCatalog::DOUBLE; + ct.colWidth = 8; + ct.scale = 0; + ac->resultType(ct); + } + else if (isp->sum_func() == Item_sum::SUM_BIT_FUNC) + { + CalpontSystemCatalog::ColType ct; + ct.colDataType = CalpontSystemCatalog::BIGINT; + ct.colWidth = 8; + ct.scale = 0; + ct.precision = -16; // borrowed to indicate skip null value check on connector + ac->resultType(ct); + } + else if (isp->sum_func() == Item_sum::GROUP_CONCAT_FUNC) + { + //Item_func_group_concat* gc = (Item_func_group_concat*)isp; + CalpontSystemCatalog::ColType ct; + ct.colDataType = CalpontSystemCatalog::VARCHAR; + ct.colWidth = isp->max_length; + ct.precision = 0; + ac->resultType(ct); + } + else + { + // UDAF result type will be set below. + ac->resultType(parm->resultType()); + } } else { - // UDAF result type will be set below. - ac->resultType(parm->resultType()); + ac->resultType(colType_MysqlToIDB(isp)); } - } - else - { - ac->resultType(colType_MysqlToIDB(isp)); - } - // adjust decimal result type according to internalDecimalScale - if (gwi.internalDecimalScale >= 0 && ac->resultType().colDataType == CalpontSystemCatalog::DECIMAL) - { - CalpontSystemCatalog::ColType ct = ac->resultType(); - ct.scale = gwi.internalDecimalScale; - ac->resultType(ct); - } - - // check for same aggregate on the select list - ac->expressionId(ci->expressionId++); - - if (gwi.clauseType != SELECT) - { - for (uint32_t i = 0; i < gwi.returnedCols.size(); i++) + // adjust decimal result type according to internalDecimalScale + if (gwi.internalDecimalScale >= 0 && ac->resultType().colDataType == CalpontSystemCatalog::DECIMAL) { - if (*ac == gwi.returnedCols[i].get()) - ac->expressionId(gwi.returnedCols[i]->expressionId()); + CalpontSystemCatalog::ColType ct = ac->resultType(); + ct.scale = gwi.internalDecimalScale; + ac->resultType(ct); } - } - // @bug5977 @note Temporary fix to avoid mysqld crash. The permanent fix will - // be applied in ExeMgr. When the ExeMgr fix is available, this checking - // will be taken out. + // check for same aggregate on the select list + ac->expressionId(ci->expressionId++); + + if (gwi.clauseType != SELECT) + { + for (uint32_t i = 0; i < gwi.returnedCols.size(); i++) + { + if (*ac == gwi.returnedCols[i].get()) + ac->expressionId(gwi.returnedCols[i]->expressionId()); + } + } + + // @bug5977 @note Temporary fix to avoid mysqld crash. The permanent fix will + // be applied in ExeMgr. When the ExeMgr fix is available, this checking + // will be taken out. if (isp->sum_func() != Item_sum::UDF_SUM_FUNC) { - if (ac->constCol() && gwi.tbList.empty() && gwi.derivedTbList.empty()) - { - gwi.fatalParseError = true; - gwi.parseErrorText = "No project column found for aggregate function"; + if (ac->constCol() && gwi.tbList.empty() && gwi.derivedTbList.empty()) + { + gwi.fatalParseError = true; + gwi.parseErrorText = "No project column found for aggregate function"; + if (ac) delete ac; - return NULL; - } - else if (ac->constCol()) - { - gwi.count_asterisk_list.push_back(ac); - } + + return NULL; + } + else if (ac->constCol()) + { + gwi.count_asterisk_list.push_back(ac); + } } - // For UDAF, populate the context and call the UDAF init() function. + // For UDAF, populate the context and call the UDAF init() function. // The return type is (should be) set in context by init(). - if (isp->sum_func() == Item_sum::UDF_SUM_FUNC) - { - UDAFColumn* udafc = dynamic_cast(ac); - - if (udafc) + if (isp->sum_func() == Item_sum::UDF_SUM_FUNC) { - mcsv1Context& context = udafc->getContext(); - context.setName(isp->func_name()); + UDAFColumn* udafc = dynamic_cast(ac); - // Set up the return type defaults for the call to init() - context.setResultType(udafc->resultType().colDataType); - context.setColWidth(udafc->resultType().colWidth); - context.setScale(udafc->resultType().scale); - context.setPrecision(udafc->resultType().precision); + if (udafc) + { + mcsv1Context& context = udafc->getContext(); + context.setName(isp->func_name()); + + // Set up the return type defaults for the call to init() + context.setResultType(udafc->resultType().colDataType); + context.setColWidth(udafc->resultType().colWidth); + context.setScale(udafc->resultType().scale); + context.setPrecision(udafc->resultType().precision); context.setParamCount(udafc->aggParms().size()); ColumnDatum colType; ColumnDatum colTypes[udafc->aggParms().size()]; + // Build the column type vector. // Modified for MCOL-1201 multi-argument aggregate for (uint32_t i = 0; i < udafc->aggParms().size(); ++i) { - const execplan::CalpontSystemCatalog::ColType& resultType + const execplan::CalpontSystemCatalog::ColType& resultType = udafc->aggParms()[i]->resultType(); colType.dataType = resultType.colDataType; colType.precision = resultType.precision; @@ -4536,65 +4551,78 @@ ReturnedColumn* buildAggregateColumn(Item* item, gp_walk_info& gwi) colTypes[i] = colType; } - // Call the user supplied init() + // Call the user supplied init() mcsv1sdk::mcsv1_UDAF* udaf = context.getFunction(); + if (!udaf) { gwi.fatalParseError = true; gwi.parseErrorText = "Aggregate Function " + context.getName() + " doesn't exist in the ColumnStore engine"; + if (ac) delete ac; + return NULL; } - if (udaf->init(&context, colTypes) == mcsv1_UDAF::ERROR) - { - gwi.fatalParseError = true; - gwi.parseErrorText = udafc->getContext().getErrorMessage(); + + if (udaf->init(&context, colTypes) == mcsv1_UDAF::ERROR) + { + gwi.fatalParseError = true; + gwi.parseErrorText = udafc->getContext().getErrorMessage(); + if (ac) delete ac; - return NULL; - } + + return NULL; + } // UDAF_OVER_REQUIRED means that this function is for Window // Function only. Reject it here in aggregate land. - if (udafc->getContext().getRunFlag(UDAF_OVER_REQUIRED)) - { - gwi.fatalParseError = true; - gwi.parseErrorText = - logging::IDBErrorInfo::instance()->errorMsg(logging::ERR_WINDOW_FUNC_ONLY, - context.getName()); + if (udafc->getContext().getRunFlag(UDAF_OVER_REQUIRED)) + { + gwi.fatalParseError = true; + gwi.parseErrorText = + logging::IDBErrorInfo::instance()->errorMsg(logging::ERR_WINDOW_FUNC_ONLY, + context.getName()); + if (ac) delete ac; - return NULL; - } - // Set the return type as set in init() - CalpontSystemCatalog::ColType ct; - ct.colDataType = context.getResultType(); - ct.colWidth = context.getColWidth(); - ct.scale = context.getScale(); - ct.precision = context.getPrecision(); - udafc->resultType(ct); + return NULL; + } + + // Set the return type as set in init() + CalpontSystemCatalog::ColType ct; + ct.colDataType = context.getResultType(); + ct.colWidth = context.getColWidth(); + ct.scale = context.getScale(); + ct.precision = context.getPrecision(); + udafc->resultType(ct); + } } } - } catch (std::logic_error e) { gwi.fatalParseError = true; gwi.parseErrorText = "error building Aggregate Function: "; gwi.parseErrorText += e.what(); + if (ac) delete ac; + return NULL; } catch (...) { gwi.fatalParseError = true; gwi.parseErrorText = "error building Aggregate Function: Unspecified exception"; + if (ac) delete ac; + return NULL; } + return ac; } @@ -7915,6 +7943,7 @@ int getSelectPlan(gp_walk_info& gwi, SELECT_LEX& select_lex, SCSEP& csep, bool i setError(gwi.thd, ER_INTERNAL_ERROR, gwi.parseErrorText, gwi); return ER_CHECK_NOT_IMPLEMENTED; } + // Replace the last (presumably constant) object with minSc if ((*coliter)->aggParms().empty()) { diff --git a/dbcon/mysql/ha_calpont_impl.cpp b/dbcon/mysql/ha_calpont_impl.cpp index 3d4ee6ac3..dcafc4c38 100644 --- a/dbcon/mysql/ha_calpont_impl.cpp +++ b/dbcon/mysql/ha_calpont_impl.cpp @@ -782,7 +782,7 @@ int fetchNextRow(uchar* buf, cal_table_info& ti, cal_connection_info* ci, bool h //double double_val = *(double*)(&value); //f2->store(double_val); if ((f2->decimals() == DECIMAL_NOT_SPECIFIED && row.getScale(s) > 0) - || f2->decimals() < row.getScale(s)) + || f2->decimals() < row.getScale(s)) { f2->dec = row.getScale(s); } @@ -5278,6 +5278,7 @@ int ha_calpont_impl_group_by_init(ha_calpont_group_by_handler* group_hand, TABLE execplan::CalpontSelectExecutionPlan::ColumnMap::iterator colMapIter; execplan::CalpontSelectExecutionPlan::ColumnMap::iterator condColMapIter; execplan::ParseTree* ptIt; + for (TABLE_LIST* tl = gi.groupByTables; tl; tl = tl->next_local) { mapiter = ci->tableMap.find(tl->table); diff --git a/dbcon/mysql/ha_window_function.cpp b/dbcon/mysql/ha_window_function.cpp index 8d68a6260..4c04a402c 100644 --- a/dbcon/mysql/ha_window_function.cpp +++ b/dbcon/mysql/ha_window_function.cpp @@ -384,7 +384,7 @@ ReturnedColumn* buildWindowFunctionColumn(Item* item, gp_walk_info& gwi, bool& n // Modified for MCOL-1201 multi-argument aggregate for (size_t i = 0; i < funcParms.size(); ++i) { - const execplan::CalpontSystemCatalog::ColType& resultType + const execplan::CalpontSystemCatalog::ColType& resultType = funcParms[i]->resultType(); colType.dataType = resultType.colDataType; colType.precision = resultType.precision; diff --git a/dbcon/mysql/is_columnstore_columns.cpp b/dbcon/mysql/is_columnstore_columns.cpp index 1aa379724..84d81042c 100644 --- a/dbcon/mysql/is_columnstore_columns.cpp +++ b/dbcon/mysql/is_columnstore_columns.cpp @@ -56,10 +56,11 @@ ST_FIELD_INFO is_columnstore_columns_fields[] = }; -static void get_cond_item(Item_func *item, String **table, String **db) +static void get_cond_item(Item_func* item, String** table, String** db) { char tmp_char[MAX_FIELD_WIDTH]; - Item_field *item_field = (Item_field*) item->arguments()[0]->real_item(); + Item_field* item_field = (Item_field*) item->arguments()[0]->real_item(); + if (strcasecmp(item_field->field_name, "table_name") == 0) { String str_buf(tmp_char, sizeof(tmp_char), system_charset_info); @@ -74,13 +75,14 @@ static void get_cond_item(Item_func *item, String **table, String **db) } } -static void get_cond_items(COND *cond, String **table, String **db) +static void get_cond_items(COND* cond, String** table, String** db) { if (cond->type() == Item::FUNC_ITEM) { Item_func* fitem = (Item_func*) cond; + if (fitem->arguments()[0]->real_item()->type() == Item::FIELD_ITEM && - fitem->arguments()[1]->const_item()) + fitem->arguments()[1]->const_item()) { get_cond_item(fitem, table, db); } @@ -88,8 +90,9 @@ static void get_cond_items(COND *cond, String **table, String **db) else if ((cond->type() == Item::COND_ITEM) && (((Item_cond*) cond)->functype() == Item_func::COND_AND_FUNC)) { List_iterator li(*((Item_cond*) cond)->argument_list()); - Item *item; - while ((item= li++)) + Item* item; + + while ((item = li++)) { if (item->type() == Item::FUNC_ITEM) { @@ -107,8 +110,8 @@ static int is_columnstore_columns_fill(THD* thd, TABLE_LIST* tables, COND* cond) { CHARSET_INFO* cs = system_charset_info; TABLE* table = tables->table; - String *table_name = NULL; - String *db_name = NULL; + String* table_name = NULL; + String* db_name = NULL; boost::shared_ptr systemCatalogPtr = execplan::CalpontSystemCatalog::makeCalpontSystemCatalog(execplan::CalpontSystemCatalog::idb_tid2sid(thd->thread_id)); @@ -133,6 +136,7 @@ static int is_columnstore_columns_fill(THD* thd, TABLE_LIST* tables, COND* cond) continue; } } + if (table_name) { if ((*it).second.table.compare(table_name->ptr()) != 0) diff --git a/dbcon/mysql/is_columnstore_extents.cpp b/dbcon/mysql/is_columnstore_extents.cpp index c6dcba567..0fd42d3cf 100644 --- a/dbcon/mysql/is_columnstore_extents.cpp +++ b/dbcon/mysql/is_columnstore_extents.cpp @@ -52,7 +52,7 @@ ST_FIELD_INFO is_columnstore_extents_fields[] = {0, 0, MYSQL_TYPE_NULL, 0, 0, 0, 0} }; -static int generate_result(BRM::OID_t oid, BRM::DBRM *emp, TABLE *table, THD *thd) +static int generate_result(BRM::OID_t oid, BRM::DBRM* emp, TABLE* table, THD* thd) { CHARSET_INFO* cs = system_charset_info; std::vector entries; @@ -77,7 +77,7 @@ static int generate_result(BRM::OID_t oid, BRM::DBRM *emp, TABLE *table, THD *th table->field[1]->store("Column", strlen("Column"), cs); if (iter->partition.cprange.lo_val == std::numeric_limits::max() || - iter->partition.cprange.lo_val <= (std::numeric_limits::min() + 2)) + iter->partition.cprange.lo_val <= (std::numeric_limits::min() + 2)) { table->field[4]->set_null(); } @@ -88,7 +88,7 @@ static int generate_result(BRM::OID_t oid, BRM::DBRM *emp, TABLE *table, THD *th } if (iter->partition.cprange.hi_val == std::numeric_limits::max() || - iter->partition.cprange.hi_val <= (std::numeric_limits::min() + 2)) + iter->partition.cprange.hi_val <= (std::numeric_limits::min() + 2)) { table->field[5]->set_null(); } @@ -179,15 +179,17 @@ static int generate_result(BRM::OID_t oid, BRM::DBRM *emp, TABLE *table, THD *th iter++; } + return 0; } -static int is_columnstore_extents_fill(THD *thd, TABLE_LIST *tables, COND *cond) +static int is_columnstore_extents_fill(THD* thd, TABLE_LIST* tables, COND* cond) { BRM::OID_t cond_oid = 0; - TABLE *table = tables->table; + TABLE* table = tables->table; + + BRM::DBRM* emp = new BRM::DBRM(); - BRM::DBRM *emp = new BRM::DBRM(); if (!emp || !emp->isDBRMReady()) { return 1; @@ -196,13 +198,15 @@ static int is_columnstore_extents_fill(THD *thd, TABLE_LIST *tables, COND *cond) if (cond && cond->type() == Item::FUNC_ITEM) { Item_func* fitem = (Item_func*) cond; + if ((fitem->functype() == Item_func::EQ_FUNC) && (fitem->argument_count() == 2)) { - if(fitem->arguments()[0]->real_item()->type() == Item::FIELD_ITEM && - fitem->arguments()[1]->const_item()) + if (fitem->arguments()[0]->real_item()->type() == Item::FIELD_ITEM && + fitem->arguments()[1]->const_item()) { // WHERE object_id = value - Item_field *item_field = (Item_field*) fitem->arguments()[0]->real_item(); + Item_field* item_field = (Item_field*) fitem->arguments()[0]->real_item(); + if (strcasecmp(item_field->field_name, "object_id") == 0) { cond_oid = fitem->arguments()[1]->val_int(); @@ -210,10 +214,11 @@ static int is_columnstore_extents_fill(THD *thd, TABLE_LIST *tables, COND *cond) } } else if (fitem->arguments()[1]->real_item()->type() == Item::FIELD_ITEM && - fitem->arguments()[0]->const_item()) + fitem->arguments()[0]->const_item()) { // WHERE value = object_id - Item_field *item_field = (Item_field*) fitem->arguments()[1]->real_item(); + Item_field* item_field = (Item_field*) fitem->arguments()[1]->real_item(); + if (strcasecmp(item_field->field_name, "object_id") == 0) { cond_oid = fitem->arguments()[0]->val_int(); @@ -224,15 +229,17 @@ static int is_columnstore_extents_fill(THD *thd, TABLE_LIST *tables, COND *cond) else if (fitem->functype() == Item_func::IN_FUNC) { // WHERE object_id in (value1, value2) - Item_field *item_field = (Item_field*) fitem->arguments()[0]->real_item(); + Item_field* item_field = (Item_field*) fitem->arguments()[0]->real_item(); + if (strcasecmp(item_field->field_name, "object_id") == 0) { - for (unsigned int i=1; i < fitem->argument_count(); i++) + for (unsigned int i = 1; i < fitem->argument_count(); i++) { cond_oid = fitem->arguments()[i]->val_int(); int result = generate_result(cond_oid, emp, table, thd); + if (result) - return 1; + return 1; } } } @@ -240,13 +247,16 @@ static int is_columnstore_extents_fill(THD *thd, TABLE_LIST *tables, COND *cond) strcasecmp(fitem->func_name(), "find_in_set") == 0) { // WHERE FIND_IN_SET(object_id, values) - String *tmp_var = fitem->arguments()[1]->val_str(); + String* tmp_var = fitem->arguments()[1]->val_str(); std::stringstream ss(tmp_var->ptr()); + while (ss >> cond_oid) { int ret = generate_result(cond_oid, emp, table, thd); + if (ret) return 1; + if (ss.peek() == ',') ss.ignore(); } @@ -256,12 +266,14 @@ static int is_columnstore_extents_fill(THD *thd, TABLE_LIST *tables, COND *cond) execplan::ObjectIDManager oidm; BRM::OID_t MaxOID = oidm.size(); - for(BRM::OID_t oid = 3000; oid <= MaxOID; oid++) + for (BRM::OID_t oid = 3000; oid <= MaxOID; oid++) { int result = generate_result(oid, emp, table, thd); + if (result) return 1; } + delete emp; return 0; } diff --git a/dbcon/mysql/is_columnstore_files.cpp b/dbcon/mysql/is_columnstore_files.cpp index 815345474..71d61958f 100644 --- a/dbcon/mysql/is_columnstore_files.cpp +++ b/dbcon/mysql/is_columnstore_files.cpp @@ -84,7 +84,7 @@ static bool get_file_sizes(messageqcpp::MessageQueueClient* msgQueueClient, cons } } -static int generate_result(BRM::OID_t oid, BRM::DBRM *emp, TABLE *table, THD *thd) +static int generate_result(BRM::OID_t oid, BRM::DBRM* emp, TABLE* table, THD* thd) { std::vector entries; CHARSET_INFO* cs = system_charset_info; @@ -185,14 +185,15 @@ static int generate_result(BRM::OID_t oid, BRM::DBRM *emp, TABLE *table, THD *th messageqcpp::MessageQueueClientPool::releaseInstance(msgQueueClient); msgQueueClient = NULL; } + return 0; } -static int is_columnstore_files_fill(THD *thd, TABLE_LIST *tables, COND *cond) +static int is_columnstore_files_fill(THD* thd, TABLE_LIST* tables, COND* cond) { - BRM::DBRM *emp = new BRM::DBRM(); + BRM::DBRM* emp = new BRM::DBRM(); BRM::OID_t cond_oid = 0; - TABLE *table = tables->table; + TABLE* table = tables->table; if (!emp || !emp->isDBRMReady()) { @@ -202,13 +203,15 @@ static int is_columnstore_files_fill(THD *thd, TABLE_LIST *tables, COND *cond) if (cond && cond->type() == Item::FUNC_ITEM) { Item_func* fitem = (Item_func*) cond; + if ((fitem->functype() == Item_func::EQ_FUNC) && (fitem->argument_count() == 2)) { - if(fitem->arguments()[0]->real_item()->type() == Item::FIELD_ITEM && - fitem->arguments()[1]->const_item()) + if (fitem->arguments()[0]->real_item()->type() == Item::FIELD_ITEM && + fitem->arguments()[1]->const_item()) { // WHERE object_id = value - Item_field *item_field = (Item_field*) fitem->arguments()[0]->real_item(); + Item_field* item_field = (Item_field*) fitem->arguments()[0]->real_item(); + if (strcasecmp(item_field->field_name, "object_id") == 0) { cond_oid = fitem->arguments()[1]->val_int(); @@ -216,10 +219,11 @@ static int is_columnstore_files_fill(THD *thd, TABLE_LIST *tables, COND *cond) } } else if (fitem->arguments()[1]->real_item()->type() == Item::FIELD_ITEM && - fitem->arguments()[0]->const_item()) + fitem->arguments()[0]->const_item()) { // WHERE value = object_id - Item_field *item_field = (Item_field*) fitem->arguments()[1]->real_item(); + Item_field* item_field = (Item_field*) fitem->arguments()[1]->real_item(); + if (strcasecmp(item_field->field_name, "object_id") == 0) { cond_oid = fitem->arguments()[0]->val_int(); @@ -230,15 +234,17 @@ static int is_columnstore_files_fill(THD *thd, TABLE_LIST *tables, COND *cond) else if (fitem->functype() == Item_func::IN_FUNC) { // WHERE object_id in (value1, value2) - Item_field *item_field = (Item_field*) fitem->arguments()[0]->real_item(); + Item_field* item_field = (Item_field*) fitem->arguments()[0]->real_item(); + if (strcasecmp(item_field->field_name, "object_id") == 0) { - for (unsigned int i=1; i < fitem->argument_count(); i++) + for (unsigned int i = 1; i < fitem->argument_count(); i++) { cond_oid = fitem->arguments()[i]->val_int(); int result = generate_result(cond_oid, emp, table, thd); + if (result) - return 1; + return 1; } } } @@ -246,13 +252,16 @@ static int is_columnstore_files_fill(THD *thd, TABLE_LIST *tables, COND *cond) strcasecmp(fitem->func_name(), "find_in_set") == 0) { // WHERE FIND_IN_SET(object_id, values) - String *tmp_var = fitem->arguments()[1]->val_str(); + String* tmp_var = fitem->arguments()[1]->val_str(); std::stringstream ss(tmp_var->ptr()); + while (ss >> cond_oid) { int ret = generate_result(cond_oid, emp, table, thd); + if (ret) return 1; + if (ss.peek() == ',') ss.ignore(); } @@ -264,9 +273,10 @@ static int is_columnstore_files_fill(THD *thd, TABLE_LIST *tables, COND *cond) if (!cond_oid) { - for(BRM::OID_t oid = 3000; oid <= MaxOID; oid++) + for (BRM::OID_t oid = 3000; oid <= MaxOID; oid++) { int result = generate_result(oid, emp, table, thd); + if (result) return 1; } diff --git a/dbcon/mysql/is_columnstore_tables.cpp b/dbcon/mysql/is_columnstore_tables.cpp index ddc21e6f5..d422c2f90 100644 --- a/dbcon/mysql/is_columnstore_tables.cpp +++ b/dbcon/mysql/is_columnstore_tables.cpp @@ -42,10 +42,11 @@ ST_FIELD_INFO is_columnstore_tables_fields[] = {0, 0, MYSQL_TYPE_NULL, 0, 0, 0, 0} }; -static void get_cond_item(Item_func *item, String **table, String **db) +static void get_cond_item(Item_func* item, String** table, String** db) { char tmp_char[MAX_FIELD_WIDTH]; - Item_field *item_field = (Item_field*) item->arguments()[0]->real_item(); + Item_field* item_field = (Item_field*) item->arguments()[0]->real_item(); + if (strcasecmp(item_field->field_name, "table_name") == 0) { String str_buf(tmp_char, sizeof(tmp_char), system_charset_info); @@ -60,13 +61,14 @@ static void get_cond_item(Item_func *item, String **table, String **db) } } -static void get_cond_items(COND *cond, String **table, String **db) +static void get_cond_items(COND* cond, String** table, String** db) { if (cond->type() == Item::FUNC_ITEM) { Item_func* fitem = (Item_func*) cond; + if (fitem->arguments()[0]->real_item()->type() == Item::FIELD_ITEM && - fitem->arguments()[1]->const_item()) + fitem->arguments()[1]->const_item()) { get_cond_item(fitem, table, db); } @@ -74,8 +76,9 @@ static void get_cond_items(COND *cond, String **table, String **db) else if ((cond->type() == Item::COND_ITEM) && (((Item_cond*) cond)->functype() == Item_func::COND_AND_FUNC)) { List_iterator li(*((Item_cond*) cond)->argument_list()); - Item *item; - while ((item= li++)) + Item* item; + + while ((item = li++)) { if (item->type() == Item::FUNC_ITEM) { @@ -93,8 +96,8 @@ static int is_columnstore_tables_fill(THD* thd, TABLE_LIST* tables, COND* cond) { CHARSET_INFO* cs = system_charset_info; TABLE* table = tables->table; - String *table_name = NULL; - String *db_name = NULL; + String* table_name = NULL; + String* db_name = NULL; boost::shared_ptr systemCatalogPtr = execplan::CalpontSystemCatalog::makeCalpontSystemCatalog(execplan::CalpontSystemCatalog::idb_tid2sid(thd->thread_id)); @@ -119,6 +122,7 @@ static int is_columnstore_tables_fill(THD* thd, TABLE_LIST* tables, COND* cond) continue; } } + if (table_name) { if ((*it).second.table.compare(table_name->ptr()) != 0) diff --git a/oam/oamcpp/liboamcpp.cpp b/oam/oamcpp/liboamcpp.cpp index 2747bf1fa..5c63ffd20 100644 --- a/oam/oamcpp/liboamcpp.cpp +++ b/oam/oamcpp/liboamcpp.cpp @@ -5811,34 +5811,35 @@ bool Oam::autoMovePmDbroot(std::string residePM) exceptionControl("autoMovePmDbroot", API_INVALID_PARAMETER); } - //detach first to make sure DBS can be detach before trying to move to another pm - DBRootConfigList::iterator pt3 = residedbrootConfigList.begin(); - for( ; pt3 != residedbrootConfigList.end() ; pt3++ ) - { - int dbrootID = *pt3; + //detach first to make sure DBS can be detach before trying to move to another pm + DBRootConfigList::iterator pt3 = residedbrootConfigList.begin(); - try - { - typedef std::vector dbrootList; - dbrootList dbrootlist; - dbrootlist.push_back(itoa(dbrootID)); + for ( ; pt3 != residedbrootConfigList.end() ; pt3++ ) + { + int dbrootID = *pt3; - amazonDetach(dbrootlist); - } - catch (exception& ) - { - writeLog("ERROR: amazonDetach failure", LOG_TYPE_ERROR ); - - //reattach - typedef std::vector dbrootList; - dbrootList dbrootlist; - dbrootlist.push_back(itoa(dbrootID)); + try + { + typedef std::vector dbrootList; + dbrootList dbrootlist; + dbrootlist.push_back(itoa(dbrootID)); - amazonAttach(residePM, dbrootlist); + amazonDetach(dbrootlist); + } + catch (exception& ) + { + writeLog("ERROR: amazonDetach failure", LOG_TYPE_ERROR ); - exceptionControl("autoMovePmDbroot", API_DETACH_FAILURE); - } - } + //reattach + typedef std::vector dbrootList; + dbrootList dbrootlist; + dbrootlist.push_back(itoa(dbrootID)); + + amazonAttach(residePM, dbrootlist); + + exceptionControl("autoMovePmDbroot", API_DETACH_FAILURE); + } + } //get dbroot id for other PMs systemStorageInfo_t t; @@ -6359,15 +6360,17 @@ bool Oam::autoUnMovePmDbroot(std::string toPM) } if (!found) - writeLog("No dbroots found in ../Calpont/local/moveDbrootTransactionLog", LOG_TYPE_DEBUG ); - cout << "No dbroots found in " << fileName << endl; + { + writeLog("No dbroots found in ../Calpont/local/moveDbrootTransactionLog", LOG_TYPE_DEBUG ); + + cout << "No dbroots found in " << fileName << endl; } oldFile.close(); unlink (fileName.c_str()); ofstream newFile (fileName.c_str()); - //create new file +//create new file int fd = open(fileName.c_str(), O_RDWR | O_CREAT, 0664); copy(lines.begin(), lines.end(), ostream_iterator(newFile, "\n")); @@ -7773,7 +7776,7 @@ void Oam::actionMysqlCalpont(MYSQLCALPONT_ACTION action) else return; - // check if mysql-Columnstore is installed + // check if mysql-Columnstore is installed string mysqlscript = InstallDir + "/mysql/mysql-Columnstore"; if (access(mysqlscript.c_str(), X_OK) != 0) @@ -10377,146 +10380,161 @@ void Oam::sendStatusUpdate(ByteStream obs, ByteStream::byte returnRequestType) * ****************************************************************************/ - void Oam::amazonDetach(dbrootList dbrootConfigList) +void Oam::amazonDetach(dbrootList dbrootConfigList) +{ + //if amazon cloud with external volumes, do the detach/attach moves + string cloud; + string DBRootStorageType; + + try { - //if amazon cloud with external volumes, do the detach/attach moves - string cloud; - string DBRootStorageType; - try { - getSystemConfig("Cloud", cloud); - getSystemConfig("DBRootStorageType", DBRootStorageType); - } - catch(...) {} + getSystemConfig("Cloud", cloud); + getSystemConfig("DBRootStorageType", DBRootStorageType); + } + catch (...) {} - if ( (cloud == "amazon-ec2" || cloud == "amazon-vpc") && - DBRootStorageType == "external" ) - { - writeLog("amazonDetach function started ", LOG_TYPE_DEBUG ); - - dbrootList::iterator pt3 = dbrootConfigList.begin(); - for( ; pt3 != dbrootConfigList.end() ; pt3++) - { - string dbrootid = *pt3; - string volumeNameID = "PMVolumeName" + dbrootid; - string volumeName = oam::UnassignedName; - string deviceNameID = "PMVolumeDeviceName" + dbrootid; - string deviceName = oam::UnassignedName; - try { - getSystemConfig( volumeNameID, volumeName); - getSystemConfig( deviceNameID, deviceName); - } - catch(...) - {} - - if ( volumeName == oam::UnassignedName || deviceName == oam::UnassignedName ) - { - cout << " ERROR: amazonDetach, invalid configure " + volumeName + ":" + deviceName << endl; - writeLog("ERROR: amazonDetach, invalid configure " + volumeName + ":" + deviceName, LOG_TYPE_ERROR ); - exceptionControl("amazonDetach", API_INVALID_PARAMETER); - } - - //send msg to to-pm to umount volume - int returnStatus = sendMsgToProcMgr(UNMOUNT, dbrootid, FORCEFUL, ACK_YES); - if (returnStatus != API_SUCCESS) { - writeLog("ERROR: amazonDetach, umount failed on " + dbrootid, LOG_TYPE_ERROR ); - } - - if (!detachEC2Volume(volumeName)) { - cout << " ERROR: amazonDetach, detachEC2Volume failed on " + volumeName << endl; - writeLog("ERROR: amazonDetach, detachEC2Volume failed on " + volumeName , LOG_TYPE_ERROR ); - exceptionControl("amazonDetach", API_FAILURE); - } - - writeLog("amazonDetach, detachEC2Volume passed on " + volumeName , LOG_TYPE_DEBUG ); - } - } - } - - /*************************************************************************** - * - * Function: amazonAttach - * - * Purpose: Amazon EC2 volume Attach needed - * - ****************************************************************************/ - - void Oam::amazonAttach(std::string toPM, dbrootList dbrootConfigList) + if ( (cloud == "amazon-ec2" || cloud == "amazon-vpc") && + DBRootStorageType == "external" ) { - //if amazon cloud with external volumes, do the detach/attach moves - string cloud; - string DBRootStorageType; - try { - getSystemConfig("Cloud", cloud); - getSystemConfig("DBRootStorageType", DBRootStorageType); - } - catch(...) {} + writeLog("amazonDetach function started ", LOG_TYPE_DEBUG ); - if ( (cloud == "amazon-ec2" || cloud == "amazon-vpc") && - DBRootStorageType == "external" ) - { - writeLog("amazonAttach function started ", LOG_TYPE_DEBUG ); + dbrootList::iterator pt3 = dbrootConfigList.begin(); - //get Instance Name for to-pm - string toInstanceName = oam::UnassignedName; - try - { - ModuleConfig moduleconfig; - getSystemConfig(toPM, moduleconfig); - HostConfigList::iterator pt1 = moduleconfig.hostConfigList.begin(); - toInstanceName = (*pt1).HostName; - } - catch(...) - {} + for ( ; pt3 != dbrootConfigList.end() ; pt3++) + { + string dbrootid = *pt3; + string volumeNameID = "PMVolumeName" + dbrootid; + string volumeName = oam::UnassignedName; + string deviceNameID = "PMVolumeDeviceName" + dbrootid; + string deviceName = oam::UnassignedName; - if ( toInstanceName == oam::UnassignedName || toInstanceName.empty() ) - { - cout << " ERROR: amazonAttach, invalid Instance Name for " << toPM << endl; - writeLog("ERROR: amazonAttach, invalid Instance Name " + toPM, LOG_TYPE_ERROR ); - exceptionControl("amazonAttach", API_INVALID_PARAMETER); - } + try + { + getSystemConfig( volumeNameID, volumeName); + getSystemConfig( deviceNameID, deviceName); + } + catch (...) + {} - dbrootList::iterator pt3 = dbrootConfigList.begin(); - for( ; pt3 != dbrootConfigList.end() ; pt3++) - { - string dbrootid = *pt3; - string volumeNameID = "PMVolumeName" + dbrootid; - string volumeName = oam::UnassignedName; - string deviceNameID = "PMVolumeDeviceName" + dbrootid; - string deviceName = oam::UnassignedName; - try { - getSystemConfig( volumeNameID, volumeName); - getSystemConfig( deviceNameID, deviceName); - } - catch(...) - {} + if ( volumeName == oam::UnassignedName || deviceName == oam::UnassignedName ) + { + cout << " ERROR: amazonDetach, invalid configure " + volumeName + ":" + deviceName << endl; + writeLog("ERROR: amazonDetach, invalid configure " + volumeName + ":" + deviceName, LOG_TYPE_ERROR ); + exceptionControl("amazonDetach", API_INVALID_PARAMETER); + } - if ( volumeName == oam::UnassignedName || deviceName == oam::UnassignedName ) - { - cout << " ERROR: amazonAttach, invalid configure " + volumeName + ":" + deviceName << endl; - writeLog("ERROR: amazonAttach, invalid configure " + volumeName + ":" + deviceName, LOG_TYPE_ERROR ); - exceptionControl("amazonAttach", API_INVALID_PARAMETER); - } + //send msg to to-pm to umount volume + int returnStatus = sendMsgToProcMgr(UNMOUNT, dbrootid, FORCEFUL, ACK_YES); - if (!attachEC2Volume(volumeName, deviceName, toInstanceName)) { - cout << " ERROR: amazonAttach, attachEC2Volume failed on " + volumeName + ":" + deviceName + ":" + toInstanceName << endl; - writeLog("ERROR: amazonAttach, attachEC2Volume failed on " + volumeName + ":" + deviceName + ":" + toInstanceName, LOG_TYPE_ERROR ); - exceptionControl("amazonAttach", API_FAILURE); - } + if (returnStatus != API_SUCCESS) + { + writeLog("ERROR: amazonDetach, umount failed on " + dbrootid, LOG_TYPE_ERROR ); + } - writeLog("amazonAttach, attachEC2Volume passed on " + volumeName + ":" + toPM, LOG_TYPE_DEBUG ); - } - } - } + if (!detachEC2Volume(volumeName)) + { + cout << " ERROR: amazonDetach, detachEC2Volume failed on " + volumeName << endl; + writeLog("ERROR: amazonDetach, detachEC2Volume failed on " + volumeName, LOG_TYPE_ERROR ); + exceptionControl("amazonDetach", API_FAILURE); + } + writeLog("amazonDetach, detachEC2Volume passed on " + volumeName, LOG_TYPE_DEBUG ); + } + } +} - /*************************************************************************** - * - * Function: amazonReattach +/*************************************************************************** * - * Purpose: Amazon EC2 volume reattach needed + * Function: amazonAttach + * + * Purpose: Amazon EC2 volume Attach needed * ****************************************************************************/ +void Oam::amazonAttach(std::string toPM, dbrootList dbrootConfigList) +{ + //if amazon cloud with external volumes, do the detach/attach moves + string cloud; + string DBRootStorageType; + + try + { + getSystemConfig("Cloud", cloud); + getSystemConfig("DBRootStorageType", DBRootStorageType); + } + catch (...) {} + + if ( (cloud == "amazon-ec2" || cloud == "amazon-vpc") && + DBRootStorageType == "external" ) + { + writeLog("amazonAttach function started ", LOG_TYPE_DEBUG ); + + //get Instance Name for to-pm + string toInstanceName = oam::UnassignedName; + + try + { + ModuleConfig moduleconfig; + getSystemConfig(toPM, moduleconfig); + HostConfigList::iterator pt1 = moduleconfig.hostConfigList.begin(); + toInstanceName = (*pt1).HostName; + } + catch (...) + {} + + if ( toInstanceName == oam::UnassignedName || toInstanceName.empty() ) + { + cout << " ERROR: amazonAttach, invalid Instance Name for " << toPM << endl; + writeLog("ERROR: amazonAttach, invalid Instance Name " + toPM, LOG_TYPE_ERROR ); + exceptionControl("amazonAttach", API_INVALID_PARAMETER); + } + + dbrootList::iterator pt3 = dbrootConfigList.begin(); + + for ( ; pt3 != dbrootConfigList.end() ; pt3++) + { + string dbrootid = *pt3; + string volumeNameID = "PMVolumeName" + dbrootid; + string volumeName = oam::UnassignedName; + string deviceNameID = "PMVolumeDeviceName" + dbrootid; + string deviceName = oam::UnassignedName; + + try + { + getSystemConfig( volumeNameID, volumeName); + getSystemConfig( deviceNameID, deviceName); + } + catch (...) + {} + + if ( volumeName == oam::UnassignedName || deviceName == oam::UnassignedName ) + { + cout << " ERROR: amazonAttach, invalid configure " + volumeName + ":" + deviceName << endl; + writeLog("ERROR: amazonAttach, invalid configure " + volumeName + ":" + deviceName, LOG_TYPE_ERROR ); + exceptionControl("amazonAttach", API_INVALID_PARAMETER); + } + + if (!attachEC2Volume(volumeName, deviceName, toInstanceName)) + { + cout << " ERROR: amazonAttach, attachEC2Volume failed on " + volumeName + ":" + deviceName + ":" + toInstanceName << endl; + writeLog("ERROR: amazonAttach, attachEC2Volume failed on " + volumeName + ":" + deviceName + ":" + toInstanceName, LOG_TYPE_ERROR ); + exceptionControl("amazonAttach", API_FAILURE); + } + + writeLog("amazonAttach, attachEC2Volume passed on " + volumeName + ":" + toPM, LOG_TYPE_DEBUG ); + } + } +} + + +/*************************************************************************** +* +* Function: amazonReattach +* +* Purpose: Amazon EC2 volume reattach needed +* +****************************************************************************/ + void Oam::amazonReattach(std::string toPM, dbrootList dbrootConfigList, bool attach) { //if amazon cloud with external volumes, do the detach/attach moves diff --git a/oam/oamcpp/liboamcpp.h b/oam/oamcpp/liboamcpp.h index 3b18ac490..8575cd4d6 100644 --- a/oam/oamcpp/liboamcpp.h +++ b/oam/oamcpp/liboamcpp.h @@ -229,7 +229,7 @@ enum API_STATUS API_CONN_REFUSED, API_CANCELLED, API_STILL_WORKING, - API_DETACH_FAILURE, + API_DETACH_FAILURE, API_MAX }; @@ -2433,8 +2433,8 @@ public: void amazonReattach(std::string toPM, dbrootList dbrootConfigList, bool attach = false); void mountDBRoot(dbrootList dbrootConfigList, bool mount = true); - void amazonDetach(dbrootList dbrootConfigList); - void amazonAttach(std::string toPM, dbrootList dbrootConfigList); + void amazonDetach(dbrootList dbrootConfigList); + void amazonAttach(std::string toPM, dbrootList dbrootConfigList); /** *@brief gluster control diff --git a/primitives/primproc/primitiveserver.cpp b/primitives/primproc/primitiveserver.cpp index f977149c1..227b494de 100644 --- a/primitives/primproc/primitiveserver.cpp +++ b/primitives/primproc/primitiveserver.cpp @@ -2067,11 +2067,11 @@ struct ReadThread case DICT_CREATE_EQUALITY_FILTER: { PriorityThreadPool::Job job; - const uint8_t *buf = bs->buf(); - uint32_t pos = sizeof(ISMPacketHeader) - 2; - job.stepID = *((uint32_t *) &buf[pos+6]); - job.uniqueID = *((uint32_t *) &buf[pos+10]); - job.sock = outIos; + const uint8_t* buf = bs->buf(); + uint32_t pos = sizeof(ISMPacketHeader) - 2; + job.stepID = *((uint32_t*) &buf[pos + 6]); + job.uniqueID = *((uint32_t*) &buf[pos + 10]); + job.sock = outIos; job.functor = boost::shared_ptr(new CreateEqualityFilter(bs)); OOBPool->addJob(job); break; @@ -2080,11 +2080,11 @@ struct ReadThread case DICT_DESTROY_EQUALITY_FILTER: { PriorityThreadPool::Job job; - const uint8_t *buf = bs->buf(); - uint32_t pos = sizeof(ISMPacketHeader) - 2; - job.stepID = *((uint32_t *) &buf[pos+6]); - job.uniqueID = *((uint32_t *) &buf[pos+10]); - job.sock = outIos; + const uint8_t* buf = bs->buf(); + uint32_t pos = sizeof(ISMPacketHeader) - 2; + job.stepID = *((uint32_t*) &buf[pos + 6]); + job.uniqueID = *((uint32_t*) &buf[pos + 10]); + job.sock = outIos; job.functor = boost::shared_ptr(new DestroyEqualityFilter(bs)); OOBPool->addJob(job); break; @@ -2118,11 +2118,14 @@ struct ReadThread job.id = hdr->Hdr.UniqueID; job.weight = LOGICAL_BLOCK_RIDS; job.priority = hdr->Hdr.Priority; - const uint8_t *buf = bs->buf(); - uint32_t pos = sizeof(ISMPacketHeader) - 2; - job.stepID = *((uint32_t *) &buf[pos+6]); - job.uniqueID = *((uint32_t *) &buf[pos+10]); - job.sock = outIos; + const uint8_t* buf = bs->buf(); + uint32_t pos = sizeof(ISMPacketHeader) - 2; + job.stepID = *((uint32_t*) &buf[pos + 6]); + job.uniqueID = *((uint32_t*) &buf[pos + 10]); + job.sock = outIos; + + if (hdr->flags & IS_SYSCAT) + { //boost::thread t(DictScanJob(outIos, bs, writeLock)); // using already-existing threads may cut latency // if it's changed back to running in an independent thread @@ -2167,11 +2170,14 @@ struct ReadThread job.id = bpps->getID(); job.weight = ismHdr->Size; job.priority = bpps->priority(); - const uint8_t *buf = bs->buf(); - uint32_t pos = sizeof(ISMPacketHeader) - 2; - job.stepID = *((uint32_t *) &buf[pos+6]); - job.uniqueID = *((uint32_t *) &buf[pos+10]); - job.sock = outIos; + const uint8_t* buf = bs->buf(); + uint32_t pos = sizeof(ISMPacketHeader) - 2; + job.stepID = *((uint32_t*) &buf[pos + 6]); + job.uniqueID = *((uint32_t*) &buf[pos + 10]); + job.sock = outIos; + + if (bpps->isSysCat()) + { //boost::thread t(*bpps); // using already-existing threads may cut latency @@ -2191,11 +2197,11 @@ struct ReadThread { PriorityThreadPool::Job job; job.functor = boost::shared_ptr(new BPPHandler::Create(fBPPHandler, bs)); - const uint8_t *buf = bs->buf(); - uint32_t pos = sizeof(ISMPacketHeader) - 2; - job.stepID = *((uint32_t *) &buf[pos+6]); - job.uniqueID = *((uint32_t *) &buf[pos+10]); - job.sock = outIos; + const uint8_t* buf = bs->buf(); + uint32_t pos = sizeof(ISMPacketHeader) - 2; + job.stepID = *((uint32_t*) &buf[pos + 6]); + job.uniqueID = *((uint32_t*) &buf[pos + 10]); + job.sock = outIos; OOBPool->addJob(job); //fBPPHandler->createBPP(*bs); break; @@ -2206,11 +2212,11 @@ struct ReadThread PriorityThreadPool::Job job; job.functor = boost::shared_ptr(new BPPHandler::AddJoiner(fBPPHandler, bs)); job.id = fBPPHandler->getUniqueID(bs, ismHdr->Command); - const uint8_t *buf = bs->buf(); - uint32_t pos = sizeof(ISMPacketHeader) - 2; - job.stepID = *((uint32_t *) &buf[pos+6]); - job.uniqueID = *((uint32_t *) &buf[pos+10]); - job.sock = outIos; + const uint8_t* buf = bs->buf(); + uint32_t pos = sizeof(ISMPacketHeader) - 2; + job.stepID = *((uint32_t*) &buf[pos + 6]); + job.uniqueID = *((uint32_t*) &buf[pos + 10]); + job.sock = outIos; OOBPool->addJob(job); //fBPPHandler->addJoinerToBPP(*bs); break; @@ -2224,11 +2230,11 @@ struct ReadThread PriorityThreadPool::Job job; job.functor = boost::shared_ptr(new BPPHandler::LastJoiner(fBPPHandler, bs)); job.id = fBPPHandler->getUniqueID(bs, ismHdr->Command); - const uint8_t *buf = bs->buf(); - uint32_t pos = sizeof(ISMPacketHeader) - 2; - job.stepID = *((uint32_t *) &buf[pos+6]); - job.uniqueID = *((uint32_t *) &buf[pos+10]); - job.sock = outIos; + const uint8_t* buf = bs->buf(); + uint32_t pos = sizeof(ISMPacketHeader) - 2; + job.stepID = *((uint32_t*) &buf[pos + 6]); + job.uniqueID = *((uint32_t*) &buf[pos + 10]); + job.sock = outIos; OOBPool->addJob(job); break; } @@ -2240,11 +2246,11 @@ struct ReadThread PriorityThreadPool::Job job; job.functor = boost::shared_ptr(new BPPHandler::Destroy(fBPPHandler, bs)); job.id = fBPPHandler->getUniqueID(bs, ismHdr->Command); - const uint8_t *buf = bs->buf(); - uint32_t pos = sizeof(ISMPacketHeader) - 2; - job.stepID = *((uint32_t *) &buf[pos+6]); - job.uniqueID = *((uint32_t *) &buf[pos+10]); - job.sock = outIos; + const uint8_t* buf = bs->buf(); + uint32_t pos = sizeof(ISMPacketHeader) - 2; + job.stepID = *((uint32_t*) &buf[pos + 6]); + job.uniqueID = *((uint32_t*) &buf[pos + 10]); + job.sock = outIos; OOBPool->addJob(job); //fBPPHandler->destroyBPP(*bs); break; @@ -2263,11 +2269,11 @@ struct ReadThread PriorityThreadPool::Job job; job.functor = boost::shared_ptr(new BPPHandler::Abort(fBPPHandler, bs)); job.id = fBPPHandler->getUniqueID(bs, ismHdr->Command); - const uint8_t *buf = bs->buf(); - uint32_t pos = sizeof(ISMPacketHeader) - 2; - job.stepID = *((uint32_t *) &buf[pos+6]); - job.uniqueID = *((uint32_t *) &buf[pos+10]); - job.sock = outIos; + const uint8_t* buf = bs->buf(); + uint32_t pos = sizeof(ISMPacketHeader) - 2; + job.stepID = *((uint32_t*) &buf[pos + 6]); + job.uniqueID = *((uint32_t*) &buf[pos + 10]); + job.sock = outIos; OOBPool->addJob(job); break; } @@ -2299,12 +2305,12 @@ struct ReadThread } } - // If this function is called, we have a "bug" of some sort. We added - // the "fIos" connection to UmSocketSelector earlier, so at the very - // least, UmSocketSelector should have been able to return that con- - // nection/port. We will try to recover by using the original fIos to - // send the response msg; but as stated, if this ever happens we have - // a bug we need to resolve. +// If this function is called, we have a "bug" of some sort. We added +// the "fIos" connection to UmSocketSelector earlier, so at the very +// least, UmSocketSelector should have been able to return that con- +// nection/port. We will try to recover by using the original fIos to +// send the response msg; but as stated, if this ever happens we have +// a bug we need to resolve. void handleUmSockSelErr(const string& cmd) { ostringstream oss; diff --git a/procmgr/main.cpp b/procmgr/main.cpp index ae662adcb..05160da1e 100644 --- a/procmgr/main.cpp +++ b/procmgr/main.cpp @@ -1688,7 +1688,7 @@ void pingDeviceThread() processManager.restartProcessType("WriteEngineServer", moduleName); //set module to enable state - processManager.enableModule(moduleName, oam::AUTO_OFFLINE, true); + processManager.enableModule(moduleName, oam::AUTO_OFFLINE, true); downActiveOAMModule = false; int retry; @@ -1784,7 +1784,7 @@ void pingDeviceThread() } else //set module to enable state - processManager.enableModule(moduleName, oam::AUTO_OFFLINE, true); + processManager.enableModule(moduleName, oam::AUTO_OFFLINE, true); //restart module processes int retry = 0; @@ -2094,7 +2094,7 @@ void pingDeviceThread() if ( PrimaryUMModuleName == moduleName ) downPrimaryUM = true; - // if disabled, skip + // if disabled, skip if (opState != oam::AUTO_DISABLED ) { //Log failure, issue alarm, set moduleOpState @@ -2140,7 +2140,9 @@ void pingDeviceThread() if ( ( moduleName.find("pm") == 0 && !amazon && ( DBRootStorageType != "internal") ) || ( moduleName.find("pm") == 0 && amazon && downActiveOAMModule ) || ( moduleName.find("pm") == 0 && amazon && AmazonPMFailover == "y") ) - string error; + { + string error; + try { log.writeLog(__LINE__, "Call autoMovePmDbroot", LOG_TYPE_DEBUG); @@ -2157,23 +2159,23 @@ void pingDeviceThread() catch (...) { log.writeLog(__LINE__, "EXCEPTION ERROR on autoMovePmDbroot: Caught unknown exception!", LOG_TYPE_ERROR); - } - - if ( error == oam.itoa(oam::API_DETACH_FAILURE) ) - { - processManager.setModuleState(moduleName, oam::AUTO_DISABLED); + } - // resume the dbrm - oam.dbrmctl("resume"); - log.writeLog(__LINE__, "'dbrmctl resume' done", LOG_TYPE_DEBUG); + if ( error == oam.itoa(oam::API_DETACH_FAILURE) ) + { + processManager.setModuleState(moduleName, oam::AUTO_DISABLED); - //enable query stats - dbrm.setSystemQueryReady(true); + // resume the dbrm + oam.dbrmctl("resume"); + log.writeLog(__LINE__, "'dbrmctl resume' done", LOG_TYPE_DEBUG); - //set query system state ready - processManager.setQuerySystemState(true); - - break; + //enable query stats + dbrm.setSystemQueryReady(true); + + //set query system state ready + processManager.setQuerySystemState(true); + + break; } } } diff --git a/procmgr/processmanager.cpp b/procmgr/processmanager.cpp index 7646a0c3d..f9f5a8d47 100644 --- a/procmgr/processmanager.cpp +++ b/procmgr/processmanager.cpp @@ -3780,7 +3780,7 @@ void ProcessManager::recycleProcess(string module, bool enableModule) restartProcessType("ExeMgr"); sleep(1); - restartProcessType("mysqld"); + restartProcessType("mysqld"); restartProcessType("WriteEngineServer"); sleep(1); @@ -3839,8 +3839,8 @@ int ProcessManager::enableModule(string target, int state, bool failover) setStandbyModule(newStandbyModule); //set recycle process - if (!failover) - recycleProcess(target); + if (!failover) + recycleProcess(target); log.writeLog(__LINE__, "enableModule request for " + target + " completed", LOG_TYPE_DEBUG); @@ -4648,7 +4648,7 @@ int ProcessManager::restartProcessType( std::string processName, std::string ski PMwithUM = "n"; } - // If mysqld is the processName, then send to modules were ExeMgr is running + // If mysqld is the processName, then send to modules were ExeMgr is running try { oam.getProcessStatus(systemprocessstatus); @@ -4659,7 +4659,7 @@ int ProcessManager::restartProcessType( std::string processName, std::string ski if ( systemprocessstatus.processstatus[i].Module == skipModule ) continue; - if ( processName == "mysqld" ) { + if ( processName == "mysqld" ) { if ( systemprocessstatus.processstatus[i].ProcessName == "ExeMgr") { @@ -9814,7 +9814,7 @@ int ProcessManager::OAMParentModuleChange() { log.writeLog(__LINE__, "System Active, restart needed processes", LOG_TYPE_DEBUG); - processManager.restartProcessType("mysqld"); + processManager.restartProcessType("mysqld"); processManager.restartProcessType("ExeMgr"); processManager.restartProcessType("WriteEngineServer"); processManager.reinitProcessType("DBRMWorkerNode"); @@ -11014,7 +11014,7 @@ void ProcessManager::stopProcessTypes(bool manualFlag) log.writeLog(__LINE__, "stopProcessTypes Called"); //front-end first - processManager.stopProcessType("mysqld", manualFlag); + processManager.stopProcessType("mysqld", manualFlag); processManager.stopProcessType("DMLProc", manualFlag); processManager.stopProcessType("DDLProc", manualFlag); processManager.stopProcessType("ExeMgr", manualFlag); diff --git a/procmon/main.cpp b/procmon/main.cpp index 41977139c..35a2ebb59 100644 --- a/procmon/main.cpp +++ b/procmon/main.cpp @@ -781,10 +781,10 @@ int main(int argc, char** argv) if ( ret != 0 ) log.writeLog(__LINE__, "pthread_create failed, return code = " + oam.itoa(ret), LOG_TYPE_ERROR); - //mysqld status monitor thread - if ( config.moduleType() == "um" || - ( config.moduleType() == "pm" && config.ServerInstallType() == oam::INSTALL_COMBINE_DM_UM_PM ) || - ( config.moduleType() == "pm" && PMwithUM == "y") ) + //mysqld status monitor thread + if ( config.moduleType() == "um" || + ( config.moduleType() == "pm" && config.ServerInstallType() == oam::INSTALL_COMBINE_DM_UM_PM ) || + ( config.moduleType() == "pm" && PMwithUM == "y") ) { pthread_t mysqlThread; ret = pthread_create (&mysqlThread, NULL, (void* (*)(void*)) &mysqlMonitorThread, NULL); @@ -1233,7 +1233,7 @@ static void mysqlMonitorThread(MonitorConfig config) catch (...) {} - sleep(5); + sleep(5); } } diff --git a/procmon/processmonitor.cpp b/procmon/processmonitor.cpp index 3da88c2df..5ff74b034 100644 --- a/procmon/processmonitor.cpp +++ b/procmon/processmonitor.cpp @@ -484,7 +484,7 @@ void ProcessMonitor::processMessage(messageqcpp::ByteStream msg, messageqcpp::IO log.writeLog(__LINE__, "MSG RECEIVED: Stop process request on " + processName); int requestStatus = API_SUCCESS; - // check for mysqld + // check for mysqld if ( processName == "mysqld" ) { try @@ -553,7 +553,7 @@ void ProcessMonitor::processMessage(messageqcpp::ByteStream msg, messageqcpp::IO msg >> manualFlag; log.writeLog(__LINE__, "MSG RECEIVED: Start process request on: " + processName); - // check for mysqld + // check for mysqld if ( processName == "mysqld" ) { try @@ -684,7 +684,7 @@ void ProcessMonitor::processMessage(messageqcpp::ByteStream msg, messageqcpp::IO log.writeLog(__LINE__, "MSG RECEIVED: Restart process request on " + processName); int requestStatus = API_SUCCESS; - // check for mysqld restart + // check for mysqld restart if ( processName == "mysqld" ) { try @@ -933,7 +933,7 @@ void ProcessMonitor::processMessage(messageqcpp::ByteStream msg, messageqcpp::IO log.writeLog(__LINE__, "Error running DBRM clearShm", LOG_TYPE_ERROR); } - //stop the mysqld daemon + //stop the mysqld daemon try { oam.actionMysqlCalpont(MYSQL_STOP); @@ -1071,13 +1071,14 @@ void ProcessMonitor::processMessage(messageqcpp::ByteStream msg, messageqcpp::IO system(cmd.c_str()); - //start the mysqld daemon + //start the mysqld daemon try { oam.actionMysqlCalpont(MYSQL_START); } catch (...) - { // mysqld didn't start, return with error + { + // mysqld didn't start, return with error // mysql didn't start, return with error log.writeLog(__LINE__, "STARTALL: MySQL failed to start, start-module failure", LOG_TYPE_CRITICAL); @@ -1366,7 +1367,7 @@ void ProcessMonitor::processMessage(messageqcpp::ByteStream msg, messageqcpp::IO //send down notification oam.sendDeviceNotification(config.moduleName(), MODULE_DOWN); - //stop the mysqld daemon and then columnstore + //stop the mysqld daemon and then columnstore try { oam.actionMysqlCalpont(MYSQL_STOP); } @@ -1548,7 +1549,7 @@ void ProcessMonitor::processMessage(messageqcpp::ByteStream msg, messageqcpp::IO } } - // install mysqld rpms if being reconfigured as a um + // install mysqld rpms if being reconfigured as a um if ( reconfigureModuleName.find("um") != string::npos ) { string cmd = startup::StartUp::installDir() + "/bin/post-mysqld-install >> /tmp/rpminstall"; diff --git a/utils/dataconvert/dataconvert.cpp b/utils/dataconvert/dataconvert.cpp index 1a2e90a88..c4dbbc728 100644 --- a/utils/dataconvert/dataconvert.cpp +++ b/utils/dataconvert/dataconvert.cpp @@ -1420,6 +1420,7 @@ DataConvert::convertColumnData(const CalpontSystemCatalog::ColType& colType, { pushWarning = true; } + value = (int64_t) * (reinterpret_cast(&aTime)); } break; @@ -1928,6 +1929,7 @@ int64_t DataConvert::convertColumnTime( { return value; } + if (dataOrgLen < 3) { // Not enough chars to be a time diff --git a/utils/funcexp/func_date.cpp b/utils/funcexp/func_date.cpp index 9f875b374..d0bc30942 100644 --- a/utils/funcexp/func_date.cpp +++ b/utils/funcexp/func_date.cpp @@ -75,7 +75,7 @@ int64_t Func_date::getIntVal(rowgroup::Row& row, break; } - // Time adds to now() and then gets value + // Time adds to now() and then gets value case CalpontSystemCatalog::TIME: { int64_t val; diff --git a/utils/funcexp/func_day.cpp b/utils/funcexp/func_day.cpp index a25a41bcf..7ff2bab9a 100644 --- a/utils/funcexp/func_day.cpp +++ b/utils/funcexp/func_day.cpp @@ -62,7 +62,7 @@ int64_t Func_day::getIntVal(rowgroup::Row& row, val = parm[0]->data()->getIntVal(row, isNull); return (uint32_t)((val >> 38) & 0x3f); - // Time adds to now() and then gets value + // Time adds to now() and then gets value case CalpontSystemCatalog::TIME: aDateTime = static_cast(nowDatetime()); aTime = parm[0]->data()->getTimeIntVal(row, isNull); diff --git a/utils/funcexp/func_dayname.cpp b/utils/funcexp/func_dayname.cpp index abb2d47dc..5da6d8943 100644 --- a/utils/funcexp/func_dayname.cpp +++ b/utils/funcexp/func_dayname.cpp @@ -73,7 +73,7 @@ int64_t Func_dayname::getIntVal(rowgroup::Row& row, day = (uint32_t)((val >> 38) & 0x3f); break; - // Time adds to now() and then gets value + // Time adds to now() and then gets value case CalpontSystemCatalog::TIME: aDateTime = static_cast(nowDatetime()); aTime = parm[0]->data()->getTimeIntVal(row, isNull); @@ -160,8 +160,10 @@ string Func_dayname::getStrVal(rowgroup::Row& row, CalpontSystemCatalog::ColType& op_ct) { int32_t weekday = getIntVal(row, parm, isNull, op_ct); + if (weekday == -1) return ""; + return helpers::weekdayFullNames[weekday]; } diff --git a/utils/funcexp/func_dayofweek.cpp b/utils/funcexp/func_dayofweek.cpp index 1ac549060..ec84f5738 100644 --- a/utils/funcexp/func_dayofweek.cpp +++ b/utils/funcexp/func_dayofweek.cpp @@ -71,7 +71,7 @@ int64_t Func_dayofweek::getIntVal(rowgroup::Row& row, day = (uint32_t)((val >> 38) & 0x3f); break; - // Time adds to now() and then gets value + // Time adds to now() and then gets value case CalpontSystemCatalog::TIME: aDateTime = static_cast(nowDatetime()); aTime = parm[0]->data()->getTimeIntVal(row, isNull); diff --git a/utils/funcexp/func_dayofyear.cpp b/utils/funcexp/func_dayofyear.cpp index 0ec48f22f..ee3b9cf30 100644 --- a/utils/funcexp/func_dayofyear.cpp +++ b/utils/funcexp/func_dayofyear.cpp @@ -71,7 +71,7 @@ int64_t Func_dayofyear::getIntVal(rowgroup::Row& row, day = (uint32_t)((val >> 38) & 0x3f); break; - // Time adds to now() and then gets value + // Time adds to now() and then gets value case CalpontSystemCatalog::TIME: aDateTime = static_cast(nowDatetime()); aTime = parm[0]->data()->getTimeIntVal(row, isNull); diff --git a/utils/funcexp/func_last_day.cpp b/utils/funcexp/func_last_day.cpp index 757e19d77..28b4c01e2 100644 --- a/utils/funcexp/func_last_day.cpp +++ b/utils/funcexp/func_last_day.cpp @@ -72,7 +72,7 @@ int64_t Func_last_day::getIntVal(rowgroup::Row& row, day = (uint32_t)((val >> 38) & 0x3f); break; - // Time adds to now() and then gets value + // Time adds to now() and then gets value case CalpontSystemCatalog::TIME: aDateTime = static_cast(nowDatetime()); aTime = parm[0]->data()->getTimeIntVal(row, isNull); diff --git a/utils/funcexp/func_month.cpp b/utils/funcexp/func_month.cpp index 4269ae22f..5479270d0 100644 --- a/utils/funcexp/func_month.cpp +++ b/utils/funcexp/func_month.cpp @@ -61,7 +61,7 @@ int64_t Func_month::getIntVal(rowgroup::Row& row, val = parm[0]->data()->getIntVal(row, isNull); return (unsigned)((val >> 44) & 0xf); - // Time adds to now() and then gets value + // Time adds to now() and then gets value case CalpontSystemCatalog::TIME: aDateTime = static_cast(nowDatetime()); aTime = parm[0]->data()->getTimeIntVal(row, isNull); diff --git a/utils/funcexp/func_monthname.cpp b/utils/funcexp/func_monthname.cpp index dfa530cdd..9657b1ea2 100644 --- a/utils/funcexp/func_monthname.cpp +++ b/utils/funcexp/func_monthname.cpp @@ -48,8 +48,10 @@ string Func_monthname::getStrVal(rowgroup::Row& row, CalpontSystemCatalog::ColType& op_ct) { int32_t month = getIntVal(row, parm, isNull, op_ct); + if (month == -1) return ""; + return helpers::monthFullNames[month]; } @@ -90,7 +92,7 @@ int64_t Func_monthname::getIntVal(rowgroup::Row& row, val = parm[0]->data()->getIntVal(row, isNull); return (unsigned)((val >> 44) & 0xf); - // Time adds to now() and then gets value + // Time adds to now() and then gets value case CalpontSystemCatalog::TIME: aDateTime = static_cast(nowDatetime()); aTime = parm[0]->data()->getTimeIntVal(row, isNull); diff --git a/utils/funcexp/func_quarter.cpp b/utils/funcexp/func_quarter.cpp index c819cbdbe..78559d68d 100644 --- a/utils/funcexp/func_quarter.cpp +++ b/utils/funcexp/func_quarter.cpp @@ -65,7 +65,7 @@ int64_t Func_quarter::getIntVal(rowgroup::Row& row, month = (val >> 44) & 0xf; break; - // Time adds to now() and then gets value + // Time adds to now() and then gets value case CalpontSystemCatalog::TIME: aDateTime = static_cast(nowDatetime()); aTime = parm[0]->data()->getTimeIntVal(row, isNull); diff --git a/utils/funcexp/func_to_days.cpp b/utils/funcexp/func_to_days.cpp index 5c58b1b11..cc2e3afa2 100644 --- a/utils/funcexp/func_to_days.cpp +++ b/utils/funcexp/func_to_days.cpp @@ -85,7 +85,7 @@ int64_t Func_to_days::getIntVal(rowgroup::Row& row, break; } - // Time adds to now() and then gets value + // Time adds to now() and then gets value case CalpontSystemCatalog::TIME: { int64_t val; diff --git a/utils/funcexp/func_week.cpp b/utils/funcexp/func_week.cpp index 9cb869c1e..a9e47bd4b 100644 --- a/utils/funcexp/func_week.cpp +++ b/utils/funcexp/func_week.cpp @@ -75,7 +75,7 @@ int64_t Func_week::getIntVal(rowgroup::Row& row, day = (uint32_t)((val >> 38) & 0x3f); break; - // Time adds to now() and then gets value + // Time adds to now() and then gets value case CalpontSystemCatalog::TIME: aDateTime = static_cast(nowDatetime()); aTime = parm[0]->data()->getTimeIntVal(row, isNull); diff --git a/utils/funcexp/func_weekday.cpp b/utils/funcexp/func_weekday.cpp index 67f535f1f..9666710f5 100644 --- a/utils/funcexp/func_weekday.cpp +++ b/utils/funcexp/func_weekday.cpp @@ -71,7 +71,7 @@ int64_t Func_weekday::getIntVal(rowgroup::Row& row, day = (uint32_t)((val >> 38) & 0x3f); break; - // Time adds to now() and then gets value + // Time adds to now() and then gets value case CalpontSystemCatalog::TIME: aDateTime = static_cast(nowDatetime()); aTime = parm[0]->data()->getTimeIntVal(row, isNull); diff --git a/utils/funcexp/func_year.cpp b/utils/funcexp/func_year.cpp index 8b3f79fa0..17ff4f2d0 100644 --- a/utils/funcexp/func_year.cpp +++ b/utils/funcexp/func_year.cpp @@ -61,7 +61,7 @@ int64_t Func_year::getIntVal(rowgroup::Row& row, val = parm[0]->data()->getIntVal(row, isNull); return (unsigned)((val >> 48) & 0xffff); - // Time adds to now() and then gets value + // Time adds to now() and then gets value case CalpontSystemCatalog::TIME: aDateTime = static_cast(nowDatetime()); aTime = parm[0]->data()->getTimeIntVal(row, isNull); diff --git a/utils/funcexp/func_yearweek.cpp b/utils/funcexp/func_yearweek.cpp index 749491e11..e567440b4 100644 --- a/utils/funcexp/func_yearweek.cpp +++ b/utils/funcexp/func_yearweek.cpp @@ -78,7 +78,7 @@ int64_t Func_yearweek::getIntVal(rowgroup::Row& row, day = (uint32_t)((val >> 38) & 0x3f); break; - // Time adds to now() and then gets value + // Time adds to now() and then gets value case CalpontSystemCatalog::TIME: aDateTime = static_cast(nowDatetime()); aTime = parm[0]->data()->getTimeIntVal(row, isNull); diff --git a/utils/libmysql_client/libmysql_client.cpp b/utils/libmysql_client/libmysql_client.cpp index 9c67d3fa8..300df8a75 100644 --- a/utils/libmysql_client/libmysql_client.cpp +++ b/utils/libmysql_client/libmysql_client.cpp @@ -119,16 +119,17 @@ int LibMySQL::run(const char* query) void LibMySQL::handleMySqlError(const char* errStr, unsigned int errCode) { - ostringstream oss; - if (mysql->getErrno()) + ostringstream oss; + + if (getErrno()) { - oss << errStr << " (" << mysql->getErrno() << ")"; - oss << " (" << mysql->getErrorMsg() << ")"; + oss << errStr << " (" << getErrno() << ")"; + oss << " (" << getErrorMsg() << ")"; } else { - oss << errStr << " (" << errCode << ")"; - oss << " (unknown)"; + oss << errStr << " (" << errCode << ")"; + oss << " (unknown)"; } throw logging::IDBExcept(oss.str(), logging::ERR_CROSS_ENGINE_CONNECT); diff --git a/utils/libmysql_client/libmysql_client.h b/utils/libmysql_client/libmysql_client.h index 7d3c258e7..5720ffd73 100644 --- a/utils/libmysql_client/libmysql_client.h +++ b/utils/libmysql_client/libmysql_client.h @@ -71,8 +71,14 @@ public: { return fErrStr; } - unsigned int getErrno() { return mysql_errno(fCon); } - const char* getErrorMsg() { return mysql_error(fCon); } + unsigned int getErrno() + { + return mysql_errno(fCon); + } + const char* getErrorMsg() + { + return mysql_error(fCon); + } private: MYSQL* fCon; diff --git a/utils/rowgroup/rowaggregation.cpp b/utils/rowgroup/rowaggregation.cpp index bead74aff..1a28de089 100644 --- a/utils/rowgroup/rowaggregation.cpp +++ b/utils/rowgroup/rowaggregation.cpp @@ -2002,7 +2002,7 @@ void RowAggregation::doStatistics(const Row& rowIn, int64_t colIn, int64_t colOu fRow.setLongDoubleField(fRow.getLongDoubleField(colAux + 1) + valIn * valIn, colAux + 1); } -void RowAggregation::doUDAF(const Row& rowIn, int64_t colIn, int64_t colOut, +void RowAggregation::doUDAF(const Row& rowIn, int64_t colIn, int64_t colOut, int64_t colAux, uint64_t& funcColsIdx) { uint32_t paramCount = fRGContext.getParameterCount(); @@ -2012,6 +2012,7 @@ void RowAggregation::doUDAF(const Row& rowIn, int64_t colIn, int64_t colOut, ConstantColumn* cc; bool bIsNull = false; execplan::CalpontSystemCatalog::ColDataType colDataType; + for (uint32_t i = 0; i < paramCount; ++i) { // If UDAF_IGNORE_NULLS is on, bIsNull gets set the first time @@ -2022,6 +2023,7 @@ void RowAggregation::doUDAF(const Row& rowIn, int64_t colIn, int64_t colOut, ++funcColsIdx; continue; } + SP_ROWAGG_FUNC_t pFunctionCol = fFunctionCols[funcColsIdx]; mcsv1sdk::ColumnDatum& datum = valsIn[i]; // Turn on NULL flags @@ -2030,13 +2032,14 @@ void RowAggregation::doUDAF(const Row& rowIn, int64_t colIn, int64_t colOut, // If this particular parameter is a constant, then we need // to acces the constant value rather than a row value. cc = NULL; + if (pFunctionCol->fpConstCol) { cc = dynamic_cast(pFunctionCol->fpConstCol.get()); } if ((cc && cc->type() == ConstantColumn::NULLDATA) - || (!cc && isNull(&fRowGroupIn, rowIn, colIn) == true)) + || (!cc && isNull(&fRowGroupIn, rowIn, colIn) == true)) { if (fRGContext.getRunFlag(mcsv1sdk::UDAF_IGNORE_NULLS)) { @@ -2044,6 +2047,7 @@ void RowAggregation::doUDAF(const Row& rowIn, int64_t colIn, int64_t colOut, ++funcColsIdx; continue; } + dataFlags[i] |= mcsv1sdk::PARAM_IS_NULL; } @@ -2055,6 +2059,7 @@ void RowAggregation::doUDAF(const Row& rowIn, int64_t colIn, int64_t colOut, { colDataType = fRowGroupIn.getColTypes()[colIn]; } + if (!(dataFlags[i] & mcsv1sdk::PARAM_IS_NULL)) { switch (colDataType) @@ -2066,6 +2071,7 @@ void RowAggregation::doUDAF(const Row& rowIn, int64_t colIn, int64_t colOut, case execplan::CalpontSystemCatalog::BIGINT: { datum.dataType = execplan::CalpontSystemCatalog::BIGINT; + if (cc) { datum.columnData = cc->getIntVal(const_cast(rowIn), bIsNull); @@ -2078,12 +2084,15 @@ void RowAggregation::doUDAF(const Row& rowIn, int64_t colIn, int64_t colOut, datum.scale = fRowGroupIn.getScale()[colIn]; datum.precision = fRowGroupIn.getPrecision()[colIn]; } + break; } + case execplan::CalpontSystemCatalog::DECIMAL: case execplan::CalpontSystemCatalog::UDECIMAL: { datum.dataType = colDataType; + if (cc) { datum.columnData = cc->getDecimalVal(const_cast(rowIn), bIsNull).value; @@ -2096,6 +2105,7 @@ void RowAggregation::doUDAF(const Row& rowIn, int64_t colIn, int64_t colOut, datum.scale = fRowGroupIn.getScale()[colIn]; datum.precision = fRowGroupIn.getPrecision()[colIn]; } + break; } @@ -2106,6 +2116,7 @@ void RowAggregation::doUDAF(const Row& rowIn, int64_t colIn, int64_t colOut, case execplan::CalpontSystemCatalog::UBIGINT: { datum.dataType = execplan::CalpontSystemCatalog::UBIGINT; + if (cc) { datum.columnData = cc->getUintVal(const_cast(rowIn), bIsNull); @@ -2114,6 +2125,7 @@ void RowAggregation::doUDAF(const Row& rowIn, int64_t colIn, int64_t colOut, { datum.columnData = rowIn.getUintField(colIn); } + break; } @@ -2121,6 +2133,7 @@ void RowAggregation::doUDAF(const Row& rowIn, int64_t colIn, int64_t colOut, case execplan::CalpontSystemCatalog::UDOUBLE: { datum.dataType = execplan::CalpontSystemCatalog::DOUBLE; + if (cc) { datum.columnData = cc->getDoubleVal(const_cast(rowIn), bIsNull); @@ -2129,6 +2142,7 @@ void RowAggregation::doUDAF(const Row& rowIn, int64_t colIn, int64_t colOut, { datum.columnData = rowIn.getDoubleField(colIn); } + break; } @@ -2136,6 +2150,7 @@ void RowAggregation::doUDAF(const Row& rowIn, int64_t colIn, int64_t colOut, case execplan::CalpontSystemCatalog::UFLOAT: { datum.dataType = execplan::CalpontSystemCatalog::FLOAT; + if (cc) { datum.columnData = cc->getFloatVal(const_cast(rowIn), bIsNull); @@ -2144,12 +2159,14 @@ void RowAggregation::doUDAF(const Row& rowIn, int64_t colIn, int64_t colOut, { datum.columnData = rowIn.getFloatField(colIn); } + break; } case execplan::CalpontSystemCatalog::DATE: { datum.dataType = execplan::CalpontSystemCatalog::UBIGINT; + if (cc) { datum.columnData = cc->getDateIntVal(const_cast(rowIn), bIsNull); @@ -2158,11 +2175,14 @@ void RowAggregation::doUDAF(const Row& rowIn, int64_t colIn, int64_t colOut, { datum.columnData = rowIn.getUintField(colIn); } + break; } + case execplan::CalpontSystemCatalog::DATETIME: { datum.dataType = execplan::CalpontSystemCatalog::UBIGINT; + if (cc) { datum.columnData = cc->getDatetimeIntVal(const_cast(rowIn), bIsNull); @@ -2171,12 +2191,14 @@ void RowAggregation::doUDAF(const Row& rowIn, int64_t colIn, int64_t colOut, { datum.columnData = rowIn.getUintField(colIn); } + break; } case execplan::CalpontSystemCatalog::TIME: { datum.dataType = execplan::CalpontSystemCatalog::BIGINT; + if (cc) { datum.columnData = cc->getTimeIntVal(const_cast(rowIn), bIsNull); @@ -2185,6 +2207,7 @@ void RowAggregation::doUDAF(const Row& rowIn, int64_t colIn, int64_t colOut, { datum.columnData = rowIn.getIntField(colIn); } + break; } @@ -2196,6 +2219,7 @@ void RowAggregation::doUDAF(const Row& rowIn, int64_t colIn, int64_t colOut, case execplan::CalpontSystemCatalog::BLOB: { datum.dataType = colDataType; + if (cc) { datum.columnData = cc->getStrVal(const_cast(rowIn), bIsNull); @@ -2204,6 +2228,7 @@ void RowAggregation::doUDAF(const Row& rowIn, int64_t colIn, int64_t colOut, { datum.columnData = rowIn.getStringField(colIn); } + break; } @@ -2221,8 +2246,8 @@ void RowAggregation::doUDAF(const Row& rowIn, int64_t colIn, int64_t colOut, // MCOL-1201: If there are multiple parameters, the next fFunctionCols // will have the column used. By incrementing the funcColsIdx (passed by // ref, we also increment the caller's index. - if (fFunctionCols.size() > funcColsIdx + 1 - && fFunctionCols[funcColsIdx+1]->fAggFunction == ROWAGG_MULTI_PARM) + if (fFunctionCols.size() > funcColsIdx + 1 + && fFunctionCols[funcColsIdx + 1]->fAggFunction == ROWAGG_MULTI_PARM) { ++funcColsIdx; SP_ROWAGG_FUNC_t pFunctionCol = fFunctionCols[funcColsIdx]; @@ -2718,6 +2743,7 @@ void RowAggregationUM::SetUDAFValue(static_any::any& valOut, int64_t colOut) std::string strOut; bool bSetSuccess = false; + switch (colDataType) { case execplan::CalpontSystemCatalog::BIT: @@ -2732,10 +2758,12 @@ void RowAggregationUM::SetUDAFValue(static_any::any& valOut, int64_t colOut) intOut = valOut.cast(); bSetSuccess = true; } + if (bSetSuccess) { fRow.setIntField<1>(intOut, colOut); } + break; case execplan::CalpontSystemCatalog::SMALLINT: @@ -2746,6 +2774,7 @@ void RowAggregationUM::SetUDAFValue(static_any::any& valOut, int64_t colOut) fRow.setIntField<2>(intOut, colOut); bSetSuccess = true; } + break; case execplan::CalpontSystemCatalog::INT: @@ -2759,10 +2788,12 @@ void RowAggregationUM::SetUDAFValue(static_any::any& valOut, int64_t colOut) intOut = valOut.cast(); bSetSuccess = true; } + if (bSetSuccess) { fRow.setIntField<4>(intOut, colOut); } + break; case execplan::CalpontSystemCatalog::BIGINT: @@ -2774,6 +2805,7 @@ void RowAggregationUM::SetUDAFValue(static_any::any& valOut, int64_t colOut) fRow.setIntField<8>(intOut, colOut); bSetSuccess = true; } + break; case execplan::CalpontSystemCatalog::UTINYINT: @@ -2783,6 +2815,7 @@ void RowAggregationUM::SetUDAFValue(static_any::any& valOut, int64_t colOut) fRow.setUintField<1>(uintOut, colOut); bSetSuccess = true; } + break; case execplan::CalpontSystemCatalog::USMALLINT: @@ -2793,6 +2826,7 @@ void RowAggregationUM::SetUDAFValue(static_any::any& valOut, int64_t colOut) fRow.setUintField<2>(uintOut, colOut); bSetSuccess = true; } + break; case execplan::CalpontSystemCatalog::UINT: @@ -2802,6 +2836,7 @@ void RowAggregationUM::SetUDAFValue(static_any::any& valOut, int64_t colOut) fRow.setUintField<4>(uintOut, colOut); bSetSuccess = true; } + break; case execplan::CalpontSystemCatalog::UBIGINT: @@ -2811,6 +2846,7 @@ void RowAggregationUM::SetUDAFValue(static_any::any& valOut, int64_t colOut) fRow.setUintField<8>(uintOut, colOut); bSetSuccess = true; } + break; case execplan::CalpontSystemCatalog::DATE: @@ -2821,6 +2857,7 @@ void RowAggregationUM::SetUDAFValue(static_any::any& valOut, int64_t colOut) fRow.setUintField<8>(uintOut, colOut); bSetSuccess = true; } + break; case execplan::CalpontSystemCatalog::FLOAT: @@ -2831,6 +2868,7 @@ void RowAggregationUM::SetUDAFValue(static_any::any& valOut, int64_t colOut) fRow.setFloatField(floatOut, colOut); bSetSuccess = true; } + break; case execplan::CalpontSystemCatalog::DOUBLE: @@ -2841,6 +2879,7 @@ void RowAggregationUM::SetUDAFValue(static_any::any& valOut, int64_t colOut) fRow.setDoubleField(doubleOut, colOut); bSetSuccess = true; } + break; case execplan::CalpontSystemCatalog::CHAR: @@ -2852,6 +2891,7 @@ void RowAggregationUM::SetUDAFValue(static_any::any& valOut, int64_t colOut) fRow.setStringField(strOut, colOut); bSetSuccess = true; } + break; case execplan::CalpontSystemCatalog::VARBINARY: @@ -2863,6 +2903,7 @@ void RowAggregationUM::SetUDAFValue(static_any::any& valOut, int64_t colOut) fRow.setVarBinaryField(strOut, colOut); bSetSuccess = true; } + break; default: @@ -2873,6 +2914,7 @@ void RowAggregationUM::SetUDAFValue(static_any::any& valOut, int64_t colOut) break; } } + if (!bSetSuccess) { SetUDAFAnyValue(valOut, colOut); @@ -3404,14 +3446,17 @@ void RowAggregationUM::doNullConstantAggregate(const ConstantAggData& aggData, u fRGContext.setInterrupted(true); throw logging::QueryDataExcept(fRGContext.getErrorMessage(), logging::aggregateFuncErr); } + #if 0 uint32_t dataFlags[fRGContext.getParameterCount()]; + for (uint32_t i = 0; i < fRGContext.getParameterCount(); ++i) { mcsv1sdk::ColumnDatum& datum = valsIn[i]; // Turn on NULL flags dataFlags[i] = 0; } + #endif // Turn the NULL and CONSTANT flags on. uint32_t flags[1]; @@ -4278,19 +4323,20 @@ void RowAggregationUMP2::doBitOp(const Row& rowIn, int64_t colIn, int64_t colOut // colAux(in) - Where the UDAF userdata resides // rowUDAF(in) - pointer to the RowUDAFFunctionCol for this UDAF instance //------------------------------------------------------------------------------ -void RowAggregationUMP2::doUDAF(const Row& rowIn, int64_t colIn, int64_t colOut, +void RowAggregationUMP2::doUDAF(const Row& rowIn, int64_t colIn, int64_t colOut, int64_t colAux, uint64_t& funcColsIdx) { static_any::any valOut; // Get the user data - boost::shared_ptr userDataIn = rowIn.getUserData(colIn+1); + boost::shared_ptr userDataIn = rowIn.getUserData(colIn + 1); // Unlike other aggregates, the data isn't in colIn, so testing it for NULL // there won't help. In case of NULL, userData will be NULL. uint32_t flags[1]; flags[0] = 0; + if (!userDataIn) { if (fRGContext.getRunFlag(mcsv1sdk::UDAF_IGNORE_NULLS)) diff --git a/utils/rowgroup/rowaggregation.h b/utils/rowgroup/rowaggregation.h index 14e4313cf..b593239cd 100644 --- a/utils/rowgroup/rowaggregation.h +++ b/utils/rowgroup/rowaggregation.h @@ -228,9 +228,9 @@ struct RowUDAFFunctionCol : public RowAggFunctionCol inputColIndex, outputColIndex, auxColIndex), bInterrupted(false) {} - RowUDAFFunctionCol(const RowUDAFFunctionCol& rhs) : - RowAggFunctionCol(ROWAGG_UDAF, ROWAGG_FUNCT_UNDEFINE, rhs.fInputColumnIndex, - rhs.fOutputColumnIndex, rhs.fAuxColumnIndex), + RowUDAFFunctionCol(const RowUDAFFunctionCol& rhs) : + RowAggFunctionCol(ROWAGG_UDAF, ROWAGG_FUNCT_UNDEFINE, rhs.fInputColumnIndex, + rhs.fOutputColumnIndex, rhs.fAuxColumnIndex), fUDAFContext(rhs.fUDAFContext), bInterrupted(false) {} @@ -249,6 +249,7 @@ inline void RowAggFunctionCol::serialize(messageqcpp::ByteStream& bs) const bs << (uint8_t)fAggFunction; bs << fInputColumnIndex; bs << fOutputColumnIndex; + if (fpConstCol) { bs << (uint8_t)1; @@ -258,7 +259,7 @@ inline void RowAggFunctionCol::serialize(messageqcpp::ByteStream& bs) const { bs << (uint8_t)0; } - + } inline void RowAggFunctionCol::deserialize(messageqcpp::ByteStream& bs) @@ -268,6 +269,7 @@ inline void RowAggFunctionCol::deserialize(messageqcpp::ByteStream& bs) bs >> fOutputColumnIndex; uint8_t t; bs >> t; + if (t) { fpConstCol.reset(new ConstantColumn); diff --git a/utils/threadpool/prioritythreadpool.cpp b/utils/threadpool/prioritythreadpool.cpp index c2326a78f..4c043ebbb 100644 --- a/utils/threadpool/prioritythreadpool.cpp +++ b/utils/threadpool/prioritythreadpool.cpp @@ -53,9 +53,9 @@ PriorityThreadPool::PriorityThreadPool(uint targetWeightPerRun, uint highThreads cout << "started " << highThreads << " high, " << midThreads << " med, " << lowThreads << " low.\n"; - defaultThreadCounts[HIGH] = threadCounts[HIGH] = highThreads; - defaultThreadCounts[MEDIUM] = threadCounts[MEDIUM] = midThreads; - defaultThreadCounts[LOW] = threadCounts[LOW] = lowThreads; + defaultThreadCounts[HIGH] = threadCounts[HIGH] = highThreads; + defaultThreadCounts[MEDIUM] = threadCounts[MEDIUM] = midThreads; + defaultThreadCounts[LOW] = threadCounts[LOW] = lowThreads; } PriorityThreadPool::~PriorityThreadPool() @@ -76,11 +76,13 @@ void PriorityThreadPool::addJob(const Job& job, bool useLock) threads.create_thread(ThreadHelper(this, HIGH)); threadCounts[HIGH]++; } + if (defaultThreadCounts[MEDIUM] != threadCounts[MEDIUM]) { threads.create_thread(ThreadHelper(this, MEDIUM)); threadCounts[MEDIUM]++; } + if (defaultThreadCounts[LOW] != threadCounts[LOW]) { threads.create_thread(ThreadHelper(this, LOW)); @@ -136,14 +138,15 @@ void PriorityThreadPool::threadFcn(const Priority preferredQueue) throw() try { - while (!_stop) { + while (!_stop) + { mutex::scoped_lock lk(mutex); queue = pickAQueue(preferredQueue); - if (jobQueues[queue].empty()) { - if (jobQueues[queue].empty()) - { + + if (jobQueues[queue].empty()) + { newJob.wait(lk); continue; } @@ -158,8 +161,8 @@ void PriorityThreadPool::threadFcn(const Priority preferredQueue) throw() // should leave some to the other threads while ((weight < weightPerRun) && (!jobQueues[queue].empty()) - && (runList.size() <= queueSize/2)) { - { + && (runList.size() <= queueSize / 2)) + { runList.push_back(jobQueues[queue].front()); jobQueues[queue].pop_front(); weight += runList.back().weight; @@ -169,25 +172,24 @@ void PriorityThreadPool::threadFcn(const Priority preferredQueue) throw() reschedule.resize(runList.size()); rescheduleCount = 0; - for (i = 0; i < runList.size() && !_stop; i++) { - { - try + + for (i = 0; i < runList.size() && !_stop; i++) { - reschedule[i] = false; - running = true; - reschedule[i] = (*(runList[i].functor))(); - running = false; - if (reschedule[i]) - rescheduleCount++; + reschedule[i] = false; + running = true; + reschedule[i] = (*(runList[i].functor))(); + running = false; + + if (reschedule[i]) + rescheduleCount++; } - { // no real work was done, prevent intensive busy waiting if (rescheduleCount == runList.size()) usleep(1000); - if (rescheduleCount > 0) { - { + if (rescheduleCount > 0) + { lk.lock(); for (i = 0; i < runList.size(); i++) @@ -205,7 +207,7 @@ void PriorityThreadPool::threadFcn(const Priority preferredQueue) throw() runList.clear(); } } - catch (std::exception &ex) + catch (std::exception& ex) { // Log the exception and exit this thread try @@ -224,6 +226,7 @@ void PriorityThreadPool::threadFcn(const Priority preferredQueue) throw() ml.logErrorMessage( message ); #endif + if (running) sendErrorMsg(runList[i].uniqueID, runList[i].stepID, runList[i].sock); } @@ -250,6 +253,7 @@ void PriorityThreadPool::threadFcn(const Priority preferredQueue) throw() ml.logErrorMessage( message ); #endif + if (running) sendErrorMsg(runList[i].uniqueID, runList[i].stepID, runList[i].sock); } @@ -261,17 +265,17 @@ void PriorityThreadPool::threadFcn(const Priority preferredQueue) throw() void PriorityThreadPool::sendErrorMsg(uint32_t id, uint32_t step, primitiveprocessor::SP_UM_IOSOCK sock) { - ISMPacketHeader ism; - PrimitiveHeader ph = {0}; + ISMPacketHeader ism; + PrimitiveHeader ph = {0}; - ism.Status = logging::primitiveServerErr; - ph.UniqueID = id; - ph.StepID = step; - ByteStream msg(sizeof(ISMPacketHeader) + sizeof(PrimitiveHeader)); - msg.append((uint8_t *) &ism, sizeof(ism)); - msg.append((uint8_t *) &ph, sizeof(ph)); + ism.Status = logging::primitiveServerErr; + ph.UniqueID = id; + ph.StepID = step; + ByteStream msg(sizeof(ISMPacketHeader) + sizeof(PrimitiveHeader)); + msg.append((uint8_t*) &ism, sizeof(ism)); + msg.append((uint8_t*) &ph, sizeof(ph)); - sock->write(msg); + sock->write(msg); } void PriorityThreadPool::stop() diff --git a/utils/udfsdk/avgx.cpp b/utils/udfsdk/avgx.cpp index 887a8418e..5af852967 100644 --- a/utils/udfsdk/avgx.cpp +++ b/utils/udfsdk/avgx.cpp @@ -161,9 +161,11 @@ mcsv1_UDAF::ReturnCode avgx::subEvaluate(mcsv1Context* context, const UserData* } struct avgx_data* outData = (struct avgx_data*)context->getUserData()->data; + struct avgx_data* inData = (struct avgx_data*)userDataIn->data; outData->sum += inData->sum; + outData->cnt += inData->cnt; return mcsv1_UDAF::SUCCESS; diff --git a/utils/udfsdk/mcsv1_udaf.cpp b/utils/udfsdk/mcsv1_udaf.cpp index ee08dcc07..b042d63f5 100644 --- a/utils/udfsdk/mcsv1_udaf.cpp +++ b/utils/udfsdk/mcsv1_udaf.cpp @@ -120,7 +120,7 @@ bool mcsv1Context::operator==(const mcsv1Context& c) const // We don't test the per row data fields. They don't determine // if it's the same Context. if (getName() != c.getName() - ||fRunFlags != c.fRunFlags + || fRunFlags != c.fRunFlags || fContextFlags != c.fContextFlags || fUserDataSize != c.fUserDataSize || fResultType != c.fResultType diff --git a/utils/udfsdk/regr_avgx.cpp b/utils/udfsdk/regr_avgx.cpp index aec4f361f..e99871f97 100644 --- a/utils/udfsdk/regr_avgx.cpp +++ b/utils/udfsdk/regr_avgx.cpp @@ -82,6 +82,7 @@ mcsv1_UDAF::ReturnCode regr_avgx::nextValue(mcsv1Context* context, ColumnDatum* { return mcsv1_UDAF::SUCCESS; // Ought not happen when UDAF_IGNORE_NULLS is on. } + if (valIn_x.empty() || valIn_y.empty()) // Usually empty if NULL. Probably redundant { return mcsv1_UDAF::SUCCESS; // Ought not happen when UDAF_IGNORE_NULLS is on. @@ -162,9 +163,11 @@ mcsv1_UDAF::ReturnCode regr_avgx::subEvaluate(mcsv1Context* context, const UserD } struct regr_avgx_data* outData = (struct regr_avgx_data*)context->getUserData()->data; + struct regr_avgx_data* inData = (struct regr_avgx_data*)userDataIn->data; outData->sum += inData->sum; + outData->cnt += inData->cnt; return mcsv1_UDAF::SUCCESS; @@ -182,6 +185,7 @@ mcsv1_UDAF::ReturnCode regr_avgx::evaluate(mcsv1Context* context, static_any::an { valOut = data->sum / (double)data->cnt; } + return mcsv1_UDAF::SUCCESS; } diff --git a/utils/udfsdk/udfmysql.cpp b/utils/udfsdk/udfmysql.cpp index dc0277ccc..b0b2ebb9c 100644 --- a/utils/udfsdk/udfmysql.cpp +++ b/utils/udfsdk/udfmysql.cpp @@ -498,159 +498,163 @@ extern "C" */ struct regr_avgx_data { - double sumx; - int64_t cnt; + double sumx; + int64_t cnt; }; - - #ifdef _MSC_VER + +#ifdef _MSC_VER __declspec(dllexport) - #endif +#endif my_bool regr_avgx_init(UDF_INIT* initid, UDF_ARGS* args, char* message) { - struct regr_avgx_data* data; - if (args->arg_count != 2) - { - strcpy(message,"regr_avgx() requires two arguments"); - return 1; - } + struct regr_avgx_data* data; - if (!(data = (struct regr_avgx_data*) malloc(sizeof(struct regr_avgx_data)))) - { - strmov(message,"Couldn't allocate memory"); - return 1; - } - data->sumx = 0; + if (args->arg_count != 2) + { + strcpy(message, "regr_avgx() requires two arguments"); + return 1; + } + + if (!(data = (struct regr_avgx_data*) malloc(sizeof(struct regr_avgx_data)))) + { + strmov(message, "Couldn't allocate memory"); + return 1; + } + + data->sumx = 0; data->cnt = 0; - initid->ptr = (char*)data; - return 0; + initid->ptr = (char*)data; + return 0; } - #ifdef _MSC_VER +#ifdef _MSC_VER __declspec(dllexport) - #endif +#endif void regr_avgx_deinit(UDF_INIT* initid) { - free(initid->ptr); - } + free(initid->ptr); + } - #ifdef _MSC_VER +#ifdef _MSC_VER __declspec(dllexport) - #endif +#endif void regr_avgx_clear(UDF_INIT* initid, char* is_null __attribute__((unused)), - char* message __attribute__((unused))) + char* message __attribute__((unused))) { - struct regr_avgx_data* data = (struct regr_avgx_data*)initid->ptr; - data->sumx = 0; + struct regr_avgx_data* data = (struct regr_avgx_data*)initid->ptr; + data->sumx = 0; data->cnt = 0; } - #ifdef _MSC_VER +#ifdef _MSC_VER __declspec(dllexport) - #endif +#endif void regr_avgx_add(UDF_INIT* initid, UDF_ARGS* args, - char* is_null, - char* message __attribute__((unused))) + char* is_null, + char* message __attribute__((unused))) { // TODO test for NULL in x and y - struct regr_avgx_data* data = (struct regr_avgx_data*)initid->ptr; - double xval = cvtArgToDouble(args->arg_type[1], args->args[0]); + struct regr_avgx_data* data = (struct regr_avgx_data*)initid->ptr; + double xval = cvtArgToDouble(args->arg_type[1], args->args[0]); ++data->cnt; - data->sumx += xval; + data->sumx += xval; } - #ifdef _MSC_VER +#ifdef _MSC_VER __declspec(dllexport) - #endif +#endif long long regr_avgx(UDF_INIT* initid, UDF_ARGS* args __attribute__((unused)), - char* is_null, char* error __attribute__((unused))) + char* is_null, char* error __attribute__((unused))) { - struct regr_avgx_data* data = (struct regr_avgx_data*)initid->ptr; - return data->sumx / data->cnt; + struct regr_avgx_data* data = (struct regr_avgx_data*)initid->ptr; + return data->sumx / data->cnt; } //======================================================================= /** - * avgx connector stub. Exactly the same functionality as the - * built in avg() function. Use to test the performance of the - * API + * avgx connector stub. Exactly the same functionality as the + * built in avg() function. Use to test the performance of the + * API */ struct avgx_data { - double sumx; - int64_t cnt; + double sumx; + int64_t cnt; }; - - #ifdef _MSC_VER + +#ifdef _MSC_VER __declspec(dllexport) - #endif +#endif my_bool avgx_init(UDF_INIT* initid, UDF_ARGS* args, char* message) { - struct avgx_data* data; - if (args->arg_count != 1) - { - strcpy(message,"avgx() requires one argument"); - return 1; - } + struct avgx_data* data; - if (!(data = (struct avgx_data*) malloc(sizeof(struct avgx_data)))) - { - strmov(message,"Couldn't allocate memory"); - return 1; - } - data->sumx = 0; + if (args->arg_count != 1) + { + strcpy(message, "avgx() requires one argument"); + return 1; + } + + if (!(data = (struct avgx_data*) malloc(sizeof(struct avgx_data)))) + { + strmov(message, "Couldn't allocate memory"); + return 1; + } + + data->sumx = 0; data->cnt = 0; - initid->ptr = (char*)data; - return 0; + initid->ptr = (char*)data; + return 0; } - #ifdef _MSC_VER +#ifdef _MSC_VER __declspec(dllexport) - #endif +#endif void avgx_deinit(UDF_INIT* initid) { - free(initid->ptr); - } + free(initid->ptr); + } - #ifdef _MSC_VER +#ifdef _MSC_VER __declspec(dllexport) - #endif +#endif void avgx_clear(UDF_INIT* initid, char* is_null __attribute__((unused)), - char* message __attribute__((unused))) + char* message __attribute__((unused))) { - struct avgx_data* data = (struct avgx_data*)initid->ptr; - data->sumx = 0; + struct avgx_data* data = (struct avgx_data*)initid->ptr; + data->sumx = 0; data->cnt = 0; } - #ifdef _MSC_VER +#ifdef _MSC_VER __declspec(dllexport) - #endif +#endif void avgx_add(UDF_INIT* initid, UDF_ARGS* args, - char* is_null, - char* message __attribute__((unused))) + char* is_null, + char* message __attribute__((unused))) { // TODO test for NULL in x and y - struct avgx_data* data = (struct avgx_data*)initid->ptr; - double xval = cvtArgToDouble(args->arg_type[1], args->args[0]); + struct avgx_data* data = (struct avgx_data*)initid->ptr; + double xval = cvtArgToDouble(args->arg_type[1], args->args[0]); ++data->cnt; - data->sumx += xval; + data->sumx += xval; } - #ifdef _MSC_VER +#ifdef _MSC_VER __declspec(dllexport) - #endif +#endif long long avgx(UDF_INIT* initid, UDF_ARGS* args __attribute__((unused)), - char* is_null, char* error __attribute__((unused))) + char* is_null, char* error __attribute__((unused))) { - struct avgx_data* data = (struct avgx_data*)initid->ptr; - return data->sumx / data->cnt; + struct avgx_data* data = (struct avgx_data*)initid->ptr; + return data->sumx / data->cnt; } } // vim:ts=4 sw=4: diff --git a/utils/windowfunction/wf_udaf.cpp b/utils/windowfunction/wf_udaf.cpp index ee48360f1..79ed61b52 100644 --- a/utils/windowfunction/wf_udaf.cpp +++ b/utils/windowfunction/wf_udaf.cpp @@ -152,7 +152,7 @@ void WF_udaf::parseParms(const std::vector& parms) { bRespectNulls = true; // The last parms: respect null | ignore null - ConstantColumn* cc = dynamic_cast(parms[parms.size()-1].get()); + ConstantColumn* cc = dynamic_cast(parms[parms.size() - 1].get()); idbassert(cc != NULL); bool isNull = false; // dummy, harded coded bRespectNulls = (cc->getIntVal(fRow, isNull) > 0); @@ -175,9 +175,10 @@ bool WF_udaf::dropValues(int64_t b, int64_t e) // Put the parameter metadata (type, scale, precision) into valsIn mcsv1sdk::ColumnDatum valsIn[getContext().getParameterCount()]; + for (uint32_t i = 0; i < getContext().getParameterCount(); ++i) { - uint64_t colIn = fFieldIndex[i+1]; + uint64_t colIn = fFieldIndex[i + 1]; mcsv1sdk::ColumnDatum& datum = valsIn[i]; datum.dataType = fRow.getColType(colIn); datum.scale = fRow.getScale(colIn); @@ -196,9 +197,10 @@ bool WF_udaf::dropValues(int64_t b, int64_t e) for (uint32_t k = 0; k < getContext().getParameterCount(); ++k) { - uint64_t colIn = fFieldIndex[k+1]; + uint64_t colIn = fFieldIndex[k + 1]; mcsv1sdk::ColumnDatum& datum = valsIn[k]; flags[k] = 0; + if (fRow.isNullValue(colIn) == true) { if (!bRespectNulls) @@ -228,6 +230,7 @@ bool WF_udaf::dropValues(int64_t b, int64_t e) datum.columnData = valIn; } + if (bHasNull) { continue; @@ -452,6 +455,7 @@ void WF_udaf::operator()(int64_t b, int64_t e, int64_t c) mcsv1sdk::mcsv1_UDAF::ReturnCode rc; uint64_t colOut = fFieldIndex[0]; bool isNull = false; + if ((fFrameUnit == WF__FRAME_ROWS) || (fPrev == -1) || (!fPeer->operator()(getPointer(fRowData->at(c)), getPointer(fRowData->at(fPrev))))) @@ -469,10 +473,12 @@ void WF_udaf::operator()(int64_t b, int64_t e, int64_t c) // Put the parameter metadata (type, scale, precision) into valsIn mcsv1sdk::ColumnDatum valsIn[getContext().getParameterCount()]; ConstantColumn* cc = NULL; + for (uint32_t i = 0; i < getContext().getParameterCount(); ++i) { mcsv1sdk::ColumnDatum& datum = valsIn[i]; cc = static_cast(fConstantParms[i].get()); + if (cc) { datum.dataType = cc->resultType().colDataType; @@ -481,7 +487,7 @@ void WF_udaf::operator()(int64_t b, int64_t e, int64_t c) } else { - uint64_t colIn = fFieldIndex[i+1]; + uint64_t colIn = fFieldIndex[i + 1]; datum.dataType = fRow.getColType(colIn); datum.scale = fRow.getScale(colIn); datum.precision = fRow.getPrecision(colIn); @@ -494,6 +500,7 @@ void WF_udaf::operator()(int64_t b, int64_t e, int64_t c) getContext().clearContextFlag(mcsv1sdk::CONTEXT_HAS_CURRENT_ROW); bool bHasNull = false; + for (int64_t i = b; i <= e; i++) { if (i % 1000 == 0 && fStep->cancelled()) @@ -504,16 +511,18 @@ void WF_udaf::operator()(int64_t b, int64_t e, int64_t c) // NULL flags uint32_t flags[getContext().getParameterCount()]; bHasNull = false; + for (uint32_t k = 0; k < getContext().getParameterCount(); ++k) { cc = static_cast(fConstantParms[k].get()); - uint64_t colIn = fFieldIndex[k+1]; + uint64_t colIn = fFieldIndex[k + 1]; mcsv1sdk::ColumnDatum& datum = valsIn[k]; // Turn on Null flags or skip based on respect nulls flags[k] = 0; + if ((!cc && fRow.isNullValue(colIn) == true) - || (cc && cc->type() == ConstantColumn::NULLDATA)) + || (cc && cc->type() == ConstantColumn::NULLDATA)) { if (!bRespectNulls) { @@ -535,6 +544,7 @@ void WF_udaf::operator()(int64_t b, int64_t e, int64_t c) case CalpontSystemCatalog::BIGINT: { int64_t valIn; + if (cc) { valIn = cc->getIntVal(fRow, isNull); @@ -543,6 +553,7 @@ void WF_udaf::operator()(int64_t b, int64_t e, int64_t c) { getValue(colIn, valIn); } + // Check for distinct, if turned on. // Currently, distinct only works on the first parameter. if (k == 0) @@ -555,6 +566,7 @@ void WF_udaf::operator()(int64_t b, int64_t e, int64_t c) if (fDistinct) fDistinctSet.insert(valIn); } + datum.columnData = valIn; break; } @@ -563,6 +575,7 @@ void WF_udaf::operator()(int64_t b, int64_t e, int64_t c) case CalpontSystemCatalog::UDECIMAL: { int64_t valIn; + if (cc) { valIn = cc->getDecimalVal(fRow, isNull).value; @@ -571,6 +584,7 @@ void WF_udaf::operator()(int64_t b, int64_t e, int64_t c) { getValue(colIn, valIn); } + // Check for distinct, if turned on. // Currently, distinct only works on the first parameter. if (k == 0) @@ -583,6 +597,7 @@ void WF_udaf::operator()(int64_t b, int64_t e, int64_t c) if (fDistinct) fDistinctSet.insert(valIn); } + datum.columnData = valIn; break; } @@ -594,6 +609,7 @@ void WF_udaf::operator()(int64_t b, int64_t e, int64_t c) case CalpontSystemCatalog::UBIGINT: { uint64_t valIn; + if (cc) { valIn = cc->getUintVal(fRow, isNull); @@ -602,6 +618,7 @@ void WF_udaf::operator()(int64_t b, int64_t e, int64_t c) { getValue(colIn, valIn); } + // Check for distinct, if turned on. // Currently, distinct only works on the first parameter. if (k == 0) @@ -614,6 +631,7 @@ void WF_udaf::operator()(int64_t b, int64_t e, int64_t c) if (fDistinct) fDistinctSet.insert(valIn); } + datum.columnData = valIn; break; } @@ -622,6 +640,7 @@ void WF_udaf::operator()(int64_t b, int64_t e, int64_t c) case CalpontSystemCatalog::UDOUBLE: { double valIn; + if (cc) { valIn = cc->getDoubleVal(fRow, isNull); @@ -630,6 +649,7 @@ void WF_udaf::operator()(int64_t b, int64_t e, int64_t c) { getValue(colIn, valIn); } + // Check for distinct, if turned on. // Currently, distinct only works on the first parameter. if (k == 0) @@ -642,6 +662,7 @@ void WF_udaf::operator()(int64_t b, int64_t e, int64_t c) if (fDistinct) fDistinctSet.insert(valIn); } + datum.columnData = valIn; break; } @@ -650,6 +671,7 @@ void WF_udaf::operator()(int64_t b, int64_t e, int64_t c) case CalpontSystemCatalog::UFLOAT: { float valIn; + if (cc) { valIn = cc->getFloatVal(fRow, isNull); @@ -658,6 +680,7 @@ void WF_udaf::operator()(int64_t b, int64_t e, int64_t c) { getValue(colIn, valIn); } + // Check for distinct, if turned on. // Currently, distinct only works on the first parameter. if (k == 0) @@ -670,6 +693,7 @@ void WF_udaf::operator()(int64_t b, int64_t e, int64_t c) if (fDistinct) fDistinctSet.insert(valIn); } + datum.columnData = valIn; break; } @@ -681,6 +705,7 @@ void WF_udaf::operator()(int64_t b, int64_t e, int64_t c) case CalpontSystemCatalog::BLOB: { string valIn; + if (cc) { valIn = cc->getStrVal(fRow, isNull); @@ -689,6 +714,7 @@ void WF_udaf::operator()(int64_t b, int64_t e, int64_t c) { getValue(colIn, valIn); } + // Check for distinct, if turned on. // Currently, distinct only works on the first parameter. if (k == 0) @@ -701,6 +727,7 @@ void WF_udaf::operator()(int64_t b, int64_t e, int64_t c) if (fDistinct) fDistinctSet.insert(valIn); } + datum.columnData = valIn; break; } @@ -717,13 +744,15 @@ void WF_udaf::operator()(int64_t b, int64_t e, int64_t c) } } } + // Skip if any value is NULL and respect nulls is off. if (bHasNull) { continue; } + getContext().setDataFlags(flags); - + rc = getContext().getFunction()->nextValue(&getContext(), valsIn); if (rc == mcsv1sdk::mcsv1_UDAF::ERROR) diff --git a/utils/windowfunction/wf_udaf.h b/utils/windowfunction/wf_udaf.h index fc3f9006d..ef2ca5853 100644 --- a/utils/windowfunction/wf_udaf.h +++ b/utils/windowfunction/wf_udaf.h @@ -93,7 +93,7 @@ protected: bool bRespectNulls; // respect null | ignore null bool bHasDropValue; // Set to false when we discover the UDAnF doesn't implement dropValue. // To hold distinct values - std::tr1::unordered_set fDistinctSet; + std::tr1::unordered_set fDistinctSet; static_any::any fValOut; // The return value public: diff --git a/utils/windowfunction/windowfunctiontype.cpp b/utils/windowfunction/windowfunctiontype.cpp index f5598a7e5..dfceb6364 100644 --- a/utils/windowfunction/windowfunctiontype.cpp +++ b/utils/windowfunction/windowfunctiontype.cpp @@ -645,6 +645,7 @@ void WindowFunctionType::constParms(const std::vector& functionParms) for (size_t i = 0; i < functionParms.size(); ++i) { ConstantColumn* cc = dynamic_cast(functionParms[i].get()); + if (cc) { fConstantParms.push_back(functionParms[i]); diff --git a/utils/windowfunction/windowfunctiontype.h b/utils/windowfunction/windowfunctiontype.h index efa1c548a..5c2f43db0 100644 --- a/utils/windowfunction/windowfunctiontype.h +++ b/utils/windowfunction/windowfunctiontype.h @@ -199,7 +199,7 @@ public: } void constParms(const std::vector& functionParms); - + static boost::shared_ptr makeWindowFunction(const std::string&, int ct, WindowFunctionColumn* wc); protected: diff --git a/writeengine/wrapper/writeengine.cpp b/writeengine/wrapper/writeengine.cpp index 754a7b464..923871ef9 100644 --- a/writeengine/wrapper/writeengine.cpp +++ b/writeengine/wrapper/writeengine.cpp @@ -2247,31 +2247,32 @@ int WriteEngineWrapper::insertColumnRecsBinary(const TxnID& txnid, { oldHwm = it->hwm; - // save hwm for the old extent - colWidth = colStructList[i].colWidth; - succFlag = colOp->calculateRowId(lastRid, BYTE_PER_BLOCK / colWidth, colWidth, curFbo, curBio); + // save hwm for the old extent + colWidth = colStructList[i].colWidth; + succFlag = colOp->calculateRowId(lastRid, BYTE_PER_BLOCK / colWidth, colWidth, curFbo, curBio); - //cout << "insertcolumnrec oid:rid:fbo:oldhwm = " << colStructList[i].dataOid << ":" << lastRid << ":" << curFbo << ":" << oldHwm << endl; - if (succFlag) - { - if ((HWM)curFbo >= oldHwm) + //cout << "insertcolumnrec oid:rid:fbo:oldhwm = " << colStructList[i].dataOid << ":" << lastRid << ":" << curFbo << ":" << oldHwm << endl; + if (succFlag) { - it->hwm = (HWM)curFbo; - } + if ((HWM)curFbo >= oldHwm) + { + it->hwm = (HWM)curFbo; + } - //@Bug 4947. set current to false for old extent. - if (newExtent) - { - it->current = false; - } + //@Bug 4947. set current to false for old extent. + if (newExtent) + { + it->current = false; + } - //cout << "updated old ext info for oid " << colStructList[i].dataOid << " dbroot:part:seg:hwm:current = " - //<< it->dbRoot<<":"<partNum<<":"<segNum<<":"<hwm<<":"<< it->current<< " and newExtent is " << newExtent << endl; - } - else - return ERR_INVALID_PARAM; + //cout << "updated old ext info for oid " << colStructList[i].dataOid << " dbroot:part:seg:hwm:current = " + //<< it->dbRoot<<":"<partNum<<":"<segNum<<":"<hwm<<":"<< it->current<< " and newExtent is " << newExtent << endl; + } + else + return ERR_INVALID_PARAM; } + //update hwm for the new extent if (newExtent) { @@ -2285,7 +2286,8 @@ int WriteEngineWrapper::insertColumnRecsBinary(const TxnID& txnid, it++; } - colWidth = newColStructList[i].colWidth; + + colWidth = newColStructList[i].colWidth; succFlag = colOp->calculateRowId(lastRidNew, BYTE_PER_BLOCK / colWidth, colWidth, curFbo, curBio); if (succFlag) @@ -2356,29 +2358,29 @@ int WriteEngineWrapper::insertColumnRecsBinary(const TxnID& txnid, curFbo)); } } - else - return ERR_INVALID_PARAM; - } + else + return ERR_INVALID_PARAM; } + } - // If we create a new extent for this batch - for (unsigned i = 0; i < newColStructList.size(); i++) + // If we create a new extent for this batch + for (unsigned i = 0; i < newColStructList.size(); i++) + { + colOp = m_colOp[op(newColStructList[i].fCompressionType)]; + width = newColStructList[i].colWidth; + successFlag = colOp->calculateRowId(lastRidNew, BYTE_PER_BLOCK / width, width, curFbo, curBio); + + if (successFlag) { - colOp = m_colOp[op(newColStructList[i].fCompressionType)]; - width = newColStructList[i].colWidth; - successFlag = colOp->calculateRowId(lastRidNew, BYTE_PER_BLOCK / width, width, curFbo, curBio); - - if (successFlag) + if (curFbo != lastFbo) { - if (curFbo != lastFbo) - { - RETURN_ON_ERROR(AddLBIDtoList(txnid, - lbids, - colDataTypes, - newColStructList[i], - curFbo)); - } + RETURN_ON_ERROR(AddLBIDtoList(txnid, + lbids, + colDataTypes, + newColStructList[i], + curFbo)); } + } else return ERR_INVALID_PARAM; } @@ -5136,7 +5138,7 @@ int WriteEngineWrapper::writeColumnRecBinary(const TxnID& txnid, bool versioning) { int rc = 0; - void* valArray = NULL; + void* valArray = NULL; string segFile; Column curCol; ColStructList::size_type totalColumn; @@ -5167,23 +5169,25 @@ int WriteEngineWrapper::writeColumnRecBinary(const TxnID& txnid, return rc; TableMetaData* aTbaleMetaData = TableMetaData::makeTableMetaData(tableOid); + if (totalRow1) { valArray = malloc(sizeof(uint64_t) * totalRow1); + for (i = 0; i < totalColumn; i++) { - //@Bug 2205 Check if all rows go to the new extent + //@Bug 2205 Check if all rows go to the new extent //Write the first batch - RID * firstPart = rowIdArray; + RID* firstPart = rowIdArray; ColumnOp* colOp = m_colOp[op(colStructList[i].fCompressionType)]; // set params colOp->initColumn(curCol); // need to pass real dbRoot, partition, and segment to setColParam colOp->setColParam(curCol, 0, colStructList[i].colWidth, - colStructList[i].colDataType, colStructList[i].colType, colStructList[i].dataOid, - colStructList[i].fCompressionType, colStructList[i].fColDbRoot, - colStructList[i].fColPartition, colStructList[i].fColSegment); + colStructList[i].colDataType, colStructList[i].colType, colStructList[i].dataOid, + colStructList[i].fCompressionType, colStructList[i].fColDbRoot, + colStructList[i].fColPartition, colStructList[i].fColSegment); ColExtsInfo aColExtsInfo = aTbaleMetaData->getColExtsInfo(colStructList[i].dataOid); ColExtsInfo::iterator it = aColExtsInfo.begin(); @@ -5199,7 +5203,7 @@ int WriteEngineWrapper::writeColumnRecBinary(const TxnID& txnid, if (it == aColExtsInfo.end()) //add this one to the list { ColExtInfo aExt; - aExt.dbRoot =colStructList[i].fColDbRoot; + aExt.dbRoot = colStructList[i].fColDbRoot; aExt.partNum = colStructList[i].fColPartition; aExt.segNum = colStructList[i].fColSegment; aExt.compType = colStructList[i].fCompressionType; @@ -5210,18 +5214,18 @@ int WriteEngineWrapper::writeColumnRecBinary(const TxnID& txnid, rc = colOp->openColumnFile(curCol, segFile, useTmpSuffix, IO_BUFF_SIZE); // @bug 5572 HDFS tmp file if (rc != NO_ERROR) - break; + break; // handling versioning vector rangeList; if (versioning) { - rc = processVersionBuffer(curCol.dataFile.pFile, txnid, colStructList[i], - colStructList[i].colWidth, totalRow1, firstPart, rangeList); - if (rc != NO_ERROR) { - if (rc != NO_ERROR) - { + rc = processVersionBuffer(curCol.dataFile.pFile, txnid, colStructList[i], + colStructList[i].colWidth, totalRow1, firstPart, rangeList); + + if (rc != NO_ERROR) + { if (colStructList[i].fCompressionType == 0) { curCol.dataFile.pFile->flush(); @@ -5241,39 +5245,39 @@ int WriteEngineWrapper::writeColumnRecBinary(const TxnID& txnid, for (size_t j = 0; j < totalRow1; j++) { - uint64_t curValue = colValueList[((totalRow1 + totalRow2)*i) + j]; + uint64_t curValue = colValueList[((totalRow1 + totalRow2) * i) + j]; switch (colStructList[i].colType) { - case WriteEngine::WR_VARBINARY : // treat same as char for now - case WriteEngine::WR_CHAR: - case WriteEngine::WR_BLOB: - case WriteEngine::WR_TEXT: + case WriteEngine::WR_VARBINARY : // treat same as char for now + case WriteEngine::WR_CHAR: + case WriteEngine::WR_BLOB: + case WriteEngine::WR_TEXT: ((uint64_t*)valArray)[j] = curValue; break; - case WriteEngine::WR_INT: - case WriteEngine::WR_UINT: - case WriteEngine::WR_FLOAT: + case WriteEngine::WR_INT: + case WriteEngine::WR_UINT: + case WriteEngine::WR_FLOAT: tmp32 = curValue; ((uint32_t*)valArray)[j] = tmp32; break; - case WriteEngine::WR_ULONGLONG: - case WriteEngine::WR_LONGLONG: - case WriteEngine::WR_DOUBLE: - case WriteEngine::WR_TOKEN: + case WriteEngine::WR_ULONGLONG: + case WriteEngine::WR_LONGLONG: + case WriteEngine::WR_DOUBLE: + case WriteEngine::WR_TOKEN: ((uint64_t*)valArray)[j] = curValue; break; - case WriteEngine::WR_BYTE: - case WriteEngine::WR_UBYTE: + case WriteEngine::WR_BYTE: + case WriteEngine::WR_UBYTE: tmp8 = curValue; ((uint8_t*)valArray)[j] = tmp8; break; - case WriteEngine::WR_SHORT: - case WriteEngine::WR_USHORT: + case WriteEngine::WR_SHORT: + case WriteEngine::WR_USHORT: tmp16 = curValue; ((uint16_t*)valArray)[j] = tmp16; break; @@ -5282,11 +5286,11 @@ int WriteEngineWrapper::writeColumnRecBinary(const TxnID& txnid, #ifdef PROFILE - timer.start("writeRow "); + timer.start("writeRow "); #endif rc = colOp->writeRow(curCol, totalRow1, firstPart, valArray); #ifdef PROFILE - timer.stop("writeRow "); + timer.stop("writeRow "); #endif colOp->closeColumnFile(curCol); @@ -5295,7 +5299,7 @@ int WriteEngineWrapper::writeColumnRecBinary(const TxnID& txnid, // check error if (rc != NO_ERROR) - break; + break; } // end of for (i = 0