fix(joblist,QA): introduced extended SC ptrs collector to resolve the issue when ParseTree consist of ArithmCol-AggCol does not return SCs from AggCol.

dbcon/execplan/aggregatecolumn.cpp

@@ -83,6 +83,18 @@ void getAggCols(execplan::ParseTree* n, void* obj)
   }
 }
 
+const std::vector<SimpleColumn*>& getSCsFromRCForExtended(const SRCP& srcp)
+{
+  srcp->setSimpleColumnListExtended();
+  return srcp->simpleColumnListExtended();
+}
+
+const std::vector<SimpleColumn*>& getSCsFromRC(const SRCP& srcp)
+{
+  srcp->setSimpleColumnList();
+  return srcp->simpleColumnList();
+}
+
 /**
  * Constructors/Destructors
  */
@@ -129,22 +141,40 @@ AggregateColumn::AggregateColumn(const AggregateColumn& rhs, const uint32_t sess
 void AggregateColumn::setSimpleColumnList()
 {
   fSimpleColumnList.clear();
+  return setSimpleColumnList_(false);
+}
 
+void AggregateColumn::setSimpleColumnListExtended()
+{
+  fSimpleColumnListExtended.clear();
+  return setSimpleColumnList_(true);
+}
+
+void AggregateColumn::setSimpleColumnList_(const bool extractSCsfromAggCol)
+{
   for (const auto& parm : aggParms())
   {
     if (!parm)
       continue;
 
-    parm->setSimpleColumnList();
-    for (auto* sc : parm->simpleColumnList())
+    auto aggParmsAsSCVec = (extractSCsfromAggCol) ? getSCsFromRCForExtended(parm) : getSCsFromRC(parm);
+
+    for (auto* sc : aggParmsAsSCVec)
     {
       if (sc)
+      {
+        if (extractSCsfromAggCol)
+        {
+          fSimpleColumnListExtended.push_back(sc);
+        }
+        else
         {
           fSimpleColumnList.push_back(sc);
         }
       }
     }
   }
+}
 
 const string AggregateColumn::toString() const
 {

dbcon/execplan/aggregatecolumn.h

@@ -161,7 +161,9 @@ class AggregateColumn : public ReturnedColumn
     fAggParms = parms;
   }
 
+  void setSimpleColumnList_(const bool extractSCsfromAggCol);
   void setSimpleColumnList() override;
+  void setSimpleColumnListExtended() override;
 
   /** return a copy of this pointer
    *

dbcon/execplan/arithmeticcolumn.cpp

@@ -447,6 +447,12 @@ void ArithmeticColumn::setSimpleColumnList()
   fExpression->walk(getSimpleCols, &fSimpleColumnList);
 }
 
+void ArithmeticColumn::setSimpleColumnListExtended()
+{
+  fSimpleColumnListExtended.clear();
+  fExpression->walk(getSimpleColsExtended, &fSimpleColumnListExtended);
+}
+
 std::optional<CalpontSystemCatalog::TableAliasName> ArithmeticColumn::singleTable()
 {
   setSimpleColumnList();

dbcon/execplan/arithmeticcolumn.h

@@ -169,12 +169,10 @@ class ArithmeticColumn : public ReturnedColumn
 
   void setDerivedTable() override;
   void replaceRealCol(std::vector<SRCP>&) override;
-  const std::vector<SimpleColumn*>& simpleColumnList() const override
-  {
-    return fSimpleColumnList;
-  }
   void setSimpleColumnList() override;
 
+  void setSimpleColumnListExtended() override;
+
   /**
    * Return the table that the column arguments belong to.
    *

dbcon/execplan/constantfilter.cpp

@@ -331,4 +331,22 @@ void ConstantFilter::setSimpleColumnList()
   }
 }
 
+const std::vector<SimpleColumn*>& ConstantFilter::simpleColumnListExtended()
+{
+  return fSimpleColumnListExtended;
+}
+
+void ConstantFilter::setSimpleColumnListExtended()
+{
+  fSimpleColumnListExtended.clear();
+
+  for (uint32_t i = 0; i < fFilterList.size(); i++)
+  {
+    fFilterList[i]->setSimpleColumnListExtended();
+    fSimpleColumnListExtended.insert(fSimpleColumnListExtended.end(),
+                                     fFilterList[i]->simpleColumnListExtended().begin(),
+                                     fFilterList[i]->simpleColumnListExtended().end());
+  }
+}
+
 }  // namespace execplan

dbcon/execplan/constantfilter.h

@@ -202,8 +202,10 @@ class ConstantFilter : public Filter
 
   // get all simple columns involved in this column
   const std::vector<SimpleColumn*>& simpleColumnList();
+  const std::vector<SimpleColumn*>& simpleColumnListExtended();
   // walk through the constant filter operands to re-populate fSimpleColumnList
   void setSimpleColumnList();
+  void setSimpleColumnListExtended();
 
   // get all aggregate columns involved in this column
   const std::vector<AggregateColumn*>& aggColumnList() const
@@ -213,6 +215,7 @@ class ConstantFilter : public Filter
 
  private:
   std::vector<SimpleColumn*> fSimpleColumnList;
+  std::vector<SimpleColumn*> fSimpleColumnListExtended{};
   std::vector<AggregateColumn*> fAggColumnList;
   std::vector<WindowFunctionColumn*> fWindowFunctionColumnList;
 };

dbcon/execplan/functioncolumn.cpp

@@ -46,7 +46,6 @@ using namespace boost;
 #include "functor_str.h"
 using namespace funcexp;
 
-
 namespace execplan
 {
 /**
@@ -141,7 +140,8 @@ string FunctionColumn::toCppCode(IncludeSet& includes) const
 
   auto fFuncParmsInString = fData.substr(fFunctionName.size() + 1, fData.size() - fFunctionName.size() - 2);
 
-  ss << "FunctionColumn(" << std::quoted(fFunctionName) << ", " << std::quoted(fFuncParmsInString) << ", " << sessionID() << ")";
+  ss << "FunctionColumn(" << std::quoted(fFunctionName) << ", " << std::quoted(fFuncParmsInString) << ", "
+     << sessionID() << ")";
 
   return ss.str();
 }
@@ -521,6 +521,14 @@ void FunctionColumn::setSimpleColumnList()
     fFunctionParms[i]->walk(getSimpleCols, &fSimpleColumnList);
 }
 
+void FunctionColumn::setSimpleColumnListExtended()
+{
+  fSimpleColumnListExtended.clear();
+
+  for (uint i = 0; i < fFunctionParms.size(); i++)
+    fFunctionParms[i]->walk(getSimpleColsExtended, &fSimpleColumnListExtended);
+}
+
 std::optional<CalpontSystemCatalog::TableAliasName> FunctionColumn::singleTable()
 {
   setSimpleColumnList();

dbcon/execplan/functioncolumn.h

@@ -159,12 +159,9 @@ class FunctionColumn : public ReturnedColumn
   bool hasWindowFunc() override;
   void setDerivedTable() override;
   void replaceRealCol(std::vector<SRCP>&) override;
-  virtual const std::vector<SimpleColumn*>& simpleColumnList() const override
-  {
-    return fSimpleColumnList;
-  }
 
   void setSimpleColumnList() override;
+  void setSimpleColumnListExtended() override;
   /**
    * Return the tableAlias name of the table that the column arguments belong to.
    *

dbcon/execplan/returnedcolumn.cpp

@@ -260,4 +260,9 @@ void ReturnedColumn::setSimpleColumnList()
   fSimpleColumnList.clear();
 }
 
+void ReturnedColumn::setSimpleColumnListExtended()
+{
+  fSimpleColumnListExtended.clear();
+}
+
 }  // namespace execplan

dbcon/execplan/returnedcolumn.h

@@ -287,6 +287,11 @@ class ReturnedColumn : public TreeNode
     return fSimpleColumnList;
   }
 
+  virtual const std::vector<SimpleColumn*>& simpleColumnListExtended() const
+  {
+    return fSimpleColumnListExtended;
+  }
+
   /* @brief traverse this ReturnedColumn and re-populate fSimpleColumnList.
    *
    * @note all ReturnedColumns that may have simple column arguments added
@@ -294,6 +299,8 @@ class ReturnedColumn : public TreeNode
    */
   virtual void setSimpleColumnList();
 
+  virtual void setSimpleColumnListExtended();
+
   // get all aggregate column list in this expression
   const std::vector<AggregateColumn*>& aggColumnList() const
   {
@@ -342,7 +349,10 @@ class ReturnedColumn : public TreeNode
   uint64_t fOrderPos;    /// for order by and group by column
   uint64_t fColSource;   /// from which subquery
   int64_t fColPosition;  /// the column position in the source subquery
+  // The difference b/w the two vectors is that the second one is used to store SCs
+  // are are used by AggegateColumn.
   std::vector<SimpleColumn*> fSimpleColumnList;
+  std::vector<SimpleColumn*> fSimpleColumnListExtended{};
   std::vector<AggregateColumn*> fAggColumnList;
   std::vector<WindowFunctionColumn*> fWindowFunctionColumnList;
   bool fHasAggregate;  /// connector internal use. no need to serialize

dbcon/execplan/simplecolumn.cpp

@@ -91,6 +91,48 @@ void getSimpleCols(execplan::ParseTree* n, void* obj)
   }
 }
 
+void getSimpleColsExtended(execplan::ParseTree* n, void* obj)
+{
+  vector<SimpleColumn*>* list = reinterpret_cast<vector<SimpleColumn*>*>(obj);
+  TreeNode* tn = n->data();
+  SimpleColumn* sc = dynamic_cast<SimpleColumn*>(tn);
+  FunctionColumn* fc = dynamic_cast<FunctionColumn*>(tn);
+  ArithmeticColumn* ac = dynamic_cast<ArithmeticColumn*>(tn);
+  SimpleFilter* sf = dynamic_cast<SimpleFilter*>(tn);
+  ConstantFilter* cf = dynamic_cast<ConstantFilter*>(tn);
+  AggregateColumn* agc = dynamic_cast<AggregateColumn*>(tn);
+
+  if (sc)
+  {
+    list->push_back(sc);
+  }
+  else if (fc)
+  {
+    fc->setSimpleColumnListExtended();
+    list->insert(list->end(), fc->simpleColumnListExtended().begin(), fc->simpleColumnListExtended().end());
+  }
+  else if (ac)
+  {
+    ac->setSimpleColumnListExtended();
+    list->insert(list->end(), ac->simpleColumnListExtended().begin(), ac->simpleColumnListExtended().end());
+  }
+  else if (agc)
+  {
+    agc->setSimpleColumnListExtended();
+    list->insert(list->end(), agc->simpleColumnListExtended().begin(), agc->simpleColumnListExtended().end());
+  }
+  else if (sf)
+  {
+    sf->setSimpleColumnListExtended();
+    list->insert(list->end(), sf->simpleColumnListExtended().begin(), sf->simpleColumnListExtended().end());
+  }
+  else if (cf)
+  {
+    cf->setSimpleColumnListExtended();
+    list->insert(list->end(), cf->simpleColumnListExtended().begin(), cf->simpleColumnListExtended().end());
+  }
+}
+
 ParseTree* replaceRefCol(ParseTree*& n, CalpontSelectExecutionPlan::ReturnedColumnList& derivedColList)
 {
   ParseTree* lhs = n->left();
@@ -782,6 +824,18 @@ void SimpleColumn::setSimpleColumnList()
   }
 }
 
+void SimpleColumn::setSimpleColumnListExtended()
+{
+  if (fSimpleColumnListExtended.empty())
+  {
+    fSimpleColumnListExtended.push_back(this);
+  }
+  else
+  {
+    fSimpleColumnListExtended.back() = this;
+  }
+}
+
 std::optional<CalpontSystemCatalog::TableAliasName> sameTableCheck(
     std::vector<SimpleColumn*> simpleColumnList)
 {

dbcon/execplan/simplecolumn.h

@@ -266,7 +266,7 @@ class SimpleColumn : public ReturnedColumn
   std::optional<CalpontSystemCatalog::TableAliasName> singleTable() override;
 
   void setSimpleColumnList() override;
-
+  void setSimpleColumnListExtended() override;
 
  protected:
   /**
@@ -407,8 +407,10 @@ std::ostream& operator<<(std::ostream& output, const SimpleColumn& rhs);
  * utility function to extract all simple columns from a parse tree
  */
 void getSimpleCols(ParseTree* n, void* obj);
+void getSimpleColsExtended(execplan::ParseTree* n, void* obj);
 ParseTree* replaceRefCol(ParseTree*& n, CalpontSelectExecutionPlan::ReturnedColumnList&);
 
-std::optional<CalpontSystemCatalog::TableAliasName> sameTableCheck(std::vector<SimpleColumn*> simpleColumnList);
+std::optional<CalpontSystemCatalog::TableAliasName> sameTableCheck(
+    std::vector<SimpleColumn*> simpleColumnList);
 
 }  // namespace execplan

dbcon/execplan/simplefilter.cpp

@@ -810,6 +810,11 @@ const std::vector<SimpleColumn*>& SimpleFilter::simpleColumnList()
   return fSimpleColumnList;
 }
 
+const std::vector<SimpleColumn*>& SimpleFilter::simpleColumnListExtended()
+{
+  return fSimpleColumnListExtended;
+}
+
 void SimpleFilter::setSimpleColumnList()
 {
   SimpleColumn* lsc = dynamic_cast<SimpleColumn*>(fLhs);
@@ -839,6 +844,35 @@ void SimpleFilter::setSimpleColumnList()
   }
 }
 
+void SimpleFilter::setSimpleColumnListExtended()
+{
+  SimpleColumn* lsc = dynamic_cast<SimpleColumn*>(fLhs);
+  SimpleColumn* rsc = dynamic_cast<SimpleColumn*>(fRhs);
+  fSimpleColumnListExtended.clear();
+
+  if (lsc)
+  {
+    fSimpleColumnListExtended.push_back(lsc);
+  }
+  else if (fLhs)
+  {
+    fLhs->setSimpleColumnListExtended();
+    fSimpleColumnList.insert(fSimpleColumnList.end(), fLhs->simpleColumnListExtended().begin(),
+                             fLhs->simpleColumnListExtended().end());
+  }
+
+  if (rsc)
+  {
+    fSimpleColumnList.push_back(rsc);
+  }
+  else if (fRhs)
+  {
+    fRhs->setSimpleColumnListExtended();
+    fSimpleColumnList.insert(fSimpleColumnList.end(), fRhs->simpleColumnListExtended().begin(),
+                             fRhs->simpleColumnListExtended().end());
+  }
+}
+
 bool SimpleFilter::hasAggregate()
 {
   if (fAggColumnList.empty())

dbcon/execplan/simplefilter.h

@@ -246,8 +246,10 @@ class SimpleFilter : public Filter
 
   // get all simple columns involved in this column
   const std::vector<SimpleColumn*>& simpleColumnList();
+  const std::vector<SimpleColumn*>& simpleColumnListExtended();
   // walk through the simple filter operands to re-populate fSimpleColumnList
   void setSimpleColumnList();
+  void setSimpleColumnListExtended();
   // walk through the simple filter operands to check existence of aggregate
   bool hasAggregate();
 
@@ -265,6 +267,7 @@ class SimpleFilter : public Filter
 
  private:
   std::vector<SimpleColumn*> fSimpleColumnList;
+  std::vector<SimpleColumn*> fSimpleColumnListExtended{};
   std::vector<AggregateColumn*> fAggColumnList;
   std::vector<WindowFunctionColumn*> fWindowFunctionColumnList;
 };

dbcon/joblist/jlf_common.cpp

@@ -67,8 +67,6 @@ uint32_t uniqTupleKey(JobInfo& jobInfo, CalpontSystemCatalog::OID& o, CalpontSys
     return iter->second;
 
   uint32_t newId = jobInfo.keyInfo->nextKey++;
-  // cout << "new id: " << newId << " -- " << o << ", " << pi << ", " << nm << ", " << vw << ", " << sn << ",
-  // " << subId << endl;
   jobInfo.keyInfo->tupleKeyMap[id] = newId;
   jobInfo.keyInfo->tupleKeyVec.push_back(id);
   jobInfo.keyInfo->tupleKeyToTableOid.insert(make_pair(newId, t));

dbcon/rbo/rbo_apply_parallel_ces.cpp

@@ -523,27 +523,15 @@ void tryToUpdateScToUseRewrittenDerived(
 }
 
 void updateSCsUsingIteration(optimizer::TableAliasToNewAliasAndSCPositionsMap& tableAliasToSCPositionsMap,
-                             std::vector<execplan::SRCP>& rcs, const bool ignoreAgg)
+                             std::vector<execplan::SRCP>& rcs)
 {
   for (auto& rc : rcs)
   {
-    rc->setSimpleColumnList();
-    for (auto* sc : rc->simpleColumnList())
+    rc->setSimpleColumnListExtended();
+    for (auto* sc : rc->simpleColumnListExtended())
     {
       tryToUpdateScToUseRewrittenDerived(sc, tableAliasToSCPositionsMap);
     }
-
-    if (!ignoreAgg && rc->hasAggregate())
-    {
-      for (auto* agc : rc->aggColumnList())
-      {
-        agc->setSimpleColumnList();
-        for (auto* sc : agc->simpleColumnList())
-        {
-          tryToUpdateScToUseRewrittenDerived(sc, tableAliasToSCPositionsMap);
-        }
-      }
-    }
   }
 }
 
@@ -551,7 +539,7 @@ void updateSCsUsingWalkers(optimizer::TableAliasToNewAliasAndSCPositionsMap& tab
                            execplan::ParseTree* pt)
 {
   std::vector<execplan::SimpleColumn*> simpleColumns;
-  pt->walk(execplan::getSimpleCols, &simpleColumns);
+  pt->walk(execplan::getSimpleColsExtended, &simpleColumns);
   for (auto* sc : simpleColumns)
   {
     tryToUpdateScToUseRewrittenDerived(sc, tableAliasToSCPositionsMap);
@@ -622,7 +610,7 @@ bool applyParallelCES(execplan::CalpontSelectExecutionPlan& csep, optimizer::RBO
   {
     for (auto& rc : csep.returnedCols())
     {
-      updateSCsUsingIteration(tableAliasToSCPositionsMap, csep.returnedCols(), false);
+      updateSCsUsingIteration(tableAliasToSCPositionsMap, csep.returnedCols());
       newReturnedColumns.push_back(rc);
     }
 
@@ -633,13 +621,13 @@ bool applyParallelCES(execplan::CalpontSelectExecutionPlan& csep, optimizer::RBO
     // 3d pass over GROUP BY columns
     if (!csep.groupByCols().empty())
     {
-      updateSCsUsingIteration(tableAliasToSCPositionsMap, csep.groupByCols(), true);
+      updateSCsUsingIteration(tableAliasToSCPositionsMap, csep.groupByCols());
     }
 
     // 4th pass over ORDER BY columns
     if (!csep.orderByCols().empty())
     {
-      updateSCsUsingIteration(tableAliasToSCPositionsMap, csep.orderByCols(), false);
+      updateSCsUsingIteration(tableAliasToSCPositionsMap, csep.orderByCols());
     }
 
     // 5th pass over filters to use derived table SCs in filters
@@ -676,7 +664,7 @@ bool applyParallelCES(execplan::CalpontSelectExecutionPlan& csep, optimizer::RBO
           if (auto subFilters = sub->filters())
           {
             std::vector<execplan::SimpleColumn*> subSCs;
-            subFilters->walk(execplan::getSimpleCols, &subSCs);
+            subFilters->walk(execplan::getSimpleColsExtended, &subSCs);
             for (auto* sc : subSCs)
             {
               if (sc)
@@ -686,7 +674,7 @@ bool applyParallelCES(execplan::CalpontSelectExecutionPlan& csep, optimizer::RBO
           if (auto subHaving = sub->having())
           {
             std::vector<execplan::SimpleColumn*> subSCs;
-            subHaving->walk(execplan::getSimpleCols, &subSCs);
+            subHaving->walk(execplan::getSimpleColsExtended, &subSCs);
             for (auto* sc : subSCs)
             {
               if (sc)