MCOL-5021 Add a new member function to the DBRM class, DBRM::addToLBIDList().

This function iterates over lbidList (populated by an earlier call to
DBRM::getUncommittedExtentLBIDs()) to find those LBIDs which belong to
the AUX column. It then finds the corresponding LBIDs for all other columns
which belong to the same table as the AUX LBID and appends them to lbidList.
The updated lbidList is used by invalidateUncommittedExtentLBIDs() to update
the casual partitioning information.

DBRM::addToLBIDList() only comes into play in case of a transaction ROLLBACK.

dbcon/dmlpackageproc/commandpackageprocessor.cpp

@@ -374,6 +374,9 @@ DMLPackageProcessor::DMLResult CommandPackageProcessor::processPackage(
 
           if (weRc == 0)
           {
+            // MCOL-5021
+            fDbrm->addToLBIDList(fSessionID, lbidList);
+
             //@Bug 4560 invalidate cp first as bulkrollback will truncate the newly added lbids.
             fDbrm->invalidateUncommittedExtentLBIDs(0, true, &lbidList);
             cpInvalidated = true;
@@ -433,6 +436,12 @@ DMLPackageProcessor::DMLResult CommandPackageProcessor::processPackage(
 
         if (!cpInvalidated)
         {
+          // MCOL-5021
+          if (stmt == "ROLLBACK")
+          {
+            fDbrm->addToLBIDList(fSessionID, lbidList);
+          }
+
           fDbrm->invalidateUncommittedExtentLBIDs(0, stmt == "ROLLBACK", &lbidList);
         }
       }

dbcon/execplan/calpontsystemcatalog.cpp

@@ -1241,7 +1241,7 @@ const CalpontSystemCatalog::ColType CalpontSystemCatalog::colType(const OID& Oid
     ct.columnOID = Oid;
   }
 
-  if ((sysDataList.size() == 0) && isAUXColumnOID(Oid))
+  if ((sysDataList.size() == 0) && isAUXColumnOID(Oid) >= 3000)
   {
     ct.colDataType = execplan::AUX_COL_DATATYPE;
     ct.colWidth = execplan::AUX_COL_WIDTH;
@@ -3554,13 +3554,13 @@ const CalpontSystemCatalog::ROPair CalpontSystemCatalog::tableRID(const TableNam
   oss << "select objectid from systable where schema='" << aTableName.schema << "' and tablename='"
       << aTableName.table << "' --tableRID/";
 
-  csep.data(oss.str());  //@bug 6078. Log the statement
-
   if (fIdentity == EC)
     oss << "EC";
   else
     oss << "FE";
 
+  csep.data(oss.str());  //@bug 6078. Log the statement
+
   NJLSysDataList sysDataList;
 
   try
@@ -3698,13 +3698,13 @@ CalpontSystemCatalog::OID CalpontSystemCatalog::tableAUXColumnOID(const TableNam
   oss << "select auxcolumnoid from systable where schema='" << aTableName.schema << "' and tablename='"
       << aTableName.table << "' --tableAUXColumnOID/";
 
-  csep.data(oss.str());  //@bug 6078. Log the statement
-
   if (fIdentity == EC)
     oss << "EC";
   else
     oss << "FE";
 
+  csep.data(oss.str());  //@bug 6078. Log the statement
+
   NJLSysDataList sysDataList;
 
   try
@@ -3749,22 +3749,26 @@ CalpontSystemCatalog::OID CalpontSystemCatalog::tableAUXColumnOID(const TableNam
   throw IDBExcept(ERR_TABLE_NOT_IN_CATALOG, args);
 }
 
-bool CalpontSystemCatalog::isAUXColumnOID(const OID& oid)
+CalpontSystemCatalog::OID CalpontSystemCatalog::isAUXColumnOID(const OID& oid)
 {
   DEBUG << "Enter isAUXColumnOID" << endl;
 
   checkSysCatVer();
 
-  // select auxcolumnoid from systable where auxcolumnoid = oid;
+  // select objectid from systable where auxcolumnoid = oid;
   CalpontSelectExecutionPlan csep;
   CalpontSelectExecutionPlan::ReturnedColumnList returnedColumnList;
   CalpontSelectExecutionPlan::FilterTokenList filterTokenList;
   CalpontSelectExecutionPlan::ColumnMap colMap;
 
   SimpleColumn* c1 =
+    new SimpleColumn(CALPONT_SCHEMA + "." + SYSTABLE_TABLE + "." + OBJECTID_COL, fSessionID);
+  SimpleColumn* c2 =
     new SimpleColumn(CALPONT_SCHEMA + "." + SYSTABLE_TABLE + "." + AUXCOLUMNOID_COL, fSessionID);
   SRCP srcp;
   srcp.reset(c1);
+  colMap.insert(CMVT_(CALPONT_SCHEMA + "." + SYSTABLE_TABLE + "." + OBJECTID_COL, srcp));
+  srcp.reset(c2);
   colMap.insert(CMVT_(CALPONT_SCHEMA + "." + SYSTABLE_TABLE + "." + AUXCOLUMNOID_COL, srcp));
   csep.columnMapNonStatic(colMap);
 
@@ -3772,23 +3776,25 @@ bool CalpontSystemCatalog::isAUXColumnOID(const OID& oid)
   returnedColumnList.push_back(srcp);
   csep.returnedCols(returnedColumnList);
 
+  CalpontSystemCatalog::OID systableOid = c1->oid();
+
   // Filters
   SimpleFilter* f1 =
-      new SimpleFilter(opeq, c1->clone(), new ConstantColumn((int64_t)oid, ConstantColumn::NUM));
+      new SimpleFilter(opeq, c2->clone(), new ConstantColumn((int64_t)oid, ConstantColumn::NUM));
   filterTokenList.push_back(f1);
   csep.filterTokenList(filterTokenList);
 
   ostringstream oss;
-  oss << "select auxcolumnoid from systable where auxcolumnoid='" << oid
+  oss << "select objectid from systable where auxcolumnoid='" << oid
       << "' --isAUXColumnOID/";
 
-  csep.data(oss.str());  //@bug 6078. Log the statement
-
   if (fIdentity == EC)
     oss << "EC";
   else
     oss << "FE";
 
+  csep.data(oss.str());  //@bug 6078. Log the statement
+
   NJLSysDataList sysDataList;
 
   try
@@ -3804,10 +3810,24 @@ bool CalpontSystemCatalog::isAUXColumnOID(const OID& oid)
     throw runtime_error(e.what());
   }
 
-  if (sysDataList.size() == 1)
+  vector<ColumnResult*>::const_iterator it;
-    return true;
 
-  return false;
+  for (it = sysDataList.begin(); it != sysDataList.end(); it++)
+  {
+    if ((*it)->ColumnOID() == systableOid)
+    {
+      if ((*it)->dataCount() == 1)
+      {
+        return (OID)((*it)->GetData(0));
+      }
+      else
+      {
+        break;
+      }
+    }
+  }
+
+  return (OID)(0);
 }
 
 #if 0

dbcon/execplan/calpontsystemcatalog.h

@@ -666,10 +666,10 @@ class CalpontSystemCatalog : public datatypes::SystemCatalog
    */
   OID tableAUXColumnOID(const TableName& tableName, int lower_case_table_names = 0);
 
-  /** returns true if the given OID is the AUX
+  /** returns the table OID if the input OID is the AUX
    *  column OID of the table.
    */
-  bool isAUXColumnOID(const OID& oid);
+  CalpontSystemCatalog::OID isAUXColumnOID(const OID& oid);
 
   /** return the RID of the index for a table
    *

versioning/BRM/dbrm.cpp

@@ -21,6 +21,7 @@
  ****************************************************************************/
 
 #include <iostream>
+#include <unordered_set>
 #include <sys/types.h>
 #include <vector>
 #ifdef __linux__
@@ -4453,6 +4454,97 @@ void DBRM::deleteAISequence(uint32_t OID)
   }
 }
 
+void DBRM::addToLBIDList(uint32_t sessionID, vector<LBID_t>& lbidList)
+{
+  boost::shared_ptr<execplan::CalpontSystemCatalog> systemCatalogPtr =
+    execplan::CalpontSystemCatalog::makeCalpontSystemCatalog(sessionID);
+
+  std::unordered_map<execplan::CalpontSystemCatalog::OID,
+    std::unordered_map<execplan::CalpontSystemCatalog::OID,
+      std::vector<struct BRM::EMEntry>>> extentMap;
+
+  int err = 0;
+
+  std::unordered_set<LBID_t> lbidSet;
+  std::unordered_set<execplan::CalpontSystemCatalog::OID> tableOidSet;
+
+  for (const auto i : lbidList)
+  {
+    lbidSet.insert(i);
+    execplan::CalpontSystemCatalog::OID oid;
+    uint16_t dbRoot, segmentNum;
+    uint32_t partitionNum, fbo;
+    err = lookupLocal(i, 0, false, oid, dbRoot, partitionNum, segmentNum, fbo);
+
+    if (err)
+    {
+      ostringstream os;
+      std::string errorMsg;
+      BRM::errString(err, errorMsg);
+      os << "lookupLocal from extent map error encountered while looking up lbid " << i
+         << " and error code is " << err << " with message " << errorMsg;
+      throw runtime_error(os.str());
+    }
+
+    execplan::CalpontSystemCatalog::OID tableOid =
+      systemCatalogPtr->isAUXColumnOID(oid);
+
+    if (tableOid >= 3000)
+    {
+      if (tableOidSet.find(tableOid) == tableOidSet.end())
+      {
+        tableOidSet.insert(tableOid);
+        execplan::CalpontSystemCatalog::TableName tableName =
+          systemCatalogPtr->tableName(tableOid);
+        execplan::CalpontSystemCatalog::RIDList tableColRidList =
+          systemCatalogPtr->columnRIDs(tableName);
+
+        for (unsigned j = 0; j < tableColRidList.size(); j++)
+        {
+          auto objNum = tableColRidList[j].objnum;
+          err = getExtents(objNum, extentMap[tableOid][objNum]);
+
+          if (err)
+          {
+            ostringstream os;
+            os << "BRM lookup error. Could not get extents for OID " << objNum;
+            throw runtime_error(os.str());
+          }
+        }
+      }
+
+      uint32_t extentNumAux = fbo / (execplan::AUX_COL_WIDTH * 1024);
+
+      for (auto iter = extentMap[tableOid].begin(); iter != extentMap[tableOid].end(); iter++)
+      {
+        const std::vector<struct BRM::EMEntry>& extents = iter->second;
+
+        for (const struct BRM::EMEntry& extent : extents)
+        {
+          uint32_t extentNum = extent.blockOffset / (extent.range.size * 1024);
+
+          if (dbRoot == extent.dbRoot && partitionNum == extent.partitionNum &&
+              segmentNum == extent.segmentNum && extentNumAux == extentNum)
+          {
+            lbidSet.insert(extent.range.start);
+            break;
+          }
+        }
+      }
+    }
+  }
+
+  lbidList.clear();
+
+  for (auto iter = lbidSet.begin(); iter != lbidSet.end(); iter++)
+  {
+    lbidList.push_back(*iter);
+  }
+
+  // Sort the vector.
+  std::sort<vector<LBID_t>::iterator>(lbidList.begin(), lbidList.end());
+}
+
 void DBRM::invalidateUncommittedExtentLBIDs(execplan::CalpontSystemCatalog::SCN txnid, bool allExtents,
                                             vector<LBID_t>* plbidList)
 {
@@ -4488,6 +4580,22 @@ void DBRM::invalidateUncommittedExtentLBIDs(execplan::CalpontSystemCatalog::SCN
   if (plbidList == NULL)
   {
     getUncommittedExtentLBIDs(static_cast<VER_t>(txnid), localLBIDList);
+
+    try
+    {
+      addToLBIDList(0, localLBIDList);
+    }
+    catch (exception& e)
+    {
+      cerr << e.what() << endl;
+      return;
+    }
+    catch (...)
+    {
+      cerr << "invalidateUncommittedExtentLBIDs: caught an exception" << endl;
+      return;
+    }
+
     plbidList = &localLBIDList;
   }
 

versioning/BRM/dbrm.h

@@ -620,6 +620,18 @@ class DBRM
    */
   EXPORT int getUncommittedExtentLBIDs(VER_t transID, std::vector<LBID_t>& lbidList) throw();
 
+  // MCOL-5021
+  /** @brief Adds LBIDs to an existing list of LBIDs retrieved via call
+   *  to getUncommittedExtentLBIDs().
+   *
+   * This function iterates over lbidList to find those LBIDs which belong to
+   * the AUX column. It then finds the corresponding LBIDs for all other columns
+   * which belong to the same table as the AUX LBID and appends them to lbidList.
+   * The updated lbidList is used by invalidateUncommittedExtentLBIDs().
+   * @param lbidList (in and out) This contains the ranges of LBIDs
+   */
+  EXPORT void addToLBIDList(uint32_t sessionID, vector<LBID_t>& lbidList);
+
   /** @brief Atomically prepare to copy data to the version buffer
    *
    * Atomically sets the copy flag on the specified LBID ranges