diff --git a/tools/dbbuilder/dbbuilder.cpp b/tools/dbbuilder/dbbuilder.cpp index 79d2a4beb..36557db2e 100644 --- a/tools/dbbuilder/dbbuilder.cpp +++ b/tools/dbbuilder/dbbuilder.cpp @@ -189,26 +189,62 @@ int main(int argc, char* argv[]) if (access(logFile.c_str(), W_OK) != 0) canWrite = false; + // MCOL-5162 Automatic syscat upgrade + // The below std::unordered_map's are used by the + // SystemCatalog::upgrade() call. + bool isUpgrade = false; + + std::unordered_map> upgradeOidMap; + upgradeOidMap[OID_SYSTABLE_AUXCOLUMNOID] = // This is the candidate OID for the upgrade. + std::make_pair(OID_SYSTABLE_OBJECTID, false); // std::pair::first is the reference OID used + // to fill the candidate OID with default vals + // std::pair::second is set to false by default + // which means the candidate OID will not be + // upgraded. This is set to true in the code + // below if a specific condition is met. Please + // note that the candidate and reference OID + // datatypes and colwidths are assumed to be the + // same in SystemCatalog::upgrade(). + + std::unordered_map upgradeOidTypeMap; + upgradeOidTypeMap[OID_SYSTABLE_AUXCOLUMNOID] = + std::make_pair(CalpontSystemCatalog::INT, 4); + + std::unordered_map upgradeOidDefaultValStrMap; + upgradeOidDefaultValStrMap[OID_SYSTABLE_AUXCOLUMNOID] = "0"; + try { if (checkNotThere(1001) != 0) { - string cmd = "echo 'FAILED: buildOption=" + oam.itoa(buildOption) + "' > " + logFile; - - if (canWrite) + for (auto iter = upgradeOidMap.begin(); iter != upgradeOidMap.end(); iter++) { - rc = system(cmd.c_str()); - } - else - { - cerr << cmd << endl; + if (checkNotThere(iter->first) == 0) + { + (iter->second).second = true; + isUpgrade = true; + } } - errorHandler(sysCatalogErr, "Build system catalog", - "System catalog appears to exist. It will remain intact " - "for reuse. The database is not recreated.", - false); - return 1; + if (!isUpgrade) + { + string cmd = "echo 'FAILED: buildOption=" + oam.itoa(buildOption) + "' > " + logFile; + + if (canWrite) + { + rc = system(cmd.c_str()); + } + else + { + cerr << cmd << endl; + } + + errorHandler(sysCatalogErr, "Build system catalog", + "System catalog appears to exist. It will remain intact " + "for reuse. The database is not recreated.", + false); + return 1; + } } //@bug5554, make sure IDBPolicy matches the Columnstore.xml config @@ -259,7 +295,15 @@ int main(int argc, char* argv[]) } SystemCatalog sysCatalog; - sysCatalog.build(); + + if (!isUpgrade) + { + sysCatalog.build(); + } + else + { + sysCatalog.upgrade(upgradeOidMap, upgradeOidTypeMap,upgradeOidDefaultValStrMap); + } std::string cmd = "echo 'OK: buildOption=" + oam.itoa(buildOption) + "' > " + logFile; diff --git a/tools/dbbuilder/systemcatalog.cpp b/tools/dbbuilder/systemcatalog.cpp index 630f30527..c55edc69e 100644 --- a/tools/dbbuilder/systemcatalog.cpp +++ b/tools/dbbuilder/systemcatalog.cpp @@ -51,15 +51,6 @@ void SystemCatalog::build() ostringstream msg; WErrorCodes ec; - //------------------------------------------------------------------------------ - // Get the DBRoot count, and rotate the tables through those DBRoots. - // All the columns in the first table (SYSTABLE) start on DBRoot1, all the - // columns in the second table (SYSCOLUMN) start on DBRoot2, etc. - //------------------------------------------------------------------------------ - config::Config* cf = config::Config::makeConfig(); - string root = cf->getConfig("SystemConfig", "DBRootCount"); - uint32_t dbRootCount = cf->uFromText(root); - //------------------------------------------------------------------------------ // Create SYSTABLE table //------------------------------------------------------------------------------ @@ -249,9 +240,6 @@ void SystemCatalog::build() //------------------------------------------------------------------------------ // Create SYSCOLUMN table //------------------------------------------------------------------------------ - // dbRoot++; - // if (dbRoot > dbRootCount) - // dbRoot = 1; // SYSCOLUMN if (rm->useHdfs()) @@ -585,14 +573,6 @@ void SystemCatalog::build() msg.str(""); - //------------------------------------------------------------------------------ - // Create SYSCONSTRAINT table - //------------------------------------------------------------------------------ - dbRoot++; - - if (dbRoot > dbRootCount) - dbRoot = 1; - // flush data files fWriteEngine.flushDataFiles(rc, 1, oids); // save brm @@ -624,3 +604,51 @@ void SystemCatalog::remove() for (int d = 2001; d <= 2312; d++) colOp.deleteFile(d); } + +int SystemCatalog::upgrade(const std::unordered_map>& upgradeOidMap, + std::unordered_map upgradeOidTypeMap, + std::unordered_map upgradeOidDefaultValStrMap) +{ + TxnID txnID = 0; + int rc = 0; + int compressionType = 0; + ostringstream msg; + WErrorCodes ec; + + cout << "Upgrading System Catalog..." << endl << endl; + + for (auto iter = upgradeOidMap.begin(); iter != upgradeOidMap.end(); iter++) + { + if ((iter->second).second == true) + { + msg.str(""); + msg << " Creating column OID: " << iter->first; + cout << msg.str() << endl; + + execplan::CalpontSystemCatalog::ColType colType; + colType.colDataType = upgradeOidTypeMap[iter->first].first; + colType.colWidth = upgradeOidTypeMap[iter->first].second; + + ColTuple defaultVal; + std::string defaultValStr = upgradeOidDefaultValStrMap[iter->first]; + bool pushWarning = false; + bool isNULL = false; + long timeZone = 0; + defaultVal.data = colType.convertColumnData(defaultValStr, pushWarning, timeZone, isNULL, false, false); + + rc = fWriteEngine.fillColumn(txnID, iter->first, colType, defaultVal, + (iter->second).first, + upgradeOidTypeMap[iter->first].first, + upgradeOidTypeMap[iter->first].second, compressionType, + isNULL, compressionType, defaultValStr, 0, false); + + if (rc) + { + throw runtime_error(msg.str() + ec.errorString(rc)); + } + } + } + + return rc; +} diff --git a/tools/dbbuilder/systemcatalog.h b/tools/dbbuilder/systemcatalog.h index f98608d13..52361c653 100644 --- a/tools/dbbuilder/systemcatalog.h +++ b/tools/dbbuilder/systemcatalog.h @@ -23,12 +23,19 @@ #pragma once #include "writeengine.h" +#include +#include + +using OidTypeT = std::pair; class SystemCatalog { public: void build(); void remove(); + int upgrade(const std::unordered_map>&, + std::unordered_map, + std::unordered_map); private: WriteEngine::WriteEngineWrapper fWriteEngine;