diff --git a/versioning/BRM/extentmap.cpp b/versioning/BRM/extentmap.cpp index 70d12a383..e5f84ea53 100644 --- a/versioning/BRM/extentmap.cpp +++ b/versioning/BRM/extentmap.cpp @@ -74,6 +74,7 @@ namespace bi = boost::interprocess; #define EM_MAGIC_V2 0x76f78b1d #define EM_MAGIC_V3 0x76f78b1e #define EM_MAGIC_V4 0x76f78b1f +#define EM_MAGIC_V5 0x76f78b20 #ifndef NDEBUG #define ASSERT(x) \ @@ -163,7 +164,7 @@ EMCasualPartition_struct& EMCasualPartition_struct::operator= (const EMCasualPar } //------------------------------------------------------------------------------ -// Version 4 EmEntry methods +// Version 5 EmEntry methods //------------------------------------------------------------------------------ EMEntry::EMEntry() @@ -1276,7 +1277,7 @@ void ExtentMap::reserveLBIDRange(LBID_t start, uint8_t size) */ -void ExtentMap::loadVersion4(IDBDataFile* in) +void ExtentMap::loadVersion4or5(IDBDataFile* in, bool upgradeV4ToV5) { int emNumElements = 0, flNumElements = 0; @@ -1287,8 +1288,8 @@ void ExtentMap::loadVersion4(IDBDataFile* in) if ((size_t) nbytes != sizeof(int) + sizeof(int)) { - log_errno("ExtentMap::loadVersion4(): read "); - throw runtime_error("ExtentMap::loadVersion4(): read failed. Check the error log."); + log_errno("ExtentMap::loadVersion4or5(): read "); + throw runtime_error("ExtentMap::loadVersion4or5(): read failed. Check the error log."); } void *fExtentMapPtr = static_cast(fExtentMap); @@ -1317,18 +1318,66 @@ void ExtentMap::loadVersion4(IDBDataFile* in) growEMShmseg(nrows); } - size_t progress = 0, writeSize = emNumElements * sizeof(EMEntry); int err; - char *writePos = (char *) fExtentMap; - while (progress < writeSize) + char* writePos; + size_t progress, writeSize; + + if (!upgradeV4ToV5) { - err = in->read(writePos + progress, writeSize - progress); - if (err <= 0) + progress = 0; + writeSize = emNumElements * sizeof(EMEntry); + writePos = (char *) fExtentMap; + + while (progress < writeSize) { - log_errno("ExtentMap::loadVersion4(): read "); - throw runtime_error("ExtentMap::loadVersion4(): read failed. Check the error log."); + err = in->read(writePos + progress, writeSize - progress); + if (err <= 0) + { + log_errno("ExtentMap::loadVersion4or5(): read "); + throw runtime_error("ExtentMap::loadVersion4or5(): read failed. Check the error log."); + } + progress += (uint) err; } - progress += (uint) err; + } + else + { + // We are upgrading extent map from v4 to v5. + for (int i = 0; i < emNumElements; i++) + { + EMEntry_v4 emEntryV4; + progress = 0; + writeSize = sizeof(EMEntry_v4); + writePos = (char *) &(emEntryV4); + while (progress < writeSize) + { + err = in->read(writePos + progress, writeSize - progress); + if (err <= 0) + { + log_errno("ExtentMap::loadVersion4or5(): read "); + throw runtime_error("ExtentMap::loadVersion4or5(): read failed during upgrade. Check the error log."); + } + progress += (uint) err; + } + + fExtentMap[i].range.start = emEntryV4.range.start; + fExtentMap[i].range.size = emEntryV4.range.size; + fExtentMap[i].fileID = emEntryV4.fileID; + fExtentMap[i].blockOffset = emEntryV4.blockOffset; + fExtentMap[i].HWM = emEntryV4.HWM; + fExtentMap[i].partitionNum = emEntryV4.partitionNum; + fExtentMap[i].segmentNum = emEntryV4.segmentNum; + fExtentMap[i].dbRoot = emEntryV4.dbRoot; + fExtentMap[i].colWid = emEntryV4.colWid; + fExtentMap[i].status = emEntryV4.status; + fExtentMap[i].partition.cprange.hi_val = emEntryV4.partition.cprange.hi_val; + fExtentMap[i].partition.cprange.lo_val = emEntryV4.partition.cprange.lo_val; + fExtentMap[i].partition.cprange.sequenceNum = emEntryV4.partition.cprange.sequenceNum; + fExtentMap[i].partition.cprange.isValid = emEntryV4.partition.cprange.isValid; + utils::int128Max(fExtentMap[i].partition.cprange.bigLoVal); + utils::int128Min(fExtentMap[i].partition.cprange.bigHiVal); + } + + std::cout<read((char*) &emVersion, sizeof(int)); - if (bytes == (int) sizeof(int) && emVersion == EM_MAGIC_V4) - loadVersion4(in.get()); + if (bytes == (int) sizeof(int) && + (emVersion == EM_MAGIC_V4 || emVersion == EM_MAGIC_V5)) + { + loadVersion4or5(in.get(), emVersion == EM_MAGIC_V4); + } else { log("ExtentMap::load(): That file is not a valid ExtentMap image"); @@ -1486,7 +1538,7 @@ void ExtentMap::save(const string& filename) throw ios_base::failure("ExtentMap::save(): open failed. Check the error log."); } - loadSize[0] = EM_MAGIC_V4; + loadSize[0] = EM_MAGIC_V5; loadSize[1] = fEMShminfo->currentSize / sizeof(EMEntry); loadSize[2] = fFLShminfo->allocdSize / sizeof(InlineLBIDRange); // needs to send all entries diff --git a/versioning/BRM/extentmap.h b/versioning/BRM/extentmap.h index e3d4634d8..3aa0da0ca 100644 --- a/versioning/BRM/extentmap.h +++ b/versioning/BRM/extentmap.h @@ -100,17 +100,43 @@ const char CP_INVALID = 0; const char CP_UPDATING = 1; const char CP_VALID = 2; +// The _v4 structs are defined below for upgrading extent map +// from v4 to v5; see ExtentMap::loadVersion4or5 for details. +struct EMCasualPartition_struct_v4 +{ + RangePartitionData_t hi_val; // This needs to be reinterpreted as unsigned for uint64_t column types. + RangePartitionData_t lo_val; + int32_t sequenceNum; + char isValid; //CP_INVALID - No min/max and no DML in progress. CP_UPDATING - Update in progress. CP_VALID- min/max is valid +}; + +struct EMPartition_struct_v4 +{ + EMCasualPartition_struct_v4 cprange; +}; + +struct EMEntry_v4 +{ + InlineLBIDRange range; + int fileID; + uint32_t blockOffset; + HWM_t HWM; + uint32_t partitionNum; // starts at 0 + uint16_t segmentNum; // starts at 0 + uint16_t dbRoot; // starts at 1 to match Columnstore.xml + uint16_t colWid; + int16_t status; //extent avail for query or not, or out of service + EMPartition_struct_v4 partition; +}; + +// MCOL-641: v5 structs of the extent map. This version supports int128_t min +// and max values for casual partitioning. struct EMCasualPartition_struct { RangePartitionData_t hi_val; // This needs to be reinterpreted as unsigned for uint64_t column types. RangePartitionData_t lo_val; int32_t sequenceNum; char isValid; //CP_INVALID - No min/max and no DML in progress. CP_UPDATING - Update in progress. CP_VALID- min/max is valid - EXPORT EMCasualPartition_struct(); - EXPORT EMCasualPartition_struct(const int64_t lo, const int64_t hi, const int32_t seqNum); - EXPORT EMCasualPartition_struct(const __int128 bigLo, const __int128 bigHi, const int32_t seqNum); - EXPORT EMCasualPartition_struct(const EMCasualPartition_struct& em); - EXPORT EMCasualPartition_struct& operator= (const EMCasualPartition_struct& em); union { __int128 bigLoVal; @@ -121,12 +147,17 @@ struct EMCasualPartition_struct __int128 bigHiVal; int64_t hiVal; }; + EXPORT EMCasualPartition_struct(); + EXPORT EMCasualPartition_struct(const int64_t lo, const int64_t hi, const int32_t seqNum); + EXPORT EMCasualPartition_struct(const __int128 bigLo, const __int128 bigHi, const int32_t seqNum); + EXPORT EMCasualPartition_struct(const EMCasualPartition_struct& em); + EXPORT EMCasualPartition_struct& operator= (const EMCasualPartition_struct& em); }; typedef EMCasualPartition_struct EMCasualPartition_t; struct EMPartition_struct { - EMCasualPartition_t cprange; + EMCasualPartition_t cprange; }; typedef EMPartition_struct EMPartition_t; @@ -973,8 +1004,13 @@ private: int _markInvalid(const LBID_t lbid, const execplan::CalpontSystemCatalog::ColDataType colDataType); - void loadVersion4(std::ifstream& in); - void loadVersion4(idbdatafile::IDBDataFile* in); + /** @brief Loads the extent map from a file into memory. + * + * @param in (in) the file to load the extent map from. + * @param upgradeV4ToV5 (in) flag indicating whether we are upgrading + * extent map from v4 to v5. + */ + void loadVersion4or5(idbdatafile::IDBDataFile* in, bool upgradeV4ToV5); ExtentMapImpl* fPExtMapImpl; FreeListImpl* fPFreeListImpl;