database/mariadb-columnstore-engine
1
0
Fork 0
mirror of https://github.com/mariadb-corporation/mariadb-columnstore-engine.git synced 2025-07-29 08:21:15 +03:00
Code Activity

MCOL-641 Upgrade code for extent map from v4 to v5.

Browse Source 
v5 (current) version of the extent map supports int128_t min/max
values. This commit provides upgrade capability to load_brm program
that when reads an extent map saved to disk earlier with save_brm,
with a EM_MAGIC_V4 entry, converts it to the current v5 "wider"
extent map.
This commit is contained in:
Gagan Goel
2020-07-27 11:02:59 -04:00
committed by Roman Nozdrin
parent d3bc68b02f
commit 5287e6860b
2 changed files with 111 additions and 23 deletions
Download Patch File Download Diff File Expand all files Collapse all files

82
versioning/BRM/extentmap.cpp
View File

@ -74,6 +74,7 @@ namespace bi = boost::interprocess;
#define EM_MAGIC_V2 0x76f78b1d #define EM_MAGIC_V2 0x76f78b1d
#define EM_MAGIC_V3 0x76f78b1e #define EM_MAGIC_V3 0x76f78b1e
#define EM_MAGIC_V4 0x76f78b1f #define EM_MAGIC_V4 0x76f78b1f
#define EM_MAGIC_V5 0x76f78b20
#ifndef NDEBUG #ifndef NDEBUG
#define ASSERT(x) \ #define ASSERT(x) \
@ -163,7 +164,7 @@ EMCasualPartition_struct& EMCasualPartition_struct::operator= (const EMCasualPar
} }
//------------------------------------------------------------------------------ //------------------------------------------------------------------------------
// Version 4 EmEntry methods // Version 5 EmEntry methods
//------------------------------------------------------------------------------ //------------------------------------------------------------------------------
EMEntry::EMEntry() EMEntry::EMEntry()
@ -1276,7 +1277,7 @@ void ExtentMap::reserveLBIDRange(LBID_t start, uint8_t size)
*/ */
void ExtentMap::loadVersion4(IDBDataFile* in) void ExtentMap::loadVersion4or5(IDBDataFile* in, bool upgradeV4ToV5)
{ {
int emNumElements = 0, flNumElements = 0; int emNumElements = 0, flNumElements = 0;
@ -1287,8 +1288,8 @@ void ExtentMap::loadVersion4(IDBDataFile* in)
if ((size_t) nbytes != sizeof(int) + sizeof(int)) if ((size_t) nbytes != sizeof(int) + sizeof(int))
{ {
log_errno("ExtentMap::loadVersion4(): read "); log_errno("ExtentMap::loadVersion4or5(): read ");
throw runtime_error("ExtentMap::loadVersion4(): read failed. Check the error log."); throw runtime_error("ExtentMap::loadVersion4or5(): read failed. Check the error log.");
} }
void *fExtentMapPtr = static_cast<void*>(fExtentMap); void *fExtentMapPtr = static_cast<void*>(fExtentMap);
@ -1317,18 +1318,66 @@ void ExtentMap::loadVersion4(IDBDataFile* in)
growEMShmseg(nrows); growEMShmseg(nrows);
} }
size_t progress = 0, writeSize = emNumElements * sizeof(EMEntry);
int err; int err;
char *writePos = (char *) fExtentMap; char* writePos;
while (progress < writeSize) size_t progress, writeSize;
if (!upgradeV4ToV5)
{ {
err = in->read(writePos + progress, writeSize - progress); progress = 0;
if (err <= 0) writeSize = emNumElements * sizeof(EMEntry);
writePos = (char *) fExtentMap;
while (progress < writeSize)
{ {
log_errno("ExtentMap::loadVersion4(): read "); err = in->read(writePos + progress, writeSize - progress);
throw runtime_error("ExtentMap::loadVersion4(): read failed. Check the error log."); if (err <= 0)
{
log_errno("ExtentMap::loadVersion4or5(): read ");
throw runtime_error("ExtentMap::loadVersion4or5(): read failed. Check the error log.");
}
progress += (uint) err;
} }
progress += (uint) err; }
else
{
// We are upgrading extent map from v4 to v5.
for (int i = 0; i < emNumElements; i++)
{
EMEntry_v4 emEntryV4;
progress = 0;
writeSize = sizeof(EMEntry_v4);
writePos = (char *) &(emEntryV4);
while (progress < writeSize)
{
err = in->read(writePos + progress, writeSize - progress);
if (err <= 0)
{
log_errno("ExtentMap::loadVersion4or5(): read ");
throw runtime_error("ExtentMap::loadVersion4or5(): read failed during upgrade. Check the error log.");
}
progress += (uint) err;
}
fExtentMap[i].range.start = emEntryV4.range.start;
fExtentMap[i].range.size = emEntryV4.range.size;
fExtentMap[i].fileID = emEntryV4.fileID;
fExtentMap[i].blockOffset = emEntryV4.blockOffset;
fExtentMap[i].HWM = emEntryV4.HWM;
fExtentMap[i].partitionNum = emEntryV4.partitionNum;
fExtentMap[i].segmentNum = emEntryV4.segmentNum;
fExtentMap[i].dbRoot = emEntryV4.dbRoot;
fExtentMap[i].colWid = emEntryV4.colWid;
fExtentMap[i].status = emEntryV4.status;
fExtentMap[i].partition.cprange.hi_val = emEntryV4.partition.cprange.hi_val;
fExtentMap[i].partition.cprange.lo_val = emEntryV4.partition.cprange.lo_val;
fExtentMap[i].partition.cprange.sequenceNum = emEntryV4.partition.cprange.sequenceNum;
fExtentMap[i].partition.cprange.isValid = emEntryV4.partition.cprange.isValid;
utils::int128Max(fExtentMap[i].partition.cprange.bigLoVal);
utils::int128Min(fExtentMap[i].partition.cprange.bigHiVal);
}
std::cout<<emNumElements<<" extents successfully upgraded"<<std::endl;
} }
for (int i = 0; i < emNumElements; i++) for (int i = 0; i < emNumElements; i++)
@ -1418,8 +1467,11 @@ void ExtentMap::load(const string& filename, bool fixFL)
int emVersion = 0; int emVersion = 0;
int bytes = in->read((char*) &emVersion, sizeof(int)); int bytes = in->read((char*) &emVersion, sizeof(int));
if (bytes == (int) sizeof(int) && emVersion == EM_MAGIC_V4) if (bytes == (int) sizeof(int) &&
loadVersion4(in.get()); (emVersion == EM_MAGIC_V4 || emVersion == EM_MAGIC_V5))
{
loadVersion4or5(in.get(), emVersion == EM_MAGIC_V4);
}
else else
{ {
log("ExtentMap::load(): That file is not a valid ExtentMap image"); log("ExtentMap::load(): That file is not a valid ExtentMap image");
@ -1486,7 +1538,7 @@ void ExtentMap::save(const string& filename)
throw ios_base::failure("ExtentMap::save(): open failed. Check the error log."); throw ios_base::failure("ExtentMap::save(): open failed. Check the error log.");
} }
loadSize[0] = EM_MAGIC_V4; loadSize[0] = EM_MAGIC_V5;
loadSize[1] = fEMShminfo->currentSize / sizeof(EMEntry); loadSize[1] = fEMShminfo->currentSize / sizeof(EMEntry);
loadSize[2] = fFLShminfo->allocdSize / sizeof(InlineLBIDRange); // needs to send all entries loadSize[2] = fFLShminfo->allocdSize / sizeof(InlineLBIDRange); // needs to send all entries

52
versioning/BRM/extentmap.h
View File

@ -100,17 +100,43 @@ const char CP_INVALID = 0;
const char CP_UPDATING = 1; const char CP_UPDATING = 1;
const char CP_VALID = 2; const char CP_VALID = 2;
// The _v4 structs are defined below for upgrading extent map
// from v4 to v5; see ExtentMap::loadVersion4or5 for details.
struct EMCasualPartition_struct_v4
{
RangePartitionData_t hi_val; // This needs to be reinterpreted as unsigned for uint64_t column types.
RangePartitionData_t lo_val;
int32_t sequenceNum;
char isValid; //CP_INVALID - No min/max and no DML in progress. CP_UPDATING - Update in progress. CP_VALID- min/max is valid
};
struct EMPartition_struct_v4
{
EMCasualPartition_struct_v4 cprange;
};
struct EMEntry_v4
{
InlineLBIDRange range;
int fileID;
uint32_t blockOffset;
HWM_t HWM;
uint32_t partitionNum; // starts at 0
uint16_t segmentNum; // starts at 0
uint16_t dbRoot; // starts at 1 to match Columnstore.xml
uint16_t colWid;
int16_t status; //extent avail for query or not, or out of service
EMPartition_struct_v4 partition;
};
// MCOL-641: v5 structs of the extent map. This version supports int128_t min
// and max values for casual partitioning.
struct EMCasualPartition_struct struct EMCasualPartition_struct
{ {
RangePartitionData_t hi_val; // This needs to be reinterpreted as unsigned for uint64_t column types. RangePartitionData_t hi_val; // This needs to be reinterpreted as unsigned for uint64_t column types.
RangePartitionData_t lo_val; RangePartitionData_t lo_val;
int32_t sequenceNum; int32_t sequenceNum;
char isValid; //CP_INVALID - No min/max and no DML in progress. CP_UPDATING - Update in progress. CP_VALID- min/max is valid char isValid; //CP_INVALID - No min/max and no DML in progress. CP_UPDATING - Update in progress. CP_VALID- min/max is valid
EXPORT EMCasualPartition_struct();
EXPORT EMCasualPartition_struct(const int64_t lo, const int64_t hi, const int32_t seqNum);
EXPORT EMCasualPartition_struct(const __int128 bigLo, const __int128 bigHi, const int32_t seqNum);
EXPORT EMCasualPartition_struct(const EMCasualPartition_struct& em);
EXPORT EMCasualPartition_struct& operator= (const EMCasualPartition_struct& em);
union union
{ {
__int128 bigLoVal; __int128 bigLoVal;
@ -121,12 +147,17 @@ struct EMCasualPartition_struct
__int128 bigHiVal; __int128 bigHiVal;
int64_t hiVal; int64_t hiVal;
}; };
EXPORT EMCasualPartition_struct();
EXPORT EMCasualPartition_struct(const int64_t lo, const int64_t hi, const int32_t seqNum);
EXPORT EMCasualPartition_struct(const __int128 bigLo, const __int128 bigHi, const int32_t seqNum);
EXPORT EMCasualPartition_struct(const EMCasualPartition_struct& em);
EXPORT EMCasualPartition_struct& operator= (const EMCasualPartition_struct& em);
}; };
typedef EMCasualPartition_struct EMCasualPartition_t; typedef EMCasualPartition_struct EMCasualPartition_t;
struct EMPartition_struct struct EMPartition_struct
{ {
EMCasualPartition_t cprange; EMCasualPartition_t cprange;
}; };
typedef EMPartition_struct EMPartition_t; typedef EMPartition_struct EMPartition_t;
@ -973,8 +1004,13 @@ private:
int _markInvalid(const LBID_t lbid, const execplan::CalpontSystemCatalog::ColDataType colDataType); int _markInvalid(const LBID_t lbid, const execplan::CalpontSystemCatalog::ColDataType colDataType);
void loadVersion4(std::ifstream& in); /** @brief Loads the extent map from a file into memory.
void loadVersion4(idbdatafile::IDBDataFile* in); *
* @param in (in) the file to load the extent map from.
* @param upgradeV4ToV5 (in) flag indicating whether we are upgrading
* extent map from v4 to v5.
*/
void loadVersion4or5(idbdatafile::IDBDataFile* in, bool upgradeV4ToV5);
ExtentMapImpl* fPExtMapImpl; ExtentMapImpl* fPExtMapImpl;
FreeListImpl* fPFreeListImpl; FreeListImpl* fPFreeListImpl;