database/mariadb-columnstore-engine
1
0
Fork 0
mirror of https://github.com/mariadb-corporation/mariadb-columnstore-engine.git synced 2025-11-02 06:13:16 +03:00
Code Activity

feat: add vacuum_partition functionality with initialization and execution logic

Browse Source
This commit is contained in:
Amr Elmohamady
2025-09-27 23:55:14 +03:00
committed by drrtuy
parent 899f0f6aae
commit 3d2e61a637
18 changed files with 832 additions and 11 deletions
Download Patch File Download Diff File Expand all files Collapse all files

2
versioning/BRM/brmtypes.h
View File

@@ -481,6 +481,8 @@ const uint8_t MARK_ALL_PARTITION_FOR_DELETION = 41;
const uint8_t CREATE_COLUMN_EXTENT_EXACT_FILE = 42;
const uint8_t DELETE_DBROOT = 43;
const uint8_t CREATE_STRIPE_COLUMN_EXTENTS = 44;
const uint8_t CREATE_HIDDEN_STRIPE_COLUMN_EXTENTS = 107;
const uint8_t MAKE_PARTITION_VISIBLE = 108;
/* SessionManager interface */
const uint8_t VER_ID = 45;

108
versioning/BRM/dbrm.cpp
View File

@@ -991,6 +991,114 @@ int DBRM::createStripeColumnExtents(const std::vector<CreateStripeColumnExtentsA
return 0;
}
//------------------------------------------------------------------------------
// Send a request to create hidden stripe column extents
//------------------------------------------------------------------------------
int DBRM::createHiddenStripeColumnExtents(const std::vector<CreateStripeColumnExtentsArgIn>& cols, uint16_t dbRoot,
uint32_t& partitionNum, uint16_t& segmentNum,
std::vector<CreateStripeColumnExtentsArgOut>& extents) DBRM_THROW
{
#ifdef BRM_INFO
if (fDebug)
{
TRACER_WRITELATER("createHiddenStripeColumnExtents");
TRACER_WRITE;
}
#endif
ByteStream command, response;
uint8_t err;
uint16_t tmp16;
uint32_t tmp32;
command << CREATE_HIDDEN_STRIPE_COLUMN_EXTENTS;
serializeInlineVector(command, cols);
command << dbRoot << partitionNum;
err = send_recv(command, response);
if (err != ERR_OK)
return err;
if (response.length() == 0)
return ERR_NETWORK;
try
{
response >> err;
if (err != 0)
return (int)err;
response >> tmp32;
partitionNum = tmp32;
response >> tmp16;
segmentNum = tmp16;
deserializeInlineVector(response, extents);
}
catch (exception& e)
{
cerr << e.what() << endl;
return ERR_FAILURE;
}
CHECK_EMPTY(response);
return 0;
}
//------------------------------------------------------------------------------
// Send a request to make a hidden partition visible
//------------------------------------------------------------------------------
int DBRM::makePartitionVisible(const std::set<OID_t>& oids, uint16_t dbRoot, uint32_t partitionNum) DBRM_THROW
{
#ifdef BRM_INFO
if (fDebug)
{
TRACER_WRITELATER("makePartitionVisible");
TRACER_WRITE;
}
#endif
ByteStream command, response;
uint8_t err;
command << MAKE_PARTITION_VISIBLE;
command << static_cast<uint32_t>(oids.size());
for (const auto& oid : oids)
{
command << oid;
}
command << dbRoot << partitionNum;
err = send_recv(command, response);
if (err != ERR_OK)
return err;
if (response.length() == 0)
return ERR_NETWORK;
try
{
response >> err;
if (err != 0)
return (int)err;
}
catch (exception& e)
{
cerr << e.what() << endl;
return ERR_FAILURE;
}
CHECK_EMPTY(response);
return 0;
}
//------------------------------------------------------------------------------
// Send a request to create a column extent for the specified OID and DBRoot.
//------------------------------------------------------------------------------

29
versioning/BRM/dbrm.h
View File

@@ -229,6 +229,35 @@ class DBRM
uint16_t dbRoot, uint32_t& partitionNum, uint16_t& segmentNum,
std::vector<CreateStripeColumnExtentsArgOut>& extents) DBRM_THROW;
/** @brief Allocate a "stripe" of hidden (out of service) extents for columns in a table (in DBRoot)
*
* Creates a new hidden partition that is not visible to normal query operations.
* The partition can be used as a destination for data movement operations like VACUUM.
* All extents are initially marked with EXTENTOUTOFSERVICE status.
*
* @param cols (in) List of column OIDs and column widths
* @param dbRoot (in) DBRoot for requested extents.
* @param partitionNum (in/out) Partition number in file path.
* @param segmentNum (out) Segment number selected for new extents.
* @param extents (out) list of lbids, numBlks, and fbo for new extents
*/
EXPORT int createHiddenStripeColumnExtents(const std::vector<CreateStripeColumnExtentsArgIn>& cols,
uint16_t dbRoot, uint32_t& partitionNum, uint16_t& segmentNum,
std::vector<CreateStripeColumnExtentsArgOut>& extents) DBRM_THROW;
/** @brief Make a hidden partition visible to normal query operations
*
* Changes the status of all extents in the specified partition from
* EXTENTOUTOFSERVICE to EXTENTAVAILABLE, making them visible to queries.
* This is used to make partitions created with createHiddenStripeColumnExtents
* visible after data movement operations like VACUUM are complete.
*
* @param oids (in) Set of column OIDs in the partition
* @param dbRoot (in) The DBRoot containing the partition
* @param partitionNum (in) The partition number to make visible
*/
EXPORT int makePartitionVisible(const std::set<OID_t>& oids, uint16_t dbRoot, uint32_t partitionNum) DBRM_THROW;
/** @brief Allocate an extent for a column file
*
* Allocate a column extent for the specified OID and DBRoot.

87
versioning/BRM/extentmap.cpp
View File

@@ -2593,6 +2593,93 @@ void ExtentMap::createStripeColumnExtents(const vector<CreateStripeColumnExtents
}
}
//------------------------------------------------------------------------------
// Creates a "stripe" of hidden extents for columns in a table (in DBRoot).
// These extents are marked as EXTENTOUTOFSERVICE and are not visible to
// normal query operations until makePartitionVisible() is called.
//------------------------------------------------------------------------------
void ExtentMap::createHiddenStripeColumnExtents(const vector<CreateStripeColumnExtentsArgIn>& cols,
uint16_t dbRoot, uint32_t& partitionNum, uint16_t& segmentNum,
vector<CreateStripeColumnExtentsArgOut>& extents)
{
LBID_t startLbid;
int allocSize;
uint32_t startBlkOffset;
grabEMEntryTable(WRITE);
grabEMIndex(WRITE);
grabFreeList(WRITE);
OID_t baselineOID = -1;
uint16_t baselineSegmentNum = -1;
uint32_t baselinePartNum = -1;
for (uint32_t i = 0; i < cols.size(); i++)
{
createColumnExtent_DBroot(cols[i].oid, cols[i].width, dbRoot, cols[i].colDataType, partitionNum,
segmentNum, startLbid, allocSize, startBlkOffset, false);
auto emIter = fExtentMapRBTree->find(startLbid);
if (emIter != fExtentMapRBTree->end())
{
makeUndoRecordRBTree(UndoRecordType::DEFAULT, emIter->second);
emIter->second.status = EXTENTOUTOFSERVICE;
}
if (i == 0)
{
baselineOID = cols[i].oid;
baselinePartNum = partitionNum;
baselineSegmentNum = segmentNum;
}
else if ((partitionNum != baselinePartNum) || (segmentNum != baselineSegmentNum))
{
ostringstream oss;
oss << "ExtentMap::createHiddenStripeColumnExtents(): "
"Inconsistent segment extent creation: "
<< "DBRoot: " << dbRoot << "OID1: " << baselineOID << "; Part#: " << baselinePartNum
<< "; Seg#: " << baselineSegmentNum << " <versus> OID2: " << cols[i].oid
<< "; Part#: " << partitionNum << "; Seg#: " << segmentNum;
log(oss.str(), logging::LOG_TYPE_CRITICAL);
throw invalid_argument(oss.str());
}
CreateStripeColumnExtentsArgOut extentInfo;
extentInfo.startLbid = startLbid;
extentInfo.allocSize = allocSize;
extentInfo.startBlkOffset = startBlkOffset;
extents.push_back(extentInfo);
}
}
//------------------------------------------------------------------------------
// Makes a hidden partition visible to normal query operations by changing
// the status of all extents from EXTENTOUTOFSERVICE to EXTENTAVAILABLE.
//------------------------------------------------------------------------------
void ExtentMap::makePartitionVisible(const set<OID_t>& oids, uint16_t dbRoot, uint32_t partitionNum)
{
grabEMEntryTable(WRITE);
for (const auto& oid : oids)
{
const auto lbids = fPExtMapIndexImpl_->find(dbRoot, oid);
auto emIterators = getEmIteratorsByLbids(lbids);
for (auto& emIter : emIterators)
{
EMEntry& emEntry = emIter->second;
if (emEntry.partitionNum == partitionNum && emEntry.dbRoot == dbRoot &&
emEntry.status == EXTENTOUTOFSERVICE)
{
makeUndoRecordRBTree(UndoRecordType::DEFAULT, emEntry);
emEntry.status = EXTENTAVAILABLE;
}
}
}
releaseEMEntryTable(WRITE);
}
//------------------------------------------------------------------------------
// Creates an extent for a column file on the specified DBRoot. This is the
// external API function referenced by the dbrm wrapper class.

33
versioning/BRM/extentmap.h
View File

@@ -639,6 +639,39 @@ class ExtentMap : public Undoable
uint16_t dbRoot, uint32_t& partitionNum, uint16_t& segmentNum,
std::vector<CreateStripeColumnExtentsArgOut>& extents);
/** @brief Allocate a "stripe" of hidden extents for columns in a table (in DBRoot)
*
* Creates a new hidden partition that is not visible to normal query operations.
* The partition can be used as a destination for data movement operations like VACUUM.
* All extents are initially marked with EXTENTOUTOFSERVICE status and remain
* hidden even when HWM is set, unlike normal extents which become visible
* when HWM is set.
*
* @param cols (in) List of column OIDs and column widths
* @param dbRoot (in) DBRoot for requested extents.
* @param partitionNum (in/out) Partition number in file path.
* If allocating OID's first extent for this DBRoot, then
* partitionNum is input, else it is an output arg.
* @param segmentNum (out) Segment number selected for new extents.
* @param extents (out) list of lbids, numBlks, and fbo for new extents
*/
EXPORT void createHiddenStripeColumnExtents(const std::vector<CreateStripeColumnExtentsArgIn>& cols,
uint16_t dbRoot, uint32_t& partitionNum, uint16_t& segmentNum,
std::vector<CreateStripeColumnExtentsArgOut>& extents);
/** @brief Make a hidden partition visible to normal query operations
*
* Changes the status of all extents in the specified partition from
* EXTENTOUTOFSERVICE to EXTENTAVAILABLE, making them visible to queries.
* This is used to make partitions created with createHiddenStripeColumnExtents
* visible after data movement operations like VACUUM are complete.
*
* @param oids (in) Set of column OIDs in the partition
* @param dbRoot (in) The DBRoot containing the partition
* @param partitionNum (in) The partition number to make visible
*/
EXPORT void makePartitionVisible(const std::set<OID_t>& oids, uint16_t dbRoot, uint32_t partitionNum);
/** @brief Allocates an extent for a column file
*
* Allocates an extent for the specified OID and DBroot.

111
versioning/BRM/slavecomm.cpp
View File

@@ -293,6 +293,10 @@ void SlaveComm::processCommand(ByteStream& msg)
{
case CREATE_STRIPE_COLUMN_EXTENTS: do_createStripeColumnExtents(msg); break;
case CREATE_HIDDEN_STRIPE_COLUMN_EXTENTS: do_createHiddenStripeColumnExtents(msg); break;
case MAKE_PARTITION_VISIBLE: do_makePartitionVisible(msg); break;
case CREATE_COLUMN_EXTENT_DBROOT: do_createColumnExtent_DBroot(msg); break;
case CREATE_COLUMN_EXTENT_EXACT_FILE: do_createColumnExtentExactFile(msg); break;
@@ -432,6 +436,113 @@ void SlaveComm::do_createStripeColumnExtents(ByteStream& msg)
doSaveDelta = true;
}
//------------------------------------------------------------------------------
// Process a request to create hidden stripe column extents
//------------------------------------------------------------------------------
void SlaveComm::do_createHiddenStripeColumnExtents(ByteStream& msg)
{
int err;
uint16_t tmp16;
uint16_t tmp32;
uint16_t dbRoot;
uint32_t partitionNum;
uint16_t segmentNum;
std::vector<CreateStripeColumnExtentsArgIn> cols;
std::vector<CreateStripeColumnExtentsArgOut> extents;
ByteStream reply;
#ifdef BRM_VERBOSE
cerr << "WorkerComm: do_createHiddenStripeColumnExtents()" << endl;
#endif
deserializeInlineVector(msg, cols);
msg >> tmp16;
dbRoot = tmp16;
msg >> tmp32;
partitionNum = tmp32;
if (printOnly)
{
cout << "createHiddenStripeColumnExtents(). " << "DBRoot=" << dbRoot << "; Part#=" << partitionNum << endl;
for (uint32_t i = 0; i < cols.size(); i++)
cout << "HiddenStripeColExt arg " << i + 1 << ": oid=" << cols[i].oid << " width=" << cols[i].width << endl;
return;
}
err = slave->createHiddenStripeColumnExtents(cols, dbRoot, partitionNum, segmentNum, extents);
reply << (uint8_t)err;
if (err == ERR_OK)
{
reply << partitionNum;
reply << segmentNum;
serializeInlineVector(reply, extents);
}
#ifdef BRM_VERBOSE
cerr << "WorkerComm: do_createHiddenStripeColumnExtents() err code is " << err << endl;
#endif
if (!standalone)
master.write(reply);
// see bug 3596. Need to make sure a snapshot file exists.
if ((cols.size() > 0) && (cols[0].oid < 3000))
takeSnapshot = true;
else
doSaveDelta = true;
}
//------------------------------------------------------------------------------
// Process a request to make a hidden partition visible
//------------------------------------------------------------------------------
void SlaveComm::do_makePartitionVisible(ByteStream& msg)
{
int err;
uint32_t numOids;
uint32_t partitionNum;
uint16_t dbRoot;
std::set<OID_t> oids;
ByteStream reply;
#ifdef BRM_VERBOSE
cerr << "WorkerComm: do_makePartitionVisible()" << endl;
#endif
msg >> numOids;
for (uint32_t i = 0; i < numOids; i++)
{
OID_t oid;
msg >> oid;
oids.insert(oid);
}
msg >> dbRoot;
msg >> partitionNum;
if (printOnly)
{
cout << "makePartitionVisible(). " << "DBRoot=" << dbRoot << "; Part#=" << partitionNum << "; OIDs: ";
for (const auto& oid : oids)
cout << oid << " ";
cout << endl;
return;
}
err = slave->makePartitionVisible(oids, dbRoot, partitionNum);
reply << (uint8_t)err;
#ifdef BRM_VERBOSE
cerr << "WorkerComm: do_makePartitionVisible() err code is " << err << endl;
#endif
if (!standalone)
master.write(reply);
doSaveDelta = true;
}
//------------------------------------------------------------------------------
// Process a request to create a column extent for a specific OID and DBRoot.
//------------------------------------------------------------------------------

2
versioning/BRM/slavecomm.h
View File

@@ -72,6 +72,8 @@ class SlaveComm
void processCommand(messageqcpp::ByteStream& msg);
void do_createStripeColumnExtents(messageqcpp::ByteStream& msg);
void do_createHiddenStripeColumnExtents(messageqcpp::ByteStream& msg);
void do_makePartitionVisible(messageqcpp::ByteStream& msg);
void do_createColumnExtent_DBroot(messageqcpp::ByteStream& msg);
void do_createColumnExtentExactFile(messageqcpp::ByteStream& msg);
void do_createDictStoreExtent(messageqcpp::ByteStream& msg);

38
versioning/BRM/slavedbrmnode.cpp
View File

@@ -95,6 +95,44 @@ int SlaveDBRMNode::createStripeColumnExtents(const std::vector<CreateStripeColum
return 0;
}
//------------------------------------------------------------------------------
// Create hidden stripe column extents
//------------------------------------------------------------------------------
int SlaveDBRMNode::createHiddenStripeColumnExtents(const std::vector<CreateStripeColumnExtentsArgIn>& cols,
uint16_t dbRoot, uint32_t& partitionNum, uint16_t& segmentNum,
std::vector<CreateStripeColumnExtentsArgOut>& extents) throw()
{
try
{
em.createHiddenStripeColumnExtents(cols, dbRoot, partitionNum, segmentNum, extents);
}
catch (exception& e)
{
cerr << e.what() << endl;
return -1;
}
return 0;
}
//------------------------------------------------------------------------------
// Make a hidden partition visible
//------------------------------------------------------------------------------
int SlaveDBRMNode::makePartitionVisible(const std::set<OID_t>& oids, uint16_t dbRoot, uint32_t partitionNum) throw()
{
try
{
em.makePartitionVisible(oids, dbRoot, partitionNum);
}
catch (exception& e)
{
cerr << e.what() << endl;
return -1;
}
return 0;
}
//------------------------------------------------------------------------------
// Create an extent for the specified OID and DBRoot.
//------------------------------------------------------------------------------

19
versioning/BRM/slavedbrmnode.h
View File

@@ -104,6 +104,25 @@ class SlaveDBRMNode
uint16_t dbRoot, uint32_t& partitionNum, uint16_t& segmentNum,
std::vector<CreateStripeColumnExtentsArgOut>& extents) throw();
/** @brief Allocate a "stripe" of hidden (out of service) extents for columns in a table
*
* Allocate a "stripe" of hidden (out of service) extents for the specified columns and DBRoot
* @param cols (in) List of column OIDs and column widths
* @param dbRoot (in) DBRoot for requested extents.
* @param partitionNum (in/out) Partition number in file path.
* If allocating OID's first extent for this DBRoot, then
* partitionNum is input, else it is an output arg.
* @param segmentNum (out) Segment number selected for new extents.
* @param extents (out) list of lbids, numBlks, and fbo for new extents
* @return 0 on success, -1 on error */
EXPORT int createHiddenStripeColumnExtents(const std::vector<CreateStripeColumnExtentsArgIn>& cols,
uint16_t dbRoot, uint32_t& partitionNum, uint16_t& segmentNum,
std::vector<CreateStripeColumnExtentsArgOut>& extents) throw();
/** @brief Make a hidden partition visible to normal query operations
*/
EXPORT int makePartitionVisible(const std::set<OID_t>& oids, uint16_t dbRoot, uint32_t partitionNum) throw();
/** @brief Allocate extent in the specified segment file
*
* Allocate column extent for the exact segment file specified by the