database/mariadb-columnstore-engine
1
0
Fork 0
mirror of https://github.com/mariadb-corporation/mariadb-columnstore-engine.git synced 2025-07-29 08:21:15 +03:00
Code Activity

[MCOL-5106] Add support to work with StorageManager.

Browse Source 
This patch eliminates boost::filesystem from `mcsRebuildEM` tool.
After this change we should be able to work with any filesystem
even S3.
This commit is contained in:
Denis Khalikov
2022-05-30 20:15:46 +03:00
parent de522ed15a
commit fb1e23bb83
5 changed files with 154 additions and 89 deletions
Download Patch File Download Diff File Expand all files Collapse all files

61
tools/rebuildEM/main.cpp
View File

@ -18,7 +18,6 @@
#include <iostream> #include <iostream>
#include <string> #include <string>
#include <ftw.h> #include <ftw.h>
#include <boost/filesystem.hpp>
#include <boost/algorithm/string/case_conv.hpp> #include <boost/algorithm/string/case_conv.hpp>
#include "configcpp.h" #include "configcpp.h"
@ -29,7 +28,7 @@
using namespace idbdatafile; using namespace idbdatafile;
using namespace RebuildExtentMap; using namespace RebuildExtentMap;
static void usage(const string& pname) static void usage(const std::string& pname)
{ {
std::cout << "usage: " << pname << " [-vdhs]" << std::endl; std::cout << "usage: " << pname << " [-vdhs]" << std::endl;
std::cout << "rebuilds the extent map from the contents of the database file " std::cout << "rebuilds the extent map from the contents of the database file "
@ -41,6 +40,20 @@ static void usage(const string& pname)
std::cout << " -s show extent map and quit" << std::endl; std::cout << " -s show extent map and quit" << std::endl;
} }
static bool isYes()
{
std::string confirmation;
std::cin >> confirmation;
if (confirmation.size() == 0)
return false;
boost::algorithm::to_lower(confirmation);
if (!(confirmation == "y" || confirmation == "yes"))
return false;
return true;
}
int main(int argc, char** argv) int main(int argc, char** argv)
{ {
int32_t option; int32_t option;
@ -78,41 +91,28 @@ int main(int argc, char** argv)
// MCOL-4685 // MCOL-4685
std::cout << "The launch of mcsRebuildEM tool must be sanctioned by MariaDB support. " << std::endl; std::cout << "The launch of mcsRebuildEM tool must be sanctioned by MariaDB support. " << std::endl;
std::cout << "Requirement: all DBRoots must be on this node. " << std::endl;
std::cout << "Do you want to continue Y/N? "; std::cout << "Do you want to continue Y/N? ";
std::string confirmation; if (!isYes())
cin >> confirmation;
if (confirmation.size() == 0)
return 0;
boost::algorithm::to_lower(confirmation);
if (!(confirmation == "y" || confirmation == "yes"))
return 0; return 0;
auto* config = config::Config::makeConfig(); auto* config = config::Config::makeConfig();
// Check for storage type.
const auto DBRootStorageType = config->getConfig("Installation", "DBRootStorageType");
if (DBRootStorageType != "internal")
{
std::cout << "Only internal DBRootStorageType is supported, provided: " << DBRootStorageType << std::endl;
return 0;
}
const auto BRMSavesEM = config->getConfig("SystemConfig", "DBRMRoot") + "_em"; const auto BRMSavesEM = config->getConfig("SystemConfig", "DBRMRoot") + "_em";
// Check for `BRM_saves_em` file presents. // Check for `BRM_saves_em` file presents.
// TODO: Should we add force option to remove file? // TODO: Should we add force option to remove file?
if (boost::filesystem::exists(BRMSavesEM)) if (IDBPolicy::exists(BRMSavesEM.c_str()))
{ {
std::cout << BRMSavesEM << " file exists. " << std::endl; std::cout << BRMSavesEM << " file exists. " << std::endl;
std::cout << "Please note: this tool is only suitable in situations " std::cout << "Do you want to delete this file Y/N? ";
"where there is no `BRM_saves_em` file. " if (!isYes())
<< std::endl; return 0;
std::cout << "If `BRM_saves_em` "
"exists extent map will be restored from it. " if (IDBPolicy::remove(BRMSavesEM.c_str()) == -1)
<< std::endl; {
std::cout << "Exiting. " << std::endl; std::cout << "Cannot remove " << BRMSavesEM << std::endl;
return 0; std::cout << "Exiting. " << std::endl;
return 0;
}
} }
// Initialize system extents from the binary blob. // Initialize system extents from the binary blob.
@ -136,10 +136,11 @@ int main(int argc, char** argv)
auto dbRootPath = config->getConfig("SystemConfig", dbRootName); auto dbRootPath = config->getConfig("SystemConfig", dbRootName);
emReBuilder.setDBRoot(dbRootNumber); emReBuilder.setDBRoot(dbRootNumber);
emReBuilder.collectExtents(dbRootPath.c_str()); emReBuilder.collectExtents(dbRootPath.c_str());
emReBuilder.rebuildExtentMap();
emReBuilder.clear();
} }
emReBuilder.rebuildExtentMap();
emReBuilder.clear();
// Save restored extent map. // Save restored extent map.
emReBuilder.getEM().save(BRMSavesEM); emReBuilder.getEM().save(BRMSavesEM);
std::cout << "Completed." << std::endl; std::cout << "Completed." << std::endl;

73
tools/rebuildEM/rebuildEM.cpp
View File

@ -16,7 +16,6 @@
MA 02110-1301, USA. */ MA 02110-1301, USA. */
#include <iostream> #include <iostream>
#include <boost/filesystem.hpp>
#include <stdint.h> #include <stdint.h>
#include "rebuildEM.h" #include "rebuildEM.h"
@ -36,16 +35,36 @@ using namespace idbdatafile;
namespace RebuildExtentMap namespace RebuildExtentMap
{ {
int32_t EMReBuilder::collectExtents(const string& dbRootPath) void EMReBuilder::collectFileNames(const std::string& partialPath, std::string currentPath,
std::vector<std::string>& fileNames)
{
currentPath.append(partialPath);
std::list<std::string> partialPathes;
IDBPolicy::listDirectory(currentPath.c_str(), partialPathes);
if (partialPathes.size() == 0)
{
fileNames.push_back(currentPath);
return;
}
currentPath.push_back('/');
for (const auto& partialPath : partialPathes)
collectFileNames(partialPath, currentPath, fileNames);
}
int32_t EMReBuilder::collectExtents(const std::string& dbRootPath)
{ {
if (doVerbose()) if (doVerbose())
{ {
std::cout << "Collect extents for the DBRoot " << dbRootPath << std::endl; std::cout << "Collect extents for the DBRoot " << dbRootPath << std::endl;
} }
for (boost::filesystem::recursive_directory_iterator dirIt(dbRootPath), dirEnd; dirIt != dirEnd; ++dirIt) std::vector<std::string> fileNames;
collectFileNames(dbRootPath, "", fileNames);
for (const auto& fileName : fileNames)
{ {
(void)collectExtent(dirIt->path().string()); (void)collectExtent(fileName);
} }
return 0; return 0;
@ -141,8 +160,8 @@ int32_t EMReBuilder::collectExtent(const std::string& fullFileName)
} }
uint64_t hwm = 0; uint64_t hwm = 0;
rc = searchHWMInSegmentFile(oid, getDBRoot(), partition, segment, colDataType, colWidth, blockCount, isDict, rc = searchHWMInSegmentFile(fullFileName, oid, getDBRoot(), partition, segment, colDataType, colWidth,
compressionType, hwm); blockCount, isDict, compressionType, hwm);
if (rc != 0) if (rc != 0)
return rc; return rc;
@ -156,13 +175,13 @@ int32_t EMReBuilder::collectExtent(const std::string& fullFileName)
for (uint32_t lbidIndex = 0; lbidIndex < lbidCount - 1; ++lbidIndex) for (uint32_t lbidIndex = 0; lbidIndex < lbidCount - 1; ++lbidIndex)
{ {
auto lbid = compress::CompressInterface::getLBIDByIndex(fileHeader, lbidIndex); auto lbid = compress::CompressInterface::getLBIDByIndex(fileHeader, lbidIndex);
FileId fileId(oid, partition, segment, colWidth, colDataType, lbid, /*hwm*/ 0, isDict); FileId fileId(oid, partition, segment, getDBRoot(), colWidth, colDataType, lbid, /*hwm*/ 0, isDict);
extentMap.push_back(fileId); extentMap.push_back(fileId);
} }
// Last one has an actual HWM. // Last one has an actual HWM.
auto lbid = compress::CompressInterface::getLBIDByIndex(fileHeader, lbidCount - 1); auto lbid = compress::CompressInterface::getLBIDByIndex(fileHeader, lbidCount - 1);
FileId fileId(oid, partition, segment, colWidth, colDataType, lbid, hwm, isDict); FileId fileId(oid, partition, segment, getDBRoot(), colWidth, colDataType, lbid, hwm, isDict);
extentMap.push_back(fileId); extentMap.push_back(fileId);
if (doVerbose()) if (doVerbose())
@ -175,7 +194,7 @@ int32_t EMReBuilder::collectExtent(const std::string& fullFileName)
{ {
// One extent per segment file. // One extent per segment file.
auto lbid = compress::CompressInterface::getLBIDByIndex(fileHeader, 0); auto lbid = compress::CompressInterface::getLBIDByIndex(fileHeader, 0);
FileId fileId(oid, partition, segment, colWidth, colDataType, lbid, hwm, isDict); FileId fileId(oid, partition, segment, getDBRoot(), colWidth, colDataType, lbid, hwm, isDict);
extentMap.push_back(fileId); extentMap.push_back(fileId);
if (doVerbose()) if (doVerbose())
@ -213,14 +232,14 @@ int32_t EMReBuilder::rebuildExtentMap()
{ {
// Create a dictionary extent for the given oid, partition, // Create a dictionary extent for the given oid, partition,
// segment, dbroot. // segment, dbroot.
getEM().createDictStoreExtent(fileId.oid, getDBRoot(), fileId.partition, fileId.segment, lbid, getEM().createDictStoreExtent(fileId.oid, fileId.dbroot, fileId.partition, fileId.segment, lbid,
allocdSize); allocdSize);
} }
else else
{ {
// Create a column extent for the given oid, partition, // Create a column extent for the given oid, partition,
// segment, dbroot and column width. // segment, dbroot and column width.
getEM().createColumnExtentExactFile(fileId.oid, fileId.colWidth, getDBRoot(), fileId.partition, getEM().createColumnExtentExactFile(fileId.oid, fileId.colWidth, fileId.dbroot, fileId.partition,
fileId.segment, fileId.colDataType, lbid, allocdSize, fileId.segment, fileId.colDataType, lbid, allocdSize,
startBlockOffset); startBlockOffset);
} }
@ -263,8 +282,8 @@ int32_t EMReBuilder::rebuildExtentMap()
return 0; return 0;
} }
int32_t EMReBuilder::searchHWMInSegmentFile(uint32_t oid, uint32_t dbRoot, uint32_t partition, int32_t EMReBuilder::searchHWMInSegmentFile(const std::string& fullFileName, uint32_t oid, uint32_t dbRoot,
uint32_t segment, uint32_t partition, uint32_t segment,
execplan::CalpontSystemCatalog::ColDataType colDataType, execplan::CalpontSystemCatalog::ColDataType colDataType,
uint32_t colWidth, uint64_t blockCount, bool isDict, uint32_t colWidth, uint64_t blockCount, bool isDict,
uint32_t compressionType, uint64_t& hwm) uint32_t compressionType, uint64_t& hwm)
@ -275,12 +294,12 @@ int32_t EMReBuilder::searchHWMInSegmentFile(uint32_t oid, uint32_t dbRoot, uint3
if (isDict) if (isDict)
{ {
chunkManagerWrapper = std::unique_ptr<ChunkManagerWrapperDict>(new ChunkManagerWrapperDict( chunkManagerWrapper = std::unique_ptr<ChunkManagerWrapperDict>(new ChunkManagerWrapperDict(
oid, dbRoot, partition, segment, colDataType, colWidth, compressionType)); fullFileName, oid, dbRoot, partition, segment, colDataType, colWidth, compressionType));
} }
else else
{ {
chunkManagerWrapper = std::unique_ptr<ChunkManagerWrapperColumn>(new ChunkManagerWrapperColumn( chunkManagerWrapper = std::unique_ptr<ChunkManagerWrapperColumn>(new ChunkManagerWrapperColumn(
oid, dbRoot, partition, segment, colDataType, colWidth, compressionType)); fullFileName, oid, dbRoot, partition, segment, colDataType, colWidth, compressionType));
} }
} }
catch (...) catch (...)
@ -348,7 +367,8 @@ int32_t EMReBuilder::initializeSystemExtents()
return 0; return 0;
} }
ChunkManagerWrapper::ChunkManagerWrapper(uint32_t oid, uint32_t dbRoot, uint32_t partition, uint32_t segment, ChunkManagerWrapper::ChunkManagerWrapper(const std::string& filename, uint32_t oid, uint32_t dbRoot,
uint32_t partition, uint32_t segment,
execplan::CalpontSystemCatalog::ColDataType colDataType, execplan::CalpontSystemCatalog::ColDataType colDataType,
uint32_t colWidth) uint32_t colWidth)
: oid(oid) : oid(oid)
@ -359,6 +379,7 @@ ChunkManagerWrapper::ChunkManagerWrapper(uint32_t oid, uint32_t dbRoot, uint32_t
, colWidth(colWidth) , colWidth(colWidth)
, size(colWidth) , size(colWidth)
, pFileOp(nullptr) , pFileOp(nullptr)
, fileName(filename)
{ {
} }
@ -370,19 +391,19 @@ int32_t ChunkManagerWrapper::readBlock(uint32_t blockNumber)
return 0; return 0;
} }
ChunkManagerWrapperColumn::ChunkManagerWrapperColumn(uint32_t oid, uint32_t dbRoot, uint32_t partition, ChunkManagerWrapperColumn::ChunkManagerWrapperColumn(const std::string& filename, uint32_t oid,
uint32_t segment, uint32_t dbRoot, uint32_t partition, uint32_t segment,
execplan::CalpontSystemCatalog::ColDataType colDataType, execplan::CalpontSystemCatalog::ColDataType colDataType,
uint32_t colWidth, uint32_t compressionType) uint32_t colWidth, uint32_t compressionType)
: ChunkManagerWrapper(oid, dbRoot, partition, segment, colDataType, colWidth) : ChunkManagerWrapper(filename, oid, dbRoot, partition, segment, colDataType, colWidth)
{ {
pFileOp = pFileOp =
std::unique_ptr<WriteEngine::ColumnOpCompress1>(new WriteEngine::ColumnOpCompress1(compressionType)); std::unique_ptr<WriteEngine::ColumnOpCompress1>(new WriteEngine::ColumnOpCompress1(compressionType));
chunkManager.fileOp(pFileOp.get()); chunkManager.fileOp(pFileOp.get());
// Open compressed column segment file. We will read block by block // Open compressed column segment file. We will read block by block
// from the compressed chunks. // from the compressed chunks.
pFile = chunkManager.getSegmentFilePtr(oid, dbRoot, partition, segment, colDataType, colWidth, fileName, pFile = chunkManager.getFilePtrByName(fileName, oid, dbRoot, partition, segment, colDataType, colWidth,
"rb", size, false, false); "rb", size, false, false);
if (!pFile) if (!pFile)
{ {
throw std::bad_alloc(); throw std::bad_alloc();
@ -425,17 +446,17 @@ bool ChunkManagerWrapperColumn::isEmptyValue(const uint8_t* value) const
return false; return false;
} }
ChunkManagerWrapperDict::ChunkManagerWrapperDict(uint32_t oid, uint32_t dbRoot, uint32_t partition, ChunkManagerWrapperDict::ChunkManagerWrapperDict(const std::string& filename, uint32_t oid, uint32_t dbRoot,
uint32_t segment, uint32_t partition, uint32_t segment,
execplan::CalpontSystemCatalog::ColDataType colDataType, execplan::CalpontSystemCatalog::ColDataType colDataType,
uint32_t colWidth, uint32_t compressionType) uint32_t colWidth, uint32_t compressionType)
: ChunkManagerWrapper(oid, dbRoot, partition, segment, colDataType, colWidth) : ChunkManagerWrapper(filename, oid, dbRoot, partition, segment, colDataType, colWidth)
{ {
pFileOp = std::unique_ptr<WriteEngine::DctnryCompress1>(new WriteEngine::DctnryCompress1(compressionType)); pFileOp = std::unique_ptr<WriteEngine::DctnryCompress1>(new WriteEngine::DctnryCompress1(compressionType));
chunkManager.fileOp(pFileOp.get()); chunkManager.fileOp(pFileOp.get());
// Open compressed dict segment file. // Open compressed dict segment file.
pFile = chunkManager.getSegmentFilePtr(oid, dbRoot, partition, segment, colDataType, colWidth, fileName, pFile = chunkManager.getFilePtrByName(fileName, oid, dbRoot, partition, segment, colDataType, colWidth,
"rb", size, false, true); "rb", size, false, true);
if (!pFile) if (!pFile)
{ {
throw std::bad_alloc(); throw std::bad_alloc();

32
tools/rebuildEM/rebuildEM.h
View File

@ -39,11 +39,12 @@ namespace RebuildExtentMap
// This struct represents a FileId. For internal purpose only. // This struct represents a FileId. For internal purpose only.
struct FileId struct FileId
{ {
FileId(uint32_t oid, uint32_t partition, uint32_t segment, uint32_t colWidth, FileId(uint32_t oid, uint32_t partition, uint32_t segment, uint32_t dbroot, uint32_t colWidth,
execplan::CalpontSystemCatalog::ColDataType colDataType, int64_t lbid, uint64_t hwm, bool isDict) execplan::CalpontSystemCatalog::ColDataType colDataType, int64_t lbid, uint64_t hwm, bool isDict)
: oid(oid) : oid(oid)
, partition(partition) , partition(partition)
, segment(segment) , segment(segment)
, dbroot(dbroot)
, colWidth(colWidth) , colWidth(colWidth)
, colDataType(colDataType) , colDataType(colDataType)
, lbid(lbid) , lbid(lbid)
@ -55,6 +56,7 @@ struct FileId
uint32_t oid; uint32_t oid;
uint32_t partition; uint32_t partition;
uint32_t segment; uint32_t segment;
uint32_t dbroot;
uint32_t colWidth; uint32_t colWidth;
execplan::CalpontSystemCatalog::ColDataType colDataType; execplan::CalpontSystemCatalog::ColDataType colDataType;
int64_t lbid; int64_t lbid;
@ -70,13 +72,17 @@ class EMReBuilder
EMReBuilder(bool verbose, bool display) : verbose(verbose), display(display) EMReBuilder(bool verbose, bool display) : verbose(verbose), display(display)
{ {
// Initalize plugins. // Initalize plugins.
IDBPolicy::init(true, false, "", 0); IDBPolicy::configIDBPolicy();
} }
~EMReBuilder() = default; ~EMReBuilder() = default;
// Collects extents from the given DBRoot path. // Collects extents from the given DBRoot path.
int32_t collectExtents(const std::string& dbRootPath); int32_t collectExtents(const std::string& dbRootPath);
// Collects file names for the given `partialPath` direcotory.
void collectFileNames(const std::string& partialPath, std::string currentPath,
std::vector<std::string>& fileNames);
// Clears collected extents. // Clears collected extents.
void clear() void clear()
{ {
@ -124,7 +130,8 @@ class EMReBuilder
int32_t rebuildExtentMap(); int32_t rebuildExtentMap();
// Search HWM in the given segment file. // Search HWM in the given segment file.
int32_t searchHWMInSegmentFile(uint32_t oid, uint32_t dbRoot, uint32_t partition, uint32_t segment, int32_t searchHWMInSegmentFile(const std::string& fullFileName, uint32_t oid, uint32_t dbRoot,
uint32_t partition, uint32_t segment,
execplan::CalpontSystemCatalog::ColDataType colDataType, uint32_t width, execplan::CalpontSystemCatalog::ColDataType colDataType, uint32_t width,
uint64_t blocksCount, bool isDict, uint32_t compressionType, uint64_t& hwm); uint64_t blocksCount, bool isDict, uint32_t compressionType, uint64_t& hwm);
@ -158,8 +165,9 @@ class EMReBuilder
class ChunkManagerWrapper class ChunkManagerWrapper
{ {
public: public:
ChunkManagerWrapper(uint32_t oid, uint32_t dbRoot, uint32_t partition, uint32_t segment, ChunkManagerWrapper(const std::string& filename, uint32_t oid, uint32_t dbRoot, uint32_t partition,
execplan::CalpontSystemCatalog::ColDataType colDataType, uint32_t colWidth); uint32_t segment, execplan::CalpontSystemCatalog::ColDataType colDataType,
uint32_t colWidth);
virtual ~ChunkManagerWrapper() = default; virtual ~ChunkManagerWrapper() = default;
ChunkManagerWrapper(const ChunkManagerWrapper& other) = delete; ChunkManagerWrapper(const ChunkManagerWrapper& other) = delete;
@ -182,8 +190,8 @@ class ChunkManagerWrapper
execplan::CalpontSystemCatalog::ColDataType colDataType; execplan::CalpontSystemCatalog::ColDataType colDataType;
uint32_t colWidth; uint32_t colWidth;
int32_t size; int32_t size;
std::string fileName;
std::unique_ptr<WriteEngine::FileOp> pFileOp; std::unique_ptr<WriteEngine::FileOp> pFileOp;
std::string fileName;
// Note: We cannot clear this pointer directly, because // Note: We cannot clear this pointer directly, because
// `ChunkManager` closes this file for us, otherwise we will get double // `ChunkManager` closes this file for us, otherwise we will get double
// free error. // free error.
@ -196,9 +204,9 @@ class ChunkManagerWrapper
class ChunkManagerWrapperColumn : public ChunkManagerWrapper class ChunkManagerWrapperColumn : public ChunkManagerWrapper
{ {
public: public:
ChunkManagerWrapperColumn(uint32_t oid, uint32_t dbRoot, uint32_t partition, uint32_t segment, ChunkManagerWrapperColumn(const std::string& filename, uint32_t oid, uint32_t dbRoot, uint32_t partition,
execplan::CalpontSystemCatalog::ColDataType colDataType, uint32_t colWidth, uint32_t segment, execplan::CalpontSystemCatalog::ColDataType colDataType,
uint32_t compressionType); uint32_t colWidth, uint32_t compressionType);
~ChunkManagerWrapperColumn() = default; ~ChunkManagerWrapperColumn() = default;
ChunkManagerWrapperColumn(const ChunkManagerWrapperColumn& other) = delete; ChunkManagerWrapperColumn(const ChunkManagerWrapperColumn& other) = delete;
@ -219,9 +227,9 @@ class ChunkManagerWrapperColumn : public ChunkManagerWrapper
class ChunkManagerWrapperDict : public ChunkManagerWrapper class ChunkManagerWrapperDict : public ChunkManagerWrapper
{ {
public: public:
ChunkManagerWrapperDict(uint32_t oid, uint32_t dbRoot, uint32_t partition, uint32_t segment, ChunkManagerWrapperDict(const std::string& filename, uint32_t oid, uint32_t dbRoot, uint32_t partition,
execplan::CalpontSystemCatalog::ColDataType colDataType, uint32_t colWidth, uint32_t segment, execplan::CalpontSystemCatalog::ColDataType colDataType,
uint32_t compressionType); uint32_t colWidth, uint32_t compressionType);
~ChunkManagerWrapperDict() = default; ~ChunkManagerWrapperDict() = default;
ChunkManagerWrapperDict(const ChunkManagerWrapperDict& other) = delete; ChunkManagerWrapperDict(const ChunkManagerWrapperDict& other) = delete;

59
writeengine/shared/we_chunkmanager.cpp
View File

@ -280,19 +280,20 @@ IDBDataFile* ChunkManager::getFilePtr(const FID& fid, uint16_t root, uint32_t pa
} }
//------------------------------------------------------------------------------ //------------------------------------------------------------------------------
// Get/Return IDBDataFile* for specified OID, root, partition, and segment. // Get/Return IDBDataFile* by the given `filename`.
// OID, partition and segment are needed for a file cache.
// Function is to be used to open column/dict segment file. // Function is to be used to open column/dict segment file.
// If the IDBDataFile* is not found, then a segment file will be opened using // If the IDBDataFile* is not found, then a segment file will be opened using
// the mode (mode) and I/O buffer size (size) that is given. Name of the // the mode (mode) and I/O buffer size (size) that is given.
// resulting file is returned in filename.
//------------------------------------------------------------------------------ //------------------------------------------------------------------------------
IDBDataFile* ChunkManager::getSegmentFilePtr(FID& fid, uint16_t root, uint32_t partition, uint16_t segment, IDBDataFile* ChunkManager::getFilePtrByName(const std::string& filename, FID& fid, uint16_t root,
execplan::CalpontSystemCatalog::ColDataType colDataType, uint32_t partition, uint16_t segment,
uint32_t colWidth, std::string& filename, const char* mode, execplan::CalpontSystemCatalog::ColDataType colDataType,
int32_t size, bool useTmpSuffix, bool isDict) const uint32_t colWidth, const char* mode, int32_t size,
bool useTmpSuffix, bool isDict) const
{ {
CompFileData* fileData = getFileData(fid, root, partition, segment, filename, mode, size, colDataType, CompFileData* fileData = getFileDataByName(filename, fid, root, partition, segment, mode, size, colDataType,
colWidth, useTmpSuffix, isDict); colWidth, useTmpSuffix, isDict);
return (fileData ? fileData->fFilePtr : NULL); return (fileData ? fileData->fFilePtr : NULL);
} }
@ -328,12 +329,39 @@ CompFileData* ChunkManager::getFileData(const FID& fid, uint16_t root, uint32_t
// New CompFileData pointer needs to be created // New CompFileData pointer needs to be created
char name[FILE_NAME_SIZE]; char name[FILE_NAME_SIZE];
if (fFileOp->getFileName(fid, name, root, partition, segment) != NO_ERROR) if (fFileOp->getFileName(fid, name, root, partition, segment) != NO_ERROR)
return NULL; return NULL;
CompFileData* fileData = new CompFileData(fileID, fid, colDataType, colWidth); // Initialize the given `filename`.
fileData->fFileName = filename = name; filename = name;
return getFileData_(fileID, filename, mode, size, colDataType, colWidth, useTmpSuffix, dctnry);
}
CompFileData* ChunkManager::getFileDataByName(const std::string& filename, const FID& fid, uint16_t root,
uint32_t partition, uint16_t segment, const char* mode,
int size, const CalpontSystemCatalog::ColDataType colDataType,
int colWidth, bool useTmpSuffix, bool dctnry) const
{
FileID fileID(fid, root, partition, segment);
map<FileID, CompFileData*>::const_iterator mit = fFileMap.find(fileID);
WE_COMP_DBG(cout << "getFileData: fid:" << fid << " root:" << root << " part:" << partition << " seg:"
<< segment << " file* " << ((mit != fFileMap.end()) ? "" : "not ") << "found." << endl;)
// Get CompFileData pointer for existing Column or Dictionary store file
if (mit != fFileMap.end())
return mit->second;
return getFileData_(fileID, filename, mode, size, colDataType, colWidth, useTmpSuffix, dctnry);
}
CompFileData* ChunkManager::getFileData_(const FileID& fileID, const string& filename, const char* mode,
int size, const CalpontSystemCatalog::ColDataType colDataType,
int colWidth, bool useTmpSuffix, bool dctnry) const
{
CompFileData* fileData = new CompFileData(fileID, fileID.fFid, colDataType, colWidth);
fileData->fFileName = filename;
if (openFile(fileData, mode, colWidth, useTmpSuffix, __LINE__) != NO_ERROR) if (openFile(fileData, mode, colWidth, useTmpSuffix, __LINE__) != NO_ERROR)
{ {
@ -369,13 +397,12 @@ CompFileData* ChunkManager::getFileData(const FID& fid, uint16_t root, uint32_t
return NULL; return NULL;
} }
int headerSize = compress::CompressInterface::getHdrSize(fileData->fFileHeader.fControlData); const int32_t headerSize = compress::CompressInterface::getHdrSize(fileData->fFileHeader.fControlData);
int ptrSecSize = headerSize - COMPRESSED_FILE_HEADER_UNIT; const int32_t ptrSecSize = headerSize - COMPRESSED_FILE_HEADER_UNIT;
// Save segment file compression type. // Save segment file compression type.
uint32_t compressionType = fileData->fCompressionType =
compress::CompressInterface::getCompressionType(fileData->fFileHeader.fControlData); compress::CompressInterface::getCompressionType(fileData->fFileHeader.fControlData);
fileData->fCompressionType = compressionType;
if (ptrSecSize > COMPRESSED_FILE_HEADER_UNIT) if (ptrSecSize > COMPRESSED_FILE_HEADER_UNIT)
{ {

18
writeengine/shared/we_chunkmanager.h
View File

@ -189,12 +189,12 @@ class ChunkManager
IDBDataFile* getFilePtr(const FID& fid, uint16_t root, uint32_t partition, uint16_t segment, IDBDataFile* getFilePtr(const FID& fid, uint16_t root, uint32_t partition, uint16_t segment,
std::string& filename, const char* mode, int size, bool useTmpSuffix) const; std::string& filename, const char* mode, int size, bool useTmpSuffix) const;
// @brief Retrieve a file pointer in the chunk manager. // @brief Retrieve a file pointer in the chunk manager by the given `filename`.
// for column/dict segment file // for column/dict segment file
IDBDataFile* getSegmentFilePtr(FID& fid, uint16_t root, uint32_t partition, uint16_t segment, IDBDataFile* getFilePtrByName(const std::string& filename, FID& fid, uint16_t root, uint32_t partition,
execplan::CalpontSystemCatalog::ColDataType colDataType, uint32_t colWidth, uint16_t segment, execplan::CalpontSystemCatalog::ColDataType colDataType,
std::string& filename, const char* mode, int32_t size, bool useTmpSuffix, uint32_t colWidth, const char* mode, int32_t size, bool useTmpSuffix,
bool isDict) const; bool isDict) const;
// @brief Create a compressed dictionary file with an appropriate header. // @brief Create a compressed dictionary file with an appropriate header.
IDBDataFile* createDctnryFile(const FID& fid, int64_t width, uint16_t root, uint32_t partition, IDBDataFile* createDctnryFile(const FID& fid, int64_t width, uint16_t root, uint32_t partition,
@ -284,6 +284,11 @@ class ChunkManager
const execplan::CalpontSystemCatalog::ColDataType colDataType, int colWidth, const execplan::CalpontSystemCatalog::ColDataType colDataType, int colWidth,
bool useTmpSuffix, bool dictnry = false) const; bool useTmpSuffix, bool dictnry = false) const;
CompFileData* getFileDataByName(const std::string& filename, const FID& fid, uint16_t root,
uint32_t partition, uint16_t segment, const char* mode, int size,
const execplan::CalpontSystemCatalog::ColDataType colDataType, int colWidth,
bool useTmpSuffix, bool dctnry) const;
// @brief Retrieve a chunk of pFile from disk. // @brief Retrieve a chunk of pFile from disk.
int fetchChunkFromFile(IDBDataFile* pFile, int64_t id, ChunkData*& chunkData); int fetchChunkFromFile(IDBDataFile* pFile, int64_t id, ChunkData*& chunkData);
@ -366,6 +371,9 @@ class ChunkManager
size_t COMPRESSED_CHUNK_SIZE; size_t COMPRESSED_CHUNK_SIZE;
private: private:
CompFileData* getFileData_(const FileID& fid, const std::string& filename, const char* mode, int size,
const execplan::CalpontSystemCatalog::ColDataType colDataType, int colWidth,
bool useTmpSuffix, bool dictnry = false) const;
}; };
} // namespace WriteEngine } // namespace WriteEngine