database/mariadb-columnstore-engine
1
0
Fork 0
mirror of https://github.com/mariadb-corporation/mariadb-columnstore-engine.git synced 2025-10-31 18:30:33 +03:00
Code Activity
Files
f6c479ab40c5b09bd230a35bdca8cde83ca03082
mariadb-columnstore-engine/writeengine/shared/we_fileop.h
Denis Khalikov 80277a5e7d DBBuilder: check syscat files instead dirs
2025-04-01 19:53:28 +04:00

513 lines
20 KiB
C++
Raw Blame History

/* Copyright (C) 2014 InfiniDB, Inc.
This program is free software; you can redistribute it and/or
modify it under the terms of the GNU General Public License
as published by the Free Software Foundation; version 2 of
the License.
This program is distributed in the hope that it will be useful,
but WITHOUT ANY WARRANTY; without even the implied warranty of
MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
GNU General Public License for more details.
You should have received a copy of the GNU General Public License
along with this program; if not, write to the Free Software
Foundation, Inc., 51 Franklin Street, Fifth Floor, Boston,
MA 02110-1301, USA. */
// $Id: we_fileop.h 4737 2013-08-14 20:45:46Z bwilkinson $
/** @file */
#pragma once
#include <sys/types.h>
#include <sys/stat.h>
#include <string>
#include <vector>
#include <map>
#include <boost/thread.hpp>
#include "we_blockop.h"
#include "we_brm.h"
#include "we_config.h"
#include "we_stats.h"
#include "idbcompress.h"
#include "calpontsystemcatalog.h"
#define EXPORT
#define MAX_NBLOCKS 8192
#include "brmtypes.h"
/** Namespace WriteEngine */
namespace WriteEngine
{
constexpr size_t MaxDirLevels = 5;
/** Class FileOp */
class FileOp : public BlockOp, public WeUIDGID
{
public:
/**
* @brief Constructor
*/
EXPORT explicit FileOp(bool doAlloc = true);
/**
* @brief Destructor
*/
EXPORT virtual ~FileOp();
/**
* @brief Close a file
*/
EXPORT void closeFile(IDBDataFile* pFile) const;
/**
* @brief Create a directory
*/
EXPORT int createDir(const char* dirName, mode_t mode) const;
int createDir(const char* dirName) const;
/**
* @brief Create a file with a fixed file size and file id
*/
EXPORT int createFile(FID fid, int& allocSize, uint16_t dbRoot, uint32_t partition,
execplan::CalpontSystemCatalog::ColDataType colDataType, const uint8_t* emptyVal,
int width = 1);
/**
* @brief Create a file with a fixed file size by its name.
* Changed to public for UT.
*/
int createFile(const char* fileName, int fileSize, const uint8_t* emptyVal, int width,
execplan::CalpontSystemCatalog::ColDataType colDataType, uint16_t dbRoot,
BRM::LBID_t lbid = -1);
/**
* @brief Delete a file
*/
EXPORT int deleteFile(const char* fileName) const;
/**
* @brief Delete the db files corresponding to the specified file id
*/
EXPORT int deleteFile(FID fid) const;
/**
* @brief Delete the db files corresponding to the specified file id
*/
EXPORT int deleteFiles(const std::vector<int32_t>& fids) const;
/**
* @brief Delete db files corresponding to specified file id and partition
*/
EXPORT int deletePartitions(const std::vector<OID>& fids,
const std::vector<BRM::PartitionInfo>& partitions) const;
/**
* @brief Delete a specific database segment file.
*/
EXPORT int deleteFile(FID fid, uint16_t dbRoot, uint32_t partition, uint16_t segment) const;
/**
* @brief Check whether a file exists or not
*/
EXPORT bool exists(const char* fileName) const;
/**
* @brief @brief Check whether file exists or not by using file id, DBRoot,
* partition, and segment number.
*/
EXPORT bool exists(FID fid, uint16_t dbRoot, uint32_t partition, uint16_t segment) const;
/**
* @brief Check whether a column exists or not by using file id. Since this
* is not enough to fully qualify a db filename, all it can do is to verify
* that the OID directory exists on one or more of the DBRoots.
*/
EXPORT bool existsOIDDir(FID fid) const;
/**
* @brief Check whether a column file exists by the given `fid` in all
* available db roots and default values for `partition` and `segment`.
*/
EXPORT bool existsDefaultFile(FID fid) const;
/**
* @brief Expand current abbreviated extent for this column to a full extent
*
* @param pFile FILE ptr of segment file we are updating.
* @param dbRoot DBRoot of the file being updated.
* @param emptyVal Empty value used in initializing extents for this column
* @param width Width of this column (in bytes)
* @param colDataType Column data type.
*/
EXPORT virtual int expandAbbrevColumnExtent(IDBDataFile* pFile, uint16_t dbRoot, const uint8_t* emptyVal,
int width,
execplan::CalpontSystemCatalog::ColDataType colDataType);
/**
* @brief Add an extent to the specified Column OID and DBRoot.
* The extent must already exist in the extentmap prior to calling this fctn.
*
* The partition, segment, and HWM of the column file where the
* extent is added is returned. If needed, the applicable column segment
* file will be created. This extendFile1 function should supplant other
* extendFile functions with Multiple-File-per-OID enhancement, "but" we
* may want to rethink when we do Shared-Nothing. When this function
* returns, the file position will be located at the end of the file.
* For shared-everything DBRoot was an output argument, as BRM selected the
* the DBRoot. For shared-nothing DBRoot is an input argument, as the
* application code must track/control the DBRoot selection.
* If this is the very first file for the specified DBRoot, then the
* partition and segment number must be specified, else the selected
* partition and segment numbers are returned.
*
* @param oid OID of the column to be extended
* @param emptyVal Empty value to be used for oid
* @param width Width of the column
* @param hwm The fbo of the column segment file where the new extent begins
* @param startLbid The starting LBID for the new extent
* @param allocSize Number of blocks allocated to the extent.
* @param dbRoot The DBRoot of the file with the new extent.
* @param partition The partnum of the file with the new extent.
* @param segment The segnum of the file with the new extent.
* @param segFile (out) Name of the segment file where extent was added.
* @param pFile (out) FILE ptr to the file where the extent is added.
* @param newFile (out) Indicates if a new file was created for the extent
* @param hdrs (in/out) Contents of headers, if file is compressed.
* @return returns NO_ERROR if success.
*/
EXPORT int extendFile(OID oid, const uint8_t* emptyVal, int width,
execplan::CalpontSystemCatalog::ColDataType colDataType, HWM hwm,
BRM::LBID_t startLbid, int allocSize, uint16_t dbRoot, uint32_t partition,
uint16_t segment, std::string& segFile, IDBDataFile*& pFile, bool& newFile,
char* hdrs);
/**
* @brief For alter table add column; add an extent to a specific file
*
* @param oid OID of the column to be extended
* @param emptyVal Empty value to be used for oid
* @param width Width of the column
* @param allocSize (out) Number of blocks allocated to the extent.
* @param dbRoot The DBRoot of the file with the new extent.
* @param partition The partnum of the file with the new extent.
* @param segment The segnum of the file with the new extent.
* @param segFile (out) Name of the segment file where extent was added.
* @param startLbid (out) The starting LBID for the new extent
* @param newFile (out) Indicates if a new file was created for the extent
* @param hdrs (in/out) Contents of headers, if file is compressed.
*/
EXPORT int addExtentExactFile(OID oid, const uint8_t* emptyVal, int width, int& allocSize, uint16_t dbRoot,
uint32_t partition, uint16_t segment,
execplan::CalpontSystemCatalog::ColDataType colDataType, std::string& segFile,
BRM::LBID_t& startLbid, bool& newFile, char* hdrs);
/**
* @brief Pad the specified compressed extent with empty chunks
* @param oid OID of relevant column
* @param width Width in bytes of this column
* @param emptyVal Empty value to be employed in filling the chunks
* @param dbRoot DBRoot of the extent to be filled
* @param partition Partition of the extent to be filled
* @param segment Segment file number of the extent to be filled
* @param colDataType Column data type
* @param hwm New HWM blk setting for the segment file after extent is padded
* @param segFile (out) Name of updated segment file
* @param errTask (out) Task that failed if error occurs
* @return returns NO_ERROR if success.
*/
EXPORT int fillCompColumnExtentEmptyChunks(OID oid, int colWidth, const uint8_t* emptyVal, uint16_t dbRoot,
uint32_t partition, uint16_t segment,
execplan::CalpontSystemCatalog::ColDataType colDataType, HWM hwm,
std::string& segFile, std::string& errTask);
/**
* @brief Write the specified header info to compressed column file pFile.
*
* @param pFile Column file to be written to
* @param hdr Header info to be written
*/
EXPORT int writeHeaders(IDBDataFile* pFile, const char* hdr) const;
/**
* @brief Write the specified header info to compressed column or
* dictionary file pFile.
*
* @param pFile Column file to be written to
* @param controlHdr Control header info to be written
* @param pointerHdr Pointer header info to be written
* @param ptrHdrSize Size (in bytes) of pointerHdr
*/
EXPORT int writeHeaders(IDBDataFile* pFile, const char* controlHdr, const char* pointerHdr,
uint64_t ptrHdrSize) const;
/**
* @brief Get the Version Buffer filename for the specified fid (OID).
*
* This version of getFileName automatically uses 0 for the partition and
* segment numbers. The applicable DBRoot is assigned based on the OID.
*
* @param fid (in) OID of the Version Buffer DB file of interest
* @param fileName (out) the name of the pertinent file that was found
*
* @return returns NO_ERROR if success; ERR_FILE_NOT_EXIST if file not found
*/
int getVBFileName(FID fid, char* fileName) const;
/**
* @brief Get the filename for the specified fid (OID). DBRoot, partition,
* and segment number.
*
* @param fid (in) OID of the DB file of interest
* @param fileName (out) the name of the pertinent file that was found
* @param dbRoot (in) DBRoot of the file of interest. If 0, then all the
* DBRoots will be searched.
* @param partition (in) partition number of the file of interest
* @param segment (in) segment number of the file of interest
*/
int getFileName(FID fid, char* fileName, uint16_t dbRoot, uint32_t partition, uint16_t segment) const;
/* Added for MCOL-3251 */
void getFileNameForPrimProc(FID fid, char* fileName, uint16_t dbRoot, uint32_t partition,
uint16_t segment) const;
/**
* @brief Construct directory path for the specified fid (OID), DBRoot, and
* partition number. Directory does not have to exist, nor is it created.
*/
int getDirName(FID fid, uint16_t dbRoot, uint32_t partition, std::string& dirName) const;
/**
* @brief Get the file size
*/
EXPORT int getFileSize(IDBDataFile* pFile, long long& fileSize) const;
EXPORT int getFileSize(FID fid, uint16_t dbRoot, uint32_t partition, uint16_t segment,
long long& fileSize) const;
/**
* @brief Initialize an extent in a dictionary store file
* @param pFile (in) IDBDataFile* of dictionary store file to be written to
* @param dbRoot (in) - DBRoot of pFile
* @param nBlocks (in) - number of blocks to be written for an extent
* @param blockHdrInit(in) - data used to initialize each block header
* @param blockHdrInitSize(in) - number of bytes in blockHdrInit
* @param bExpandExtent (in) - Expand existing extent, or initialize new one
* @param bOptExtension (in) - skip or optimize full extent preallocation
*/
EXPORT int initDctnryExtent(IDBDataFile* pFile, uint16_t dbRoot, int nBlocks, unsigned char* blockHdrInit,
int blockHdrInitSize, bool bExpandExtent, bool bOptExtension = false,
int64_t lbid = 0);
/**
* @brief Check whether it is an directory
*/
EXPORT bool isDir(const char* dirName) const;
/**
* @brief See if there is room in the file system for specific number of blks
* @param fileName Name of file to extend (does not have to be full name)
* @param nBlocks Number of 8192-byte blocks to be added
* @return returns TRUE if file system has room for 'nBlocks', else FALSE
*/
EXPORT bool isDiskSpaceAvail(const std::string& fileName, int nBlocks) const;
/**
* @brief Convert an oid to a full file name
*/
EXPORT int oid2FileName(FID fid, char* fullFileName, bool bCreateDir, uint16_t dbRoot, uint32_t partition,
uint16_t segment) const;
EXPORT int oid2DirName(FID fid, char* oidDirName) const;
/**
* @brief Open a file using a filename.
* @param fileName Name of the file to open.
* @param mode Mode to use in opening the file (ex: "r+b").
* @param ioBuffSize Buffer size to be employed by setvbuf().
* @return returns the IDBDataFile* of the opened file.
*/
EXPORT IDBDataFile* openFile(const char* fileName, const char* mode = "r+b", int ioColSize = DEFAULT_COLSIZ,
bool useTmpSuffix = false) const;
/**
* @brief Open a file using an OID, dbroot, partition, and segment number.
* @param fid OID of the file to be opened.
* @param dbRoot DBRoot of the file to be opened.
* @param partition Partition number of the file to be opened.
* @param segment Segment number of the file to be opened.
* @param mode Mode to use in opening the file (default of "r+b" will open
* an existing binary file as read/write.
* @param ioBuffSize Buffer size to be employed by setvbuf().
* @return returns the IDBDataFile* of the opened file.
*/
EXPORT IDBDataFile* openFile(FID fid, uint16_t dbRoot, uint32_t partition, uint16_t segment,
std::string& segFile, const char* mode = "r+b", int ioColSize = DEFAULT_COLSIZ,
bool useTmpSuffix = false) const;
/**
* @brief Read to a buffer from a file at current location
*/
EXPORT int readFile(IDBDataFile* pFile, unsigned char* readBuf, int readSize) const;
/**
* @brief Reads in 2 compression header blocks from a column segment file.
* IDBDataFile* points to start of data when function returns.
* @param pFile (in) IDBDataFile* of column segment file to be read.
* @param hdrs (out) Contents of headers that are read.
*/
EXPORT int readHeaders(IDBDataFile* pFile, char* hdrs) const;
EXPORT int readHeaders(IDBDataFile* pFile, char* hdr1, char* hdr2) const;
/**
* @brief Reinitialize a partial extent in a column segment file
* @param pFile (in) IDBDataFile* of column segment file to be written to
* @param startOffset (in) - file offset where blocks are to be written
* @param nBlocks (in) - number of blocks to be written to the extent
* @param emptyVal(in) - empty value to be used for column data values
* width (in) - width of the applicable column
*/
EXPORT int reInitPartialColumnExtent(IDBDataFile* pFile, long long startOffset, int nBlocks,
const uint8_t* emptyVal, int width);
/**
* @brief Reinitialize an extent in a dictionary store file
* @param pFile (in) IDBDataFile* of dictionary store file to be written to
* @param startOffset (in) - file offset where blocks are to be written
* @param nBlocks (in) - number of blocks to be written to the extent
* @param blockHdrInit(in) - data used to initialize each block header
* @param blockHdrInitSize(in) - number of bytes in blockHdrInit
*/
EXPORT int reInitPartialDctnryExtent(IDBDataFile* pFile, long long startOffset, int nBlocks,
unsigned char* blockHdrInit, int blockHdrInitSize);
/**
* @brief Set the file to specified location based on the offset
*/
EXPORT int setFileOffset(IDBDataFile* pFile, long long offset, int origin = SEEK_SET) const;
EXPORT int setFileOffsetBlock(IDBDataFile* pFile, uint64_t lbid, int origin = SEEK_SET) const;
/**
* @brief Truncate the file to the specified file size
*/
EXPORT int truncateFile(IDBDataFile* pFile, long long fileSize) const;
/**
* @brief Write a buffer to a file at current location
*/
EXPORT int writeFile(IDBDataFile* pFile, const unsigned char* buf, int bufSize) const;
/**
* @brief set the flag to use the instance to access the brm wrapper class
*/
EXPORT virtual void setTransId(const TxnID& transId);
EXPORT virtual void setBulkFlag(bool isBulkLoad);
EXPORT virtual void setFixFlag(bool isFix);
TxnID getTransId() const;
void compressionType(int t);
int compressionType() const;
EXPORT virtual int flushFile(int rc, std::map<FID, FID>& oids);
// Initialize an extent in a column segment file
// pFile (in) IDBDataFile* of column segment file to be written to
// dbRoot (in) - DBRoot of pFile
// nBlocks (in) - number of blocks to be written for an extent
// emptyVal(in) - empty value to be used for column data values
// width (in) - width of the applicable column
// bNewFile (in) - Adding extent to new file
// bExpandExtent (in) - Expand existing extent, or initialize new one
// bAbbrevExtent (in) - If adding new extent, is it abbreviated
// bOptExtension(in) - skip or optimize full extent preallocation
int initColumnExtent(IDBDataFile* pFile, uint16_t dbRoot, int nBlocks, const uint8_t* emptyVal, int width,
execplan::CalpontSystemCatalog::ColDataType colDataType, bool bNewFile,
bool bExpandExtent, bool bAbbrevExtent, bool bOptExtension = false, int64_t lbid = 0);
// Calls a chown and logs an error message
bool chownDataPath(const std::string& fileName) const;
protected:
EXPORT virtual int updateColumnExtent(IDBDataFile* pFile, int nBlocks, int64_t lbid);
EXPORT virtual int updateDctnryExtent(IDBDataFile* pFile, int nBlocks, int64_t lbid);
int m_compressionType; // compresssion type
private:
// not copyable
FileOp(const FileOp& rhs);
FileOp& operator=(const FileOp& rhs);
int expandAbbrevColumnChunk(IDBDataFile* pFile, const uint8_t* emptyVal, int colWidth,
const compress::CompChunkPtr& chunkInPtr, compress::CompChunkPtr& chunkOutPt,
const char* hdrs = nullptr);
int initAbbrevCompColumnExtent(IDBDataFile* pFile, uint16_t dbRoot, int nBlocks, const uint8_t* emptyVal,
int width, BRM::LBID_t lbid,
execplan::CalpontSystemCatalog::ColDataType colDataType);
static void initDbRootExtentMutexes();
int writeInitialCompColumnChunk(IDBDataFile* pFile, int nBlocksAllocated, int nRows,
const uint8_t* emptyVal, int width, BRM::LBID_t lbid,
execplan::CalpontSystemCatalog::ColDataType colDataType, char* hdrs);
TxnID m_transId;
bool m_isBulk;
bool m_isFix;
// protect creation of m_DbRootAddExtentMutexes
static boost::mutex m_createDbRootMutexes;
// Mutexes used to serialize extent creation within each DBRoot
static std::map<int, boost::mutex> m_DbRootAddExtentMutexes;
// protect race condition in creating directories
static boost::mutex m_mkdirMutex;
char* m_buffer; // buffer used with setvbuf()
};
//------------------------------------------------------------------------------
// Inline functions
//------------------------------------------------------------------------------
inline void FileOp::compressionType(int t)
{
m_compressionType = t;
}
inline int FileOp::compressionType() const
{
return m_compressionType;
}
inline int FileOp::createDir(const char* dirName) const
{
return createDir(dirName, S_IRWXU | S_IRWXG | S_IROTH | S_IXOTH);
}
inline int FileOp::getVBFileName(FID fid, char* fileName) const
{
uint16_t dbRoot = 0;
uint32_t partition = 0;
uint16_t segment = 0;
return oid2FileName(fid, fileName, true, dbRoot, partition, segment);
}
inline int FileOp::getFileName(FID fid, char* fileName, uint16_t dbRoot, uint32_t partition,
uint16_t segment) const
{
return oid2FileName(fid, fileName, false, dbRoot, partition, segment);
}
inline TxnID FileOp::getTransId() const
{
return m_transId;
}
} // namespace WriteEngine
#undef EXPORT
View Git Blame Copy Permalink