/* Copyright (C) 2014 InfiniDB, Inc. This program is free software; you can redistribute it and/or modify it under the terms of the GNU General Public License as published by the Free Software Foundation; version 2 of the License. This program is distributed in the hope that it will be useful, but WITHOUT ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the GNU General Public License for more details. You should have received a copy of the GNU General Public License along with this program; if not, write to the Free Software Foundation, Inc., 51 Franklin Street, Fifth Floor, Boston, MA 02110-1301, USA. */ // $Id: we_fileop.h 4737 2013-08-14 20:45:46Z bwilkinson $ /** @file */ #pragma once #include #include #include #include #include #include #include "we_blockop.h" #include "we_brm.h" #include "we_config.h" #include "we_stats.h" #include "idbcompress.h" #include "calpontsystemcatalog.h" #define EXPORT #define MAX_NBLOCKS 8192 #include "brmtypes.h" /** Namespace WriteEngine */ namespace WriteEngine { constexpr size_t MaxDirLevels = 5; /** Class FileOp */ class FileOp : public BlockOp, public WeUIDGID { public: /** * @brief Constructor */ EXPORT explicit FileOp(bool doAlloc = true); /** * @brief Destructor */ EXPORT virtual ~FileOp(); /** * @brief Close a file */ EXPORT void closeFile(IDBDataFile* pFile) const; /** * @brief Create a directory */ EXPORT int createDir(const char* dirName, mode_t mode) const; int createDir(const char* dirName) const; /** * @brief Create a file with a fixed file size and file id */ EXPORT int createFile(FID fid, int& allocSize, uint16_t dbRoot, uint32_t partition, execplan::CalpontSystemCatalog::ColDataType colDataType, const uint8_t* emptyVal, int width = 1); /** * @brief Create a file with a fixed file size by its name. * Changed to public for UT. */ int createFile(const char* fileName, int fileSize, const uint8_t* emptyVal, int width, execplan::CalpontSystemCatalog::ColDataType colDataType, uint16_t dbRoot, BRM::LBID_t lbid = -1); /** * @brief Delete a file */ EXPORT int deleteFile(const char* fileName) const; /** * @brief Delete the db files corresponding to the specified file id */ EXPORT int deleteFile(FID fid) const; /** * @brief Delete the db files corresponding to the specified file id */ EXPORT int deleteFiles(const std::vector& fids) const; /** * @brief Delete db files corresponding to specified file id and partition */ EXPORT int deletePartitions(const std::vector& fids, const std::vector& partitions) const; /** * @brief Delete a specific database segment file. */ EXPORT int deleteFile(FID fid, uint16_t dbRoot, uint32_t partition, uint16_t segment) const; /** * @brief Check whether a file exists or not */ EXPORT bool exists(const char* fileName) const; /** * @brief @brief Check whether file exists or not by using file id, DBRoot, * partition, and segment number. */ EXPORT bool exists(FID fid, uint16_t dbRoot, uint32_t partition, uint16_t segment) const; /** * @brief Check whether a column exists or not by using file id. Since this * is not enough to fully qualify a db filename, all it can do is to verify * that the OID directory exists on one or more of the DBRoots. */ EXPORT bool existsOIDDir(FID fid) const; /** * @brief Check whether a column file exists by the given `fid` in all * available db roots and default values for `partition` and `segment`. */ EXPORT bool existsDefaultFile(FID fid) const; /** * @brief Expand current abbreviated extent for this column to a full extent * * @param pFile FILE ptr of segment file we are updating. * @param dbRoot DBRoot of the file being updated. * @param emptyVal Empty value used in initializing extents for this column * @param width Width of this column (in bytes) * @param colDataType Column data type. */ EXPORT virtual int expandAbbrevColumnExtent(IDBDataFile* pFile, uint16_t dbRoot, const uint8_t* emptyVal, int width, execplan::CalpontSystemCatalog::ColDataType colDataType); /** * @brief Add an extent to the specified Column OID and DBRoot. * The extent must already exist in the extentmap prior to calling this fctn. * * The partition, segment, and HWM of the column file where the * extent is added is returned. If needed, the applicable column segment * file will be created. This extendFile1 function should supplant other * extendFile functions with Multiple-File-per-OID enhancement, "but" we * may want to rethink when we do Shared-Nothing. When this function * returns, the file position will be located at the end of the file. * For shared-everything DBRoot was an output argument, as BRM selected the * the DBRoot. For shared-nothing DBRoot is an input argument, as the * application code must track/control the DBRoot selection. * If this is the very first file for the specified DBRoot, then the * partition and segment number must be specified, else the selected * partition and segment numbers are returned. * * @param oid OID of the column to be extended * @param emptyVal Empty value to be used for oid * @param width Width of the column * @param hwm The fbo of the column segment file where the new extent begins * @param startLbid The starting LBID for the new extent * @param allocSize Number of blocks allocated to the extent. * @param dbRoot The DBRoot of the file with the new extent. * @param partition The partnum of the file with the new extent. * @param segment The segnum of the file with the new extent. * @param segFile (out) Name of the segment file where extent was added. * @param pFile (out) FILE ptr to the file where the extent is added. * @param newFile (out) Indicates if a new file was created for the extent * @param hdrs (in/out) Contents of headers, if file is compressed. * @return returns NO_ERROR if success. */ EXPORT int extendFile(OID oid, const uint8_t* emptyVal, int width, execplan::CalpontSystemCatalog::ColDataType colDataType, HWM hwm, BRM::LBID_t startLbid, int allocSize, uint16_t dbRoot, uint32_t partition, uint16_t segment, std::string& segFile, IDBDataFile*& pFile, bool& newFile, char* hdrs); /** * @brief For alter table add column; add an extent to a specific file * * @param oid OID of the column to be extended * @param emptyVal Empty value to be used for oid * @param width Width of the column * @param allocSize (out) Number of blocks allocated to the extent. * @param dbRoot The DBRoot of the file with the new extent. * @param partition The partnum of the file with the new extent. * @param segment The segnum of the file with the new extent. * @param segFile (out) Name of the segment file where extent was added. * @param startLbid (out) The starting LBID for the new extent * @param newFile (out) Indicates if a new file was created for the extent * @param hdrs (in/out) Contents of headers, if file is compressed. */ EXPORT int addExtentExactFile(OID oid, const uint8_t* emptyVal, int width, int& allocSize, uint16_t dbRoot, uint32_t partition, uint16_t segment, execplan::CalpontSystemCatalog::ColDataType colDataType, std::string& segFile, BRM::LBID_t& startLbid, bool& newFile, char* hdrs); /** * @brief Pad the specified compressed extent with empty chunks * @param oid OID of relevant column * @param width Width in bytes of this column * @param emptyVal Empty value to be employed in filling the chunks * @param dbRoot DBRoot of the extent to be filled * @param partition Partition of the extent to be filled * @param segment Segment file number of the extent to be filled * @param colDataType Column data type * @param hwm New HWM blk setting for the segment file after extent is padded * @param segFile (out) Name of updated segment file * @param errTask (out) Task that failed if error occurs * @return returns NO_ERROR if success. */ EXPORT int fillCompColumnExtentEmptyChunks(OID oid, int colWidth, const uint8_t* emptyVal, uint16_t dbRoot, uint32_t partition, uint16_t segment, execplan::CalpontSystemCatalog::ColDataType colDataType, HWM hwm, std::string& segFile, std::string& errTask); /** * @brief Write the specified header info to compressed column file pFile. * * @param pFile Column file to be written to * @param hdr Header info to be written */ EXPORT int writeHeaders(IDBDataFile* pFile, const char* hdr) const; /** * @brief Write the specified header info to compressed column or * dictionary file pFile. * * @param pFile Column file to be written to * @param controlHdr Control header info to be written * @param pointerHdr Pointer header info to be written * @param ptrHdrSize Size (in bytes) of pointerHdr */ EXPORT int writeHeaders(IDBDataFile* pFile, const char* controlHdr, const char* pointerHdr, uint64_t ptrHdrSize) const; /** * @brief Get the Version Buffer filename for the specified fid (OID). * * This version of getFileName automatically uses 0 for the partition and * segment numbers. The applicable DBRoot is assigned based on the OID. * * @param fid (in) OID of the Version Buffer DB file of interest * @param fileName (out) the name of the pertinent file that was found * * @return returns NO_ERROR if success; ERR_FILE_NOT_EXIST if file not found */ int getVBFileName(FID fid, char* fileName) const; /** * @brief Get the filename for the specified fid (OID). DBRoot, partition, * and segment number. * * @param fid (in) OID of the DB file of interest * @param fileName (out) the name of the pertinent file that was found * @param dbRoot (in) DBRoot of the file of interest. If 0, then all the * DBRoots will be searched. * @param partition (in) partition number of the file of interest * @param segment (in) segment number of the file of interest */ int getFileName(FID fid, char* fileName, uint16_t dbRoot, uint32_t partition, uint16_t segment) const; /* Added for MCOL-3251 */ void getFileNameForPrimProc(FID fid, char* fileName, uint16_t dbRoot, uint32_t partition, uint16_t segment) const; /** * @brief Construct directory path for the specified fid (OID), DBRoot, and * partition number. Directory does not have to exist, nor is it created. */ int getDirName(FID fid, uint16_t dbRoot, uint32_t partition, std::string& dirName) const; /** * @brief Get the file size */ EXPORT int getFileSize(IDBDataFile* pFile, long long& fileSize) const; EXPORT int getFileSize(FID fid, uint16_t dbRoot, uint32_t partition, uint16_t segment, long long& fileSize) const; /** * @brief Initialize an extent in a dictionary store file * @param pFile (in) IDBDataFile* of dictionary store file to be written to * @param dbRoot (in) - DBRoot of pFile * @param nBlocks (in) - number of blocks to be written for an extent * @param blockHdrInit(in) - data used to initialize each block header * @param blockHdrInitSize(in) - number of bytes in blockHdrInit * @param bExpandExtent (in) - Expand existing extent, or initialize new one * @param bOptExtension (in) - skip or optimize full extent preallocation */ EXPORT int initDctnryExtent(IDBDataFile* pFile, uint16_t dbRoot, int nBlocks, unsigned char* blockHdrInit, int blockHdrInitSize, bool bExpandExtent, bool bOptExtension = false, int64_t lbid = 0); /** * @brief Check whether it is an directory */ EXPORT bool isDir(const char* dirName) const; /** * @brief See if there is room in the file system for specific number of blks * @param fileName Name of file to extend (does not have to be full name) * @param nBlocks Number of 8192-byte blocks to be added * @return returns TRUE if file system has room for 'nBlocks', else FALSE */ EXPORT bool isDiskSpaceAvail(const std::string& fileName, int nBlocks) const; /** * @brief Convert an oid to a full file name */ EXPORT int oid2FileName(FID fid, char* fullFileName, bool bCreateDir, uint16_t dbRoot, uint32_t partition, uint16_t segment) const; EXPORT int oid2DirName(FID fid, char* oidDirName) const; /** * @brief Open a file using a filename. * @param fileName Name of the file to open. * @param mode Mode to use in opening the file (ex: "r+b"). * @param ioBuffSize Buffer size to be employed by setvbuf(). * @return returns the IDBDataFile* of the opened file. */ EXPORT IDBDataFile* openFile(const char* fileName, const char* mode = "r+b", int ioColSize = DEFAULT_COLSIZ, bool useTmpSuffix = false) const; /** * @brief Open a file using an OID, dbroot, partition, and segment number. * @param fid OID of the file to be opened. * @param dbRoot DBRoot of the file to be opened. * @param partition Partition number of the file to be opened. * @param segment Segment number of the file to be opened. * @param mode Mode to use in opening the file (default of "r+b" will open * an existing binary file as read/write. * @param ioBuffSize Buffer size to be employed by setvbuf(). * @return returns the IDBDataFile* of the opened file. */ EXPORT IDBDataFile* openFile(FID fid, uint16_t dbRoot, uint32_t partition, uint16_t segment, std::string& segFile, const char* mode = "r+b", int ioColSize = DEFAULT_COLSIZ, bool useTmpSuffix = false) const; /** * @brief Read to a buffer from a file at current location */ EXPORT int readFile(IDBDataFile* pFile, unsigned char* readBuf, int readSize) const; /** * @brief Reads in 2 compression header blocks from a column segment file. * IDBDataFile* points to start of data when function returns. * @param pFile (in) IDBDataFile* of column segment file to be read. * @param hdrs (out) Contents of headers that are read. */ EXPORT int readHeaders(IDBDataFile* pFile, char* hdrs) const; EXPORT int readHeaders(IDBDataFile* pFile, char* hdr1, char* hdr2) const; /** * @brief Reinitialize a partial extent in a column segment file * @param pFile (in) IDBDataFile* of column segment file to be written to * @param startOffset (in) - file offset where blocks are to be written * @param nBlocks (in) - number of blocks to be written to the extent * @param emptyVal(in) - empty value to be used for column data values * width (in) - width of the applicable column */ EXPORT int reInitPartialColumnExtent(IDBDataFile* pFile, long long startOffset, int nBlocks, const uint8_t* emptyVal, int width); /** * @brief Reinitialize an extent in a dictionary store file * @param pFile (in) IDBDataFile* of dictionary store file to be written to * @param startOffset (in) - file offset where blocks are to be written * @param nBlocks (in) - number of blocks to be written to the extent * @param blockHdrInit(in) - data used to initialize each block header * @param blockHdrInitSize(in) - number of bytes in blockHdrInit */ EXPORT int reInitPartialDctnryExtent(IDBDataFile* pFile, long long startOffset, int nBlocks, unsigned char* blockHdrInit, int blockHdrInitSize); /** * @brief Set the file to specified location based on the offset */ EXPORT int setFileOffset(IDBDataFile* pFile, long long offset, int origin = SEEK_SET) const; EXPORT int setFileOffsetBlock(IDBDataFile* pFile, uint64_t lbid, int origin = SEEK_SET) const; /** * @brief Truncate the file to the specified file size */ EXPORT int truncateFile(IDBDataFile* pFile, long long fileSize) const; /** * @brief Write a buffer to a file at current location */ EXPORT int writeFile(IDBDataFile* pFile, const unsigned char* buf, int bufSize) const; /** * @brief set the flag to use the instance to access the brm wrapper class */ EXPORT virtual void setTransId(const TxnID& transId); EXPORT virtual void setBulkFlag(bool isBulkLoad); EXPORT virtual void setFixFlag(bool isFix); TxnID getTransId() const; void compressionType(int t); int compressionType() const; EXPORT virtual int flushFile(int rc, std::map& oids); // Initialize an extent in a column segment file // pFile (in) IDBDataFile* of column segment file to be written to // dbRoot (in) - DBRoot of pFile // nBlocks (in) - number of blocks to be written for an extent // emptyVal(in) - empty value to be used for column data values // width (in) - width of the applicable column // bNewFile (in) - Adding extent to new file // bExpandExtent (in) - Expand existing extent, or initialize new one // bAbbrevExtent (in) - If adding new extent, is it abbreviated // bOptExtension(in) - skip or optimize full extent preallocation int initColumnExtent(IDBDataFile* pFile, uint16_t dbRoot, int nBlocks, const uint8_t* emptyVal, int width, execplan::CalpontSystemCatalog::ColDataType colDataType, bool bNewFile, bool bExpandExtent, bool bAbbrevExtent, bool bOptExtension = false, int64_t lbid = 0); // Calls a chown and logs an error message bool chownDataPath(const std::string& fileName) const; protected: EXPORT virtual int updateColumnExtent(IDBDataFile* pFile, int nBlocks, int64_t lbid); EXPORT virtual int updateDctnryExtent(IDBDataFile* pFile, int nBlocks, int64_t lbid); int m_compressionType; // compresssion type private: // not copyable FileOp(const FileOp& rhs); FileOp& operator=(const FileOp& rhs); int expandAbbrevColumnChunk(IDBDataFile* pFile, const uint8_t* emptyVal, int colWidth, const compress::CompChunkPtr& chunkInPtr, compress::CompChunkPtr& chunkOutPt, const char* hdrs = nullptr); int initAbbrevCompColumnExtent(IDBDataFile* pFile, uint16_t dbRoot, int nBlocks, const uint8_t* emptyVal, int width, BRM::LBID_t lbid, execplan::CalpontSystemCatalog::ColDataType colDataType); static void initDbRootExtentMutexes(); int writeInitialCompColumnChunk(IDBDataFile* pFile, int nBlocksAllocated, int nRows, const uint8_t* emptyVal, int width, BRM::LBID_t lbid, execplan::CalpontSystemCatalog::ColDataType colDataType, char* hdrs); TxnID m_transId; bool m_isBulk; bool m_isFix; // protect creation of m_DbRootAddExtentMutexes static boost::mutex m_createDbRootMutexes; // Mutexes used to serialize extent creation within each DBRoot static std::map m_DbRootAddExtentMutexes; // protect race condition in creating directories static boost::mutex m_mkdirMutex; char* m_buffer; // buffer used with setvbuf() }; //------------------------------------------------------------------------------ // Inline functions //------------------------------------------------------------------------------ inline void FileOp::compressionType(int t) { m_compressionType = t; } inline int FileOp::compressionType() const { return m_compressionType; } inline int FileOp::createDir(const char* dirName) const { return createDir(dirName, S_IRWXU | S_IRWXG | S_IROTH | S_IXOTH); } inline int FileOp::getVBFileName(FID fid, char* fileName) const { uint16_t dbRoot = 0; uint32_t partition = 0; uint16_t segment = 0; return oid2FileName(fid, fileName, true, dbRoot, partition, segment); } inline int FileOp::getFileName(FID fid, char* fileName, uint16_t dbRoot, uint32_t partition, uint16_t segment) const { return oid2FileName(fid, fileName, false, dbRoot, partition, segment); } inline TxnID FileOp::getTransId() const { return m_transId; } } // namespace WriteEngine #undef EXPORT