database/mariadb-columnstore-engine
1
0
Fork 0
mirror of https://github.com/mariadb-corporation/mariadb-columnstore-engine.git synced 2025-07-30 19:23:07 +03:00
Code Activity

Reformat all code to coding standard

Browse Source
This commit is contained in:
Andrew Hutchings
2017-10-26 17:18:17 +01:00
parent 4985f3456e
commit 01446d1e22
1296 changed files with 403852 additions and 353747 deletions
Download Patch File Download Diff File Expand all files Collapse all files

4752
writeengine/wrapper/tdriver.cpp
View File

File diff suppressed because it is too large Load Diff

2996
writeengine/wrapper/we_colop.cpp
View File

File diff suppressed because it is too large Load Diff

468
writeengine/wrapper/we_colop.h
View File

@ -39,7 +39,7 @@
/** Namespace WriteEngine */
namespace WriteEngine
{
class Log;
class Log;
//const int COL_DATATYPE_NUM = 15;
//const char ColDataTypeStr[COL_DATATYPE_NUM][20] = { "bit", "tinyint", "char", "smallint", "decimal", "medint", "integer", "float", "date", "bigint",
@ -49,141 +49,144 @@ namespace WriteEngine
class ColumnOp : public DbFileOp
{
public:
/**
* @brief Constructor
*/
EXPORT ColumnOp();
EXPORT ColumnOp(Log* logger);
/**
* @brief Constructor
*/
EXPORT ColumnOp();
EXPORT ColumnOp(Log* logger);
/**
* @brief Default Destructor
*/
EXPORT virtual ~ColumnOp();
/**
* @brief Default Destructor
*/
EXPORT virtual ~ColumnOp();
EXPORT virtual int allocRowId(const TxnID& txnid,
bool useStartingExtent,
Column& column,
uint64_t totalRow,
RID* rowIdArray,
HWM& hwm,
bool& newExtent,
uint64_t& rowsLeft,
HWM& newHwm,
bool& newFile,
ColStructList& newColStructList,
DctnryStructList& newDctnryStructList,
std::vector<boost::shared_ptr<DBRootExtentTracker> > & dbRootExtentTrackers,
bool insertSelect = false,
bool isBatchInsert = false,
OID tableOid = 0, bool isFirstBatchPm = false);
EXPORT virtual int allocRowId(const TxnID& txnid,
bool useStartingExtent,
Column& column,
uint64_t totalRow,
RID* rowIdArray,
HWM& hwm,
bool& newExtent,
uint64_t& rowsLeft,
HWM& newHwm,
bool& newFile,
ColStructList& newColStructList,
DctnryStructList& newDctnryStructList,
std::vector<boost::shared_ptr<DBRootExtentTracker> >& dbRootExtentTrackers,
bool insertSelect = false,
bool isBatchInsert = false,
OID tableOid = 0, bool isFirstBatchPm = false);
/**
* @brief Create column file(s)
*/
EXPORT virtual int createColumn(Column& column,
int colNo,
int colWidth,
execplan::CalpontSystemCatalog::ColDataType colDataType,
ColType colType,
FID dataFid,
uint16_t dbRoot,
uint32_t partition);
/**
* @brief Create column file(s)
*/
EXPORT virtual int createColumn(Column& column,
int colNo,
int colWidth,
execplan::CalpontSystemCatalog::ColDataType colDataType,
ColType colType,
FID dataFid,
uint16_t dbRoot,
uint32_t partition);
/**
* @brief Fills up a column with null/default values in all non-empty rows as the reference column. The reference column
* would typically be another column of the same table.
*
* @param column The new colum
* @param refCol The reference column for identifying valid rows
* @param defaultVal The default value of the new column
* @param dictOid The dictionary store OID for a dictionary column
* @param dictColWidth The dictionary string width for a dictionary column
*/
//BUG931
EXPORT virtual int fillColumn(const TxnID& txnid,
Column& column,
Column& refCol,
void* defaultVal,
Dctnry* dctnry,
ColumnOp* refColOp,
const OID dictOid = 0,
const int dictColWidth = 0,
const std::string defaultValStr = "",
bool autoincrement = false);
/**
* @brief Fills up a column with null/default values in all non-empty rows as the reference column. The reference column
* would typically be another column of the same table.
*
* @param column The new colum
* @param refCol The reference column for identifying valid rows
* @param defaultVal The default value of the new column
* @param dictOid The dictionary store OID for a dictionary column
* @param dictColWidth The dictionary string width for a dictionary column
*/
//BUG931
EXPORT virtual int fillColumn(const TxnID& txnid,
Column& column,
Column& refCol,
void* defaultVal,
Dctnry* dctnry,
ColumnOp* refColOp,
const OID dictOid = 0,
const int dictColWidth = 0,
const std::string defaultValStr = "",
bool autoincrement = false);
/**
* @brief Create a table file
*/
EXPORT virtual int createTable() const;
/**
* @brief Create a table file
*/
EXPORT virtual int createTable() const;
/**
* @brief Drop column file(s)
*/
EXPORT virtual int dropColumn(const FID dataFid);
/**
* @brief Drop column file(s)
*/
EXPORT virtual int dropColumn(const FID dataFid);
/**
* @brief Drop column and dictionary file(s)
*/
EXPORT virtual int dropFiles(const std::vector<int32_t>& dataFids);
EXPORT virtual int dropFiles(const std::vector<int32_t>& dataFids);
/**
* @brief Delete file(s) for the given partition
*/
EXPORT virtual int dropPartitions(const std::vector<OID>& dataFids,
const std::vector<BRM::PartitionInfo>& partitions);
/**
* @brief Delete file(s) for the given partition
*/
EXPORT virtual int dropPartitions(const std::vector<OID>& dataFids,
const std::vector<BRM::PartitionInfo>& partitions);
EXPORT virtual int deleteOIDsFromExtentMap(const std::vector<int32_t>& dataFids);
/**
* @brief Drop a table file
*/
int dropTable() const { return NO_ERROR; }
EXPORT virtual int deleteOIDsFromExtentMap(const std::vector<int32_t>& dataFids);
/**
* @brief Drop a table file
*/
int dropTable() const
{
return NO_ERROR;
}
/**
* @brief Expand the abbreviated extent to a full extent for this column.
*/
EXPORT virtual int expandAbbrevExtent(const Column& column);
/**
* @brief Expand the abbreviated extent to a full extent for this column.
*/
EXPORT virtual int expandAbbrevExtent(const Column& column);
/**
* @brief Add an extent to the specified column OID and DBRoot.
* When this function returns, the file position will be located at the
* end of the file. If the applicable column segment file does not exist,
* extendColumn() will create the new segment file.
* The extent must already exist in the extentmap prior to calling this fctn.
*
* @param column Column struct with input column attributes.
* @param leaveFileOpen Leave the db file open when leaving this function
* @param hwm The fbo of the column segment file where the new extent begins
* @param startLbid The starting LBID for the new extent.
* @param allocSize Number of blocks to be written for an extent
* @param dbRoot The DBRoot of the file with the new extent.
* @param partition Partition num of the file with the new extent.
* @param segment The segment number of the file with the new extent.
* @param segFile (out) Name of segment file to which the extent is added.
* @param pFile (out) FILE ptr to the file where the extent is added.
* @param newFile (out) Indicates if extent was added to new or existing file
* @param hdrs (out) Contents of headers if file is compressed.
* @return returns NO_ERROR if success.
*/
EXPORT int extendColumn(const Column& column,
bool leaveFileOpen,
HWM hwm,
BRM::LBID_t startLbid,
int allocSize,
uint16_t dbRoot,
uint32_t partition,
uint16_t segment,
std::string& segFile,
IDBDataFile*& pFile,
bool& newFile,
char* hdrs = NULL);
/**
* @brief Add an extent to the specified column OID and DBRoot.
* When this function returns, the file position will be located at the
* end of the file. If the applicable column segment file does not exist,
* extendColumn() will create the new segment file.
* The extent must already exist in the extentmap prior to calling this fctn.
*
* @param column Column struct with input column attributes.
* @param leaveFileOpen Leave the db file open when leaving this function
* @param hwm The fbo of the column segment file where the new extent begins
* @param startLbid The starting LBID for the new extent.
* @param allocSize Number of blocks to be written for an extent
* @param dbRoot The DBRoot of the file with the new extent.
* @param partition Partition num of the file with the new extent.
* @param segment The segment number of the file with the new extent.
* @param segFile (out) Name of segment file to which the extent is added.
* @param pFile (out) FILE ptr to the file where the extent is added.
* @param newFile (out) Indicates if extent was added to new or existing file
* @param hdrs (out) Contents of headers if file is compressed.
* @return returns NO_ERROR if success.
*/
EXPORT int extendColumn(const Column& column,
bool leaveFileOpen,
HWM hwm,
BRM::LBID_t startLbid,
int allocSize,
uint16_t dbRoot,
uint32_t partition,
uint16_t segment,
std::string& segFile,
IDBDataFile*& pFile,
bool& newFile,
char* hdrs = NULL);
/**
/**
* @brief Add an extent to the OID specified in the column argument.
* When this function returns, the file position will be located at the
* end of the file.
* end of the file.
*
* @param column Column struct with input column attributes.
* @param column Column struct with input column attributes.
* @param dbRoot (in) The DBRoot of the file with the new extent.
* @param partition (in) The partition num of the file with the new extent.
* @param segment (in) The segment number of the file with the new extent.
@ -191,144 +194,147 @@ public:
* @param startLbid (out) The starting LBID for the new extent.
* @param newFile (out) Indicates if extent was added to new or existing file
* @param hdsr (out) Contents of headers if file is compressed.
* @param allocSize (out) number of blocks to be written for an extent
* @param allocSize (out) number of blocks to be written for an extent
* @return returns NO_ERROR if success.
*/
EXPORT int addExtent(const Column& column,
uint16_t dbRoot,
uint32_t partition,
uint16_t segment,
std::string& segFile,
BRM::LBID_t& startLbid,
bool& newFile,
int& allocSize,
char* hdrs = NULL);
EXPORT int addExtent(const Column& column,
uint16_t dbRoot,
uint32_t partition,
uint16_t segment,
std::string& segFile,
BRM::LBID_t& startLbid,
bool& newFile,
int& allocSize,
char* hdrs = NULL);
/**
* @brief Get columne data type
*/
EXPORT virtual bool getColDataType(const char* name, execplan::CalpontSystemCatalog::ColDataType& colDataType) const;
/**
* @brief Get columne data type
*/
EXPORT virtual bool getColDataType(const char* name, execplan::CalpontSystemCatalog::ColDataType& colDataType) const;
/**
* @brief Initialize the column
*/
EXPORT virtual void initColumn(Column& column) const;
/**
* @brief Initialize the column
*/
EXPORT virtual void initColumn(Column& column) const;
/**
* @brief Check whether it is an empty row
*/
EXPORT virtual bool isEmptyRow(unsigned char* buf, int offset, const Column& column);
/**
* @brief Check whether it is an empty row
*/
EXPORT virtual bool isEmptyRow(unsigned char* buf, int offset, const Column& column);
/**
* @brief Check whether it is a valid column
*/
EXPORT virtual bool isValid(Column& column) const;
/**
* @brief Check whether it is a valid column
*/
EXPORT virtual bool isValid(Column& column) const;
/**
* @brief Open column file, segFile is set to the name of the column
* segment file that is opened.
*/
EXPORT virtual int openColumnFile(Column& column,
std::string& segFile,
bool useTmpSuffix,
int ioBuffSize=DEFAULT_BUFSIZ) const;
/**
* @brief Open column file, segFile is set to the name of the column
* segment file that is opened.
*/
EXPORT virtual int openColumnFile(Column& column,
std::string& segFile,
bool useTmpSuffix,
int ioBuffSize = DEFAULT_BUFSIZ) const;
/**
* @brief Open table file
*/
int openTableFile() const { return NO_ERROR; }
/**
* @brief Open table file
*/
int openTableFile() const
{
return NO_ERROR;
}
/**
* @brief Delete a file
*/
EXPORT virtual void setColParam(Column& column, int colNo = 0,
int colWidth = 0,
execplan::CalpontSystemCatalog::ColDataType colDataType = execplan::CalpontSystemCatalog::INT,
ColType colType = WR_INT,
FID dataFid = 0,
int comppre = 0,
uint16_t dbRoot = 0,
uint32_t partition = 0,
uint16_t segment = 0) const;
/**
* @brief Delete a file
*/
EXPORT virtual void setColParam(Column& column, int colNo = 0,
int colWidth = 0,
execplan::CalpontSystemCatalog::ColDataType colDataType = execplan::CalpontSystemCatalog::INT,
ColType colType = WR_INT,
FID dataFid = 0,
int comppre = 0,
uint16_t dbRoot = 0,
uint32_t partition = 0,
uint16_t segment = 0) const;
/**
* @brief Write row(s)
*/
EXPORT virtual int writeRow(Column& curCol,
uint64_t totalRow,
const RID* rowIdArray,
const void* valArray,
bool bDelete = false);
/**
* @brief Write row(s)
*/
EXPORT virtual int writeRow(Column& curCol,
uint64_t totalRow,
const RID* rowIdArray,
const void* valArray,
bool bDelete = false);
/**
* @brief Write row(s) for update and delete @Bug 1886,2870
*/
EXPORT virtual int writeRows(Column& curCol,
uint64_t totalRow,
const RIDList& ridList,
const void* valArray,
const void* oldValArray=0,
bool bDelete = false);
/**
* @brief Write row(s) for update and delete @Bug 1886,2870
*/
EXPORT virtual int writeRows(Column& curCol,
uint64_t totalRow,
const RIDList& ridList,
const void* valArray,
const void* oldValArray = 0,
bool bDelete = false);
/**
* @brief Write row(s) for update and delete @Bug 1886,2870
*/
EXPORT virtual int writeRowsValues(Column& curCol,
uint64_t totalRow,
const RIDList& ridList,
const void* valArray);
/**
* @brief Write row(s) for update and delete @Bug 1886,2870
*/
EXPORT virtual int writeRowsValues(Column& curCol,
uint64_t totalRow,
const RIDList& ridList,
const void* valArray);
/**
* @brief Test if the pFile is an abbreviated extent.
*/
virtual bool abbreviatedExtent(IDBDataFile* pFile, int colWidth) const = 0;
/**
* @brief Test if the pFile is an abbreviated extent.
*/
virtual bool abbreviatedExtent(IDBDataFile* pFile, int colWidth) const = 0;
/**
* @brief Caculate the number of blocks in file.
*/
virtual int blocksInFile(IDBDataFile* pFile) const = 0;
/**
* @brief Caculate the number of blocks in file.
*/
virtual int blocksInFile(IDBDataFile* pFile) const = 0;
/**
* @brief Clear a column
*/
EXPORT void clearColumn(Column& column) const;
/**
* @brief Clear a column
*/
EXPORT void clearColumn(Column& column) const;
/**
* @brief open a data file of column
*/
virtual IDBDataFile* openFile(const Column& column, uint16_t dbRoot, uint32_t partition,
uint16_t segment, std::string& segFile, bool useTmpSuffix, const char* mode = "r+b",
int ioBuffSize = DEFAULT_BUFSIZ) const = 0;
/**
* @brief open a data file of column
*/
virtual IDBDataFile* openFile(const Column& column, uint16_t dbRoot, uint32_t partition,
uint16_t segment, std::string& segFile, bool useTmpSuffix, const char* mode = "r+b",
int ioBuffSize = DEFAULT_BUFSIZ) const = 0;
/**
* @brief backup blocks to version buffer
*/
int writeVB(IDBDataFile* pSource, const OID sourceOid, IDBDataFile* pTarget, const OID targetOid,
const std::vector<uint32_t>& fboList, const BRM::VBRange& freeList,
size_t& nBlocksProcessed, const size_t fboCurrentOffset);
/**
* @brief backup blocks to version buffer
*/
int writeVB(IDBDataFile* pSource, const OID sourceOid, IDBDataFile* pTarget, const OID targetOid,
const std::vector<uint32_t>& fboList, const BRM::VBRange& freeList,
size_t& nBlocksProcessed, const size_t fboCurrentOffset);
/**
* @brief restore blocks from version buffer
*/
int copyVB(IDBDataFile* pSource, const BRM::VER_t txnD, const OID oid, std::vector<uint32_t>& fboList,
/**
* @brief restore blocks from version buffer
*/
int copyVB(IDBDataFile* pSource, const BRM::VER_t txnD, const OID oid, std::vector<uint32_t>& fboList,
std::vector<BRM::LBIDRange>& rangeList);
/**
* @brief close column file
*/
EXPORT virtual void closeColumnFile(Column& column) const;
/**
* @brief close column file
*/
EXPORT virtual void closeColumnFile(Column& column) const;
protected:
/**
* @brief populate readBuf with data in block #lbid
*/
virtual int readBlock(IDBDataFile* pFile, unsigned char* readBuf, const uint64_t fbo) = 0;
/**
* @brief populate readBuf with data in block #lbid
*/
virtual int readBlock(IDBDataFile* pFile, unsigned char* readBuf, const uint64_t fbo) = 0;
/**
* @brief output writeBuf to pFile starting at position fbo
*/
virtual int saveBlock(IDBDataFile* pFile, const unsigned char* writeBuf, const uint64_t fbo) = 0;
/**
* @brief output writeBuf to pFile starting at position fbo
*/
virtual int saveBlock(IDBDataFile* pFile, const unsigned char* writeBuf, const uint64_t fbo) = 0;
private:
};

92
writeengine/wrapper/we_colopcompress.cpp
View File

@ -44,15 +44,15 @@ class ChunkManager;
*/
ColumnOpCompress0::ColumnOpCompress0()
{
m_compressionType = 0;
m_compressionType = 0;
}
ColumnOpCompress0::ColumnOpCompress0(Log* logger)
{
m_compressionType = 0;
setDebugLevel( logger->getDebugLevel() );
setLogger ( logger );
m_compressionType = 0;
setDebugLevel( logger->getDebugLevel() );
setLogger ( logger );
}
/**
@ -64,21 +64,23 @@ ColumnOpCompress0::~ColumnOpCompress0()
// @bug 5572 - HDFS usage: add *.tmp file backup flag
IDBDataFile* ColumnOpCompress0::openFile(
const Column& column, const uint16_t dbRoot, const uint32_t partition, const uint16_t segment,
std::string& segFile, bool useTmpSuffix, const char* mode, const int ioBuffSize) const
const Column& column, const uint16_t dbRoot, const uint32_t partition, const uint16_t segment,
std::string& segFile, bool useTmpSuffix, const char* mode, const int ioBuffSize) const
{
return FileOp::openFile(column.dataFile.fid, dbRoot, partition, segment, segFile,
mode, column.colWidth, useTmpSuffix);
return FileOp::openFile(column.dataFile.fid, dbRoot, partition, segment, segFile,
mode, column.colWidth, useTmpSuffix);
}
bool ColumnOpCompress0::abbreviatedExtent(IDBDataFile* pFile, int colWidth) const
{
long long fsize;
if (getFileSize(pFile, fsize) == NO_ERROR)
{
return (fsize == INITIAL_EXTENT_ROWS_TO_DISK*colWidth);
return (fsize == INITIAL_EXTENT_ROWS_TO_DISK * colWidth);
}
// TODO: Log error
return false;
}
@ -87,10 +89,12 @@ bool ColumnOpCompress0::abbreviatedExtent(IDBDataFile* pFile, int colWidth) cons
int ColumnOpCompress0::blocksInFile(IDBDataFile* pFile) const
{
long long fsize;
if (getFileSize(pFile, fsize) == NO_ERROR)
{
return (fsize / BYTE_PER_BLOCK);
}
// TODO: Log error
return 0;
}
@ -98,13 +102,13 @@ int ColumnOpCompress0::blocksInFile(IDBDataFile* pFile) const
int ColumnOpCompress0::readBlock(IDBDataFile* pFile, unsigned char* readBuf, const uint64_t fbo)
{
return readDBFile(pFile, readBuf, fbo, true);
return readDBFile(pFile, readBuf, fbo, true);
}
int ColumnOpCompress0::saveBlock(IDBDataFile* pFile, const unsigned char* writeBuf, const uint64_t fbo)
{
return writeDBFileFbo(pFile, writeBuf, fbo, 1);
return writeDBFileFbo(pFile, writeBuf, fbo, 1);
}
@ -119,14 +123,16 @@ int ColumnOpCompress0::saveBlock(IDBDataFile* pFile, const unsigned char* writeB
ColumnOpCompress1::ColumnOpCompress1(Log* logger)
{
m_compressionType = 1;
m_chunkManager = new ChunkManager();
if (logger)
{
setDebugLevel( logger->getDebugLevel() );
setLogger ( logger );
}
m_chunkManager->fileOp(this);
m_compressionType = 1;
m_chunkManager = new ChunkManager();
if (logger)
{
setDebugLevel( logger->getDebugLevel() );
setLogger ( logger );
}
m_chunkManager->fileOp(this);
}
/**
@ -134,77 +140,77 @@ ColumnOpCompress1::ColumnOpCompress1(Log* logger)
*/
ColumnOpCompress1::~ColumnOpCompress1()
{
if (m_chunkManager)
{
delete m_chunkManager;
}
if (m_chunkManager)
{
delete m_chunkManager;
}
}
// @bug 5572 - HDFS usage: add *.tmp file backup flag
IDBDataFile* ColumnOpCompress1::openFile(
const Column& column, const uint16_t dbRoot, const uint32_t partition, const uint16_t segment,
std::string& segFile, bool useTmpSuffix, const char* mode, const int ioBuffSize) const
const Column& column, const uint16_t dbRoot, const uint32_t partition, const uint16_t segment,
std::string& segFile, bool useTmpSuffix, const char* mode, const int ioBuffSize) const
{
return m_chunkManager->getFilePtr(column, dbRoot, partition, segment, segFile,
mode, ioBuffSize, useTmpSuffix);
mode, ioBuffSize, useTmpSuffix);
}
bool ColumnOpCompress1::abbreviatedExtent(IDBDataFile* pFile, int colWidth) const
{
return (blocksInFile(pFile) == INITIAL_EXTENT_ROWS_TO_DISK*colWidth/BYTE_PER_BLOCK);
return (blocksInFile(pFile) == INITIAL_EXTENT_ROWS_TO_DISK * colWidth / BYTE_PER_BLOCK);
}
int ColumnOpCompress1::blocksInFile(IDBDataFile* pFile) const
{
CompFileHeader compFileHeader;
readHeaders(pFile, compFileHeader.fControlData, compFileHeader.fPtrSection);
CompFileHeader compFileHeader;
readHeaders(pFile, compFileHeader.fControlData, compFileHeader.fPtrSection);
compress::IDBCompressInterface compressor;
return compressor.getBlockCount(compFileHeader.fControlData);
compress::IDBCompressInterface compressor;
return compressor.getBlockCount(compFileHeader.fControlData);
}
int ColumnOpCompress1::readBlock(IDBDataFile* pFile, unsigned char* readBuf, const uint64_t fbo)
{
return m_chunkManager->readBlock(pFile, readBuf, fbo);
return m_chunkManager->readBlock(pFile, readBuf, fbo);
}
int ColumnOpCompress1::saveBlock(IDBDataFile* pFile, const unsigned char* writeBuf, const uint64_t fbo)
{
return m_chunkManager->saveBlock(pFile, writeBuf, fbo);
return m_chunkManager->saveBlock(pFile, writeBuf, fbo);
}
int ColumnOpCompress1::flushFile(int rc, std::map<FID,FID> & columnOids)
int ColumnOpCompress1::flushFile(int rc, std::map<FID, FID>& columnOids)
{
return m_chunkManager->flushChunks(rc, columnOids);
return m_chunkManager->flushChunks(rc, columnOids);
}
int ColumnOpCompress1::expandAbbrevColumnExtent(
IDBDataFile* pFile, uint16_t dbRoot, uint64_t emptyVal, int width)
IDBDataFile* pFile, uint16_t dbRoot, uint64_t emptyVal, int width)
{
// update the uncompressed initial chunk to full chunk
RETURN_ON_ERROR(m_chunkManager->expandAbbrevColumnExtent(pFile, emptyVal, width));
// update the uncompressed initial chunk to full chunk
RETURN_ON_ERROR(m_chunkManager->expandAbbrevColumnExtent(pFile, emptyVal, width));
// let the base to physically expand extent.
return FileOp::expandAbbrevColumnExtent(pFile, dbRoot, emptyVal, width);
// let the base to physically expand extent.
return FileOp::expandAbbrevColumnExtent(pFile, dbRoot, emptyVal, width);
}
int ColumnOpCompress1::updateColumnExtent(IDBDataFile* pFile, int nBlocks)
{
return m_chunkManager->updateColumnExtent(pFile, nBlocks);
return m_chunkManager->updateColumnExtent(pFile, nBlocks);
}
void ColumnOpCompress1::closeColumnFile(Column& column) const
{
// Leave file closing to chunk manager.
column.dataFile.pFile = NULL;
// Leave file closing to chunk manager.
column.dataFile.pFile = NULL;
}
} //end of namespace

196
writeengine/wrapper/we_colopcompress.h
View File

@ -42,46 +42,46 @@ namespace WriteEngine
class ColumnOpCompress0 : public ColumnOp
{
public:
/**
* @brief Constructor
*/
EXPORT ColumnOpCompress0();
EXPORT ColumnOpCompress0(Log* logger);
/**
* @brief Constructor
*/
EXPORT ColumnOpCompress0();
EXPORT ColumnOpCompress0(Log* logger);
/**
* @brief Default Destructor
*/
EXPORT virtual ~ColumnOpCompress0();
/**
* @brief Default Destructor
*/
EXPORT virtual ~ColumnOpCompress0();
/**
* @brief virtual method in ColumnOp
*/
IDBDataFile* openFile(const Column& column, uint16_t dbRoot, uint32_t partition,
uint16_t segment, std::string& segFile, bool useTmpSuffix, const char* mode = "r+b",
int ioBuffSize = DEFAULT_BUFSIZ) const;
/**
* @brief virtual method in ColumnOp
*/
IDBDataFile* openFile(const Column& column, uint16_t dbRoot, uint32_t partition,
uint16_t segment, std::string& segFile, bool useTmpSuffix, const char* mode = "r+b",
int ioBuffSize = DEFAULT_BUFSIZ) const;
/**
* @brief virtual method in ColumnOp
*/
bool abbreviatedExtent(IDBDataFile* pFile, int colWidth) const;
/**
* @brief virtual method in ColumnOp
*/
bool abbreviatedExtent(IDBDataFile* pFile, int colWidth) const;
/**
* @brief virtual method in ColumnOp
*/
int blocksInFile(IDBDataFile* pFile) const;
/**
* @brief virtual method in ColumnOp
*/
int blocksInFile(IDBDataFile* pFile) const;
protected:
/**
* @brief virtual method in ColumnOp
*/
int readBlock(IDBDataFile* pFile, unsigned char* readBuf, const uint64_t fbo);
/**
* @brief virtual method in ColumnOp
*/
int readBlock(IDBDataFile* pFile, unsigned char* readBuf, const uint64_t fbo);
/**
* @brief virtual method in ColumnOp
*/
int saveBlock(IDBDataFile* pFile, const unsigned char* writeBuf, const uint64_t fbo);
/**
* @brief virtual method in ColumnOp
*/
int saveBlock(IDBDataFile* pFile, const unsigned char* writeBuf, const uint64_t fbo);
private:
@ -93,81 +93,95 @@ private:
class ColumnOpCompress1 : public ColumnOp
{
public:
/**
* @brief Constructor
*/
EXPORT ColumnOpCompress1(Log* logger=0);
/**
* @brief Constructor
*/
EXPORT ColumnOpCompress1(Log* logger = 0);
/**
* @brief Default Destructor
*/
EXPORT virtual ~ColumnOpCompress1();
/**
* @brief Default Destructor
*/
EXPORT virtual ~ColumnOpCompress1();
/**
* @brief virtual method in FileOp
*/
EXPORT int flushFile(int rc, std::map<FID,FID> & columnOids);
/**
* @brief virtual method in FileOp
*/
EXPORT int flushFile(int rc, std::map<FID, FID>& columnOids);
/**
* @brief virtual method in FileOp
*/
int expandAbbrevColumnExtent(IDBDataFile* pFile, uint16_t dbRoot, uint64_t emptyVal, int width);
/**
* @brief virtual method in FileOp
*/
int expandAbbrevColumnExtent(IDBDataFile* pFile, uint16_t dbRoot, uint64_t emptyVal, int width);
/**
* @brief virtual method in ColumnOp
*/
IDBDataFile* openFile(const Column& column, uint16_t dbRoot, uint32_t partition,
uint16_t segment, std::string& segFile, bool useTmpSuffix, const char* mode = "r+b",
int ioBuffSize = DEFAULT_BUFSIZ) const;
/**
* @brief virtual method in ColumnOp
*/
IDBDataFile* openFile(const Column& column, uint16_t dbRoot, uint32_t partition,
uint16_t segment, std::string& segFile, bool useTmpSuffix, const char* mode = "r+b",
int ioBuffSize = DEFAULT_BUFSIZ) const;
/**
* @brief virtual method in ColumnOp
*/
bool abbreviatedExtent(IDBDataFile* pFile, int colWidth) const;
/**
* @brief virtual method in ColumnOp
*/
bool abbreviatedExtent(IDBDataFile* pFile, int colWidth) const;
/**
* @brief virtual method in ColumnOp
*/
int blocksInFile(IDBDataFile* pFile) const;
/**
* @brief virtual method in ColumnOp
*/
int blocksInFile(IDBDataFile* pFile) const;
// void chunkManager(ChunkManager* cm);
/**
* @brief virtual method in FileOp
*/
void setTransId(const TxnID& transId) {ColumnOp::setTransId(transId); if (m_chunkManager) m_chunkManager->setTransId(transId);}
void setBulkFlag(bool isBulkLoad) {m_chunkManager->setBulkFlag(isBulkLoad);};
void setFixFlag(bool isFix) {m_chunkManager->setFixFlag(isFix);};
/**
* @brief virtual method in FileOp
*/
void setTransId(const TxnID& transId)
{
ColumnOp::setTransId(transId);
if (m_chunkManager) m_chunkManager->setTransId(transId);
}
void setBulkFlag(bool isBulkLoad)
{
m_chunkManager->setBulkFlag(isBulkLoad);
};
void setFixFlag(bool isFix)
{
m_chunkManager->setFixFlag(isFix);
};
protected:
/**
* @brief virtual method in FileOp
*/
int updateColumnExtent(IDBDataFile* pFile, int nBlocks);
/**
* @brief virtual method in FileOp
*/
int updateColumnExtent(IDBDataFile* pFile, int nBlocks);
/**
* @brief virtual method in ColumnOp
*/
void closeColumnFile(Column& column) const;
/**
* @brief virtual method in ColumnOp
*/
void closeColumnFile(Column& column) const;
/**
* @brief virtual method in ColumnOp
*/
int readBlock(IDBDataFile* pFile, unsigned char* readBuf, const uint64_t fbo);
/**
* @brief virtual method in ColumnOp
*/
int readBlock(IDBDataFile* pFile, unsigned char* readBuf, const uint64_t fbo);
/**
* @brief virtual method in ColumnOp
*/
int saveBlock(IDBDataFile* pFile, const unsigned char* writeBuf, const uint64_t fbo);
/**
* @brief virtual method in ColumnOp
*/
int saveBlock(IDBDataFile* pFile, const unsigned char* writeBuf, const uint64_t fbo);
/**
* @brief Set the IsInsert flag in the ChunkManager.
* This forces flush at end of statement. Used only for bulk insert.
*/
void setIsInsert(bool isInsert) { m_chunkManager->setIsInsert(isInsert); }
/**
* @brief Set the IsInsert flag in the ChunkManager.
* This forces flush at end of statement. Used only for bulk insert.
*/
void setIsInsert(bool isInsert)
{
m_chunkManager->setIsInsert(isInsert);
}
private:
};

59
writeengine/wrapper/we_dctnrycompress.cpp
View File

@ -44,14 +44,14 @@ class ChunkManager;
*/
DctnryCompress0::DctnryCompress0()
{
m_compressionType = 0;
m_compressionType = 0;
}
DctnryCompress0::DctnryCompress0(Log* logger)
{
m_compressionType = 0;
setDebugLevel( logger->getDebugLevel() );
setLogger ( logger );
m_compressionType = 0;
setDebugLevel( logger->getDebugLevel() );
setLogger ( logger );
}
/**
@ -69,15 +69,16 @@ DctnryCompress0::~DctnryCompress0()
*/
DctnryCompress1::DctnryCompress1(Log* logger)
{
m_compressionType = 1;
m_chunkManager = new ChunkManager();
m_compressionType = 1;
m_chunkManager = new ChunkManager();
if (logger)
{
setDebugLevel( logger->getDebugLevel() );
setLogger ( logger );
}
m_chunkManager->fileOp(this);
if (logger)
{
setDebugLevel( logger->getDebugLevel() );
setLogger ( logger );
}
m_chunkManager->fileOp(this);
}
/**
@ -85,37 +86,38 @@ DctnryCompress1::DctnryCompress1(Log* logger)
*/
DctnryCompress1::~DctnryCompress1()
{
if (m_chunkManager)
delete m_chunkManager;
if (m_chunkManager)
delete m_chunkManager;
}
int DctnryCompress1::updateDctnryExtent(IDBDataFile* pFile, int nBlocks)
{
return m_chunkManager->updateDctnryExtent(pFile, nBlocks);
return m_chunkManager->updateDctnryExtent(pFile, nBlocks);
}
IDBDataFile* DctnryCompress1::createDctnryFile(const char *name,int width,const char *mode,int ioBuffSize)
IDBDataFile* DctnryCompress1::createDctnryFile(const char* name, int width, const char* mode, int ioBuffSize)
{
return m_chunkManager->createDctnryFile(
m_dctnryOID, width, m_dbRoot, m_partition, m_segment, name, mode, ioBuffSize);
return m_chunkManager->createDctnryFile(
m_dctnryOID, width, m_dbRoot, m_partition, m_segment, name, mode, ioBuffSize);
}
// @bug 5572 - HDFS usage: add *.tmp file backup flag
IDBDataFile* DctnryCompress1::openDctnryFile(bool useTmpSuffix)
{
return m_chunkManager->getFilePtr(
m_dctnryOID, m_dbRoot, m_partition, m_segment, m_segFileName, "r+b", DEFAULT_BUFSIZ, useTmpSuffix);
return m_chunkManager->getFilePtr(
m_dctnryOID, m_dbRoot, m_partition, m_segment, m_segFileName, "r+b", DEFAULT_BUFSIZ, useTmpSuffix);
}
void DctnryCompress1::closeDctnryFile(bool doFlush, std::map<FID,FID> & columnOids)
void DctnryCompress1::closeDctnryFile(bool doFlush, std::map<FID, FID>& columnOids)
{
if (doFlush)
m_chunkManager->flushChunks(NO_ERROR, columnOids);
else
m_chunkManager->cleanUp(columnOids);
m_dFile = NULL;
}
@ -130,10 +132,11 @@ int DctnryCompress1::readDBFile(IDBDataFile* pFile, unsigned char* readBuf, cons
const bool isFbo)
{
int fbo = lbid;
if (!isFbo)
RETURN_ON_ERROR(lbidToFbo(lbid, fbo));
return m_chunkManager->readBlock(pFile, readBuf, fbo);
return m_chunkManager->readBlock(pFile, readBuf, fbo);
}
@ -144,27 +147,27 @@ int DctnryCompress1::writeDBFile(IDBDataFile* pFile, const unsigned char* writeB
RETURN_ON_ERROR(lbidToFbo(lbid, fbo));
for (int i = 0; i < numOfBlock; i++)
RETURN_ON_ERROR(m_chunkManager->saveBlock(pFile, writeBuf, fbo+i));
RETURN_ON_ERROR(m_chunkManager->saveBlock(pFile, writeBuf, fbo + i));
return NO_ERROR;
}
int DctnryCompress1::writeDBFileNoVBCache(IDBDataFile* pFile,
const unsigned char * writeBuf, const int fbo,
const int numOfBlock)
const unsigned char* writeBuf, const int fbo,
const int numOfBlock)
{
//int fbo = 0;
//RETURN_ON_ERROR(lbidToFbo(lbid, fbo));
for (int i = 0; i < numOfBlock; i++)
RETURN_ON_ERROR(m_chunkManager->saveBlock(pFile, writeBuf, fbo+i));
RETURN_ON_ERROR(m_chunkManager->saveBlock(pFile, writeBuf, fbo + i));
return NO_ERROR;
}
int DctnryCompress1::flushFile(int rc, std::map<FID,FID> & columnOids)
int DctnryCompress1::flushFile(int rc, std::map<FID, FID>& columnOids)
{
return m_chunkManager->flushChunks(rc, columnOids);
return m_chunkManager->flushChunks(rc, columnOids);
}

178
writeengine/wrapper/we_dctnrycompress.h
View File

@ -41,16 +41,16 @@ namespace WriteEngine
class DctnryCompress0 : public Dctnry
{
public:
/**
* @brief Constructor
*/
EXPORT DctnryCompress0();
EXPORT DctnryCompress0(Log* logger);
/**
* @brief Constructor
*/
EXPORT DctnryCompress0();
EXPORT DctnryCompress0(Log* logger);
/**
* @brief Default Destructor
*/
EXPORT virtual ~DctnryCompress0();
/**
* @brief Default Destructor
*/
EXPORT virtual ~DctnryCompress0();
};
@ -59,92 +59,112 @@ public:
class DctnryCompress1 : public Dctnry
{
public:
/**
* @brief Constructor
*/
EXPORT DctnryCompress1(Log* logger=0);
/**
* @brief Constructor
*/
EXPORT DctnryCompress1(Log* logger = 0);
/**
* @brief Default Destructor
*/
EXPORT virtual ~DctnryCompress1();
/**
* @brief Default Destructor
*/
EXPORT virtual ~DctnryCompress1();
/**
* @brief virtual method in FileOp
*/
EXPORT int flushFile(int rc, std::map<FID,FID> & columnOids);
/**
* @brief virtual method in FileOp
*/
EXPORT int flushFile(int rc, std::map<FID, FID>& columnOids);
/**
* @brief virtual method in DBFileOp
*/
EXPORT int readDBFile(IDBDataFile* pFile, unsigned char* readBuf, const uint64_t lbid,
const bool isFbo = false );
/**
* @brief virtual method in DBFileOp
*/
EXPORT int readDBFile(IDBDataFile* pFile, unsigned char* readBuf, const uint64_t lbid,
const bool isFbo = false );
/**
* @brief virtual method in DBFileOp
*/
EXPORT int writeDBFile(IDBDataFile* pFile, const unsigned char* writeBuf, const uint64_t lbid,
const int numOfBlock = 1);
/**
* @brief virtual method in DBFileOp
*/
EXPORT int writeDBFile(IDBDataFile* pFile, const unsigned char* writeBuf, const uint64_t lbid,
const int numOfBlock = 1);
/**
* @brief virtual method in DBFileOp
*/
EXPORT int writeDBFileNoVBCache(IDBDataFile* pFile,
const unsigned char * writeBuf, const int fbo,
const int numOfBlock = 1);
/**
* @brief virtual method in DBFileOp
*/
EXPORT int writeDBFileNoVBCache(IDBDataFile* pFile,
const unsigned char* writeBuf, const int fbo,
const int numOfBlock = 1);
/**
* @brief virtual method in Dctnry
*/
IDBDataFile* createDctnryFile(const char *name, int width, const char *mode, int ioBuffSize);
/**
* @brief virtual method in Dctnry
*/
IDBDataFile* createDctnryFile(const char* name, int width, const char* mode, int ioBuffSize);
/**
* @brief virtual method in Dctnry
*/
IDBDataFile* openDctnryFile(bool useTmpSuffix);
/**
* @brief virtual method in Dctnry
*/
IDBDataFile* openDctnryFile(bool useTmpSuffix);
/**
* @brief virtual method in Dctnry
*/
void closeDctnryFile(bool doFlush, std::map<FID,FID> & columnOids);
/**
* @brief virtual method in Dctnry
*/
int numOfBlocksInFile();
/**
* @brief For bulkload to use
*/
void setMaxActiveChunkNum(unsigned int maxActiveChunkNum) { m_chunkManager->setMaxActiveChunkNum(maxActiveChunkNum); };
void setBulkFlag(bool isBulkLoad) {m_chunkManager->setBulkFlag(isBulkLoad);};
void setFixFlag(bool isFix) {m_chunkManager->setFixFlag(isFix);};
int checkFixLastDictChunk () { return m_chunkManager->checkFixLastDictChunk(m_dctnryOID, m_dbRoot, m_partition, m_segment); };
/**
* @brief virtual method in Dctnry
*/
void closeDctnryFile(bool doFlush, std::map<FID, FID>& columnOids);
/**
* @brief virtual method in Dctnry
*/
int numOfBlocksInFile();
/**
* @brief For bulkload to use
*/
void setMaxActiveChunkNum(unsigned int maxActiveChunkNum)
{
m_chunkManager->setMaxActiveChunkNum(maxActiveChunkNum);
};
void setBulkFlag(bool isBulkLoad)
{
m_chunkManager->setBulkFlag(isBulkLoad);
};
void setFixFlag(bool isFix)
{
m_chunkManager->setFixFlag(isFix);
};
int checkFixLastDictChunk ()
{
return m_chunkManager->checkFixLastDictChunk(m_dctnryOID, m_dbRoot, m_partition, m_segment);
};
// void chunkManager(ChunkManager* cm);
/**
* @brief virtual method in FileOp
*/
void setTransId(const TxnID& transId) {Dctnry::setTransId(transId); if(m_chunkManager) m_chunkManager->setTransId(transId);}
/**
* @brief virtual method in FileOp
*/
void setTransId(const TxnID& transId)
{
Dctnry::setTransId(transId);
/**
* @brief Set the IsInsert flag in the ChunkManager.
* This forces flush at end of statement. Used only for bulk insert.
*/
void setIsInsert(bool isInsert) { m_chunkManager->setIsInsert(isInsert); }
if (m_chunkManager) m_chunkManager->setTransId(transId);
}
/**
* @brief Set the IsInsert flag in the ChunkManager.
* This forces flush at end of statement. Used only for bulk insert.
*/
void setIsInsert(bool isInsert)
{
m_chunkManager->setIsInsert(isInsert);
}
protected:
/**
* @brief virtual method in FileOp
*/
int updateDctnryExtent(IDBDataFile* pFile, int nBlocks);
/**
* @brief virtual method in FileOp
*/
int updateDctnryExtent(IDBDataFile* pFile, int nBlocks);
/**
* @brief convert lbid to fbo
*/
int lbidToFbo(const uint64_t lbid, int& fbo);
/**
* @brief convert lbid to fbo
*/
int lbidToFbo(const uint64_t lbid, int& fbo);
};

69
writeengine/wrapper/we_tablemetadata.cpp
View File

@ -38,16 +38,18 @@ TableMetaData::TableMetaDataMap TableMetaData::fTableMetaDataMap;
TableMetaData* TableMetaData::makeTableMetaData(uint32_t tableOid)
{
boost::mutex::scoped_lock lock(map_mutex);
TableMetaData* instance;
boost::mutex::scoped_lock lock(map_mutex);
TableMetaData* instance;
TableMetaDataMap::const_iterator it = fTableMetaDataMap.find(tableOid);
if (it == fTableMetaDataMap.end())
{
instance = new TableMetaData();
fTableMetaDataMap[tableOid] = instance;
return instance;
}
return it->second;
{
instance = new TableMetaData();
fTableMetaDataMap[tableOid] = instance;
return instance;
}
return it->second;
}
/* static */
@ -55,11 +57,12 @@ void TableMetaData::removeTableMetaData(uint32_t tableOid)
{
boost::mutex::scoped_lock lock(map_mutex);
TableMetaDataMap::iterator it = fTableMetaDataMap.find(tableOid);
if (it != fTableMetaDataMap.end())
{
delete (*it).second;
fTableMetaDataMap.erase(it);
}
}
}
TableMetaData::TableMetaData()
@ -69,40 +72,42 @@ TableMetaData::~TableMetaData()
{
}
ColExtsInfo & TableMetaData::getColExtsInfo (OID columnOid)
ColExtsInfo& TableMetaData::getColExtsInfo (OID columnOid)
{
boost::mutex::scoped_lock lock(fColsExtsInfoLock);
ColsExtsInfoMap::iterator it = fColsExtsInfoMap.find(columnOid);
boost::mutex::scoped_lock lock(fColsExtsInfoLock);
ColsExtsInfoMap::iterator it = fColsExtsInfoMap.find(columnOid);
if (it != fColsExtsInfoMap.end())
{
return it->second;
}
else
{
ColExtsInfo colExtsInfo;
fColsExtsInfoMap[columnOid] = colExtsInfo;
return fColsExtsInfoMap[columnOid];
}
{
return it->second;
}
else
{
ColExtsInfo colExtsInfo;
fColsExtsInfoMap[columnOid] = colExtsInfo;
return fColsExtsInfoMap[columnOid];
}
}
void TableMetaData::setColExtsInfo (OID columnOid, ColExtsInfo colExtsInfo)
{
boost::mutex::scoped_lock lock(fColsExtsInfoLock);
ColsExtsInfoMap::iterator it = fColsExtsInfoMap.find(columnOid);
boost::mutex::scoped_lock lock(fColsExtsInfoLock);
ColsExtsInfoMap::iterator it = fColsExtsInfoMap.find(columnOid);
if (it != fColsExtsInfoMap.end())
{
it->second = colExtsInfo;
}
else
{
fColsExtsInfoMap[columnOid] = colExtsInfo;
}
{
it->second = colExtsInfo;
}
else
{
fColsExtsInfoMap[columnOid] = colExtsInfo;
}
}
ColsExtsInfoMap& TableMetaData::getColsExtsInfoMap()
{
boost::mutex::scoped_lock lock(fColsExtsInfoLock);
return fColsExtsInfoMap;
boost::mutex::scoped_lock lock(fColsExtsInfoLock);
return fColsExtsInfoMap;
}
} //end of namespace
// vim:ts=4 sw=4:

81
writeengine/wrapper/we_tablemetadata.h
View File

@ -37,32 +37,33 @@
namespace WriteEngine
{
struct ColExtInfo {
uint16_t dbRoot;
uint32_t partNum;
uint16_t segNum;
HWM hwm;
RID lastRid;
int64_t max;
int64_t min;
bool isNewExt;
bool current;
uint16_t compType;
bool isDict;
ColExtInfo() :
dbRoot(0),
partNum(0),
segNum(0),
hwm(0),
lastRid(0),
max(0),
min(0),
isNewExt(false),
current(true),
compType(2),
isDict(false)
{}
struct ColExtInfo
{
uint16_t dbRoot;
uint32_t partNum;
uint16_t segNum;
HWM hwm;
RID lastRid;
int64_t max;
int64_t min;
bool isNewExt;
bool current;
uint16_t compType;
bool isDict;
ColExtInfo() :
dbRoot(0),
partNum(0),
segNum(0),
hwm(0),
lastRid(0),
max(0),
min(0),
isNewExt(false),
current(true),
compType(2),
isDict(false)
{}
};
typedef std::vector<ColExtInfo> ColExtsInfo;
@ -72,26 +73,26 @@ typedef std::map<OID, ColExtsInfo> ColsExtsInfoMap;
class TableMetaData
{
public:
typedef std::map <uint32_t, TableMetaData*> TableMetaDataMap;
EXPORT static TableMetaData* makeTableMetaData(uint32_t tableOid);
/** remove and delete the instance map to the tableOid
typedef std::map <uint32_t, TableMetaData*> TableMetaDataMap;
EXPORT static TableMetaData* makeTableMetaData(uint32_t tableOid);
/** remove and delete the instance map to the tableOid
* @param tableOid
*/
*/
EXPORT static void removeTableMetaData(uint32_t tableOid);
EXPORT ColExtsInfo & getColExtsInfo (OID columnOid);
EXPORT void setColExtsInfo (OID columnOid, ColExtsInfo colExtsInfo);
EXPORT ColsExtsInfoMap& getColsExtsInfoMap();
EXPORT ColExtsInfo& getColExtsInfo (OID columnOid);
EXPORT void setColExtsInfo (OID columnOid, ColExtsInfo colExtsInfo);
EXPORT ColsExtsInfoMap& getColsExtsInfoMap();
private:
/** Constuctors */
/** Constuctors */
explicit TableMetaData();
explicit TableMetaData(const TableMetaData& rhs);
~TableMetaData();
static boost::mutex map_mutex;
static TableMetaDataMap fTableMetaDataMap;
boost::mutex fColsExtsInfoLock;
ColsExtsInfoMap fColsExtsInfoMap;
~TableMetaData();
static boost::mutex map_mutex;
static TableMetaDataMap fTableMetaDataMap;
boost::mutex fColsExtsInfoLock;
ColsExtsInfoMap fColsExtsInfoMap;
};
} //end of namespace

8975
writeengine/wrapper/writeengine.cpp
View File

File diff suppressed because it is too large Load Diff

1039
writeengine/wrapper/writeengine.h
View File

File diff suppressed because it is too large Load Diff