database/mariadb-columnstore-engine
1
0
Fork 0
mirror of https://github.com/mariadb-corporation/mariadb-columnstore-engine.git synced 2025-07-29 08:21:15 +03:00
Code Activity

clang format apply

Browse Source
This commit is contained in:
Leonid Fedorov
2022-01-21 16:43:49 +00:00
parent 6b6411229f
commit 04752ec546
1376 changed files with 393460 additions and 412662 deletions
Download Patch File Download Diff File Expand all files Collapse all files

123
writeengine/shared/tconfig.cpp
View File

@ -13,92 +13,73 @@ using namespace WriteEngine;
void test()
{
std::cout << "getDBRootByIdx(1): " <<
Config::getDBRootByIdx(1) << std::endl;
std::cout << "getDBRootByIdx(3): " <<
Config::getDBRootByIdx(3) << std::endl;
std::cout << "getDBRootByIdx(1): " << Config::getDBRootByIdx(1) << std::endl;
std::cout << "getDBRootByIdx(3): " << Config::getDBRootByIdx(3) << std::endl;
std::cout << "getDBRootByNum(1): " <<
Config::getDBRootByNum(1) << std::endl;
std::cout << "getDBRootByNum(3): " <<
Config::getDBRootByNum(3) << std::endl;
std::cout << "getDBRootByNum(1): " << Config::getDBRootByNum(1) << std::endl;
std::cout << "getDBRootByNum(3): " << Config::getDBRootByNum(3) << std::endl;
std::vector<unsigned short> dbRootIds;
Config::getRootIdList( dbRootIds );
std::cout << "getRootIdList: ";
std::vector<unsigned short> dbRootIds;
Config::getRootIdList(dbRootIds);
std::cout << "getRootIdList: ";
for (unsigned k = 0; k < dbRootIds.size(); k++)
{
std::cout << dbRootIds[k] << ' ';
}
for (unsigned k = 0; k < dbRootIds.size(); k++)
{
std::cout << dbRootIds[k] << ' ';
}
std::cout << std::endl;
std::cout << std::endl;
std::vector<std::string> dbRootPathList;
Config::getDBRootPathList( dbRootPathList );
std::cout << "getDBRootPathList: " << std::endl;
std::vector<std::string> dbRootPathList;
Config::getDBRootPathList(dbRootPathList);
std::cout << "getDBRootPathList: " << std::endl;
for (unsigned k = 0; k < dbRootPathList.size(); k++)
std::cout << " " << k << ". " << dbRootPathList[k] << std::endl;
for (unsigned k = 0; k < dbRootPathList.size(); k++)
std::cout << " " << k << ". " << dbRootPathList[k] << std::endl;
std::cout << "getBulkRoot(): " <<
Config::getBulkRoot() << std::endl;
std::cout << "DBRootCount(): " <<
Config::DBRootCount() << std::endl;
std::cout << "totalDBRootCount(): " <<
Config::totalDBRootCount() << std::endl;
std::cout << "getWaitPeriod(): " <<
Config::getWaitPeriod() << std::endl;
std::cout << "getFilePerColumnPartition(): " <<
Config::getFilesPerColumnPartition() << std::endl;
std::cout << "getExtentsPerSegmentFile(): " <<
Config::getExtentsPerSegmentFile() << std::endl;
std::cout << "getBulkProcessPriority(): " <<
Config::getBulkProcessPriority() << std::endl;
std::cout << "getBulkRollbackDir(): " <<
Config::getBulkRollbackDir() << std::endl;
std::cout << "getMaxFileSystemDiskUsage(): " <<
Config::getMaxFileSystemDiskUsage() << std::endl;
std::cout << "getNumCompressedPadBlks(): " <<
Config::getNumCompressedPadBlks() << std::endl;
std::cout << "getParentOAMModuleFlag(): " <<
Config::getParentOAMModuleFlag() << std::endl;
std::cout << "getLocalModuleType(): " <<
Config::getLocalModuleType() << std::endl;
std::cout << "getLocalModuleID(): " <<
Config::getLocalModuleID() << std::endl;
std::cout << "getVBRoot(): " <<
Config::getVBRoot() << std::endl;
std::cout << "getBulkRoot(): " << Config::getBulkRoot() << std::endl;
std::cout << "DBRootCount(): " << Config::DBRootCount() << std::endl;
std::cout << "totalDBRootCount(): " << Config::totalDBRootCount() << std::endl;
std::cout << "getWaitPeriod(): " << Config::getWaitPeriod() << std::endl;
std::cout << "getFilePerColumnPartition(): " << Config::getFilesPerColumnPartition() << std::endl;
std::cout << "getExtentsPerSegmentFile(): " << Config::getExtentsPerSegmentFile() << std::endl;
std::cout << "getBulkProcessPriority(): " << Config::getBulkProcessPriority() << std::endl;
std::cout << "getBulkRollbackDir(): " << Config::getBulkRollbackDir() << std::endl;
std::cout << "getMaxFileSystemDiskUsage(): " << Config::getMaxFileSystemDiskUsage() << std::endl;
std::cout << "getNumCompressedPadBlks(): " << Config::getNumCompressedPadBlks() << std::endl;
std::cout << "getParentOAMModuleFlag(): " << Config::getParentOAMModuleFlag() << std::endl;
std::cout << "getLocalModuleType(): " << Config::getLocalModuleType() << std::endl;
std::cout << "getLocalModuleID(): " << Config::getLocalModuleID() << std::endl;
std::cout << "getVBRoot(): " << Config::getVBRoot() << std::endl;
}
int main()
{
Config::initConfigCache();
char resp;
Config::initConfigCache();
char resp;
int nTest = 0;
int nTest = 0;
std::cout << std::endl;
while (1)
{
std::cout << "test" << nTest << "..." << std::endl;
test();
std::cout << "Pause..." << std::endl;
std::cin >> resp;
std::cout << std::endl;
while (1)
if (resp == 'c')
{
std::cout << "test" << nTest << "..." << std::endl;
test();
std::cout << "Pause..." << std::endl;
std::cin >> resp;
std::cout << std::endl;
if (resp == 'c')
{
std::cout << "Has local DBRootList changed: " <<
(bool)Config::hasLocalDBRootListChanged() << std::endl;
}
else if (resp == 'q')
{
break;
}
nTest++;
std::cout << "Has local DBRootList changed: " << (bool)Config::hasLocalDBRootListChanged() << std::endl;
}
else if (resp == 'q')
{
break;
}
return 0;
nTest++;
}
return 0;
}

153
writeengine/shared/we_blockop.cpp
View File

@ -16,9 +16,9 @@
MA 02110-1301, USA. */
/*******************************************************************************
* $Id: we_blockop.cpp 4500 2013-01-31 20:25:42Z dhall $
*
*******************************************************************************/
* $Id: we_blockop.cpp 4500 2013-01-31 20:25:42Z dhall $
*
*******************************************************************************/
/** @file */
#include <stdio.h>
@ -37,18 +37,19 @@ using namespace execplan;
namespace WriteEngine
{
/**
* Constructor
*/
BlockOp::BlockOp(): m_typeHandler(nullptr)
{}
BlockOp::BlockOp() : m_typeHandler(nullptr)
{
}
/**
* Default Destructor
*/
BlockOp::~BlockOp()
{}
{
}
/***********************************************************
* DESCRIPTION:
@ -62,16 +63,15 @@ BlockOp::~BlockOp()
* bio - Block internal offset
* true if success, false otherwise
***********************************************************/
bool BlockOp::calculateRowId(
RID rowId, const int epb, const int width, int& fbo, int& bio ) const
bool BlockOp::calculateRowId(RID rowId, const int epb, const int width, int& fbo, int& bio) const
{
if ( std::numeric_limits<WriteEngine::RID>::max() == rowId )
return false;
if (std::numeric_limits<WriteEngine::RID>::max() == rowId)
return false;
fbo = (int)( rowId / epb );
bio = ( rowId & ( epb - 1 )) * width;
fbo = (int)(rowId / epb);
bio = (rowId & (epb - 1)) * width;
return true;
return true;
}
/***********************************************************
@ -83,20 +83,18 @@ bool BlockOp::calculateRowId(
* RETURN:
* emptyVal - the value of empty row
***********************************************************/
const uint8_t* BlockOp::getEmptyRowValue(
const CalpontSystemCatalog::ColDataType colDataType,
const int width) const
const uint8_t* BlockOp::getEmptyRowValue(const CalpontSystemCatalog::ColDataType colDataType,
const int width) const
{
auto attrs = datatypes::SystemCatalog::TypeAttributesStd(width, 0, -1);
// Bulk operation runtime should have m_typeHandler nullptr calling this
// Non-bulk operations runtime branch
if (m_typeHandler)
return m_typeHandler->getEmptyValueForType(attrs);
auto attrs = datatypes::SystemCatalog::TypeAttributesStd(width, 0, -1);
// Bulk operation runtime should have m_typeHandler nullptr calling this
// Non-bulk operations runtime branch
if (m_typeHandler)
return m_typeHandler->getEmptyValueForType(attrs);
// Bulk operation branch
auto* typeHandler = datatypes::TypeHandler::find(colDataType,
attrs);
return typeHandler->getEmptyValueForType(attrs);
// Bulk operation branch
auto* typeHandler = datatypes::TypeHandler::find(colDataType, attrs);
return typeHandler->getEmptyValueForType(attrs);
}
/***********************************************************
@ -108,10 +106,9 @@ const uint8_t* BlockOp::getEmptyRowValue(
* RETURN:
* emptyVal - the value of empty row
***********************************************************/
int BlockOp::getCorrectRowWidth(
const CalpontSystemCatalog::ColDataType colDataType, const int width ) const
int BlockOp::getCorrectRowWidth(const CalpontSystemCatalog::ColDataType colDataType, const int width) const
{
return Convertor::getCorrectRowWidth(colDataType, width);
return Convertor::getCorrectRowWidth(colDataType, width);
}
/***********************************************************
@ -124,12 +121,11 @@ int BlockOp::getCorrectRowWidth(
* RETURN:
* row id
***********************************************************/
RID BlockOp::getRowId(
const long fbo, const int width, const int rowPos
/*const int bio, const int bbo*/ ) const
RID BlockOp::getRowId(const long fbo, const int width, const int rowPos
/*const int bio, const int bbo*/) const
{
// return fbo*BYTE_PER_BLOCK*ROW_PER_BYTE + bio*ROW_PER_BYTE + bbo;
return (BYTE_PER_BLOCK / width) * fbo + rowPos;
// return fbo*BYTE_PER_BLOCK*ROW_PER_BYTE + bio*ROW_PER_BYTE + bbo;
return (BYTE_PER_BLOCK / width) * fbo + rowPos;
}
/***********************************************************
@ -141,10 +137,9 @@ RID BlockOp::getRowId(
* RETURN:
* val - buffer value
***********************************************************/
void BlockOp::readBufValue(
const unsigned char* buf, void* val, const short width ) const
void BlockOp::readBufValue(const unsigned char* buf, void* val, const short width) const
{
memcpy( val, buf, width );
memcpy(val, buf, width);
}
/***********************************************************
@ -156,9 +151,9 @@ void BlockOp::readBufValue(
* RETURN:
* none
***********************************************************/
void BlockOp::resetBuf( unsigned char* buf, const int bufSize ) const
void BlockOp::resetBuf(unsigned char* buf, const int bufSize) const
{
memset( buf, 0, bufSize );
memset(buf, 0, bufSize);
}
/***********************************************************
@ -173,45 +168,38 @@ void BlockOp::resetBuf( unsigned char* buf, const int bufSize ) const
* none
***********************************************************/
/* static */
void BlockOp::setEmptyBuf(
unsigned char* buf,
const int bufSize,
const uint8_t* emptyVal,
const int width )
void BlockOp::setEmptyBuf(unsigned char* buf, const int bufSize, const uint8_t* emptyVal, const int width)
{
const int ARRAY_COUNT = 128;
const int NBYTES_IN_ARRAY = width * ARRAY_COUNT;
//unsigned char emptyValArray[NBYTES_IN_ARRAY];
unsigned char* emptyValArray = (unsigned char*)alloca(NBYTES_IN_ARRAY);
const int ARRAY_COUNT = 128;
const int NBYTES_IN_ARRAY = width * ARRAY_COUNT;
// unsigned char emptyValArray[NBYTES_IN_ARRAY];
unsigned char* emptyValArray = (unsigned char*)alloca(NBYTES_IN_ARRAY);
// Optimize buffer initialization by constructing and copying in an array
// instead of individual values. This reduces the number of calls to
// memcpy().
for(uint8_t* pos = emptyValArray, * end = pos + NBYTES_IN_ARRAY; pos < end; pos += width) //FIXME for no loop
{
memcpy(pos, emptyVal, width);
}
// Optimize buffer initialization by constructing and copying in an array
// instead of individual values. This reduces the number of calls to
// memcpy().
int countFull128 = (bufSize / width) / ARRAY_COUNT;
int countRemain = (bufSize / width) % ARRAY_COUNT;
for (uint8_t *pos = emptyValArray, *end = pos + NBYTES_IN_ARRAY; pos < end;
pos += width) // FIXME for no loop
{
memcpy(pos, emptyVal, width);
}
// Copy in the 128 element array into "buf" as many times as needed
if (countFull128 > 0)
{
for ( int i = 0; i < countFull128; i++ )
memcpy( buf + (i * (NBYTES_IN_ARRAY)),
emptyValArray,
NBYTES_IN_ARRAY );
}
int countFull128 = (bufSize / width) / ARRAY_COUNT;
int countRemain = (bufSize / width) % ARRAY_COUNT;
// Initialize the remainder of "buf" that is leftover
if (countRemain > 0)
{
memcpy( buf + (countFull128 * NBYTES_IN_ARRAY),
emptyValArray,
width * countRemain );
}
// Copy in the 128 element array into "buf" as many times as needed
if (countFull128 > 0)
{
for (int i = 0; i < countFull128; i++)
memcpy(buf + (i * (NBYTES_IN_ARRAY)), emptyValArray, NBYTES_IN_ARRAY);
}
// Initialize the remainder of "buf" that is leftover
if (countRemain > 0)
{
memcpy(buf + (countFull128 * NBYTES_IN_ARRAY), emptyValArray, width * countRemain);
}
}
/***********************************************************
@ -224,25 +212,20 @@ void BlockOp::setEmptyBuf(
* RETURN:
* none
***********************************************************/
void BlockOp::writeBufValue(
unsigned char* buf, const void* val, const size_t width, const bool clear ) const
void BlockOp::writeBufValue(unsigned char* buf, const void* val, const size_t width, const bool clear) const
{
if ( clear )
memset( buf, 0, width );
if (clear)
memset(buf, 0, width);
memcpy( buf, val, width );
memcpy(buf, val, width);
}
void BlockOp::findTypeHandler(const int colWidth,
const execplan::CalpontSystemCatalog::ColDataType colDataType)
{
auto attrs = datatypes::SystemCatalog::TypeAttributesStd(colWidth,
0,
-1);
m_typeHandler = datatypes::TypeHandler::find(colDataType,
attrs);
auto attrs = datatypes::SystemCatalog::TypeAttributesStd(colWidth, 0, -1);
m_typeHandler = datatypes::TypeHandler::find(colDataType, attrs);
}
} //end of namespace
} // namespace WriteEngine

156
writeengine/shared/we_blockop.h
View File

@ -16,9 +16,9 @@
MA 02110-1301, USA. */
/*******************************************************************************
* $Id: we_blockop.h 4450 2013-01-21 14:13:24Z rdempsey $
*
*******************************************************************************/
* $Id: we_blockop.h 4450 2013-01-21 14:13:24Z rdempsey $
*
*******************************************************************************/
/** @file */
#pragma once
@ -34,106 +34,88 @@
/** Namespace WriteEngine */
namespace WriteEngine
{
/** Class BlockOp */
class BlockOp : public WEObj
{
public:
/**
* @brief Constructor
*/
EXPORT BlockOp();
public:
/**
* @brief Constructor
*/
EXPORT BlockOp();
/**
* @brief Default Destructor
*/
EXPORT ~BlockOp();
/**
* @brief Default Destructor
*/
EXPORT ~BlockOp();
/**
* @brief Calculate the location of Row ID
*/
EXPORT bool calculateRowId(RID rowId, const int epb, const int width, int& fbo, int& bio) const;
/**
* @brief Calculate the location of Row ID
*/
EXPORT bool calculateRowId( RID rowId,
const int epb,
const int width,
int& fbo,
int& bio ) const;
/**
* @brief Calculate the location of Row ID
*/
void clearBlock(DataBlock* block)
{
memset(block->data, 0, sizeof(block->data));
block->no = -1;
block->dirty = false;
}
/**
* @brief Calculate the location of Row ID
*/
void clearBlock( DataBlock* block )
{
memset(block->data, 0, sizeof(block->data));
block->no = -1;
block->dirty = false;
}
/**
* @brief Get bit value after shift
*/
uint64_t getBitValue(uint64_t val, int shiftBit, uint64_t mask) const
{
return (val >> shiftBit) & mask;
}
/**
* @brief Get bit value after shift
*/
uint64_t getBitValue( uint64_t val,
int shiftBit,
uint64_t mask ) const
{
return ( val >> shiftBit ) & mask ;
}
/**
* @brief Get correct row width
*/
EXPORT int getCorrectRowWidth(const execplan::CalpontSystemCatalog::ColDataType colDataType,
const int width) const;
/**
* @brief Get correct row width
*/
EXPORT int getCorrectRowWidth( const execplan::CalpontSystemCatalog::ColDataType colDataType,
const int width ) const;
/**
* @brief Get an empty row value
*/
EXPORT const uint8_t* getEmptyRowValue(const execplan::CalpontSystemCatalog::ColDataType colDataType,
const int width) const;
/**
* @brief Get an empty row value
*/
EXPORT const uint8_t* getEmptyRowValue(const execplan::CalpontSystemCatalog::ColDataType colDataType,
const int width) const;
/**
* @brief Calculate row id
*/
EXPORT RID getRowId(const long fbo, const int width, const int rowPos) const;
/**
* @brief Get buffer value
*/
EXPORT void readBufValue(const unsigned char* buf, void* val, const short width) const;
/**
* @brief Calculate row id
*/
EXPORT RID getRowId( const long fbo,
const int width,
const int rowPos ) const;
/**
* @brief Reset a buffer
*/
EXPORT void resetBuf(unsigned char* buf, const int bufSize) const;
/**
* @brief Get buffer value
*/
EXPORT void readBufValue( const unsigned char* buf,
void* val, const short width ) const;
/**
* @brief Fill buffer with empty values
*/
EXPORT void static setEmptyBuf(unsigned char* buf, const int bufSize, const uint8_t* emptyVal,
const int width);
/**
* @brief Reset a buffer
*/
EXPORT void resetBuf( unsigned char* buf,
const int bufSize ) const;
/**
* @brief Fill buffer with empty values
*/
EXPORT void static setEmptyBuf( unsigned char* buf,
const int bufSize,
const uint8_t* emptyVal,
const int width );
/**
* @brief Set a value in a buffer
*/
EXPORT void writeBufValue( unsigned char* buf,
const void* val,
const size_t width,
const bool clear = false ) const;
EXPORT void findTypeHandler( const int colWidth,
/**
* @brief Set a value in a buffer
*/
EXPORT void writeBufValue(unsigned char* buf, const void* val, const size_t width,
const bool clear = false) const;
EXPORT void findTypeHandler(const int colWidth,
const execplan::CalpontSystemCatalog::ColDataType colDataType);
private:
const datatypes::TypeHandler* m_typeHandler;
private:
const datatypes::TypeHandler* m_typeHandler;
};
} //end of namespace
} // namespace WriteEngine
#undef EXPORT

2566
writeengine/shared/we_brm.cpp
View File

File diff suppressed because it is too large Load Diff

1000
writeengine/shared/we_brm.h
View File

File diff suppressed because it is too large Load Diff

548
writeengine/shared/we_bulkrollbackfile.cpp
View File

@ -16,8 +16,8 @@
MA 02110-1301, USA. */
/*
* $Id: we_bulkrollbackfile.cpp 4737 2013-08-14 20:45:46Z bwilkinson $
*/
* $Id: we_bulkrollbackfile.cpp 4737 2013-08-14 20:45:46Z bwilkinson $
*/
#include "we_bulkrollbackfile.h"
#include "we_bulkrollbackmgr.h"
@ -34,26 +34,23 @@ using namespace execplan;
namespace WriteEngine
{
//------------------------------------------------------------------------------
// BulkRollbackFile constructor
//------------------------------------------------------------------------------
BulkRollbackFile::BulkRollbackFile(BulkRollbackMgr* mgr) : fMgr(mgr)
{
// Initialize empty dictionary header block used when reinitializing
// dictionary store extents.
const uint16_t freeSpace = BYTE_PER_BLOCK -
(HDR_UNIT_SIZE + NEXT_PTR_BYTES + HDR_UNIT_SIZE + HDR_UNIT_SIZE);
const uint64_t nextPtr = NOT_USED_PTR;
const uint16_t offSetZero = BYTE_PER_BLOCK;
const uint16_t endHeader = DCTNRY_END_HEADER;
// Initialize empty dictionary header block used when reinitializing
// dictionary store extents.
const uint16_t freeSpace =
BYTE_PER_BLOCK - (HDR_UNIT_SIZE + NEXT_PTR_BYTES + HDR_UNIT_SIZE + HDR_UNIT_SIZE);
const uint64_t nextPtr = NOT_USED_PTR;
const uint16_t offSetZero = BYTE_PER_BLOCK;
const uint16_t endHeader = DCTNRY_END_HEADER;
memcpy(fDctnryHdr, &freeSpace, HDR_UNIT_SIZE);
memcpy(fDctnryHdr + HDR_UNIT_SIZE, &nextPtr, NEXT_PTR_BYTES);
memcpy(fDctnryHdr + HDR_UNIT_SIZE + NEXT_PTR_BYTES,
&offSetZero, HDR_UNIT_SIZE);
memcpy(fDctnryHdr + HDR_UNIT_SIZE + NEXT_PTR_BYTES + HDR_UNIT_SIZE,
&endHeader, HDR_UNIT_SIZE);
memcpy(fDctnryHdr, &freeSpace, HDR_UNIT_SIZE);
memcpy(fDctnryHdr + HDR_UNIT_SIZE, &nextPtr, NEXT_PTR_BYTES);
memcpy(fDctnryHdr + HDR_UNIT_SIZE + NEXT_PTR_BYTES, &offSetZero, HDR_UNIT_SIZE);
memcpy(fDctnryHdr + HDR_UNIT_SIZE + NEXT_PTR_BYTES + HDR_UNIT_SIZE, &endHeader, HDR_UNIT_SIZE);
}
//------------------------------------------------------------------------------
@ -73,35 +70,24 @@ BulkRollbackFile::~BulkRollbackFile()
// segNum - Segment number of segment file to be found
// segFileName (out) - Name of segment file
//------------------------------------------------------------------------------
void BulkRollbackFile::buildSegmentFileName(
OID columnOID,
bool fileTypeFlag,
uint32_t dbRoot,
uint32_t partNum,
uint32_t segNum,
std::string& segFileName )
void BulkRollbackFile::buildSegmentFileName(OID columnOID, bool fileTypeFlag, uint32_t dbRoot,
uint32_t partNum, uint32_t segNum, std::string& segFileName)
{
char fileName[FILE_NAME_SIZE];
int rc = fDbFile.getFileName( columnOID, fileName,
dbRoot, partNum, segNum );
char fileName[FILE_NAME_SIZE];
int rc = fDbFile.getFileName(columnOID, fileName, dbRoot, partNum, segNum);
if (rc != NO_ERROR)
{
WErrorCodes ec;
std::ostringstream oss;
oss << "Error constructing " <<
(fileTypeFlag ? "column" : "dictionary store") <<
" filename for deletion" <<
"; columnOID-" << columnOID <<
"; dbRoot-" << dbRoot <<
"; partNum-" << partNum <<
"; segNum-" << segNum <<
"; " << ec.errorString(rc);
if (rc != NO_ERROR)
{
WErrorCodes ec;
std::ostringstream oss;
oss << "Error constructing " << (fileTypeFlag ? "column" : "dictionary store") << " filename for deletion"
<< "; columnOID-" << columnOID << "; dbRoot-" << dbRoot << "; partNum-" << partNum << "; segNum-"
<< segNum << "; " << ec.errorString(rc);
throw WeException( oss.str(), rc );
}
throw WeException(oss.str(), rc);
}
segFileName = fileName;
segFileName = fileName;
}
//------------------------------------------------------------------------------
@ -114,41 +100,31 @@ void BulkRollbackFile::buildSegmentFileName(
// segNum - Segment number of segment file to be deleted
// segFileName - Name of file to be deleted
//------------------------------------------------------------------------------
void BulkRollbackFile::deleteSegmentFile(
OID columnOID,
bool fileTypeFlag,
uint32_t dbRoot,
uint32_t partNum,
uint32_t segNum,
const std::string& segFileName )
void BulkRollbackFile::deleteSegmentFile(OID columnOID, bool fileTypeFlag, uint32_t dbRoot, uint32_t partNum,
uint32_t segNum, const std::string& segFileName)
{
std::ostringstream msgText;
msgText << "Deleting " << (fileTypeFlag ? "column" : "dictionary store") <<
" file: dbRoot-" << dbRoot <<
"; part#-" << partNum <<
"; seg#-" << segNum;
fMgr->logAMessage( logging::LOG_TYPE_INFO,
logging::M0075, columnOID, msgText.str() );
std::ostringstream msgText;
msgText << "Deleting " << (fileTypeFlag ? "column" : "dictionary store") << " file: dbRoot-" << dbRoot
<< "; part#-" << partNum << "; seg#-" << segNum;
fMgr->logAMessage(logging::LOG_TYPE_INFO, logging::M0075, columnOID, msgText.str());
// delete the db segment file if it exists
int rc = fDbFile.deleteFile( segFileName.c_str() );
// delete the db segment file if it exists
int rc = fDbFile.deleteFile(segFileName.c_str());
if (rc != NO_ERROR)
if (rc != NO_ERROR)
{
if (rc != ERR_FILE_NOT_EXIST)
{
if (rc != ERR_FILE_NOT_EXIST)
{
WErrorCodes ec;
std::ostringstream oss;
oss << "Error deleting segment file"
"; columnOID-" << columnOID <<
"; dbRoot-" << dbRoot <<
"; partNum-" << partNum <<
"; segNum-" << segNum <<
"; " << ec.errorString(rc);
WErrorCodes ec;
std::ostringstream oss;
oss << "Error deleting segment file"
"; columnOID-"
<< columnOID << "; dbRoot-" << dbRoot << "; partNum-" << partNum << "; segNum-" << segNum << "; "
<< ec.errorString(rc);
throw WeException( oss.str(), rc );
}
throw WeException(oss.str(), rc);
}
}
}
//------------------------------------------------------------------------------
@ -161,59 +137,47 @@ void BulkRollbackFile::deleteSegmentFile(
// fileSizeBlocks - Number of blocks to be left in the file. Remainder of file
// is to be truncated.
//------------------------------------------------------------------------------
void BulkRollbackFile::truncateSegmentFile(
OID columnOID,
uint32_t dbRoot,
uint32_t partNum,
uint32_t segNum,
long long fileSizeBlocks )
void BulkRollbackFile::truncateSegmentFile(OID columnOID, uint32_t dbRoot, uint32_t partNum, uint32_t segNum,
long long fileSizeBlocks)
{
long long fileSizeBytes = fileSizeBlocks * BYTE_PER_BLOCK;
long long fileSizeBytes = fileSizeBlocks * BYTE_PER_BLOCK;
std::ostringstream msgText;
msgText << "Truncating column file"
": dbRoot-" << dbRoot <<
"; part#-" << partNum <<
"; seg#-" << segNum <<
"; totBlks-" << fileSizeBlocks <<
"; fileSize(bytes)-" << fileSizeBytes;
fMgr->logAMessage( logging::LOG_TYPE_INFO,
logging::M0075, columnOID, msgText.str() );
std::ostringstream msgText;
msgText << "Truncating column file"
": dbRoot-"
<< dbRoot << "; part#-" << partNum << "; seg#-" << segNum << "; totBlks-" << fileSizeBlocks
<< "; fileSize(bytes)-" << fileSizeBytes;
fMgr->logAMessage(logging::LOG_TYPE_INFO, logging::M0075, columnOID, msgText.str());
std::string segFile;
IDBDataFile* pFile = fDbFile.openFile(columnOID, dbRoot, partNum, segNum, segFile);
std::string segFile;
IDBDataFile* pFile = fDbFile.openFile(columnOID, dbRoot, partNum, segNum, segFile);
if (pFile == 0)
{
std::ostringstream oss;
oss << "Error opening column segment file to rollback extents "
"from DB for" <<
": OID-" << columnOID <<
"; DbRoot-" << dbRoot <<
"; partition-" << partNum <<
"; segment-" << segNum;
if (pFile == 0)
{
std::ostringstream oss;
oss << "Error opening column segment file to rollback extents "
"from DB for"
<< ": OID-" << columnOID << "; DbRoot-" << dbRoot << "; partition-" << partNum << "; segment-"
<< segNum;
throw WeException( oss.str(), ERR_FILE_OPEN );
}
throw WeException(oss.str(), ERR_FILE_OPEN);
}
int rc = fDbFile.truncateFile( pFile, fileSizeBytes );
int rc = fDbFile.truncateFile(pFile, fileSizeBytes);
if (rc != NO_ERROR)
{
WErrorCodes ec;
std::ostringstream oss;
oss << "Error truncating column extents from DB for" <<
": OID-" << columnOID <<
"; DbRoot-" << dbRoot <<
"; partition-" << partNum <<
"; segment-" << segNum <<
"; " << ec.errorString(rc);
if (rc != NO_ERROR)
{
WErrorCodes ec;
std::ostringstream oss;
oss << "Error truncating column extents from DB for"
<< ": OID-" << columnOID << "; DbRoot-" << dbRoot << "; partition-" << partNum << "; segment-"
<< segNum << "; " << ec.errorString(rc);
fDbFile.closeFile( pFile );
throw WeException( oss.str(), rc );
}
fDbFile.closeFile(pFile);
throw WeException(oss.str(), rc);
}
fDbFile.closeFile( pFile );
fDbFile.closeFile(pFile);
}
//------------------------------------------------------------------------------
@ -233,122 +197,98 @@ void BulkRollbackFile::truncateSegmentFile(
// uncompressed, but defined in this base class for the
// compressed derived class.
//------------------------------------------------------------------------------
void BulkRollbackFile::reInitTruncColumnExtent(
OID columnOID,
uint32_t dbRoot,
uint32_t partNum,
uint32_t segNum,
long long startOffsetBlk,
int nBlocks,
CalpontSystemCatalog::ColDataType colType,
uint32_t colWidth,
bool /*restoreHwmChk*/ )
void BulkRollbackFile::reInitTruncColumnExtent(OID columnOID, uint32_t dbRoot, uint32_t partNum,
uint32_t segNum, long long startOffsetBlk, int nBlocks,
CalpontSystemCatalog::ColDataType colType, uint32_t colWidth,
bool /*restoreHwmChk*/)
{
long long startOffset = startOffsetBlk * BYTE_PER_BLOCK;
long long startOffset = startOffsetBlk * BYTE_PER_BLOCK;
std::ostringstream msgText;
msgText << "Reinit HWM column extent in db file"
": dbRoot-" << dbRoot <<
"; part#-" << partNum <<
"; seg#-" << segNum <<
"; offset(bytes)-" << startOffset <<
"; freeBlks-" << nBlocks;
fMgr->logAMessage( logging::LOG_TYPE_INFO,
logging::M0075, columnOID, msgText.str() );
std::ostringstream msgText;
msgText << "Reinit HWM column extent in db file"
": dbRoot-"
<< dbRoot << "; part#-" << partNum << "; seg#-" << segNum << "; offset(bytes)-" << startOffset
<< "; freeBlks-" << nBlocks;
fMgr->logAMessage(logging::LOG_TYPE_INFO, logging::M0075, columnOID, msgText.str());
std::string segFile;
IDBDataFile* pFile = fDbFile.openFile(columnOID, dbRoot, partNum, segNum, segFile);
std::string segFile;
IDBDataFile* pFile = fDbFile.openFile(columnOID, dbRoot, partNum, segNum, segFile);
if (pFile == 0)
if (pFile == 0)
{
std::ostringstream oss;
oss << "Error opening HWM column segment file to rollback extents "
"from DB for"
<< ": OID-" << columnOID << "; DbRoot-" << dbRoot << "; partition-" << partNum << "; segment-"
<< segNum;
throw WeException(oss.str(), ERR_FILE_OPEN);
}
// nBlocks is based on full extents, but if the database file only has an
// abbreviated extent, then we reset nBlocks to reflect the size of the file
// (Only the 1st extent in part0, seg0 employs an abbreviated extent.)
// DMC-SHARED_NOTHING_NOTE: Is it safe to assume only part0 seg0 is abbreviated?
if ((partNum == 0) && (segNum == 0))
{
long long nBytesInAbbrevExtent = INITIAL_EXTENT_ROWS_TO_DISK * colWidth;
if (startOffset <= nBytesInAbbrevExtent)
{
std::ostringstream oss;
oss << "Error opening HWM column segment file to rollback extents "
"from DB for" <<
": OID-" << columnOID <<
"; DbRoot-" << dbRoot <<
"; partition-" << partNum <<
"; segment-" << segNum;
// This check would prevent us from truncating back to an
// abbreviated extent if the failed import expanded the initial
// extent; but when adding compression, decided to go ahead and
// truncate back to an abbreviated extent.
// long long fileSizeBytes;
// int rc = fDbFile.getFileSize2(pFile,fileSizeBytes);
// if (fileSizeBytes == nBytesInAbbrevExtent)
{
nBlocks = (nBytesInAbbrevExtent - startOffset) / BYTE_PER_BLOCK;
throw WeException( oss.str(), ERR_FILE_OPEN );
std::ostringstream msgText2;
msgText2 << "Reinit (abbrev) HWM column extent in db file"
": dbRoot-"
<< dbRoot << "; part#-" << partNum << "; seg#-" << segNum << "; offset(bytes)-"
<< startOffset << "; freeBlks-" << nBlocks;
fMgr->logAMessage(logging::LOG_TYPE_INFO, logging::M0075, columnOID, msgText2.str());
}
}
}
// nBlocks is based on full extents, but if the database file only has an
// abbreviated extent, then we reset nBlocks to reflect the size of the file
// (Only the 1st extent in part0, seg0 employs an abbreviated extent.)
// DMC-SHARED_NOTHING_NOTE: Is it safe to assume only part0 seg0 is abbreviated?
if ((partNum == 0) && (segNum == 0))
{
long long nBytesInAbbrevExtent = INITIAL_EXTENT_ROWS_TO_DISK * colWidth;
// Initialize the remainder of the extent after the HWM block
const uint8_t* emptyVal = fDbFile.getEmptyRowValue(colType, colWidth);
if (startOffset <= nBytesInAbbrevExtent)
{
// This check would prevent us from truncating back to an
// abbreviated extent if the failed import expanded the initial
// extent; but when adding compression, decided to go ahead and
// truncate back to an abbreviated extent.
//long long fileSizeBytes;
//int rc = fDbFile.getFileSize2(pFile,fileSizeBytes);
//if (fileSizeBytes == nBytesInAbbrevExtent)
{
nBlocks = (nBytesInAbbrevExtent - startOffset) / BYTE_PER_BLOCK;
int rc = fDbFile.reInitPartialColumnExtent(pFile, startOffset, nBlocks, emptyVal, colWidth);
std::ostringstream msgText2;
msgText2 << "Reinit (abbrev) HWM column extent in db file"
": dbRoot-" << dbRoot <<
"; part#-" << partNum <<
"; seg#-" << segNum <<
"; offset(bytes)-" << startOffset <<
"; freeBlks-" << nBlocks;
fMgr->logAMessage( logging::LOG_TYPE_INFO,
logging::M0075, columnOID, msgText2.str() );
}
}
}
if (rc != NO_ERROR)
{
WErrorCodes ec;
std::ostringstream oss;
oss << "Error rolling back HWM column extent from DB for"
<< ": OID-" << columnOID << "; DbRoot-" << dbRoot << "; partition-" << partNum << "; segment-"
<< segNum << "; " << ec.errorString(rc);
// Initialize the remainder of the extent after the HWM block
const uint8_t* emptyVal = fDbFile.getEmptyRowValue( colType, colWidth );
fDbFile.closeFile(pFile);
throw WeException(oss.str(), rc);
}
int rc = fDbFile.reInitPartialColumnExtent( pFile,
startOffset,
nBlocks,
emptyVal,
colWidth );
// Truncate the remainder of the file
rc = fDbFile.truncateFile(pFile, pFile->tell());
if (rc != NO_ERROR)
{
WErrorCodes ec;
std::ostringstream oss;
oss << "Error rolling back HWM column extent from DB for" <<
": OID-" << columnOID <<
"; DbRoot-" << dbRoot <<
"; partition-" << partNum <<
"; segment-" << segNum <<
"; " << ec.errorString(rc);
if (rc != NO_ERROR)
{
WErrorCodes ec;
std::ostringstream oss;
oss << "Error truncating post-HWM column extents "
"from HWM segment DB file for"
<< ": OID-" << columnOID << "; DbRoot-" << dbRoot << "; partition-" << partNum << "; segment-"
<< segNum << "; " << ec.errorString(rc);
fDbFile.closeFile( pFile );
throw WeException( oss.str(), rc );
}
fDbFile.closeFile(pFile);
throw WeException(oss.str(), rc);
}
// Truncate the remainder of the file
rc = fDbFile.truncateFile( pFile, pFile->tell() );
if (rc != NO_ERROR)
{
WErrorCodes ec;
std::ostringstream oss;
oss << "Error truncating post-HWM column extents "
"from HWM segment DB file for" <<
": OID-" << columnOID <<
"; DbRoot-" << dbRoot <<
"; partition-" << partNum <<
"; segment-" << segNum <<
"; " << ec.errorString(rc);
fDbFile.closeFile( pFile );
throw WeException( oss.str(), rc );
}
fDbFile.closeFile( pFile );
fDbFile.closeFile(pFile);
}
//------------------------------------------------------------------------------
@ -363,116 +303,92 @@ void BulkRollbackFile::reInitTruncColumnExtent(
// to be reinitialized. Value is in raw data blocks.
// nBlocks - Number of blocks to be reinitialized
//------------------------------------------------------------------------------
void BulkRollbackFile::reInitTruncDctnryExtent(
OID dStoreOID,
uint32_t dbRoot,
uint32_t partNum,
uint32_t segNum,
long long startOffsetBlk,
int nBlocks )
void BulkRollbackFile::reInitTruncDctnryExtent(OID dStoreOID, uint32_t dbRoot, uint32_t partNum,
uint32_t segNum, long long startOffsetBlk, int nBlocks)
{
long long startOffset = startOffsetBlk * BYTE_PER_BLOCK;
long long startOffset = startOffsetBlk * BYTE_PER_BLOCK;
std::ostringstream msgText;
msgText << "Reinit dictionary store extent in db file"
": dbRoot-" << dbRoot <<
"; part#-" << partNum <<
"; seg#-" << segNum <<
"; offset(bytes)-" << startOffset <<
"; numblks-" << nBlocks;
fMgr->logAMessage( logging::LOG_TYPE_INFO,
logging::M0075, dStoreOID, msgText.str() );
std::ostringstream msgText;
msgText << "Reinit dictionary store extent in db file"
": dbRoot-"
<< dbRoot << "; part#-" << partNum << "; seg#-" << segNum << "; offset(bytes)-" << startOffset
<< "; numblks-" << nBlocks;
fMgr->logAMessage(logging::LOG_TYPE_INFO, logging::M0075, dStoreOID, msgText.str());
std::string segFile;
IDBDataFile* pFile = fDbFile.openFile(dStoreOID, dbRoot, partNum, segNum, segFile);
std::string segFile;
IDBDataFile* pFile = fDbFile.openFile(dStoreOID, dbRoot, partNum, segNum, segFile);
if (pFile == 0)
if (pFile == 0)
{
std::ostringstream oss;
oss << "Error opening dictionary store segment file to rollback extents"
" from DB for"
<< ": OID-" << dStoreOID << "; DbRoot-" << dbRoot << "; partition-" << partNum << "; segment-"
<< segNum;
throw WeException(oss.str(), ERR_FILE_OPEN);
}
// nBlocks is based on full extents, but if the database file only has an
// abbreviated extent, then we reset nBlocks to reflect the size of the file
// (Unlike column files which only employ an abbreviated extent for the
// 1st extent in part0, seg0, all new store files start with abbrev extent)
const uint32_t PSEUDO_COL_WIDTH = 8; // simulated col width for dictionary
long long nBytesInAbbrevExtent = INITIAL_EXTENT_ROWS_TO_DISK * PSEUDO_COL_WIDTH;
if (startOffset <= nBytesInAbbrevExtent)
{
// This check would prevent us from truncating back to an
// abbreviated extent if the failed import expanded the initial
// extent; but when adding compression, decided to go ahead and
// truncate back to an abbreviated extent.
// long long fileSizeBytes;
// int rc = fDbFile.getFileSize2(pFile,fileSizeBytes);
// if (fileSizeBytes == nBytesInAbbrevExtent)
{
std::ostringstream oss;
oss << "Error opening dictionary store segment file to rollback extents"
" from DB for" <<
": OID-" << dStoreOID <<
"; DbRoot-" << dbRoot <<
"; partition-" << partNum <<
"; segment-" << segNum;
nBlocks = (nBytesInAbbrevExtent - startOffset) / BYTE_PER_BLOCK;
throw WeException( oss.str(), ERR_FILE_OPEN );
std::ostringstream msgText2;
msgText2 << "Reinit (abbrev) dictionary store extent in db file"
": dbRoot-"
<< dbRoot << "; part#-" << partNum << "; seg#-" << segNum << "; offset(bytes)-" << startOffset
<< "; numblks-" << nBlocks;
fMgr->logAMessage(logging::LOG_TYPE_INFO, logging::M0075, dStoreOID, msgText2.str());
}
}
// nBlocks is based on full extents, but if the database file only has an
// abbreviated extent, then we reset nBlocks to reflect the size of the file
// (Unlike column files which only employ an abbreviated extent for the
// 1st extent in part0, seg0, all new store files start with abbrev extent)
const uint32_t PSEUDO_COL_WIDTH = 8; // simulated col width for dictionary
long long nBytesInAbbrevExtent = INITIAL_EXTENT_ROWS_TO_DISK *
PSEUDO_COL_WIDTH;
// Initialize the remainder of the extent after the HWM block
int rc = fDbFile.reInitPartialDctnryExtent(pFile, startOffset, nBlocks, fDctnryHdr, DCTNRY_HEADER_SIZE);
if (startOffset <= nBytesInAbbrevExtent)
{
// This check would prevent us from truncating back to an
// abbreviated extent if the failed import expanded the initial
// extent; but when adding compression, decided to go ahead and
// truncate back to an abbreviated extent.
//long long fileSizeBytes;
//int rc = fDbFile.getFileSize2(pFile,fileSizeBytes);
//if (fileSizeBytes == nBytesInAbbrevExtent)
{
nBlocks = (nBytesInAbbrevExtent - startOffset) / BYTE_PER_BLOCK;
if (rc != NO_ERROR)
{
WErrorCodes ec;
std::ostringstream oss;
oss << "Error rolling back HWM dictionary store extent from DB for"
<< ": OID-" << dStoreOID << "; DbRoot-" << dbRoot << "; partition-" << partNum << "; segment-"
<< segNum << "; " << ec.errorString(rc);
std::ostringstream msgText2;
msgText2 << "Reinit (abbrev) dictionary store extent in db file"
": dbRoot-" << dbRoot <<
"; part#-" << partNum <<
"; seg#-" << segNum <<
"; offset(bytes)-" << startOffset <<
"; numblks-" << nBlocks;
fMgr->logAMessage( logging::LOG_TYPE_INFO,
logging::M0075, dStoreOID, msgText2.str() );
}
}
fDbFile.closeFile(pFile);
throw WeException(oss.str(), rc);
}
// Initialize the remainder of the extent after the HWM block
int rc = fDbFile.reInitPartialDctnryExtent( pFile,
startOffset,
nBlocks,
fDctnryHdr,
DCTNRY_HEADER_SIZE );
// Truncate the remainder of the file
rc = fDbFile.truncateFile(pFile, pFile->tell());
if (rc != NO_ERROR)
{
WErrorCodes ec;
std::ostringstream oss;
oss << "Error rolling back HWM dictionary store extent from DB for" <<
": OID-" << dStoreOID <<
"; DbRoot-" << dbRoot <<
"; partition-" << partNum <<
"; segment-" << segNum <<
"; " << ec.errorString(rc);
if (rc != NO_ERROR)
{
WErrorCodes ec;
std::ostringstream oss;
oss << "Error truncating post-HWM dictionary store extents "
"from DB file for"
<< ": OID-" << dStoreOID << "; DbRoot-" << dbRoot << "; partition-" << partNum << "; segment-"
<< segNum << "; " << ec.errorString(rc);
fDbFile.closeFile( pFile );
throw WeException( oss.str(), rc );
}
fDbFile.closeFile(pFile);
throw WeException(oss.str(), rc);
}
// Truncate the remainder of the file
rc = fDbFile.truncateFile( pFile, pFile->tell() );
if (rc != NO_ERROR)
{
WErrorCodes ec;
std::ostringstream oss;
oss << "Error truncating post-HWM dictionary store extents "
"from DB file for" <<
": OID-" << dStoreOID <<
"; DbRoot-" << dbRoot <<
"; partition-" << partNum <<
"; segment-" << segNum <<
"; " << ec.errorString(rc);
fDbFile.closeFile( pFile );
throw WeException( oss.str(), rc );
}
fDbFile.closeFile( pFile );
fDbFile.closeFile(pFile);
}
//------------------------------------------------------------------------------
@ -483,12 +399,10 @@ void BulkRollbackFile::reInitTruncDctnryExtent(
// class can override this functionality, and return true or false depending
// on whether the HWM chunk was modified and backed up to disk.
//------------------------------------------------------------------------------
bool BulkRollbackFile::doWeReInitExtent( OID columnOID,
uint32_t dbRoot,
uint32_t partNum,
uint32_t segNum) const
bool BulkRollbackFile::doWeReInitExtent(OID columnOID, uint32_t dbRoot, uint32_t partNum,
uint32_t segNum) const
{
return true;
return true;
}
} //end of namespace
} // namespace WriteEngine

253
writeengine/shared/we_bulkrollbackfile.h
View File

@ -16,8 +16,8 @@
MA 02110-1301, USA. */
/*
* $Id: we_bulkrollbackfile.h 4675 2013-06-13 15:20:37Z dcathey $
*/
* $Id: we_bulkrollbackfile.h 4675 2013-06-13 15:20:37Z dcathey $
*/
/** @file
* Contains class to restore db files on behalf of BulkRollBackMgr.
@ -45,161 +45,130 @@ class BulkRollbackMgr;
//------------------------------------------------------------------------------
class BulkRollbackFile
{
public:
public:
/** @brief BulkRollbackFile constructor
* @param mgr The controlling BulkRollbackMgr object.
*/
BulkRollbackFile(BulkRollbackMgr* mgr);
/** @brief BulkRollbackFile constructor
* @param mgr The controlling BulkRollbackMgr object.
*/
BulkRollbackFile(BulkRollbackMgr* mgr);
/** @brief BulkRollbackFile destructor
*/
virtual ~BulkRollbackFile();
/** @brief BulkRollbackFile destructor
*/
virtual ~BulkRollbackFile();
/** @brief Construct the relevant db filename.
* Warning: This function may throw a WeException.
*
* @param columnOID OID of the segment file to be deleted
* @param fileTypeFlag file type (true->column; false->dictionary)
* @param dbRoot DBRoot of the segment file to be deleted
* @param partNum Partition number of the segment file to be deleted
* @param segNum Segment number of the segment file to be deleted
* @param segFileName (out) Name of segment file
*/
void buildSegmentFileName(OID columnOID, bool fileTypeFlag, uint32_t dbRoot, uint32_t partNum,
uint32_t segNum, std::string& segFileName);
/** @brief Construct the relevant db filename.
* Warning: This function may throw a WeException.
*
* @param columnOID OID of the segment file to be deleted
* @param fileTypeFlag file type (true->column; false->dictionary)
* @param dbRoot DBRoot of the segment file to be deleted
* @param partNum Partition number of the segment file to be deleted
* @param segNum Segment number of the segment file to be deleted
* @param segFileName (out) Name of segment file
*/
void buildSegmentFileName(OID columnOID,
bool fileTypeFlag,
uint32_t dbRoot,
uint32_t partNum,
uint32_t segNum,
std::string& segFileName);
/** @brief Delete a segment file.
* Warning: This function may throw a WeException.
*
* @param columnOID OID of the segment file to be deleted
* @param fileTypeFlag file type (true->column; false->dictionary)
* @param dbRoot DBRoot of the segment file to be deleted
* @param partNum Partition number of the segment file to be deleted
* @param segNum Segment number of the segment file to be deleted
* @param segFileName Name of segment file to be deleted
*/
void deleteSegmentFile(OID columnOID, bool fileTypeFlag, uint32_t dbRoot, uint32_t partNum, uint32_t segNum,
const std::string& segFileName);
/** @brief Delete a segment file.
* Warning: This function may throw a WeException.
*
* @param columnOID OID of the segment file to be deleted
* @param fileTypeFlag file type (true->column; false->dictionary)
* @param dbRoot DBRoot of the segment file to be deleted
* @param partNum Partition number of the segment file to be deleted
* @param segNum Segment number of the segment file to be deleted
* @param segFileName Name of segment file to be deleted
*/
void deleteSegmentFile(OID columnOID,
bool fileTypeFlag,
uint32_t dbRoot,
uint32_t partNum,
uint32_t segNum,
const std::string& segFileName );
/** @brief Construct a directory path.
*
* @param oid (in) OID to use in constructing directory path
* @param dbRoot (in) DBRoot to use in constructing directory path
* @param partition (in) Partition number to use in constructing dir path
* @param dirName (out)Directory path constructed from input arguments
* @return returns NO_ERROR if success
*/
int buildDirName(OID oid, uint16_t dbRoot, uint32_t partition, std::string& dirName);
/** @brief Construct a directory path.
*
* @param oid (in) OID to use in constructing directory path
* @param dbRoot (in) DBRoot to use in constructing directory path
* @param partition (in) Partition number to use in constructing dir path
* @param dirName (out)Directory path constructed from input arguments
* @return returns NO_ERROR if success
*/
int buildDirName( OID oid,
uint16_t dbRoot,
uint32_t partition,
std::string& dirName);
/** @brief Do we reinit trailing blocks in the HWM extent for the specified
* segment file
*
* The base behavior of this function always returns true, to reinit
* any trailing blocks in the HWM extent (for uncompressed data) to
* empty values.
*
* @param columnOID OID of the segment file in question
* @param dbRoot DBRoot for the segment file in question
* @param partNum Partition number for the segment file in question
* @param segNum Segment number for the segment file in question
*/
virtual bool doWeReInitExtent(OID columnOID, uint32_t dbRoot, uint32_t partNum, uint32_t segNum) const;
/** @brief Do we reinit trailing blocks in the HWM extent for the specified
* segment file
*
* The base behavior of this function always returns true, to reinit
* any trailing blocks in the HWM extent (for uncompressed data) to
* empty values.
*
* @param columnOID OID of the segment file in question
* @param dbRoot DBRoot for the segment file in question
* @param partNum Partition number for the segment file in question
* @param segNum Segment number for the segment file in question
*/
virtual bool doWeReInitExtent( OID columnOID,
uint32_t dbRoot,
uint32_t partNum,
uint32_t segNum) const;
/** @brief Reinitialize the specified column segment file starting at
* startOffsetBlk, and truncate trailing extents.
* Warning: This function may throw a WeException.
*
* @param columnOID OID of the relevant segment file
* @param dbRoot DBRoot of the relevant segment file
* @param partNum Partition number of the relevant segment file
* @param segNum Segment number of the relevant segment file
* @param startOffsetBlk Starting block offset where file is to be
* reinitialized
* @param nBlocks Number of blocks to be reinitialized
* @param colType Column type of the relevant segment file
* @param colWidth Width in bytes of column.
* @param restoreHwmChk Restore HWM chunk (n/a to uncompressed)
*/
virtual void reInitTruncColumnExtent(OID columnOID, uint32_t dbRoot, uint32_t partNum, uint32_t segNum,
long long startOffsetBlk, int nBlocks,
execplan::CalpontSystemCatalog::ColDataType colType, uint32_t colWidth,
bool restoreHwmChk);
/** @brief Reinitialize the specified column segment file starting at
* startOffsetBlk, and truncate trailing extents.
* Warning: This function may throw a WeException.
*
* @param columnOID OID of the relevant segment file
* @param dbRoot DBRoot of the relevant segment file
* @param partNum Partition number of the relevant segment file
* @param segNum Segment number of the relevant segment file
* @param startOffsetBlk Starting block offset where file is to be
* reinitialized
* @param nBlocks Number of blocks to be reinitialized
* @param colType Column type of the relevant segment file
* @param colWidth Width in bytes of column.
* @param restoreHwmChk Restore HWM chunk (n/a to uncompressed)
*/
virtual void reInitTruncColumnExtent(OID columnOID,
uint32_t dbRoot,
uint32_t partNum,
uint32_t segNum,
long long startOffsetBlk,
int nBlocks,
execplan::CalpontSystemCatalog::ColDataType colType,
uint32_t colWidth,
bool restoreHwmChk );
/** @brief Reinitialize the specified dictionary store segment file starting
* at startOffsetBlk, and truncate trailing extents.
* Warning: This function may throw a WeException.
*
* @param columnOID OID of the relevant segment file
* @param dbRoot DBRoot of the relevant segment file
* @param partNum Partition number of the relevant segment file
* @param segNum Segment number of the relevant segment file
* @param startOffsetBlk Starting block offset where file is to be
* reinitialized
* @param nBlocks Number of blocks to be reinitialized
*/
virtual void reInitTruncDctnryExtent(OID columnOID, uint32_t dbRoot, uint32_t partNum, uint32_t segNum,
long long startOffsetBlk, int nBlocks);
/** @brief Reinitialize the specified dictionary store segment file starting
* at startOffsetBlk, and truncate trailing extents.
* Warning: This function may throw a WeException.
*
* @param columnOID OID of the relevant segment file
* @param dbRoot DBRoot of the relevant segment file
* @param partNum Partition number of the relevant segment file
* @param segNum Segment number of the relevant segment file
* @param startOffsetBlk Starting block offset where file is to be
* reinitialized
* @param nBlocks Number of blocks to be reinitialized
*/
virtual void reInitTruncDctnryExtent(OID columnOID,
uint32_t dbRoot,
uint32_t partNum,
uint32_t segNum,
long long startOffsetBlk,
int nBlocks );
/** @brief Truncate the specified segment file to a specified num of bytes
* Warning: This function may throw a WeException.
*
* @param columnOID OID of the relevant segment file
* @param dbRoot DBRoot of the relevant segment file
* @param partNum Partition number of the relevant segment file
* @param segNum Segment number of the relevant segment file
* @param fileSizeBlocks Number of blocks to retain in the file
*/
virtual void truncateSegmentFile(OID columnOID, uint32_t dbRoot, uint32_t partNum, uint32_t segNum,
long long filesSizeBlocks);
/** @brief Truncate the specified segment file to a specified num of bytes
* Warning: This function may throw a WeException.
*
* @param columnOID OID of the relevant segment file
* @param dbRoot DBRoot of the relevant segment file
* @param partNum Partition number of the relevant segment file
* @param segNum Segment number of the relevant segment file
* @param fileSizeBlocks Number of blocks to retain in the file
*/
virtual void truncateSegmentFile( OID columnOID,
uint32_t dbRoot,
uint32_t partNum,
uint32_t segNum,
long long filesSizeBlocks );
protected:
BulkRollbackMgr* fMgr; // Bulk Rollback controller
FileOp fDbFile; // interface to DB file
unsigned char fDctnryHdr[DCTNRY_HEADER_SIZE]; // empty dctnry store blk
protected:
BulkRollbackMgr* fMgr; // Bulk Rollback controller
FileOp fDbFile; // interface to DB file
unsigned char fDctnryHdr[DCTNRY_HEADER_SIZE]; // empty dctnry store blk
private:
// Disable unnecessary copy constructor and assignment operator
BulkRollbackFile(const BulkRollbackFile& rhs);
BulkRollbackFile& operator=(const BulkRollbackFile& rhs);
private:
// Disable unnecessary copy constructor and assignment operator
BulkRollbackFile(const BulkRollbackFile& rhs);
BulkRollbackFile& operator=(const BulkRollbackFile& rhs);
};
//------------------------------------------------------------------------------
// Inline functions
//------------------------------------------------------------------------------
inline int BulkRollbackFile::buildDirName( OID oid,
uint16_t dbRoot,
uint32_t partition,
std::string& dirName)
inline int BulkRollbackFile::buildDirName(OID oid, uint16_t dbRoot, uint32_t partition, std::string& dirName)
{
return fDbFile.getDirName( oid, dbRoot, partition, dirName );
return fDbFile.getDirName(oid, dbRoot, partition, dirName);
}
} //end of namespace
} // namespace WriteEngine

1512
writeengine/shared/we_bulkrollbackfilecompressed.cpp
View File

File diff suppressed because it is too large Load Diff

171
writeengine/shared/we_bulkrollbackfilecompressed.h
View File

@ -16,8 +16,8 @@
MA 02110-1301, USA. */
/*
* $Id: we_bulkrollbackfilecompressed.h 4726 2013-08-07 03:38:36Z bwilkinson $
*/
* $Id: we_bulkrollbackfilecompressed.h 4726 2013-08-07 03:38:36Z bwilkinson $
*/
/** @file
* Contains class to restore compressed db files on behalf of BulkRollBackMgr.
@ -45,111 +45,82 @@ class BulkRollbackMgr;
//------------------------------------------------------------------------------
class BulkRollbackFileCompressed : public BulkRollbackFile
{
public:
public:
/** @brief BulkRollbackFile constructor
* @param mgr The controlling BulkRollbackMgr object.
*/
BulkRollbackFileCompressed(BulkRollbackMgr* mgr);
/** @brief BulkRollbackFile constructor
* @param mgr The controlling BulkRollbackMgr object.
*/
BulkRollbackFileCompressed(BulkRollbackMgr* mgr);
/** @brief BulkRollbackFile destructor
*/
virtual ~BulkRollbackFileCompressed();
/** @brief BulkRollbackFile destructor
*/
virtual ~BulkRollbackFileCompressed();
/** @brief Do we reinit trailing blocks in the HWM extent for the specified
* segment file
*
* @param columnOID OID of the segment file in question
* @param dbRoot DBRoot for the segment file in question
* @param partNum Partition number for the segment file in question
* @param segNum Segment number for the segment file in question
*/
virtual bool doWeReInitExtent(OID columnOID, uint32_t dbRoot, uint32_t partNum, uint32_t segNum) const;
/** @brief Do we reinit trailing blocks in the HWM extent for the specified
* segment file
*
* @param columnOID OID of the segment file in question
* @param dbRoot DBRoot for the segment file in question
* @param partNum Partition number for the segment file in question
* @param segNum Segment number for the segment file in question
*/
virtual bool doWeReInitExtent( OID columnOID,
uint32_t dbRoot,
uint32_t partNum,
uint32_t segNum) const;
/** @brief Reinitialize the specified column segment file starting at
* startOffsetBlk, and truncate trailing extents.
* @param columnOID OID of the relevant segment file
* @param dbRoot DBRoot of the relevant segment file
* @param partNum Partition number of the relevant segment file
* @param segNum Segment number of the relevant segment file
* @param startOffsetBlk Starting block offset where file is to be
* reinitialized
* @param nBlocks Number of blocks to be reinitialized
* @param colType Column type of the relevant segment file
* @param colWidth Width in bytes of column.
* @param restoreHwmChk Restore HWM chunk
*/
virtual void reInitTruncColumnExtent(OID columnOID, uint32_t dbRoot, uint32_t partNum, uint32_t segNum,
long long startOffsetBlk, int nBlocks,
execplan::CalpontSystemCatalog::ColDataType colType, uint32_t colWidth,
bool restoreHwmChk);
/** @brief Reinitialize the specified column segment file starting at
* startOffsetBlk, and truncate trailing extents.
* @param columnOID OID of the relevant segment file
* @param dbRoot DBRoot of the relevant segment file
* @param partNum Partition number of the relevant segment file
* @param segNum Segment number of the relevant segment file
* @param startOffsetBlk Starting block offset where file is to be
* reinitialized
* @param nBlocks Number of blocks to be reinitialized
* @param colType Column type of the relevant segment file
* @param colWidth Width in bytes of column.
* @param restoreHwmChk Restore HWM chunk
*/
virtual void reInitTruncColumnExtent(OID columnOID,
uint32_t dbRoot,
uint32_t partNum,
uint32_t segNum,
long long startOffsetBlk,
int nBlocks,
execplan::CalpontSystemCatalog::ColDataType colType,
uint32_t colWidth,
bool restoreHwmChk );
/** @brief Reinitialize the specified dictionary store segment file starting
* at startOffsetBlk, and truncate trailing extents.
* @param columnOID OID of the relevant segment file
* @param dbRoot DBRoot of the relevant segment file
* @param partNum Partition number of the relevant segment file
* @param segNum Segment number of the relevant segment file
* @param startOffsetBlk Starting block offset where file is to be
* reinitialized
* @param nBlocks Number of blocks to be reinitialized
*/
virtual void reInitTruncDctnryExtent(OID columnOID, uint32_t dbRoot, uint32_t partNum, uint32_t segNum,
long long startOffsetBlk, int nBlocks);
/** @brief Reinitialize the specified dictionary store segment file starting
* at startOffsetBlk, and truncate trailing extents.
* @param columnOID OID of the relevant segment file
* @param dbRoot DBRoot of the relevant segment file
* @param partNum Partition number of the relevant segment file
* @param segNum Segment number of the relevant segment file
* @param startOffsetBlk Starting block offset where file is to be
* reinitialized
* @param nBlocks Number of blocks to be reinitialized
*/
virtual void reInitTruncDctnryExtent(OID columnOID,
uint32_t dbRoot,
uint32_t partNum,
uint32_t segNum,
long long startOffsetBlk,
int nBlocks );
/** @brief Truncate the specified segment file to a specified num of bytes
* @param columnOID OID of the relevant segment file
* @param dbRoot DBRoot of the relevant segment file
* @param partNum Partition number of the relevant segment file
* @param segNum Segment number of the relevant segment file
* @param fileSizeBlocks Number of blocks to retain in the file
*/
virtual void truncateSegmentFile(OID columnOID, uint32_t dbRoot, uint32_t partNum, uint32_t segNum,
long long filesSizeBlocks);
/** @brief Truncate the specified segment file to a specified num of bytes
* @param columnOID OID of the relevant segment file
* @param dbRoot DBRoot of the relevant segment file
* @param partNum Partition number of the relevant segment file
* @param segNum Segment number of the relevant segment file
* @param fileSizeBlocks Number of blocks to retain in the file
*/
virtual void truncateSegmentFile( OID columnOID,
uint32_t dbRoot,
uint32_t partNum,
uint32_t segNum,
long long filesSizeBlocks );
private:
// Disable unnecessary copy constructor and assignment operator
BulkRollbackFileCompressed(const BulkRollbackFileCompressed& rhs);
BulkRollbackFileCompressed& operator=(const BulkRollbackFileCompressed& rhs);
private:
// Disable unnecessary copy constructor and assignment operator
BulkRollbackFileCompressed(const BulkRollbackFileCompressed& rhs);
BulkRollbackFileCompressed& operator=(const BulkRollbackFileCompressed& rhs);
size_t readFillBuffer(IDBDataFile* pFile, char* buffer, size_t bytesReq) const;
int restoreHWMChunk(IDBDataFile* pFile, OID columnOID, uint32_t partNum, uint32_t segNum,
uint64_t fileOffsetByteForRestoredChunk, uint64_t& restoredChunkLen,
uint64_t& restoredFileSize, std::string& errMsg);
int loadColumnHdrPtrs(IDBDataFile* pFile, char* hdrs, compress::CompChunkPtrList& chunkPtrs,
std::string& errMsg) const;
int loadDctnryHdrPtrs(IDBDataFile* pFile, char* controlHdr, compress::CompChunkPtrList& chunkPtrs,
uint64_t& ptrHdrSize, std::string& errMsg) const;
size_t readFillBuffer ( IDBDataFile* pFile,
char* buffer,
size_t bytesReq) const;
int restoreHWMChunk ( IDBDataFile* pFile,
OID columnOID,
uint32_t partNum,
uint32_t segNum,
uint64_t fileOffsetByteForRestoredChunk,
uint64_t& restoredChunkLen,
uint64_t& restoredFileSize,
std::string& errMsg );
int loadColumnHdrPtrs ( IDBDataFile* pFile,
char* hdrs,
compress::CompChunkPtrList& chunkPtrs,
std::string& errMsg) const;
int loadDctnryHdrPtrs ( IDBDataFile* pFile,
char* controlHdr,
compress::CompChunkPtrList& chunkPtrs,
uint64_t& ptrHdrSize,
std::string& errMsg ) const;
compress::CompressorPool fCompressorPool;
compress::CompressorPool fCompressorPool;
};
} //end of namespace
} // namespace WriteEngine

255
writeengine/shared/we_bulkrollbackfilecompressedhdfs.cpp
View File

@ -38,17 +38,14 @@ namespace
{
const char* DATA_DIR_SUFFIX = "_data";
const char* OLD_FILE_SUFFIX = ".old_bulk";
}
} // namespace
namespace WriteEngine
{
//------------------------------------------------------------------------------
// BulkRollbackFileCompressedHdfs constructor
//------------------------------------------------------------------------------
BulkRollbackFileCompressedHdfs::BulkRollbackFileCompressedHdfs(
BulkRollbackMgr* mgr) :
BulkRollbackFile(mgr)
BulkRollbackFileCompressedHdfs::BulkRollbackFileCompressedHdfs(BulkRollbackMgr* mgr) : BulkRollbackFile(mgr)
{
}
@ -58,7 +55,7 @@ BulkRollbackFileCompressedHdfs::BulkRollbackFileCompressedHdfs(
BulkRollbackFileCompressedHdfs::~BulkRollbackFileCompressedHdfs()
{
}
//------------------------------------------------------------------------------
// Truncate the specified database segment file to the extent specified by
// the given file offset. Also updates the header(s) as well.
@ -70,25 +67,18 @@ BulkRollbackFileCompressedHdfs::~BulkRollbackFileCompressedHdfs()
// fileSizeBlocks - Number of raw data blocks to be left in the file.
// Remainder of file is to be truncated.
//------------------------------------------------------------------------------
void BulkRollbackFileCompressedHdfs::truncateSegmentFile(
OID columnOID,
uint32_t dbRoot,
uint32_t partNum,
uint32_t segNum,
long long fileSizeBlocks )
void BulkRollbackFileCompressedHdfs::truncateSegmentFile(OID columnOID, uint32_t dbRoot, uint32_t partNum,
uint32_t segNum, long long fileSizeBlocks)
{
std::ostringstream msgText;
msgText << "Truncating compressed HDFS column file"
": dbRoot-" << dbRoot <<
"; part#-" << partNum <<
"; seg#-" << segNum <<
"; rawTotBlks-" << fileSizeBlocks;
fMgr->logAMessage( logging::LOG_TYPE_INFO,
logging::M0075, columnOID, msgText.str() );
std::ostringstream msgText;
msgText << "Truncating compressed HDFS column file"
": dbRoot-"
<< dbRoot << "; part#-" << partNum << "; seg#-" << segNum << "; rawTotBlks-" << fileSizeBlocks;
fMgr->logAMessage(logging::LOG_TYPE_INFO, logging::M0075, columnOID, msgText.str());
restoreFromBackup( "column", columnOID, dbRoot, partNum, segNum );
restoreFromBackup("column", columnOID, dbRoot, partNum, segNum);
}
//------------------------------------------------------------------------------
// Reinitialize a column segment extent (in the db file) to empty values,
// following the HWM. Remaining extents in the file are truncated.
@ -105,32 +95,23 @@ void BulkRollbackFileCompressedHdfs::truncateSegmentFile(
// colWidth - Width in bytes, of the applicable column
// restoreHwmChk - Specifies whether HWM chunk is to be restored.
//------------------------------------------------------------------------------
void BulkRollbackFileCompressedHdfs::reInitTruncColumnExtent(
OID columnOID,
uint32_t dbRoot,
uint32_t partNum,
uint32_t segNum,
long long startOffsetBlk,
int nBlocks,
CalpontSystemCatalog::ColDataType colType,
uint32_t colWidth,
bool restoreHwmChk )
void BulkRollbackFileCompressedHdfs::reInitTruncColumnExtent(OID columnOID, uint32_t dbRoot, uint32_t partNum,
uint32_t segNum, long long startOffsetBlk,
int nBlocks,
CalpontSystemCatalog::ColDataType colType,
uint32_t colWidth, bool restoreHwmChk)
{
long long startOffset = startOffsetBlk * BYTE_PER_BLOCK;
long long startOffset = startOffsetBlk * BYTE_PER_BLOCK;
std::ostringstream msgText;
msgText << "Reinit HWM compressed column extent in HDFS db file" <<
": dbRoot-" << dbRoot <<
"; part#-" << partNum <<
"; seg#-" << segNum <<
"; rawOffset(bytes)-" << startOffset <<
"; rawFreeBlks-" << nBlocks;
fMgr->logAMessage( logging::LOG_TYPE_INFO,
logging::M0075, columnOID, msgText.str() );
std::ostringstream msgText;
msgText << "Reinit HWM compressed column extent in HDFS db file"
<< ": dbRoot-" << dbRoot << "; part#-" << partNum << "; seg#-" << segNum << "; rawOffset(bytes)-"
<< startOffset << "; rawFreeBlks-" << nBlocks;
fMgr->logAMessage(logging::LOG_TYPE_INFO, logging::M0075, columnOID, msgText.str());
restoreFromBackup( "column", columnOID, dbRoot, partNum, segNum );
restoreFromBackup("column", columnOID, dbRoot, partNum, segNum);
}
//------------------------------------------------------------------------------
// Reinitialize a dictionary segment extent (in the db file) to empty blocks,
// following the HWM. Remaining extents in the file are truncated.
@ -144,42 +125,33 @@ void BulkRollbackFileCompressedHdfs::reInitTruncColumnExtent(
// to be reinitialized. Value is in raw data blocks.
// nBlocks - Number of blocks to be reinitialized
//------------------------------------------------------------------------------
void BulkRollbackFileCompressedHdfs::reInitTruncDctnryExtent(
OID dStoreOID,
uint32_t dbRoot,
uint32_t partNum,
uint32_t segNum,
long long startOffsetBlk,
int nBlocks )
void BulkRollbackFileCompressedHdfs::reInitTruncDctnryExtent(OID dStoreOID, uint32_t dbRoot, uint32_t partNum,
uint32_t segNum, long long startOffsetBlk,
int nBlocks)
{
long long startOffset = startOffsetBlk * BYTE_PER_BLOCK;
long long startOffset = startOffsetBlk * BYTE_PER_BLOCK;
std::ostringstream msgText;
msgText << "Reinit HWM compressed dictionary store extent in HDFS db file"
": dbRoot-" << dbRoot <<
"; part#-" << partNum <<
"; seg#-" << segNum <<
"; rawOffset(bytes)-" << startOffset <<
"; rawFreeBlks-" << nBlocks;
fMgr->logAMessage( logging::LOG_TYPE_INFO,
logging::M0075, dStoreOID, msgText.str() );
std::ostringstream msgText;
msgText << "Reinit HWM compressed dictionary store extent in HDFS db file"
": dbRoot-"
<< dbRoot << "; part#-" << partNum << "; seg#-" << segNum << "; rawOffset(bytes)-" << startOffset
<< "; rawFreeBlks-" << nBlocks;
fMgr->logAMessage(logging::LOG_TYPE_INFO, logging::M0075, dStoreOID, msgText.str());
restoreFromBackup( "dictionary store", dStoreOID, dbRoot, partNum, segNum );
restoreFromBackup("dictionary store", dStoreOID, dbRoot, partNum, segNum);
}
//------------------------------------------------------------------------------
// For HDFS system, just always return true.
// Let ConfirmHdfsDbFile later determine when/if/how to restore from any
// existing backup file.
//------------------------------------------------------------------------------
bool BulkRollbackFileCompressedHdfs::doWeReInitExtent( OID columnOID,
uint32_t dbRoot,
uint32_t partNum,
uint32_t segNum) const
bool BulkRollbackFileCompressedHdfs::doWeReInitExtent(OID columnOID, uint32_t dbRoot, uint32_t partNum,
uint32_t segNum) const
{
return true;
return true;
}
//------------------------------------------------------------------------------
// Replace the currently specified db file with it's corresponding backup file.
// The backup file is a complete backup, not just a backup of a single chunk.
@ -193,96 +165,79 @@ bool BulkRollbackFileCompressedHdfs::doWeReInitExtent( OID columnOID,
// in case the user did not upgrade cleanly, and we have to restore using an
// old leftover backup file.
//------------------------------------------------------------------------------
void BulkRollbackFileCompressedHdfs::restoreFromBackup(const char* colType,
OID columnOID,
uint32_t dbRoot,
uint32_t partNum,
uint32_t segNum)
void BulkRollbackFileCompressedHdfs::restoreFromBackup(const char* colType, OID columnOID, uint32_t dbRoot,
uint32_t partNum, uint32_t segNum)
{
// Construct file name for db file to be restored
char dbFileName[FILE_NAME_SIZE];
int rc = fDbFile.getFileName( columnOID, dbFileName,
dbRoot, partNum, segNum );
// Construct file name for db file to be restored
char dbFileName[FILE_NAME_SIZE];
int rc = fDbFile.getFileName(columnOID, dbFileName, dbRoot, partNum, segNum);
if (rc != NO_ERROR)
if (rc != NO_ERROR)
{
std::ostringstream oss;
oss << "Error restoring " << colType << " HDFS file for OID " << columnOID
<< "; Can't construct file name for DBRoot" << dbRoot << "; partition-" << partNum << "; segment-"
<< segNum;
throw WeException(oss.str(), rc);
}
// Construct file name for backup copy of db file
std::ostringstream ossFile;
ossFile << "/" << columnOID << ".p" << partNum << ".s" << segNum;
std::string backupFileName(fMgr->getMetaFileName());
backupFileName += DATA_DIR_SUFFIX;
backupFileName += ossFile.str();
std::string dbFileNameTmp = dbFileName;
dbFileNameTmp += OLD_FILE_SUFFIX;
// For backwards compatibility...
// Restore from backup file used in initial HDFS release, in case the user
// upgraded without going down cleanly. In that case we might need to
// rollback using an old backup file left from previous release.
if (IDBPolicy::exists(backupFileName.c_str()))
{
// Rename current db file to make room for restored file
rc = IDBPolicy::rename(dbFileName, dbFileNameTmp.c_str());
if (rc != 0)
{
std::ostringstream oss;
oss << "Error restoring " << colType <<
" HDFS file for OID " << columnOID <<
"; Can't construct file name for DBRoot" << dbRoot <<
"; partition-" << partNum <<
"; segment-" << segNum;
throw WeException( oss.str(), rc );
std::ostringstream oss;
oss << "Error restoring " << colType << " HDFS file for OID " << columnOID
<< "; Can't move old file for DBRoot" << dbRoot << "; partition-" << partNum << "; segment-"
<< segNum;
throw WeException(oss.str(), ERR_COMP_RENAME_FILE);
}
// Construct file name for backup copy of db file
std::ostringstream ossFile;
ossFile << "/" << columnOID << ".p" << partNum << ".s" << segNum;
std::string backupFileName( fMgr->getMetaFileName() );
backupFileName += DATA_DIR_SUFFIX;
backupFileName += ossFile.str();
// Rename backup file to replace current db file
rc = IDBPolicy::rename(backupFileName.c_str(), dbFileName);
std::string dbFileNameTmp = dbFileName;
dbFileNameTmp += OLD_FILE_SUFFIX;
// For backwards compatibility...
// Restore from backup file used in initial HDFS release, in case the user
// upgraded without going down cleanly. In that case we might need to
// rollback using an old backup file left from previous release.
if ( IDBPolicy::exists(backupFileName.c_str()) )
if (rc != 0)
{
// Rename current db file to make room for restored file
rc = IDBPolicy::rename( dbFileName, dbFileNameTmp.c_str() );
if (rc != 0)
{
std::ostringstream oss;
oss << "Error restoring " << colType <<
" HDFS file for OID " << columnOID <<
"; Can't move old file for DBRoot" << dbRoot <<
"; partition-" << partNum <<
"; segment-" << segNum;
throw WeException( oss.str(), ERR_COMP_RENAME_FILE );
}
// Rename backup file to replace current db file
rc = IDBPolicy::rename( backupFileName.c_str(), dbFileName );
if (rc != 0)
{
std::ostringstream oss;
oss << "Error restoring " << colType <<
" HDFS file for OID " << columnOID <<
"; Can't rename backup file for DBRoot" << dbRoot <<
"; partition-" << partNum <<
"; segment-" << segNum;
throw WeException( oss.str(), ERR_METADATABKUP_COMP_RENAME );
}
// Delete db file we just replaced with backup
IDBPolicy::remove( dbFileNameTmp.c_str() );
std::ostringstream oss;
oss << "Error restoring " << colType << " HDFS file for OID " << columnOID
<< "; Can't rename backup file for DBRoot" << dbRoot << "; partition-" << partNum << "; segment-"
<< segNum;
throw WeException(oss.str(), ERR_METADATABKUP_COMP_RENAME);
}
else // Restore from HDFS temp swap backup file; This is the normal case
// Delete db file we just replaced with backup
IDBPolicy::remove(dbFileNameTmp.c_str());
}
else // Restore from HDFS temp swap backup file; This is the normal case
{
std::string errMsg;
ConfirmHdfsDbFile confirmHdfs;
rc = confirmHdfs.endDbFileChange(std::string("tmp"), dbFileName, false, errMsg);
if (rc != 0)
{
std::string errMsg;
ConfirmHdfsDbFile confirmHdfs;
rc = confirmHdfs.endDbFileChange( std::string("tmp"),
dbFileName,
false,
errMsg);
if (rc != 0)
{
std::ostringstream oss;
oss << "Error restoring " << colType <<
" HDFS file for OID " << columnOID <<
"; DBRoot" << dbRoot <<
"; partition-" << partNum <<
"; segment-" << segNum <<
"; " << errMsg;
throw WeException( oss.str(), rc );
}
std::ostringstream oss;
oss << "Error restoring " << colType << " HDFS file for OID " << columnOID << "; DBRoot" << dbRoot
<< "; partition-" << partNum << "; segment-" << segNum << "; " << errMsg;
throw WeException(oss.str(), rc);
}
}
}
} //end of namespace
} // namespace WriteEngine

145
writeengine/shared/we_bulkrollbackfilecompressedhdfs.h
View File

@ -43,95 +43,74 @@ class BulkRollbackMgr;
//------------------------------------------------------------------------------
class BulkRollbackFileCompressedHdfs : public BulkRollbackFile
{
public:
public:
/** @brief BulkRollbackFile constructor
* @param mgr The controlling BulkRollbackMgr object.
*/
BulkRollbackFileCompressedHdfs(BulkRollbackMgr* mgr);
/** @brief BulkRollbackFile constructor
* @param mgr The controlling BulkRollbackMgr object.
*/
BulkRollbackFileCompressedHdfs(BulkRollbackMgr* mgr);
/** @brief BulkRollbackFile destructor
*/
virtual ~BulkRollbackFileCompressedHdfs();
/** @brief BulkRollbackFile destructor
*/
virtual ~BulkRollbackFileCompressedHdfs();
/** @brief Do we reinit trailing blocks in the HWM extent for the specified
* segment file
*
* @param columnOID OID of the segment file in question
* @param dbRoot DBRoot for the segment file in question
* @param partNum Partition number for the segment file in question
* @param segNum Segment number for the segment file in question
*/
virtual bool doWeReInitExtent(OID columnOID, uint32_t dbRoot, uint32_t partNum, uint32_t segNum) const;
/** @brief Do we reinit trailing blocks in the HWM extent for the specified
* segment file
*
* @param columnOID OID of the segment file in question
* @param dbRoot DBRoot for the segment file in question
* @param partNum Partition number for the segment file in question
* @param segNum Segment number for the segment file in question
*/
virtual bool doWeReInitExtent(OID columnOID,
uint32_t dbRoot,
uint32_t partNum,
uint32_t segNum) const;
/** @brief Reinitialize the specified column segment file starting at
* startOffsetBlk, and truncate trailing extents.
* @param columnOID OID of the relevant segment file
* @param dbRoot DBRoot of the relevant segment file
* @param partNum Partition number of the relevant segment file
* @param segNum Segment number of the relevant segment file
* @param startOffsetBlk Starting block offset where file is to be
* reinitialized
* @param nBlocks Number of blocks to be reinitialized
* @param colType Column type of the relevant segment file
* @param colWidth Width in bytes of column.
* @param restoreHwmChk Restore HWM chunk
*/
virtual void reInitTruncColumnExtent(OID columnOID, uint32_t dbRoot, uint32_t partNum, uint32_t segNum,
long long startOffsetBlk, int nBlocks,
execplan::CalpontSystemCatalog::ColDataType colType, uint32_t colWidth,
bool restoreHwmChk);
/** @brief Reinitialize the specified column segment file starting at
* startOffsetBlk, and truncate trailing extents.
* @param columnOID OID of the relevant segment file
* @param dbRoot DBRoot of the relevant segment file
* @param partNum Partition number of the relevant segment file
* @param segNum Segment number of the relevant segment file
* @param startOffsetBlk Starting block offset where file is to be
* reinitialized
* @param nBlocks Number of blocks to be reinitialized
* @param colType Column type of the relevant segment file
* @param colWidth Width in bytes of column.
* @param restoreHwmChk Restore HWM chunk
*/
virtual void reInitTruncColumnExtent(OID columnOID,
uint32_t dbRoot,
uint32_t partNum,
uint32_t segNum,
long long startOffsetBlk,
int nBlocks,
execplan::CalpontSystemCatalog::ColDataType colType,
uint32_t colWidth,
bool restoreHwmChk );
/** @brief Reinitialize the specified dictionary store segment file starting
* at startOffsetBlk, and truncate trailing extents.
* @param columnOID OID of the relevant segment file
* @param dbRoot DBRoot of the relevant segment file
* @param partNum Partition number of the relevant segment file
* @param segNum Segment number of the relevant segment file
* @param startOffsetBlk Starting block offset where file is to be
* reinitialized
* @param nBlocks Number of blocks to be reinitialized
*/
virtual void reInitTruncDctnryExtent(OID columnOID, uint32_t dbRoot, uint32_t partNum, uint32_t segNum,
long long startOffsetBlk, int nBlocks);
/** @brief Reinitialize the specified dictionary store segment file starting
* at startOffsetBlk, and truncate trailing extents.
* @param columnOID OID of the relevant segment file
* @param dbRoot DBRoot of the relevant segment file
* @param partNum Partition number of the relevant segment file
* @param segNum Segment number of the relevant segment file
* @param startOffsetBlk Starting block offset where file is to be
* reinitialized
* @param nBlocks Number of blocks to be reinitialized
*/
virtual void reInitTruncDctnryExtent(OID columnOID,
uint32_t dbRoot,
uint32_t partNum,
uint32_t segNum,
long long startOffsetBlk,
int nBlocks );
/** @brief Truncate the specified segment file to a specified num of bytes
* @param columnOID OID of the relevant segment file
* @param dbRoot DBRoot of the relevant segment file
* @param partNum Partition number of the relevant segment file
* @param segNum Segment number of the relevant segment file
* @param fileSizeBlocks Number of blocks to retain in the file
*/
virtual void truncateSegmentFile(OID columnOID, uint32_t dbRoot, uint32_t partNum, uint32_t segNum,
long long filesSizeBlocks);
/** @brief Truncate the specified segment file to a specified num of bytes
* @param columnOID OID of the relevant segment file
* @param dbRoot DBRoot of the relevant segment file
* @param partNum Partition number of the relevant segment file
* @param segNum Segment number of the relevant segment file
* @param fileSizeBlocks Number of blocks to retain in the file
*/
virtual void truncateSegmentFile(OID columnOID,
uint32_t dbRoot,
uint32_t partNum,
uint32_t segNum,
long long filesSizeBlocks );
private:
// Disable unnecessary copy constructor and assignment operator
BulkRollbackFileCompressedHdfs(const BulkRollbackFileCompressedHdfs& rhs);
BulkRollbackFileCompressedHdfs& operator=(const BulkRollbackFileCompressedHdfs& rhs);
private:
// Disable unnecessary copy constructor and assignment operator
BulkRollbackFileCompressedHdfs(const BulkRollbackFileCompressedHdfs& rhs);
BulkRollbackFileCompressedHdfs& operator=(
const BulkRollbackFileCompressedHdfs& rhs);
void restoreFromBackup(const char* colType,
OID columnOID,
uint32_t dbRoot,
uint32_t partNum,
uint32_t segNum );
void restoreFromBackup(const char* colType, OID columnOID, uint32_t dbRoot, uint32_t partNum,
uint32_t segNum);
};
} //end of namespace
} // namespace WriteEngine

2863
writeengine/shared/we_bulkrollbackmgr.cpp
View File

File diff suppressed because it is too large Load Diff

300
writeengine/shared/we_bulkrollbackmgr.h
View File

@ -16,8 +16,8 @@
MA 02110-1301, USA. */
/*
* $Id: we_bulkrollbackmgr.h 4726 2013-08-07 03:38:36Z bwilkinson $
*/
* $Id: we_bulkrollbackmgr.h 4726 2013-08-07 03:38:36Z bwilkinson $
*/
/** @file
* Contains class to clear a database table lock, and rolls back extents
@ -59,177 +59,165 @@ class BulkRollbackFile;
//------------------------------------------------------------------------------
class BulkRollbackMgr
{
enum BulkRollbackVersion
{
BULK_RB_VERSION_OTHER,
BULK_RB_VERSION3 = 3,
BULK_RB_VERSION4 = 4
};
enum BulkRollbackVersion
{
BULK_RB_VERSION_OTHER,
BULK_RB_VERSION3 = 3,
BULK_RB_VERSION4 = 4
};
public:
/**
* @brief BulkRollbackMgr constructor
* @param tableOID table to be rolled back.
* @param lockID Table lock id of the table to be rolled back.
* Currently used for logging only.
* @param tableName name of table associated with tableOID.
* Currently used for logging only.
* @param applName application that is driving this bulk rollback.
* Currently used for logging only.
*/
EXPORT BulkRollbackMgr(OID tableOID,
uint64_t lockID,
const std::string& tableName,
const std::string& applName,
Log* logger = 0);
public:
/**
* @brief BulkRollbackMgr constructor
* @param tableOID table to be rolled back.
* @param lockID Table lock id of the table to be rolled back.
* Currently used for logging only.
* @param tableName name of table associated with tableOID.
* Currently used for logging only.
* @param applName application that is driving this bulk rollback.
* Currently used for logging only.
*/
EXPORT BulkRollbackMgr(OID tableOID, uint64_t lockID, const std::string& tableName,
const std::string& applName, Log* logger = 0);
/**
* @brief BulkRollbackMgr destructor
*/
EXPORT ~BulkRollbackMgr( )
{
closeMetaDataFile ( );
}
/**
* @brief BulkRollbackMgr destructor
*/
EXPORT ~BulkRollbackMgr()
{
closeMetaDataFile();
}
/**
* @brief Clear table lock and rollback extents for fTableOID
* @param keepMetaFile controls whether the meta data file is deleted
* @return NO_ERROR upon success
*/
EXPORT int rollback ( bool keepMetaFile );
/**
* @brief Clear table lock and rollback extents for fTableOID
* @param keepMetaFile controls whether the meta data file is deleted
* @return NO_ERROR upon success
*/
EXPORT int rollback(bool keepMetaFile);
/**
* @brief Accessor to any error msg related to a bad return code.
* @return error message if rollback rejected or failed.
*/
const std::string& getErrorMsg( ) const
{
return fErrorMsg;
}
/**
* @brief Accessor to any error msg related to a bad return code.
* @return error message if rollback rejected or failed.
*/
const std::string& getErrorMsg() const
{
return fErrorMsg;
}
/**
* @brief Accessor to the name of the meta file we are processing
*/
const std::string& getMetaFileName() const
{
return fMetaFileName;
}
/**
* @brief Accessor to the name of the meta file we are processing
*/
const std::string& getMetaFileName() const
{
return fMetaFileName;
}
/**
* @brief Mutator to enable/disable debug logging to console.
*/
void setDebugConsole ( bool debug )
{
fDebugConsole = debug;
}
/**
* @brief Mutator to enable/disable debug logging to console.
*/
void setDebugConsole(bool debug)
{
fDebugConsole = debug;
}
/**
* @brief Log the specified message.
* @param logType type of message to be logged
* @param msgId message id to be used
* @param columnOID column OID
* @param text message text to be logged
*/
void logAMessage ( logging::LOG_TYPE logType, //log a message
logging::Message::MessageID msgId,
OID columnOID,
const std::string& text );
/**
* @brief Standalone utility that can be used to delete the bulk rollback
* meta data files. Caution: this function can throw an exception.
* @param tableOID Bulk rollback files for this table are to be deleted
*/
EXPORT static void deleteMetaFile( OID tableOID );
/**
* @brief Log the specified message.
* @param logType type of message to be logged
* @param msgId message id to be used
* @param columnOID column OID
* @param text message text to be logged
*/
void logAMessage(logging::LOG_TYPE logType, // log a message
logging::Message::MessageID msgId, OID columnOID, const std::string& text);
/**
* @brief Standalone utility that can be used to delete the bulk rollback
* meta data files. Caution: this function can throw an exception.
* @param tableOID Bulk rollback files for this table are to be deleted
*/
EXPORT static void deleteMetaFile(OID tableOID);
/*
* @brief Get list of segment file numbers found in dirName directory
* @param dirName Directory path to be searched
* @param bIncludeAlternateSegFileNames Include *.orig and *.tmp in search
* @param segList List of segment files found in dirName
* @param errMsg Error msg if return code is not NO_ERROR
*/
EXPORT static int getSegFileList( const std::string& dirName,
bool bIncludeAlternateSegFileNames,
std::vector<uint32_t>& segList,
std::string& errMsg );
/*
* @brief Get list of segment file numbers found in dirName directory
* @param dirName Directory path to be searched
* @param bIncludeAlternateSegFileNames Include *.orig and *.tmp in search
* @param segList List of segment files found in dirName
* @param errMsg Error msg if return code is not NO_ERROR
*/
EXPORT static int getSegFileList(const std::string& dirName, bool bIncludeAlternateSegFileNames,
std::vector<uint32_t>& segList, std::string& errMsg);
private:
// Declare but don't define copy constructor and assignment operator
BulkRollbackMgr(const BulkRollbackMgr& rhs);
BulkRollbackMgr& operator=(const BulkRollbackMgr& rhs);
private:
// Declare but don't define copy constructor and assignment operator
BulkRollbackMgr(const BulkRollbackMgr& rhs);
BulkRollbackMgr& operator=(const BulkRollbackMgr& rhs);
// Structure used to store info for the list of dictionary store
// segment files in the last partition.
struct RollbackData
{
uint32_t fDbRoot;
uint32_t fPartNum;
uint32_t fSegNum;
HWM fHwm;
bool fWithHwm;
};
// Structure used to store info for the list of dictionary store
// segment files in the last partition.
struct RollbackData
{
uint32_t fDbRoot;
uint32_t fPartNum;
uint32_t fSegNum;
HWM fHwm;
bool fWithHwm;
};
void createFileDeletionEntry( OID columnOID,
bool fileTypeFlag,
uint32_t dbRoot,
uint32_t partNum,
uint32_t segNum,
const std::string& segFileName );
void deleteColumn1Extents ( const char* inBuf ); // delete col extents
void deleteColumn1ExtentsV3(const char* inBuf );
void deleteColumn1ExtentsV4(const char* inBuf );
void deleteColumn2Extents ( const char* inBuf ); // delete col extents
void deleteColumn2ExtentsV3(const char* inBuf );
void deleteColumn2ExtentsV4(const char* inBuf );
void deleteDbFiles ( ); // delete DB files waiting to be deleted
void deleteDctnryExtents ( ); // delete dictionary store extents
void deleteDctnryExtentsV3( );
void deleteDctnryExtentsV4( );
void deleteExtents ( std::istringstream& metaDataStream );
// function that drives extent deletion
void readMetaDataRecDctnry(const char* inBuf );//read meta-data dct rec
void createFileDeletionEntry(OID columnOID, bool fileTypeFlag, uint32_t dbRoot, uint32_t partNum,
uint32_t segNum, const std::string& segFileName);
void deleteColumn1Extents(const char* inBuf); // delete col extents
void deleteColumn1ExtentsV3(const char* inBuf);
void deleteColumn1ExtentsV4(const char* inBuf);
void deleteColumn2Extents(const char* inBuf); // delete col extents
void deleteColumn2ExtentsV3(const char* inBuf);
void deleteColumn2ExtentsV4(const char* inBuf);
void deleteDbFiles(); // delete DB files waiting to be deleted
void deleteDctnryExtents(); // delete dictionary store extents
void deleteDctnryExtentsV3();
void deleteDctnryExtentsV4();
void deleteExtents(std::istringstream& metaDataStream);
// function that drives extent deletion
void readMetaDataRecDctnry(const char* inBuf); // read meta-data dct rec
void deleteSubDir ( const std::string& metaFileName ); // delete
// subdirectory used for backup chunks
EXPORT void closeMetaDataFile ( ); // close a metafile
void deleteMetaDataFiles ( ); // delete metafiles
int metaDataFileExists ( bool& exists ); // does meta-data file exists
BulkRollbackFile* makeFileRestorer(int compressionType);
bool openMetaDataFile ( uint16_t dbRoot, // open a metadata file
std::istringstream& metaDataStream );
void validateAllMetaFilesExist(const std::vector<uint16_t>& dbRoots) const;
void deleteSubDir(const std::string& metaFileName); // delete
// subdirectory used for backup chunks
EXPORT void closeMetaDataFile(); // close a metafile
void deleteMetaDataFiles(); // delete metafiles
int metaDataFileExists(bool& exists); // does meta-data file exists
BulkRollbackFile* makeFileRestorer(int compressionType);
bool openMetaDataFile(uint16_t dbRoot, // open a metadata file
std::istringstream& metaDataStream);
void validateAllMetaFilesExist(const std::vector<uint16_t>& dbRoots) const;
// Data members
OID fTableOID; // table to be rolled back
uint64_t fLockID; // unique lock ID associated with table lock
std::string fTableName; // name of table associated with fTableOID
uint32_t fProcessId; // pid associated with current table lock
std::string fProcessName; // processName associated with fProcessId
IDBDataFile* fMetaFile; // current meta data file we are reading
std::string fMetaFileName;// name of current meta data file
std::vector<std::string> fMetaFileNames; // all relevant meta data files
std::string fErrorMsg;
unsigned char fDctnryHdr[DCTNRY_HEADER_SIZE]; // empty dctnry store blk
// Data members
OID fTableOID; // table to be rolled back
uint64_t fLockID; // unique lock ID associated with table lock
std::string fTableName; // name of table associated with fTableOID
uint32_t fProcessId; // pid associated with current table lock
std::string fProcessName; // processName associated with fProcessId
IDBDataFile* fMetaFile; // current meta data file we are reading
std::string fMetaFileName; // name of current meta data file
std::vector<std::string> fMetaFileNames; // all relevant meta data files
std::string fErrorMsg;
unsigned char fDctnryHdr[DCTNRY_HEADER_SIZE]; // empty dctnry store blk
// Dictionary store extents for an OID are read in and managed as a
// group. The following data members are used to collect this info.
OID fPendingDctnryStoreOID;// Dctnry OID of pending dctnry extents
uint32_t fPendingDctnryStoreDbRoot; // DbRoot of pending dctnry extents
int fPendingDctnryStoreCompressionType; // Dctnry compression type
std::vector<RollbackData> fPendingDctnryExtents;
std::set<OID> fAllColDctOIDs; // List of all affected col and dctnry OIDS
// Dictionary store extents for an OID are read in and managed as a
// group. The following data members are used to collect this info.
OID fPendingDctnryStoreOID; // Dctnry OID of pending dctnry extents
uint32_t fPendingDctnryStoreDbRoot; // DbRoot of pending dctnry extents
int fPendingDctnryStoreCompressionType; // Dctnry compression type
std::vector<RollbackData> fPendingDctnryExtents;
std::set<OID> fAllColDctOIDs; // List of all affected col and dctnry OIDS
// List of DB Files to be deleted. Files are deleted in reverse order.
std::vector<File> fPendingFilesToDelete;
// List of DB Files to be deleted. Files are deleted in reverse order.
std::vector<File> fPendingFilesToDelete;
logging::MessageLog fSysLogger; // Used for syslogging
bool fDebugConsole; // control debug logging to console
Log* fLog; // optional logger object
std::string fApplName; // application initiating the bulk rollback
int fVersion; // version of meta data file being read
logging::MessageLog fSysLogger; // Used for syslogging
bool fDebugConsole; // control debug logging to console
Log* fLog; // optional logger object
std::string fApplName; // application initiating the bulk rollback
int fVersion; // version of meta data file being read
};
} //end of namespace
} // namespace WriteEngine
#undef EXPORT

525
writeengine/shared/we_cache.cpp
View File

@ -16,9 +16,9 @@
MA 02110-1301, USA. */
/******************************************************************************************
* $Id: we_cache.cpp 33 2006-10-30 13:45:13Z wzhou $
*
******************************************************************************************/
* $Id: we_cache.cpp 33 2006-10-30 13:45:13Z wzhou $
*
******************************************************************************************/
/** @file */
#include <we_cache.h>
@ -27,15 +27,14 @@ using namespace std;
namespace WriteEngine
{
CacheControl* Cache::m_cacheParam = NULL;
FreeBufList* Cache::m_freeList = NULL;
CacheMap* Cache::m_lruList = NULL;
CacheMap* Cache::m_writeList = NULL;
CacheControl* Cache::m_cacheParam = NULL;
FreeBufList* Cache::m_freeList = NULL;
CacheMap* Cache::m_lruList = NULL;
CacheMap* Cache::m_writeList = NULL;
#ifdef _MSC_VER
__declspec(dllexport)
#endif
bool Cache::m_useCache = false;
bool Cache::m_useCache = false;
/***********************************************************
* DESCRIPTION:
* Clear all list and free memory
@ -44,47 +43,47 @@ bool Cache::m_useCache = false;
* RETURN:
* NO_ERROR if success, other otherwise
***********************************************************/
void Cache::clear()
void Cache::clear()
{
CacheMapIt it;
BlockBuffer* block;
size_t i;
CacheMapIt it;
BlockBuffer* block;
size_t i;
// free list
if ( m_freeList != NULL )
// free list
if (m_freeList != NULL)
{
for (i = 0; i < m_freeList->size(); i++)
{
for ( i = 0; i < m_freeList->size(); i++ )
{
block = m_freeList->at(i);
block->clear();
}
block = m_freeList->at(i);
block->clear();
}
}
// LRU list
if (m_lruList != NULL)
{
for (it = m_lruList->begin(); it != m_lruList->end(); it++)
{
block = it->second;
block->clear();
m_freeList->push_back(block);
}
// LRU list
if ( m_lruList != NULL )
{
for ( it = m_lruList->begin(); it != m_lruList->end(); it++ )
{
block = it->second;
block->clear();
m_freeList->push_back( block );
}
m_lruList->clear();
}
m_lruList->clear();
// Write list
if (m_writeList != NULL)
{
for (it = m_writeList->begin(); it != m_writeList->end(); it++)
{
block = it->second;
block->clear();
m_freeList->push_back(block);
}
// Write list
if ( m_writeList != NULL )
{
for ( it = m_writeList->begin(); it != m_writeList->end(); it++ )
{
block = it->second;
block->clear();
m_freeList->push_back( block );
}
m_writeList->clear();
}
m_writeList->clear();
}
}
/***********************************************************
@ -95,50 +94,50 @@ void Cache::clear()
* RETURN:
* NO_ERROR if success, other otherwise
***********************************************************/
int Cache::flushCache()
int Cache::flushCache()
{
bool bHasReadBlock = false;
BlockBuffer* curBuf;
bool bHasReadBlock = false;
BlockBuffer* curBuf;
// add lock here
if ( m_lruList && m_lruList->size() > 0 )
// add lock here
if (m_lruList && m_lruList->size() > 0)
{
bHasReadBlock = true;
for (CacheMapIt it = m_lruList->begin(); it != m_lruList->end(); it++)
{
bHasReadBlock = true;
for ( CacheMapIt it = m_lruList->begin(); it != m_lruList->end(); it++ )
{
curBuf = it->second;
curBuf->clear();
m_freeList->push_back( curBuf );
}
m_lruList->clear();
curBuf = it->second;
curBuf->clear();
m_freeList->push_back(curBuf);
}
// must write to disk first
if ( m_writeList && m_writeList->size() > 0 )
{
if ( !bHasReadBlock )
for ( CacheMapIt it = m_writeList->begin(); it != m_writeList->end(); it++ )
{
curBuf = it->second;
curBuf->clear();
m_freeList->push_back( curBuf );
}
else
for ( CacheMapIt it = m_writeList->begin(); it != m_writeList->end(); it++ )
{
curBuf = it->second;
(*curBuf).block.dirty = false;
processCacheMap( m_lruList, curBuf, INSERT );
}
m_lruList->clear();
}
m_writeList->clear();
// must write to disk first
if (m_writeList && m_writeList->size() > 0)
{
if (!bHasReadBlock)
for (CacheMapIt it = m_writeList->begin(); it != m_writeList->end(); it++)
{
curBuf = it->second;
curBuf->clear();
m_freeList->push_back(curBuf);
}
else
for (CacheMapIt it = m_writeList->begin(); it != m_writeList->end(); it++)
{
curBuf = it->second;
(*curBuf).block.dirty = false;
processCacheMap(m_lruList, curBuf, INSERT);
}
} // end of if( m_writeList->size()
m_writeList->clear();
// add unlock here
return NO_ERROR;
} // end of if( m_writeList->size()
// add unlock here
return NO_ERROR;
}
/***********************************************************
@ -149,63 +148,63 @@ int Cache::flushCache()
* RETURN:
* NO_ERROR if success, other otherwise
***********************************************************/
void Cache::freeMemory()
void Cache::freeMemory()
{
CacheMapIt it;
BlockBuffer* block;
size_t i;
CacheMapIt it;
BlockBuffer* block;
size_t i;
// free list
if ( m_freeList != NULL )
// free list
if (m_freeList != NULL)
{
for (i = 0; i < m_freeList->size(); i++)
{
for ( i = 0; i < m_freeList->size(); i++ )
{
block = m_freeList->at(i);
block->freeMem();
delete block;
}
m_freeList->clear();
delete m_freeList;
m_freeList = NULL;
block = m_freeList->at(i);
block->freeMem();
delete block;
}
// LRU list
if ( m_lruList != NULL )
{
for ( it = m_lruList->begin(); it != m_lruList->end(); it++ )
{
block = it->second;
block->freeMem();
delete block;
}
m_freeList->clear();
delete m_freeList;
m_freeList = NULL;
}
m_lruList->clear();
delete m_lruList;
m_lruList = NULL;
// LRU list
if (m_lruList != NULL)
{
for (it = m_lruList->begin(); it != m_lruList->end(); it++)
{
block = it->second;
block->freeMem();
delete block;
}
// Write list
if ( m_writeList != NULL )
{
for ( it = m_writeList->begin(); it != m_writeList->end(); it++ )
{
block = it->second;
block->freeMem();
delete block;
}
m_lruList->clear();
delete m_lruList;
m_lruList = NULL;
}
m_writeList->clear();
delete m_writeList;
m_writeList = NULL;
// Write list
if (m_writeList != NULL)
{
for (it = m_writeList->begin(); it != m_writeList->end(); it++)
{
block = it->second;
block->freeMem();
delete block;
}
// param
if ( m_cacheParam != NULL )
{
delete m_cacheParam;
m_cacheParam = NULL;
}
m_writeList->clear();
delete m_writeList;
m_writeList = NULL;
}
// param
if (m_cacheParam != NULL)
{
delete m_cacheParam;
m_cacheParam = NULL;
}
}
/***********************************************************
@ -216,30 +215,24 @@ void Cache::freeMemory()
* RETURN:
* NO_ERROR if success, other otherwise
***********************************************************/
int Cache::getListSize( const CacheListType listType )
int Cache::getListSize(const CacheListType listType)
{
int size = 0;
if ( !m_useCache )
return size;
switch ( listType )
{
case FREE_LIST:
size = m_freeList->size();
break;
case LRU_LIST:
size = m_lruList->size();
break;
case WRITE_LIST:
default:
size = m_writeList->size();
break;
}
int size = 0;
if (!m_useCache)
return size;
switch (listType)
{
case FREE_LIST: size = m_freeList->size(); break;
case LRU_LIST: size = m_lruList->size(); break;
case WRITE_LIST:
default: size = m_writeList->size(); break;
}
return size;
}
/***********************************************************
@ -252,28 +245,28 @@ int Cache::getListSize( const CacheListType listType )
* RETURN:
* NO_ERROR if success, other otherwise
***********************************************************/
void Cache::init( const int totalBlock, const int chkPoint, const int pctFree )
void Cache::init(const int totalBlock, const int chkPoint, const int pctFree)
{
BlockBuffer* buffer;
BlockBuffer* buffer;
if ( m_cacheParam && m_freeList && m_lruList && m_writeList )
return;
if (m_cacheParam && m_freeList && m_lruList && m_writeList)
return;
m_cacheParam = new CacheControl();
m_cacheParam->totalBlock = totalBlock;
m_cacheParam->checkInterval = chkPoint;
m_cacheParam->pctFree = pctFree;
m_cacheParam = new CacheControl();
m_cacheParam->totalBlock = totalBlock;
m_cacheParam->checkInterval = chkPoint;
m_cacheParam->pctFree = pctFree;
m_freeList = new FreeBufList();
m_lruList = new CacheMap();
m_writeList = new CacheMap();
m_freeList = new FreeBufList();
m_lruList = new CacheMap();
m_writeList = new CacheMap();
for ( int i = 0; i < m_cacheParam->totalBlock; i++ )
{
buffer = new BlockBuffer();
buffer->init();
m_freeList->push_back( buffer );
}
for (int i = 0; i < m_cacheParam->totalBlock; i++)
{
buffer = new BlockBuffer();
buffer->init();
m_freeList->push_back(buffer);
}
}
/***********************************************************
@ -287,30 +280,30 @@ void Cache::init( const int totalBlock, const int chkPoint, const int pctFree )
* RETURN:
* NO_ERROR if success, other otherwise
***********************************************************/
int Cache::insertLRUList( CommBlock& cb, const uint64_t lbid, const uint64_t fbo, const unsigned char* buf )
int Cache::insertLRUList(CommBlock& cb, const uint64_t lbid, const uint64_t fbo, const unsigned char* buf)
{
BlockBuffer* buffer;
vector<BlockBuffer*>::iterator it;
BlockBuffer* buffer;
vector<BlockBuffer*>::iterator it;
if ( m_freeList->size() == 0 )
return ERR_FREE_LIST_EMPTY;
if (m_freeList->size() == 0)
return ERR_FREE_LIST_EMPTY;
// make sure flush first if necessary
it = m_freeList->begin();
buffer = *it;
memcpy( (*buffer).block.data, buf, BYTE_PER_BLOCK );
(*buffer).listType = LRU_LIST;
(*buffer).block.lbid = lbid;
(*buffer).block.fbo = fbo;
(*buffer).block.dirty = false;
(*buffer).block.hitCount = 1;
(*buffer).cb.file.oid = cb.file.oid;
(*buffer).cb.file.pFile = cb.file.pFile;
// make sure flush first if necessary
it = m_freeList->begin();
buffer = *it;
memcpy((*buffer).block.data, buf, BYTE_PER_BLOCK);
(*buffer).listType = LRU_LIST;
(*buffer).block.lbid = lbid;
(*buffer).block.fbo = fbo;
(*buffer).block.dirty = false;
(*buffer).block.hitCount = 1;
(*buffer).cb.file.oid = cb.file.oid;
(*buffer).cb.file.pFile = cb.file.pFile;
RETURN_ON_ERROR( processCacheMap( m_lruList, buffer, INSERT ) );
m_freeList->erase( it );
RETURN_ON_ERROR(processCacheMap(m_lruList, buffer, INSERT));
m_freeList->erase(it);
return NO_ERROR;
return NO_ERROR;
}
/***********************************************************
@ -322,29 +315,29 @@ int Cache::insertLRUList( CommBlock& cb, const uint64_t lbid, const uint64_t fb
* RETURN:
* NO_ERROR if success, other otherwise
***********************************************************/
int Cache::loadCacheBlock( const CacheKey& key, unsigned char* buf )
int Cache::loadCacheBlock(const CacheKey& key, unsigned char* buf)
{
BlockBuffer* buffer;
CacheMapIt iter;
BlockBuffer* buffer;
CacheMapIt iter;
iter = m_lruList->find( key );
iter = m_lruList->find(key);
if ( iter != m_lruList->end() )
buffer = iter->second;
if (iter != m_lruList->end())
buffer = iter->second;
else
{
iter = m_writeList->find(key);
if (iter != m_writeList->end())
buffer = iter->second;
else
{
iter = m_writeList->find( key );
return ERR_CACHE_KEY_NOT_EXIST;
}
if ( iter != m_writeList->end() )
buffer = iter->second;
else
return ERR_CACHE_KEY_NOT_EXIST;
}
memcpy(buf, (*buffer).block.data, BYTE_PER_BLOCK);
(*buffer).block.hitCount++;
memcpy( buf, (*buffer).block.data, BYTE_PER_BLOCK );
(*buffer).block.hitCount++;
return NO_ERROR;
return NO_ERROR;
}
/***********************************************************
@ -356,37 +349,36 @@ int Cache::loadCacheBlock( const CacheKey& key, unsigned char* buf )
* RETURN:
* NO_ERROR if success, other otherwise
***********************************************************/
int Cache::modifyCacheBlock( const CacheKey& key, const unsigned char* buf )
int Cache::modifyCacheBlock(const CacheKey& key, const unsigned char* buf)
{
BlockBuffer* buffer;
CacheMapIt iter;
BlockBuffer* buffer;
CacheMapIt iter;
iter = m_lruList->find( key );
iter = m_lruList->find(key);
if ( iter != m_lruList->end() )
{
buffer = iter->second;
(*buffer).listType = WRITE_LIST;
(*buffer).block.dirty = true;
if (iter != m_lruList->end())
{
buffer = iter->second;
(*buffer).listType = WRITE_LIST;
(*buffer).block.dirty = true;
(*m_writeList)[key] = iter->second;
m_lruList->erase( iter );
(*m_writeList)[key] = iter->second;
m_lruList->erase(iter);
}
else
{
iter = m_writeList->find(key);
}
if (iter != m_writeList->end())
buffer = iter->second;
else
{
iter = m_writeList->find( key );
return ERR_CACHE_KEY_NOT_EXIST;
}
if ( iter != m_writeList->end() )
buffer = iter->second;
else
return ERR_CACHE_KEY_NOT_EXIST;
}
memcpy((*buffer).block.data, buf, BYTE_PER_BLOCK);
(*buffer).block.hitCount++;
memcpy( (*buffer).block.data, buf, BYTE_PER_BLOCK );
(*buffer).block.hitCount++;
return NO_ERROR;
return NO_ERROR;
}
/***********************************************************
@ -397,35 +389,35 @@ int Cache::modifyCacheBlock( const CacheKey& key, const unsigned char* buf )
* RETURN:
* none
***********************************************************/
void Cache::printCacheList()
void Cache::printCacheList()
{
BlockBuffer* buffer;
int i = 0;
BlockBuffer* buffer;
int i = 0;
if ( !m_useCache )
return;
if (!m_useCache)
return;
cout << "\nFree List has " << m_freeList->size() << " elements" << endl;
cout << "LRU List has " << m_lruList->size() << " elements" << endl;
cout << "\nFree List has " << m_freeList->size() << " elements" << endl;
cout << "LRU List has " << m_lruList->size() << " elements" << endl;
for ( CacheMapIt it = m_lruList->begin(); it != m_lruList->end(); it++ )
{
buffer = it->second;
cout << "\t[" << i++ << "] key=" << it->first << " listType=" << buffer->listType
<< " oid=" << (*buffer).cb.file.oid << " fbo=" << (*buffer).block.fbo
<< " dirty=" << (*buffer).block.dirty << " hitCount=" << (*buffer).block.hitCount << endl;
}
for (CacheMapIt it = m_lruList->begin(); it != m_lruList->end(); it++)
{
buffer = it->second;
cout << "\t[" << i++ << "] key=" << it->first << " listType=" << buffer->listType
<< " oid=" << (*buffer).cb.file.oid << " fbo=" << (*buffer).block.fbo
<< " dirty=" << (*buffer).block.dirty << " hitCount=" << (*buffer).block.hitCount << endl;
}
i = 0;
cout << "Write List has " << m_writeList->size() << " elements" << endl;
i = 0;
cout << "Write List has " << m_writeList->size() << " elements" << endl;
for ( CacheMapIt it = m_writeList->begin(); it != m_writeList->end(); it++ )
{
buffer = it->second;
cout << "\t[" << i++ << "] key=" << it->first << " listType=" << buffer->listType
<< " oid=" << (*buffer).cb.file.oid << " fbo=" << (*buffer).block.fbo
<< " dirty=" << (*buffer).block.dirty << " hitCount=" << (*buffer).block.hitCount << endl;
}
for (CacheMapIt it = m_writeList->begin(); it != m_writeList->end(); it++)
{
buffer = it->second;
cout << "\t[" << i++ << "] key=" << it->first << " listType=" << buffer->listType
<< " oid=" << (*buffer).cb.file.oid << " fbo=" << (*buffer).block.fbo
<< " dirty=" << (*buffer).block.dirty << " hitCount=" << (*buffer).block.hitCount << endl;
}
}
/***********************************************************
@ -437,34 +429,31 @@ void Cache::printCacheList()
* RETURN:
* NO_ERROR if success, other otherwise
***********************************************************/
int Cache::processCacheMap( CacheMap* map, BlockBuffer* buffer, OpType opType )
int Cache::processCacheMap(CacheMap* map, BlockBuffer* buffer, OpType opType)
{
RETURN_ON_NULL( buffer, ERR_NULL_BLOCK );
CacheMapIt iter;
RETURN_ON_NULL(buffer, ERR_NULL_BLOCK);
CacheMapIt iter;
CacheKey key = getCacheKey( buffer );
iter = map->find( key );
CacheKey key = getCacheKey(buffer);
iter = map->find(key);
// only handle insert and delete
if ( iter == map->end() )
{
if ( opType == INSERT )
(*map)[key] = buffer;
else
return ERR_CACHE_KEY_NOT_EXIST;
}
// only handle insert and delete
if (iter == map->end())
{
if (opType == INSERT)
(*map)[key] = buffer;
else
{
if ( opType == INSERT )
return ERR_CACHE_KEY_EXIST;
else
map->erase( iter );
}
return ERR_CACHE_KEY_NOT_EXIST;
}
else
{
if (opType == INSERT)
return ERR_CACHE_KEY_EXIST;
else
map->erase(iter);
}
return NO_ERROR;
return NO_ERROR;
}
} //end of namespace
} // namespace WriteEngine

291
writeengine/shared/we_cache.h
View File

@ -16,9 +16,9 @@
MA 02110-1301, USA. */
/******************************************************************************************
* $Id: we_cache.h 33 2006-10-30 13:45:13Z wzhou $
*
******************************************************************************************/
* $Id: we_cache.h 33 2006-10-30 13:45:13Z wzhou $
*
******************************************************************************************/
/** @file */
#pragma once
@ -45,184 +45,191 @@
/** Namespace WriteEngine */
namespace WriteEngine
{
typedef std::vector<BlockBuffer*> FreeBufList; /** @brief Free buffer list */
//typedef std::string CacheKey; /** @brief Key definition */
typedef uint64_t CacheKey; /** @brief Key definition */
typedef std::vector<BlockBuffer*> FreeBufList; /** @brief Free buffer list */
// typedef std::string CacheKey; /** @brief Key definition */
typedef uint64_t CacheKey; /** @brief Key definition */
//typedef std::map<CacheKey, BlockBuffer*, std::greater<CacheKey> > CacheMap; /** @brief Cache map */
//typedef CacheMap::iterator CacheMapIt; /** @brief CacheMap iterator */
// typedef std::map<CacheKey, BlockBuffer*, std::greater<CacheKey> > CacheMap; /** @brief Cache map */
// typedef CacheMap::iterator CacheMapIt; /** @brief CacheMap iterator */
template<class T>struct hashCacheKey { };
template<> struct hashCacheKey<CacheKey>
template <class T>
struct hashCacheKey
{
size_t
operator()(CacheKey __x) const
{
return __x;
}
};
template <>
struct hashCacheKey<CacheKey>
{
size_t operator()(CacheKey __x) const
{
return __x;
}
};
struct eqCacheKey
{
bool operator() (const CacheKey k1, const CacheKey k2 ) const
{
return k1 == k2;
}
bool operator()(const CacheKey k1, const CacheKey k2) const
{
return k1 == k2;
}
};
//typedef hash_map<Signature, TokenStruc, signatureHash<Signature>, eqSig> DCTNRYHASHMAP;
// typedef hash_map<Signature, TokenStruc, signatureHash<Signature>, eqSig> DCTNRYHASHMAP;
#if __GNUC__ == 4 && __GNUC_MINOR__ < 2
typedef __gnu_cxx::hash_map<CacheKey, BlockBuffer*, hashCacheKey<CacheKey>, eqCacheKey> CacheMap;
#else
typedef std::tr1::unordered_map<CacheKey, BlockBuffer*, hashCacheKey<CacheKey>, eqCacheKey> CacheMap;
#endif
//typedef __gnu_cxx::hash_map<CacheKey, BlockBuffer*> CacheMap;
typedef CacheMap::iterator CacheMapIt;
// typedef __gnu_cxx::hash_map<CacheKey, BlockBuffer*> CacheMap;
typedef CacheMap::iterator CacheMapIt;
//typedef CacheMap LRUBufList; /** @brief Least Recent Used Buffer list */
//typedef CacheMap WriteBufList; /** @brief Write buffer list */
// typedef CacheMap LRUBufList; /** @brief Least Recent Used Buffer list */
// typedef CacheMap WriteBufList; /** @brief Write buffer list */
/** Class Cache */
class Cache
{
public:
/**
* @brief Constructor
*/
Cache() {}
public:
/**
* @brief Constructor
*/
Cache()
{
}
/**
* @brief Default Destructor
*/
~Cache() {}
/**
* @brief Default Destructor
*/
~Cache()
{
}
/**
* @brief Check whether cache key exists
*/
static bool cacheKeyExist( CacheMap* map, const OID oid, const uint64_t lbid )
{
CacheKey key = getCacheKey( oid, lbid );
return map->find(key) == map->end() ? false : true;
}
static bool cacheKeyExist( CacheMap* map, BlockBuffer* buffer )
{
return cacheKeyExist( map, (*buffer).cb.file.oid, (*buffer).block.lbid );
}
static bool cacheKeyExist( const OID oid, const uint64_t lbid )
{
return cacheKeyExist( m_lruList, oid, lbid ) || cacheKeyExist( m_writeList, oid, lbid );
}
/**
* @brief Check whether cache key exists
*/
static bool cacheKeyExist(CacheMap* map, const OID oid, const uint64_t lbid)
{
CacheKey key = getCacheKey(oid, lbid);
return map->find(key) == map->end() ? false : true;
}
static bool cacheKeyExist(CacheMap* map, BlockBuffer* buffer)
{
return cacheKeyExist(map, (*buffer).cb.file.oid, (*buffer).block.lbid);
}
static bool cacheKeyExist(const OID oid, const uint64_t lbid)
{
return cacheKeyExist(m_lruList, oid, lbid) || cacheKeyExist(m_writeList, oid, lbid);
}
/**
* @brief Clear the buffer
*/
EXPORT static void clear();
/**
* @brief Clear the buffer
*/
EXPORT static void clear();
/**
* @brief Free the buffer memory
*/
EXPORT static void freeMemory();
/**
* @brief Free the buffer memory
*/
EXPORT static void freeMemory();
/**
* @brief Flush the write cache
*/
EXPORT static int flushCache();
/**
* @brief Flush the write cache
*/
EXPORT static int flushCache();
/**
* @brief Get the cache key
*/
static CacheKey getCacheKey( const OID oid, const uint64_t lbid )
{
CacheKey key = lbid; /*Convertor::int2Str( oid ) + "|" + Convertor::int2Str(lbid)*/;
return key;
}
static CacheKey getCacheKey( const BlockBuffer* buffer )
{
return getCacheKey( (*buffer).cb.file.oid, (*buffer).block.lbid );
}
/**
* @brief Get the cache key
*/
static CacheKey getCacheKey(const OID oid, const uint64_t lbid)
{
CacheKey key = lbid; /*Convertor::int2Str( oid ) + "|" + Convertor::int2Str(lbid)*/
;
return key;
}
static CacheKey getCacheKey(const BlockBuffer* buffer)
{
return getCacheKey((*buffer).cb.file.oid, (*buffer).block.lbid);
}
EXPORT static int getListSize( const CacheListType listType );
EXPORT static int getListSize(const CacheListType listType);
/**
* @brief Init the buffers
*/
EXPORT static void init( const int totalBlock, const int chkPoint, const int pctFree );
static void init()
{
init( DEFAULT_CACHE_BLOCK, DEFAULT_CHK_INTERVAL, DEFAULT_CACHE_PCT_FREE );
}
/**
* @brief Init the buffers
*/
EXPORT static void init(const int totalBlock, const int chkPoint, const int pctFree);
static void init()
{
init(DEFAULT_CACHE_BLOCK, DEFAULT_CHK_INTERVAL, DEFAULT_CACHE_PCT_FREE);
}
/**
* @brief Insert into LRU list
*/
EXPORT static int insertLRUList( CommBlock& cb, const uint64_t lbid, const uint64_t fbo, const unsigned char* buf );
static int insertLRUList( CommBlock& cb, const uint64_t lbid, const uint64_t fbo, const DataBlock& block )
{
return insertLRUList( cb, lbid, fbo, block.data );
}
/**
* @brief Insert into LRU list
*/
EXPORT static int insertLRUList(CommBlock& cb, const uint64_t lbid, const uint64_t fbo,
const unsigned char* buf);
static int insertLRUList(CommBlock& cb, const uint64_t lbid, const uint64_t fbo, const DataBlock& block)
{
return insertLRUList(cb, lbid, fbo, block.data);
}
/**
* @brief Insert into Write list
*/
// static const int insertWriteList( const CacheKey& key );
/**
* @brief Insert into Write list
*/
// static const int insertWriteList( const CacheKey& key );
/**
* @brief Load cache block to a buffer
*/
static int loadCacheBlock( const CacheKey& key, DataBlock& block )
{
return loadCacheBlock( key, block.data );
}
EXPORT static int loadCacheBlock( const CacheKey& key, unsigned char* buf );
/**
* @brief Load cache block to a buffer
*/
static int loadCacheBlock(const CacheKey& key, DataBlock& block)
{
return loadCacheBlock(key, block.data);
}
EXPORT static int loadCacheBlock(const CacheKey& key, unsigned char* buf);
/**
* @brief Modify a cache block
*/
static int modifyCacheBlock( const CacheKey& key, const DataBlock& block )
{
return modifyCacheBlock( key, block.data );
}
EXPORT static int modifyCacheBlock( const CacheKey& key, const unsigned char* buf );
/**
* @brief Modify a cache block
*/
static int modifyCacheBlock(const CacheKey& key, const DataBlock& block)
{
return modifyCacheBlock(key, block.data);
}
EXPORT static int modifyCacheBlock(const CacheKey& key, const unsigned char* buf);
/**
* @brief Print
*/
EXPORT static void printCacheMapList( const CacheMap* map );
EXPORT static void printCacheList();
/**
* @brief Print
*/
EXPORT static void printCacheMapList(const CacheMap* map);
EXPORT static void printCacheList();
/**
* @brief Insert/Delete an element in cache map
*/
EXPORT static int processCacheMap( CacheMap* map, BlockBuffer* buffer, OpType opType );
/**
* @brief Insert/Delete an element in cache map
*/
EXPORT static int processCacheMap(CacheMap* map, BlockBuffer* buffer, OpType opType);
// accessory
static int getTotalBlock()
{
return m_cacheParam->totalBlock;
}
static bool getUseCache()
{
return m_useCache;
}
static void setUseCache( const bool flag )
{
m_useCache = flag;
}
// accessory
static int getTotalBlock()
{
return m_cacheParam->totalBlock;
}
static bool getUseCache()
{
return m_useCache;
}
static void setUseCache(const bool flag)
{
m_useCache = flag;
}
static CacheControl* m_cacheParam; // Cache parameters
static FreeBufList* m_freeList; // free buffer list
static CacheMap* m_lruList; // LRU buffer list
static CacheMap* m_writeList; // Write buffer list
static CacheControl* m_cacheParam; // Cache parameters
static FreeBufList* m_freeList; // free buffer list
static CacheMap* m_lruList; // LRU buffer list
static CacheMap* m_writeList; // Write buffer list
#if defined(_MSC_VER) && !defined(WRITEENGINE_DLLEXPORT)
__declspec(dllimport)
__declspec(dllimport)
#endif
EXPORT static bool m_useCache; // Use cache flag
private:
EXPORT static bool m_useCache; // Use cache flag
private:
};
} //end of namespace
} // namespace WriteEngine
#undef EXPORT

3860
writeengine/shared/we_chunkmanager.cpp
View File

File diff suppressed because it is too large Load Diff

478
writeengine/shared/we_chunkmanager.h
View File

@ -17,7 +17,6 @@
// $Id: we_chunkmanager.h 4726 2013-08-07 03:38:36Z bwilkinson $
/** @file */
#pragma once
@ -41,13 +40,20 @@
#endif
#ifdef _MSC_VER
#define WE_COMP_DBG(x) {}
#define WE_COMP_DBG(x) \
{ \
}
#else
//#define IDB_COMP_DEBUG
#ifdef IDB_COMP_DEBUG
#define WE_COMP_DBG(x) {x}
#define WE_COMP_DBG(x) \
{ \
x \
}
#else
#define WE_COMP_DBG(x) {}
#define WE_COMP_DBG(x) \
{ \
}
#endif
#endif
@ -55,11 +61,10 @@ namespace logging
{
// use Logger (not we_log) for now.
class Logger;
}
} // namespace logging
namespace WriteEngine
{
// forward reference
class FileOp;
@ -75,321 +80,294 @@ const int MAXOFFSET_PER_CHUNK = 511 * BYTE_PER_BLOCK;
typedef int64_t ChunkId;
struct ChunkData
{
ChunkId fChunkId;
unsigned int fLenUnCompressed;
char fBufUnCompressed[UNCOMPRESSED_CHUNK_SIZE];
bool fWriteToFile;
ChunkId fChunkId;
unsigned int fLenUnCompressed;
char fBufUnCompressed[UNCOMPRESSED_CHUNK_SIZE];
bool fWriteToFile;
ChunkData(ChunkId id = 0) : fChunkId(id), fLenUnCompressed(0), fWriteToFile(false) {}
bool operator < (const ChunkData& rhs) const
{
return fChunkId < rhs.fChunkId;
}
ChunkData(ChunkId id = 0) : fChunkId(id), fLenUnCompressed(0), fWriteToFile(false)
{
}
bool operator<(const ChunkData& rhs) const
{
return fChunkId < rhs.fChunkId;
}
};
// compressed DB file header information
struct CompFileHeader
{
char fHeaderData[COMPRESSED_FILE_HEADER_UNIT * 2];
char* fControlData;
char* fPtrSection;
boost::scoped_array<char> fLongPtrSectData;
char fHeaderData[COMPRESSED_FILE_HEADER_UNIT * 2];
char* fControlData;
char* fPtrSection;
boost::scoped_array<char> fLongPtrSectData;
CompFileHeader() :
fControlData(fHeaderData), fPtrSection(fHeaderData + COMPRESSED_FILE_HEADER_UNIT) {}
CompFileHeader() : fControlData(fHeaderData), fPtrSection(fHeaderData + COMPRESSED_FILE_HEADER_UNIT)
{
}
};
// unique ID of a DB file
struct FileID
{
FID fFid;
uint32_t fDbRoot;
uint32_t fPartition;
uint32_t fSegment;
FID fFid;
uint32_t fDbRoot;
uint32_t fPartition;
uint32_t fSegment;
FileID(FID f, uint32_t r, uint32_t p, uint32_t s) :
fFid(f), fDbRoot(r), fPartition(p), fSegment(s) {}
FileID(FID f, uint32_t r, uint32_t p, uint32_t s) : fFid(f), fDbRoot(r), fPartition(p), fSegment(s)
{
}
bool operator < (const FileID& rhs) const
{
return (
(fFid < rhs.fFid) ||
(fFid == rhs.fFid && fDbRoot < rhs.fDbRoot) ||
(fFid == rhs.fFid && fDbRoot == rhs.fDbRoot && fPartition < rhs.fPartition) ||
(fFid == rhs.fFid && fDbRoot == rhs.fDbRoot && fPartition == rhs.fPartition && fSegment < rhs.fSegment));
}
bool operator == (const FileID& rhs) const
{
return (
fFid == rhs.fFid && fDbRoot == rhs.fDbRoot && fPartition == rhs.fPartition && fSegment == rhs.fSegment);
}
bool operator<(const FileID& rhs) const
{
return ((fFid < rhs.fFid) || (fFid == rhs.fFid && fDbRoot < rhs.fDbRoot) ||
(fFid == rhs.fFid && fDbRoot == rhs.fDbRoot && fPartition < rhs.fPartition) ||
(fFid == rhs.fFid && fDbRoot == rhs.fDbRoot && fPartition == rhs.fPartition &&
fSegment < rhs.fSegment));
}
bool operator==(const FileID& rhs) const
{
return (fFid == rhs.fFid && fDbRoot == rhs.fDbRoot && fPartition == rhs.fPartition &&
fSegment == rhs.fSegment);
}
};
// compressed DB file information
class CompFileData
{
public:
CompFileData(const FileID& id, const FID& fid, const execplan::CalpontSystemCatalog::ColDataType colDataType, int colWidth) :
fFileID(id), fFid(fid), fColDataType(colDataType), fColWidth(colWidth), fDctnryCol(false),
fFilePtr(NULL), fIoBSize(0), fCompressionType(1) {}
public:
CompFileData(const FileID& id, const FID& fid,
const execplan::CalpontSystemCatalog::ColDataType colDataType, int colWidth)
: fFileID(id)
, fFid(fid)
, fColDataType(colDataType)
, fColWidth(colWidth)
, fDctnryCol(false)
, fFilePtr(NULL)
, fIoBSize(0)
, fCompressionType(1)
{
}
ChunkData* findChunk(int64_t cid) const;
ChunkData* findChunk(int64_t cid) const;
protected:
FileID fFileID;
FID fFid;
execplan::CalpontSystemCatalog::ColDataType fColDataType;
int fColWidth;
bool fDctnryCol;
IDBDataFile* fFilePtr;
std::string fFileName;
CompFileHeader fFileHeader;
std::list<ChunkData*> fChunkList;
boost::scoped_array<char> fIoBuffer;
size_t fIoBSize;
uint32_t fCompressionType;
protected:
FileID fFileID;
FID fFid;
execplan::CalpontSystemCatalog::ColDataType fColDataType;
int fColWidth;
bool fDctnryCol;
IDBDataFile* fFilePtr;
std::string fFileName;
CompFileHeader fFileHeader;
std::list<ChunkData*> fChunkList;
boost::scoped_array<char> fIoBuffer;
size_t fIoBSize;
uint32_t fCompressionType;
friend class ChunkManager;
friend class ChunkManager;
};
class ChunkManager
{
public:
// @brief constructor
EXPORT ChunkManager();
public:
// @brief constructor
EXPORT ChunkManager();
// @brief destructor
EXPORT virtual ~ChunkManager();
// @brief destructor
EXPORT virtual ~ChunkManager();
// @brief Retrieve a file pointer in the chunk manager.
// for column file
IDBDataFile* getFilePtr(const Column& column,
uint16_t root,
uint32_t partition,
uint16_t segment,
std::string& filename,
const char* mode,
int size,
bool useTmpSuffix) const;
// @brief Retrieve a file pointer in the chunk manager.
// for column file
IDBDataFile* getFilePtr(const Column& column, uint16_t root, uint32_t partition, uint16_t segment,
std::string& filename, const char* mode, int size, bool useTmpSuffix) const;
// @brief Retrieve a file pointer in the chunk manager.
// for dictionary file
IDBDataFile* getFilePtr(const FID& fid,
uint16_t root,
uint32_t partition,
uint16_t segment,
std::string& filename,
const char* mode,
int size,
bool useTmpSuffix) const;
// @brief Retrieve a file pointer in the chunk manager.
// for dictionary file
IDBDataFile* getFilePtr(const FID& fid, uint16_t root, uint32_t partition, uint16_t segment,
std::string& filename, const char* mode, int size, bool useTmpSuffix) const;
// @brief Retrieve a file pointer in the chunk manager.
// for column/dict segment file
IDBDataFile* getSegmentFilePtr(
FID& fid, uint16_t root, uint32_t partition, uint16_t segment,
execplan::CalpontSystemCatalog::ColDataType colDataType,
uint32_t colWidth, std::string& filename, const char* mode,
int32_t size, bool useTmpSuffix, bool isDict) const;
// @brief Retrieve a file pointer in the chunk manager.
// for column/dict segment file
IDBDataFile* getSegmentFilePtr(FID& fid, uint16_t root, uint32_t partition, uint16_t segment,
execplan::CalpontSystemCatalog::ColDataType colDataType, uint32_t colWidth,
std::string& filename, const char* mode, int32_t size, bool useTmpSuffix,
bool isDict) const;
// @brief Create a compressed dictionary file with an appropriate header.
IDBDataFile* createDctnryFile(const FID& fid,
int64_t width,
uint16_t root,
uint32_t partition,
uint16_t segment,
const char* filename,
const char* mode,
int size,
int64_t lbid);
// @brief Create a compressed dictionary file with an appropriate header.
IDBDataFile* createDctnryFile(const FID& fid, int64_t width, uint16_t root, uint32_t partition,
uint16_t segment, const char* filename, const char* mode, int size,
int64_t lbid);
// @brief Read a block from pFile at offset fbo.
// The data may copied from memory if the chunk it belongs to is already available.
int readBlock(IDBDataFile* pFile, unsigned char* readBuf, uint64_t fbo);
// @brief Read a block from pFile at offset fbo.
// The data may copied from memory if the chunk it belongs to is already available.
int readBlock(IDBDataFile* pFile, unsigned char* readBuf, uint64_t fbo);
// @brief Save a block to a chunk in pFile.
// The block is not written to disk immediately, will be delayed until flush.
int saveBlock(IDBDataFile* pFile, const unsigned char* writeBuf, uint64_t fbo);
// @brief Save a block to a chunk in pFile.
// The block is not written to disk immediately, will be delayed until flush.
int saveBlock(IDBDataFile* pFile, const unsigned char* writeBuf, uint64_t fbo);
// @brief Write all active chunks to disk, and reset all repository.
EXPORT int flushChunks(int rc, const std::map<FID, FID>& columOids);
// @brief Write all active chunks to disk, and reset all repository.
EXPORT int flushChunks(int rc, const std::map<FID, FID>& columOids);
// @brief Reset all repository without writing anything to disk.
void cleanUp(const std::map<FID, FID>& columOids);
// @brief Reset all repository without writing anything to disk.
void cleanUp(const std::map<FID, FID>& columOids);
// @brief Expand an initial column, not dictionary, extent to a full extent.
int expandAbbrevColumnExtent(IDBDataFile* pFile, const uint8_t* emptyVal, int width);
// @brief Expand an initial column, not dictionary, extent to a full extent.
int expandAbbrevColumnExtent(IDBDataFile* pFile, const uint8_t* emptyVal, int width);
// @brief Update column extent
int updateColumnExtent(IDBDataFile* pFile, int addBlockCount,
int64_t lbid);
// @brief Update column extent
int updateColumnExtent(IDBDataFile* pFile, int addBlockCount, int64_t lbid);
// @brief Update dictionary extent
int updateDctnryExtent(IDBDataFile* pFile, int addBlockCount,
int64_t lbid);
// @brief Update dictionary extent
int updateDctnryExtent(IDBDataFile* pFile, int addBlockCount, int64_t lbid);
// @brief Read in n continuous blocks to read buffer.
// for backing up blocks to version buffer
int readBlocks(IDBDataFile* pFile, unsigned char* readBuf, uint64_t fbo, size_t n);
// @brief Read in n continuous blocks to read buffer.
// for backing up blocks to version buffer
int readBlocks(IDBDataFile* pFile, unsigned char* readBuf, uint64_t fbo, size_t n);
// @brief Restore the data block at offset fbo from version buffer
// for rollback
int restoreBlock(IDBDataFile* pFile, const unsigned char* writeBuf, uint64_t fbo);
// @brief Restore the data block at offset fbo from version buffer
// for rollback
int restoreBlock(IDBDataFile* pFile, const unsigned char* writeBuf, uint64_t fbo);
// @brief Retrieve the total block count of a DB file.
int getBlockCount(IDBDataFile* pFile);
// @brief Retrieve the total block count of a DB file.
int getBlockCount(IDBDataFile* pFile);
// @brief Set FileOp pointer (for compression type, empty value, txnId, etc.)
void fileOp(FileOp* fileOp);
// @brief Set FileOp pointer (for compression type, empty value, txnId, etc.)
void fileOp(FileOp* fileOp);
// @brief Control the number of active chunks being stored in memory
void setMaxActiveChunkNum(unsigned int maxActiveChunkNum)
{
fMaxActiveChunkNum = maxActiveChunkNum;
}
// @brief Control the number of active chunks being stored in memory
void setMaxActiveChunkNum(unsigned int maxActiveChunkNum)
{
fMaxActiveChunkNum = maxActiveChunkNum;
}
// @brief Use this flag to avoid logging and backing up chunks, tmp files.
void setBulkFlag(bool isBulkLoad)
{
fIsBulkLoad = isBulkLoad;
}
// @brief Use this flag to avoid logging and backing up chunks, tmp files.
void setBulkFlag(bool isBulkLoad)
{
fIsBulkLoad = isBulkLoad;
}
// @brief Use this flag to flush chunk when is full.
void setIsInsert(bool isInsert)
{
fIsInsert = isInsert;
}
bool getIsInsert()
{
return fIsInsert;
}
// @brief Use this flag to flush chunk when is full.
void setIsInsert(bool isInsert)
{
fIsInsert = isInsert;
}
bool getIsInsert()
{
return fIsInsert;
}
void setTransId(const TxnID& transId)
{
fTransId = transId;
}
void setTransId(const TxnID& transId)
{
fTransId = transId;
}
// @brief bug5504, Use non transactional DML for InfiniDB with HDFS
EXPORT int startTransaction(const TxnID& transId) const;
EXPORT int confirmTransaction(const TxnID& transId) const;
EXPORT int endTransaction(const TxnID& transId, bool success) const;
// @brief Use this flag to fix bad chunk.
void setFixFlag(bool isFix)
{
fIsFix = isFix;
}
// @brief bug5504, Use non transactional DML for InfiniDB with HDFS
EXPORT int startTransaction(const TxnID& transId) const;
EXPORT int confirmTransaction(const TxnID& transId) const;
EXPORT int endTransaction(const TxnID& transId, bool success) const;
// @brief Use this flag to fix bad chunk.
void setFixFlag(bool isFix)
{
fIsFix = isFix;
}
EXPORT int checkFixLastDictChunk(const FID& fid,
uint16_t root,
uint32_t partition,
uint16_t segment);
EXPORT int checkFixLastDictChunk(const FID& fid, uint16_t root, uint32_t partition, uint16_t segment);
protected:
// @brief Retrieve pointer to a compressed DB file.
CompFileData* getFileData(const FID& fid,
uint16_t root,
uint32_t partition,
uint16_t segment,
std::string& filename,
const char* mode,
int size,
const execplan::CalpontSystemCatalog::ColDataType colDataType,
int colWidth,
bool useTmpSuffix,
bool dictnry = false) const;
protected:
// @brief Retrieve pointer to a compressed DB file.
CompFileData* getFileData(const FID& fid, uint16_t root, uint32_t partition, uint16_t segment,
std::string& filename, const char* mode, int size,
const execplan::CalpontSystemCatalog::ColDataType colDataType, int colWidth,
bool useTmpSuffix, bool dictnry = false) const;
// @brief Retrieve a chunk of pFile from disk.
int fetchChunkFromFile(IDBDataFile* pFile, int64_t id, ChunkData*& chunkData);
// @brief Retrieve a chunk of pFile from disk.
int fetchChunkFromFile(IDBDataFile* pFile, int64_t id, ChunkData*& chunkData);
// @brief Compress a chunk and write it to file.
int writeChunkToFile(CompFileData* fileData, int64_t id);
int writeChunkToFile(CompFileData* fileData, ChunkData* chunkData);
// @brief Compress a chunk and write it to file.
int writeChunkToFile(CompFileData* fileData, int64_t id);
int writeChunkToFile(CompFileData* fileData, ChunkData* chunkData);
// @brief Write the compressed data to file and log a recover entry.
int writeCompressedChunk(CompFileData* fileData, int64_t offset, int64_t size);
inline int writeCompressedChunk_(CompFileData* fileData, int64_t offset);
// @brief Write the compressed data to file and log a recover entry.
int writeCompressedChunk(CompFileData* fileData, int64_t offset, int64_t size);
inline int writeCompressedChunk_(CompFileData* fileData, int64_t offset);
// @brief Write the file header to disk.
int writeHeader(CompFileData* fileData, int ln);
inline int writeHeader_(CompFileData* fileData, int ptrSecSize);
// @brief Write the file header to disk.
int writeHeader(CompFileData* fileData, int ln);
inline int writeHeader_(CompFileData* fileData, int ptrSecSize);
// @brief open a compressed DB file.
int openFile(CompFileData* fileData, const char* mode, int colWidth,
bool useTmpSuffix, int ln) const;
// @brief open a compressed DB file.
int openFile(CompFileData* fileData, const char* mode, int colWidth, bool useTmpSuffix, int ln) const;
// @brief set offset in a compressed DB file from beginning.
int setFileOffset(IDBDataFile* pFile, const std::string& fileName, off64_t offset, int ln) const;
// @brief set offset in a compressed DB file from beginning.
int setFileOffset(IDBDataFile* pFile, const std::string& fileName, off64_t offset, int ln) const;
// @brief read from a compressed DB file.
int readFile(IDBDataFile* pFile, const std::string& fileName, void* buf, size_t size, int ln) const;
// @brief read from a compressed DB file.
int readFile(IDBDataFile* pFile, const std::string& fileName, void* buf, size_t size, int ln) const;
// @brief write to a compressed DB file.
int writeFile(IDBDataFile* pFile, const std::string& fileName, void* buf, size_t size, int ln) const;
// @brief write to a compressed DB file.
int writeFile(IDBDataFile* pFile, const std::string& fileName, void* buf, size_t size, int ln) const;
// @brief Close a compressed DB file.
int closeFile(CompFileData* fileData);
// @brief Close a compressed DB file.
int closeFile(CompFileData* fileData);
// @brief Set empty values to a chunk.
void initializeColumnChunk(char* buf, CompFileData* fileData);
void initializeDctnryChunk(char* buf, int size);
// @brief Set empty values to a chunk.
void initializeColumnChunk(char* buf, CompFileData* fileData);
void initializeDctnryChunk(char* buf, int size);
// @brief Calculate the header size based on column width.
int calculateHeaderSize(int width);
// @brief Calculate the header size based on column width.
int calculateHeaderSize(int width);
// @brief Moving chunks as a result of expanding a chunk.
int reallocateChunks(CompFileData* fileData);
// @brief Moving chunks as a result of expanding a chunk.
int reallocateChunks(CompFileData* fileData);
// @brief verify chunks in the file are OK
int verifyChunksAfterRealloc(CompFileData* fileData);
// @brief verify chunks in the file are OK
int verifyChunksAfterRealloc(CompFileData* fileData);
// @brief log a message to the syslog
void logMessage(int code, int level, int lineNum, int fromLine = -1) const;
void logMessage(const std::string& msg, int level) const;
// @brief log a message to the syslog
void logMessage(int code, int level, int lineNum, int fromLine = -1) const;
void logMessage(const std::string& msg, int level) const;
// @brief Write a DML recovery log
int writeLog(TxnID txnId, std::string backUpFileType, std::string filename,
std::string& aDMLLogFileName, int64_t size = 0, int64_t offset = 0) const;
// @brief Write a DML recovery log
int writeLog(TxnID txnId, std::string backUpFileType, std::string filename, std::string& aDMLLogFileName,
int64_t size = 0, int64_t offset = 0) const;
// @brief remove DML recovery logs
int removeBackups(TxnID txnId);
// @brief remove DML recovery logs
int removeBackups(TxnID txnId);
// @brief swap the src file to dest file
int swapTmpFile(const std::string& src, const std::string& dest);
// @brief swap the src file to dest file
int swapTmpFile(const std::string& src, const std::string& dest);
// @brief construnct a DML log file name
int getDMLLogFileName(std::string& aDMLLogFileName, const TxnID& txnId) const;
// @brief construnct a DML log file name
int getDMLLogFileName(std::string& aDMLLogFileName, const TxnID& txnId) const;
mutable std::map<FileID, CompFileData*> fFileMap;
mutable std::map<IDBDataFile*, CompFileData*> fFilePtrMap;
std::list<std::pair<FileID, ChunkData*> > fActiveChunks;
unsigned int fMaxActiveChunkNum; // max active chunks per file
char* fBufCompressed;
size_t fLenCompressed;
size_t fMaxCompressedBufSize;
size_t fUserPaddings;
bool fIsBulkLoad;
bool fDropFdCache;
bool fIsInsert;
bool fIsHdfs;
FileOp* fFileOp;
compress::CompressorPool fCompressorPool;
logging::Logger* fSysLogger;
TxnID fTransId;
int fLocalModuleId;
idbdatafile::IDBFileSystem& fFs;
bool fIsFix;
size_t COMPRESSED_CHUNK_SIZE;
mutable std::map<FileID, CompFileData*> fFileMap;
mutable std::map<IDBDataFile*, CompFileData*> fFilePtrMap;
std::list<std::pair<FileID, ChunkData*> > fActiveChunks;
unsigned int fMaxActiveChunkNum; // max active chunks per file
char* fBufCompressed;
size_t fLenCompressed;
size_t fMaxCompressedBufSize;
size_t fUserPaddings;
bool fIsBulkLoad;
bool fDropFdCache;
bool fIsInsert;
bool fIsHdfs;
FileOp* fFileOp;
compress::CompressorPool fCompressorPool;
logging::Logger* fSysLogger;
TxnID fTransId;
int fLocalModuleId;
idbdatafile::IDBFileSystem& fFs;
bool fIsFix;
size_t COMPRESSED_CHUNK_SIZE;
private:
private:
};
}
} // namespace WriteEngine
#undef EXPORT

568
writeengine/shared/we_config.cpp
View File

@ -16,9 +16,9 @@
MA 02110-1301, USA. */
/*******************************************************************************
* $Id: we_config.cpp 4737 2013-08-14 20:45:46Z bwilkinson $
*
*******************************************************************************/
* $Id: we_config.cpp 4737 2013-08-14 20:45:46Z bwilkinson $
*
*******************************************************************************/
/** @file */
#include <string>
@ -41,42 +41,39 @@ using namespace idbdatafile;
namespace WriteEngine
{
const int DEFAULT_WAIT_PERIOD = 10;
const unsigned DEFAULT_FILES_PER_COLUMN_PARTITION = 4;
const unsigned DEFAULT_EXTENTS_PER_SEGMENT_FILE = 2;
const int DEFAULT_BULK_PROCESS_PRIORITY = -1;
const unsigned DEFAULT_MAX_FILESYSTEM_DISK_USAGE = 98; // allow 98% full
const unsigned DEFAULT_COMPRESSED_PADDING_BLKS = 1;
const int DEFAULT_LOCAL_MODULE_ID = 1;
const bool DEFAULT_PARENT_OAM = true;
const char* DEFAULT_LOCAL_MODULE_TYPE = "pm";
const int DEFAULT_WAIT_PERIOD = 10;
const unsigned DEFAULT_FILES_PER_COLUMN_PARTITION = 4;
const unsigned DEFAULT_EXTENTS_PER_SEGMENT_FILE = 2;
const int DEFAULT_BULK_PROCESS_PRIORITY = -1;
const unsigned DEFAULT_MAX_FILESYSTEM_DISK_USAGE = 98; // allow 98% full
const unsigned DEFAULT_COMPRESSED_PADDING_BLKS = 1;
const int DEFAULT_LOCAL_MODULE_ID = 1;
const bool DEFAULT_PARENT_OAM = true;
const char* DEFAULT_LOCAL_MODULE_TYPE = "pm";
int Config::m_dbRootCount = 0;
int Config::m_dbRootCount = 0;
Config::strvec_t Config::m_dbRootPath;
Config::intstrmap_t Config::m_dbRootPathMap;
Config::uint16vec_t Config::m_dbRootId;
string Config::m_bulkRoot;
string Config::m_bulkRoot;
unsigned long Config::fDBRootChangeCount = 0;
time_t Config::fCacheTime = 0;
boost::mutex Config::fCacheLock;
unsigned long Config::fDBRootChangeCount = 0;
time_t Config::fCacheTime = 0;
boost::mutex Config::fCacheLock;
#ifdef SHARED_NOTHING_DEMO_2
boost::mutex Config::m_bulkRoot_lk;
boost::mutex Config::m_bulkRoot_lk;
#endif
int Config::m_WaitPeriod = DEFAULT_WAIT_PERIOD;
unsigned Config::m_FilesPerColumnPartition =
DEFAULT_FILES_PER_COLUMN_PARTITION;
unsigned Config::m_ExtentsPerSegmentFile =
DEFAULT_EXTENTS_PER_SEGMENT_FILE;
int Config::m_BulkProcessPriority = DEFAULT_BULK_PROCESS_PRIORITY;
string Config::m_BulkRollbackDir;
unsigned Config::m_MaxFileSystemDiskUsage =
DEFAULT_MAX_FILESYSTEM_DISK_USAGE;
unsigned Config::m_NumCompressedPadBlks = DEFAULT_COMPRESSED_PADDING_BLKS;
bool Config::m_ParentOAMModuleFlag = DEFAULT_PARENT_OAM;
string Config::m_LocalModuleType;
int Config::m_LocalModuleID = DEFAULT_LOCAL_MODULE_ID;
string Config::m_VersionBufferDir;
int Config::m_WaitPeriod = DEFAULT_WAIT_PERIOD;
unsigned Config::m_FilesPerColumnPartition = DEFAULT_FILES_PER_COLUMN_PARTITION;
unsigned Config::m_ExtentsPerSegmentFile = DEFAULT_EXTENTS_PER_SEGMENT_FILE;
int Config::m_BulkProcessPriority = DEFAULT_BULK_PROCESS_PRIORITY;
string Config::m_BulkRollbackDir;
unsigned Config::m_MaxFileSystemDiskUsage = DEFAULT_MAX_FILESYSTEM_DISK_USAGE;
unsigned Config::m_NumCompressedPadBlks = DEFAULT_COMPRESSED_PADDING_BLKS;
bool Config::m_ParentOAMModuleFlag = DEFAULT_PARENT_OAM;
string Config::m_LocalModuleType;
int Config::m_LocalModuleID = DEFAULT_LOCAL_MODULE_ID;
string Config::m_VersionBufferDir;
/*******************************************************************************
* DESCRIPTION:
@ -88,8 +85,8 @@ string Config::m_VersionBufferDir;
******************************************************************************/
void Config::initConfigCache()
{
boost::mutex::scoped_lock lk(fCacheLock);
checkReload( );
boost::mutex::scoped_lock lk(fCacheLock);
checkReload();
}
/*******************************************************************************
@ -98,221 +95,218 @@ void Config::initConfigCache()
* PARAMETERS:
* none
******************************************************************************/
void Config::checkReload( )
void Config::checkReload()
{
bool bFirstLoad = false;
bool bFirstLoad = false;
if (fCacheTime == 0)
bFirstLoad = true;
if (fCacheTime == 0)
bFirstLoad = true;
config::Config* cf = config::Config::makeConfig();
config::Config* cf = config::Config::makeConfig();
// Immediately return if Columnstore.xml timestamp has not changed
if (cf->getCurrentMTime() == fCacheTime)
return;
// Immediately return if Columnstore.xml timestamp has not changed
if (cf->getCurrentMTime() == fCacheTime)
return;
//std::cout << "RELOADING cache..." << std::endl;
// std::cout << "RELOADING cache..." << std::endl;
//--------------------------------------------------------------------------
// Initialize bulk root directory
//--------------------------------------------------------------------------
m_bulkRoot = cf->getConfig("WriteEngine", "BulkRoot");
//--------------------------------------------------------------------------
// Initialize bulk root directory
//--------------------------------------------------------------------------
m_bulkRoot = cf->getConfig("WriteEngine", "BulkRoot");
if ( m_bulkRoot.length() == 0 )
{
m_bulkRoot = "/var/log/mariadb/columnstore";
if (m_bulkRoot.length() == 0)
{
m_bulkRoot = "/var/log/mariadb/columnstore";
#ifndef _MSC_VER
m_bulkRoot += "/data";
m_bulkRoot += "/data";
#endif
m_bulkRoot += "/bulk";
}
m_bulkRoot += "/bulk";
}
// Get latest Columnstore.xml timestamp after first access forced a reload
fCacheTime = cf ->getLastMTime();
// Get latest Columnstore.xml timestamp after first access forced a reload
fCacheTime = cf->getLastMTime();
//--------------------------------------------------------------------------
// Initialize time interval (in seconds) between retries
//--------------------------------------------------------------------------
m_WaitPeriod = DEFAULT_WAIT_PERIOD;
string waitPeriodStr = cf->getConfig("SystemConfig", "WaitPeriod");
//--------------------------------------------------------------------------
// Initialize time interval (in seconds) between retries
//--------------------------------------------------------------------------
m_WaitPeriod = DEFAULT_WAIT_PERIOD;
string waitPeriodStr = cf->getConfig("SystemConfig", "WaitPeriod");
if ( waitPeriodStr.length() != 0 )
m_WaitPeriod = static_cast<int>(config::Config::fromText(
waitPeriodStr));
if (waitPeriodStr.length() != 0)
m_WaitPeriod = static_cast<int>(config::Config::fromText(waitPeriodStr));
//--------------------------------------------------------------------------
// Initialize files per column partition
//--------------------------------------------------------------------------
m_FilesPerColumnPartition = DEFAULT_FILES_PER_COLUMN_PARTITION;
string fpc = cf->getConfig("ExtentMap", "FilesPerColumnPartition");
//--------------------------------------------------------------------------
// Initialize files per column partition
//--------------------------------------------------------------------------
m_FilesPerColumnPartition = DEFAULT_FILES_PER_COLUMN_PARTITION;
string fpc = cf->getConfig("ExtentMap", "FilesPerColumnPartition");
if ( fpc.length() != 0 )
m_FilesPerColumnPartition = cf->uFromText(fpc);
if (fpc.length() != 0)
m_FilesPerColumnPartition = cf->uFromText(fpc);
//--------------------------------------------------------------------------
// Initialize extents per segment file
//--------------------------------------------------------------------------
m_ExtentsPerSegmentFile = DEFAULT_EXTENTS_PER_SEGMENT_FILE;
//--------------------------------------------------------------------------
// Initialize extents per segment file
//--------------------------------------------------------------------------
m_ExtentsPerSegmentFile = DEFAULT_EXTENTS_PER_SEGMENT_FILE;
//--------------------------------------------------------------------------
// Initialize bulk load process priority
//--------------------------------------------------------------------------
m_BulkProcessPriority = DEFAULT_BULK_PROCESS_PRIORITY;
string prior = cf->getConfig("WriteEngine", "Priority");
//--------------------------------------------------------------------------
// Initialize bulk load process priority
//--------------------------------------------------------------------------
m_BulkProcessPriority = DEFAULT_BULK_PROCESS_PRIORITY;
string prior = cf->getConfig("WriteEngine", "Priority");
if ( prior.length() != 0 )
{
int initialBPP = cf->fromText(prior);
if (prior.length() != 0)
{
int initialBPP = cf->fromText(prior);
// config file priority is 40..1 (highest..lowest)
// convert config file value to setpriority(2) value(-20..19, -1 is the
// default)
if (initialBPP > 0)
m_BulkProcessPriority = 20 - initialBPP;
else if (initialBPP < 0)
m_BulkProcessPriority = 19;
// config file priority is 40..1 (highest..lowest)
// convert config file value to setpriority(2) value(-20..19, -1 is the
// default)
if (initialBPP > 0)
m_BulkProcessPriority = 20 - initialBPP;
else if (initialBPP < 0)
m_BulkProcessPriority = 19;
if (m_BulkProcessPriority < -20)
m_BulkProcessPriority = -20;
}
if (m_BulkProcessPriority < -20)
m_BulkProcessPriority = -20;
}
//--------------------------------------------------------------------------
// Initialize bulk rollback directory
// Note this uses m_bulkRoot, so this init section must be after the section
// that sets m_bulkRoot.
//--------------------------------------------------------------------------
m_BulkRollbackDir = cf->getConfig("WriteEngine", "BulkRollbackDir");
//--------------------------------------------------------------------------
// Initialize bulk rollback directory
// Note this uses m_bulkRoot, so this init section must be after the section
// that sets m_bulkRoot.
//--------------------------------------------------------------------------
m_BulkRollbackDir = cf->getConfig("WriteEngine", "BulkRollbackDir");
if (m_BulkRollbackDir.length() == 0)
{
m_BulkRollbackDir.assign( m_bulkRoot );
m_BulkRollbackDir += "/rollback";
}
if (m_BulkRollbackDir.length() == 0)
{
m_BulkRollbackDir.assign(m_bulkRoot);
m_BulkRollbackDir += "/rollback";
}
//--------------------------------------------------------------------------
// Initialize max disk usage
//--------------------------------------------------------------------------
//--------------------------------------------------------------------------
// Initialize max disk usage
//--------------------------------------------------------------------------
m_MaxFileSystemDiskUsage = DEFAULT_MAX_FILESYSTEM_DISK_USAGE;
string usg = cf->getConfig("WriteEngine", "MaxFileSystemDiskUsagePct");
if (usg.length() != 0)
m_MaxFileSystemDiskUsage = cf->uFromText(usg);
if (m_MaxFileSystemDiskUsage > 100)
m_MaxFileSystemDiskUsage = DEFAULT_MAX_FILESYSTEM_DISK_USAGE;
string usg = cf->getConfig("WriteEngine", "MaxFileSystemDiskUsagePct");
if ( usg.length() != 0 )
m_MaxFileSystemDiskUsage = cf->uFromText(usg);
//--------------------------------------------------------------------------
// Number of compressed padding blocks
//--------------------------------------------------------------------------
m_NumCompressedPadBlks = DEFAULT_COMPRESSED_PADDING_BLKS;
string ncpb = cf->getConfig("WriteEngine", "CompressedPaddingBlocks");
if (m_MaxFileSystemDiskUsage > 100)
m_MaxFileSystemDiskUsage = DEFAULT_MAX_FILESYSTEM_DISK_USAGE;
if (ncpb.length() != 0)
m_NumCompressedPadBlks = cf->uFromText(ncpb);
//--------------------------------------------------------------------------
// Number of compressed padding blocks
//--------------------------------------------------------------------------
m_NumCompressedPadBlks = DEFAULT_COMPRESSED_PADDING_BLKS;
string ncpb = cf->getConfig("WriteEngine", "CompressedPaddingBlocks");
IDBPolicy::configIDBPolicy();
if ( ncpb.length() != 0 )
m_NumCompressedPadBlks = cf->uFromText(ncpb);
//--------------------------------------------------------------------------
// Initialize Parent OAM Module flag
// Initialize Module Type
// Initialize Local Module ID
//--------------------------------------------------------------------------
oam::Oam oam;
oam::oamModuleInfo_t t;
IDBPolicy::configIDBPolicy();
try
{
t = oam.getModuleInfo();
m_ParentOAMModuleFlag = boost::get<4>(t);
m_LocalModuleType = boost::get<1>(t);
m_LocalModuleID = boost::get<2>(t);
}
catch (exception&)
{
m_ParentOAMModuleFlag = DEFAULT_PARENT_OAM;
m_LocalModuleType.assign(DEFAULT_LOCAL_MODULE_TYPE);
m_LocalModuleID = DEFAULT_LOCAL_MODULE_ID;
}
//--------------------------------------------------------------------------
// Initialize Parent OAM Module flag
// Initialize Module Type
// Initialize Local Module ID
//--------------------------------------------------------------------------
oam::Oam oam;
oam::oamModuleInfo_t t;
//--------------------------------------------------------------------------
// Initialize Version Buffer
//--------------------------------------------------------------------------
m_VersionBufferDir = cf->getConfig("SystemConfig", "DBRMRoot");
if (m_VersionBufferDir.length() == 0)
{
m_VersionBufferDir = "/var/lib/columnstore/data1/systemFiles/dbrm/BRM_saves";
}
//--------------------------------------------------------------------------
// Initialize m_dbRootCount, m_dbRootPath, m_dbRootPathMap, m_dbRootId.
// Note this uses m_localModuleType and m_LocalModuleID, so this init
// section must be after the section(s) that set m_localModuleType and
// m_LocalModuleID.
//--------------------------------------------------------------------------
uint16vec_t dbRootIdPrevious(m_dbRootId); // save current settings
strvec_t dbRootPathPrevious(m_dbRootPath); // save current setttings
m_dbRootPath.clear();
m_dbRootPathMap.clear();
m_dbRootId.clear();
if (m_LocalModuleType == "pm")
{
oam::DBRootConfigList oamRootList;
try
{
t = oam.getModuleInfo();
m_ParentOAMModuleFlag = boost::get<4>(t);
m_LocalModuleType = boost::get<1>(t);
m_LocalModuleID = boost::get<2>(t);
oam.getPmDbrootConfig(m_LocalModuleID, oamRootList);
std::sort(oamRootList.begin(), oamRootList.end());
m_dbRootCount = oamRootList.size();
for (unsigned int idx = 0; idx < oamRootList.size(); idx++)
{
ostringstream oss;
oss << "DBRoot" << oamRootList[idx];
std::string DbRootPath = cf->getConfig("SystemConfig", oss.str());
m_dbRootPath.push_back(DbRootPath);
m_dbRootPathMap[oamRootList[idx]] = DbRootPath;
m_dbRootId.push_back(oamRootList[idx]);
}
}
catch (exception&)
{
m_ParentOAMModuleFlag = DEFAULT_PARENT_OAM;
m_LocalModuleType.assign( DEFAULT_LOCAL_MODULE_TYPE );
m_LocalModuleID = DEFAULT_LOCAL_MODULE_ID;
m_dbRootCount = 0;
}
}
else
{
m_dbRootCount = 0;
}
//--------------------------------------------------------------------------
// Initialize Version Buffer
//--------------------------------------------------------------------------
m_VersionBufferDir = cf->getConfig("SystemConfig", "DBRMRoot");
if ( m_VersionBufferDir.length() == 0 )
// Update counter used to track changes to local PM DBRoot list
if (!bFirstLoad)
{
if ((dbRootIdPrevious != m_dbRootId) || (dbRootPathPrevious != m_dbRootPath))
{
m_VersionBufferDir = "/var/lib/columnstore/data1/systemFiles/dbrm/BRM_saves";
fDBRootChangeCount++;
}
}
//--------------------------------------------------------------------------
// Initialize m_dbRootCount, m_dbRootPath, m_dbRootPathMap, m_dbRootId.
// Note this uses m_localModuleType and m_LocalModuleID, so this init
// section must be after the section(s) that set m_localModuleType and
// m_LocalModuleID.
//--------------------------------------------------------------------------
uint16vec_t dbRootIdPrevious( m_dbRootId ); // save current settings
strvec_t dbRootPathPrevious( m_dbRootPath ); // save current setttings
m_dbRootPath.clear();
m_dbRootPathMap.clear();
m_dbRootId.clear();
if (m_LocalModuleType == "pm")
{
oam::DBRootConfigList oamRootList;
try
{
oam.getPmDbrootConfig( m_LocalModuleID, oamRootList );
std::sort( oamRootList.begin(), oamRootList.end() );
m_dbRootCount = oamRootList.size();
for (unsigned int idx = 0; idx < oamRootList.size(); idx++)
{
ostringstream oss;
oss << "DBRoot" << oamRootList[idx];
std::string DbRootPath =
cf->getConfig("SystemConfig", oss.str());
m_dbRootPath.push_back( DbRootPath );
m_dbRootPathMap[ oamRootList[idx] ] = DbRootPath;
m_dbRootId.push_back( oamRootList[idx] );
}
}
catch (exception&)
{
m_dbRootCount = 0;
}
}
else
{
m_dbRootCount = 0;
}
// Update counter used to track changes to local PM DBRoot list
if (!bFirstLoad)
{
if ((dbRootIdPrevious != m_dbRootId) ||
(dbRootPathPrevious != m_dbRootPath))
{
fDBRootChangeCount++;
}
}
// for (unsigned int n=0; n<m_dbRootPath.size(); n++)
// {
// std::cout << "dbrootpath: " << n << ". " << m_dbRootPath[n] <<std::endl;
// }
// for (unsigned int n=0; n<m_dbRootId.size(); n++)
// {
// std::cout << "dbrootId: " << n << ". " << m_dbRootId[n] << std::endl;
// }
// for (Config::intstrmap_t::iterator k=m_dbRootPathMap.begin();
// k!=m_dbRootPathMap.end(); ++k)
// {
// std::cout << "dbrootmap: " << k->first << "," << k->second << std::endl;
// }
// for (unsigned int n=0; n<m_dbRootPath.size(); n++)
// {
// std::cout << "dbrootpath: " << n << ". " << m_dbRootPath[n] <<std::endl;
// }
// for (unsigned int n=0; n<m_dbRootId.size(); n++)
// {
// std::cout << "dbrootId: " << n << ". " << m_dbRootId[n] << std::endl;
// }
// for (Config::intstrmap_t::iterator k=m_dbRootPathMap.begin();
// k!=m_dbRootPathMap.end(); ++k)
// {
// std::cout << "dbrootmap: " << k->first << "," << k->second << std::endl;
// }
}
/*******************************************************************************
@ -325,10 +319,10 @@ void Config::checkReload( )
******************************************************************************/
size_t Config::DBRootCount()
{
boost::mutex::scoped_lock lk(fCacheLock);
checkReload( );
boost::mutex::scoped_lock lk(fCacheLock);
checkReload();
return m_dbRootCount;
return m_dbRootCount;
}
/*******************************************************************************
@ -341,16 +335,16 @@ size_t Config::DBRootCount()
******************************************************************************/
std::string Config::getDBRootByIdx(unsigned idx)
{
boost::mutex::scoped_lock lk(fCacheLock);
checkReload( );
boost::mutex::scoped_lock lk(fCacheLock);
checkReload();
if (idx >= m_dbRootPath.size())
{
std::string emptyResult;
return emptyResult;
}
if (idx >= m_dbRootPath.size())
{
std::string emptyResult;
return emptyResult;
}
return m_dbRootPath[idx];
return m_dbRootPath[idx];
}
/*******************************************************************************
@ -361,13 +355,13 @@ std::string Config::getDBRootByIdx(unsigned idx)
* RETURN:
* none
******************************************************************************/
void Config::getDBRootPathList( std::vector<std::string>& dbRootPathList )
void Config::getDBRootPathList(std::vector<std::string>& dbRootPathList)
{
boost::mutex::scoped_lock lk(fCacheLock);
checkReload( );
boost::mutex::scoped_lock lk(fCacheLock);
checkReload();
dbRootPathList.clear();
dbRootPathList = m_dbRootPath;
dbRootPathList.clear();
dbRootPathList = m_dbRootPath;
}
/*******************************************************************************
@ -380,18 +374,18 @@ void Config::getDBRootPathList( std::vector<std::string>& dbRootPathList )
******************************************************************************/
std::string Config::getDBRootByNum(unsigned num)
{
boost::mutex::scoped_lock lk(fCacheLock);
checkReload( );
boost::mutex::scoped_lock lk(fCacheLock);
checkReload();
Config::intstrmap_t::const_iterator iter = m_dbRootPathMap.find( num );
Config::intstrmap_t::const_iterator iter = m_dbRootPathMap.find(num);
if (iter == m_dbRootPathMap.end())
{
std::string emptyResult;
return emptyResult;
}
if (iter == m_dbRootPathMap.end())
{
std::string emptyResult;
return emptyResult;
}
return iter->second;
return iter->second;
}
/*******************************************************************************
@ -402,12 +396,12 @@ std::string Config::getDBRootByNum(unsigned num)
* RETURN:
* The list of DBRoot ids
******************************************************************************/
void Config::getRootIdList( std::vector<uint16_t>& rootIds )
void Config::getRootIdList(std::vector<uint16_t>& rootIds)
{
boost::mutex::scoped_lock lk(fCacheLock);
checkReload( );
boost::mutex::scoped_lock lk(fCacheLock);
checkReload();
rootIds = m_dbRootId;
rootIds = m_dbRootId;
}
/*******************************************************************************
@ -420,21 +414,20 @@ void Config::getRootIdList( std::vector<uint16_t>& rootIds )
******************************************************************************/
std::string Config::getBulkRoot()
{
boost::mutex::scoped_lock lk(fCacheLock);
checkReload( );
boost::mutex::scoped_lock lk(fCacheLock);
checkReload();
return m_bulkRoot;
return m_bulkRoot;
}
#ifdef SHARED_NOTHING_DEMO_2
void Config::getSharedNothingRoot(char* ret)
{
string root;
boost::mutex::scoped_lock lk(m_bulkRoot_lk);
string root;
boost::mutex::scoped_lock lk(m_bulkRoot_lk);
root = config::Config::makeConfig()->getConfig(
"WriteEngine", "SharedNothingRoot");
strncpy(ret, root.c_str(), FILE_NAME_SIZE);
root = config::Config::makeConfig()->getConfig("WriteEngine", "SharedNothingRoot");
strncpy(ret, root.c_str(), FILE_NAME_SIZE);
}
#endif
@ -448,10 +441,10 @@ void Config::getSharedNothingRoot(char* ret)
******************************************************************************/
int Config::getWaitPeriod()
{
boost::mutex::scoped_lock lk(fCacheLock);
checkReload( );
boost::mutex::scoped_lock lk(fCacheLock);
checkReload();
return m_WaitPeriod;
return m_WaitPeriod;
}
/*******************************************************************************
@ -464,10 +457,10 @@ int Config::getWaitPeriod()
******************************************************************************/
unsigned Config::getFilesPerColumnPartition()
{
boost::mutex::scoped_lock lk(fCacheLock);
checkReload( );
boost::mutex::scoped_lock lk(fCacheLock);
checkReload();
return m_FilesPerColumnPartition;
return m_FilesPerColumnPartition;
}
/*******************************************************************************
@ -480,10 +473,10 @@ unsigned Config::getFilesPerColumnPartition()
******************************************************************************/
unsigned Config::getExtentsPerSegmentFile()
{
boost::mutex::scoped_lock lk(fCacheLock);
checkReload( );
boost::mutex::scoped_lock lk(fCacheLock);
checkReload();
return m_ExtentsPerSegmentFile;
return m_ExtentsPerSegmentFile;
}
/*******************************************************************************
@ -503,10 +496,10 @@ unsigned Config::getExtentsPerSegmentFile()
******************************************************************************/
int Config::getBulkProcessPriority()
{
boost::mutex::scoped_lock lk(fCacheLock);
checkReload( );
boost::mutex::scoped_lock lk(fCacheLock);
checkReload();
return m_BulkProcessPriority;
return m_BulkProcessPriority;
}
/*******************************************************************************
@ -517,10 +510,10 @@ int Config::getBulkProcessPriority()
******************************************************************************/
std::string Config::getBulkRollbackDir()
{
boost::mutex::scoped_lock lk(fCacheLock);
checkReload( );
boost::mutex::scoped_lock lk(fCacheLock);
checkReload();
return m_BulkRollbackDir;
return m_BulkRollbackDir;
}
/*******************************************************************************
@ -531,10 +524,10 @@ std::string Config::getBulkRollbackDir()
******************************************************************************/
unsigned Config::getMaxFileSystemDiskUsage()
{
boost::mutex::scoped_lock lk(fCacheLock);
checkReload( );
boost::mutex::scoped_lock lk(fCacheLock);
checkReload();
return m_MaxFileSystemDiskUsage;
return m_MaxFileSystemDiskUsage;
}
/*******************************************************************************
@ -546,10 +539,10 @@ unsigned Config::getMaxFileSystemDiskUsage()
******************************************************************************/
unsigned Config::getNumCompressedPadBlks()
{
boost::mutex::scoped_lock lk(fCacheLock);
checkReload( );
boost::mutex::scoped_lock lk(fCacheLock);
checkReload();
return m_NumCompressedPadBlks;
return m_NumCompressedPadBlks;
}
/*******************************************************************************
@ -560,10 +553,10 @@ unsigned Config::getNumCompressedPadBlks()
******************************************************************************/
bool Config::getParentOAMModuleFlag()
{
boost::mutex::scoped_lock lk(fCacheLock);
checkReload( );
boost::mutex::scoped_lock lk(fCacheLock);
checkReload();
return m_ParentOAMModuleFlag;
return m_ParentOAMModuleFlag;
}
/*******************************************************************************
@ -574,10 +567,10 @@ bool Config::getParentOAMModuleFlag()
******************************************************************************/
std::string Config::getLocalModuleType()
{
boost::mutex::scoped_lock lk(fCacheLock);
checkReload( );
boost::mutex::scoped_lock lk(fCacheLock);
checkReload();
return m_LocalModuleType;
return m_LocalModuleType;
}
/*******************************************************************************
@ -588,10 +581,10 @@ std::string Config::getLocalModuleType()
******************************************************************************/
uint16_t Config::getLocalModuleID()
{
boost::mutex::scoped_lock lk(fCacheLock);
checkReload( );
boost::mutex::scoped_lock lk(fCacheLock);
checkReload();
return m_LocalModuleID;
return m_LocalModuleID;
}
/*******************************************************************************
@ -604,10 +597,10 @@ uint16_t Config::getLocalModuleID()
******************************************************************************/
std::string Config::getVBRoot()
{
boost::mutex::scoped_lock lk(fCacheLock);
checkReload( );
boost::mutex::scoped_lock lk(fCacheLock);
checkReload();
return m_VersionBufferDir;
return m_VersionBufferDir;
}
/*******************************************************************************
@ -621,16 +614,15 @@ std::string Config::getVBRoot()
******************************************************************************/
bool Config::hasLocalDBRootListChanged()
{
boost::mutex::scoped_lock lk(fCacheLock);
boost::mutex::scoped_lock lk(fCacheLock);
if (fDBRootChangeCount > 0)
{
fDBRootChangeCount = 0;
return true;
}
if (fDBRootChangeCount > 0)
{
fDBRootChangeCount = 0;
return true;
}
return false;
return false;
}
} //end of namespace
} // namespace WriteEngine

249
writeengine/shared/we_config.h
View File

@ -16,9 +16,9 @@
MA 02110-1301, USA. */
/*******************************************************************************
* $Id: we_config.h 4726 2013-08-07 03:38:36Z bwilkinson $
*
*******************************************************************************/
* $Id: we_config.h 4726 2013-08-07 03:38:36Z bwilkinson $
*
*******************************************************************************/
/** @file */
#pragma once
@ -41,159 +41,160 @@
/** Namespace WriteEngine */
namespace WriteEngine
{
/** Class Config */
class Config
{
public:
/**
* @brief Constructor
*/
Config() {}
public:
/**
* @brief Constructor
*/
Config()
{
}
/**
* @brief Default Destructor
*/
~Config() {}
/**
* @brief Default Destructor
*/
~Config()
{
}
/**
* @brief Get DB root (for local PM)
* @param idx Index of the DBRootn entry to fetch (0 fetches DBRoot[0],etc.)
*/
EXPORT static std::string getDBRootByIdx(unsigned idx);
/**
* @brief Get DB root (for local PM)
* @param idx Index of the DBRootn entry to fetch (0 fetches DBRoot[0],etc.)
*/
EXPORT static std::string getDBRootByIdx(unsigned idx);
/**
* @brief Get complete DBRoot path list for the local PM
* @param dbRootPathList vector of DBRoot paths
*/
EXPORT static void getDBRootPathList(
std::vector<std::string>& dbRootPathList );
/**
* @brief Get complete DBRoot path list for the local PM
* @param dbRootPathList vector of DBRoot paths
*/
EXPORT static void getDBRootPathList(std::vector<std::string>& dbRootPathList);
/**
* @brief Get DB root (for local PM)
* @param num DBRootN entry to fetch (1 fetches DBRoot1, etc.)
*/
EXPORT static std::string getDBRootByNum(unsigned num);
/**
* @brief Get DB root (for local PM)
* @param num DBRootN entry to fetch (1 fetches DBRoot1, etc.)
*/
EXPORT static std::string getDBRootByNum(unsigned num);
/**
* @brief Get list of applicable DBRoot ids for this job.
*/
EXPORT static void getRootIdList( std::vector<uint16_t>& dbRootIds );
/**
* @brief Get list of applicable DBRoot ids for this job.
*/
EXPORT static void getRootIdList(std::vector<uint16_t>& dbRootIds);
#ifdef SHARED_NOTHING_DEMO_2
EXPORT static void getSharedNothingRoot(char*); // pass in an char[FILE_NAME_SIZE]
EXPORT static void getSharedNothingRoot(char*); // pass in an char[FILE_NAME_SIZE]
#endif
/**
* @brief Bulkload DB root
*/
EXPORT static std::string getBulkRoot();
/**
* @brief Bulkload DB root
*/
EXPORT static std::string getBulkRoot();
/**
* @brief DBRoot count for local PM
*/
EXPORT static size_t DBRootCount();
/**
* @brief DBRoot count for local PM
*/
EXPORT static size_t DBRootCount();
/**
* @brief Wait Period
*/
EXPORT static int getWaitPeriod();
/**
* @brief Wait Period
*/
EXPORT static int getWaitPeriod();
/**
* @brief FilesPerColumnPartition
*/
EXPORT static unsigned getFilesPerColumnPartition();
/**
* @brief FilesPerColumnPartition
*/
EXPORT static unsigned getFilesPerColumnPartition();
/**
* @brief ExtentsPerSegmentFile
*/
EXPORT static unsigned getExtentsPerSegmentFile();
/**
* @brief ExtentsPerSegmentFile
*/
EXPORT static unsigned getExtentsPerSegmentFile();
/**
* @brief Process Priority for cpimport.bin
* Return value is in range -20..19 (highest...lowest, 0=normal)
*/
EXPORT static int getBulkProcessPriority();
/**
* @brief Process Priority for cpimport.bin
* Return value is in range -20..19 (highest...lowest, 0=normal)
*/
EXPORT static int getBulkProcessPriority();
/**
* @brief Directory carrying Bulk Rollback meta data files
*/
EXPORT static std::string getBulkRollbackDir();
/**
* @brief Directory carrying Bulk Rollback meta data files
*/
EXPORT static std::string getBulkRollbackDir();
/**
* @brief Max percentage of allowable file system disk usage for each DBRoot
*/
EXPORT static unsigned getMaxFileSystemDiskUsage();
/**
* @brief Max percentage of allowable file system disk usage for each DBRoot
*/
EXPORT static unsigned getMaxFileSystemDiskUsage();
/**
* @brief Number of Blocks to pad each compressed chunk.
*/
EXPORT static unsigned getNumCompressedPadBlks();
/**
* @brief Number of Blocks to pad each compressed chunk.
*/
EXPORT static unsigned getNumCompressedPadBlks();
/**
* @brief Parent OAM Module flag (is this the parent OAM node, ex: pm1)
*/
EXPORT static bool getParentOAMModuleFlag();
/**
* @brief Parent OAM Module flag (is this the parent OAM node, ex: pm1)
*/
EXPORT static bool getParentOAMModuleFlag();
/**
* @brief Local Module Type (ex: "pm")
*/
EXPORT static std::string getLocalModuleType();
/**
* @brief Local Module Type (ex: "pm")
*/
EXPORT static std::string getLocalModuleType();
/**
* @brief Local Module ID (ex: 1 )
*/
EXPORT static uint16_t getLocalModuleID();
/**
* @brief Local Module ID (ex: 1 )
*/
EXPORT static uint16_t getLocalModuleID();
/**
* @brief Version Buffer root
*/
EXPORT static std::string getVBRoot();
/**
* @brief Version Buffer root
*/
EXPORT static std::string getVBRoot();
/**
* @brief Cache the config parameters locally
* Initialize Config cache. Cache will be updated as needed.
*/
EXPORT static void initConfigCache();
/**
* @brief Cache the config parameters locally
* Initialize Config cache. Cache will be updated as needed.
*/
EXPORT static void initConfigCache();
/**
* @brief Has Local PM DBRoot info changed since last time this function
* was called. Can be used to monitor changes to DBRoot info.
*/
EXPORT static bool hasLocalDBRootListChanged();
/**
* @brief Has Local PM DBRoot info changed since last time this function
* was called. Can be used to monitor changes to DBRoot info.
*/
EXPORT static bool hasLocalDBRootListChanged();
private:
typedef std::vector<std::string> strvec_t;
typedef std::map<int, std::string> intstrmap_t;
typedef std::vector<uint16_t> uint16vec_t;
private:
typedef std::vector<std::string> strvec_t;
typedef std::map<int, std::string> intstrmap_t;
typedef std::vector<uint16_t> uint16vec_t;
static void checkReload();
static void checkReload();
static int m_dbRootCount; // num DBRoots for local PM
static strvec_t m_dbRootPath; // root paths for open files
static intstrmap_t m_dbRootPathMap; // map of root id to root paths
static uint16vec_t m_dbRootId; // list of root ids
static std::string m_bulkRoot; // root path for bulk operation
static unsigned long fDBRootChangeCount; // track recent DBRoot changes
static time_t fCacheTime; // timestamp associated w/cache
static boost::mutex fCacheLock; // mutex for m_dbRoot sync
static int m_dbRootCount; // num DBRoots for local PM
static strvec_t m_dbRootPath; // root paths for open files
static intstrmap_t m_dbRootPathMap; // map of root id to root paths
static uint16vec_t m_dbRootId; // list of root ids
static std::string m_bulkRoot; // root path for bulk operation
static unsigned long fDBRootChangeCount; // track recent DBRoot changes
static time_t fCacheTime; // timestamp associated w/cache
static boost::mutex fCacheLock; // mutex for m_dbRoot sync
#ifdef SHARED_NOTHING_DEMO_2
static boost::mutex m_bulkRoot_lk; // mutex for m_bulkRoot sync
static boost::mutex m_bulkRoot_lk; // mutex for m_bulkRoot sync
#endif
static int m_WaitPeriod; // secs to wait for transaction
static unsigned m_FilesPerColumnPartition;//# seg files per partition
static unsigned m_ExtentsPerSegmentFile; // # extents per segment file
static int m_BulkProcessPriority; // cpimport.bin proc priority
static std::string m_BulkRollbackDir; // bulk rollback meta data dir
static unsigned m_MaxFileSystemDiskUsage;// max file system % disk usage
static unsigned m_NumCompressedPadBlks; // num blks to pad comp chunks
static bool m_ParentOAMModuleFlag; // are we running on parent PM
static std::string m_LocalModuleType; // local node type (ex: "pm")
static int m_LocalModuleID; // local node id (ex: 1 )
static std::string m_VersionBufferDir; // Version buffer directory
static int m_WaitPeriod; // secs to wait for transaction
static unsigned m_FilesPerColumnPartition; //# seg files per partition
static unsigned m_ExtentsPerSegmentFile; // # extents per segment file
static int m_BulkProcessPriority; // cpimport.bin proc priority
static std::string m_BulkRollbackDir; // bulk rollback meta data dir
static unsigned m_MaxFileSystemDiskUsage; // max file system % disk usage
static unsigned m_NumCompressedPadBlks; // num blks to pad comp chunks
static bool m_ParentOAMModuleFlag; // are we running on parent PM
static std::string m_LocalModuleType; // local node type (ex: "pm")
static int m_LocalModuleID; // local node id (ex: 1 )
static std::string m_VersionBufferDir; // Version buffer directory
};
} //end of namespace
} // namespace WriteEngine
#undef EXPORT

1217
writeengine/shared/we_confirmhdfsdbfile.cpp
View File

File diff suppressed because it is too large Load Diff

146
writeengine/shared/we_confirmhdfsdbfile.h
View File

@ -32,7 +32,6 @@
namespace WriteEngine
{
/** @brief Encapsulates logic to confirm and finalize (or abort) changes
* to an HDFS db file. Class should only be used for HDFS db files.
*
@ -46,92 +45,83 @@ namespace WriteEngine
*/
class ConfirmHdfsDbFile
{
public:
EXPORT ConfirmHdfsDbFile( );
EXPORT ~ConfirmHdfsDbFile( );
public:
EXPORT ConfirmHdfsDbFile();
EXPORT ~ConfirmHdfsDbFile();
/** @brief Confirm changes to the specified db file
* @param backUpFileType Backup file type to confirm. Types:
* "rlc" - reallocated chunk file
* "tmp" - updated db file
* @param filename Name of db file to be confirmed.
* @param errMsg (out) Error msg associated with a bad return code
* @return Returns NO_ERROR if call is successful
*/
EXPORT int confirmDbFileChange( const std::string& backUpFileType,
const std::string& filename,
std::string& errMsg ) const;
/** @brief Confirm changes to the specified db file
* @param backUpFileType Backup file type to confirm. Types:
* "rlc" - reallocated chunk file
* "tmp" - updated db file
* @param filename Name of db file to be confirmed.
* @param errMsg (out) Error msg associated with a bad return code
* @return Returns NO_ERROR if call is successful
*/
EXPORT int confirmDbFileChange(const std::string& backUpFileType, const std::string& filename,
std::string& errMsg) const;
/** @brief Finalize changes to the specified db file
*
* If success flag is true:
* The old version of the db file is deleted.
* If success flag is false:
* The old version is retained, and any temporary file with pending
* changes is deleted.
*
* @param backUpFileType Backup file type to finalize. Types:
* "rlc" - reallocated chunk file
* "tmp" - updated db file
* @param filename Name of db file to be finalized.
* @param success Final success/fail status of db file changes
* @param errMsg (out) Error msg associated with a bad return code
* @return Returns NO_ERROR if call is successful
*/
EXPORT int endDbFileChange( const std::string& backUpFileType,
const std::string& filename,
bool success,
std::string& errMsg ) const;
/** @brief Finalize changes to the specified db file
*
* If success flag is true:
* The old version of the db file is deleted.
* If success flag is false:
* The old version is retained, and any temporary file with pending
* changes is deleted.
*
* @param backUpFileType Backup file type to finalize. Types:
* "rlc" - reallocated chunk file
* "tmp" - updated db file
* @param filename Name of db file to be finalized.
* @param success Final success/fail status of db file changes
* @param errMsg (out) Error msg associated with a bad return code
* @return Returns NO_ERROR if call is successful
*/
EXPORT int endDbFileChange(const std::string& backUpFileType, const std::string& filename, bool success,
std::string& errMsg) const;
/** @brief Confirm changes to the db files modified for tableOID
*
* The HWM db file for each DBRoot, as listed in the bulk rollback meta
* data file (for the specified tableOID), is confirmed.
*
* @param tableOID Table that has changes to be confirmed
* @param errMsg (out) Error msg associated with a bad return code
* @return Returns NO_ERROR if call is successful
*/
EXPORT int confirmDbFileListFromMetaFile( OID tableOID,
std::string& errMsg );
/** @brief Confirm changes to the db files modified for tableOID
*
* The HWM db file for each DBRoot, as listed in the bulk rollback meta
* data file (for the specified tableOID), is confirmed.
*
* @param tableOID Table that has changes to be confirmed
* @param errMsg (out) Error msg associated with a bad return code
* @return Returns NO_ERROR if call is successful
*/
EXPORT int confirmDbFileListFromMetaFile(OID tableOID, std::string& errMsg);
/** @brief Finalize changes to the db files modified for tableOID
*
* The HWM db file for each DBRoot, as listed in the bulk rollback meta
* data file (for the specified tableOID), is finalized.
*
* If success flag is true:
* The old version of the db files are deleted.
* If success flag is false:
* The old versions are retained, and any temporary files with pending
* changes are deleted.
*
* @param tableOID Table that has changes to be confirmed
* @param errMsg (out) Error msg associated with a bad return code
* @return Returns NO_ERROR if call is successful
*/
EXPORT int endDbFileListFromMetaFile( OID tableOID,
bool success,
std::string& errMsg );
/** @brief Finalize changes to the db files modified for tableOID
*
* The HWM db file for each DBRoot, as listed in the bulk rollback meta
* data file (for the specified tableOID), is finalized.
*
* If success flag is true:
* The old version of the db files are deleted.
* If success flag is false:
* The old versions are retained, and any temporary files with pending
* changes are deleted.
*
* @param tableOID Table that has changes to be confirmed
* @param errMsg (out) Error msg associated with a bad return code
* @return Returns NO_ERROR if call is successful
*/
EXPORT int endDbFileListFromMetaFile(OID tableOID, bool success, std::string& errMsg);
private:
void openMetaDataFile( OID tableOID,
uint16_t dbRoot,
std::istringstream& metaDataStream );
private:
void openMetaDataFile(OID tableOID, uint16_t dbRoot, std::istringstream& metaDataStream);
void confirmDbFiles( std::istringstream& metaDataStream ) const;
void confirmColumnDbFile( const char* inBuf ) const;
void confirmDctnryStoreDbFile( const char* inBuf ) const;
void confirmDbFiles(std::istringstream& metaDataStream) const;
void confirmColumnDbFile(const char* inBuf) const;
void confirmDctnryStoreDbFile(const char* inBuf) const;
void endDbFiles( std::istringstream& metaDataStream, bool success ) const;
void endColumnDbFile( const char* inBuf, bool success ) const;
void endDctnryStoreDbFile( const char* inBuf, bool success ) const;
void endDbFiles(std::istringstream& metaDataStream, bool success) const;
void endColumnDbFile(const char* inBuf, bool success) const;
void endDctnryStoreDbFile(const char* inBuf, bool success) const;
idbdatafile::IDBFileSystem& fFs;
std::string fMetaFileName;
idbdatafile::IDBFileSystem& fFs;
std::string fMetaFileName;
};
} // end of namespace
} // namespace WriteEngine
#undef EXPORT

1196
writeengine/shared/we_convertor.cpp
View File

File diff suppressed because it is too large Load Diff

177
writeengine/shared/we_convertor.h
View File

@ -16,9 +16,9 @@
MA 02110-1301, USA. */
/*******************************************************************************
* $Id: we_convertor.h 4726 2013-08-07 03:38:36Z bwilkinson $
*
*******************************************************************************/
* $Id: we_convertor.h 4726 2013-08-07 03:38:36Z bwilkinson $
*
*******************************************************************************/
/** @file */
#pragma once
@ -44,108 +44,103 @@
/** Namespace WriteEngine */
namespace WriteEngine
{
/** Class Convertor */
class Convertor
{
public:
/**
* @brief Default Constructor
*/
Convertor() {}
public:
/**
* @brief Default Constructor
*/
Convertor()
{
}
/**
* @brief Destructor
*/
~Convertor() {}
/**
* @brief Destructor
*/
~Convertor()
{
}
/**
* @brief Get date/time string based on current date and time
*/
EXPORT static const std::string getTimeStr();
/**
* @brief Get date/time string based on current date and time
*/
EXPORT static const std::string getTimeStr();
/**
* @brief Convert specified integer value to a string
*
* @param val Integer value to be converted to a string
*/
EXPORT static const std::string int2Str(int val);
/**
* @brief Convert specified integer value to a string
*
* @param val Integer value to be converted to a string
*/
EXPORT static const std::string int2Str(int val);
/**
* @brief Convert an oid to a full file name (with partition and segment
* being included in the filename). This is used for all column and
* dictionary store db files. If dealing with a version buffer file,
* a partition and segment number of 0 should be used.
*/
EXPORT static int oid2FileName(FID fid, char* fullFileName,
char dbDirName[][MAX_DB_DIR_NAME_SIZE],
uint32_t partition, uint16_t segment);
/**
* @brief Convert an oid to a full file name (with partition and segment
* being included in the filename). This is used for all column and
* dictionary store db files. If dealing with a version buffer file,
* a partition and segment number of 0 should be used.
*/
EXPORT static int oid2FileName(FID fid, char* fullFileName, char dbDirName[][MAX_DB_DIR_NAME_SIZE],
uint32_t partition, uint16_t segment);
EXPORT static int fileName2Oid(const std::string &fullFileName,
uint32_t &oid, uint32_t &partition,
uint32_t &segment);
/**
* @brief Convert specified errno to associated error msg string
*
* @param errNum System errno to be converted.
* @param errString Error msg string associated with the specified errno.
*/
EXPORT static void mapErrnoToString(int errNum, std::string& errString);
EXPORT static int fileName2Oid(const std::string& fullFileName, uint32_t& oid, uint32_t& partition,
uint32_t& segment);
/**
* @brief Convert specified errno to associated error msg string
*
* @param errNum System errno to be converted.
* @param errString Error msg string associated with the specified errno.
*/
EXPORT static void mapErrnoToString(int errNum, std::string& errString);
/**
* @brief Convert specified ColDataType to internal storage type (ColType)
*
* @param dataType Interface data-type
* @param internalType Internal data-type used for storing
*/
//BUG931
EXPORT static void convertColType(execplan::CalpontSystemCatalog::ColDataType dataType,
int colWidth, ColType& internalType,
bool isToken = false);
/**
* @brief Convert specified internal storage type (ColType) to
* ColDataType
*
* @param internalType Internal data-type used for storing
* @param dataType Interface data-type
*/
EXPORT static void convertWEColType(ColType internalType,
execplan::CalpontSystemCatalog::ColDataType& dataType);
/**
* @brief Convert specified ColDataType to internal storage type (ColType)
*
* @param dataType Interface data-type
* @param internalType Internal data-type used for storing
*/
// BUG931
EXPORT static void convertColType(execplan::CalpontSystemCatalog::ColDataType dataType, int colWidth,
ColType& internalType, bool isToken = false);
/**
* @brief Convert specified internal storage type (ColType) to
* ColDataType
*
* @param internalType Internal data-type used for storing
* @param dataType Interface data-type
*/
EXPORT static void convertWEColType(ColType internalType,
execplan::CalpontSystemCatalog::ColDataType& dataType);
/**
* @brief Convert interface column type to a internal column type.
* curStruct is interpreted as a ColStruct.
*/
EXPORT static void convertColType(ColStruct* curStruct);
/**
* @brief Convert interface column type to a internal column type.
* curStruct is interpreted as a ColStruct.
*/
EXPORT static void convertColType(ColStruct* curStruct);
/*
* @brief Get the correct width for a row
*/
EXPORT static int getCorrectRowWidth( execplan::CalpontSystemCatalog::ColDataType dataType, int width );
/*
* @brief Get the correct width for a row
*/
EXPORT static int getCorrectRowWidth(execplan::CalpontSystemCatalog::ColDataType dataType, int width);
/*
* @brief Convert a Decimal string to it's equivalent integer value.
* errno can be checked upon return to see if input value was
* out of range (ERANGE).
*
* field decimal string to be converted
* fieldLengh length of "field" in bytes
* scale decimal scale to be applied to value
*/
EXPORT static long long convertDecimalString ( const char* field,
int fieldLength,
int scale );
/*
* @brief Convert a Decimal string to it's equivalent integer value.
* errno can be checked upon return to see if input value was
* out of range (ERANGE).
*
* field decimal string to be converted
* fieldLengh length of "field" in bytes
* scale decimal scale to be applied to value
*/
EXPORT static long long convertDecimalString(const char* field, int fieldLength, int scale);
private:
struct dmFilePathArgs_t;
static int dmOid2FPath(uint32_t oid, uint32_t partition, uint32_t segment,
dmFilePathArgs_t* pArgs);
static int32_t dmFPath2Oid(const dmFilePathArgs_t& pArgs, uint32_t& oid,
uint32_t& partition, uint32_t& segment);
private:
struct dmFilePathArgs_t;
static int dmOid2FPath(uint32_t oid, uint32_t partition, uint32_t segment, dmFilePathArgs_t* pArgs);
static int32_t dmFPath2Oid(const dmFilePathArgs_t& pArgs, uint32_t& oid, uint32_t& partition,
uint32_t& segment);
};
} //end of namespace
} // namespace WriteEngine
#undef EXPORT

474
writeengine/shared/we_dbfileop.cpp
View File

@ -36,18 +36,19 @@ using namespace BRM;
namespace WriteEngine
{
/**
* Constructor
*/
DbFileOp::DbFileOp() : m_chunkManager(NULL)
{}
{
}
/**
* Default Destructor
*/
DbFileOp::~DbFileOp()
{}
{
}
/***********************************************************
* DESCRIPTION:
@ -59,24 +60,20 @@ DbFileOp::~DbFileOp()
***********************************************************/
int DbFileOp::flushCache()
{
BlockBuffer* curBuf;
BlockBuffer* curBuf;
if ( !Cache::getUseCache() )
return NO_ERROR;
for ( CacheMapIt it = Cache::m_writeList->begin();
it != Cache::m_writeList->end(); it++ )
{
curBuf = it->second;
RETURN_ON_ERROR( writeDBFile( (*curBuf).cb.file.pFile,
(*curBuf).block.data,
(*curBuf).block.lbid ) );
}
RETURN_ON_ERROR( Cache::flushCache() );
if (!Cache::getUseCache())
return NO_ERROR;
}
for (CacheMapIt it = Cache::m_writeList->begin(); it != Cache::m_writeList->end(); it++)
{
curBuf = it->second;
RETURN_ON_ERROR(writeDBFile((*curBuf).cb.file.pFile, (*curBuf).block.data, (*curBuf).block.lbid));
}
RETURN_ON_ERROR(Cache::flushCache());
return NO_ERROR;
}
/***********************************************************
* DESCRIPTION:
@ -93,14 +90,13 @@ int DbFileOp::flushCache()
* RETURN:
* none
***********************************************************/
void DbFileOp::getSubBlockEntry( unsigned char* blockBuf,
const int sbid, const int entryNo,
const int width, void* pStruct )
void DbFileOp::getSubBlockEntry(unsigned char* blockBuf, const int sbid, const int entryNo, const int width,
void* pStruct)
{
unsigned char* pBlock;
unsigned char* pBlock;
pBlock = blockBuf + BYTE_PER_SUBBLOCK * sbid + entryNo * MAX_COLUMN_BOUNDARY;
memcpy( pStruct, pBlock, width );
pBlock = blockBuf + BYTE_PER_SUBBLOCK * sbid + entryNo * MAX_COLUMN_BOUNDARY;
memcpy(pStruct, pBlock, width);
}
/***********************************************************
@ -114,89 +110,80 @@ void DbFileOp::getSubBlockEntry( unsigned char* blockBuf,
* NO_ERROR if success
* other number if something wrong
***********************************************************/
int DbFileOp::readDBFile( IDBDataFile* pFile,
unsigned char* readBuf,
const uint64_t lbid,
const bool isFbo )
int DbFileOp::readDBFile(IDBDataFile* pFile, unsigned char* readBuf, const uint64_t lbid, const bool isFbo)
{
long long fboOffset = 0;
long long fboOffset = 0;
if ( !isFbo )
{
RETURN_ON_ERROR( setFileOffsetBlock( pFile, lbid ) );
}
else
{
fboOffset = (lbid) * (long)BYTE_PER_BLOCK;
RETURN_ON_ERROR( setFileOffset( pFile, fboOffset ) );
}
if (!isFbo)
{
RETURN_ON_ERROR(setFileOffsetBlock(pFile, lbid));
}
else
{
fboOffset = (lbid) * (long)BYTE_PER_BLOCK;
RETURN_ON_ERROR(setFileOffset(pFile, fboOffset));
}
return readFile( pFile, readBuf, BYTE_PER_BLOCK );
return readFile(pFile, readBuf, BYTE_PER_BLOCK);
}
int DbFileOp::readDBFile( IDBDataFile* pFile,
DataBlock* block,
const uint64_t lbid,
const bool isFbo )
int DbFileOp::readDBFile(IDBDataFile* pFile, DataBlock* block, const uint64_t lbid, const bool isFbo)
{
block->dirty = false;
block->no = lbid;
block->dirty = false;
block->no = lbid;
Stats::incIoBlockRead();
Stats::incIoBlockRead();
return readDBFile( pFile, block->data, lbid, isFbo );
return readDBFile(pFile, block->data, lbid, isFbo);
}
int DbFileOp::readDBFile( CommBlock& cb,
unsigned char* readBuf,
const uint64_t lbid )
int DbFileOp::readDBFile(CommBlock& cb, unsigned char* readBuf, const uint64_t lbid)
{
CacheKey key;
CacheKey key;
if ( Cache::getUseCache() )
if (Cache::getUseCache())
{
if (Cache::cacheKeyExist(cb.file.oid, lbid))
{
if ( Cache::cacheKeyExist( cb.file.oid, lbid ) )
{
key = Cache::getCacheKey( cb.file.oid, lbid );
RETURN_ON_ERROR( Cache::loadCacheBlock( key, readBuf ) );
return NO_ERROR;
}
key = Cache::getCacheKey(cb.file.oid, lbid);
RETURN_ON_ERROR(Cache::loadCacheBlock(key, readBuf));
return NO_ERROR;
}
}
RETURN_ON_ERROR(readDBFile(cb.file.pFile, readBuf, lbid));
if (Cache::getUseCache())
{
int fbo = lbid;
uint16_t dbRoot;
uint32_t partition;
uint16_t segment;
RETURN_ON_ERROR(BRMWrapper::getInstance()->getFboOffset(lbid, dbRoot, partition, segment, fbo));
if (Cache::getListSize(FREE_LIST) == 0)
{
if (isDebug(DEBUG_1))
{
printf("\nBefore flushing cache ");
Cache::printCacheList();
}
// flush cache to give up more space
RETURN_ON_ERROR(flushCache());
if (isDebug(DEBUG_1))
{
printf("\nAfter flushing cache ");
Cache::printCacheList();
}
}
RETURN_ON_ERROR( readDBFile( cb.file.pFile, readBuf, lbid ) );
RETURN_ON_ERROR(Cache::insertLRUList(cb, lbid, fbo, readBuf));
}
if ( Cache::getUseCache() )
{
int fbo = lbid;
uint16_t dbRoot;
uint32_t partition;
uint16_t segment;
RETURN_ON_ERROR( BRMWrapper::getInstance()->getFboOffset(
lbid, dbRoot, partition, segment, fbo ) );
if ( Cache::getListSize( FREE_LIST ) == 0 )
{
if ( isDebug( DEBUG_1 ) )
{
printf( "\nBefore flushing cache " );
Cache::printCacheList();
}
// flush cache to give up more space
RETURN_ON_ERROR( flushCache() );
if ( isDebug( DEBUG_1 ) )
{
printf( "\nAfter flushing cache " );
Cache::printCacheList();
}
}
RETURN_ON_ERROR( Cache::insertLRUList( cb, lbid, fbo, readBuf ) );
}
return NO_ERROR;
return NO_ERROR;
}
/***********************************************************
@ -217,30 +204,24 @@ int DbFileOp::readDBFile( CommBlock& cb,
* NO_ERROR if success
* other number if something wrong
***********************************************************/
int DbFileOp::readSubBlockEntry( IDBDataFile* pFile, DataBlock* block,
const uint64_t lbid, const int sbid,
const int entryNo, const int width,
void* pStruct )
int DbFileOp::readSubBlockEntry(IDBDataFile* pFile, DataBlock* block, const uint64_t lbid, const int sbid,
const int entryNo, const int width, void* pStruct)
{
RETURN_ON_ERROR( readDBFile( pFile, block->data, lbid ) );
getSubBlockEntry( block->data, sbid, entryNo, width, pStruct );
RETURN_ON_ERROR(readDBFile(pFile, block->data, lbid));
getSubBlockEntry(block->data, sbid, entryNo, width, pStruct);
return NO_ERROR;
return NO_ERROR;
}
int DbFileOp::readSubBlockEntry( CommBlock& cb, DataBlock* block,
const uint64_t lbid, const int sbid,
const int entryNo, const int width,
void* pStruct )
int DbFileOp::readSubBlockEntry(CommBlock& cb, DataBlock* block, const uint64_t lbid, const int sbid,
const int entryNo, const int width, void* pStruct)
{
RETURN_ON_ERROR( readDBFile( cb, block->data, lbid ) );
getSubBlockEntry( block->data, sbid, entryNo, width, pStruct );
RETURN_ON_ERROR(readDBFile(cb, block->data, lbid));
getSubBlockEntry(block->data, sbid, entryNo, width, pStruct);
return NO_ERROR;
return NO_ERROR;
}
/***********************************************************
* DESCRIPTION:
* Set an entry within a sub block
@ -256,14 +237,13 @@ int DbFileOp::readSubBlockEntry( CommBlock& cb, DataBlock* block,
* RETURN:
* none
***********************************************************/
void DbFileOp::setSubBlockEntry( unsigned char* blockBuf, const int sbid,
const int entryNo, const int width,
const void* pStruct )
void DbFileOp::setSubBlockEntry(unsigned char* blockBuf, const int sbid, const int entryNo, const int width,
const void* pStruct)
{
unsigned char* pBlock;
unsigned char* pBlock;
pBlock = blockBuf + BYTE_PER_SUBBLOCK * sbid + entryNo * MAX_COLUMN_BOUNDARY;
memcpy( pBlock, pStruct, width );
pBlock = blockBuf + BYTE_PER_SUBBLOCK * sbid + entryNo * MAX_COLUMN_BOUNDARY;
memcpy(pBlock, pStruct, width);
}
/***********************************************************
@ -278,47 +258,45 @@ void DbFileOp::setSubBlockEntry( unsigned char* blockBuf, const int sbid,
* NO_ERROR if success
* other number if something wrong
***********************************************************/
int DbFileOp::writeDBFile( CommBlock& cb, const unsigned char* writeBuf,
const uint64_t lbid, const int numOfBlock )
int DbFileOp::writeDBFile(CommBlock& cb, const unsigned char* writeBuf, const uint64_t lbid,
const int numOfBlock)
{
CacheKey key;
int ret;
if ( Cache::getUseCache() )
CacheKey key;
int ret;
if (Cache::getUseCache())
{
if (Cache::cacheKeyExist(cb.file.oid, lbid))
{
if ( Cache::cacheKeyExist( cb.file.oid, lbid ) )
{
key = Cache::getCacheKey( cb.file.oid, lbid );
RETURN_ON_ERROR( Cache::modifyCacheBlock( key, writeBuf ) );
return NO_ERROR;
}
key = Cache::getCacheKey(cb.file.oid, lbid);
RETURN_ON_ERROR(Cache::modifyCacheBlock(key, writeBuf));
return NO_ERROR;
}
}
if (BRMWrapper::getUseVb())
{
RETURN_ON_ERROR( writeVB( cb.file.pFile, cb.file.oid, lbid ) );
}
if (BRMWrapper::getUseVb())
{
RETURN_ON_ERROR(writeVB(cb.file.pFile, cb.file.oid, lbid));
}
ret = writeDBFile( cb.file.pFile, writeBuf, lbid, numOfBlock );
ret = writeDBFile(cb.file.pFile, writeBuf, lbid, numOfBlock);
if (BRMWrapper::getUseVb())
{
LBIDRange_v ranges;
LBIDRange range;
range.start = lbid;
range.size = 1;
ranges.push_back(range);
BRMWrapper::getInstance()->writeVBEnd(getTransId(), ranges);
}
if (BRMWrapper::getUseVb())
{
LBIDRange_v ranges;
LBIDRange range;
range.start = lbid;
range.size = 1;
ranges.push_back(range);
BRMWrapper::getInstance()->writeVBEnd(getTransId(), ranges);
}
return ret;
return ret;
}
int DbFileOp::writeDBFileNoVBCache(CommBlock& cb,
const unsigned char* writeBuf,
const int fbo,
int DbFileOp::writeDBFileNoVBCache(CommBlock& cb, const unsigned char* writeBuf, const int fbo,
const int numOfBlock)
{
return writeDBFileNoVBCache( cb.file.pFile, writeBuf, fbo, numOfBlock );
return writeDBFileNoVBCache(cb.file.pFile, writeBuf, fbo, numOfBlock);
}
/***********************************************************
@ -326,65 +304,63 @@ int DbFileOp::writeDBFileNoVBCache(CommBlock& cb,
* Core function for writing data w/o using VB cache
* (bulk load dictionary store inserts)
***********************************************************/
int DbFileOp::writeDBFileNoVBCache( IDBDataFile* pFile,
const unsigned char* writeBuf,
const int fbo,
const int numOfBlock )
int DbFileOp::writeDBFileNoVBCache(IDBDataFile* pFile, const unsigned char* writeBuf, const int fbo,
const int numOfBlock)
{
#ifdef PROFILE
// This function is only used by bulk load for dictionary store files,
// so we log as such.
Stats::startParseEvent(WE_STATS_WRITE_DCT);
// This function is only used by bulk load for dictionary store files,
// so we log as such.
Stats::startParseEvent(WE_STATS_WRITE_DCT);
#endif
for ( int i = 0; i < numOfBlock; i++ )
{
Stats::incIoBlockWrite();
RETURN_ON_ERROR( writeFile( pFile, writeBuf, BYTE_PER_BLOCK ) );
}
for (int i = 0; i < numOfBlock; i++)
{
Stats::incIoBlockWrite();
RETURN_ON_ERROR(writeFile(pFile, writeBuf, BYTE_PER_BLOCK));
}
#ifdef PROFILE
Stats::stopParseEvent(WE_STATS_WRITE_DCT);
Stats::stopParseEvent(WE_STATS_WRITE_DCT);
#endif
return NO_ERROR;
return NO_ERROR;
}
/***********************************************************
* DESCRIPTION:
* Core function for writing data using VB cache
***********************************************************/
int DbFileOp::writeDBFile( IDBDataFile* pFile, const unsigned char* writeBuf,
const uint64_t lbid, const int numOfBlock )
int DbFileOp::writeDBFile(IDBDataFile* pFile, const unsigned char* writeBuf, const uint64_t lbid,
const int numOfBlock)
{
RETURN_ON_ERROR( setFileOffsetBlock( pFile, lbid ) );
RETURN_ON_ERROR(setFileOffsetBlock(pFile, lbid));
for ( int i = 0; i < numOfBlock; i++ )
{
Stats::incIoBlockWrite();
RETURN_ON_ERROR( writeFile( pFile, writeBuf, BYTE_PER_BLOCK ) );
}
for (int i = 0; i < numOfBlock; i++)
{
Stats::incIoBlockWrite();
RETURN_ON_ERROR(writeFile(pFile, writeBuf, BYTE_PER_BLOCK));
}
return NO_ERROR;
return NO_ERROR;
}
// just don't have a good solution to consolidate with above functions
// Note: This is used with absolute FBO, no lbid involved
int DbFileOp::writeDBFileFbo(IDBDataFile* pFile, const unsigned char* writeBuf,
const uint64_t fbo, const int numOfBlock )
int DbFileOp::writeDBFileFbo(IDBDataFile* pFile, const unsigned char* writeBuf, const uint64_t fbo,
const int numOfBlock)
{
long long fboOffset = 0;
long long fboOffset = 0;
fboOffset = (fbo) * (long)BYTE_PER_BLOCK;
RETURN_ON_ERROR( setFileOffset( pFile, fboOffset ) );
fboOffset = (fbo) * (long)BYTE_PER_BLOCK;
RETURN_ON_ERROR(setFileOffset(pFile, fboOffset));
for ( int i = 0; i < numOfBlock; i++ )
{
Stats::incIoBlockWrite();
RETURN_ON_ERROR( writeFile( pFile, writeBuf, BYTE_PER_BLOCK ) );
}
for (int i = 0; i < numOfBlock; i++)
{
Stats::incIoBlockWrite();
RETURN_ON_ERROR(writeFile(pFile, writeBuf, BYTE_PER_BLOCK));
}
return NO_ERROR;
return NO_ERROR;
}
/***********************************************************
@ -405,26 +381,22 @@ int DbFileOp::writeDBFileFbo(IDBDataFile* pFile, const unsigned char* writeBuf,
* NO_ERROR if success
* other number if something wrong
***********************************************************/
int DbFileOp::writeSubBlockEntry( IDBDataFile* pFile, DataBlock* block,
const uint64_t lbid, const int sbid,
const int entryNo, const int width,
void* pStruct )
int DbFileOp::writeSubBlockEntry(IDBDataFile* pFile, DataBlock* block, const uint64_t lbid, const int sbid,
const int entryNo, const int width, void* pStruct)
{
setSubBlockEntry( block->data, sbid, entryNo, width, pStruct );
block->dirty = false;
setSubBlockEntry(block->data, sbid, entryNo, width, pStruct);
block->dirty = false;
return writeDBFile( pFile, block->data, lbid );
return writeDBFile(pFile, block->data, lbid);
}
int DbFileOp::writeSubBlockEntry( CommBlock& cb, DataBlock* block,
const uint64_t lbid, const int sbid,
const int entryNo, const int width,
void* pStruct )
int DbFileOp::writeSubBlockEntry(CommBlock& cb, DataBlock* block, const uint64_t lbid, const int sbid,
const int entryNo, const int width, void* pStruct)
{
setSubBlockEntry( block->data, sbid, entryNo, width, pStruct );
block->dirty = false;
setSubBlockEntry(block->data, sbid, entryNo, width, pStruct);
block->dirty = false;
return writeDBFile( cb, block->data, lbid );
return writeDBFile(cb, block->data, lbid);
}
/***********************************************************
@ -437,89 +409,77 @@ int DbFileOp::writeSubBlockEntry( CommBlock& cb, DataBlock* block,
* NO_ERROR if success
* other number if something wrong
***********************************************************/
int DbFileOp::writeVB( IDBDataFile* pFile, const OID oid, const uint64_t lbid )
int DbFileOp::writeVB(IDBDataFile* pFile, const OID oid, const uint64_t lbid)
{
if ( !BRMWrapper::getUseVb() )
return NO_ERROR;
int rc;
const TxnID transId = getTransId();
if (transId != ((TxnID)INVALID_NUM))
{
rc = BRMWrapper::getInstance()->writeVB( pFile,
(VER_t)transId,
oid, lbid, this );
//@Bug 4671. The error is already logged by worker node.
/* if (rc != NO_ERROR)
{
char msg[2048];
snprintf(msg, 2048,
"we_dbfileop->BRMWrapper::getInstance()->writeVB "
"transId %i oid %i lbid "
#if __LP64__
"%lu"
#else
"%llu"
#endif
" Error Code %i", transId, oid, lbid, rc);
puts(msg);
{
logging::MessageLog ml(logging::LoggingID(19));
logging::Message m;
logging::Message::Args args;
args.add(msg);
m.format(args);
ml.logCriticalMessage(m);
}
return rc;
} */
return rc;
}
if (!BRMWrapper::getUseVb())
return NO_ERROR;
int rc;
const TxnID transId = getTransId();
if (transId != ((TxnID)INVALID_NUM))
{
rc = BRMWrapper::getInstance()->writeVB(pFile, (VER_t)transId, oid, lbid, this);
//@Bug 4671. The error is already logged by worker node.
/* if (rc != NO_ERROR)
{
char msg[2048];
snprintf(msg, 2048,
"we_dbfileop->BRMWrapper::getInstance()->writeVB "
"transId %i oid %i lbid "
#if __LP64__
"%lu"
#else
"%llu"
#endif
" Error Code %i", transId, oid, lbid, rc);
puts(msg);
{
logging::MessageLog ml(logging::LoggingID(19));
logging::Message m;
logging::Message::Args args;
args.add(msg);
m.format(args);
ml.logCriticalMessage(m);
}
return rc;
} */
return rc;
}
return NO_ERROR;
}
int DbFileOp::readDbBlocks(IDBDataFile* pFile,
unsigned char* readBuf,
uint64_t fbo,
size_t n)
int DbFileOp::readDbBlocks(IDBDataFile* pFile, unsigned char* readBuf, uint64_t fbo, size_t n)
{
if (m_chunkManager)
{
return m_chunkManager->readBlocks(pFile, readBuf, fbo, n);
}
if (m_chunkManager)
{
return m_chunkManager->readBlocks(pFile, readBuf, fbo, n);
}
if (setFileOffset(pFile, fbo * BYTE_PER_BLOCK, SEEK_SET) != NO_ERROR)
return -1;
if (setFileOffset(pFile, fbo * BYTE_PER_BLOCK, SEEK_SET) != NO_ERROR)
return -1;
return pFile->read(readBuf, BYTE_PER_BLOCK * n) / BYTE_PER_BLOCK;
return pFile->read(readBuf, BYTE_PER_BLOCK * n) / BYTE_PER_BLOCK;
}
int DbFileOp::restoreBlock(IDBDataFile* pFile, const unsigned char* writeBuf, uint64_t fbo)
{
if (m_chunkManager)
return m_chunkManager->restoreBlock(pFile, writeBuf, fbo);
if (m_chunkManager)
return m_chunkManager->restoreBlock(pFile, writeBuf, fbo);
if (setFileOffset(pFile, fbo * BYTE_PER_BLOCK, SEEK_SET) != NO_ERROR)
return -1;
if (setFileOffset(pFile, fbo * BYTE_PER_BLOCK, SEEK_SET) != NO_ERROR)
return -1;
return pFile->write(writeBuf, BYTE_PER_BLOCK);
return pFile->write(writeBuf, BYTE_PER_BLOCK);
}
// @bug 5572 - HDFS usage: add *.tmp file backup flag
IDBDataFile* DbFileOp::getFilePtr(const Column& column, bool useTmpSuffix)
{
string filename;
return m_chunkManager->getFilePtr(column,
column.dataFile.fDbRoot,
column.dataFile.fPartition,
column.dataFile.fSegment,
filename,
"r+b",
column.colWidth,
useTmpSuffix);
string filename;
return m_chunkManager->getFilePtr(column, column.dataFile.fDbRoot, column.dataFile.fPartition,
column.dataFile.fSegment, filename, "r+b", column.colWidth, useTmpSuffix);
}
} //end of namespace
} // namespace WriteEngine

291
writeengine/shared/we_dbfileop.h
View File

@ -26,7 +26,6 @@
#include "we_blockop.h"
#include "we_cache.h"
#if defined(_MSC_VER) && defined(WRITEENGINE_DLLEXPORT)
#define EXPORT __declspec(dllexport)
#else
@ -36,217 +35,147 @@
/** Namespace WriteEngine */
namespace WriteEngine
{
// forward reference
class ChunkManager;
/** Class DbFileOp */
class DbFileOp : public FileOp
{
public:
/**
* @brief Constructor
*/
EXPORT DbFileOp();
public:
/**
* @brief Constructor
*/
EXPORT DbFileOp();
/**
* @brief Default Destructor
*/
EXPORT virtual ~DbFileOp();
/**
* @brief Default Destructor
*/
EXPORT virtual ~DbFileOp();
EXPORT virtual int flushCache();
EXPORT virtual int flushCache();
/**
* @brief Get an entry within a subblock
*/
EXPORT void getSubBlockEntry( unsigned char* blockBuf,
const int sbid,
const int entryNo,
const int width,
void* pStruct ) ;
/**
* @brief Get an entry within a subblock
*/
EXPORT void getSubBlockEntry(unsigned char* blockBuf, const int sbid, const int entryNo, const int width,
void* pStruct);
/**
* @brief Get an entry within a subblock using block information
*/
void getSubBlockEntry( DataBlock* block,
const int sbid,
const int entryNo,
const int width,
void* pStruct )
{
getSubBlockEntry( block->data, sbid, entryNo, width, pStruct );
}
/**
* @brief Get an entry within a subblock using block information
*/
void getSubBlockEntry(DataBlock* block, const int sbid, const int entryNo, const int width, void* pStruct)
{
getSubBlockEntry(block->data, sbid, entryNo, width, pStruct);
}
/**
* @brief Read DB file to a buffer
*/
EXPORT virtual int readDBFile( IDBDataFile* pFile,
unsigned char* readBuf,
const uint64_t lbid,
const bool isFbo = false );
EXPORT int readDBFile( CommBlock& cb,
unsigned char* readBuf,
const uint64_t lbid );
/**
* @brief Read DB file to a buffer
*/
EXPORT virtual int readDBFile(IDBDataFile* pFile, unsigned char* readBuf, const uint64_t lbid,
const bool isFbo = false);
EXPORT int readDBFile(CommBlock& cb, unsigned char* readBuf, const uint64_t lbid);
EXPORT int readDBFile( IDBDataFile* pFile,
DataBlock* block,
const uint64_t lbid,
const bool isFbo = false );
int readDBFile( CommBlock& cb,
DataBlock* block,
const uint64_t lbid )
{
return readDBFile( cb, block->data, lbid );
}
EXPORT int readDBFile(IDBDataFile* pFile, DataBlock* block, const uint64_t lbid, const bool isFbo = false);
int readDBFile(CommBlock& cb, DataBlock* block, const uint64_t lbid)
{
return readDBFile(cb, block->data, lbid);
}
/**
* @brief Get an entry within a subblock and also populate block buffer
*
*/
EXPORT int readSubBlockEntry(IDBDataFile* pFile,
DataBlock* block,
const uint64_t lbid,
const int sbid,
const int entryNo,
const int width,
void* pStruct ) ;
/**
* @brief Get an entry within a subblock and also populate block buffer
*
*/
EXPORT int readSubBlockEntry(IDBDataFile* pFile, DataBlock* block, const uint64_t lbid, const int sbid,
const int entryNo, const int width, void* pStruct);
EXPORT int readSubBlockEntry(CommBlock& cb,
DataBlock* block,
const uint64_t lbid,
const int sbid,
const int entryNo,
const int width,
void* pStruct );
EXPORT int readSubBlockEntry(CommBlock& cb, DataBlock* block, const uint64_t lbid, const int sbid,
const int entryNo, const int width, void* pStruct);
/**
* @brief Set an entry within a subblock
*/
EXPORT void setSubBlockEntry( unsigned char* blockBuf,
const int sbid,
const int entryNo,
const int width,
const void* pStruct ) ;
/**
* @brief Set an entry within a subblock
*/
EXPORT void setSubBlockEntry(unsigned char* blockBuf, const int sbid, const int entryNo, const int width,
const void* pStruct);
/**
* @brief Set an entry within a subblock using block information
*/
void setSubBlockEntry( DataBlock* block,
const int sbid,
const int entryNo,
const int width,
const void* pStruct )
{
block->dirty = true;
setSubBlockEntry( block->data, sbid, entryNo, width, pStruct );
}
/**
* @brief Set an entry within a subblock using block information
*/
void setSubBlockEntry(DataBlock* block, const int sbid, const int entryNo, const int width,
const void* pStruct)
{
block->dirty = true;
setSubBlockEntry(block->data, sbid, entryNo, width, pStruct);
}
/**
* @brief Lbid Write a buffer to a DB file
*/
EXPORT virtual int writeDBFile( IDBDataFile* pFile,
const unsigned char* writeBuf,
const uint64_t lbid,
const int numOfBlock = 1 );
EXPORT int writeDBFile( CommBlock& cb,
const unsigned char* writeBuf,
const uint64_t lbid,
const int numOfBlock = 1 );
/**
* @brief Lbid Write a buffer to a DB file
*/
EXPORT virtual int writeDBFile(IDBDataFile* pFile, const unsigned char* writeBuf, const uint64_t lbid,
const int numOfBlock = 1);
EXPORT int writeDBFile(CommBlock& cb, const unsigned char* writeBuf, const uint64_t lbid,
const int numOfBlock = 1);
/**
* @brief Write designated block(s) w/o writing to Version Buffer or cache.
*/
EXPORT int writeDBFileNoVBCache(CommBlock& cb,
const unsigned char* writeBuf,
const int fbo,
const int numOfBlock = 1);
EXPORT virtual int writeDBFileNoVBCache(IDBDataFile* pFile,
const unsigned char* writeBuf,
const int fbo,
const int numOfBlock = 1);
/**
* @brief Write designated block(s) w/o writing to Version Buffer or cache.
*/
EXPORT int writeDBFileNoVBCache(CommBlock& cb, const unsigned char* writeBuf, const int fbo,
const int numOfBlock = 1);
EXPORT virtual int writeDBFileNoVBCache(IDBDataFile* pFile, const unsigned char* writeBuf, const int fbo,
const int numOfBlock = 1);
int writeDBFile( IDBDataFile* pFile,
DataBlock* block,
const uint64_t lbid )
{
block->dirty = false;
return writeDBFile( pFile, block->data, lbid );
}
int writeDBFile( CommBlock& cb,
DataBlock* block,
const uint64_t lbid )
{
return writeDBFile( cb, block->data, lbid );
}
int writeDBFile(IDBDataFile* pFile, DataBlock* block, const uint64_t lbid)
{
block->dirty = false;
return writeDBFile(pFile, block->data, lbid);
}
int writeDBFile(CommBlock& cb, DataBlock* block, const uint64_t lbid)
{
return writeDBFile(cb, block->data, lbid);
}
EXPORT virtual int writeDBFileFbo( IDBDataFile* pFile,
const unsigned char* writeBuf,
const uint64_t fbo,
const int numOfBlock );
EXPORT virtual int writeDBFileFbo(IDBDataFile* pFile, const unsigned char* writeBuf, const uint64_t fbo,
const int numOfBlock);
int writeDBFileNoVBCache(CommBlock& cb,
DataBlock* block,
const int fbo)
{
return writeDBFileNoVBCache(cb, block->data, fbo);
}
int writeDBFileNoVBCache(CommBlock& cb, DataBlock* block, const int fbo)
{
return writeDBFileNoVBCache(cb, block->data, fbo);
}
/**
* @brief Write a sub block entry directly to a DB file
*/
EXPORT int writeSubBlockEntry(IDBDataFile* pFile,
DataBlock* block,
const uint64_t lbid,
const int sbid,
const int entryNo,
const int width,
void* pStruct );
/**
* @brief Write a sub block entry directly to a DB file
*/
EXPORT int writeSubBlockEntry(IDBDataFile* pFile, DataBlock* block, const uint64_t lbid, const int sbid,
const int entryNo, const int width, void* pStruct);
EXPORT int writeSubBlockEntry(CommBlock& cb,
DataBlock* block,
const uint64_t lbid,
const int sbid,
const int entryNo,
const int width,
void* pStruct ) ;
EXPORT int writeSubBlockEntry(CommBlock& cb, DataBlock* block, const uint64_t lbid, const int sbid,
const int entryNo, const int width, void* pStruct);
/**
* @brief Write to version buffer
*/
EXPORT int writeVB( IDBDataFile* pFile,
const OID oid,
const uint64_t lbid );
/**
* @brief Write to version buffer
*/
EXPORT int writeVB(IDBDataFile* pFile, const OID oid, const uint64_t lbid);
EXPORT virtual int readDbBlocks( IDBDataFile* pFile,
unsigned char* readBuf,
uint64_t fbo,
size_t n);
EXPORT virtual int readDbBlocks(IDBDataFile* pFile, unsigned char* readBuf, uint64_t fbo, size_t n);
EXPORT virtual int restoreBlock( IDBDataFile* pFile,
const unsigned char* writeBuf,
uint64_t fbo);
EXPORT virtual int restoreBlock(IDBDataFile* pFile, const unsigned char* writeBuf, uint64_t fbo);
EXPORT virtual IDBDataFile* getFilePtr(const Column& column,
bool useTmpSuffix);
EXPORT virtual IDBDataFile* getFilePtr(const Column& column, bool useTmpSuffix);
virtual void chunkManager(ChunkManager* ptr)
{
m_chunkManager = ptr;
}
virtual ChunkManager* chunkManager()
{
return m_chunkManager;
}
virtual void chunkManager(ChunkManager* ptr)
{
m_chunkManager = ptr;
}
virtual ChunkManager* chunkManager()
{
return m_chunkManager;
}
protected:
ChunkManager* m_chunkManager;
private:
protected:
ChunkManager* m_chunkManager;
private:
};
} //end of namespace
} // namespace WriteEngine
#undef EXPORT

720
writeengine/shared/we_dbrootextenttracker.cpp
View File

@ -16,8 +16,8 @@
MA 02110-1301, USA. */
/*
* $Id: we_dbrootextenttracker.cpp 4631 2013-05-02 15:21:09Z dcathey $
*/
* $Id: we_dbrootextenttracker.cpp 4631 2013-05-02 15:21:09Z dcathey $
*/
#include "we_dbrootextenttracker.h"
#include <algorithm>
@ -29,199 +29,167 @@
namespace
{
const char* stateStrings[] = { "initState",
"PartialExtent",
"EmptyDbRoot",
"ExtentBoundary",
"OutOfService"
};
const char* stateStrings[] = {"initState", "PartialExtent", "EmptyDbRoot", "ExtentBoundary", "OutOfService"};
}
namespace WriteEngine
{
//------------------------------------------------------------------------------
// DBRootExtentInfo constructor
//------------------------------------------------------------------------------
DBRootExtentInfo::DBRootExtentInfo(
uint16_t dbRoot,
uint32_t partition,
uint16_t segment,
BRM::LBID_t startLbid,
HWM localHwm,
uint64_t dbrootTotalBlocks,
DBRootExtentInfoState state) :
fPartition(partition),
fDbRoot(dbRoot),
fSegment(segment),
fStartLbid(startLbid),
fLocalHwm(localHwm),
fDBRootTotalBlocks(dbrootTotalBlocks),
fState(state)
DBRootExtentInfo::DBRootExtentInfo(uint16_t dbRoot, uint32_t partition, uint16_t segment,
BRM::LBID_t startLbid, HWM localHwm, uint64_t dbrootTotalBlocks,
DBRootExtentInfoState state)
: fPartition(partition)
, fDbRoot(dbRoot)
, fSegment(segment)
, fStartLbid(startLbid)
, fLocalHwm(localHwm)
, fDBRootTotalBlocks(dbrootTotalBlocks)
, fState(state)
{
}
//------------------------------------------------------------------------------
// LessThan operator used to sort DBRootExtentInfo objects by DBRoot.
//------------------------------------------------------------------------------
bool DBRootExtentInfo::operator<(
const DBRootExtentInfo& entry) const
bool DBRootExtentInfo::operator<(const DBRootExtentInfo& entry) const
{
if (fDbRoot < entry.fDbRoot)
return true;
if (fDbRoot < entry.fDbRoot)
return true;
return false;
return false;
}
//------------------------------------------------------------------------------
// DBRootExtentTracker constructor
//
// Mutex lock not needed in this function as it is only called from main thread
// before processing threads are spawned.
//------------------------------------------------------------------------------
DBRootExtentTracker::DBRootExtentTracker ( OID oid,
const std::vector<int>& colWidths,
const std::vector<BRM::EmDbRootHWMInfo_v>& dbRootHWMInfoColVec,
unsigned int columnIdx,
Log* logger ) :
fOID(oid),
fLog(logger),
fCurrentDBRootIdx(-1),
fEmptyOrDisabledPM(false),
fEmptyPM(true),
fDisabledHWM(false)
DBRootExtentTracker::DBRootExtentTracker(OID oid, const std::vector<int>& colWidths,
const std::vector<BRM::EmDbRootHWMInfo_v>& dbRootHWMInfoColVec,
unsigned int columnIdx, Log* logger)
: fOID(oid)
, fLog(logger)
, fCurrentDBRootIdx(-1)
, fEmptyOrDisabledPM(false)
, fEmptyPM(true)
, fDisabledHWM(false)
{
const BRM::EmDbRootHWMInfo_v& emDbRootHWMInfo =
dbRootHWMInfoColVec[columnIdx];
int colWidth = colWidths[columnIdx];
const BRM::EmDbRootHWMInfo_v& emDbRootHWMInfo = dbRootHWMInfoColVec[columnIdx];
int colWidth = colWidths[columnIdx];
fBlksPerExtent = (long long)BRMWrapper::getInstance()->getExtentRows() *
(long long)colWidth / (long long)BYTE_PER_BLOCK;
fBlksPerExtent =
(long long)BRMWrapper::getInstance()->getExtentRows() * (long long)colWidth / (long long)BYTE_PER_BLOCK;
std::vector<bool> resetState;
std::vector<bool> resetState;
for (unsigned int i = 0; i < emDbRootHWMInfo.size(); i++)
for (unsigned int i = 0; i < emDbRootHWMInfo.size(); i++)
{
resetState.push_back(false);
DBRootExtentInfoState state = determineState(colWidths[columnIdx], emDbRootHWMInfo[i].localHWM,
emDbRootHWMInfo[i].totalBlocks, emDbRootHWMInfo[i].status);
// For a full extent...
// check to see if any of the column HWMs are partially full, in which
// case we consider all the columns for that DBRoot to be partially
// full. (This can happen if a table has columns with varying widths,
// as the HWM may be at the last extent block for a shorter column, and
// still have free blocks for wider columns.)
if (state == DBROOT_EXTENT_EXTENT_BOUNDARY)
{
resetState.push_back(false);
DBRootExtentInfoState state = determineState(
colWidths[columnIdx],
emDbRootHWMInfo[i].localHWM,
emDbRootHWMInfo[i].totalBlocks,
emDbRootHWMInfo[i].status);
for (unsigned int kCol = 0; kCol < dbRootHWMInfoColVec.size(); kCol++)
{
const BRM::EmDbRootHWMInfo_v& emDbRootHWMInfo2 = dbRootHWMInfoColVec[kCol];
DBRootExtentInfoState state2 =
determineState(colWidths[kCol], emDbRootHWMInfo2[i].localHWM, emDbRootHWMInfo2[i].totalBlocks,
emDbRootHWMInfo2[i].status);
// For a full extent...
// check to see if any of the column HWMs are partially full, in which
// case we consider all the columns for that DBRoot to be partially
// full. (This can happen if a table has columns with varying widths,
// as the HWM may be at the last extent block for a shorter column, and
// still have free blocks for wider columns.)
if (state == DBROOT_EXTENT_EXTENT_BOUNDARY)
if (state2 == DBROOT_EXTENT_PARTIAL_EXTENT)
{
for (unsigned int kCol = 0; kCol < dbRootHWMInfoColVec.size(); kCol++)
{
const BRM::EmDbRootHWMInfo_v& emDbRootHWMInfo2 =
dbRootHWMInfoColVec[kCol];
DBRootExtentInfoState state2 = determineState(
colWidths[kCol],
emDbRootHWMInfo2[i].localHWM,
emDbRootHWMInfo2[i].totalBlocks,
emDbRootHWMInfo2[i].status);
if (state2 == DBROOT_EXTENT_PARTIAL_EXTENT)
{
state = DBROOT_EXTENT_PARTIAL_EXTENT;
resetState[ resetState.size() - 1 ] = true;
break;
}
}
state = DBROOT_EXTENT_PARTIAL_EXTENT;
resetState[resetState.size() - 1] = true;
break;
}
DBRootExtentInfo dbRootExtent(
emDbRootHWMInfo[i].dbRoot,
emDbRootHWMInfo[i].partitionNum,
emDbRootHWMInfo[i].segmentNum,
emDbRootHWMInfo[i].startLbid,
emDbRootHWMInfo[i].localHWM,
emDbRootHWMInfo[i].totalBlocks,
state);
fDBRootExtentList.push_back( dbRootExtent );
}
}
std::sort( fDBRootExtentList.begin(), fDBRootExtentList.end() );
DBRootExtentInfo dbRootExtent(emDbRootHWMInfo[i].dbRoot, emDbRootHWMInfo[i].partitionNum,
emDbRootHWMInfo[i].segmentNum, emDbRootHWMInfo[i].startLbid,
emDbRootHWMInfo[i].localHWM, emDbRootHWMInfo[i].totalBlocks, state);
if (fLog)
fDBRootExtentList.push_back(dbRootExtent);
}
std::sort(fDBRootExtentList.begin(), fDBRootExtentList.end());
if (fLog)
{
// Always log this info for now; may control with debug later
// if (fLog->isDebug(DEBUG_1))
{
// Always log this info for now; may control with debug later
//if (fLog->isDebug(DEBUG_1))
{
std::ostringstream oss;
oss << "Starting DBRoot info for OID " << fOID;
std::ostringstream oss;
oss << "Starting DBRoot info for OID " << fOID;
for (unsigned int k = 0; k < fDBRootExtentList.size(); k++)
{
oss << std::endl;
oss << " DBRoot-" << fDBRootExtentList[k].fDbRoot <<
", part/seg/hwm/LBID/totBlks/state: " <<
fDBRootExtentList[k].fPartition <<
"/" << fDBRootExtentList[k].fSegment <<
"/" << fDBRootExtentList[k].fLocalHwm <<
"/" << fDBRootExtentList[k].fStartLbid <<
"/" << fDBRootExtentList[k].fDBRootTotalBlocks <<
"/" << stateStrings[ fDBRootExtentList[k].fState ];
for (unsigned int k = 0; k < fDBRootExtentList.size(); k++)
{
oss << std::endl;
oss << " DBRoot-" << fDBRootExtentList[k].fDbRoot
<< ", part/seg/hwm/LBID/totBlks/state: " << fDBRootExtentList[k].fPartition << "/"
<< fDBRootExtentList[k].fSegment << "/" << fDBRootExtentList[k].fLocalHwm << "/"
<< fDBRootExtentList[k].fStartLbid << "/" << fDBRootExtentList[k].fDBRootTotalBlocks << "/"
<< stateStrings[fDBRootExtentList[k].fState];
if (resetState[k])
oss << ".";
}
if (resetState[k])
oss << ".";
}
fLog->logMsg( oss.str(), MSGLVL_INFO2 );
}
fLog->logMsg(oss.str(), MSGLVL_INFO2);
}
}
}
//------------------------------------------------------------------------------
// Determines the state of the HWM extent (for a DBRoot); considering the
// current BRM status, HWM, and total block count for the DBRoot.
//------------------------------------------------------------------------------
DBRootExtentInfoState DBRootExtentTracker::determineState(int colWidth,
HWM localHwm,
uint64_t dbRootTotalBlocks,
int16_t status)
DBRootExtentInfoState DBRootExtentTracker::determineState(int colWidth, HWM localHwm,
uint64_t dbRootTotalBlocks, int16_t status)
{
DBRootExtentInfoState extentState;
DBRootExtentInfoState extentState;
if (status == BRM::EXTENTOUTOFSERVICE)
if (status == BRM::EXTENTOUTOFSERVICE)
{
extentState = DBROOT_EXTENT_OUT_OF_SERVICE;
}
else
{
if (dbRootTotalBlocks == 0)
{
extentState = DBROOT_EXTENT_OUT_OF_SERVICE;
extentState = DBROOT_EXTENT_EMPTY_DBROOT;
}
else
{
if (dbRootTotalBlocks == 0)
{
extentState = DBROOT_EXTENT_EMPTY_DBROOT;
}
else
{
extentState = DBROOT_EXTENT_PARTIAL_EXTENT;
extentState = DBROOT_EXTENT_PARTIAL_EXTENT;
// See if local hwm is on an extent bndry,in which case the extent
// is full and we won't be adding rows to the current HWM extent;
// we will instead need to allocate a new extent in order to begin
// adding any rows.
long long nRows = ((long long)(localHwm + 1) *
(long long)BYTE_PER_BLOCK) / (long long)colWidth;
long long nRem = nRows % BRMWrapper::getInstance()->getExtentRows();
// See if local hwm is on an extent bndry,in which case the extent
// is full and we won't be adding rows to the current HWM extent;
// we will instead need to allocate a new extent in order to begin
// adding any rows.
long long nRows = ((long long)(localHwm + 1) * (long long)BYTE_PER_BLOCK) / (long long)colWidth;
long long nRem = nRows % BRMWrapper::getInstance()->getExtentRows();
if (nRem == 0)
{
extentState = DBROOT_EXTENT_EXTENT_BOUNDARY;
}
}
if (nRem == 0)
{
extentState = DBROOT_EXTENT_EXTENT_BOUNDARY;
}
}
}
return extentState;
return extentState;
}
//------------------------------------------------------------------------------
// Select the first segment file to add rows to for the local PM.
// Function will first try to find the HWM extent with the fewest blocks.
@ -232,126 +200,118 @@ DBRootExtentInfoState DBRootExtentTracker::determineState(int colWidth,
// Mutex lock not needed in this function as it is only called from main thread
// before processing threads are spawned.
//------------------------------------------------------------------------------
int DBRootExtentTracker::selectFirstSegFile(
DBRootExtentInfo& dbRootExtent, bool& bNoStartExtentOnThisPM,
bool& bEmptyPM,
std::string& errMsg )
int DBRootExtentTracker::selectFirstSegFile(DBRootExtentInfo& dbRootExtent, bool& bNoStartExtentOnThisPM,
bool& bEmptyPM, std::string& errMsg)
{
int startExtentIdx = -1;
int fewestLocalBlocksIdx = -1; // track HWM extent with fewest blocks
int fewestTotalBlocksIdx = -1; // track DBRoot with fewest total blocks
bNoStartExtentOnThisPM = false;
int startExtentIdx = -1;
int fewestLocalBlocksIdx = -1; // track HWM extent with fewest blocks
int fewestTotalBlocksIdx = -1; // track DBRoot with fewest total blocks
bNoStartExtentOnThisPM = false;
unsigned int fewestTotalBlks = UINT_MAX;
unsigned int fewestLocalBlks = UINT_MAX;
uint16_t fewestTotalBlkSegNum = USHRT_MAX;
uint16_t fewestLocalBlkSegNum = USHRT_MAX;
unsigned int fewestTotalBlks = UINT_MAX;
unsigned int fewestLocalBlks = UINT_MAX;
uint16_t fewestTotalBlkSegNum = USHRT_MAX;
uint16_t fewestLocalBlkSegNum = USHRT_MAX;
// Find DBRoot having HWM extent with fewest blocks. If all HWM extents
// are full (remblks=0), then fall-back on selecting the DBRoot with fewest
// total blks.
//
// Selecting HWM extent with fewest blocks should be straight forward, be-
// cause all the DBRoots on a PM should end on an extent boundary except
// for the current last extent. But if the user has moved a DBRoot, then
// we can end up with 2 partially filled HWM extents on 2 DBRoots, on the
// same PM. That's why we loop through the DBRoots to see if we have more
// than 1 partially filled HWM extent.
for (unsigned int iroot = 0;
iroot < fDBRootExtentList.size();
iroot++)
// Find DBRoot having HWM extent with fewest blocks. If all HWM extents
// are full (remblks=0), then fall-back on selecting the DBRoot with fewest
// total blks.
//
// Selecting HWM extent with fewest blocks should be straight forward, be-
// cause all the DBRoots on a PM should end on an extent boundary except
// for the current last extent. But if the user has moved a DBRoot, then
// we can end up with 2 partially filled HWM extents on 2 DBRoots, on the
// same PM. That's why we loop through the DBRoots to see if we have more
// than 1 partially filled HWM extent.
for (unsigned int iroot = 0; iroot < fDBRootExtentList.size(); iroot++)
{
// Skip over DBRoots which have no extents
if (fDBRootExtentList[iroot].fState == DBROOT_EXTENT_EMPTY_DBROOT)
continue;
fEmptyPM = false;
// Find DBRoot and segment file with most incomplete extent.
// Break a tie by selecting the lowest segment number.
long long remBlks = (long long)(fDBRootExtentList[iroot].fLocalHwm + 1) % fBlksPerExtent;
if (remBlks > 0)
{
// Skip over DBRoots which have no extents
if (fDBRootExtentList[iroot].fState == DBROOT_EXTENT_EMPTY_DBROOT)
continue;
fEmptyPM = false;
// Find DBRoot and segment file with most incomplete extent.
// Break a tie by selecting the lowest segment number.
long long remBlks = (long long)(fDBRootExtentList[iroot].fLocalHwm + 1) %
fBlksPerExtent;
if (remBlks > 0)
{
if ( (remBlks < fewestLocalBlks) ||
((remBlks == fewestLocalBlks) &&
(fDBRootExtentList[iroot].fSegment < fewestLocalBlkSegNum)) )
{
fewestLocalBlocksIdx = iroot;
fewestLocalBlks = remBlks;
fewestLocalBlkSegNum = fDBRootExtentList[iroot].fSegment;
}
}
// Find DBRoot with fewest total of blocks.
// Break a tie by selecting the highest segment number.
if ( (fDBRootExtentList[iroot].fDBRootTotalBlocks < fewestTotalBlks) ||
((fDBRootExtentList[iroot].fDBRootTotalBlocks == fewestTotalBlks) &&
(fDBRootExtentList[iroot].fSegment > fewestTotalBlkSegNum)) )
{
fewestTotalBlocksIdx = iroot;
fewestTotalBlks = fDBRootExtentList[iroot].fDBRootTotalBlocks;
fewestTotalBlkSegNum = fDBRootExtentList[iroot].fSegment;
}
if ((remBlks < fewestLocalBlks) ||
((remBlks == fewestLocalBlks) && (fDBRootExtentList[iroot].fSegment < fewestLocalBlkSegNum)))
{
fewestLocalBlocksIdx = iroot;
fewestLocalBlks = remBlks;
fewestLocalBlkSegNum = fDBRootExtentList[iroot].fSegment;
}
}
// Select HWM extent with fewest number of blocks;
// If chosen extent is disabled, then treat like an empty PM,
// meaning we have to allocate a new extent before adding any rows
if (fewestLocalBlocksIdx != -1)
// Find DBRoot with fewest total of blocks.
// Break a tie by selecting the highest segment number.
if ((fDBRootExtentList[iroot].fDBRootTotalBlocks < fewestTotalBlks) ||
((fDBRootExtentList[iroot].fDBRootTotalBlocks == fewestTotalBlks) &&
(fDBRootExtentList[iroot].fSegment > fewestTotalBlkSegNum)))
{
startExtentIdx = fewestLocalBlocksIdx;
if (fDBRootExtentList[startExtentIdx].fState ==
DBROOT_EXTENT_OUT_OF_SERVICE)
{
fDisabledHWM = true;
}
fewestTotalBlocksIdx = iroot;
fewestTotalBlks = fDBRootExtentList[iroot].fDBRootTotalBlocks;
fewestTotalBlkSegNum = fDBRootExtentList[iroot].fSegment;
}
}
// If the HWM on each DBRoot ends on an extent boundary, then
// select the DBRoot with the fewest total number of blocks;
// If chosen extent is disabled, then treat like an empty PM,
// meaning we have to allocate a new extent before adding any rows
else if (fewestTotalBlocksIdx != -1)
// Select HWM extent with fewest number of blocks;
// If chosen extent is disabled, then treat like an empty PM,
// meaning we have to allocate a new extent before adding any rows
if (fewestLocalBlocksIdx != -1)
{
startExtentIdx = fewestLocalBlocksIdx;
if (fDBRootExtentList[startExtentIdx].fState == DBROOT_EXTENT_OUT_OF_SERVICE)
{
startExtentIdx = fewestTotalBlocksIdx;
if (fDBRootExtentList[startExtentIdx].fState ==
DBROOT_EXTENT_OUT_OF_SERVICE)
{
fDisabledHWM = true;
}
fDisabledHWM = true;
}
}
// PM with no extents (or all extents disabled), so select DBRoot/segment
// file from DBRoot list, where we will start inserting rows.
// Select lowest segment file number.
else
// If the HWM on each DBRoot ends on an extent boundary, then
// select the DBRoot with the fewest total number of blocks;
// If chosen extent is disabled, then treat like an empty PM,
// meaning we have to allocate a new extent before adding any rows
else if (fewestTotalBlocksIdx != -1)
{
startExtentIdx = fewestTotalBlocksIdx;
if (fDBRootExtentList[startExtentIdx].fState == DBROOT_EXTENT_OUT_OF_SERVICE)
{
RETURN_ON_ERROR( selectFirstSegFileForEmptyPM( errMsg ) );
startExtentIdx = fCurrentDBRootIdx;
fDisabledHWM = true;
}
}
if ((fEmptyOrDisabledPM) || (fDisabledHWM))
bNoStartExtentOnThisPM = true;
// PM with no extents (or all extents disabled), so select DBRoot/segment
// file from DBRoot list, where we will start inserting rows.
// Select lowest segment file number.
else
{
RETURN_ON_ERROR(selectFirstSegFileForEmptyPM(errMsg));
bEmptyPM = fEmptyPM;
fCurrentDBRootIdx = startExtentIdx;
startExtentIdx = fCurrentDBRootIdx;
}
// Finish Initializing DBRootExtentList for empty DBRoots w/o any extents
initEmptyDBRoots( );
if ((fEmptyOrDisabledPM) || (fDisabledHWM))
bNoStartExtentOnThisPM = true;
logFirstDBRootSelection( );
bEmptyPM = fEmptyPM;
fCurrentDBRootIdx = startExtentIdx;
dbRootExtent = fDBRootExtentList[startExtentIdx];
fDBRootExtentList[startExtentIdx].fState = DBROOT_EXTENT_EXTENT_BOUNDARY;
// Finish Initializing DBRootExtentList for empty DBRoots w/o any extents
initEmptyDBRoots();
return NO_ERROR;
logFirstDBRootSelection();
dbRootExtent = fDBRootExtentList[startExtentIdx];
fDBRootExtentList[startExtentIdx].fState = DBROOT_EXTENT_EXTENT_BOUNDARY;
return NO_ERROR;
}
//------------------------------------------------------------------------------
// If we have encountered a PM with no extents (or all extent disabled), then
// this function can be called to determine the DBRoot to be used for the 1st
@ -361,73 +321,66 @@ int DBRootExtentTracker::selectFirstSegFile(
// Mutex lock not needed in this function as it is only called from main thread
// before processing threads are spawned.
//------------------------------------------------------------------------------
int DBRootExtentTracker::selectFirstSegFileForEmptyPM( std::string& errMsg )
int DBRootExtentTracker::selectFirstSegFileForEmptyPM(std::string& errMsg)
{
fEmptyOrDisabledPM = true;
fEmptyOrDisabledPM = true;
fCurrentDBRootIdx = 0; // Start with first DBRoot for this PM
fCurrentDBRootIdx = 0; // Start with first DBRoot for this PM
// Always start empty PM with partition number 0. If the DBRoot has a HWM
// extent that is disabled, then BRM will override this partition number.
fDBRootExtentList[fCurrentDBRootIdx].fPartition = 0;
// Always start empty PM with partition number 0. If the DBRoot has a HWM
// extent that is disabled, then BRM will override this partition number.
fDBRootExtentList[fCurrentDBRootIdx].fPartition = 0;
return NO_ERROR;
return NO_ERROR;
}
//------------------------------------------------------------------------------
// Finish Initializing fDBRootExtentList for any empty DBRoots w/o any extents.
//------------------------------------------------------------------------------
void DBRootExtentTracker::initEmptyDBRoots( )
void DBRootExtentTracker::initEmptyDBRoots()
{
int startExtentIdx = fCurrentDBRootIdx;
bool bAnyChanges = false; // If fDBRootExtentList changes, log the contents
int startExtentIdx = fCurrentDBRootIdx;
bool bAnyChanges = false; // If fDBRootExtentList changes, log the contents
// Fill in starting partition for any DBRoots having no extents
for (unsigned int iroot = 0;
iroot < fDBRootExtentList.size();
iroot++)
// Fill in starting partition for any DBRoots having no extents
for (unsigned int iroot = 0; iroot < fDBRootExtentList.size(); iroot++)
{
if ((fDBRootExtentList[iroot].fState == DBROOT_EXTENT_EMPTY_DBROOT) &&
((int)iroot != startExtentIdx)) // skip over selected dbroot
{
if ((fDBRootExtentList[iroot].fState == DBROOT_EXTENT_EMPTY_DBROOT) &&
((int)iroot != startExtentIdx)) // skip over selected dbroot
{
if (fDBRootExtentList[iroot].fPartition !=
fDBRootExtentList[startExtentIdx].fPartition)
{
bAnyChanges = true;
if (fDBRootExtentList[iroot].fPartition != fDBRootExtentList[startExtentIdx].fPartition)
{
bAnyChanges = true;
fDBRootExtentList[iroot].fPartition =
fDBRootExtentList[startExtentIdx].fPartition;
}
}
fDBRootExtentList[iroot].fPartition = fDBRootExtentList[startExtentIdx].fPartition;
}
}
}
// Log fDBRootExtentList if modifications were made
if ((bAnyChanges) && (fLog))
// Log fDBRootExtentList if modifications were made
if ((bAnyChanges) && (fLog))
{
// Always log this info for now; may control with debug later
// if (fLog->isDebug(DEBUG_1))
{
// Always log this info for now; may control with debug later
//if (fLog->isDebug(DEBUG_1))
{
std::ostringstream oss;
oss << "Updated starting (empty) DBRoot info for OID " << fOID;
std::ostringstream oss;
oss << "Updated starting (empty) DBRoot info for OID " << fOID;
for (unsigned int k = 0; k < fDBRootExtentList.size(); k++)
{
oss << std::endl;
oss << " DBRoot-" << fDBRootExtentList[k].fDbRoot <<
", part/seg/hwm/LBID/totBlks/state: " <<
fDBRootExtentList[k].fPartition <<
"/" << fDBRootExtentList[k].fSegment <<
"/" << fDBRootExtentList[k].fLocalHwm <<
"/" << fDBRootExtentList[k].fStartLbid <<
"/" << fDBRootExtentList[k].fDBRootTotalBlocks <<
"/" << stateStrings[ fDBRootExtentList[k].fState ];
}
for (unsigned int k = 0; k < fDBRootExtentList.size(); k++)
{
oss << std::endl;
oss << " DBRoot-" << fDBRootExtentList[k].fDbRoot
<< ", part/seg/hwm/LBID/totBlks/state: " << fDBRootExtentList[k].fPartition << "/"
<< fDBRootExtentList[k].fSegment << "/" << fDBRootExtentList[k].fLocalHwm << "/"
<< fDBRootExtentList[k].fStartLbid << "/" << fDBRootExtentList[k].fDBRootTotalBlocks << "/"
<< stateStrings[fDBRootExtentList[k].fState];
}
fLog->logMsg( oss.str(), MSGLVL_INFO2 );
}
fLog->logMsg(oss.str(), MSGLVL_INFO2);
}
}
}
//------------------------------------------------------------------------------
// Assign the DBRoot/segment file to be used for extent loading based on the
// setting in the specified reference tracker (set for a reference column).
@ -435,79 +388,75 @@ void DBRootExtentTracker::initEmptyDBRoots( )
// Mutex lock not needed in this function as it is only called from main thread
// before processing threads are spawned.
//------------------------------------------------------------------------------
void DBRootExtentTracker::assignFirstSegFile(
const DBRootExtentTracker& refTracker,
DBRootExtentInfo& dbRootExtent)
void DBRootExtentTracker::assignFirstSegFile(const DBRootExtentTracker& refTracker,
DBRootExtentInfo& dbRootExtent)
{
// Start with the same DBRoot index as the reference tracker; assumes that
// DBRoots for each column are listed in same order in fDBRootExtentList.
// That should be a safe assumption since DBRootExtentTracker constructor
// sorts the entries in fDBRootExtentList by fDbRoot.
int startExtentIdx = refTracker.fCurrentDBRootIdx;
fEmptyOrDisabledPM = refTracker.fEmptyOrDisabledPM;
fEmptyPM = refTracker.fEmptyPM;
fDisabledHWM = refTracker.fDisabledHWM;
// Start with the same DBRoot index as the reference tracker; assumes that
// DBRoots for each column are listed in same order in fDBRootExtentList.
// That should be a safe assumption since DBRootExtentTracker constructor
// sorts the entries in fDBRootExtentList by fDbRoot.
int startExtentIdx = refTracker.fCurrentDBRootIdx;
fEmptyOrDisabledPM = refTracker.fEmptyOrDisabledPM;
fEmptyPM = refTracker.fEmptyPM;
fDisabledHWM = refTracker.fDisabledHWM;
// Always start empty PM with partition number 0. If the DBRoot has a HWM
// extent that is disabled, then BRM will override this partition number.
if (fEmptyOrDisabledPM)
{
fDBRootExtentList[startExtentIdx].fPartition = 0;
}
// Always start empty PM with partition number 0. If the DBRoot has a HWM
// extent that is disabled, then BRM will override this partition number.
if (fEmptyOrDisabledPM)
{
fDBRootExtentList[startExtentIdx].fPartition = 0;
}
fCurrentDBRootIdx = startExtentIdx;
fCurrentDBRootIdx = startExtentIdx;
// Finish Initializing DBRootExtentList for empty DBRoots w/o any extents
initEmptyDBRoots( );
// Finish Initializing DBRootExtentList for empty DBRoots w/o any extents
initEmptyDBRoots();
logFirstDBRootSelection( );
logFirstDBRootSelection();
dbRootExtent = fDBRootExtentList[startExtentIdx];
fDBRootExtentList[startExtentIdx].fState = DBROOT_EXTENT_EXTENT_BOUNDARY;
dbRootExtent = fDBRootExtentList[startExtentIdx];
fDBRootExtentList[startExtentIdx].fState = DBROOT_EXTENT_EXTENT_BOUNDARY;
}
//------------------------------------------------------------------------------
// Log information about the first DBRoot/segment file that is selected.
//------------------------------------------------------------------------------
void DBRootExtentTracker::logFirstDBRootSelection( ) const
void DBRootExtentTracker::logFirstDBRootSelection() const
{
if (fLog)
{
int extentIdx = fCurrentDBRootIdx;
if (fLog)
{
int extentIdx = fCurrentDBRootIdx;
if (fEmptyOrDisabledPM)
{
std::ostringstream oss;
oss << "No active extents; will add partition to start adding "
"rows for oid-" << fOID <<
"; DBRoot-" << fDBRootExtentList[extentIdx].fDbRoot;
fLog->logMsg( oss.str(), MSGLVL_INFO2 );
}
else if (fDisabledHWM)
{
std::ostringstream oss;
oss << "HWM extent disabled; will add partition to start adding "
"rows for oid-" << fOID <<
"; DBRoot-" << fDBRootExtentList[extentIdx].fDbRoot;
fLog->logMsg( oss.str(), MSGLVL_INFO2 );
}
else
{
std::ostringstream oss;
oss << "Selecting existing segFile to begin adding rows: oid-" << fOID <<
"; DBRoot-" << fDBRootExtentList[extentIdx].fDbRoot <<
", part/seg/hwm/LBID/totBlks/state: " <<
fDBRootExtentList[extentIdx].fPartition <<
"/" << fDBRootExtentList[extentIdx].fSegment <<
"/" << fDBRootExtentList[extentIdx].fLocalHwm <<
"/" << fDBRootExtentList[extentIdx].fStartLbid <<
"/" << fDBRootExtentList[extentIdx].fDBRootTotalBlocks <<
"/" << stateStrings[ fDBRootExtentList[extentIdx].fState ];
fLog->logMsg( oss.str(), MSGLVL_INFO2 );
}
if (fEmptyOrDisabledPM)
{
std::ostringstream oss;
oss << "No active extents; will add partition to start adding "
"rows for oid-"
<< fOID << "; DBRoot-" << fDBRootExtentList[extentIdx].fDbRoot;
fLog->logMsg(oss.str(), MSGLVL_INFO2);
}
else if (fDisabledHWM)
{
std::ostringstream oss;
oss << "HWM extent disabled; will add partition to start adding "
"rows for oid-"
<< fOID << "; DBRoot-" << fDBRootExtentList[extentIdx].fDbRoot;
fLog->logMsg(oss.str(), MSGLVL_INFO2);
}
else
{
std::ostringstream oss;
oss << "Selecting existing segFile to begin adding rows: oid-" << fOID << "; DBRoot-"
<< fDBRootExtentList[extentIdx].fDbRoot
<< ", part/seg/hwm/LBID/totBlks/state: " << fDBRootExtentList[extentIdx].fPartition << "/"
<< fDBRootExtentList[extentIdx].fSegment << "/" << fDBRootExtentList[extentIdx].fLocalHwm << "/"
<< fDBRootExtentList[extentIdx].fStartLbid << "/" << fDBRootExtentList[extentIdx].fDBRootTotalBlocks
<< "/" << stateStrings[fDBRootExtentList[extentIdx].fState];
fLog->logMsg(oss.str(), MSGLVL_INFO2);
}
}
}
//------------------------------------------------------------------------------
// Iterate/return next DBRoot to be used for the next extent
//
@ -522,57 +471,52 @@ void DBRootExtentTracker::logFirstDBRootSelection( ) const
// Returns true if new extent needs to be allocated, else false indicates that
// the extent is partially full.
//------------------------------------------------------------------------------
bool DBRootExtentTracker::nextSegFile(
uint16_t& dbRoot,
uint32_t& partition,
uint16_t& segment,
HWM& localHwm,
BRM::LBID_t& startLbid)
bool DBRootExtentTracker::nextSegFile(uint16_t& dbRoot, uint32_t& partition, uint16_t& segment, HWM& localHwm,
BRM::LBID_t& startLbid)
{
boost::mutex::scoped_lock lock(fDBRootExtTrkMutex);
boost::mutex::scoped_lock lock(fDBRootExtTrkMutex);
fCurrentDBRootIdx++;
fCurrentDBRootIdx++;
if ((unsigned int)fCurrentDBRootIdx >= fDBRootExtentList.size())
fCurrentDBRootIdx = 0;
if ((unsigned int)fCurrentDBRootIdx >= fDBRootExtentList.size())
fCurrentDBRootIdx = 0;
dbRoot = fDBRootExtentList[fCurrentDBRootIdx].fDbRoot;
segment = fDBRootExtentList[fCurrentDBRootIdx].fSegment;
partition = fDBRootExtentList[fCurrentDBRootIdx].fPartition;
localHwm = fDBRootExtentList[fCurrentDBRootIdx].fLocalHwm;
startLbid = fDBRootExtentList[fCurrentDBRootIdx].fStartLbid;
//std::cout << "NextSegFile: Current idx: " << fCurrentDBRootIdx <<
//"; new dbroot: " << dbRoot <<
//"; segment: " << segment <<
//"; partition: " << partition <<
//"; localHwm: " << localHwm <<
//"; startLbid: " << startLbid <<
//"; state: " << stateStrings[fDBRootExtentList[fCurrentDBRootIdx].fState]
// << std::endl;
dbRoot = fDBRootExtentList[fCurrentDBRootIdx].fDbRoot;
segment = fDBRootExtentList[fCurrentDBRootIdx].fSegment;
partition = fDBRootExtentList[fCurrentDBRootIdx].fPartition;
localHwm = fDBRootExtentList[fCurrentDBRootIdx].fLocalHwm;
startLbid = fDBRootExtentList[fCurrentDBRootIdx].fStartLbid;
// std::cout << "NextSegFile: Current idx: " << fCurrentDBRootIdx <<
//"; new dbroot: " << dbRoot <<
//"; segment: " << segment <<
//"; partition: " << partition <<
//"; localHwm: " << localHwm <<
//"; startLbid: " << startLbid <<
//"; state: " << stateStrings[fDBRootExtentList[fCurrentDBRootIdx].fState]
// << std::endl;
bool bAllocExtentFlag = true;
bool bAllocExtentFlag = true;
if (fDBRootExtentList[fCurrentDBRootIdx].fState ==
DBROOT_EXTENT_PARTIAL_EXTENT)
bAllocExtentFlag = false;
if (fDBRootExtentList[fCurrentDBRootIdx].fState == DBROOT_EXTENT_PARTIAL_EXTENT)
bAllocExtentFlag = false;
// After we have taken care of the "first" extent for each DBRoot, we can
// zero out everything. The only thing we need to continue rotating thru
// the DBRoots, is the DBRoot number itself.
fDBRootExtentList[fCurrentDBRootIdx].fSegment = 0;
fDBRootExtentList[fCurrentDBRootIdx].fPartition = 0;
fDBRootExtentList[fCurrentDBRootIdx].fLocalHwm = 0;
fDBRootExtentList[fCurrentDBRootIdx].fStartLbid = 0;
fDBRootExtentList[fCurrentDBRootIdx].fDBRootTotalBlocks = 0;
fDBRootExtentList[fCurrentDBRootIdx].fState = DBROOT_EXTENT_EXTENT_BOUNDARY;
// After we have taken care of the "first" extent for each DBRoot, we can
// zero out everything. The only thing we need to continue rotating thru
// the DBRoots, is the DBRoot number itself.
fDBRootExtentList[fCurrentDBRootIdx].fSegment = 0;
fDBRootExtentList[fCurrentDBRootIdx].fPartition = 0;
fDBRootExtentList[fCurrentDBRootIdx].fLocalHwm = 0;
fDBRootExtentList[fCurrentDBRootIdx].fStartLbid = 0;
fDBRootExtentList[fCurrentDBRootIdx].fDBRootTotalBlocks = 0;
fDBRootExtentList[fCurrentDBRootIdx].fState = DBROOT_EXTENT_EXTENT_BOUNDARY;
return bAllocExtentFlag;
return bAllocExtentFlag;
}
const std::vector<DBRootExtentInfo>& DBRootExtentTracker::getDBRootExtentList()
{
boost::mutex::scoped_lock lock(fDBRootExtTrkMutex);
return fDBRootExtentList;
boost::mutex::scoped_lock lock(fDBRootExtTrkMutex);
return fDBRootExtentList;
}
} // end of namespace
} // namespace WriteEngine

276
writeengine/shared/we_dbrootextenttracker.h
View File

@ -16,8 +16,8 @@
MA 02110-1301, USA. */
/*
* $Id: we_dbrootextenttracker.h 4631 2013-05-02 15:21:09Z dcathey $
*/
* $Id: we_dbrootextenttracker.h 4631 2013-05-02 15:21:09Z dcathey $
*/
/** @file we_dbrootextenttracker.h
* Contains classes to track the order of placement (rotation) of extents as
@ -61,10 +61,10 @@ class Log;
//
enum DBRootExtentInfoState
{
DBROOT_EXTENT_PARTIAL_EXTENT = 1,
DBROOT_EXTENT_EMPTY_DBROOT = 2,
DBROOT_EXTENT_EXTENT_BOUNDARY = 3,
DBROOT_EXTENT_OUT_OF_SERVICE = 4
DBROOT_EXTENT_PARTIAL_EXTENT = 1,
DBROOT_EXTENT_EMPTY_DBROOT = 2,
DBROOT_EXTENT_EXTENT_BOUNDARY = 3,
DBROOT_EXTENT_OUT_OF_SERVICE = 4
};
//------------------------------------------------------------------------------
@ -73,33 +73,29 @@ enum DBRootExtentInfoState
//------------------------------------------------------------------------------
struct DBRootExtentInfo
{
uint32_t fPartition;
uint16_t fDbRoot;
uint16_t fSegment;
BRM::LBID_t fStartLbid;
HWM fLocalHwm;
uint64_t fDBRootTotalBlocks;
DBRootExtentInfoState fState;
uint32_t fPartition;
uint16_t fDbRoot;
uint16_t fSegment;
BRM::LBID_t fStartLbid;
HWM fLocalHwm;
uint64_t fDBRootTotalBlocks;
DBRootExtentInfoState fState;
DBRootExtentInfo() :
fPartition(0),
fDbRoot(0),
fSegment(0),
fStartLbid(0),
fLocalHwm(0),
fDBRootTotalBlocks(0),
fState(DBROOT_EXTENT_PARTIAL_EXTENT) { }
DBRootExtentInfo()
: fPartition(0)
, fDbRoot(0)
, fSegment(0)
, fStartLbid(0)
, fLocalHwm(0)
, fDBRootTotalBlocks(0)
, fState(DBROOT_EXTENT_PARTIAL_EXTENT)
{
}
DBRootExtentInfo(
uint16_t dbRoot,
uint32_t partition,
uint16_t segment,
BRM::LBID_t startLbid,
HWM localHwm,
uint64_t dbrootTotalBlocks,
DBRootExtentInfoState state);
DBRootExtentInfo(uint16_t dbRoot, uint32_t partition, uint16_t segment, BRM::LBID_t startLbid, HWM localHwm,
uint64_t dbrootTotalBlocks, DBRootExtentInfoState state);
bool operator<(const DBRootExtentInfo& entry) const;
bool operator<(const DBRootExtentInfo& entry) const;
};
//------------------------------------------------------------------------------
@ -109,130 +105,118 @@ struct DBRootExtentInfo
//------------------------------------------------------------------------------
class DBRootExtentTracker
{
public:
public:
/** @brief DBRootExtentTracker constructor
* @param oid Column OID of interest.
* @param colWidths Widths (in bytes) of all the columns in the table.
* @param dbRootHWMInfoColVec Column HWM, DBRoots, etc for this table.
* @param columnIdx Index (into colWidths and dbRootHWMInfoColVec)
* referencing the column that applies to this ExtentTracker.
* @param logger Logger to be used for logging messages.
*/
EXPORT DBRootExtentTracker(OID oid, const std::vector<int>& colWidths,
const std::vector<BRM::EmDbRootHWMInfo_v>& dbRootHWMInfoColVec,
unsigned int columnIdx, Log* logger);
/** @brief DBRootExtentTracker constructor
* @param oid Column OID of interest.
* @param colWidths Widths (in bytes) of all the columns in the table.
* @param dbRootHWMInfoColVec Column HWM, DBRoots, etc for this table.
* @param columnIdx Index (into colWidths and dbRootHWMInfoColVec)
* referencing the column that applies to this ExtentTracker.
* @param logger Logger to be used for logging messages.
*/
EXPORT DBRootExtentTracker ( OID oid,
const std::vector<int>& colWidths,
const std::vector<BRM::EmDbRootHWMInfo_v>& dbRootHWMInfoColVec,
unsigned int columnIdx,
Log* logger );
/** @brief Select the first DBRoot/segment file to add rows to, for this PM.
* @param dbRootExtent Dbroot/segment file selected for first set of rows.
* @param bNoStartExtentOnThisPM Is starting HWM extent missing or disabled.
* If HWM extent is missing or disabled, the app will have to allo-
* cate a new extent (at the DBRoot returned in dbRootExtent)) in
* order to add any rows.
* @param bEmptyPM Is this PM void of any available extents
* @return Returns NO_ERROR if success, else returns error code.
*/
EXPORT int selectFirstSegFile(DBRootExtentInfo& dbRootExtent, bool& bNoStartExtentOnThisPM, bool& bEmptyPM,
std::string& errMsg);
/** @brief Select the first DBRoot/segment file to add rows to, for this PM.
* @param dbRootExtent Dbroot/segment file selected for first set of rows.
* @param bNoStartExtentOnThisPM Is starting HWM extent missing or disabled.
* If HWM extent is missing or disabled, the app will have to allo-
* cate a new extent (at the DBRoot returned in dbRootExtent)) in
* order to add any rows.
* @param bEmptyPM Is this PM void of any available extents
* @return Returns NO_ERROR if success, else returns error code.
*/
EXPORT int selectFirstSegFile ( DBRootExtentInfo& dbRootExtent,
bool& bNoStartExtentOnThisPM,
bool& bEmptyPM,
std::string& errMsg );
/** @brief Set up this Tracker to select the same first DBRoot/segment file
* as the reference DBRootExtentTracker that is specified from a ref column.
*
* Application code should call selectFirstSegFile for a reference column,
* and assignFirstSegFile for all other columns in the same table.
* @param refTracker Tracker object used to assign first DBRoot/segment.
* @param dbRootExtent Dbroot/segment file selected for first set of rows.
*/
EXPORT void assignFirstSegFile(const DBRootExtentTracker& refTracker, DBRootExtentInfo& dbRootExtent);
/** @brief Set up this Tracker to select the same first DBRoot/segment file
* as the reference DBRootExtentTracker that is specified from a ref column.
*
* Application code should call selectFirstSegFile for a reference column,
* and assignFirstSegFile for all other columns in the same table.
* @param refTracker Tracker object used to assign first DBRoot/segment.
* @param dbRootExtent Dbroot/segment file selected for first set of rows.
*/
EXPORT void assignFirstSegFile( const DBRootExtentTracker& refTracker,
DBRootExtentInfo& dbRootExtent );
/** @brief Iterate/return next DBRoot to be used for the next extent.
*
* Case 1)
* If it is the "very" first extent for the specified DBRoot, then the
* applicable partition to be used for the first extent is also returned.
*
* Case 2)
* If the user moves a DBRoot to a different PM, then the next cpimport.bin
* job on the recepient PM may encounter 2 partially filled in extents.
* This differs from the norm, where we only have 1 partially filled extent
* at any given time, on a PM. When a DBRoot is moved, we may finish an ex-
* tent on 1 DBRoot, and instead of advancing to start a new extent, we ro-
* tate to the recently moved DBRoot, and have to first fill in a partilly
* filled in extent instead of adding a new extent. Case 2 is intended to
* cover this use case.
* In this case, in the middle of an import, if the next extent to receive
* rows is a partially filled in extent, then the DBRoot, partition, and
* segment number for the partial extent are returned. In addition, the
* current HWM and starting LBID for the relevant extent are returned.
*
* Case 3)
* If we are just finishing one extent and adding the next extent, then
* only the DBRoot argument is relevant, telling us where to create the
* next extent. Return value will be true. This case also applies to
* the instance where the HWM extent for the next DBRoot is disabled.
*
* @param dbRoot DBRoot for the next extent
* @param partition If first extent on dbRoot (or partial extent), then
* this is the partition #
* @param segment If partially full extent, then this is the segment #
* @param localHwm If partially full extent, then this is current HWM.
* @param startLbid If partially full extent, then this is starting LBID of
* the current HWM extent.
*
* @return Returns true if new extent needs to be allocated, returns false
* if extent is partially full, and has room for more rows.
*/
EXPORT bool nextSegFile(uint16_t& dbRoot, uint32_t& partition, uint16_t& segment, HWM& localHwm,
BRM::LBID_t& startLbid);
/** @brief Iterate/return next DBRoot to be used for the next extent.
*
* Case 1)
* If it is the "very" first extent for the specified DBRoot, then the
* applicable partition to be used for the first extent is also returned.
*
* Case 2)
* If the user moves a DBRoot to a different PM, then the next cpimport.bin
* job on the recepient PM may encounter 2 partially filled in extents.
* This differs from the norm, where we only have 1 partially filled extent
* at any given time, on a PM. When a DBRoot is moved, we may finish an ex-
* tent on 1 DBRoot, and instead of advancing to start a new extent, we ro-
* tate to the recently moved DBRoot, and have to first fill in a partilly
* filled in extent instead of adding a new extent. Case 2 is intended to
* cover this use case.
* In this case, in the middle of an import, if the next extent to receive
* rows is a partially filled in extent, then the DBRoot, partition, and
* segment number for the partial extent are returned. In addition, the
* current HWM and starting LBID for the relevant extent are returned.
*
* Case 3)
* If we are just finishing one extent and adding the next extent, then
* only the DBRoot argument is relevant, telling us where to create the
* next extent. Return value will be true. This case also applies to
* the instance where the HWM extent for the next DBRoot is disabled.
*
* @param dbRoot DBRoot for the next extent
* @param partition If first extent on dbRoot (or partial extent), then
* this is the partition #
* @param segment If partially full extent, then this is the segment #
* @param localHwm If partially full extent, then this is current HWM.
* @param startLbid If partially full extent, then this is starting LBID of
* the current HWM extent.
*
* @return Returns true if new extent needs to be allocated, returns false
* if extent is partially full, and has room for more rows.
*/
EXPORT bool nextSegFile( uint16_t& dbRoot,
uint32_t& partition,
uint16_t& segment,
HWM& localHwm,
BRM::LBID_t& startLbid );
/** @brief get the DBRootExtentInfo list
*/
const std::vector<DBRootExtentInfo>& getDBRootExtentList();
/** @brief get the DBRootExtentInfo list
*/
const std::vector<DBRootExtentInfo>& getDBRootExtentList();
/** @brief get the CurrentDBRootIdx
*/
inline int getCurrentDBRootIdx()
{
boost::mutex::scoped_lock lock(fDBRootExtTrkMutex);
return fCurrentDBRootIdx;
}
/** @brief get the CurrentDBRootIdx
*/
inline int getCurrentDBRootIdx()
{
boost::mutex::scoped_lock lock(fDBRootExtTrkMutex);
return fCurrentDBRootIdx;
}
private:
DBRootExtentInfoState determineState(int colWidth, HWM localHwm, uint64_t dbRootTotalBlocks,
int16_t status);
// Select First DBRoot/segment file on a PM having no extents for fOID
int selectFirstSegFileForEmptyPM(std::string& errMsg);
void initEmptyDBRoots(); // init ExtentList for empty DBRoots
void logFirstDBRootSelection() const;
private:
DBRootExtentInfoState determineState(int colWidth,
HWM localHwm,
uint64_t dbRootTotalBlocks,
int16_t status);
// Select First DBRoot/segment file on a PM having no extents for fOID
int selectFirstSegFileForEmptyPM ( std::string& errMsg );
void initEmptyDBRoots(); // init ExtentList for empty DBRoots
void logFirstDBRootSelection() const;
OID fOID; // applicable colunn OID
long long fBlksPerExtent; // blocks per extent for fOID
Log* fLog; // logger
boost::mutex fDBRootExtTrkMutex; // mutex to access fDBRootExtentList
int fCurrentDBRootIdx; // Index into fDBRootExtentList,
// DBRoot where current extent is
// being added
std::vector<DBRootExtentInfo> fDBRootExtentList; // List of current pending
// DBRoot/extents for each DBRoot
// assigned to the local PM.
bool fEmptyOrDisabledPM; // true if PM has no extents or all
// extents are disabled
bool fEmptyPM; // true if PM has no available or
// disabled extents
bool fDisabledHWM; // Did job start with disabled HWM
OID fOID; // applicable colunn OID
long long fBlksPerExtent; // blocks per extent for fOID
Log* fLog; // logger
boost::mutex fDBRootExtTrkMutex; // mutex to access fDBRootExtentList
int fCurrentDBRootIdx; // Index into fDBRootExtentList,
// DBRoot where current extent is
// being added
std::vector<DBRootExtentInfo> fDBRootExtentList; // List of current pending
// DBRoot/extents for each DBRoot
// assigned to the local PM.
bool fEmptyOrDisabledPM; // true if PM has no extents or all
// extents are disabled
bool fEmptyPM; // true if PM has no available or
// disabled extents
bool fDisabledHWM; // Did job start with disabled HWM
};
} //end of namespace
} // namespace WriteEngine
#undef EXPORT

488
writeengine/shared/we_define.cpp
View File

@ -16,9 +16,9 @@
MA 02110-1301, USA. */
/*******************************************************************************
* $Id: we_define.cpp 4726 2013-08-07 03:38:36Z bwilkinson $
*
*******************************************************************************/
* $Id: we_define.cpp 4726 2013-08-07 03:38:36Z bwilkinson $
*
*******************************************************************************/
/** @file" **/
#include <string>
@ -33,269 +33,277 @@
/** Namespace WriteEngine **/
namespace WriteEngine
{
WErrorCodes::WErrorCodes() : fErrorCodes()
{
fErrorCodes[ERR_UNKNOWN] = " a Generic (unknown) error";
fErrorCodes[ERR_INVALID_PARAM] = " due to Invalid parameters";
fErrorCodes[ERR_STRUCT_EMPTY] = " because the Structure is empty";
fErrorCodes[ERR_VALUE_OUTOFRANGE] = " because a Value is out of range";
fErrorCodes[ERR_PARSING] = " a Value is out of range";
fErrorCodes[ERR_NO_MEM] = " out of memory";
fErrorCodes[ERR_DML_LOG_NAME] = " construct DML log filename failed";
fErrorCodes[ERR_OPEN_DML_LOG] = " open DML log file failed";
fErrorCodes[ERR_HDFS_BACKUP] = " DML backup error in HDFS";
fErrorCodes[ERR_UNKNOWN] = " a Generic (unknown) error";
fErrorCodes[ERR_INVALID_PARAM] = " due to Invalid parameters";
fErrorCodes[ERR_STRUCT_EMPTY] = " because the Structure is empty";
fErrorCodes[ERR_VALUE_OUTOFRANGE] = " because a Value is out of range";
fErrorCodes[ERR_PARSING] = " a Value is out of range";
fErrorCodes[ERR_NO_MEM] = " out of memory";
fErrorCodes[ERR_DML_LOG_NAME] = " construct DML log filename failed";
fErrorCodes[ERR_OPEN_DML_LOG] = " open DML log file failed";
fErrorCodes[ERR_HDFS_BACKUP] = " DML backup error in HDFS";
// File level error
fErrorCodes[ERR_FILE_CREATE] = " The column file could not be created; it may already exist or be inaccessible.";
fErrorCodes[ERR_FILE_OPEN] = " opening a column file. The file was not found or was inaccessible.";
fErrorCodes[ERR_FILE_DELETE] = " it can not delete the file, because it does not exist or was inaccessible. ";
fErrorCodes[ERR_FILE_EXIST] = " The File already exists. ";
fErrorCodes[ERR_FILE_NOT_EXIST] = " The File does not exist. " ;
fErrorCodes[ERR_FILE_NULL] = " The FILE pointer is null." ;
fErrorCodes[ERR_FILE_WRITE] = " Error writing to a database file. ";
fErrorCodes[ERR_FILE_READ] = " Error reading from a database file. ";
fErrorCodes[ERR_FILE_SEEK] = " Error in positioning file handle. ";
fErrorCodes[ERR_FILE_READ_IMPORT] = " Error reading import source file. ";
fErrorCodes[ERR_DIR_CREATE] = " Error in creating a directory. ";
fErrorCodes[ERR_FILE_NEW_EXTENT_FBO] = " New extent FBO too high for current file. ";
fErrorCodes[ERR_FILE_FBO_NEG] = " Specified file FBO is negative. ";
fErrorCodes[ERR_FILE_TRUNCATE] = " Error truncating db file. ";
fErrorCodes[ERR_FILE_DISK_SPACE] = "Not able to add extent; adding extent "
"would exceed max file system disk usage. ";
fErrorCodes[ERR_FILE_STAT] = " Error getting stats on db file. ";
fErrorCodes[ERR_VB_FILE_NOT_EXIST] = " Version buffer file does not exists.";
fErrorCodes[ERR_FILE_FLUSH] = " Error flushing db file. ";
fErrorCodes[ERR_FILE_GLOBBING] = " Error globbing a file name. ";
// File level error
fErrorCodes[ERR_FILE_CREATE] =
" The column file could not be created; it may already exist or be inaccessible.";
fErrorCodes[ERR_FILE_OPEN] = " opening a column file. The file was not found or was inaccessible.";
fErrorCodes[ERR_FILE_DELETE] =
" it can not delete the file, because it does not exist or was inaccessible. ";
fErrorCodes[ERR_FILE_EXIST] = " The File already exists. ";
fErrorCodes[ERR_FILE_NOT_EXIST] = " The File does not exist. ";
fErrorCodes[ERR_FILE_NULL] = " The FILE pointer is null.";
fErrorCodes[ERR_FILE_WRITE] = " Error writing to a database file. ";
fErrorCodes[ERR_FILE_READ] = " Error reading from a database file. ";
fErrorCodes[ERR_FILE_SEEK] = " Error in positioning file handle. ";
fErrorCodes[ERR_FILE_READ_IMPORT] = " Error reading import source file. ";
fErrorCodes[ERR_DIR_CREATE] = " Error in creating a directory. ";
fErrorCodes[ERR_FILE_NEW_EXTENT_FBO] = " New extent FBO too high for current file. ";
fErrorCodes[ERR_FILE_FBO_NEG] = " Specified file FBO is negative. ";
fErrorCodes[ERR_FILE_TRUNCATE] = " Error truncating db file. ";
fErrorCodes[ERR_FILE_DISK_SPACE] =
"Not able to add extent; adding extent "
"would exceed max file system disk usage. ";
fErrorCodes[ERR_FILE_STAT] = " Error getting stats on db file. ";
fErrorCodes[ERR_VB_FILE_NOT_EXIST] = " Version buffer file does not exists.";
fErrorCodes[ERR_FILE_FLUSH] = " Error flushing db file. ";
fErrorCodes[ERR_FILE_GLOBBING] = " Error globbing a file name. ";
// XML level error
fErrorCodes[ERR_XML_FILE] = " An xml file error, usually because the file does not exist";
fErrorCodes[ERR_XML_ROOT_ELEM] = " An xml file Root element error ";
fErrorCodes[ERR_XML_EMPTY] = " An Empty XML file ";
fErrorCodes[ERR_XML_PARSE] = " An XML Parsing error";
// XML level error
fErrorCodes[ERR_XML_FILE] = " An xml file error, usually because the file does not exist";
fErrorCodes[ERR_XML_ROOT_ELEM] = " An xml file Root element error ";
fErrorCodes[ERR_XML_EMPTY] = " An Empty XML file ";
fErrorCodes[ERR_XML_PARSE] = " An XML Parsing error";
// table lock level error
fErrorCodes[ERR_TBLLOCK_LOCK_NOT_FOUND] = "Table is not locked.";
fErrorCodes[ERR_TBLLOCK_GET_LOCK] = "Error getting table lock.";
fErrorCodes[ERR_TBLLOCK_GET_LOCK_LOCKED] = "Table locked by another user.";
fErrorCodes[ERR_TBLLOCK_RELEASE_LOCK] = "Error releasing table lock.";
fErrorCodes[ERR_TBLLOCK_CHANGE_STATE] = "Error changing table lock state.";
fErrorCodes[ERR_TBLLOCK_GET_INFO] = "Error getting table lock info.";
fErrorCodes[ERR_TBLLOCK_LOCKID_CONFLICT] = "Table LockID for different table than expected.";
// table lock level error
fErrorCodes[ERR_TBLLOCK_LOCK_NOT_FOUND] = "Table is not locked.";
fErrorCodes[ERR_TBLLOCK_GET_LOCK] = "Error getting table lock.";
fErrorCodes[ERR_TBLLOCK_GET_LOCK_LOCKED] = "Table locked by another user.";
fErrorCodes[ERR_TBLLOCK_RELEASE_LOCK] = "Error releasing table lock.";
fErrorCodes[ERR_TBLLOCK_CHANGE_STATE] = "Error changing table lock state.";
fErrorCodes[ERR_TBLLOCK_GET_INFO] = "Error getting table lock info.";
fErrorCodes[ERR_TBLLOCK_LOCKID_CONFLICT] = "Table LockID for different table than expected.";
// DDL/DML Interface level error
fErrorCodes[ERR_STRUCT_VALUE_NOT_MATCH] = " the number of structs does not match with the number of value sets";
fErrorCodes[ERR_ROWID_VALUE_NOT_MATCH] = " the number of rowids does not match with the number of values";
fErrorCodes[ERR_TBL_SYSCAT_ERROR] = "Error occured when querying systemcatalog.";
// DDL/DML Interface level error
fErrorCodes[ERR_STRUCT_VALUE_NOT_MATCH] =
" the number of structs does not match with the number of value sets";
fErrorCodes[ERR_ROWID_VALUE_NOT_MATCH] = " the number of rowids does not match with the number of values";
fErrorCodes[ERR_TBL_SYSCAT_ERROR] = "Error occured when querying systemcatalog.";
// index error
fErrorCodes[ERR_IDX_TREE_MOVE_ENTRY] = " an error in moving part of an index tree to a new subblock";
fErrorCodes[ERR_IDX_TREE_INVALID_TYPE] = " an Invalid index tree entry type";
fErrorCodes[ERR_IDX_TREE_BITTEST_VAL] = " a Wrong bit test value in the index tree entry";
fErrorCodes[ERR_IDX_TREE_INVALID_LEVEL] = " an Invalid testbit index tree level";
fErrorCodes[ERR_IDX_TREE_INVALID_GRP] = "an Invalid index tree group type ";
fErrorCodes[ERR_IDX_TREE_LISTPTR_CHANGE] = " an index tree List pointer change";
//index list error
fErrorCodes[ERR_IDX_LIST_INVALID_ADDHDR] = " a Create indexlist header error";
fErrorCodes[ERR_IDX_LIST_INVALID_UPDATE] = " an pdate Index List error ";
fErrorCodes[ERR_IDX_LIST_INVALID_DELETE] = " a Delete rowid in indexlist err";
fErrorCodes[ERR_IDX_LIST_INVALID_KEY] = " an Invalid index listbppseeder.cpp Key passed";
fErrorCodes[ERR_IDX_LIST_GET_RID_ARRARY] = " an index list RID array";
fErrorCodes[ERR_IDX_LIST_WRONG_KEY ] = " a not matched Key passed to an index list";
fErrorCodes[ERR_IDX_LIST_HDR_EMPTY] = " an empty index list header";
fErrorCodes[ERR_IDX_LIST_GET_SEGMT] = " in an index list Get Segment";
fErrorCodes[ERR_IDX_LIST_WRONG_LBID_WRITE] = " an index list incorrect LBID write";
fErrorCodes[ERR_IDX_LIST_UPDATE_SUB] = " in an index list update sub";
fErrorCodes[ERR_IDX_LIST_UPDATE_NARRAY] = " in an index list update narray";
fErrorCodes[ERR_IDX_LIST_LAST_FBO_NEG] = " the last index list FBO neg";
fErrorCodes[ERR_IDX_LIST_INIT_NEW_BLKS] = " in an index list initialize new blocks";
fErrorCodes[ERR_IDX_LIST_INIT_LINK_BLKS] = " in an index list initialize link blocks";
fErrorCodes[ERR_IDX_LIST_UPDATE_COUNT] = " in an index list update count";
fErrorCodes[ERR_IDX_LIST_SET_NEXT_LBID] = " in an index list set next LBID";
fErrorCodes[ERR_IDX_LIST_INVALID_LBID] = "an index list invalid LBID";
fErrorCodes[ERR_IDX_LIST_INVALID_BLK_READ] = " in an index list invalid LBID read";
fErrorCodes[ERR_IDX_LIST_UPDATE_HDR_COUNT] = " in an index list update header count";
fErrorCodes[ERR_IDX_LIST_WRONG_BLK] = " an index list wrong block";
fErrorCodes[ERR_IDX_LIST_WRONG_TYPE] = " an index list wrong type";
fErrorCodes[ERR_IDX_LIST_GET_COUNT] = " in an index list get count";
fErrorCodes[ERR_IDX_LIST_GET_NEXT] = " in an index list get next";
fErrorCodes[ERR_IDX_LIST_GET_PARENT] = " in an index list get parent";
fErrorCodes[ERR_IDX_LIST_GET_SUB_BLK] = " in an index list get sub block";
fErrorCodes[ERR_IDX_LIST_INVALID_UP_HDR] = " an invalid Update Index List header ";
fErrorCodes[ERR_IDX_LIST_INVALID_ADD_LIST] = " an invalid add Index List";
fErrorCodes[ERR_IDX_LIST_INVALID_UP] = " an invalid Update Index List";
// index error
fErrorCodes[ERR_IDX_TREE_MOVE_ENTRY] = " an error in moving part of an index tree to a new subblock";
fErrorCodes[ERR_IDX_TREE_INVALID_TYPE] = " an Invalid index tree entry type";
fErrorCodes[ERR_IDX_TREE_BITTEST_VAL] = " a Wrong bit test value in the index tree entry";
fErrorCodes[ERR_IDX_TREE_INVALID_LEVEL] = " an Invalid testbit index tree level";
fErrorCodes[ERR_IDX_TREE_INVALID_GRP] = "an Invalid index tree group type ";
fErrorCodes[ERR_IDX_TREE_LISTPTR_CHANGE] = " an index tree List pointer change";
// index list error
fErrorCodes[ERR_IDX_LIST_INVALID_ADDHDR] = " a Create indexlist header error";
fErrorCodes[ERR_IDX_LIST_INVALID_UPDATE] = " an pdate Index List error ";
fErrorCodes[ERR_IDX_LIST_INVALID_DELETE] = " a Delete rowid in indexlist err";
fErrorCodes[ERR_IDX_LIST_INVALID_KEY] = " an Invalid index listbppseeder.cpp Key passed";
fErrorCodes[ERR_IDX_LIST_GET_RID_ARRARY] = " an index list RID array";
fErrorCodes[ERR_IDX_LIST_WRONG_KEY] = " a not matched Key passed to an index list";
fErrorCodes[ERR_IDX_LIST_HDR_EMPTY] = " an empty index list header";
fErrorCodes[ERR_IDX_LIST_GET_SEGMT] = " in an index list Get Segment";
fErrorCodes[ERR_IDX_LIST_WRONG_LBID_WRITE] = " an index list incorrect LBID write";
fErrorCodes[ERR_IDX_LIST_UPDATE_SUB] = " in an index list update sub";
fErrorCodes[ERR_IDX_LIST_UPDATE_NARRAY] = " in an index list update narray";
fErrorCodes[ERR_IDX_LIST_LAST_FBO_NEG] = " the last index list FBO neg";
fErrorCodes[ERR_IDX_LIST_INIT_NEW_BLKS] = " in an index list initialize new blocks";
fErrorCodes[ERR_IDX_LIST_INIT_LINK_BLKS] = " in an index list initialize link blocks";
fErrorCodes[ERR_IDX_LIST_UPDATE_COUNT] = " in an index list update count";
fErrorCodes[ERR_IDX_LIST_SET_NEXT_LBID] = " in an index list set next LBID";
fErrorCodes[ERR_IDX_LIST_INVALID_LBID] = "an index list invalid LBID";
fErrorCodes[ERR_IDX_LIST_INVALID_BLK_READ] = " in an index list invalid LBID read";
fErrorCodes[ERR_IDX_LIST_UPDATE_HDR_COUNT] = " in an index list update header count";
fErrorCodes[ERR_IDX_LIST_WRONG_BLK] = " an index list wrong block";
fErrorCodes[ERR_IDX_LIST_WRONG_TYPE] = " an index list wrong type";
fErrorCodes[ERR_IDX_LIST_GET_COUNT] = " in an index list get count";
fErrorCodes[ERR_IDX_LIST_GET_NEXT] = " in an index list get next";
fErrorCodes[ERR_IDX_LIST_GET_PARENT] = " in an index list get parent";
fErrorCodes[ERR_IDX_LIST_GET_SUB_BLK] = " in an index list get sub block";
fErrorCodes[ERR_IDX_LIST_INVALID_UP_HDR] = " an invalid Update Index List header ";
fErrorCodes[ERR_IDX_LIST_INVALID_ADD_LIST] = " an invalid add Index List";
fErrorCodes[ERR_IDX_LIST_INVALID_UP] = " an invalid Update Index List";
//freemgr error
fErrorCodes[ERR_FM_ASSIGN_ERR] = " in an assignment";
fErrorCodes[ERR_FM_RELEASE_ERR] = " in a release";
fErrorCodes[ERR_FM_BAD_FBO] = " an invalid File Block Offset";
fErrorCodes[ERR_FM_BAD_TYPE] = "an invalid type that must be pointer or list";
fErrorCodes[ERR_FM_NO_SPACE] = " that No blocks are available";
fErrorCodes[ERR_FM_EXTEND] = " while extending a file";
// freemgr error
fErrorCodes[ERR_FM_ASSIGN_ERR] = " in an assignment";
fErrorCodes[ERR_FM_RELEASE_ERR] = " in a release";
fErrorCodes[ERR_FM_BAD_FBO] = " an invalid File Block Offset";
fErrorCodes[ERR_FM_BAD_TYPE] = "an invalid type that must be pointer or list";
fErrorCodes[ERR_FM_NO_SPACE] = " that No blocks are available";
fErrorCodes[ERR_FM_EXTEND] = " while extending a file";
// Dictionary error
fErrorCodes[ERR_DICT_NO_SPACE_INSERT] = " no space for a dictionary insert";
fErrorCodes[ERR_DICT_SIZE_GT_2G] = " the dictionary size was > 2GB";
fErrorCodes[ERR_DICT_NO_OP_DELETE] = " in the dictionary no op delete";
fErrorCodes[ERR_DICT_NO_OFFSET_DELETE] = " a dictionary bad Delete offset";
fErrorCodes[ERR_DICT_INVALID_HDR] = " a dictionary bad Delete Hdr";
fErrorCodes[ERR_DICT_ZERO_LEN] = " a dictionary zero len";
fErrorCodes[ERR_DICT_TOKEN_NOT_FOUND] = " a dictionary token not found";
fErrorCodes[ERR_DICT_FILE_NOT_FOUND] = " a dictionary file not found";
fErrorCodes[ERR_DICT_BAD_TOKEN_LBID] = " a dictionary token lbid is bad";
fErrorCodes[ERR_DICT_BAD_TOKEN_OP] = " a dictionary token op is bad";
// Dictionary error
fErrorCodes[ERR_DICT_NO_SPACE_INSERT] = " no space for a dictionary insert";
fErrorCodes[ERR_DICT_SIZE_GT_2G] = " the dictionary size was > 2GB";
fErrorCodes[ERR_DICT_NO_OP_DELETE] = " in the dictionary no op delete";
fErrorCodes[ERR_DICT_NO_OFFSET_DELETE] = " a dictionary bad Delete offset";
fErrorCodes[ERR_DICT_INVALID_HDR] = " a dictionary bad Delete Hdr";
fErrorCodes[ERR_DICT_ZERO_LEN] = " a dictionary zero len";
fErrorCodes[ERR_DICT_TOKEN_NOT_FOUND] = " a dictionary token not found";
fErrorCodes[ERR_DICT_FILE_NOT_FOUND] = " a dictionary file not found";
fErrorCodes[ERR_DICT_BAD_TOKEN_LBID] = " a dictionary token lbid is bad";
fErrorCodes[ERR_DICT_BAD_TOKEN_OP] = " a dictionary token op is bad";
// Bulk error
fErrorCodes[ERR_BULK_MAX_ERR_NUM] = " the Maximum number of error rows reached";
fErrorCodes[ERR_BULK_DATA_COL_NUM] = " the total number of data column not match with column definitions";
fErrorCodes[ERR_BULK_SEND_MSG_ERR] = " in a bulk load send msg";
fErrorCodes[ERR_BULK_MISSING_EXTENT_ENTRY] = " missing Extent Entry when trying to save LBID info for CP";
fErrorCodes[ERR_BULK_MISSING_EXTENT_ROW] = " missing Extent Row when trying to save LBID info for CP";
fErrorCodes[ERR_BULK_ROW_FILL_BUFFER] = " Single row fills read buffer; try larger read buffer.";
fErrorCodes[ERR_BULK_DBROOT_CHANGE] = " Local PM DBRoot settings changed during bulk load.";
fErrorCodes[ERR_BULK_ROLLBACK_MISS_ROOT] = " Mode3 automatic rollback not performed. DBRoot missing.";
fErrorCodes[ERR_BULK_ROLLBACK_SEG_LIST] = " Error building segment file list in a directory.";
fErrorCodes[ERR_BULK_BINARY_PARTIAL_REC] = " Binary import did not end on fixed length record boundary.";
fErrorCodes[ERR_BULK_BINARY_IGNORE_FLD] = " <IgnoreField> tag not supported for binary imports.";
// Bulk error
fErrorCodes[ERR_BULK_MAX_ERR_NUM] = " the Maximum number of error rows reached";
fErrorCodes[ERR_BULK_DATA_COL_NUM] = " the total number of data column not match with column definitions";
fErrorCodes[ERR_BULK_SEND_MSG_ERR] = " in a bulk load send msg";
fErrorCodes[ERR_BULK_MISSING_EXTENT_ENTRY] = " missing Extent Entry when trying to save LBID info for CP";
fErrorCodes[ERR_BULK_MISSING_EXTENT_ROW] = " missing Extent Row when trying to save LBID info for CP";
fErrorCodes[ERR_BULK_ROW_FILL_BUFFER] = " Single row fills read buffer; try larger read buffer.";
fErrorCodes[ERR_BULK_DBROOT_CHANGE] = " Local PM DBRoot settings changed during bulk load.";
fErrorCodes[ERR_BULK_ROLLBACK_MISS_ROOT] = " Mode3 automatic rollback not performed. DBRoot missing.";
fErrorCodes[ERR_BULK_ROLLBACK_SEG_LIST] = " Error building segment file list in a directory.";
fErrorCodes[ERR_BULK_BINARY_PARTIAL_REC] = " Binary import did not end on fixed length record boundary.";
fErrorCodes[ERR_BULK_BINARY_IGNORE_FLD] = " <IgnoreField> tag not supported for binary imports.";
// BRM error
fErrorCodes[ERR_BRM_LOOKUP_LBID] = " a BRM Lookup LBID error.";
fErrorCodes[ERR_BRM_LOOKUP_FBO] = " a BRM Lookup FBO error.";
fErrorCodes[ERR_BRM_ALLOC_EXTEND] = " a BRM Allocate extent error.";
fErrorCodes[ERR_BRM_COMMIT] = " a BRM Commit error.";
fErrorCodes[ERR_BRM_ROLLBACK] = " a BRM Rollback error.";
fErrorCodes[ERR_BRM_GET_UNCOMM_LBID] = " a BRM get uncommitted lbid list error.";
fErrorCodes[ERR_BRM_DEL_OID] = " a BRM Delete oid error.";
fErrorCodes[ERR_BRM_BEGIN_COPY] = " a BRM Begin copy error.";
fErrorCodes[ERR_BRM_END_COPY] = " a BRM End copy error.";
fErrorCodes[ERR_BRM_GET_HWM] = " a BRM get hwm error.";
fErrorCodes[ERR_BRM_SET_HWM] = " a BRM Set hwm error.";
fErrorCodes[ERR_BRM_WR_VB_ENTRY] = " a BRM VB entry error.";
fErrorCodes[ERR_BRM_VB_COPY_READ] = " a BRM VB copy read error.";
fErrorCodes[ERR_BRM_VB_COPY_SEEK_DB] = " a BRM VB copy seek error against DB file.";
fErrorCodes[ERR_BRM_VB_COPY_SEEK_VB] = " a BRM VB copy seek error against VB file.";
fErrorCodes[ERR_BRM_VB_COPY_WRITE] = " a BRM VB copy write.";
fErrorCodes[ERR_BRM_DEAD_LOCK] = " a BRM DEAD lock error.";
fErrorCodes[ERR_BRM_MARK_INVALID] = " a BRM Mark extent invalid error from casual paritioning.";
fErrorCodes[ERR_BRM_SAVE_STATE] = " a BRM Save state error.";
fErrorCodes[ERR_BRM_GET_START_EXTENT] = " a BRM get start Extent error.";
fErrorCodes[ERR_BRM_VB_OVERFLOW] = "BRM block version buffer overflow error.";
fErrorCodes[ERR_BRM_READ_ONLY] = "BRM is in read-only state.";
fErrorCodes[ERR_BRM_GET_READ_WRITE] = "BRM error getting read-write state.";
fErrorCodes[ERR_BRM_BULK_RB_COLUMN] = "BRM error performing bulk rollback of column extents.";
fErrorCodes[ERR_BRM_BULK_RB_DCTNRY] = "BRM error performing bulk rollback of dictionary store extents.";
fErrorCodes[ERR_BRM_DELETE_EXTENT_COLUMN] = "BRM error deleting column extents.";
fErrorCodes[ERR_BRM_DELETE_EXTENT_DCTNRY] = "BRM error deleting dictionary extents.";
fErrorCodes[ERR_BRM_TAKE_SNAPSHOT] = "BRM error requesting snapshot of BRM state.";
fErrorCodes[ERR_BRM_LOOKUP_START_LBID] = "BRM start LBID lookup error.";
fErrorCodes[ERR_BRM_BULK_UPDATE] = "BRM error executing bulk update of HWM and CP.";
fErrorCodes[ERR_BRM_GET_EXT_STATE] = "BRM error getting segment file extent state.";
fErrorCodes[ERR_EXTENTMAP_LOOKUP] = " a extent map Lookup error.";
fErrorCodes[ERR_BRM_LOOKUP_VERSION] = " a vssLookup version info error.";
fErrorCodes[ERR_BRM_LOOKUP_LBID_RANGES] = " BRM error getting LBID ranges.";
fErrorCodes[ERR_BRM_HWMS_NOT_EQUAL] = " HWMs for same width columns not equal. ";
fErrorCodes[ERR_BRM_HWMS_OUT_OF_SYNC] = " HWMs for different width columns not in sync. ";
fErrorCodes[ERR_BRM_DBROOT_HWMS] = " BRM error getting HWMs for DBRoots. ";
fErrorCodes[ERR_BRM_NETWORK] = " Network error in DBRM call. ";
fErrorCodes[ERR_BRM_READONLY] = " DBRM is read only. ";
fErrorCodes[ERR_INVALID_VBOID] = " The VB oid is invalid ";
fErrorCodes[ERR_BRM_SET_EXTENTS_CP] = " BRM error setting extents min/max ";
fErrorCodes[ERR_BRM_SHUTDOWN] = " The system is being shutdown ";
fErrorCodes[ERR_BRM_GET_SHUTDOWN] = " BRM error get the system shutdown flag ";
fErrorCodes[ERR_BRM_SUSPEND] = " The system is in write suspended mode";
fErrorCodes[ERR_BRM_GET_SUSPEND] = " BRM error get the system suspend flag ";
fErrorCodes[ERR_BRM_BAD_STRIPE_CNT] = " Incorrect number of column extents allocated in stripe";
fErrorCodes[ERR_BRM_UNSUPP_WIDTH] = " Unsupported non-dictionary column width";
fErrorCodes[ERR_BRM_GET_EXTENT_CP] = " BRM error getting extent max/min";
// BRM error
fErrorCodes[ERR_BRM_LOOKUP_LBID] = " a BRM Lookup LBID error.";
fErrorCodes[ERR_BRM_LOOKUP_FBO] = " a BRM Lookup FBO error.";
fErrorCodes[ERR_BRM_ALLOC_EXTEND] = " a BRM Allocate extent error.";
fErrorCodes[ERR_BRM_COMMIT] = " a BRM Commit error.";
fErrorCodes[ERR_BRM_ROLLBACK] = " a BRM Rollback error.";
fErrorCodes[ERR_BRM_GET_UNCOMM_LBID] = " a BRM get uncommitted lbid list error.";
fErrorCodes[ERR_BRM_DEL_OID] = " a BRM Delete oid error.";
fErrorCodes[ERR_BRM_BEGIN_COPY] = " a BRM Begin copy error.";
fErrorCodes[ERR_BRM_END_COPY] = " a BRM End copy error.";
fErrorCodes[ERR_BRM_GET_HWM] = " a BRM get hwm error.";
fErrorCodes[ERR_BRM_SET_HWM] = " a BRM Set hwm error.";
fErrorCodes[ERR_BRM_WR_VB_ENTRY] = " a BRM VB entry error.";
fErrorCodes[ERR_BRM_VB_COPY_READ] = " a BRM VB copy read error.";
fErrorCodes[ERR_BRM_VB_COPY_SEEK_DB] = " a BRM VB copy seek error against DB file.";
fErrorCodes[ERR_BRM_VB_COPY_SEEK_VB] = " a BRM VB copy seek error against VB file.";
fErrorCodes[ERR_BRM_VB_COPY_WRITE] = " a BRM VB copy write.";
fErrorCodes[ERR_BRM_DEAD_LOCK] = " a BRM DEAD lock error.";
fErrorCodes[ERR_BRM_MARK_INVALID] = " a BRM Mark extent invalid error from casual paritioning.";
fErrorCodes[ERR_BRM_SAVE_STATE] = " a BRM Save state error.";
fErrorCodes[ERR_BRM_GET_START_EXTENT] = " a BRM get start Extent error.";
fErrorCodes[ERR_BRM_VB_OVERFLOW] = "BRM block version buffer overflow error.";
fErrorCodes[ERR_BRM_READ_ONLY] = "BRM is in read-only state.";
fErrorCodes[ERR_BRM_GET_READ_WRITE] = "BRM error getting read-write state.";
fErrorCodes[ERR_BRM_BULK_RB_COLUMN] = "BRM error performing bulk rollback of column extents.";
fErrorCodes[ERR_BRM_BULK_RB_DCTNRY] = "BRM error performing bulk rollback of dictionary store extents.";
fErrorCodes[ERR_BRM_DELETE_EXTENT_COLUMN] = "BRM error deleting column extents.";
fErrorCodes[ERR_BRM_DELETE_EXTENT_DCTNRY] = "BRM error deleting dictionary extents.";
fErrorCodes[ERR_BRM_TAKE_SNAPSHOT] = "BRM error requesting snapshot of BRM state.";
fErrorCodes[ERR_BRM_LOOKUP_START_LBID] = "BRM start LBID lookup error.";
fErrorCodes[ERR_BRM_BULK_UPDATE] = "BRM error executing bulk update of HWM and CP.";
fErrorCodes[ERR_BRM_GET_EXT_STATE] = "BRM error getting segment file extent state.";
fErrorCodes[ERR_EXTENTMAP_LOOKUP] = " a extent map Lookup error.";
fErrorCodes[ERR_BRM_LOOKUP_VERSION] = " a vssLookup version info error.";
fErrorCodes[ERR_BRM_LOOKUP_LBID_RANGES] = " BRM error getting LBID ranges.";
fErrorCodes[ERR_BRM_HWMS_NOT_EQUAL] = " HWMs for same width columns not equal. ";
fErrorCodes[ERR_BRM_HWMS_OUT_OF_SYNC] = " HWMs for different width columns not in sync. ";
fErrorCodes[ERR_BRM_DBROOT_HWMS] = " BRM error getting HWMs for DBRoots. ";
fErrorCodes[ERR_BRM_NETWORK] = " Network error in DBRM call. ";
fErrorCodes[ERR_BRM_READONLY] = " DBRM is read only. ";
fErrorCodes[ERR_INVALID_VBOID] = " The VB oid is invalid ";
fErrorCodes[ERR_BRM_SET_EXTENTS_CP] = " BRM error setting extents min/max ";
fErrorCodes[ERR_BRM_SHUTDOWN] = " The system is being shutdown ";
fErrorCodes[ERR_BRM_GET_SHUTDOWN] = " BRM error get the system shutdown flag ";
fErrorCodes[ERR_BRM_SUSPEND] = " The system is in write suspended mode";
fErrorCodes[ERR_BRM_GET_SUSPEND] = " BRM error get the system suspend flag ";
fErrorCodes[ERR_BRM_BAD_STRIPE_CNT] = " Incorrect number of column extents allocated in stripe";
fErrorCodes[ERR_BRM_UNSUPP_WIDTH] = " Unsupported non-dictionary column width";
fErrorCodes[ERR_BRM_GET_EXTENT_CP] = " BRM error getting extent max/min";
// DM error
fErrorCodes[ERR_DM_CONVERT_OID] = " a DM Conversion error";
// DM error
fErrorCodes[ERR_DM_CONVERT_OID] = " a DM Conversion error";
// Cache error
fErrorCodes[ERR_CACHE_KEY_EXIST ] = " a Cache key exists";
fErrorCodes[ERR_CACHE_KEY_NOT_EXIST] = " a Cache key does not exist";
fErrorCodes[ERR_NULL_BLOCK] = " a Block is NULL";
fErrorCodes[ERR_FREE_LIST_EMPTY] = " a Free list is empty";
// Cache error
fErrorCodes[ERR_CACHE_KEY_EXIST] = " a Cache key exists";
fErrorCodes[ERR_CACHE_KEY_NOT_EXIST] = " a Cache key does not exist";
fErrorCodes[ERR_NULL_BLOCK] = " a Block is NULL";
fErrorCodes[ERR_FREE_LIST_EMPTY] = " a Free list is empty";
// Compression error
fErrorCodes[ERR_COMP_COMPRESS] = " Error in compressing data. ";
fErrorCodes[ERR_COMP_UNCOMPRESS] = " Error in uncompressing data. ";
fErrorCodes[ERR_COMP_PARSE_HDRS] = " Error parsing compression headers. ";
fErrorCodes[ERR_COMP_VERIFY_HDRS] = " Error verifying compression headers. ";
fErrorCodes[ERR_COMP_PAD_DATA] = " Error in padding compressed data. ";
fErrorCodes[ERR_COMP_READ_BLOCK] = " Error in reading a data block. ";
fErrorCodes[ERR_COMP_SAVE_BLOCK] = " Error in saving a data block. ";
fErrorCodes[ERR_COMP_WRONG_PTR] = " Invalid pointer in compression headers. ";
fErrorCodes[ERR_COMP_FILE_NOT_FOUND] = " Error searching for a compressed file. ";
fErrorCodes[ERR_COMP_CHUNK_NOT_FOUND] = " Error searching for a compressed chunk. ";
fErrorCodes[ERR_COMP_UNAVAIL_TYPE] = " Unavailable compressino type. ";
fErrorCodes[ERR_COMP_REMOVE_FILE] = " Failed to remove a file. ";
fErrorCodes[ERR_COMP_RENAME_FILE] = " Failed to rename a file. ";
fErrorCodes[ERR_COMP_OPEN_FILE] = " Failed to open a compressed data file. ";
fErrorCodes[ERR_COMP_SET_OFFSET] = " Failed to set offset in a compressed data file. ";
fErrorCodes[ERR_COMP_READ_FILE] = " Failed to read from a compressed data file. ";
fErrorCodes[ERR_COMP_WRITE_FILE] = " Failed to write to a compresssed data file. ";
fErrorCodes[ERR_COMP_CLOSE_FILE] = " Failed to close a compressed data file. ";
fErrorCodes[ERR_COMP_TRUNCATE_ZERO] = " Attempting to truncate compressed file to 0 bytes. ";
// Compression error
fErrorCodes[ERR_COMP_COMPRESS] = " Error in compressing data. ";
fErrorCodes[ERR_COMP_UNCOMPRESS] = " Error in uncompressing data. ";
fErrorCodes[ERR_COMP_PARSE_HDRS] = " Error parsing compression headers. ";
fErrorCodes[ERR_COMP_VERIFY_HDRS] = " Error verifying compression headers. ";
fErrorCodes[ERR_COMP_PAD_DATA] = " Error in padding compressed data. ";
fErrorCodes[ERR_COMP_READ_BLOCK] = " Error in reading a data block. ";
fErrorCodes[ERR_COMP_SAVE_BLOCK] = " Error in saving a data block. ";
fErrorCodes[ERR_COMP_WRONG_PTR] = " Invalid pointer in compression headers. ";
fErrorCodes[ERR_COMP_FILE_NOT_FOUND] = " Error searching for a compressed file. ";
fErrorCodes[ERR_COMP_CHUNK_NOT_FOUND] = " Error searching for a compressed chunk. ";
fErrorCodes[ERR_COMP_UNAVAIL_TYPE] = " Unavailable compressino type. ";
fErrorCodes[ERR_COMP_REMOVE_FILE] = " Failed to remove a file. ";
fErrorCodes[ERR_COMP_RENAME_FILE] = " Failed to rename a file. ";
fErrorCodes[ERR_COMP_OPEN_FILE] = " Failed to open a compressed data file. ";
fErrorCodes[ERR_COMP_SET_OFFSET] = " Failed to set offset in a compressed data file. ";
fErrorCodes[ERR_COMP_READ_FILE] = " Failed to read from a compressed data file. ";
fErrorCodes[ERR_COMP_WRITE_FILE] = " Failed to write to a compresssed data file. ";
fErrorCodes[ERR_COMP_CLOSE_FILE] = " Failed to close a compressed data file. ";
fErrorCodes[ERR_COMP_TRUNCATE_ZERO] = " Attempting to truncate compressed file to 0 bytes. ";
// Auto-increment error
fErrorCodes[ERR_AUTOINC_GEN_EXCEED_MAX] = " Generated auto-increment value "
"exceeds maximum value for the column type.";
fErrorCodes[ERR_AUTOINC_USER_OUT_OF_RANGE] = " User specified auto-"
"increment value is out of range for the column type.";
fErrorCodes[ERR_AUTOINC_TABLE_NAME] = " Invalid schema/tablename for auto increment. ";
fErrorCodes[ERR_AUTOINC_INIT1] = " Unable to initialize auto-increment value. ";
fErrorCodes[ERR_AUTOINC_INIT2] = " Unable to initialize auto-increment value. Unknown exception. ";
fErrorCodes[ERR_AUTOINC_RID] = " Failed to get row information from calpontsystemcatalog.";
fErrorCodes[ERR_AUTOINC_START_SEQ] = " Unable to setup AI sequence in BRM.";
fErrorCodes[ERR_AUTOINC_GET_RANGE] = " Unable to reserve AI range from BRM.";
fErrorCodes[ERR_AUTOINC_GET_LOCK] = " Unable to lock AI column in BRM.";
fErrorCodes[ERR_AUTOINC_REL_LOCK] = " Unable to release AI column in BRM.";
fErrorCodes[ERR_AUTOINC_UPDATE] = " Unable to update nextValue in system catalog.";
// Auto-increment error
fErrorCodes[ERR_AUTOINC_GEN_EXCEED_MAX] =
" Generated auto-increment value "
"exceeds maximum value for the column type.";
fErrorCodes[ERR_AUTOINC_USER_OUT_OF_RANGE] =
" User specified auto-"
"increment value is out of range for the column type.";
fErrorCodes[ERR_AUTOINC_TABLE_NAME] = " Invalid schema/tablename for auto increment. ";
fErrorCodes[ERR_AUTOINC_INIT1] = " Unable to initialize auto-increment value. ";
fErrorCodes[ERR_AUTOINC_INIT2] = " Unable to initialize auto-increment value. Unknown exception. ";
fErrorCodes[ERR_AUTOINC_RID] = " Failed to get row information from calpontsystemcatalog.";
fErrorCodes[ERR_AUTOINC_START_SEQ] = " Unable to setup AI sequence in BRM.";
fErrorCodes[ERR_AUTOINC_GET_RANGE] = " Unable to reserve AI range from BRM.";
fErrorCodes[ERR_AUTOINC_GET_LOCK] = " Unable to lock AI column in BRM.";
fErrorCodes[ERR_AUTOINC_REL_LOCK] = " Unable to release AI column in BRM.";
fErrorCodes[ERR_AUTOINC_UPDATE] = " Unable to update nextValue in system catalog.";
// Block cache flush error
fErrorCodes[ERR_BLKCACHE_FLUSH_LIST] = " Failed to flush list of blocks from PrimProc cache. ";
// Block cache flush error
fErrorCodes[ERR_BLKCACHE_FLUSH_LIST] = " Failed to flush list of blocks from PrimProc cache. ";
// Backup bulk meta data file error
fErrorCodes[ERR_METADATABKUP_FILE_RENAME] = " Unable to rename temporary bulk meta data file. ";
fErrorCodes[ERR_METADATABKUP_COMP_PARSE_HDRS] = " Error parsing compression headers in bulk backup file. ";
fErrorCodes[ERR_METADATABKUP_COMP_VERIFY_HDRS] = " Error verifying compression headers in bulk backup file. ";
fErrorCodes[ERR_METADATABKUP_COMP_CHUNK_NOT_FOUND] = " Error searching for compressed chunk in db file being backed up. ";
fErrorCodes[ERR_METADATABKUP_COMP_OPEN_BULK_BKUP] = " Error opening compressed chunk in bulk backup file. ";
fErrorCodes[ERR_METADATABKUP_COMP_WRITE_BULK_BKUP] = " Error writing compressed chunk to bulk backup file. ";
fErrorCodes[ERR_METADATABKUP_COMP_READ_BULK_BKUP] = " Error reading compressed chunk from bulk backup file. ";
fErrorCodes[ERR_METADATABKUP_COMP_RENAME] = " Unable to rename compressed chunk bulk backup file. ";
// Backup bulk meta data file error
fErrorCodes[ERR_METADATABKUP_FILE_RENAME] = " Unable to rename temporary bulk meta data file. ";
fErrorCodes[ERR_METADATABKUP_COMP_PARSE_HDRS] = " Error parsing compression headers in bulk backup file. ";
fErrorCodes[ERR_METADATABKUP_COMP_VERIFY_HDRS] =
" Error verifying compression headers in bulk backup file. ";
fErrorCodes[ERR_METADATABKUP_COMP_CHUNK_NOT_FOUND] =
" Error searching for compressed chunk in db file being backed up. ";
fErrorCodes[ERR_METADATABKUP_COMP_OPEN_BULK_BKUP] = " Error opening compressed chunk in bulk backup file. ";
fErrorCodes[ERR_METADATABKUP_COMP_WRITE_BULK_BKUP] =
" Error writing compressed chunk to bulk backup file. ";
fErrorCodes[ERR_METADATABKUP_COMP_READ_BULK_BKUP] =
" Error reading compressed chunk from bulk backup file. ";
fErrorCodes[ERR_METADATABKUP_COMP_RENAME] = " Unable to rename compressed chunk bulk backup file. ";
}
std::string WErrorCodes::errorString(int code)
{
// Look for error message overrides from system-wide error messages
switch (code)
// Look for error message overrides from system-wide error messages
switch (code)
{
case ERR_FILE_DISK_SPACE:
{
case ERR_FILE_DISK_SPACE:
{
logging::Message::Args args;
std::string msgArg; // empty str arg; no extra info in this context
args.add( msgArg );
return logging::IDBErrorInfo::instance()->errorMsg(
logging::ERR_EXTENT_DISK_SPACE, args);
break;
}
logging::Message::Args args;
std::string msgArg; // empty str arg; no extra info in this context
args.add(msgArg);
return logging::IDBErrorInfo::instance()->errorMsg(logging::ERR_EXTENT_DISK_SPACE, args);
break;
}
}
int brmRc = BRMWrapper::getBrmRc();
int brmRc = BRMWrapper::getBrmRc();
if (brmRc == BRM::ERR_OK)
return (fErrorCodes[code]);
if (brmRc == BRM::ERR_OK)
return (fErrorCodes[code]);
std::string errMsg( fErrorCodes[code] );
std::string brmMsg;
errMsg += " [BRM error status: ";
BRM::errString(brmRc, brmMsg);
errMsg += brmMsg;
errMsg += "]";
std::string errMsg(fErrorCodes[code]);
std::string brmMsg;
errMsg += " [BRM error status: ";
BRM::errString(brmRc, brmMsg);
errMsg += brmMsg;
errMsg += "]";
return errMsg;
return errMsg;
}
} //end of namespace
} // namespace WriteEngine

529
writeengine/shared/we_define.h
View File

@ -37,363 +37,374 @@ namespace WriteEngine
{
// Max column size is 16 bytes since MCOL-641. However left this value
// for backward compatibility
const short MAX_COLUMN_BOUNDARY = 8; // Max bytes for one column
const int MAX_SIGNATURE_SIZE = 8000; // Max len of dict sig val
const int MAX_FIELD_SIZE = 1000; // Max len non-dict fld val
const int MAX_DB_DIR_LEVEL = 6; // Max lvl of db dir struct
const int MAX_DB_DIR_NAME_SIZE = 20; // Max len of db dir size
const short ROW_PER_BYTE = 8; // Rows/byte in bitmap file
const int BYTE_PER_BLOCK = 8192; // Num bytes per data block
const int BYTE_PER_SUBBLOCK = 256; // Num bytes per sub block
const int ENTRY_PER_SUBBLOCK = 32; // Num entries per sub block
const int INITIAL_EXTENT_ROWS_TO_DISK = 256 * 1024; // Used for initial number of blocks calculation
const int MAX_INITIAL_EXTENT_BLOCKS_TO_DISK = 256; // Number of blocks in abbrev extent for 8byte col.
const short MAX_COLUMN_BOUNDARY = 8; // Max bytes for one column
const int MAX_SIGNATURE_SIZE = 8000; // Max len of dict sig val
const int MAX_FIELD_SIZE = 1000; // Max len non-dict fld val
const int MAX_DB_DIR_LEVEL = 6; // Max lvl of db dir struct
const int MAX_DB_DIR_NAME_SIZE = 20; // Max len of db dir size
const short ROW_PER_BYTE = 8; // Rows/byte in bitmap file
const int BYTE_PER_BLOCK = 8192; // Num bytes per data block
const int BYTE_PER_SUBBLOCK = 256; // Num bytes per sub block
const int ENTRY_PER_SUBBLOCK = 32; // Num entries per sub block
const int INITIAL_EXTENT_ROWS_TO_DISK = 256 * 1024; // Used for initial number of blocks calculation
const int MAX_INITIAL_EXTENT_BLOCKS_TO_DISK = 256; // Number of blocks in abbrev extent for 8byte col.
// Num rows reserved to disk for 'initial' extent
const int FILE_NAME_SIZE = 200; // Max size of file name
const long long MAX_ALLOW_ERROR_COUNT = 100000; //Max allowable error count
const int FILE_NAME_SIZE = 200; // Max size of file name
const long long MAX_ALLOW_ERROR_COUNT = 100000; // Max allowable error count
//--------------------------------------------------------------------------
// Dictionary related constants
//--------------------------------------------------------------------------
const uint16_t DCTNRY_END_HEADER = 0xffff ; // end of header
const uint64_t NOT_USED_PTR = 0x0 ; // not continuous ptr
const int HDR_UNIT_SIZE = 2; // hdr unit size
const int NEXT_PTR_BYTES = 8; // const ptr size
const int MAX_OP_COUNT = 1024; // op max size
const int DCTNRY_HEADER_SIZE = 14; // header total size
const int MAX_STRING_CACHE_SIZE = 1000;
const uint16_t DCTNRY_END_HEADER = 0xffff; // end of header
const uint64_t NOT_USED_PTR = 0x0; // not continuous ptr
const int HDR_UNIT_SIZE = 2; // hdr unit size
const int NEXT_PTR_BYTES = 8; // const ptr size
const int MAX_OP_COUNT = 1024; // op max size
const int DCTNRY_HEADER_SIZE = 14; // header total size
const int MAX_STRING_CACHE_SIZE = 1000;
// End of Dictionary related constants
const int COLPOSPAIR_NULL_TOKEN_OFFSET = -1; // offset value denoting a null token
const uint32_t BULK_SYSCAT_SESSION_ID = 0; // SessionID for syscat queries
const int COLPOSPAIR_NULL_TOKEN_OFFSET = -1; // offset value denoting a null token
const uint32_t BULK_SYSCAT_SESSION_ID = 0; // SessionID for syscat queries
const char COL_TYPE_DICT = 'D'; // Dictionary type
const char COL_TYPE_DICT = 'D'; // Dictionary type
const uint64_t INVALID_LBID = 0xFFFFFFFFFULL; // 2**36 - 1
const uint64_t INVALID_LBID = 0xFFFFFFFFFULL; // 2**36 - 1
const unsigned int SUBSYSTEM_ID_DDLPROC = 15;
const unsigned int SUBSYSTEM_ID_DMLPROC = 20;
const unsigned int SUBSYSTEM_ID_WE = 19;
const unsigned int SUBSYSTEM_ID_WE_SRV = 32;
const unsigned int SUBSYSTEM_ID_WE = 19;
const unsigned int SUBSYSTEM_ID_WE_SRV = 32;
const unsigned int SUBSYSTEM_ID_WE_SPLIT = 33;
const unsigned int SUBSYSTEM_ID_WE_BULK = 34;
//--------------------------------------------------------------------------
// Default definitions
//--------------------------------------------------------------------------
const int DEFAULT_CACHE_BLOCK = 256; // Max num of cache blocks
const int DEFAULT_CHK_INTERVAL = 3; // Checkpoint in seconds
const int DEFAULT_CACHE_PCT_FREE = 25; // Min % of free cache
const int DEFAULT_BUFSIZ = 1 * 1024 * 1024; // setvbuf buffer size
const int DEFAULT_COLSIZ = 8; // col size for hdfs rdwr buf
const int DEFAULT_CACHE_BLOCK = 256; // Max num of cache blocks
const int DEFAULT_CHK_INTERVAL = 3; // Checkpoint in seconds
const int DEFAULT_CACHE_PCT_FREE = 25; // Min % of free cache
const int DEFAULT_BUFSIZ = 1 * 1024 * 1024; // setvbuf buffer size
const int DEFAULT_COLSIZ = 8; // col size for hdfs rdwr buf
const int BLK_INIT = 0;
const int BLK_READ = 1;
const int BLK_WRITE = 2;
const int BLK_INIT = 0;
const int BLK_READ = 1;
const int BLK_WRITE = 2;
//--------------------------------------------------------------------------
// Return code definitions
//--------------------------------------------------------------------------
const int NO_ERROR = 0; // No error
const int NOT_FOUND = -1; // Not found
const int INVALID_NUM = -1; // Invalid number
const int NO_ERROR = 0; // No error
const int NOT_FOUND = -1; // Not found
const int INVALID_NUM = -1; // Invalid number
//--------------------------------------------------------------------------
// Error code definition
//--------------------------------------------------------------------------
const int ERR_CODEBASE = 1000; // Generic error codes
const int ERR_FILEBASE = 1050; // File-related error codes
const int ERR_XMLBASE = 1150; // XML job file error codes
const int ERR_TBLLOCKBASE = 1200; // Table-lock error codes
const int ERR_WRAPPERBASE = 1250; // DDL/DML API related errors
const int ERR_INDEXBASE = 1300; // Index-related error codes
const int ERR_FMGRBASE = 1350; // Freemgr errors
const int ERR_DCTNRYBASE = 1400; // Dictionary errors
const int ERR_BULKBASE = 1450; // Bulk specific errors
const int ERR_BRMBASE = 1500; // BRM errors
const int ERR_DMBASE = 1550; // Disk manager errors
const int ERR_CACHEBASE = 1600; // Cche management errors
const int ERR_COMPBASE = 1650; // Compression errors
const int ERR_AUTOINCBASE = 1700; // Auto-increment errors
const int ERR_BLKCACHEBASE = 1750; // Block cache flush errors
const int ERR_METABKUPBASE = 1800; // Backup bulk meta file errors
const int ERR_CODEBASE = 1000; // Generic error codes
const int ERR_FILEBASE = 1050; // File-related error codes
const int ERR_XMLBASE = 1150; // XML job file error codes
const int ERR_TBLLOCKBASE = 1200; // Table-lock error codes
const int ERR_WRAPPERBASE = 1250; // DDL/DML API related errors
const int ERR_INDEXBASE = 1300; // Index-related error codes
const int ERR_FMGRBASE = 1350; // Freemgr errors
const int ERR_DCTNRYBASE = 1400; // Dictionary errors
const int ERR_BULKBASE = 1450; // Bulk specific errors
const int ERR_BRMBASE = 1500; // BRM errors
const int ERR_DMBASE = 1550; // Disk manager errors
const int ERR_CACHEBASE = 1600; // Cche management errors
const int ERR_COMPBASE = 1650; // Compression errors
const int ERR_AUTOINCBASE = 1700; // Auto-increment errors
const int ERR_BLKCACHEBASE = 1750; // Block cache flush errors
const int ERR_METABKUPBASE = 1800; // Backup bulk meta file errors
//--------------------------------------------------------------------------
// Generic error
//--------------------------------------------------------------------------
const int ERR_UNKNOWN = ERR_CODEBASE + 1; // Generic error
const int ERR_INVALID_PARAM = ERR_CODEBASE + 2; // Invalid parms
const int ERR_STRUCT_EMPTY = ERR_CODEBASE + 3; // Struct is empty
const int ERR_VALUE_OUTOFRANGE = ERR_CODEBASE + 4; // Val out of range
const int ERR_PARSING = ERR_CODEBASE + 5; // Parsing error
const int ERR_NO_MEM = ERR_CODEBASE + 6; // Mem alloc error
const int ERR_DML_LOG_NAME = ERR_CODEBASE + 7; // DML log filename error
const int ERR_OPEN_DML_LOG = ERR_CODEBASE + 8; // Open DML log file error
const int ERR_HDFS_BACKUP = ERR_CODEBASE + 9; // HDFS backup error
const int ERR_UNKNOWN = ERR_CODEBASE + 1; // Generic error
const int ERR_INVALID_PARAM = ERR_CODEBASE + 2; // Invalid parms
const int ERR_STRUCT_EMPTY = ERR_CODEBASE + 3; // Struct is empty
const int ERR_VALUE_OUTOFRANGE = ERR_CODEBASE + 4; // Val out of range
const int ERR_PARSING = ERR_CODEBASE + 5; // Parsing error
const int ERR_NO_MEM = ERR_CODEBASE + 6; // Mem alloc error
const int ERR_DML_LOG_NAME = ERR_CODEBASE + 7; // DML log filename error
const int ERR_OPEN_DML_LOG = ERR_CODEBASE + 8; // Open DML log file error
const int ERR_HDFS_BACKUP = ERR_CODEBASE + 9; // HDFS backup error
//--------------------------------------------------------------------------
// File level error
//--------------------------------------------------------------------------
const int ERR_FILE_CREATE = ERR_FILEBASE + 1; // File creation error, mostly because file has already existed
const int ERR_FILE_OPEN = ERR_FILEBASE + 2; // Can not open the file, mostly because file not found
const int ERR_FILE_DELETE = ERR_FILEBASE + 3; // Can not delete the file, common reason is file not exist
const int ERR_FILE_EXIST = ERR_FILEBASE + 4; // File alreay exists
const int ERR_FILE_NOT_EXIST = ERR_FILEBASE + 5; // File not exists
const int ERR_FILE_NULL = ERR_FILEBASE + 6; // File is empty
const int ERR_FILE_WRITE = ERR_FILEBASE + 7; // Error writing to a DB file
const int ERR_FILE_READ = ERR_FILEBASE + 8; // Error reading from a DB file
const int ERR_FILE_SEEK = ERR_FILEBASE + 9; // Error in positioning file handle
const int ERR_FILE_READ_IMPORT = ERR_FILEBASE + 10;// Error reading import source file
const int ERR_DIR_CREATE = ERR_FILEBASE + 11;// Error in creating directory
const int ERR_FILE_NEW_EXTENT_FBO = ERR_FILEBASE + 12;// New extent fbo too large
const int ERR_FILE_FBO_NEG = ERR_FILEBASE + 13;// File FBO is negative
const int ERR_FILE_TRUNCATE = ERR_FILEBASE + 14;// Error truncating file
const int ERR_FILE_DISK_SPACE = ERR_FILEBASE + 15;// Out of space on file system
const int ERR_FILE_STAT = ERR_FILEBASE + 16;// Error getting stats on file
const int ERR_VB_FILE_NOT_EXIST = ERR_FILEBASE + 17;// Version buffer file not exists
const int ERR_FILE_FLUSH = ERR_FILEBASE + 18;// Error flushing file
const int ERR_FILE_GLOBBING = ERR_FILEBASE + 19;// Error globbing a file name
const int ERR_FILE_EOF = ERR_FILEBASE + 20;// EOF
const int ERR_FILE_CHOWN = ERR_FILEBASE + 21;// EOF
const int ERR_FILE_CREATE = ERR_FILEBASE + 1; // File creation error, mostly because file has already existed
const int ERR_FILE_OPEN = ERR_FILEBASE + 2; // Can not open the file, mostly because file not found
const int ERR_FILE_DELETE = ERR_FILEBASE + 3; // Can not delete the file, common reason is file not exist
const int ERR_FILE_EXIST = ERR_FILEBASE + 4; // File alreay exists
const int ERR_FILE_NOT_EXIST = ERR_FILEBASE + 5; // File not exists
const int ERR_FILE_NULL = ERR_FILEBASE + 6; // File is empty
const int ERR_FILE_WRITE = ERR_FILEBASE + 7; // Error writing to a DB file
const int ERR_FILE_READ = ERR_FILEBASE + 8; // Error reading from a DB file
const int ERR_FILE_SEEK = ERR_FILEBASE + 9; // Error in positioning file handle
const int ERR_FILE_READ_IMPORT = ERR_FILEBASE + 10; // Error reading import source file
const int ERR_DIR_CREATE = ERR_FILEBASE + 11; // Error in creating directory
const int ERR_FILE_NEW_EXTENT_FBO = ERR_FILEBASE + 12; // New extent fbo too large
const int ERR_FILE_FBO_NEG = ERR_FILEBASE + 13; // File FBO is negative
const int ERR_FILE_TRUNCATE = ERR_FILEBASE + 14; // Error truncating file
const int ERR_FILE_DISK_SPACE = ERR_FILEBASE + 15; // Out of space on file system
const int ERR_FILE_STAT = ERR_FILEBASE + 16; // Error getting stats on file
const int ERR_VB_FILE_NOT_EXIST = ERR_FILEBASE + 17; // Version buffer file not exists
const int ERR_FILE_FLUSH = ERR_FILEBASE + 18; // Error flushing file
const int ERR_FILE_GLOBBING = ERR_FILEBASE + 19; // Error globbing a file name
const int ERR_FILE_EOF = ERR_FILEBASE + 20; // EOF
const int ERR_FILE_CHOWN = ERR_FILEBASE + 21; // EOF
//--------------------------------------------------------------------------
// XML level error
//--------------------------------------------------------------------------
const int ERR_XML_FILE = ERR_XMLBASE + 1; // File error, probably because file does not exist
const int ERR_XML_ROOT_ELEM = ERR_XMLBASE + 2; // Root element err
const int ERR_XML_EMPTY = ERR_XMLBASE + 3; // Empty XML file
const int ERR_XML_PARSE = ERR_XMLBASE + 4; // Parsing error
const int ERR_XML_FILE = ERR_XMLBASE + 1; // File error, probably because file does not exist
const int ERR_XML_ROOT_ELEM = ERR_XMLBASE + 2; // Root element err
const int ERR_XML_EMPTY = ERR_XMLBASE + 3; // Empty XML file
const int ERR_XML_PARSE = ERR_XMLBASE + 4; // Parsing error
//--------------------------------------------------------------------------
// table lock level error
//--------------------------------------------------------------------------
const int ERR_TBLLOCK_LOCK_NOT_FOUND = ERR_TBLLOCKBASE + 1; // table has no lock
const int ERR_TBLLOCK_GET_LOCK = ERR_TBLLOCKBASE + 2; // error acquiring a table lock
const int ERR_TBLLOCK_GET_LOCK_LOCKED = ERR_TBLLOCKBASE + 3; // table currently locked
const int ERR_TBLLOCK_RELEASE_LOCK = ERR_TBLLOCKBASE + 4; // error releasing a table lock
const int ERR_TBLLOCK_CHANGE_STATE = ERR_TBLLOCKBASE + 5; // error changing state of lock
const int ERR_TBLLOCK_GET_INFO = ERR_TBLLOCKBASE + 6; // error getting info about a lock
const int ERR_TBLLOCK_LOCKID_CONFLICT = ERR_TBLLOCKBASE + 7; // lockID for different table than expected
const int ERR_TBLLOCK_LOCK_NOT_FOUND = ERR_TBLLOCKBASE + 1; // table has no lock
const int ERR_TBLLOCK_GET_LOCK = ERR_TBLLOCKBASE + 2; // error acquiring a table lock
const int ERR_TBLLOCK_GET_LOCK_LOCKED = ERR_TBLLOCKBASE + 3; // table currently locked
const int ERR_TBLLOCK_RELEASE_LOCK = ERR_TBLLOCKBASE + 4; // error releasing a table lock
const int ERR_TBLLOCK_CHANGE_STATE = ERR_TBLLOCKBASE + 5; // error changing state of lock
const int ERR_TBLLOCK_GET_INFO = ERR_TBLLOCKBASE + 6; // error getting info about a lock
const int ERR_TBLLOCK_LOCKID_CONFLICT = ERR_TBLLOCKBASE + 7; // lockID for different table than expected
//--------------------------------------------------------------------------
// DDL/DML Interface level error
//--------------------------------------------------------------------------
const int ERR_STRUCT_VALUE_NOT_MATCH = ERR_WRAPPERBASE + 1; // The number of struct not match with the number of value set
const int ERR_ROWID_VALUE_NOT_MATCH = ERR_WRAPPERBASE + 2; // The number of rowid not match with the number of values
const int ERR_TBL_SYSCAT_ERROR = ERR_WRAPPERBASE + 3; /** @brief Syscatalog query error */
const int ERR_STRUCT_VALUE_NOT_MATCH =
ERR_WRAPPERBASE + 1; // The number of struct not match with the number of value set
const int ERR_ROWID_VALUE_NOT_MATCH =
ERR_WRAPPERBASE + 2; // The number of rowid not match with the number of values
const int ERR_TBL_SYSCAT_ERROR = ERR_WRAPPERBASE + 3; /** @brief Syscatalog query error */
//--------------------------------------------------------------------------
// index error
//--------------------------------------------------------------------------
const int ERR_IDX_TREE_MOVE_ENTRY = ERR_INDEXBASE + 1; // The error in move part of tree to a new subblock
const int ERR_IDX_TREE_INVALID_TYPE = ERR_INDEXBASE + 2; // Invalid tree entry type
const int ERR_IDX_TREE_BITTEST_VAL = ERR_INDEXBASE + 3; // Wrong bit test value in the entry
const int ERR_IDX_TREE_INVALID_LEVEL = ERR_INDEXBASE + 4; // Invalid testbit treel level
const int ERR_IDX_TREE_INVALID_GRP = ERR_INDEXBASE + 5; // Invalid group type
const int ERR_IDX_TREE_LISTPTR_CHANGE = ERR_INDEXBASE + 6; // List pointer change
//index list error
const int ERR_IDX_LIST_INVALID_ADDHDR = ERR_INDEXBASE + 7; // Create indexlist header error
const int ERR_IDX_LIST_INVALID_UPDATE = ERR_INDEXBASE + 8; // Update Index List error
const int ERR_IDX_LIST_INVALID_DELETE = ERR_INDEXBASE + 9; // Delete rowid in indexlist err*/
const int ERR_IDX_LIST_INVALID_KEY = ERR_INDEXBASE + 10;// Invalid Key passed
const int ERR_IDX_LIST_GET_RID_ARRARY = ERR_INDEXBASE + 11;// RID array
const int ERR_IDX_LIST_WRONG_KEY = ERR_INDEXBASE + 12;// not matched Key passed
const int ERR_IDX_LIST_HDR_EMPTY = ERR_INDEXBASE + 13;// Delete rowid in indexlist err
const int ERR_IDX_LIST_GET_SEGMT = ERR_INDEXBASE + 14;// Get Segment
const int ERR_IDX_LIST_WRONG_LBID_WRITE = ERR_INDEXBASE + 15;
const int ERR_IDX_LIST_UPDATE_SUB = ERR_INDEXBASE + 16;
const int ERR_IDX_LIST_UPDATE_NARRAY = ERR_INDEXBASE + 17;
const int ERR_IDX_LIST_LAST_FBO_NEG = ERR_INDEXBASE + 18;
const int ERR_IDX_LIST_INIT_NEW_BLKS = ERR_INDEXBASE + 19;
const int ERR_IDX_LIST_INIT_LINK_BLKS = ERR_INDEXBASE + 20;
const int ERR_IDX_LIST_UPDATE_COUNT = ERR_INDEXBASE + 21;
const int ERR_IDX_LIST_SET_NEXT_LBID = ERR_INDEXBASE + 22;
const int ERR_IDX_LIST_INVALID_LBID = ERR_INDEXBASE + 23;
const int ERR_IDX_LIST_INVALID_BLK_READ = ERR_INDEXBASE + 24;
const int ERR_IDX_LIST_UPDATE_HDR_COUNT = ERR_INDEXBASE + 25;
const int ERR_IDX_LIST_WRONG_BLK = ERR_INDEXBASE + 26;
const int ERR_IDX_LIST_WRONG_TYPE = ERR_INDEXBASE + 27;
const int ERR_IDX_LIST_GET_COUNT = ERR_INDEXBASE + 28;
const int ERR_IDX_LIST_GET_NEXT = ERR_INDEXBASE + 29;
const int ERR_IDX_LIST_GET_PARENT = ERR_INDEXBASE + 30;
const int ERR_IDX_LIST_GET_SUB_BLK = ERR_INDEXBASE + 31;
const int ERR_IDX_LIST_INVALID_UP_HDR = ERR_INDEXBASE + 32;// Update Index List error
const int ERR_IDX_LIST_INVALID_ADD_LIST = ERR_INDEXBASE + 33; // Update Index List error
const int ERR_IDX_LIST_INVALID_UP = ERR_INDEXBASE + 34;// Update Index List error
const int ERR_IDX_TREE_MOVE_ENTRY = ERR_INDEXBASE + 1; // The error in move part of tree to a new subblock
const int ERR_IDX_TREE_INVALID_TYPE = ERR_INDEXBASE + 2; // Invalid tree entry type
const int ERR_IDX_TREE_BITTEST_VAL = ERR_INDEXBASE + 3; // Wrong bit test value in the entry
const int ERR_IDX_TREE_INVALID_LEVEL = ERR_INDEXBASE + 4; // Invalid testbit treel level
const int ERR_IDX_TREE_INVALID_GRP = ERR_INDEXBASE + 5; // Invalid group type
const int ERR_IDX_TREE_LISTPTR_CHANGE = ERR_INDEXBASE + 6; // List pointer change
// index list error
const int ERR_IDX_LIST_INVALID_ADDHDR = ERR_INDEXBASE + 7; // Create indexlist header error
const int ERR_IDX_LIST_INVALID_UPDATE = ERR_INDEXBASE + 8; // Update Index List error
const int ERR_IDX_LIST_INVALID_DELETE = ERR_INDEXBASE + 9; // Delete rowid in indexlist err*/
const int ERR_IDX_LIST_INVALID_KEY = ERR_INDEXBASE + 10; // Invalid Key passed
const int ERR_IDX_LIST_GET_RID_ARRARY = ERR_INDEXBASE + 11; // RID array
const int ERR_IDX_LIST_WRONG_KEY = ERR_INDEXBASE + 12; // not matched Key passed
const int ERR_IDX_LIST_HDR_EMPTY = ERR_INDEXBASE + 13; // Delete rowid in indexlist err
const int ERR_IDX_LIST_GET_SEGMT = ERR_INDEXBASE + 14; // Get Segment
const int ERR_IDX_LIST_WRONG_LBID_WRITE = ERR_INDEXBASE + 15;
const int ERR_IDX_LIST_UPDATE_SUB = ERR_INDEXBASE + 16;
const int ERR_IDX_LIST_UPDATE_NARRAY = ERR_INDEXBASE + 17;
const int ERR_IDX_LIST_LAST_FBO_NEG = ERR_INDEXBASE + 18;
const int ERR_IDX_LIST_INIT_NEW_BLKS = ERR_INDEXBASE + 19;
const int ERR_IDX_LIST_INIT_LINK_BLKS = ERR_INDEXBASE + 20;
const int ERR_IDX_LIST_UPDATE_COUNT = ERR_INDEXBASE + 21;
const int ERR_IDX_LIST_SET_NEXT_LBID = ERR_INDEXBASE + 22;
const int ERR_IDX_LIST_INVALID_LBID = ERR_INDEXBASE + 23;
const int ERR_IDX_LIST_INVALID_BLK_READ = ERR_INDEXBASE + 24;
const int ERR_IDX_LIST_UPDATE_HDR_COUNT = ERR_INDEXBASE + 25;
const int ERR_IDX_LIST_WRONG_BLK = ERR_INDEXBASE + 26;
const int ERR_IDX_LIST_WRONG_TYPE = ERR_INDEXBASE + 27;
const int ERR_IDX_LIST_GET_COUNT = ERR_INDEXBASE + 28;
const int ERR_IDX_LIST_GET_NEXT = ERR_INDEXBASE + 29;
const int ERR_IDX_LIST_GET_PARENT = ERR_INDEXBASE + 30;
const int ERR_IDX_LIST_GET_SUB_BLK = ERR_INDEXBASE + 31;
const int ERR_IDX_LIST_INVALID_UP_HDR = ERR_INDEXBASE + 32; // Update Index List error
const int ERR_IDX_LIST_INVALID_ADD_LIST = ERR_INDEXBASE + 33; // Update Index List error
const int ERR_IDX_LIST_INVALID_UP = ERR_INDEXBASE + 34; // Update Index List error
//--------------------------------------------------------------------------
// freemgr error
//--------------------------------------------------------------------------
const int ERR_FM_ASSIGN_ERR = ERR_FMGRBASE + 1; // General assignment error
const int ERR_FM_RELEASE_ERR = ERR_FMGRBASE + 2; // General release error
const int ERR_FM_BAD_FBO = ERR_FMGRBASE + 3; // File Block Offset err
const int ERR_FM_BAD_TYPE = ERR_FMGRBASE + 4; // type must be pointer or list
const int ERR_FM_NO_SPACE = ERR_FMGRBASE + 5; // No blocks available
const int ERR_FM_EXTEND = ERR_FMGRBASE + 6; // Error extending file
const int ERR_FM_ASSIGN_ERR = ERR_FMGRBASE + 1; // General assignment error
const int ERR_FM_RELEASE_ERR = ERR_FMGRBASE + 2; // General release error
const int ERR_FM_BAD_FBO = ERR_FMGRBASE + 3; // File Block Offset err
const int ERR_FM_BAD_TYPE = ERR_FMGRBASE + 4; // type must be pointer or list
const int ERR_FM_NO_SPACE = ERR_FMGRBASE + 5; // No blocks available
const int ERR_FM_EXTEND = ERR_FMGRBASE + 6; // Error extending file
//--------------------------------------------------------------------------
// Dictionary error
//--------------------------------------------------------------------------
const int ERR_DICT_NO_SPACE_INSERT = ERR_DCTNRYBASE + 1; // ins no space
const int ERR_DICT_SIZE_GT_2G = ERR_DCTNRYBASE + 2; // ins size >8000
const int ERR_DICT_NO_OP_DELETE = ERR_DCTNRYBASE + 3; // del no op
const int ERR_DICT_NO_OFFSET_DELETE = ERR_DCTNRYBASE + 4; // del bad offset
const int ERR_DICT_INVALID_HDR = ERR_DCTNRYBASE + 5; // Delete Hdr
const int ERR_DICT_ZERO_LEN = ERR_DCTNRYBASE + 6; // Delete zero len
const int ERR_DICT_TOKEN_NOT_FOUND = ERR_DCTNRYBASE + 7; // token not found
const int ERR_DICT_FILE_NOT_FOUND = ERR_DCTNRYBASE + 8; // dict file not found
const int ERR_DICT_BAD_TOKEN_LBID = ERR_DCTNRYBASE + 9; // bad token lbid
const int ERR_DICT_BAD_TOKEN_OP = ERR_DCTNRYBASE + 10; // token op is bad
const int ERR_DICT_NO_SPACE_INSERT = ERR_DCTNRYBASE + 1; // ins no space
const int ERR_DICT_SIZE_GT_2G = ERR_DCTNRYBASE + 2; // ins size >8000
const int ERR_DICT_NO_OP_DELETE = ERR_DCTNRYBASE + 3; // del no op
const int ERR_DICT_NO_OFFSET_DELETE = ERR_DCTNRYBASE + 4; // del bad offset
const int ERR_DICT_INVALID_HDR = ERR_DCTNRYBASE + 5; // Delete Hdr
const int ERR_DICT_ZERO_LEN = ERR_DCTNRYBASE + 6; // Delete zero len
const int ERR_DICT_TOKEN_NOT_FOUND = ERR_DCTNRYBASE + 7; // token not found
const int ERR_DICT_FILE_NOT_FOUND = ERR_DCTNRYBASE + 8; // dict file not found
const int ERR_DICT_BAD_TOKEN_LBID = ERR_DCTNRYBASE + 9; // bad token lbid
const int ERR_DICT_BAD_TOKEN_OP = ERR_DCTNRYBASE + 10; // token op is bad
//--------------------------------------------------------------------------
// Bulk error
//--------------------------------------------------------------------------
const int ERR_BULK_MAX_ERR_NUM = ERR_BULKBASE + 1; // Maximum number of error rows reached
const int ERR_BULK_DATA_COL_NUM = ERR_BULKBASE + 2; // The total number of data column not match with column definitions
const int ERR_BULK_SEND_MSG_ERR = ERR_BULKBASE + 3; // send msg to primproc to flush cache
const int ERR_BULK_MISSING_EXTENT_ENTRY = ERR_BULKBASE + 4; // Missing Extent Entry when trying to save LBID info
const int ERR_BULK_MISSING_EXTENT_ROW = ERR_BULKBASE + 5; // Missing Extent Row when trying to save LBID info
const int ERR_BULK_ROW_FILL_BUFFER = ERR_BULKBASE + 6; // Single row fills read buffer
const int ERR_BULK_DBROOT_CHANGE = ERR_BULKBASE + 7; // Local DBRoot settings changed during an import
const int ERR_BULK_ROLLBACK_MISS_ROOT = ERR_BULKBASE + 8; // Mode3 automatic rollback skipped with missing DBRoot
const int ERR_BULK_ROLLBACK_SEG_LIST = ERR_BULKBASE + 9; // Error building segment file list in a directory
const int ERR_BULK_BINARY_PARTIAL_REC = ERR_BULKBASE + 10;// Binary input did not end on fixed length record boundary
const int ERR_BULK_BINARY_IGNORE_FLD = ERR_BULKBASE + 11;// <IgnoreField> tag not supported for binary import
const int ERR_BULK_MAX_ERR_NUM = ERR_BULKBASE + 1; // Maximum number of error rows reached
const int ERR_BULK_DATA_COL_NUM =
ERR_BULKBASE + 2; // The total number of data column not match with column definitions
const int ERR_BULK_SEND_MSG_ERR = ERR_BULKBASE + 3; // send msg to primproc to flush cache
const int ERR_BULK_MISSING_EXTENT_ENTRY =
ERR_BULKBASE + 4; // Missing Extent Entry when trying to save LBID info
const int ERR_BULK_MISSING_EXTENT_ROW = ERR_BULKBASE + 5; // Missing Extent Row when trying to save LBID info
const int ERR_BULK_ROW_FILL_BUFFER = ERR_BULKBASE + 6; // Single row fills read buffer
const int ERR_BULK_DBROOT_CHANGE = ERR_BULKBASE + 7; // Local DBRoot settings changed during an import
const int ERR_BULK_ROLLBACK_MISS_ROOT =
ERR_BULKBASE + 8; // Mode3 automatic rollback skipped with missing DBRoot
const int ERR_BULK_ROLLBACK_SEG_LIST = ERR_BULKBASE + 9; // Error building segment file list in a directory
const int ERR_BULK_BINARY_PARTIAL_REC =
ERR_BULKBASE + 10; // Binary input did not end on fixed length record boundary
const int ERR_BULK_BINARY_IGNORE_FLD =
ERR_BULKBASE + 11; // <IgnoreField> tag not supported for binary import
//--------------------------------------------------------------------------
// BRM error
//--------------------------------------------------------------------------
const int ERR_BRM_LOOKUP_LBID = ERR_BRMBASE + 1; // Lookup LBID error
const int ERR_BRM_LOOKUP_FBO = ERR_BRMBASE + 2; // Lookup FBO error
const int ERR_BRM_ALLOC_EXTEND = ERR_BRMBASE + 3; // Allocate extent error
const int ERR_BRM_COMMIT = ERR_BRMBASE + 4; // Commit error
const int ERR_BRM_ROLLBACK = ERR_BRMBASE + 5; // Rollback error
const int ERR_BRM_GET_UNCOMM_LBID = ERR_BRMBASE + 6; // Get uncommitted lbid list error
const int ERR_BRM_DEL_OID = ERR_BRMBASE + 7; // Delete oid error
const int ERR_BRM_BEGIN_COPY = ERR_BRMBASE + 8; // Begin copy error
const int ERR_BRM_END_COPY = ERR_BRMBASE + 9; // End copy error
const int ERR_BRM_GET_HWM = ERR_BRMBASE + 10;// Get hwm error
const int ERR_BRM_SET_HWM = ERR_BRMBASE + 11;// Set hwm error
const int ERR_BRM_WR_VB_ENTRY = ERR_BRMBASE + 12;// Write VB entry error
const int ERR_BRM_VB_COPY_READ = ERR_BRMBASE + 13;// VB copy read error
const int ERR_BRM_VB_COPY_SEEK_DB = ERR_BRMBASE + 14;// VB copy seek error to DB file
const int ERR_BRM_VB_COPY_SEEK_VB = ERR_BRMBASE + 15;// VB copy seek error to VB file
const int ERR_BRM_VB_COPY_WRITE = ERR_BRMBASE + 16;// VB copy write
const int ERR_BRM_DEAD_LOCK = ERR_BRMBASE + 17;// DEAD lock error
const int ERR_BRM_MARK_INVALID = ERR_BRMBASE + 18;// Mark extent invalid error from casual paritioning
const int ERR_BRM_SAVE_STATE = ERR_BRMBASE + 19;// Save state error
const int ERR_BRM_GET_START_EXTENT = ERR_BRMBASE + 20; // Get starting Extent error
const int ERR_BRM_VB_OVERFLOW = ERR_BRMBASE + 21;// Version buffer overflow
const int ERR_BRM_READ_ONLY = ERR_BRMBASE + 22;// BRM is in READ-ONLY state
const int ERR_BRM_GET_READ_WRITE = ERR_BRMBASE + 23;// error getting BRM READ/WRITE state
const int ERR_BRM_BULK_RB_COLUMN = ERR_BRMBASE + 24;// error during column bulk rollback
const int ERR_BRM_BULK_RB_DCTNRY = ERR_BRMBASE + 25;// error during dctnry bulk rollback
const int ERR_BRM_DELETE_EXTENT_COLUMN = ERR_BRMBASE + 26; // error during delete column extents
const int ERR_BRM_DELETE_EXTENT_DCTNRY = ERR_BRMBASE + 27; // error during delete dictionary extents
const int ERR_BRM_TAKE_SNAPSHOT = ERR_BRMBASE + 28;// Taking snapshot of BRM state
const int ERR_BRM_LOOKUP_START_LBID = ERR_BRMBASE + 29; // Lookup starting LBID error
const int ERR_BRM_BULK_UPDATE = ERR_BRMBASE + 30;// Error with bulk update of HWM and CP
const int ERR_BRM_GET_EXT_STATE = ERR_BRMBASE + 31;// Error getting extent state
const int ERR_EXTENTMAP_LOOKUP = ERR_BRMBASE + 32;// Lookup extent map error
const int ERR_BRM_LOOKUP_VERSION = ERR_BRMBASE + 33;// Lookup version error
const int ERR_BRM_LOOKUP_LBID_RANGES = ERR_BRMBASE + 34;// Lookup LBID Ranges error
const int ERR_BRM_HWMS_NOT_EQUAL = ERR_BRMBASE + 35;// HWMs of same col width not equal
const int ERR_BRM_HWMS_OUT_OF_SYNC = ERR_BRMBASE + 36; // HWMs for dif col width not in sync
const int ERR_BRM_DBROOT_HWMS = ERR_BRMBASE + 37;// Error getting HWMs for each DBRoot
const int ERR_BRM_NETWORK = ERR_BRMBASE + 38;// Network error when calling BRM functions
const int ERR_BRM_READONLY = ERR_BRMBASE + 39;// DBRM is readonly
const int ERR_INVALID_VBOID = ERR_BRMBASE + 40;// returned if the given vboid is invalid
const int ERR_BRM_SET_EXTENTS_CP = ERR_BRMBASE + 41;// Error setting extents min/max
const int ERR_BRM_SHUTDOWN = ERR_BRMBASE + 42;// BRM is set to shutdown
const int ERR_BRM_GET_SHUTDOWN = ERR_BRMBASE + 43;// error getting BRM Shutdown flag
const int ERR_BRM_SUSPEND = ERR_BRMBASE + 44;// BRM is set to Suspend writes
const int ERR_BRM_GET_SUSPEND = ERR_BRMBASE + 45;// error getting BRM Suspend flag
const int ERR_BRM_BAD_STRIPE_CNT = ERR_BRMBASE + 46;// Incorrect num of cols allocated in stripe
const int ERR_BRM_UNSUPP_WIDTH = ERR_BRMBASE + 47;// Non-dict column Width > allowed MAX.
const int ERR_BRM_GET_EXTENT_CP = ERR_BRMBASE + 48;// Error getting extent's CPInfo
const int ERR_BRM_LOOKUP_LBID = ERR_BRMBASE + 1; // Lookup LBID error
const int ERR_BRM_LOOKUP_FBO = ERR_BRMBASE + 2; // Lookup FBO error
const int ERR_BRM_ALLOC_EXTEND = ERR_BRMBASE + 3; // Allocate extent error
const int ERR_BRM_COMMIT = ERR_BRMBASE + 4; // Commit error
const int ERR_BRM_ROLLBACK = ERR_BRMBASE + 5; // Rollback error
const int ERR_BRM_GET_UNCOMM_LBID = ERR_BRMBASE + 6; // Get uncommitted lbid list error
const int ERR_BRM_DEL_OID = ERR_BRMBASE + 7; // Delete oid error
const int ERR_BRM_BEGIN_COPY = ERR_BRMBASE + 8; // Begin copy error
const int ERR_BRM_END_COPY = ERR_BRMBASE + 9; // End copy error
const int ERR_BRM_GET_HWM = ERR_BRMBASE + 10; // Get hwm error
const int ERR_BRM_SET_HWM = ERR_BRMBASE + 11; // Set hwm error
const int ERR_BRM_WR_VB_ENTRY = ERR_BRMBASE + 12; // Write VB entry error
const int ERR_BRM_VB_COPY_READ = ERR_BRMBASE + 13; // VB copy read error
const int ERR_BRM_VB_COPY_SEEK_DB = ERR_BRMBASE + 14; // VB copy seek error to DB file
const int ERR_BRM_VB_COPY_SEEK_VB = ERR_BRMBASE + 15; // VB copy seek error to VB file
const int ERR_BRM_VB_COPY_WRITE = ERR_BRMBASE + 16; // VB copy write
const int ERR_BRM_DEAD_LOCK = ERR_BRMBASE + 17; // DEAD lock error
const int ERR_BRM_MARK_INVALID = ERR_BRMBASE + 18; // Mark extent invalid error from casual paritioning
const int ERR_BRM_SAVE_STATE = ERR_BRMBASE + 19; // Save state error
const int ERR_BRM_GET_START_EXTENT = ERR_BRMBASE + 20; // Get starting Extent error
const int ERR_BRM_VB_OVERFLOW = ERR_BRMBASE + 21; // Version buffer overflow
const int ERR_BRM_READ_ONLY = ERR_BRMBASE + 22; // BRM is in READ-ONLY state
const int ERR_BRM_GET_READ_WRITE = ERR_BRMBASE + 23; // error getting BRM READ/WRITE state
const int ERR_BRM_BULK_RB_COLUMN = ERR_BRMBASE + 24; // error during column bulk rollback
const int ERR_BRM_BULK_RB_DCTNRY = ERR_BRMBASE + 25; // error during dctnry bulk rollback
const int ERR_BRM_DELETE_EXTENT_COLUMN = ERR_BRMBASE + 26; // error during delete column extents
const int ERR_BRM_DELETE_EXTENT_DCTNRY = ERR_BRMBASE + 27; // error during delete dictionary extents
const int ERR_BRM_TAKE_SNAPSHOT = ERR_BRMBASE + 28; // Taking snapshot of BRM state
const int ERR_BRM_LOOKUP_START_LBID = ERR_BRMBASE + 29; // Lookup starting LBID error
const int ERR_BRM_BULK_UPDATE = ERR_BRMBASE + 30; // Error with bulk update of HWM and CP
const int ERR_BRM_GET_EXT_STATE = ERR_BRMBASE + 31; // Error getting extent state
const int ERR_EXTENTMAP_LOOKUP = ERR_BRMBASE + 32; // Lookup extent map error
const int ERR_BRM_LOOKUP_VERSION = ERR_BRMBASE + 33; // Lookup version error
const int ERR_BRM_LOOKUP_LBID_RANGES = ERR_BRMBASE + 34; // Lookup LBID Ranges error
const int ERR_BRM_HWMS_NOT_EQUAL = ERR_BRMBASE + 35; // HWMs of same col width not equal
const int ERR_BRM_HWMS_OUT_OF_SYNC = ERR_BRMBASE + 36; // HWMs for dif col width not in sync
const int ERR_BRM_DBROOT_HWMS = ERR_BRMBASE + 37; // Error getting HWMs for each DBRoot
const int ERR_BRM_NETWORK = ERR_BRMBASE + 38; // Network error when calling BRM functions
const int ERR_BRM_READONLY = ERR_BRMBASE + 39; // DBRM is readonly
const int ERR_INVALID_VBOID = ERR_BRMBASE + 40; // returned if the given vboid is invalid
const int ERR_BRM_SET_EXTENTS_CP = ERR_BRMBASE + 41; // Error setting extents min/max
const int ERR_BRM_SHUTDOWN = ERR_BRMBASE + 42; // BRM is set to shutdown
const int ERR_BRM_GET_SHUTDOWN = ERR_BRMBASE + 43; // error getting BRM Shutdown flag
const int ERR_BRM_SUSPEND = ERR_BRMBASE + 44; // BRM is set to Suspend writes
const int ERR_BRM_GET_SUSPEND = ERR_BRMBASE + 45; // error getting BRM Suspend flag
const int ERR_BRM_BAD_STRIPE_CNT = ERR_BRMBASE + 46; // Incorrect num of cols allocated in stripe
const int ERR_BRM_UNSUPP_WIDTH = ERR_BRMBASE + 47; // Non-dict column Width > allowed MAX.
const int ERR_BRM_GET_EXTENT_CP = ERR_BRMBASE + 48; // Error getting extent's CPInfo
//--------------------------------------------------------------------------
// DM error
//--------------------------------------------------------------------------
const int ERR_DM_CONVERT_OID = ERR_DMBASE + 1; // Conversion error
const int ERR_DM_CONVERT_OID = ERR_DMBASE + 1; // Conversion error
//--------------------------------------------------------------------------
// Cache error
//--------------------------------------------------------------------------
const int ERR_CACHE_KEY_EXIST = ERR_CACHEBASE + 1; // Cache key exist
const int ERR_CACHE_KEY_NOT_EXIST = ERR_CACHEBASE + 2; // Cache key not exist
const int ERR_NULL_BLOCK = ERR_CACHEBASE + 3; // Block is NULL
const int ERR_FREE_LIST_EMPTY = ERR_CACHEBASE + 4; // Empty Free list
const int ERR_CACHE_KEY_EXIST = ERR_CACHEBASE + 1; // Cache key exist
const int ERR_CACHE_KEY_NOT_EXIST = ERR_CACHEBASE + 2; // Cache key not exist
const int ERR_NULL_BLOCK = ERR_CACHEBASE + 3; // Block is NULL
const int ERR_FREE_LIST_EMPTY = ERR_CACHEBASE + 4; // Empty Free list
//--------------------------------------------------------------------------
// Compression error
//--------------------------------------------------------------------------
const int ERR_COMP_COMPRESS = ERR_COMPBASE + 1; // Error compressing data
const int ERR_COMP_UNCOMPRESS = ERR_COMPBASE + 2; // Error uncompressing data
const int ERR_COMP_PARSE_HDRS = ERR_COMPBASE + 3; // Error parsing compression headers
const int ERR_COMP_VERIFY_HDRS = ERR_COMPBASE + 4; // Error verifying compression headers
const int ERR_COMP_PAD_DATA = ERR_COMPBASE + 5; // Pad compressed data failed
const int ERR_COMP_READ_BLOCK = ERR_COMPBASE + 6; // Failed to read a block
const int ERR_COMP_SAVE_BLOCK = ERR_COMPBASE + 7; // Failed to save a block
const int ERR_COMP_WRONG_PTR = ERR_COMPBASE + 8; // Pointer in header is wrong
const int ERR_COMP_FILE_NOT_FOUND = ERR_COMPBASE + 9; // File not found in map
const int ERR_COMP_CHUNK_NOT_FOUND = ERR_COMPBASE + 10; // Chunk not found in map
const int ERR_COMP_UNAVAIL_TYPE = ERR_COMPBASE + 11;// Unavailable compression type
const int ERR_COMP_REMOVE_FILE = ERR_COMPBASE + 12;// Failed to remove a file
const int ERR_COMP_RENAME_FILE = ERR_COMPBASE + 13;// Failed to rename a file
const int ERR_COMP_OPEN_FILE = ERR_COMPBASE + 14;// Failed to open a compressed data file
const int ERR_COMP_SET_OFFSET = ERR_COMPBASE + 15;// Failed to set offset in a compressed data file
const int ERR_COMP_READ_FILE = ERR_COMPBASE + 16;// Failed to read from a compressed data file
const int ERR_COMP_WRITE_FILE = ERR_COMPBASE + 17;// Failed to write to a compresssed data file
const int ERR_COMP_CLOSE_FILE = ERR_COMPBASE + 18;// Failed to close a compressed data file
const int ERR_COMP_TRUNCATE_ZERO = ERR_COMPBASE + 19;// Invalid attempt to truncate file to 0 bytes
const int ERR_COMP_WRONG_COMP_TYPE = ERR_COMPBASE + 20;// Invalid compression type.
const int ERR_COMP_COMPRESS = ERR_COMPBASE + 1; // Error compressing data
const int ERR_COMP_UNCOMPRESS = ERR_COMPBASE + 2; // Error uncompressing data
const int ERR_COMP_PARSE_HDRS = ERR_COMPBASE + 3; // Error parsing compression headers
const int ERR_COMP_VERIFY_HDRS = ERR_COMPBASE + 4; // Error verifying compression headers
const int ERR_COMP_PAD_DATA = ERR_COMPBASE + 5; // Pad compressed data failed
const int ERR_COMP_READ_BLOCK = ERR_COMPBASE + 6; // Failed to read a block
const int ERR_COMP_SAVE_BLOCK = ERR_COMPBASE + 7; // Failed to save a block
const int ERR_COMP_WRONG_PTR = ERR_COMPBASE + 8; // Pointer in header is wrong
const int ERR_COMP_FILE_NOT_FOUND = ERR_COMPBASE + 9; // File not found in map
const int ERR_COMP_CHUNK_NOT_FOUND = ERR_COMPBASE + 10; // Chunk not found in map
const int ERR_COMP_UNAVAIL_TYPE = ERR_COMPBASE + 11; // Unavailable compression type
const int ERR_COMP_REMOVE_FILE = ERR_COMPBASE + 12; // Failed to remove a file
const int ERR_COMP_RENAME_FILE = ERR_COMPBASE + 13; // Failed to rename a file
const int ERR_COMP_OPEN_FILE = ERR_COMPBASE + 14; // Failed to open a compressed data file
const int ERR_COMP_SET_OFFSET = ERR_COMPBASE + 15; // Failed to set offset in a compressed data file
const int ERR_COMP_READ_FILE = ERR_COMPBASE + 16; // Failed to read from a compressed data file
const int ERR_COMP_WRITE_FILE = ERR_COMPBASE + 17; // Failed to write to a compresssed data file
const int ERR_COMP_CLOSE_FILE = ERR_COMPBASE + 18; // Failed to close a compressed data file
const int ERR_COMP_TRUNCATE_ZERO = ERR_COMPBASE + 19; // Invalid attempt to truncate file to 0 bytes
const int ERR_COMP_WRONG_COMP_TYPE = ERR_COMPBASE + 20; // Invalid compression type.
//--------------------------------------------------------------------------
// Auto-increment error
//--------------------------------------------------------------------------
const int ERR_AUTOINC_GEN_EXCEED_MAX = ERR_AUTOINCBASE + 1; // Generated autoinc value exceeds max auto increment value/
const int ERR_AUTOINC_USER_OUT_OF_RANGE = ERR_AUTOINCBASE + 2; // User specified autoinc value is out of range
const int ERR_AUTOINC_TABLE_NAME = ERR_AUTOINCBASE + 3; // Invalid schema/tablename for auto increment
const int ERR_AUTOINC_INIT1 = ERR_AUTOINCBASE + 4; // Error initializing auto increment (known exception)
const int ERR_AUTOINC_INIT2 = ERR_AUTOINCBASE + 5; // Error initializing auto increment (unknown exception)
const int ERR_AUTOINC_RID = ERR_AUTOINCBASE + 6; // Error initializing auto increment (unknown exception)
const int ERR_AUTOINC_START_SEQ = ERR_AUTOINCBASE + 7; // Error setting up an auto-increment sequence
const int ERR_AUTOINC_GET_RANGE = ERR_AUTOINCBASE + 8; // Error reserving an auto-increment range
const int ERR_AUTOINC_GET_LOCK = ERR_AUTOINCBASE + 9; // Error getting a lock to update auto-inc next value
const int ERR_AUTOINC_REL_LOCK = ERR_AUTOINCBASE + 10; // Error releasing lock to update auto-inc next value
const int ERR_AUTOINC_UPDATE = ERR_AUTOINCBASE + 11; // Error updating nextValue in system catalog
const int ERR_AUTOINC_GEN_EXCEED_MAX =
ERR_AUTOINCBASE + 1; // Generated autoinc value exceeds max auto increment value/
const int ERR_AUTOINC_USER_OUT_OF_RANGE =
ERR_AUTOINCBASE + 2; // User specified autoinc value is out of range
const int ERR_AUTOINC_TABLE_NAME = ERR_AUTOINCBASE + 3; // Invalid schema/tablename for auto increment
const int ERR_AUTOINC_INIT1 = ERR_AUTOINCBASE + 4; // Error initializing auto increment (known exception)
const int ERR_AUTOINC_INIT2 = ERR_AUTOINCBASE + 5; // Error initializing auto increment (unknown exception)
const int ERR_AUTOINC_RID = ERR_AUTOINCBASE + 6; // Error initializing auto increment (unknown exception)
const int ERR_AUTOINC_START_SEQ = ERR_AUTOINCBASE + 7; // Error setting up an auto-increment sequence
const int ERR_AUTOINC_GET_RANGE = ERR_AUTOINCBASE + 8; // Error reserving an auto-increment range
const int ERR_AUTOINC_GET_LOCK = ERR_AUTOINCBASE + 9; // Error getting a lock to update auto-inc next value
const int ERR_AUTOINC_REL_LOCK = ERR_AUTOINCBASE + 10; // Error releasing lock to update auto-inc next value
const int ERR_AUTOINC_UPDATE = ERR_AUTOINCBASE + 11; // Error updating nextValue in system catalog
//--------------------------------------------------------------------------
// Block cache flush error
//--------------------------------------------------------------------------
const int ERR_BLKCACHE_FLUSH_LIST = ERR_BLKCACHEBASE + 1; // Error flushing list of blocks to PrimProc
const int ERR_BLKCACHE_FLUSH_LIST = ERR_BLKCACHEBASE + 1; // Error flushing list of blocks to PrimProc
//--------------------------------------------------------------------------
// Bulk backup metadata file and corresponding HWM compressed chunk files
//--------------------------------------------------------------------------
const int ERR_METADATABKUP_FILE_RENAME = ERR_METABKUPBASE + 1; // Error renaming meta file */
const int ERR_METADATABKUP_COMP_PARSE_HDRS = ERR_METABKUPBASE + 2; // Error parsing compression headers */
const int ERR_METADATABKUP_COMP_VERIFY_HDRS = ERR_METABKUPBASE + 3; // Error verifying compression headers */
const int ERR_METADATABKUP_COMP_CHUNK_NOT_FOUND = ERR_METABKUPBASE + 4; // Chunk not found in file */
const int ERR_METADATABKUP_COMP_OPEN_BULK_BKUP = ERR_METABKUPBASE + 5; // Error opening backup chunk file */
const int ERR_METADATABKUP_COMP_WRITE_BULK_BKUP = ERR_METABKUPBASE + 6; // Error writing to backup chunk file */
const int ERR_METADATABKUP_COMP_READ_BULK_BKUP = ERR_METABKUPBASE + 7; // Error reading from backup chunk file */
const int ERR_METADATABKUP_COMP_RENAME = ERR_METABKUPBASE + 8; // Error renaming chunk file */
const int ERR_METADATABKUP_FILE_RENAME = ERR_METABKUPBASE + 1; // Error renaming meta file */
const int ERR_METADATABKUP_COMP_PARSE_HDRS = ERR_METABKUPBASE + 2; // Error parsing compression headers */
const int ERR_METADATABKUP_COMP_VERIFY_HDRS = ERR_METABKUPBASE + 3; // Error verifying compression headers */
const int ERR_METADATABKUP_COMP_CHUNK_NOT_FOUND = ERR_METABKUPBASE + 4; // Chunk not found in file */
const int ERR_METADATABKUP_COMP_OPEN_BULK_BKUP = ERR_METABKUPBASE + 5; // Error opening backup chunk file */
const int ERR_METADATABKUP_COMP_WRITE_BULK_BKUP =
ERR_METABKUPBASE + 6; // Error writing to backup chunk file */
const int ERR_METADATABKUP_COMP_READ_BULK_BKUP =
ERR_METABKUPBASE + 7; // Error reading from backup chunk file */
const int ERR_METADATABKUP_COMP_RENAME = ERR_METABKUPBASE + 8; // Error renaming chunk file */
//------------------------------------------------------------------------------
// Class used to convert an error code to a corresponding error message string
//------------------------------------------------------------------------------
struct WErrorCodes
{
EXPORT WErrorCodes();
EXPORT std::string errorString(int code);
private:
typedef std::map<int, std::string> CodeMap;
CodeMap fErrorCodes;
EXPORT WErrorCodes();
EXPORT std::string errorString(int code);
private:
typedef std::map<int, std::string> CodeMap;
CodeMap fErrorCodes;
};
} //end of namespace
} // namespace WriteEngine
#undef EXPORT

3533
writeengine/shared/we_fileop.cpp
View File

File diff suppressed because it is too large Load Diff

838
writeengine/shared/we_fileop.h
View File

@ -57,510 +57,425 @@
/** Namespace WriteEngine */
namespace WriteEngine
{
constexpr size_t MaxDirLevels = 5;
/** Class FileOp */
class FileOp : public BlockOp, public WeUIDGID
{
public:
/**
* @brief Constructor
*/
EXPORT explicit FileOp(bool doAlloc = true);
public:
/**
* @brief Constructor
*/
EXPORT explicit FileOp(bool doAlloc = true);
/**
* @brief Destructor
*/
EXPORT virtual ~FileOp();
/**
* @brief Destructor
*/
EXPORT virtual ~FileOp();
/**
* @brief Close a file
*/
EXPORT void closeFile( IDBDataFile* pFile ) const;
/**
* @brief Close a file
*/
EXPORT void closeFile(IDBDataFile* pFile) const;
/**
* @brief Create a directory
*/
EXPORT int createDir( const char* dirName, mode_t mode ) const;
int createDir( const char* dirName ) const;
/**
* @brief Create a directory
*/
EXPORT int createDir(const char* dirName, mode_t mode) const;
int createDir(const char* dirName) const;
/**
* @brief Create a file with a fixed file size and file id
*/
EXPORT int createFile( FID fid,
int& allocSize,
uint16_t dbRoot, uint32_t partition,
execplan::CalpontSystemCatalog::ColDataType colDataType,
const uint8_t* emptyVal, int width = 1 ) ;
/**
* @brief Create a file with a fixed file size and file id
*/
EXPORT int createFile(FID fid, int& allocSize, uint16_t dbRoot, uint32_t partition,
execplan::CalpontSystemCatalog::ColDataType colDataType, const uint8_t* emptyVal,
int width = 1);
/**
* @brief Create a file with a fixed file size by its name.
* Changed to public for UT.
*/
int createFile(const char* fileName, int fileSize, const uint8_t* emptyVal, int width,
execplan::CalpontSystemCatalog::ColDataType colDataType, uint16_t dbRoot,
BRM::LBID_t lbid = -1);
/**
* @brief Create a file with a fixed file size by its name.
* Changed to public for UT.
*/
int createFile( const char* fileName, int fileSize,
const uint8_t* emptyVal, int width,
execplan::CalpontSystemCatalog::ColDataType colDataType,
uint16_t dbRoot ,
BRM::LBID_t lbid = -1 );
/**
* @brief Delete a file
*/
EXPORT int deleteFile(const char* fileName) const;
/**
* @brief Delete a file
*/
EXPORT int deleteFile( const char* fileName ) const;
/**
* @brief Delete the db files corresponding to the specified file id
*/
EXPORT int deleteFile(FID fid) const;
/**
* @brief Delete the db files corresponding to the specified file id
*/
EXPORT int deleteFile( FID fid ) const;
/**
* @brief Delete the db files corresponding to the specified file id
*/
EXPORT int deleteFiles(const std::vector<int32_t>& fids) const;
/**
* @brief Delete the db files corresponding to the specified file id
*/
EXPORT int deleteFiles( const std::vector<int32_t>& fids ) const;
/**
* @brief Delete db files corresponding to specified file id and partition
*/
EXPORT int deletePartitions(const std::vector<OID>& fids,
const std::vector<BRM::PartitionInfo>& partitions) const;
/**
* @brief Delete a specific database segment file.
*/
EXPORT int deleteFile(FID fid, uint16_t dbRoot, uint32_t partition, uint16_t segment) const;
/**
* @brief Delete db files corresponding to specified file id and partition
*/
EXPORT int deletePartitions( const std::vector<OID>& fids,
const std::vector<BRM::PartitionInfo>& partitions )
const;
/**
* @brief Check whether a file exists or not
*/
EXPORT bool exists(const char* fileName) const;
/**
* @brief Delete a specific database segment file.
*/
EXPORT int deleteFile( FID fid, uint16_t dbRoot,
uint32_t partition,
uint16_t segment ) const;
/**
* @brief @brief Check whether file exists or not by using file id, DBRoot,
* partition, and segment number.
*/
EXPORT bool exists(FID fid, uint16_t dbRoot, uint32_t partition, uint16_t segment) const;
/**
* @brief Check whether a file exists or not
*/
EXPORT bool exists( const char* fileName ) const;
/**
* @brief Check whether a column exists or not by using file id. Since this
* is not enough to fully qualify a db filename, all it can do is to verify
* that the OID directory exists on one or more of the DBRoots.
*/
EXPORT bool existsOIDDir(FID fid) const;
/**
* @brief @brief Check whether file exists or not by using file id, DBRoot,
* partition, and segment number.
*/
EXPORT bool exists( FID fid, uint16_t dbRoot,
uint32_t partition, uint16_t segment ) const;
/**
* @brief Expand current abbreviated extent for this column to a full extent
*
* @param pFile FILE ptr of segment file we are updating.
* @param dbRoot DBRoot of the file being updated.
* @param emptyVal Empty value used in initializing extents for this column
* @param width Width of this column (in bytes)
* @param colDataType Column data type.
*/
EXPORT virtual int expandAbbrevColumnExtent(IDBDataFile* pFile, uint16_t dbRoot, const uint8_t* emptyVal,
int width,
execplan::CalpontSystemCatalog::ColDataType colDataType);
/**
* @brief Check whether a column exists or not by using file id. Since this
* is not enough to fully qualify a db filename, all it can do is to verify
* that the OID directory exists on one or more of the DBRoots.
*/
EXPORT bool existsOIDDir( FID fid ) const;
/**
* @brief Add an extent to the specified Column OID and DBRoot.
* The extent must already exist in the extentmap prior to calling this fctn.
*
* The partition, segment, and HWM of the column file where the
* extent is added is returned. If needed, the applicable column segment
* file will be created. This extendFile1 function should supplant other
* extendFile functions with Multiple-File-per-OID enhancement, "but" we
* may want to rethink when we do Shared-Nothing. When this function
* returns, the file position will be located at the end of the file.
* For shared-everything DBRoot was an output argument, as BRM selected the
* the DBRoot. For shared-nothing DBRoot is an input argument, as the
* application code must track/control the DBRoot selection.
* If this is the very first file for the specified DBRoot, then the
* partition and segment number must be specified, else the selected
* partition and segment numbers are returned.
*
* @param oid OID of the column to be extended
* @param emptyVal Empty value to be used for oid
* @param width Width of the column
* @param hwm The fbo of the column segment file where the new extent begins
* @param startLbid The starting LBID for the new extent
* @param allocSize Number of blocks allocated to the extent.
* @param dbRoot The DBRoot of the file with the new extent.
* @param partition The partnum of the file with the new extent.
* @param segment The segnum of the file with the new extent.
* @param segFile (out) Name of the segment file where extent was added.
* @param pFile (out) FILE ptr to the file where the extent is added.
* @param newFile (out) Indicates if a new file was created for the extent
* @param hdrs (in/out) Contents of headers, if file is compressed.
* @return returns NO_ERROR if success.
*/
EXPORT int extendFile(OID oid, const uint8_t* emptyVal, int width,
execplan::CalpontSystemCatalog::ColDataType colDataType, HWM hwm,
BRM::LBID_t startLbid, int allocSize, uint16_t dbRoot, uint32_t partition,
uint16_t segment, std::string& segFile, IDBDataFile*& pFile, bool& newFile,
char* hdrs);
/**
* @brief Expand current abbreviated extent for this column to a full extent
*
* @param pFile FILE ptr of segment file we are updating.
* @param dbRoot DBRoot of the file being updated.
* @param emptyVal Empty value used in initializing extents for this column
* @param width Width of this column (in bytes)
* @param colDataType Column data type.
*/
EXPORT virtual int expandAbbrevColumnExtent(
IDBDataFile* pFile,
uint16_t dbRoot,
const uint8_t* emptyVal,
int width,
execplan::CalpontSystemCatalog::ColDataType colDataType);
/**
* @brief For alter table add column; add an extent to a specific file
*
* @param oid OID of the column to be extended
* @param emptyVal Empty value to be used for oid
* @param width Width of the column
* @param allocSize (out) Number of blocks allocated to the extent.
* @param dbRoot The DBRoot of the file with the new extent.
* @param partition The partnum of the file with the new extent.
* @param segment The segnum of the file with the new extent.
* @param segFile (out) Name of the segment file where extent was added.
* @param startLbid (out) The starting LBID for the new extent
* @param newFile (out) Indicates if a new file was created for the extent
* @param hdrs (in/out) Contents of headers, if file is compressed.
*/
EXPORT int addExtentExactFile(OID oid, const uint8_t* emptyVal, int width, int& allocSize, uint16_t dbRoot,
uint32_t partition, uint16_t segment,
execplan::CalpontSystemCatalog::ColDataType colDataType, std::string& segFile,
BRM::LBID_t& startLbid, bool& newFile, char* hdrs);
/**
* @brief Add an extent to the specified Column OID and DBRoot.
* The extent must already exist in the extentmap prior to calling this fctn.
*
* The partition, segment, and HWM of the column file where the
* extent is added is returned. If needed, the applicable column segment
* file will be created. This extendFile1 function should supplant other
* extendFile functions with Multiple-File-per-OID enhancement, "but" we
* may want to rethink when we do Shared-Nothing. When this function
* returns, the file position will be located at the end of the file.
* For shared-everything DBRoot was an output argument, as BRM selected the
* the DBRoot. For shared-nothing DBRoot is an input argument, as the
* application code must track/control the DBRoot selection.
* If this is the very first file for the specified DBRoot, then the
* partition and segment number must be specified, else the selected
* partition and segment numbers are returned.
*
* @param oid OID of the column to be extended
* @param emptyVal Empty value to be used for oid
* @param width Width of the column
* @param hwm The fbo of the column segment file where the new extent begins
* @param startLbid The starting LBID for the new extent
* @param allocSize Number of blocks allocated to the extent.
* @param dbRoot The DBRoot of the file with the new extent.
* @param partition The partnum of the file with the new extent.
* @param segment The segnum of the file with the new extent.
* @param segFile (out) Name of the segment file where extent was added.
* @param pFile (out) FILE ptr to the file where the extent is added.
* @param newFile (out) Indicates if a new file was created for the extent
* @param hdrs (in/out) Contents of headers, if file is compressed.
* @return returns NO_ERROR if success.
*/
EXPORT int extendFile(OID oid, const uint8_t* emptyVal,
int width,
execplan::CalpontSystemCatalog::ColDataType colDataType,
HWM hwm,
BRM::LBID_t startLbid,
int allocSize,
uint16_t dbRoot,
uint32_t partition,
uint16_t segment,
std::string& segFile,
IDBDataFile*& pFile,
bool& newFile,
char* hdrs);
/**
* @brief Pad the specified compressed extent with empty chunks
* @param oid OID of relevant column
* @param width Width in bytes of this column
* @param emptyVal Empty value to be employed in filling the chunks
* @param dbRoot DBRoot of the extent to be filled
* @param partition Partition of the extent to be filled
* @param segment Segment file number of the extent to be filled
* @param colDataType Column data type
* @param hwm New HWM blk setting for the segment file after extent is padded
* @param segFile (out) Name of updated segment file
* @param errTask (out) Task that failed if error occurs
* @return returns NO_ERROR if success.
*/
EXPORT int fillCompColumnExtentEmptyChunks(OID oid, int colWidth, const uint8_t* emptyVal, uint16_t dbRoot,
uint32_t partition, uint16_t segment,
execplan::CalpontSystemCatalog::ColDataType colDataType, HWM hwm,
std::string& segFile, std::string& errTask);
/**
* @brief For alter table add column; add an extent to a specific file
*
* @param oid OID of the column to be extended
* @param emptyVal Empty value to be used for oid
* @param width Width of the column
* @param allocSize (out) Number of blocks allocated to the extent.
* @param dbRoot The DBRoot of the file with the new extent.
* @param partition The partnum of the file with the new extent.
* @param segment The segnum of the file with the new extent.
* @param segFile (out) Name of the segment file where extent was added.
* @param startLbid (out) The starting LBID for the new extent
* @param newFile (out) Indicates if a new file was created for the extent
* @param hdrs (in/out) Contents of headers, if file is compressed.
*/
EXPORT int addExtentExactFile(OID oid, const uint8_t* emptyVal,
int width,
int& allocSize,
uint16_t dbRoot,
uint32_t partition,
uint16_t segment,
execplan::CalpontSystemCatalog::ColDataType colDataType,
std::string& segFile,
BRM::LBID_t& startLbid,
bool& newFile,
char* hdrs);
/**
* @brief Write the specified header info to compressed column file pFile.
*
* @param pFile Column file to be written to
* @param hdr Header info to be written
*/
EXPORT int writeHeaders(IDBDataFile* pFile, const char* hdr) const;
/**
* @brief Pad the specified compressed extent with empty chunks
* @param oid OID of relevant column
* @param width Width in bytes of this column
* @param emptyVal Empty value to be employed in filling the chunks
* @param dbRoot DBRoot of the extent to be filled
* @param partition Partition of the extent to be filled
* @param segment Segment file number of the extent to be filled
* @param colDataType Column data type
* @param hwm New HWM blk setting for the segment file after extent is padded
* @param segFile (out) Name of updated segment file
* @param errTask (out) Task that failed if error occurs
* @return returns NO_ERROR if success.
*/
EXPORT int fillCompColumnExtentEmptyChunks(OID oid,
int colWidth,
const uint8_t* emptyVal,
uint16_t dbRoot,
uint32_t partition,
uint16_t segment,
execplan::CalpontSystemCatalog::ColDataType colDataType,
HWM hwm,
std::string& segFile,
std::string& errTask);
/**
* @brief Write the specified header info to compressed column or
* dictionary file pFile.
*
* @param pFile Column file to be written to
* @param controlHdr Control header info to be written
* @param pointerHdr Pointer header info to be written
* @param ptrHdrSize Size (in bytes) of pointerHdr
*/
EXPORT int writeHeaders(IDBDataFile* pFile, const char* controlHdr, const char* pointerHdr,
uint64_t ptrHdrSize) const;
/**
* @brief Write the specified header info to compressed column file pFile.
*
* @param pFile Column file to be written to
* @param hdr Header info to be written
*/
EXPORT int writeHeaders(IDBDataFile* pFile, const char* hdr) const;
/**
* @brief Get the Version Buffer filename for the specified fid (OID).
*
* This version of getFileName automatically uses 0 for the partition and
* segment numbers. The applicable DBRoot is assigned based on the OID.
*
* @param fid (in) OID of the Version Buffer DB file of interest
* @param fileName (out) the name of the pertinent file that was found
*
* @return returns NO_ERROR if success; ERR_FILE_NOT_EXIST if file not found
*/
int getVBFileName(FID fid, char* fileName) const;
/**
* @brief Write the specified header info to compressed column or
* dictionary file pFile.
*
* @param pFile Column file to be written to
* @param controlHdr Control header info to be written
* @param pointerHdr Pointer header info to be written
* @param ptrHdrSize Size (in bytes) of pointerHdr
*/
EXPORT int writeHeaders(IDBDataFile* pFile,
const char* controlHdr,
const char* pointerHdr,
uint64_t ptrHdrSize) const;
/**
* @brief Get the filename for the specified fid (OID). DBRoot, partition,
* and segment number.
*
* @param fid (in) OID of the DB file of interest
* @param fileName (out) the name of the pertinent file that was found
* @param dbRoot (in) DBRoot of the file of interest. If 0, then all the
* DBRoots will be searched.
* @param partition (in) partition number of the file of interest
* @param segment (in) segment number of the file of interest
*/
int getFileName(FID fid, char* fileName, uint16_t dbRoot, uint32_t partition, uint16_t segment) const;
/**
* @brief Get the Version Buffer filename for the specified fid (OID).
*
* This version of getFileName automatically uses 0 for the partition and
* segment numbers. The applicable DBRoot is assigned based on the OID.
*
* @param fid (in) OID of the Version Buffer DB file of interest
* @param fileName (out) the name of the pertinent file that was found
*
* @return returns NO_ERROR if success; ERR_FILE_NOT_EXIST if file not found
*/
int getVBFileName( FID fid, char* fileName ) const;
/* Added for MCOL-3251 */
void getFileNameForPrimProc(FID fid, char* fileName, uint16_t dbRoot, uint32_t partition,
uint16_t segment) const;
/**
* @brief Get the filename for the specified fid (OID). DBRoot, partition,
* and segment number.
*
* @param fid (in) OID of the DB file of interest
* @param fileName (out) the name of the pertinent file that was found
* @param dbRoot (in) DBRoot of the file of interest. If 0, then all the
* DBRoots will be searched.
* @param partition (in) partition number of the file of interest
* @param segment (in) segment number of the file of interest
*/
int getFileName( FID fid, char* fileName,
uint16_t dbRoot,
uint32_t partition,
uint16_t segment ) const;
/**
* @brief Construct directory path for the specified fid (OID), DBRoot, and
* partition number. Directory does not have to exist, nor is it created.
*/
int getDirName(FID fid, uint16_t dbRoot, uint32_t partition, std::string& dirName) const;
/* Added for MCOL-3251 */
void getFileNameForPrimProc(FID fid, char* fileName,
uint16_t dbRoot,
uint32_t partition,
uint16_t segment ) const;
/**
* @brief Get the file size
*/
EXPORT int getFileSize(IDBDataFile* pFile, long long& fileSize) const;
EXPORT int getFileSize(FID fid, uint16_t dbRoot, uint32_t partition, uint16_t segment,
long long& fileSize) const;
/**
* @brief Construct directory path for the specified fid (OID), DBRoot, and
* partition number. Directory does not have to exist, nor is it created.
*/
int getDirName( FID fid, uint16_t dbRoot,
uint32_t partition,
std::string& dirName) const;
/**
* @brief Initialize an extent in a dictionary store file
* @param pFile (in) IDBDataFile* of dictionary store file to be written to
* @param dbRoot (in) - DBRoot of pFile
* @param nBlocks (in) - number of blocks to be written for an extent
* @param blockHdrInit(in) - data used to initialize each block header
* @param blockHdrInitSize(in) - number of bytes in blockHdrInit
* @param bExpandExtent (in) - Expand existing extent, or initialize new one
* @param bOptExtension (in) - skip or optimize full extent preallocation
*/
EXPORT int initDctnryExtent(IDBDataFile* pFile, uint16_t dbRoot, int nBlocks, unsigned char* blockHdrInit,
int blockHdrInitSize, bool bExpandExtent, bool bOptExtension = false,
int64_t lbid = 0);
/**
* @brief Get the file size
*/
EXPORT int getFileSize( IDBDataFile* pFile, long long& fileSize ) const;
EXPORT int getFileSize( FID fid, uint16_t dbRoot,
uint32_t partition,
uint16_t segment,
long long& fileSize ) const;
/**
* @brief Check whether it is an directory
*/
EXPORT bool isDir(const char* dirName) const;
/**
* @brief Initialize an extent in a dictionary store file
* @param pFile (in) IDBDataFile* of dictionary store file to be written to
* @param dbRoot (in) - DBRoot of pFile
* @param nBlocks (in) - number of blocks to be written for an extent
* @param blockHdrInit(in) - data used to initialize each block header
* @param blockHdrInitSize(in) - number of bytes in blockHdrInit
* @param bExpandExtent (in) - Expand existing extent, or initialize new one
* @param bOptExtension (in) - skip or optimize full extent preallocation
*/
EXPORT int initDctnryExtent( IDBDataFile* pFile,
uint16_t dbRoot,
int nBlocks,
unsigned char* blockHdrInit,
int blockHdrInitSize,
bool bExpandExtent,
bool bOptExtension = false,
int64_t lbid = 0);
/**
* @brief See if there is room in the file system for specific number of blks
* @param fileName Name of file to extend (does not have to be full name)
* @param nBlocks Number of 8192-byte blocks to be added
* @return returns TRUE if file system has room for 'nBlocks', else FALSE
*/
EXPORT bool isDiskSpaceAvail(const std::string& fileName, int nBlocks) const;
/**
* @brief Check whether it is an directory
*/
EXPORT bool isDir( const char* dirName ) const;
/**
* @brief Convert an oid to a full file name
*/
EXPORT int oid2FileName(FID fid, char* fullFileName, bool bCreateDir, uint16_t dbRoot, uint32_t partition,
uint16_t segment) const;
EXPORT int oid2DirName(FID fid, char* oidDirName) const;
/**
* @brief See if there is room in the file system for specific number of blks
* @param fileName Name of file to extend (does not have to be full name)
* @param nBlocks Number of 8192-byte blocks to be added
* @return returns TRUE if file system has room for 'nBlocks', else FALSE
*/
EXPORT bool isDiskSpaceAvail(const std::string& fileName,
int nBlocks) const;
/**
* @brief Open a file using a filename.
* @param fileName Name of the file to open.
* @param mode Mode to use in opening the file (ex: "r+b").
* @param ioBuffSize Buffer size to be employed by setvbuf().
* @return returns the IDBDataFile* of the opened file.
*/
EXPORT IDBDataFile* openFile(const char* fileName, const char* mode = "r+b", int ioColSize = DEFAULT_COLSIZ,
bool useTmpSuffix = false) const;
/**
* @brief Convert an oid to a full file name
*/
EXPORT int oid2FileName( FID fid, char* fullFileName,
bool bCreateDir, uint16_t dbRoot,
uint32_t partition, uint16_t segment ) const;
EXPORT int oid2DirName( FID fid, char* oidDirName ) const;
/**
* @brief Open a file using an OID, dbroot, partition, and segment number.
* @param fid OID of the file to be opened.
* @param dbRoot DBRoot of the file to be opened.
* @param partition Partition number of the file to be opened.
* @param segment Segment number of the file to be opened.
* @param mode Mode to use in opening the file (default of "r+b" will open
* an existing binary file as read/write.
* @param ioBuffSize Buffer size to be employed by setvbuf().
* @return returns the IDBDataFile* of the opened file.
*/
EXPORT IDBDataFile* openFile(FID fid, uint16_t dbRoot, uint32_t partition, uint16_t segment,
std::string& segFile, const char* mode = "r+b", int ioColSize = DEFAULT_COLSIZ,
bool useTmpSuffix = false) const;
/**
* @brief Open a file using a filename.
* @param fileName Name of the file to open.
* @param mode Mode to use in opening the file (ex: "r+b").
* @param ioBuffSize Buffer size to be employed by setvbuf().
* @return returns the IDBDataFile* of the opened file.
*/
EXPORT IDBDataFile* openFile( const char* fileName,
const char* mode = "r+b",
int ioColSize = DEFAULT_COLSIZ,
bool useTmpSuffix = false) const;
/**
* @brief Read to a buffer from a file at current location
*/
EXPORT int readFile(IDBDataFile* pFile, unsigned char* readBuf, int readSize) const;
/**
* @brief Open a file using an OID, dbroot, partition, and segment number.
* @param fid OID of the file to be opened.
* @param dbRoot DBRoot of the file to be opened.
* @param partition Partition number of the file to be opened.
* @param segment Segment number of the file to be opened.
* @param mode Mode to use in opening the file (default of "r+b" will open
* an existing binary file as read/write.
* @param ioBuffSize Buffer size to be employed by setvbuf().
* @return returns the IDBDataFile* of the opened file.
*/
EXPORT IDBDataFile* openFile( FID fid,
uint16_t dbRoot,
uint32_t partition,
uint16_t segment,
std::string& segFile,
const char* mode = "r+b",
int ioColSize = DEFAULT_COLSIZ,
bool useTmpSuffix = false) const;
/**
* @brief Reads in 2 compression header blocks from a column segment file.
* IDBDataFile* points to start of data when function returns.
* @param pFile (in) IDBDataFile* of column segment file to be read.
* @param hdrs (out) Contents of headers that are read.
*/
EXPORT int readHeaders(IDBDataFile* pFile, char* hdrs) const;
EXPORT int readHeaders(IDBDataFile* pFile, char* hdr1, char* hdr2) const;
/**
* @brief Read to a buffer from a file at current location
*/
EXPORT int readFile( IDBDataFile* pFile, unsigned char* readBuf,
int readSize ) const;
/**
* @brief Reinitialize a partial extent in a column segment file
* @param pFile (in) IDBDataFile* of column segment file to be written to
* @param startOffset (in) - file offset where blocks are to be written
* @param nBlocks (in) - number of blocks to be written to the extent
* @param emptyVal(in) - empty value to be used for column data values
* width (in) - width of the applicable column
*/
EXPORT int reInitPartialColumnExtent(IDBDataFile* pFile, long long startOffset, int nBlocks,
const uint8_t* emptyVal, int width);
/**
* @brief Reads in 2 compression header blocks from a column segment file.
* IDBDataFile* points to start of data when function returns.
* @param pFile (in) IDBDataFile* of column segment file to be read.
* @param hdrs (out) Contents of headers that are read.
*/
EXPORT int readHeaders( IDBDataFile* pFile, char* hdrs ) const;
EXPORT int readHeaders( IDBDataFile* pFile, char* hdr1, char* hdr2 )const;
/**
* @brief Reinitialize an extent in a dictionary store file
* @param pFile (in) IDBDataFile* of dictionary store file to be written to
* @param startOffset (in) - file offset where blocks are to be written
* @param nBlocks (in) - number of blocks to be written to the extent
* @param blockHdrInit(in) - data used to initialize each block header
* @param blockHdrInitSize(in) - number of bytes in blockHdrInit
*/
EXPORT int reInitPartialDctnryExtent(IDBDataFile* pFile, long long startOffset, int nBlocks,
unsigned char* blockHdrInit, int blockHdrInitSize);
/**
* @brief Reinitialize a partial extent in a column segment file
* @param pFile (in) IDBDataFile* of column segment file to be written to
* @param startOffset (in) - file offset where blocks are to be written
* @param nBlocks (in) - number of blocks to be written to the extent
* @param emptyVal(in) - empty value to be used for column data values
* width (in) - width of the applicable column
*/
EXPORT int reInitPartialColumnExtent( IDBDataFile* pFile,
long long startOffset,
int nBlocks,
const uint8_t* emptyVal,
int width );
/**
* @brief Set the file to specified location based on the offset
*/
EXPORT int setFileOffset(IDBDataFile* pFile, long long offset, int origin = SEEK_SET) const;
EXPORT int setFileOffsetBlock(IDBDataFile* pFile, uint64_t lbid, int origin = SEEK_SET) const;
/**
* @brief Reinitialize an extent in a dictionary store file
* @param pFile (in) IDBDataFile* of dictionary store file to be written to
* @param startOffset (in) - file offset where blocks are to be written
* @param nBlocks (in) - number of blocks to be written to the extent
* @param blockHdrInit(in) - data used to initialize each block header
* @param blockHdrInitSize(in) - number of bytes in blockHdrInit
*/
EXPORT int reInitPartialDctnryExtent( IDBDataFile* pFile,
long long startOffset,
int nBlocks,
unsigned char* blockHdrInit,
int blockHdrInitSize );
/**
* @brief Truncate the file to the specified file size
*/
EXPORT int truncateFile(IDBDataFile* pFile, long long fileSize) const;
/**
* @brief Set the file to specified location based on the offset
*/
EXPORT int setFileOffset( IDBDataFile* pFile,
long long offset,
int origin = SEEK_SET ) const;
EXPORT int setFileOffsetBlock( IDBDataFile* pFile,
uint64_t lbid,
int origin = SEEK_SET ) const;
/**
* @brief Write a buffer to a file at current location
*/
EXPORT int writeFile(IDBDataFile* pFile, const unsigned char* buf, int bufSize) const;
/**
* @brief Truncate the file to the specified file size
*/
EXPORT int truncateFile( IDBDataFile* pFile,
long long fileSize ) const;
/**
* @brief set the flag to use the instance to access the brm wrapper class
*/
EXPORT virtual void setTransId(const TxnID& transId);
EXPORT virtual void setBulkFlag(bool isBulkLoad);
EXPORT virtual void setFixFlag(bool isFix);
TxnID getTransId() const;
/**
* @brief Write a buffer to a file at current location
*/
EXPORT int writeFile( IDBDataFile* pFile,
const unsigned char* buf, int bufSize ) const;
void compressionType(int t);
int compressionType() const;
/**
* @brief set the flag to use the instance to access the brm wrapper class
*/
EXPORT virtual void setTransId( const TxnID& transId);
EXPORT virtual void setBulkFlag(bool isBulkLoad);
EXPORT virtual void setFixFlag(bool isFix);
TxnID getTransId() const;
EXPORT virtual int flushFile(int rc, std::map<FID, FID>& oids);
// Initialize an extent in a column segment file
// pFile (in) IDBDataFile* of column segment file to be written to
// dbRoot (in) - DBRoot of pFile
// nBlocks (in) - number of blocks to be written for an extent
// emptyVal(in) - empty value to be used for column data values
// width (in) - width of the applicable column
// bNewFile (in) - Adding extent to new file
// bExpandExtent (in) - Expand existing extent, or initialize new one
// bAbbrevExtent (in) - If adding new extent, is it abbreviated
// bOptExtension(in) - skip or optimize full extent preallocation
int initColumnExtent(IDBDataFile* pFile, uint16_t dbRoot, int nBlocks, const uint8_t* emptyVal, int width,
execplan::CalpontSystemCatalog::ColDataType colDataType, bool bNewFile,
bool bExpandExtent, bool bAbbrevExtent, bool bOptExtension = false, int64_t lbid = 0);
void compressionType(int t);
int compressionType() const;
// Calls a chown and logs an error message
bool chownDataPath(const std::string& fileName) const;
EXPORT virtual int flushFile(int rc, std::map<FID, FID>& oids);
// Initialize an extent in a column segment file
// pFile (in) IDBDataFile* of column segment file to be written to
// dbRoot (in) - DBRoot of pFile
// nBlocks (in) - number of blocks to be written for an extent
// emptyVal(in) - empty value to be used for column data values
// width (in) - width of the applicable column
// bNewFile (in) - Adding extent to new file
// bExpandExtent (in) - Expand existing extent, or initialize new one
// bAbbrevExtent (in) - If adding new extent, is it abbreviated
// bOptExtension(in) - skip or optimize full extent preallocation
int initColumnExtent( IDBDataFile* pFile,
uint16_t dbRoot,
int nBlocks,
const uint8_t* emptyVal,
int width,
execplan::CalpontSystemCatalog::ColDataType colDataType,
bool bNewFile,
bool bExpandExtent,
bool bAbbrevExtent,
bool bOptExtension=false,
int64_t lbid = 0 );
protected:
EXPORT virtual int updateColumnExtent(IDBDataFile* pFile, int nBlocks, int64_t lbid);
EXPORT virtual int updateDctnryExtent(IDBDataFile* pFile, int nBlocks, int64_t lbid);
// Calls a chown and logs an error message
bool chownDataPath(const std::string& fileName) const;
int m_compressionType; // compresssion type
protected:
EXPORT virtual int updateColumnExtent(IDBDataFile* pFile, int nBlocks, int64_t lbid);
EXPORT virtual int updateDctnryExtent(IDBDataFile* pFile, int nBlocks, int64_t lbid);
private:
// not copyable
FileOp(const FileOp& rhs);
FileOp& operator=(const FileOp& rhs);
int m_compressionType; // compresssion type
int expandAbbrevColumnChunk(IDBDataFile* pFile, const uint8_t* emptyVal, int colWidth,
const compress::CompChunkPtr& chunkInPtr, compress::CompChunkPtr& chunkOutPt,
const char* hdrs = nullptr);
private:
//not copyable
FileOp(const FileOp& rhs);
FileOp& operator=(const FileOp& rhs);
int initAbbrevCompColumnExtent(IDBDataFile* pFile, uint16_t dbRoot, int nBlocks, const uint8_t* emptyVal,
int width, BRM::LBID_t lbid,
execplan::CalpontSystemCatalog::ColDataType colDataType);
int expandAbbrevColumnChunk(IDBDataFile* pFile, const uint8_t* emptyVal,
int colWidth,
const compress::CompChunkPtr& chunkInPtr,
compress::CompChunkPtr& chunkOutPt,
const char* hdrs = nullptr);
static void initDbRootExtentMutexes();
static void removeDbRootExtentMutexes();
int initAbbrevCompColumnExtent(
IDBDataFile* pFile, uint16_t dbRoot, int nBlocks,
const uint8_t* emptyVal, int width, BRM::LBID_t lbid,
execplan::CalpontSystemCatalog::ColDataType colDataType);
int writeInitialCompColumnChunk(IDBDataFile* pFile, int nBlocksAllocated, int nRows,
const uint8_t* emptyVal, int width, BRM::LBID_t lbid,
execplan::CalpontSystemCatalog::ColDataType colDataType, char* hdrs);
static void initDbRootExtentMutexes();
static void removeDbRootExtentMutexes();
TxnID m_transId;
bool m_isBulk;
bool m_isFix;
int writeInitialCompColumnChunk(
IDBDataFile* pFile, int nBlocksAllocated, int nRows,
const uint8_t* emptyVal, int width, BRM::LBID_t lbid,
execplan::CalpontSystemCatalog::ColDataType colDataType, char* hdrs);
// protect creation of m_DbRootAddExtentMutexes
static boost::mutex m_createDbRootMutexes;
TxnID m_transId;
bool m_isBulk;
bool m_isFix;
// Mutexes used to serialize extent creation within each DBRoot
static std::map<int, boost::mutex*> m_DbRootAddExtentMutexes;
// protect creation of m_DbRootAddExtentMutexes
static boost::mutex m_createDbRootMutexes;
// protect race condition in creating directories
static boost::mutex m_mkdirMutex;
// Mutexes used to serialize extent creation within each DBRoot
static std::map<int, boost::mutex*> m_DbRootAddExtentMutexes;
// protect race condition in creating directories
static boost::mutex m_mkdirMutex;
char* m_buffer; // buffer used with setvbuf()
char* m_buffer; // buffer used with setvbuf()
};
//------------------------------------------------------------------------------
@ -568,42 +483,39 @@ private:
//------------------------------------------------------------------------------
inline void FileOp::compressionType(int t)
{
m_compressionType = t;
m_compressionType = t;
}
inline int FileOp::compressionType() const
{
return m_compressionType;
return m_compressionType;
}
inline int FileOp::createDir( const char* dirName ) const
inline int FileOp::createDir(const char* dirName) const
{
return createDir( dirName, S_IRWXU | S_IRWXG | S_IROTH | S_IXOTH );
return createDir(dirName, S_IRWXU | S_IRWXG | S_IROTH | S_IXOTH);
}
inline int FileOp::getVBFileName( FID fid, char* fileName ) const
inline int FileOp::getVBFileName(FID fid, char* fileName) const
{
uint16_t dbRoot = 0;
uint32_t partition = 0;
uint16_t segment = 0;
uint16_t dbRoot = 0;
uint32_t partition = 0;
uint16_t segment = 0;
return oid2FileName( fid, fileName, true, dbRoot, partition, segment );
return oid2FileName(fid, fileName, true, dbRoot, partition, segment);
}
inline int FileOp::getFileName( FID fid, char* fileName,
uint16_t dbRoot,
uint32_t partition,
uint16_t segment ) const
inline int FileOp::getFileName(FID fid, char* fileName, uint16_t dbRoot, uint32_t partition,
uint16_t segment) const
{
return oid2FileName( fid, fileName, false, dbRoot, partition, segment );
return oid2FileName(fid, fileName, false, dbRoot, partition, segment);
}
inline TxnID FileOp::getTransId() const
{
return m_transId;
return m_transId;
}
} //end of namespace
} // namespace WriteEngine
#undef EXPORT

568
writeengine/shared/we_index.h
View File

@ -16,9 +16,9 @@
MA 02110-1301, USA. */
/******************************************************************************************
* $Id: we_index.h 4450 2013-01-21 14:13:24Z rdempsey $
*
******************************************************************************************/
* $Id: we_index.h 4450 2013-01-21 14:13:24Z rdempsey $
*
******************************************************************************************/
/** @file */
#pragma once
@ -27,117 +27,114 @@
#include "we_type.h"
/** Namespace WriteEngine */
namespace WriteEngine
{
/*****************************************************
* index definition
******************************************************/
const int IDX_BITTEST_SIZE = 10; /** @brief The bit size of bit test */
const int IDX_GROUP_SIZE = 3; /** @brief The bit size of group */
const int IDX_INSTRU_SIZE = 4; /** @brief The bit size of instruction */
const int IDX_PTR_SIZE = 46; /** @brief The bit size of address pointer */
const int IDX_TYPE_SIZE = 3; /** @brief The bit size of type */
* index definition
******************************************************/
const int IDX_BITTEST_SIZE = 10; /** @brief The bit size of bit test */
const int IDX_GROUP_SIZE = 3; /** @brief The bit size of group */
const int IDX_INSTRU_SIZE = 4; /** @brief The bit size of instruction */
const int IDX_PTR_SIZE = 46; /** @brief The bit size of address pointer */
const int IDX_TYPE_SIZE = 3; /** @brief The bit size of type */
const int IDX_BITMAP_SUBBLOCK_NO = 1; /** @brief Subblock 1 of root block is for bitmap pointer*/
const int IDX_MAX_TREE_LEVEL = 128; /** @brief The maximum depth of a tree */
const int IDX_MAX_MULTI_COL_BIT = 256; /** @brief The maximum bits of a multi-column tree (256 bit)*/
const int IDX_MAX_MULTI_COL_IDX_LEVEL = 52; /** @brief The maximum depth of a multi-column tree */
const int IDX_MAX_MULTI_COL_IDX_NUM = 64; /** @brief The maximum number of columns for a multi-column index */
const int MAX_IDX_RID = 1024; /** @brief Maximum index rids for one shot */
const int IDX_DEFAULT_READ_ROW = 10000; /** @brief Default number of rows for one read for index */
const int IDX_BITMAP_SUBBLOCK_NO = 1; /** @brief Subblock 1 of root block is for bitmap pointer*/
const int IDX_MAX_TREE_LEVEL = 128; /** @brief The maximum depth of a tree */
const int IDX_MAX_MULTI_COL_BIT = 256; /** @brief The maximum bits of a multi-column tree (256 bit)*/
const int IDX_MAX_MULTI_COL_IDX_LEVEL = 52; /** @brief The maximum depth of a multi-column tree */
const int IDX_MAX_MULTI_COL_IDX_NUM = 64; /** @brief The maximum number of columns for a multi-column index */
const int MAX_IDX_RID = 1024; /** @brief Maximum index rids for one shot */
const int IDX_DEFAULT_READ_ROW = 10000; /** @brief Default number of rows for one read for index */
// todo: need to move a higher level share file for dictionary
const int RID_SIZE = 46;
const int RID_SIZE = 46;
// const int OID_SIZE = 24; /** @brief The bit size of object id */
const int FBO_SIZE = 36; /** @brief The bit size of file block offset */
const int SBID_SIZE = 5; /** @brief The bit size of sub block id */
const int ENTRY_SIZE = 5; /** @brief The bit size of entry location with a sub block */
const int FBO_SIZE = 36; /** @brief The bit size of file block offset */
const int SBID_SIZE = 5; /** @brief The bit size of sub block id */
const int ENTRY_SIZE = 5; /** @brief The bit size of entry location with a sub block */
const int LIST_SIZE_TYPE = 0;
const int LIST_RID_TYPE = 3;
const int LIST_NOT_USED_TYPE = 7;
const int LIST_HDR_SIZE = 32;
const int LIST_SUBBLOCK_TYPE = 4 ;
const int LIST_BLOCK_TYPE = 5 ;
const int LIST_LLP_TYPE = 6 ;
const int SUBBLOCK_TOTAL_BYTES = 256;
const int LIST_SUB_LLP_POS = 31;
const int LIST_LAST_LBID_POS = 30;
const int LIST_BLOCK_LLP_POS = 1023;
const int MAX_BLOCK_ENTRY = 1024;
const int MAX_SUB_RID_CNT = 30;
const int MAX_BLK_RID_CNT = 1023;
const int MAX_BLK_NARRAY_RID_CNT = 1018;
const int LBID_SBID_ENTRY = 46;
const int RID_COUNT_SIZE = 10;
const int CUR_BLK_POS_WIDTH = 2;
const int LLP_STATUS_WIDTH = 2;
const int LIST_ENTRY_WIDTH = 8;
const int LIST_BLK_LLP_ENTRY_WIDTH = 48;
const int BEGIN_LIST_BLK_LLP_POS = 1018;
const int NEXT_BLK_PTR_OFFSET = 5;
const int PARENT_PTR_OFFSET = 4;
const int TOTAL_NUM_ARRAY_PTR = 4;
const int ARRAY_LLP_EXIST = 1;
const int LLP_NOT_FULL = 0;
const int LLP_FULL = 1;
const int TOTAL_CUR_LEVEL = 10;
const int CUR_LEVEL_POS_WIDTH = 20;
const uint64_t INVALID_KEY = -1LL; /** @brief Invalid number */
const int LIST_SIZE_TYPE = 0;
const int LIST_RID_TYPE = 3;
const int LIST_NOT_USED_TYPE = 7;
const int LIST_HDR_SIZE = 32;
const int LIST_SUBBLOCK_TYPE = 4;
const int LIST_BLOCK_TYPE = 5;
const int LIST_LLP_TYPE = 6;
const int SUBBLOCK_TOTAL_BYTES = 256;
const int LIST_SUB_LLP_POS = 31;
const int LIST_LAST_LBID_POS = 30;
const int LIST_BLOCK_LLP_POS = 1023;
const int MAX_BLOCK_ENTRY = 1024;
const int MAX_SUB_RID_CNT = 30;
const int MAX_BLK_RID_CNT = 1023;
const int MAX_BLK_NARRAY_RID_CNT = 1018;
const int LBID_SBID_ENTRY = 46;
const int RID_COUNT_SIZE = 10;
const int CUR_BLK_POS_WIDTH = 2;
const int LLP_STATUS_WIDTH = 2;
const int LIST_ENTRY_WIDTH = 8;
const int LIST_BLK_LLP_ENTRY_WIDTH = 48;
const int BEGIN_LIST_BLK_LLP_POS = 1018;
const int NEXT_BLK_PTR_OFFSET = 5;
const int PARENT_PTR_OFFSET = 4;
const int TOTAL_NUM_ARRAY_PTR = 4;
const int ARRAY_LLP_EXIST = 1;
const int LLP_NOT_FULL = 0;
const int LLP_FULL = 1;
const int TOTAL_CUR_LEVEL = 10;
const int CUR_LEVEL_POS_WIDTH = 20;
const uint64_t INVALID_KEY = -1LL; /** @brief Invalid number */
/*****************************************************
* mask definition
******************************************************/
const int BIT_MASK_ARRAY[] = { 0x0,
0x01, /** @brief 1 bit mask */
0x03, /** @brief 2 bit mask */
0x07, /** @brief 3 bit mask */
0x0F, /** @brief 4 bit mask */
0x1F, /** @brief 5 bit mask */
0x3F /** @brief 6 bit mask */
};
* mask definition
******************************************************/
const int BIT_MASK_ARRAY[] = {
0x0, 0x01, /** @brief 1 bit mask */
0x03, /** @brief 2 bit mask */
0x07, /** @brief 3 bit mask */
0x0F, /** @brief 4 bit mask */
0x1F, /** @brief 5 bit mask */
0x3F /** @brief 6 bit mask */
};
/************************************************************************
* Type enumerations
************************************************************************/
enum IdxTreeEntryType /** @brief Index tree entry types */
enum IdxTreeEntryType /** @brief Index tree entry types */
{
EMPTY_ENTRY = 0, /** @brief Empty entry */
UNIQUE_VAL = 7, /** @brief Unique value */
EMPTY_LIST = 1, /** @brief Empty list pointer entry */
EMPTY_PTR = 2, /** @brief Empty pointer entry */
BIT_TEST = 3, /** @brief Bit test entry */
LEAF_LIST = 4, /** @brief Leaf list pointer */
BITMAP_PTR = 5, /** @brief Bitmap pointer */
// SORT_LIST = 5, /** @brief Sorted list pointer */
MULTI_COL = 6 /** @brief Multi-column index pointer */
EMPTY_ENTRY = 0, /** @brief Empty entry */
UNIQUE_VAL = 7, /** @brief Unique value */
EMPTY_LIST = 1, /** @brief Empty list pointer entry */
EMPTY_PTR = 2, /** @brief Empty pointer entry */
BIT_TEST = 3, /** @brief Bit test entry */
LEAF_LIST = 4, /** @brief Leaf list pointer */
BITMAP_PTR = 5, /** @brief Bitmap pointer */
// SORT_LIST = 5, /** @brief Sorted list pointer */
MULTI_COL = 6 /** @brief Multi-column index pointer */
};
enum IdxTreeGroupType /** @brief Index tree group types */
enum IdxTreeGroupType /** @brief Index tree group types */
{
ENTRY_1 = 0, /** @brief 1 entry per group */
ENTRY_2 = 1, /** @brief 2 entry per group */
ENTRY_4 = 2, /** @brief 4 entry per group */
ENTRY_8 = 3, /** @brief 8 entry per group */
ENTRY_16 = 4, /** @brief 16 entry per group */
ENTRY_32 = 5, /** @brief 32 entry per group */
ENTRY_BLK = 6 /** @brief 1k entry per group */
ENTRY_1 = 0, /** @brief 1 entry per group */
ENTRY_2 = 1, /** @brief 2 entry per group */
ENTRY_4 = 2, /** @brief 4 entry per group */
ENTRY_8 = 3, /** @brief 8 entry per group */
ENTRY_16 = 4, /** @brief 16 entry per group */
ENTRY_32 = 5, /** @brief 32 entry per group */
ENTRY_BLK = 6 /** @brief 1k entry per group */
};
enum IdxBitCompareType /** @brief Index bit compare types */
enum IdxBitCompareType /** @brief Index bit compare types */
{
BIT_5 = 0, /** @brief 5-bit compare */
BIT_10 = 1 /** @brief 10-bit compare */
BIT_5 = 0, /** @brief 5-bit compare */
BIT_10 = 1 /** @brief 10-bit compare */
};
enum IdxFreeMgrType /** @brief Index free manager types */
enum IdxFreeMgrType /** @brief Index free manager types */
{
TREE = 0, /** @brief Index tree type */
LIST = 1 /** @brief Index list type */
TREE = 0, /** @brief Index tree type */
LIST = 1 /** @brief Index list type */
};
/************************************************************************
@ -145,164 +142,119 @@ enum IdxFreeMgrType /** @brief Index free manager types */
************************************************************************/
typedef struct
{
uint64_t type :
IDX_TYPE_SIZE; /** @brief entry type */
uint64_t spare : 12; /** @brief spare bits */
uint64_t group :
IDX_GROUP_SIZE; /** @brief entry group type */
// The following is related to ptr
uint64_t fbo :
FBO_SIZE; /** @brief file block offset */
uint64_t sbid :
SBID_SIZE; /** @brief sub block id */
uint64_t entry :
ENTRY_SIZE; /** @brief entry within sub block */
} IdxStartSubBlockEntry; /** @brief Index start block entry structure */
uint64_t type : IDX_TYPE_SIZE; /** @brief entry type */
uint64_t spare : 12; /** @brief spare bits */
uint64_t group : IDX_GROUP_SIZE; /** @brief entry group type */
// The following is related to ptr
uint64_t fbo : FBO_SIZE; /** @brief file block offset */
uint64_t sbid : SBID_SIZE; /** @brief sub block id */
uint64_t entry : ENTRY_SIZE; /** @brief entry within sub block */
} IdxStartSubBlockEntry; /** @brief Index start block entry structure */
typedef struct
{
uint64_t type :
IDX_TYPE_SIZE; /** @brief entry type */
uint64_t spare : 2; /** @brief spare bits */
uint64_t group :
IDX_GROUP_SIZE; /** @brief entry group type */
// The following is related to ptr
uint64_t spare2 : 10; /** @brief spare bits */
uint64_t fbo :
FBO_SIZE; /** @brief file block offset */
uint64_t sbid :
SBID_SIZE; /** @brief sub block id */
uint64_t entry :
ENTRY_SIZE; /** @brief entry within sub block */
} IdxEmptyListEntry; /** @brief Index empty list entry structure */
uint64_t type : IDX_TYPE_SIZE; /** @brief entry type */
uint64_t spare : 2; /** @brief spare bits */
uint64_t group : IDX_GROUP_SIZE; /** @brief entry group type */
// The following is related to ptr
uint64_t spare2 : 10; /** @brief spare bits */
uint64_t fbo : FBO_SIZE; /** @brief file block offset */
uint64_t sbid : SBID_SIZE; /** @brief sub block id */
uint64_t entry : ENTRY_SIZE; /** @brief entry within sub block */
} IdxEmptyListEntry; /** @brief Index empty list entry structure */
typedef struct
{
uint64_t type :
IDX_TYPE_SIZE; /** @brief entry type */
uint64_t spare : 15; /** @brief spare bits */
// The following is related to ptr
uint64_t fbo :
FBO_SIZE; /** @brief file block offset */
uint64_t sbid :
SBID_SIZE; /** @brief sub block id */
uint64_t entry :
ENTRY_SIZE; /** @brief entry within sub block */
} IdxBitmapPointerEntry; /** @brief Index bitmap pointer entry structure */
uint64_t type : IDX_TYPE_SIZE; /** @brief entry type */
uint64_t spare : 15; /** @brief spare bits */
// The following is related to ptr
uint64_t fbo : FBO_SIZE; /** @brief file block offset */
uint64_t sbid : SBID_SIZE; /** @brief sub block id */
uint64_t entry : ENTRY_SIZE; /** @brief entry within sub block */
} IdxBitmapPointerEntry; /** @brief Index bitmap pointer entry structure */
typedef struct
{
uint64_t type :
IDX_TYPE_SIZE; /** @brief entry type */
uint64_t bitTest :
IDX_BITTEST_SIZE; /** @brief index bittest */
uint64_t group :
IDX_GROUP_SIZE; /** @brief entry group type */
uint64_t bitCompare : 1;
uint64_t spare : 1; /** @brief spare bits */
// The following is related to ptr
uint64_t fbo :
FBO_SIZE; /** @brief file block offset */
uint64_t sbid :
SBID_SIZE; /** @brief sub block id */
uint64_t entry :
ENTRY_SIZE; /** @brief entry within sub block */
} IdxBitTestEntry; /** @brief Index bit test entry structure */
uint64_t type : IDX_TYPE_SIZE; /** @brief entry type */
uint64_t bitTest : IDX_BITTEST_SIZE; /** @brief index bittest */
uint64_t group : IDX_GROUP_SIZE; /** @brief entry group type */
uint64_t bitCompare : 1;
uint64_t spare : 1; /** @brief spare bits */
// The following is related to ptr
uint64_t fbo : FBO_SIZE; /** @brief file block offset */
uint64_t sbid : SBID_SIZE; /** @brief sub block id */
uint64_t entry : ENTRY_SIZE; /** @brief entry within sub block */
} IdxBitTestEntry; /** @brief Index bit test entry structure */
typedef struct
{
uint64_t type :
IDX_TYPE_SIZE; /** @brief entry type */
uint64_t spare : 15; /** @brief spare bits */
// The following is related to ptr
uint64_t fbo :
FBO_SIZE; /** @brief file block offset */
uint64_t sbid :
SBID_SIZE; /** @brief sub block id */
uint64_t entry :
ENTRY_SIZE; /** @brief entry within sub block */
} IdxTreePointerEntry; /** @brief Index tree pointer entry structure */
uint64_t type : IDX_TYPE_SIZE; /** @brief entry type */
uint64_t spare : 15; /** @brief spare bits */
// The following is related to ptr
uint64_t fbo : FBO_SIZE; /** @brief file block offset */
uint64_t sbid : SBID_SIZE; /** @brief sub block id */
uint64_t entry : ENTRY_SIZE; /** @brief entry within sub block */
} IdxTreePointerEntry; /** @brief Index tree pointer entry structure */
/************************************************************************
* @brief index list node defintions
************************************************************************/
typedef struct
{
uint64_t type :
IDX_TYPE_SIZE; /** @brief entry type 3 */
uint64_t spare : 15; /** @brief spare bits */
RID rid :
RID_SIZE; /** @brief row id */
} IdxRidListEntry; /** @brief Index rid list entry structure */
uint64_t type : IDX_TYPE_SIZE; /** @brief entry type 3 */
uint64_t spare : 15; /** @brief spare bits */
RID rid : RID_SIZE; /** @brief row id */
} IdxRidListEntry; /** @brief Index rid list entry structure */
typedef struct
{
uint64_t type :
IDX_TYPE_SIZE; /** @brief entry type */
uint64_t spare : 5;
uint64_t count :
RID_COUNT_SIZE; /** the count of rids on the current blk */
uint64_t llp :
LBID_SBID_ENTRY; /** @brief size */
uint64_t type : IDX_TYPE_SIZE; /** @brief entry type */
uint64_t spare : 5;
uint64_t count : RID_COUNT_SIZE; /** the count of rids on the current blk */
uint64_t llp : LBID_SBID_ENTRY; /** @brief size */
} IdxRidListPtr;
typedef struct
{
uint64_t type :
IDX_TYPE_SIZE; /** @brief entry type */
uint64_t spare : 5;
uint64_t count :
RID_COUNT_SIZE; /** the count of rids on the current blk */
uint64_t lbid :
FBO_SIZE; /** @brief size */
uint64_t sbid :
SBID_SIZE; /** @brief sub block id */
uint64_t entry :
ENTRY_SIZE; /** @brief entry within sub block */
uint64_t type : IDX_TYPE_SIZE; /** @brief entry type */
uint64_t spare : 5;
uint64_t count : RID_COUNT_SIZE; /** the count of rids on the current blk */
uint64_t lbid : FBO_SIZE; /** @brief size */
uint64_t sbid : SBID_SIZE; /** @brief sub block id */
uint64_t entry : ENTRY_SIZE; /** @brief entry within sub block */
} IdxRidLastListPtr;
typedef struct
{
uint64_t type :
IDX_TYPE_SIZE; /** @brief entry type */
uint64_t spare : 13;
uint64_t llpStat :
LLP_STATUS_WIDTH; /** llp status */
uint64_t childLbid :
FBO_SIZE; /** @brief file block offset */
uint64_t spare2 : 10;
uint64_t type : IDX_TYPE_SIZE; /** @brief entry type */
uint64_t spare : 13;
uint64_t llpStat : LLP_STATUS_WIDTH; /** llp status */
uint64_t childLbid : FBO_SIZE; /** @brief file block offset */
uint64_t spare2 : 10;
} IdxRidChildListPtr;
typedef struct
{
uint64_t type :
IDX_TYPE_SIZE; /** @brief entry type 0 or 6 */
uint64_t spare : 5;
uint64_t count :
RID_COUNT_SIZE; /** the count of rids on the current blk */
uint64_t nextLbid :
FBO_SIZE; /** @brief file block offset */
uint64_t curLevel :
TOTAL_CUR_LEVEL;
uint64_t type : IDX_TYPE_SIZE; /** @brief entry type 0 or 6 */
uint64_t spare : 5;
uint64_t count : RID_COUNT_SIZE; /** the count of rids on the current blk */
uint64_t nextLbid : FBO_SIZE; /** @brief file block offset */
uint64_t curLevel : TOTAL_CUR_LEVEL;
} IdxRidNextListPtr;
typedef struct
{
uint64_t type :
IDX_TYPE_SIZE; /** @brief entry type 6*/
uint64_t spare : 3; /** @brief spare bits */
uint64_t curLevelPos :
CUR_LEVEL_POS_WIDTH;
uint64_t curBlkPos :
CUR_BLK_POS_WIDTH; /** the position of current blk */
uint64_t parentLbid :
FBO_SIZE; /** @brief file block offset */
uint64_t type : IDX_TYPE_SIZE; /** @brief entry type 6*/
uint64_t spare : 3; /** @brief spare bits */
uint64_t curLevelPos : CUR_LEVEL_POS_WIDTH;
uint64_t curBlkPos : CUR_BLK_POS_WIDTH; /** the position of current blk */
uint64_t parentLbid : FBO_SIZE; /** @brief file block offset */
} IdxRidParentListPtr;
typedef struct
{
IdxRidChildListPtr childIdxRidListPtr[4];
IdxRidParentListPtr parentIdxListPtr;
IdxRidNextListPtr nextIdxListPtr;
IdxRidChildListPtr childIdxRidListPtr[4];
IdxRidParentListPtr parentIdxListPtr;
IdxRidNextListPtr nextIdxListPtr;
} IdxRidListArrayPtr;
/************************************************************************
@ -310,144 +262,144 @@ typedef struct
************************************************************************/
typedef struct
{
uint64_t type :
IDX_TYPE_SIZE; /** @brief entry type */
uint64_t spare : 15; /** @brief spare bits */
uint64_t size :
RID_SIZE; /** @brief size */
uint64_t type : IDX_TYPE_SIZE; /** @brief entry type */
uint64_t spare : 15; /** @brief spare bits */
uint64_t size : RID_SIZE; /** @brief size */
} IdxRidListHdrSize;
typedef struct
{
uint64_t type :
IDX_TYPE_SIZE; /** @brief entry type */
uint64_t spare : 15; /** @brief spare bits */
uint64_t llp :
RID_SIZE; /** @brief size */
uint64_t type : IDX_TYPE_SIZE; /** @brief entry type */
uint64_t spare : 15; /** @brief spare bits */
uint64_t llp : RID_SIZE; /** @brief size */
} IdxRidListHdrPtr;
typedef struct
{
IdxRidListHdrSize idxRidListSize;
uint64_t key;
IdxRidListEntry firstIdxRidListEntry;
IdxRidListHdrPtr nextIdxRidListPtr;
IdxRidListHdrSize idxRidListSize;
uint64_t key;
IdxRidListEntry firstIdxRidListEntry;
IdxRidListHdrPtr nextIdxRidListPtr;
} IdxRidListHdr;
typedef struct
{
uint64_t part1 : 15; /** @brief entry type */
uint64_t part2 : 15; /** @brief spare bits */
uint64_t spare : 34; /** @brief size */
uint64_t part1 : 15; /** @brief entry type */
uint64_t part2 : 15; /** @brief spare bits */
uint64_t spare : 34; /** @brief size */
} IdxRidListOffSet;
/************************************************************************
* @brief index tree node defintions
************************************************************************/
typedef struct
{
IdxBitTestEntry next; /** @brief next in the node */
IdxBitTestEntry current; /** @brief current addr */
uint16_t level; /** @brief tree level */
uint16_t allocCount; /** @brief allocated entry cound from free mgr */
uint16_t useCount; /** @brief actual use entry count */
uint16_t offset; /** @brief entry offset */
bool used; /** @brief used flag */
} IdxTreeNode; /** @brief Index tree node */
IdxBitTestEntry next; /** @brief next in the node */
IdxBitTestEntry current; /** @brief current addr */
uint16_t level; /** @brief tree level */
uint16_t allocCount; /** @brief allocated entry cound from free mgr */
uint16_t useCount; /** @brief actual use entry count */
uint16_t offset; /** @brief entry offset */
bool used; /** @brief used flag */
} IdxTreeNode; /** @brief Index tree node */
typedef struct
{
IdxTreeNode node[IDX_MAX_TREE_LEVEL]; /** @brief node array */
uint16_t maxLevel; /** @brief max level */
RID rid; /** @brief current row id */
uint64_t key; /** @brief current key */
uint16_t width; /** @brief current width */
} IdxTree; /** @brief Index tree */
IdxTreeNode node[IDX_MAX_TREE_LEVEL]; /** @brief node array */
uint16_t maxLevel; /** @brief max level */
RID rid; /** @brief current row id */
uint64_t key; /** @brief current key */
uint16_t width; /** @brief current width */
} IdxTree; /** @brief Index tree */
struct IdxTreeCacheNode
{
RID rid; /** @brief RID */
uint64_t key; /** @brief Key */
IdxEmptyListEntry entry; /** @brief List pointer */
bool used; /** @brief Used flag */
IdxTreeCacheNode()
{
used = false;
}
RID rid; /** @brief RID */
uint64_t key; /** @brief Key */
IdxEmptyListEntry entry; /** @brief List pointer */
bool used; /** @brief Used flag */
IdxTreeCacheNode()
{
used = false;
}
};
struct IdxMultiColKey
{
std::bitset<IDX_MAX_MULTI_COL_BIT> bitSet; /** @brief BitArray for all bits */
std::bitset<IDX_MAX_MULTI_COL_BIT> curBitset;/** @brief Current working column */
std::bitset<IDX_MAX_MULTI_COL_BIT> curMask; /** @brief Current bitset mask */
unsigned char keyBuf[IDX_MAX_MULTI_COL_BIT / 8]; /** @brief Key buffer */
int curLevel; /** @brief Current index level */
int maxLevel; /** @brief Maximum index level */
int totalBit; /** @brief Total bits */
int testbitArray[IDX_MAX_MULTI_COL_IDX_LEVEL]; /** @brief Test bit array */
void clear()
{
bitSet.reset();
curBitset.reset();
curMask.reset();
curLevel = maxLevel = 0;
totalBit = 0;
memset( testbitArray, 0, IDX_MAX_MULTI_COL_IDX_LEVEL * sizeof(testbitArray[0]));
memset( keyBuf, 0, IDX_MAX_MULTI_COL_BIT / 8 );
curMask = 0x1F;
curMask = curMask << (IDX_MAX_MULTI_COL_BIT - 5);
}
IdxMultiColKey()
{
clear();
}
std::bitset<IDX_MAX_MULTI_COL_BIT> bitSet; /** @brief BitArray for all bits */
std::bitset<IDX_MAX_MULTI_COL_BIT> curBitset; /** @brief Current working column */
std::bitset<IDX_MAX_MULTI_COL_BIT> curMask; /** @brief Current bitset mask */
unsigned char keyBuf[IDX_MAX_MULTI_COL_BIT / 8]; /** @brief Key buffer */
int curLevel; /** @brief Current index level */
int maxLevel; /** @brief Maximum index level */
int totalBit; /** @brief Total bits */
int testbitArray[IDX_MAX_MULTI_COL_IDX_LEVEL]; /** @brief Test bit array */
void clear()
{
bitSet.reset();
curBitset.reset();
curMask.reset();
curLevel = maxLevel = 0;
totalBit = 0;
memset(testbitArray, 0, IDX_MAX_MULTI_COL_IDX_LEVEL * sizeof(testbitArray[0]));
memset(keyBuf, 0, IDX_MAX_MULTI_COL_BIT / 8);
curMask = 0x1F;
curMask = curMask << (IDX_MAX_MULTI_COL_BIT - 5);
}
IdxMultiColKey()
{
clear();
}
};
struct IdxMultiRid
{
RID* ridArray; /** @brief RID array */
int totalRid; /** @brief Total number of row id */
IdxMultiRid()
{
totalRid = 0;
ridArray = NULL;
}
void setMultiRid( RID* rids, const int size )
{
totalRid = size;
ridArray = rids;
/* ridArray = new RID[size];
memcpy( ridArray, rids, size * sizeof( RID ) ); */
}
void clearMultiRid() { /*if( ridArray != NULL ) delete [] ridArray; ridArray = NULL;*/ } // we don't want to get into this mem business
RID* ridArray; /** @brief RID array */
int totalRid; /** @brief Total number of row id */
IdxMultiRid()
{
totalRid = 0;
ridArray = NULL;
}
void setMultiRid(RID* rids, const int size)
{
totalRid = size;
ridArray = rids;
/* ridArray = new RID[size];
memcpy( ridArray, rids, size * sizeof( RID ) ); */
}
void clearMultiRid()
{ /*if( ridArray != NULL ) delete [] ridArray; ridArray = NULL;*/
} // we don't want to get into this mem business
};
struct IdxLoadParam
{
File sourceFile; /** @brief Source file contatin values */
File sourceFile; /** @brief Source file contatin values */
OID indexTreeOid; /** @brief Target index tree oid */
OID indexListOid; /** @brief Target index list oid */
execplan::CalpontSystemCatalog::ColDataType indexColDataType; /** @brief Target index column type */
int indexWidth; /** @brief Target index width */
OID indexTreeOid; /** @brief Target index tree oid */
OID indexListOid; /** @brief Target index list oid */
execplan::CalpontSystemCatalog::ColDataType indexColDataType; /** @brief Target index column type */
int indexWidth; /** @brief Target index width */
int maxLoadRow; /** @brief Max rows for one load */
int maxLoadRow; /** @brief Max rows for one load */
void setIdxLoadParam( const OID treeOid, const OID listOid, const execplan::CalpontSystemCatalog::ColDataType colDataType, const int width, const int maxRow )
{
indexTreeOid = treeOid;
indexListOid = listOid;
indexColDataType = colDataType;
indexWidth = width;
maxLoadRow = maxRow;
}
bool isValid()
{
return indexTreeOid && indexListOid && indexWidth && maxLoadRow;
}
IdxLoadParam()
{
indexTreeOid = indexListOid = indexWidth = maxLoadRow = 0;
}
void setIdxLoadParam(const OID treeOid, const OID listOid,
const execplan::CalpontSystemCatalog::ColDataType colDataType, const int width,
const int maxRow)
{
indexTreeOid = treeOid;
indexListOid = listOid;
indexColDataType = colDataType;
indexWidth = width;
maxLoadRow = maxRow;
}
bool isValid()
{
return indexTreeOid && indexListOid && indexWidth && maxLoadRow;
}
IdxLoadParam()
{
indexTreeOid = indexListOid = indexWidth = maxLoadRow = 0;
}
};
} //end of namespace
} // namespace WriteEngine

213
writeengine/shared/we_log.cpp
View File

@ -16,9 +16,9 @@
MA 02110-1301, USA. */
/*******************************************************************************
* $Id: we_log.cpp 4504 2013-02-02 00:07:43Z bpaul $
*
*******************************************************************************/
* $Id: we_log.cpp 4504 2013-02-02 00:07:43Z bpaul $
*
*******************************************************************************/
#include "we_log.h"
@ -29,16 +29,14 @@
namespace WriteEngine
{
WriteEngine::WErrorCodes ec; // referenced as extern by chunkmanager
WriteEngine::WErrorCodes ec; // referenced as extern by chunkmanager
//------------------------------------------------------------------------------
// Constructor
//------------------------------------------------------------------------------
Log::Log() : m_bConsoleOutput( true ),
m_logFileName( "" ),
m_errlogFileName( "" )
Log::Log() : m_bConsoleOutput(true), m_logFileName(""), m_errlogFileName("")
{
m_pid = ::getpid();
m_pid = ::getpid();
}
//------------------------------------------------------------------------------
@ -46,8 +44,8 @@ Log::Log() : m_bConsoleOutput( true ),
//------------------------------------------------------------------------------
Log::~Log()
{
m_logFile.close();
m_errLogFile.close();
m_logFile.close();
m_errLogFile.close();
}
//------------------------------------------------------------------------------
@ -61,35 +59,31 @@ Log::~Log()
// RETURN:
// none
//------------------------------------------------------------------------------
void Log::formatMsg( const std::string& msg,
MsgLevel level,
std::ostringstream& oss,
int code ) const
void Log::formatMsg(const std::string& msg, MsgLevel level, std::ostringstream& oss, int code) const
{
// Constructing and logging the entire message as one string, should
// help avoid any thread contention that could cause logging output
// to be interweaved between threads.
oss << Convertor::getTimeStr();
// Constructing and logging the entire message as one string, should
// help avoid any thread contention that could cause logging output
// to be interweaved between threads.
oss << Convertor::getTimeStr();
// Include thread id in log message based on debug level
if (isDebug( DEBUG_2 ))
{
oss << " (" << m_pid << ":" <<
// Include thread id in log message based on debug level
if (isDebug(DEBUG_2))
{
oss << " (" << m_pid << ":" <<
#ifdef _MSC_VER
GetCurrentThreadId()
GetCurrentThreadId()
#else
pthread_self()
pthread_self()
#endif
<< ") " <<
MSG_LEVEL_STR[level] << " : " << msg ;
}
else
{
oss << " (" << m_pid << ") " << MSG_LEVEL_STR[level] << " : " << msg ;
}
<< ") " << MSG_LEVEL_STR[level] << " : " << msg;
}
else
{
oss << " (" << m_pid << ") " << MSG_LEVEL_STR[level] << " : " << msg;
}
if ( code > 0 )
oss << " [" << code << "]";
if (code > 0)
oss << " [" << code << "]";
}
//------------------------------------------------------------------------------
@ -107,46 +101,44 @@ void Log::formatMsg( const std::string& msg,
// RETURN:
// none
//------------------------------------------------------------------------------
void Log::logMsg( const char* msg,
int code,
MsgLevel level )
void Log::logMsg(const char* msg, int code, MsgLevel level)
{
std::ostringstream oss;
formatMsg( msg, level, oss, code );
std::ostringstream oss;
formatMsg(msg, level, oss, code);
// log error and critical msgs to syslog
if ( level == MSGLVL_ERROR || level == MSGLVL_CRITICAL )
// log error and critical msgs to syslog
if (level == MSGLVL_ERROR || level == MSGLVL_CRITICAL)
{
{
{
//log to log file and error log file within scope of mutex lock.
//logSyslog uses SimpleSyslog which has it's own lock.
boost::mutex::scoped_lock lk(m_WriteLockMutex);
// log to log file and error log file within scope of mutex lock.
// logSyslog uses SimpleSyslog which has it's own lock.
boost::mutex::scoped_lock lk(m_WriteLockMutex);
m_errLogFile << oss.str() << std::endl;
m_logFile << oss.str() << std::endl;
m_errLogFile << oss.str() << std::endl;
m_logFile << oss.str() << std::endl;
std::cerr << oss.str() << std::endl;
}
logSyslog( std::string(msg), code );
std::cerr << oss.str() << std::endl;
}
else
{
std::ostringstream oss2;
// Format msg again without including the status code.
// Only log INFO2 msgs to console if m_bConsoleOutput is TRUE;
// All other msg levels always go to console.
if ( (level != MSGLVL_INFO2) || (m_bConsoleOutput) )
formatMsg ( msg, level, oss2 );
logSyslog(std::string(msg), code);
}
else
{
std::ostringstream oss2;
boost::mutex::scoped_lock lk(m_WriteLockMutex);
// Format msg again without including the status code.
// Only log INFO2 msgs to console if m_bConsoleOutput is TRUE;
// All other msg levels always go to console.
if ((level != MSGLVL_INFO2) || (m_bConsoleOutput))
formatMsg(msg, level, oss2);
m_logFile << oss.str() << std::endl;
boost::mutex::scoped_lock lk(m_WriteLockMutex);
if ( (level != MSGLVL_INFO2) || (m_bConsoleOutput) )
std::cout << oss2.str() << std::endl;
}
m_logFile << oss.str() << std::endl;
if ((level != MSGLVL_INFO2) || (m_bConsoleOutput))
std::cout << oss2.str() << std::endl;
}
}
//------------------------------------------------------------------------------
@ -158,30 +150,26 @@ void Log::logMsg( const char* msg,
// RETURN:
// none
//------------------------------------------------------------------------------
void Log::setLogFileName( const char* logfile,
const char* errlogfile,
bool consoleFlag )
void Log::setLogFileName(const char* logfile, const char* errlogfile, bool consoleFlag)
{
m_logFileName = logfile;
m_errlogFileName = errlogfile;
m_bConsoleOutput = consoleFlag;
m_logFileName = logfile;
m_errlogFileName = errlogfile;
m_bConsoleOutput = consoleFlag;
#ifdef _MSC_VER
// cpimport.bin calls BulkLoad::loadJobInfo() before calling
// BulkLoad::processJob(). loadJobInfo() attempts to write to this log
// before it's opened (by processJob()). This doesn't seem to bother Linux
// but puts Windows in a bad state. Once this logic is fixed, this hack can
// go away.
// This code probably wouldn't hurt if run on Linux, but I'll leave this
// here as a reminder to fix the logic for all platforms.
m_logFile.close();
m_logFile.clear();
m_errLogFile.close();
m_errLogFile.clear();
// cpimport.bin calls BulkLoad::loadJobInfo() before calling
// BulkLoad::processJob(). loadJobInfo() attempts to write to this log
// before it's opened (by processJob()). This doesn't seem to bother Linux
// but puts Windows in a bad state. Once this logic is fixed, this hack can
// go away.
// This code probably wouldn't hurt if run on Linux, but I'll leave this
// here as a reminder to fix the logic for all platforms.
m_logFile.close();
m_logFile.clear();
m_errLogFile.close();
m_errLogFile.clear();
#endif
m_logFile.open( m_logFileName.c_str(),
std::ofstream::out | std::ofstream::app );
m_errLogFile.open(m_errlogFileName.c_str(),
std::ofstream::out | std::ofstream::app);
m_logFile.open(m_logFileName.c_str(), std::ofstream::out | std::ofstream::app);
m_errLogFile.open(m_errlogFileName.c_str(), std::ofstream::out | std::ofstream::app);
}
//------------------------------------------------------------------------------
@ -193,38 +181,34 @@ void Log::setLogFileName( const char* logfile,
// RETURN:
// none
//------------------------------------------------------------------------------
void Log::logSyslog( const std::string& msg,
int statusCode )
void Log::logSyslog(const std::string& msg, int statusCode)
{
logging::Message::MessageID msgId = logging::M0087;
logging::Message::MessageID msgId = logging::M0087;
switch (statusCode)
switch (statusCode)
{
case ERR_FILE_DISK_SPACE:
{
case ERR_FILE_DISK_SPACE:
{
msgId = logging::M0076;
break;
}
case ERR_UNKNOWN:
{
msgId = logging::M0017;
break;
}
default:
{
msgId = logging::M0087;
break;
}
msgId = logging::M0076;
break;
}
logging::Message::Args errMsgArgs;
errMsgArgs.add( msg );
SimpleSysLog::instance()->logMsg(
errMsgArgs,
logging::LOG_TYPE_ERROR,
msgId);
case ERR_UNKNOWN:
{
msgId = logging::M0017;
break;
}
default:
{
msgId = logging::M0087;
break;
}
}
logging::Message::Args errMsgArgs;
errMsgArgs.add(msg);
SimpleSysLog::instance()->logMsg(errMsgArgs, logging::LOG_TYPE_ERROR, msgId);
}
//------------------------------------------------------------------------------
// DESCRIPTION:
@ -239,9 +223,8 @@ void Log::logSyslog( const std::string& msg,
void Log::closeLog()
{
m_logFile.close();
m_errLogFile.close();
m_logFile.close();
m_errLogFile.close();
}
} //end of namespace
} // namespace WriteEngine

115
writeengine/shared/we_log.h
View File

@ -16,9 +16,9 @@
MA 02110-1301, USA. */
/*******************************************************************************
* $Id: we_log.h 4504 2013-02-02 00:07:43Z bpaul $
*
*******************************************************************************/
* $Id: we_log.h 4504 2013-02-02 00:07:43Z bpaul $
*
*******************************************************************************/
/** @file */
#pragma once
@ -44,83 +44,68 @@
/** Namespace WriteEngine */
namespace WriteEngine
{
const std::string MSG_LEVEL_STR[] =
{
"INFO",
"INFO2",
"WARN",
"ERR ",
"CRIT"
};
const std::string MSG_LEVEL_STR[] = {"INFO", "INFO2", "WARN", "ERR ", "CRIT"};
/** @brief Class is used to format and write log messages to cpimport.bin log
* file. When applicable, messages are also logged to syslog logs as well.
*/
class Log : public WEObj
{
public:
/**
* @brief Constructor
*/
EXPORT Log();
public:
/**
* @brief Constructor
*/
EXPORT Log();
/**
* @brief Destructor
*/
EXPORT ~Log();
/**
* @brief Destructor
*/
EXPORT ~Log();
/**
* @brief Log a cpimport.bin logfile message; logs errors to syslog as well
*/
EXPORT void logMsg( const char* msg, int code, MsgLevel level );
EXPORT void logMsg( const char* msg, MsgLevel level )
{
logMsg( msg, 0, level );
}
EXPORT void logMsg( const std::string& msg, MsgLevel level )
{
logMsg( msg.c_str(), level );
}
EXPORT void logMsg( const std::string& msg, int code, MsgLevel level )
{
logMsg( msg.c_str(), code, level );
}
/**
* @brief Log a cpimport.bin logfile message; logs errors to syslog as well
*/
EXPORT void logMsg(const char* msg, int code, MsgLevel level);
EXPORT void logMsg(const char* msg, MsgLevel level)
{
logMsg(msg, 0, level);
}
EXPORT void logMsg(const std::string& msg, MsgLevel level)
{
logMsg(msg.c_str(), level);
}
EXPORT void logMsg(const std::string& msg, int code, MsgLevel level)
{
logMsg(msg.c_str(), code, level);
}
/**
* @brief Set log file name
*/
EXPORT void setLogFileName( const char* logfile,
const char* errlogfile,
bool consoleFlag = true );
/**
* @brief Set log file name
*/
EXPORT void setLogFileName(const char* logfile, const char* errlogfile, bool consoleFlag = true);
// BUG 5022
/**
* @brief Set log files close other than calling d'tor
*/
EXPORT void closeLog();
// BUG 5022
/**
* @brief Set log files close other than calling d'tor
*/
EXPORT void closeLog();
private:
void logSyslog(const std::string& msg, int statusCode);
void formatMsg(const std::string& msg, MsgLevel level, std::ostringstream& oss, int code = 0) const;
private:
void logSyslog ( const std::string& msg,
int statusCode);
void formatMsg( const std::string& msg,
MsgLevel level,
std::ostringstream& oss,
int code = 0 ) const;
bool m_bConsoleOutput; // flag allowing INFO2 msg
// to display to console
std::string m_logFileName; // log file name
std::string m_errlogFileName; // error log file name
pid_t m_pid; // current pid
bool m_bConsoleOutput; // flag allowing INFO2 msg
// to display to console
std::string m_logFileName; // log file name
std::string m_errlogFileName; // error log file name
pid_t m_pid; // current pid
std::ofstream m_logFile; // log file stream
std::ofstream m_errLogFile; // error log file stream
std::ofstream m_logFile; // log file stream
std::ofstream m_errLogFile; // error log file stream
boost::mutex m_WriteLockMutex; // logging mutex
boost::mutex m_WriteLockMutex; // logging mutex
};
} //end of namespace
} // namespace WriteEngine
#undef EXPORT

41
writeengine/shared/we_macro.h
View File

@ -16,36 +16,37 @@
MA 02110-1301, USA. */
/******************************************************************************************
* $Id: we_macro.h 33 2006-10-18 14:37:12Z wzhou $
*
******************************************************************************************/
* $Id: we_macro.h 33 2006-10-18 14:37:12Z wzhou $
*
******************************************************************************************/
/** @file */
#pragma once
#include <we_type.h>
/** Namespace WriteEngine */
namespace WriteEngine
{
#define RETURN_ON_ERROR( statement ) \
{ int rcVal = (statement); \
if( rcVal != NO_ERROR ) \
return rcVal; }
#define RETURN_ON_ERROR(statement) \
{ \
int rcVal = (statement); \
if (rcVal != NO_ERROR) \
return rcVal; \
}
#define RETURN_ON_NULL( obj, rc ) \
if( obj == NULL ) \
return rc;
#define RETURN_ON_NULL(obj, rc) \
if (obj == NULL) \
return rc;
#define RETURN_ON_WE_ERROR( oldRc, newRc ) \
if( oldRc != NO_ERROR ) \
return newRc;
#define RETURN_ON_WE_ERROR(oldRc, newRc) \
if (oldRc != NO_ERROR) \
return newRc;
#define RETURN_RC( oldRc, newRc ) \
if( oldRc != NO_ERROR ) \
return newRc; \
else \
return NO_ERROR;
#define RETURN_RC(oldRc, newRc) \
if (oldRc != NO_ERROR) \
return newRc; \
else \
return NO_ERROR;
} //end of namespace
} // namespace WriteEngine

107
writeengine/shared/we_obj.h
View File

@ -16,9 +16,9 @@
MA 02110-1301, USA. */
/******************************************************************************************
* $Id: we_obj.h 33 2006-08-17 10:31:20Z wzhou $
*
******************************************************************************************/
* $Id: we_obj.h 33 2006-08-17 10:31:20Z wzhou $
*
******************************************************************************************/
/** @file */
#pragma once
@ -34,61 +34,64 @@ class Log;
/** Class WEObj */
class WEObj
{
public:
/**
* @brief Constructor
*/
WEObj() : m_debugLevel( DEBUG_0 ), m_log( 0 ) {}
public:
/**
* @brief Constructor
*/
WEObj() : m_debugLevel(DEBUG_0), m_log(0)
{
}
/**
* @brief Default Destructor
*/
~WEObj() {}
/**
* @brief Default Destructor
*/
~WEObj()
{
}
/**
* @brief Is it required to debug
*/
bool isDebug( const DebugLevel level ) const
{
return level <= m_debugLevel;
}
/**
* @brief Is it required to debug
*/
bool isDebug(const DebugLevel level) const
{
return level <= m_debugLevel;
}
/**
* @brief Get debug level
*/
DebugLevel getDebugLevel() const
{
return m_debugLevel;
}
/**
* @brief Get debug level
*/
DebugLevel getDebugLevel() const
{
return m_debugLevel;
}
/**
* @brief Get Logger object
*/
Log* getLogger() const
{
return m_log;
}
/**
* @brief Get Logger object
*/
Log* getLogger() const
{
return m_log;
}
/**
* @brief Set debug level
*/
void setDebugLevel( const DebugLevel level )
{
m_debugLevel = level;
}
/**
* @brief Set debug level
*/
void setDebugLevel(const DebugLevel level)
{
m_debugLevel = level;
}
/**
* @brief Set debug logger and debug level
*/
void setLogger( Log* logger )
{
m_log = logger;
}
/**
* @brief Set debug logger and debug level
*/
void setLogger(Log* logger)
{
m_log = logger;
}
private:
DebugLevel m_debugLevel; // internal use debug level
Log* m_log; // logger object for debug output
private:
DebugLevel m_debugLevel; // internal use debug level
Log* m_log; // logger object for debug output
};
} //end of namespace
} // namespace WriteEngine

2013
writeengine/shared/we_rbmetawriter.cpp
View File

File diff suppressed because it is too large Load Diff

403
writeengine/shared/we_rbmetawriter.h
View File

@ -16,8 +16,8 @@
MA 02110-1301, USA. */
/*
* $Id: we_rbmetawriter.h 4450 2013-01-21 14:13:24Z rdempsey $
*/
* $Id: we_rbmetawriter.h 4450 2013-01-21 14:13:24Z rdempsey $
*/
/** @file we_rbmetawriter.h
* Contains class to write HWM-related information used to rollback a
@ -59,24 +59,24 @@ class Log;
//------------------------------------------------------------------------------
struct RBChunkInfo
{
OID fOid; // dctnry store OID containing relevant chunk
uint16_t fDbRoot; // dbroot, partition, segment of file
uint32_t fPartition; // containing relevant HWM chunk
uint16_t fSegment; //
HWM fHwm; // HWM block of interest
RBChunkInfo(OID oid, uint16_t dbRoot, uint32_t partition,
uint16_t segment, HWM hwm ) :
fOid(oid), fDbRoot(dbRoot), fPartition(partition),
fSegment(segment), fHwm(hwm) { }
OID fOid; // dctnry store OID containing relevant chunk
uint16_t fDbRoot; // dbroot, partition, segment of file
uint32_t fPartition; // containing relevant HWM chunk
uint16_t fSegment; //
HWM fHwm; // HWM block of interest
RBChunkInfo(OID oid, uint16_t dbRoot, uint32_t partition, uint16_t segment, HWM hwm)
: fOid(oid), fDbRoot(dbRoot), fPartition(partition), fSegment(segment), fHwm(hwm)
{
}
};
class RBChunkInfoCompare
{
public:
bool operator()(const RBChunkInfo& lhs, const RBChunkInfo& rhs) const;
public:
bool operator()(const RBChunkInfo& lhs, const RBChunkInfo& rhs) const;
};
typedef std::set< RBChunkInfo, RBChunkInfoCompare > RBChunkSet;
typedef std::set<RBChunkInfo, RBChunkInfoCompare> RBChunkSet;
//------------------------------------------------------------------------------
/** @brief Class to write HWM-related information to support bulk rollbacks.
@ -135,245 +135,194 @@ typedef std::set< RBChunkInfo, RBChunkInfoCompare > RBChunkSet;
* parallel by several threads for different dictionary columns.
*/
//------------------------------------------------------------------------------
class RBMetaWriter: public WeUIDGID
class RBMetaWriter : public WeUIDGID
{
public:
public:
/** @brief RBMetaWriter constructor
* @param appDesc Description of application that is using RBMetaWriter
* @param logger Logger to be used for logging messages.
*/
EXPORT RBMetaWriter(const std::string& appDesc, Log* logger);
/** @brief RBMetaWriter constructor
* @param appDesc Description of application that is using RBMetaWriter
* @param logger Logger to be used for logging messages.
*/
EXPORT RBMetaWriter ( const std::string& appDesc,
Log* logger );
/** @brief RBMetaWriter destructor
*/
EXPORT ~RBMetaWriter()
{
closeMetaFile();
}
/** @brief RBMetaWriter destructor
*/
EXPORT ~RBMetaWriter ( )
{
closeMetaFile ( );
}
/** @brief Initialize this RBMetaWriter object
* Warning: This function may throw a WeException.
*
* @param tableOID OID of the table whose state is to be saved.
* @param tableName Name of the table associated with tableOID.
*/
EXPORT void init(OID tableOID, const std::string& tableName);
/** @brief Initialize this RBMetaWriter object
* Warning: This function may throw a WeException.
*
* @param tableOID OID of the table whose state is to be saved.
* @param tableName Name of the table associated with tableOID.
*/
EXPORT void init ( OID tableOID,
const std::string& tableName );
/** @brief Make a backup copy of the specified HWM dictionary store chunk.
* This operation only applies to compressed columns. Backup may not be
* necessary. Return value indicates whether the specified file needs to
* be backed up or not.
* Warning: This function may throw a WeException.
*
* This function is thread-safe since concurrent calls could be made by
* different threads, each for a different dictionary column.
*
* @param dctnryOID column OID to be saved
* @param dbRoot current dbRoot of last local HWM for columnOID
* @param partition current partition of last local HWM for columnOID
* @param segment current segment of last local HWM for columnOID
* @return Indicates whether it is necessary to perform backup
*/
EXPORT bool backupDctnryHWMChunk(OID dctnryOID, uint16_t dbRoot, uint32_t partition, uint16_t segment);
/** @brief Make a backup copy of the specified HWM dictionary store chunk.
* This operation only applies to compressed columns. Backup may not be
* necessary. Return value indicates whether the specified file needs to
* be backed up or not.
* Warning: This function may throw a WeException.
*
* This function is thread-safe since concurrent calls could be made by
* different threads, each for a different dictionary column.
*
* @param dctnryOID column OID to be saved
* @param dbRoot current dbRoot of last local HWM for columnOID
* @param partition current partition of last local HWM for columnOID
* @param segment current segment of last local HWM for columnOID
* @return Indicates whether it is necessary to perform backup
*/
EXPORT bool backupDctnryHWMChunk (
OID dctnryOID,
uint16_t dbRoot,
uint32_t partition,
uint16_t segment );
/** @brief Delete the rollback meta files associated with this table
* Warning: This function may throw a WeException.
*/
EXPORT void deleteFile();
/** @brief Delete the rollback meta files associated with this table
* Warning: This function may throw a WeException.
*/
EXPORT void deleteFile ( );
/** @brief Helper function that creates the primary meta data file.
* Warning: This function may throw a WeException.
*
* See class description for more details.
* @param columns Vector of column information. The dataFile member in
* each columns entry should be filled in with HWM information about
* the start extent where rows will be added. This information is
* needed so that the HWM chunk in that extent can be backed up.
* @param dctnryStoreOids Vector of dictionary store OID associated with
* columns vector. dctnryStoreOid[n] should be 0 if columns[n] is
* not a dictionary column.
* @param dbRootHwmInfoVecCol Vector of EmDbRootHWMInfo_v objects obtained
* from multiple calls to DBRM::getDbRootHWMInfo(). There is one
* EmDbRootHWMInfo_v entry per column. Each
* EmDbRootHWMInfo_v object carries a vector of DBRoot, HWM, etc
* objects representing the current HWM extents for a column's
* DBRoots on the local PM.
*/
EXPORT void saveBulkRollbackMetaData(const std::vector<Column>& columns,
const std::vector<OID>& dctnryStoreOids,
const std::vector<BRM::EmDbRootHWMInfo_v>& dbRootHWMInfoVecCol);
/** @brief Helper function that creates the primary meta data file.
* Warning: This function may throw a WeException.
*
* See class description for more details.
* @param columns Vector of column information. The dataFile member in
* each columns entry should be filled in with HWM information about
* the start extent where rows will be added. This information is
* needed so that the HWM chunk in that extent can be backed up.
* @param dctnryStoreOids Vector of dictionary store OID associated with
* columns vector. dctnryStoreOid[n] should be 0 if columns[n] is
* not a dictionary column.
* @param dbRootHwmInfoVecCol Vector of EmDbRootHWMInfo_v objects obtained
* from multiple calls to DBRM::getDbRootHWMInfo(). There is one
* EmDbRootHWMInfo_v entry per column. Each
* EmDbRootHWMInfo_v object carries a vector of DBRoot, HWM, etc
* objects representing the current HWM extents for a column's
* DBRoots on the local PM.
*/
EXPORT void saveBulkRollbackMetaData(
const std::vector<Column>& columns,
const std::vector<OID>& dctnryStoreOids,
const std::vector<BRM::EmDbRootHWMInfo_v>& dbRootHWMInfoVecCol );
/** @brief Verify that specified version record is for Version 3 */
static bool verifyVersion3(const char* versionRec);
/** @brief Verify that specified version record is for Version 3 */
static bool verifyVersion3(const char* versionRec);
/** @brief Verify that specified version record is for Version 4 */
static bool verifyVersion4(const char* versionRec);
/** @brief Verify that specified version record is for Version 4 */
static bool verifyVersion4(const char* versionRec);
/** @brief Verify that specified record type is a Column1 record */
static bool verifyColumn1Rec(const char* recType);
/** @brief Verify that specified record type is a Column1 record */
static bool verifyColumn1Rec(const char* recType);
/** @brief Verify that specified record type is a Column2 record */
static bool verifyColumn2Rec(const char* recType);
/** @brief Verify that specified record type is a Column2 record */
static bool verifyColumn2Rec(const char* recType);
/** @brief Verify that specified record type is a DStore1 record */
static bool verifyDStore1Rec(const char* recType);
/** @brief Verify that specified record type is a DStore1 record */
static bool verifyDStore1Rec(const char* recType);
/** @brief Verify that specified record type is a DStore2 record */
static bool verifyDStore2Rec(const char* recType);
/** @brief Verify that specified record type is a DStore2 record */
static bool verifyDStore2Rec(const char* recType);
private:
// disable copy constructor and assignment operator
RBMetaWriter(const RBMetaWriter&);
RBMetaWriter& operator=(const RBMetaWriter&);
private:
// disable copy constructor and assignment operator
RBMetaWriter(const RBMetaWriter&);
RBMetaWriter& operator=(const RBMetaWriter&);
// Make a backup copy of the specified HWM column chunk.
// This operation only applies to compressed columns.
// Warning: This function may throw a WeException.
// columnOID column OID to be saved
// dbRoot current dbRoot of last local HWM for columnOID
// partition current partition of last local HWM for columnOID
// segment current segment of last local HWM for columnOID
// lastLocalHwm current last local for column OID
void backupColumnHWMChunk (
OID columnOID,
uint16_t dbRoot,
uint32_t partition,
uint16_t segment,
HWM lastLocalHwm );
// Make a backup copy of the specified HWM column chunk.
// This operation only applies to compressed columns.
// Warning: This function may throw a WeException.
// columnOID column OID to be saved
// dbRoot current dbRoot of last local HWM for columnOID
// partition current partition of last local HWM for columnOID
// segment current segment of last local HWM for columnOID
// lastLocalHwm current last local for column OID
void backupColumnHWMChunk(OID columnOID, uint16_t dbRoot, uint32_t partition, uint16_t segment,
HWM lastLocalHwm);
// This function must be thread-safe since it is called directly by
// backupDctnryHWMChunk(). Employed by non-hdfs.
void backupHWMChunk (
bool bColumnFile,
OID columnOID,
uint16_t dbRoot,
uint32_t partition,
uint16_t segment,
HWM lastLocalHwm );
// This function must be thread-safe since it is called directly by
// backupDctnryHWMChunk(). Employed by non-hdfs.
void backupHWMChunk(bool bColumnFile, OID columnOID, uint16_t dbRoot, uint32_t partition, uint16_t segment,
HWM lastLocalHwm);
// This function must be thread-safe since it is called directly by
// backupDctnryHWMFile(). Employed by hdfs.
void backupHWMFile (
bool bColumnFile,
OID columnOID,
uint16_t dbRoot,
uint32_t partition,
uint16_t segment,
HWM lastLocalHwm );
// This function must be thread-safe since it is called directly by
// backupDctnryHWMFile(). Employed by hdfs.
void backupHWMFile(bool bColumnFile, OID columnOID, uint16_t dbRoot, uint32_t partition, uint16_t segment,
HWM lastLocalHwm);
// Close the current meta data file.
EXPORT void closeMetaFile ( );
// Close the current meta data file.
EXPORT void closeMetaFile();
void createSubDir( const std::string& metaFileName );
void deleteSubDir( const std::string& metaFileName );
int getSubDirPath(const uint16_t dbRoot,
std::string& subDirPath ) const;
void createSubDir(const std::string& metaFileName);
void deleteSubDir(const std::string& metaFileName);
int getSubDirPath(const uint16_t dbRoot, std::string& subDirPath) const;
// Open a meta data file to save HWM bulk rollback info for tableOID
// Warning: This function may throw a WeException.
// dbRoot is the DBRoot of interest for the applicable table.
std::string openMetaFile ( uint16_t dbRoot );
// Open a meta data file to save HWM bulk rollback info for tableOID
// Warning: This function may throw a WeException.
// dbRoot is the DBRoot of interest for the applicable table.
std::string openMetaFile(uint16_t dbRoot);
// Rename temporary metadata control file(s) to the permanent name.
// Filenames are taken from fMetaFileNames.
// Warning: This function may throw a WeException.
void renameMetaFile( );
// Rename temporary metadata control file(s) to the permanent name.
// Filenames are taken from fMetaFileNames.
// Warning: This function may throw a WeException.
void renameMetaFile();
// Save column meta data to the currently open file.
// This is the Shared-Nothing version of this function.
// Warning: This function may throw a WeException.
// metaFileName name of metafile to be written
// columnOID column OID to be saved
// dbRoot current dbRoot of last local HWM for columnOID
// partition current partition of last local HWM for columnOID
// segment current segment of last local HWM for columnOID
// lastLocalHwm current last local for column OID
// colType type of columnOID
// colTypeName type name of columnOID
// colWidth width (in bytes) of columnOID
// compressionType compression type
void writeColumnMetaData (
const std::string& metaFileName,
bool withHWM,
OID columnOID,
uint16_t dbRoot,
uint32_t partition,
uint16_t segment,
HWM lastLocalHwm,
execplan::CalpontSystemCatalog::ColDataType colType,
const std::string& colTypeName,
int colWidth,
int compressionType );
// Save column meta data to the currently open file.
// This is the Shared-Nothing version of this function.
// Warning: This function may throw a WeException.
// metaFileName name of metafile to be written
// columnOID column OID to be saved
// dbRoot current dbRoot of last local HWM for columnOID
// partition current partition of last local HWM for columnOID
// segment current segment of last local HWM for columnOID
// lastLocalHwm current last local for column OID
// colType type of columnOID
// colTypeName type name of columnOID
// colWidth width (in bytes) of columnOID
// compressionType compression type
void writeColumnMetaData(const std::string& metaFileName, bool withHWM, OID columnOID, uint16_t dbRoot,
uint32_t partition, uint16_t segment, HWM lastLocalHwm,
execplan::CalpontSystemCatalog::ColDataType colType,
const std::string& colTypeName, int colWidth, int compressionType);
// Save dictionary store meta data to the currently open file.
// This is the Shared-Nothing version of this function.
// dictionaryStoreOID dictionary store OID to be saved
// dbRoot dbRoot of store file
// partition partition of store file
// segment segment of store file
// localHwm current local HWM for specified partition and seg file
// compressionType compression type
void writeDictionaryStoreMetaData (
OID columnOID,
OID dictionaryStoreOID,
uint16_t dbRoot,
uint32_t partition,
uint16_t segment,
HWM localHwm,
int compressionType );
// Save dictionary store meta data to the currently open file.
// This is the Shared-Nothing version of this function.
// dictionaryStoreOID dictionary store OID to be saved
// dbRoot dbRoot of store file
// partition partition of store file
// segment segment of store file
// localHwm current local HWM for specified partition and seg file
// compressionType compression type
void writeDictionaryStoreMetaData(OID columnOID, OID dictionaryStoreOID, uint16_t dbRoot,
uint32_t partition, uint16_t segment, HWM localHwm, int compressionType);
// For first extent stripe in a partition, this function is used to
// to log a marker to denote a trailing segment file that does not exist.
// This is the Shared-Nothing version of this function.
// dictionaryStoreOID dictionary store OID to be saved
// dbRoot dbRoot of store file
// partition partition of store file
// segment segment of store file
// compressionType compression type
void writeDictionaryStoreMetaNoDataMarker (
OID columnOID,
OID dictionaryStoreOID,
uint16_t dbRoot,
uint32_t partition,
uint16_t segment,
int compressionType );
// For first extent stripe in a partition, this function is used to
// to log a marker to denote a trailing segment file that does not exist.
// This is the Shared-Nothing version of this function.
// dictionaryStoreOID dictionary store OID to be saved
// dbRoot dbRoot of store file
// partition partition of store file
// segment segment of store file
// compressionType compression type
void writeDictionaryStoreMetaNoDataMarker(OID columnOID, OID dictionaryStoreOID, uint16_t dbRoot,
uint32_t partition, uint16_t segment, int compressionType);
// This function must be thread-safe since it is called indirectly by
// backupDctnryHWMChunk() (through backupHWMChunk()).
int writeHWMChunk (
bool bColumnFile,
OID columnOID,
uint16_t dbRoot,
uint32_t partition,
uint16_t segment,
const unsigned char* compressedOutBuf,
uint64_t chunkSize,
uint64_t fileSize,
HWM chunkHwm,
std::string& errMsg) const;
void printDctnryChunkList(const RBChunkInfo& rbChk, const char* action);
// This function must be thread-safe since it is called indirectly by
// backupDctnryHWMChunk() (through backupHWMChunk()).
int writeHWMChunk(bool bColumnFile, OID columnOID, uint16_t dbRoot, uint32_t partition, uint16_t segment,
const unsigned char* compressedOutBuf, uint64_t chunkSize, uint64_t fileSize,
HWM chunkHwm, std::string& errMsg) const;
void printDctnryChunkList(const RBChunkInfo& rbChk, const char* action);
IDBDataFile* fMetaDataFile; // current meta data file to write
std::ostringstream fMetaDataStream; // adapter for IDBDataFile
std::map<uint16_t, std::string> fMetaFileNames;//map of dbroots to metafiles
std::string fAppDesc; // description of application user
Log* fLog; // import log file
bool fCreatedSubDir; // has subdir path been created
RBChunkSet fRBChunkDctnrySet; // Dctnry HWM chunk info
boost::mutex fRBChunkDctnryMutex;//Mutex lock for RBChunkSet
OID fTableOID; // OID of relevant table
std::string fTableName; // Name of relevant table
IDBDataFile* fMetaDataFile; // current meta data file to write
std::ostringstream fMetaDataStream; // adapter for IDBDataFile
std::map<uint16_t, std::string> fMetaFileNames; // map of dbroots to metafiles
std::string fAppDesc; // description of application user
Log* fLog; // import log file
bool fCreatedSubDir; // has subdir path been created
RBChunkSet fRBChunkDctnrySet; // Dctnry HWM chunk info
boost::mutex fRBChunkDctnryMutex; // Mutex lock for RBChunkSet
OID fTableOID; // OID of relevant table
std::string fTableName; // Name of relevant table
};
} //end of namespace
} // namespace WriteEngine
#undef EXPORT

96
writeengine/shared/we_simplesyslog.cpp
View File

@ -16,9 +16,9 @@
MA 02110-1301, USA. */
/*******************************************************************************
* $Id: we_simplesyslog.cpp 4607 2013-04-11 21:38:09Z rdempsey $
*
*******************************************************************************/
* $Id: we_simplesyslog.cpp 4607 2013-04-11 21:38:09Z rdempsey $
*
*******************************************************************************/
#include "we_simplesyslog.h"
@ -37,75 +37,73 @@ namespace WriteEngine
/* static */
SimpleSysLog* SimpleSysLog::instance()
{
if ( !fSysLogger )
fSysLogger = new SimpleSysLog();
if (!fSysLogger)
fSysLogger = new SimpleSysLog();
return fSysLogger;
return fSysLogger;
}
//------------------------------------------------------------------------------
// SimpleSysLog constructor.
//------------------------------------------------------------------------------
SimpleSysLog::SimpleSysLog() : fLoggingID( SUBSYSTEM_ID_WE )
SimpleSysLog::SimpleSysLog() : fLoggingID(SUBSYSTEM_ID_WE)
{
}
//------------------------------------------------------------------------------
// Reset LoggingID (in order to set the subsystem id)
//------------------------------------------------------------------------------
void SimpleSysLog::setLoggingID( const logging::LoggingID& loggingID )
void SimpleSysLog::setLoggingID(const logging::LoggingID& loggingID)
{
fLoggingID = loggingID;
fLoggingID = loggingID;
}
//------------------------------------------------------------------------------
// Log arguments (msgArgs) for specified msgId to the requested log (logType).
//------------------------------------------------------------------------------
void SimpleSysLog::logMsg( const logging::Message::Args& msgArgs,
logging::LOG_TYPE logType,
logging::Message::MessageID msgId )
void SimpleSysLog::logMsg(const logging::Message::Args& msgArgs, logging::LOG_TYPE logType,
logging::Message::MessageID msgId)
{
logging::MessageLog ml( fLoggingID );
logging::MessageLog ml(fLoggingID);
logging::Message m(msgId);
m.format(msgArgs);
logging::Message m(msgId);
m.format(msgArgs);
boost::mutex::scoped_lock lk(fWriteLockMutex);
boost::mutex::scoped_lock lk(fWriteLockMutex);
switch (logType)
switch (logType)
{
case logging::LOG_TYPE_DEBUG:
{
case logging::LOG_TYPE_DEBUG:
{
ml.logDebugMessage(m);
break;
}
case logging::LOG_TYPE_INFO:
default:
{
ml.logInfoMessage(m);
break;
}
case logging::LOG_TYPE_WARNING:
{
ml.logWarningMessage(m);
break;
}
case logging::LOG_TYPE_ERROR:
{
ml.logErrorMessage(m);
break;
}
case logging::LOG_TYPE_CRITICAL:
{
ml.logCriticalMessage(m);
break;
}
ml.logDebugMessage(m);
break;
}
case logging::LOG_TYPE_INFO:
default:
{
ml.logInfoMessage(m);
break;
}
case logging::LOG_TYPE_WARNING:
{
ml.logWarningMessage(m);
break;
}
case logging::LOG_TYPE_ERROR:
{
ml.logErrorMessage(m);
break;
}
case logging::LOG_TYPE_CRITICAL:
{
ml.logCriticalMessage(m);
break;
}
}
}
} //end of namespace
} // namespace WriteEngine

54
writeengine/shared/we_simplesyslog.h
View File

@ -16,9 +16,9 @@
MA 02110-1301, USA. */
/*******************************************************************************
* $Id: we_simplesyslog.h 4450 2013-01-21 14:13:24Z rdempsey $
*
*******************************************************************************/
* $Id: we_simplesyslog.h 4450 2013-01-21 14:13:24Z rdempsey $
*
*******************************************************************************/
/** @file */
#pragma once
@ -37,7 +37,6 @@
/** Namespace WriteEngine */
namespace WriteEngine
{
/**
* @brief SimpleSysLog class is a simple logger that only logs to syslog.
*
@ -46,35 +45,34 @@ namespace WriteEngine
*/
class SimpleSysLog
{
public:
/**
* @brief Singleton accessor.
*/
EXPORT static SimpleSysLog* instance();
public:
/**
* @brief Singleton accessor.
*/
EXPORT static SimpleSysLog* instance();
/**
* @brief Modify the LoggingID to be used. Mainly used to control the
* subsystem ID.
*/
EXPORT void setLoggingID( const logging::LoggingID& loggingID );
/**
* @brief Modify the LoggingID to be used. Mainly used to control the
* subsystem ID.
*/
EXPORT void setLoggingID(const logging::LoggingID& loggingID);
/**
* @brief Function that logs a syslog msg.
*/
EXPORT void logMsg( const logging::Message::Args& msgArgs,
logging::LOG_TYPE logType,
logging::Message::MessageID msgId );
/**
* @brief Function that logs a syslog msg.
*/
EXPORT void logMsg(const logging::Message::Args& msgArgs, logging::LOG_TYPE logType,
logging::Message::MessageID msgId);
private:
SimpleSysLog( );
SimpleSysLog( const SimpleSysLog& );
SimpleSysLog& operator= ( const SimpleSysLog& );
private:
SimpleSysLog();
SimpleSysLog(const SimpleSysLog&);
SimpleSysLog& operator=(const SimpleSysLog&);
static SimpleSysLog* fSysLogger;
logging::LoggingID fLoggingID;
boost::mutex fWriteLockMutex; // logging mutex
static SimpleSysLog* fSysLogger;
logging::LoggingID fLoggingID;
boost::mutex fWriteLockMutex; // logging mutex
};
#undef EXPORT
} //end of namespace
} // namespace WriteEngine

181
writeengine/shared/we_stats.cpp
View File

@ -16,9 +16,9 @@
MA 02110-1301, USA. */
/*******************************************************************************
* $Id: we_stats.cpp 4450 2013-01-21 14:13:24Z rdempsey $
*
*******************************************************************************/
* $Id: we_stats.cpp 4450 2013-01-21 14:13:24Z rdempsey $
*
*******************************************************************************/
/** @file */
#include <we_stats.h>
@ -28,16 +28,16 @@ using namespace std;
namespace WriteEngine
{
#ifdef PROFILE
/* static */ bool Stats::fProfiling = false;
/* static */ boost::mutex Stats::fRegisterReaderMutex;
/* static */ boost::mutex Stats::fRegisterParseMutex;
/* static */ std::vector<pthread_t> Stats::fReadProfThreads;
/* static */ std::vector<pthread_t> Stats::fParseProfThreads;
/* static */ bool Stats::fProfiling = false;
/* static */ boost::mutex Stats::fRegisterReaderMutex;
/* static */ boost::mutex Stats::fRegisterParseMutex;
/* static */ std::vector<pthread_t> Stats::fReadProfThreads;
/* static */ std::vector<pthread_t> Stats::fParseProfThreads;
/* static */ std::vector<logging::StopWatch> Stats::fReadStopWatch;
/* static */ std::vector<logging::StopWatch> Stats::fParseStopWatch;
#endif
struct IoStats Stats::m_ioStats = { 0, 0 };
struct IoStats Stats::m_ioStats = {0, 0};
bool Stats::m_bUseStats = false;
/***********************************************************
* DESCRIPTION:
@ -47,12 +47,12 @@ bool Stats::m_bUseStats = false;
* RETURN:
* none
***********************************************************/
void Stats::incIoBlockRead( const int blockNum )
void Stats::incIoBlockRead(const int blockNum)
{
if ( !m_bUseStats )
return;
if (!m_bUseStats)
return;
m_ioStats.blockRead += blockNum;
m_ioStats.blockRead += blockNum;
}
/***********************************************************
@ -63,12 +63,12 @@ void Stats::incIoBlockRead( const int blockNum )
* RETURN:
* none
***********************************************************/
void Stats::incIoBlockWrite( const int blockNum )
void Stats::incIoBlockWrite(const int blockNum)
{
if ( !m_bUseStats )
return;
if (!m_bUseStats)
return;
m_ioStats.blockWrite += blockNum;
m_ioStats.blockWrite += blockNum;
}
#ifdef PROFILE
@ -87,18 +87,18 @@ void Stats::incIoBlockWrite( const int blockNum )
***********************************************************/
void Stats::enableProfiling(int nReadThreads, int nParseThreads)
{
fProfiling = true;
fProfiling = true;
// @bug 2625: pre-reserve space for our vectors; else we could have a race
// condition whereby one parsing thread is adding itself to the vectors
// and thus "growing" the vector (in registerParseProfThread), at the
// same time that another parsing thread is reading the vector in parse-
// Event(). By pre-reserving the space, the vectors won't be growing,
// thus eliminating the problem with this race condition.
fReadProfThreads.reserve ( nReadThreads );
fReadStopWatch.reserve ( nReadThreads );
fParseProfThreads.reserve( nParseThreads );
fParseStopWatch.reserve ( nParseThreads );
// @bug 2625: pre-reserve space for our vectors; else we could have a race
// condition whereby one parsing thread is adding itself to the vectors
// and thus "growing" the vector (in registerParseProfThread), at the
// same time that another parsing thread is reading the vector in parse-
// Event(). By pre-reserving the space, the vectors won't be growing,
// thus eliminating the problem with this race condition.
fReadProfThreads.reserve(nReadThreads);
fReadStopWatch.reserve(nReadThreads);
fParseProfThreads.reserve(nParseThreads);
fParseStopWatch.reserve(nParseThreads);
}
/***********************************************************
@ -109,13 +109,13 @@ void Stats::enableProfiling(int nReadThreads, int nParseThreads)
* RETURN:
* none
***********************************************************/
void Stats::registerReadProfThread( )
void Stats::registerReadProfThread()
{
boost::mutex::scoped_lock lk(fRegisterReaderMutex);
boost::mutex::scoped_lock lk(fRegisterReaderMutex);
fReadProfThreads.push_back( pthread_self() );
logging::StopWatch readStopWatch;
fReadStopWatch.push_back ( readStopWatch );
fReadProfThreads.push_back(pthread_self());
logging::StopWatch readStopWatch;
fReadStopWatch.push_back(readStopWatch);
}
/***********************************************************
@ -126,13 +126,13 @@ void Stats::registerReadProfThread( )
* RETURN:
* none
***********************************************************/
void Stats::registerParseProfThread( )
void Stats::registerParseProfThread()
{
boost::mutex::scoped_lock lk(fRegisterParseMutex);
boost::mutex::scoped_lock lk(fRegisterParseMutex);
fParseProfThreads.push_back( pthread_self() );
logging::StopWatch parseStopWatch;
fParseStopWatch.push_back ( parseStopWatch );
fParseProfThreads.push_back(pthread_self());
logging::StopWatch parseStopWatch;
fParseStopWatch.push_back(parseStopWatch);
}
/***********************************************************
@ -145,25 +145,25 @@ void Stats::registerParseProfThread( )
* RETURN:
* none
***********************************************************/
void Stats::readEvent ( const std::string& eventString, bool start )
void Stats::readEvent(const std::string& eventString, bool start)
{
if (fProfiling)
if (fProfiling)
{
pthread_t thread = pthread_self();
for (unsigned i = 0; i < fReadProfThreads.size(); i++)
{
pthread_t thread = pthread_self();
if (fReadProfThreads[i] == thread)
{
if (start)
fReadStopWatch[i].start(eventString);
else
fReadStopWatch[i].stop(eventString);
for (unsigned i = 0; i < fReadProfThreads.size(); i++)
{
if (fReadProfThreads[i] == thread)
{
if (start)
fReadStopWatch[i].start( eventString );
else
fReadStopWatch[i].stop ( eventString );
break;
}
}
break;
}
}
}
}
/***********************************************************
@ -176,25 +176,25 @@ void Stats::readEvent ( const std::string& eventString, bool start )
* RETURN:
* none
***********************************************************/
void Stats::parseEvent ( const std::string& eventString, bool start )
void Stats::parseEvent(const std::string& eventString, bool start)
{
if (fProfiling)
if (fProfiling)
{
pthread_t thread = pthread_self();
for (unsigned i = 0; i < fParseProfThreads.size(); i++)
{
pthread_t thread = pthread_self();
if (fParseProfThreads[i] == thread)
{
if (start)
fParseStopWatch[i].start(eventString);
else
fParseStopWatch[i].stop(eventString);
for (unsigned i = 0; i < fParseProfThreads.size(); i++)
{
if (fParseProfThreads[i] == thread)
{
if (start)
fParseStopWatch[i].start( eventString );
else
fParseStopWatch[i].stop ( eventString );
break;
}
}
break;
}
}
}
}
/***********************************************************
@ -205,33 +205,30 @@ void Stats::parseEvent ( const std::string& eventString, bool start )
* RETURN:
* none
***********************************************************/
void Stats::printProfilingResults ( )
void Stats::printProfilingResults()
{
if (fProfiling)
if (fProfiling)
{
std::cout << endl;
for (unsigned j = 0; j < fReadStopWatch.size(); j++)
{
std::cout << endl;
for (unsigned j = 0; j < fReadStopWatch.size(); j++)
{
std::cout << "Execution Stats for Read Thread " << j << " (" <<
fReadProfThreads[j] << ")" << std::endl <<
"-------------------------------" << std::endl;
fReadStopWatch[j].finish();
std::cout << std::endl;
}
for (unsigned j = 0; j < fParseStopWatch.size(); j++)
{
std::cout << "Execution Stats for Parse Thread " << j << " (" <<
fParseProfThreads[j] << ")" << std::endl <<
"--------------------------------" << std::endl;
fParseStopWatch[j].finish();
std::cout << std::endl;
}
std::cout << "Execution Stats for Read Thread " << j << " (" << fReadProfThreads[j] << ")" << std::endl
<< "-------------------------------" << std::endl;
fReadStopWatch[j].finish();
std::cout << std::endl;
}
for (unsigned j = 0; j < fParseStopWatch.size(); j++)
{
std::cout << "Execution Stats for Parse Thread " << j << " (" << fParseProfThreads[j] << ")"
<< std::endl
<< "--------------------------------" << std::endl;
fParseStopWatch[j].finish();
std::cout << std::endl;
}
}
}
#endif
} //end of namespace
} // namespace WriteEngine

245
writeengine/shared/we_stats.h
View File

@ -16,9 +16,9 @@
MA 02110-1301, USA. */
/*******************************************************************************
* $Id: we_stats.h 4450 2013-01-21 14:13:24Z rdempsey $
*
*******************************************************************************/
* $Id: we_stats.h 4450 2013-01-21 14:13:24Z rdempsey $
*
*******************************************************************************/
/** @file */
#pragma once
@ -33,149 +33,152 @@ namespace WriteEngine
{
struct IoStats
{
long blockRead;
long blockWrite;
long blockRead;
long blockWrite;
};
/** Class Stats */
class Stats
{
public:
/**
* @brief Constructor
*/
Stats() {}
public:
/**
* @brief Constructor
*/
Stats()
{
}
/**
* @brief Default Destructor
*/
~Stats() {}
/**
* @brief Default Destructor
*/
~Stats()
{
}
/**
* @brief I/O
*/
static long getIoBlockRead()
{
return m_ioStats.blockRead;
}
static long getIoBlockWrite()
{
return m_ioStats.blockWrite;
}
/**
* @brief I/O
*/
static long getIoBlockRead()
{
return m_ioStats.blockRead;
}
static long getIoBlockWrite()
{
return m_ioStats.blockWrite;
}
static void incIoBlockRead( const int blockNum = 1 );
static void incIoBlockWrite( const int blockNum = 1 );
static void incIoBlockRead(const int blockNum = 1);
static void incIoBlockWrite(const int blockNum = 1);
static bool getUseStats()
{
return m_bUseStats;
}
static void setUseStats( const bool flag )
{
m_bUseStats = flag;
}
static bool getUseStats()
{
return m_bUseStats;
}
static void setUseStats(const bool flag)
{
m_bUseStats = flag;
}
static IoStats m_ioStats; // IO
static IoStats m_ioStats; // IO
#ifdef PROFILE
// Prefined event labels
#define WE_STATS_ALLOC_DCT_EXTENT "AllocDctExtent"
#define WE_STATS_COMPACT_VARBINARY "CompactingVarBinary"
#define WE_STATS_COMPLETING_PARSE "CompletingParse"
#define WE_STATS_COMPLETING_READ "CompletingRead"
// Prefined event labels
#define WE_STATS_ALLOC_DCT_EXTENT "AllocDctExtent"
#define WE_STATS_COMPACT_VARBINARY "CompactingVarBinary"
#define WE_STATS_COMPLETING_PARSE "CompletingParse"
#define WE_STATS_COMPLETING_READ "CompletingRead"
#define WE_STATS_COMPRESS_COL_INIT_ABBREV_EXT "CmpColInitAbbrevExtent"
#define WE_STATS_COMPRESS_COL_INIT_BUF "CmpColInitBuf"
#define WE_STATS_COMPRESS_COL_COMPRESS "CmpColCompress"
#define WE_STATS_COMPRESS_COL_FINISH_EXTENT "CmpColFinishExtent"
#define WE_STATS_COMPRESS_DCT_INIT_BUF "CmpDctInitBuf"
#define WE_STATS_COMPRESS_DCT_COMPRESS "CmpDctCompress"
#define WE_STATS_COMPRESS_DCT_SEEKO_CHUNK "CmpDctSeekOutputChunk"
#define WE_STATS_COMPRESS_DCT_WRITE_CHUNK "CmpDctWriteChunk"
#define WE_STATS_COMPRESS_DCT_SEEKO_HDR "CmpDctSeekOutputHdr"
#define WE_STATS_COMPRESS_DCT_WRITE_HDR "CmpDctWriteHdr"
#define WE_STATS_COMPRESS_DCT_BACKUP_CHUNK "CmpDctBackupChunk"
#define WE_STATS_CREATE_COL_EXTENT "CreateColExtent"
#define WE_STATS_CREATE_DCT_EXTENT "CreateDctExtent"
#define WE_STATS_EXPAND_COL_EXTENT "ExpandColExtent"
#define WE_STATS_EXPAND_DCT_EXTENT "ExpandDctExtent"
#define WE_STATS_FLUSH_PRIMPROC_BLOCKS "FlushPrimProcBlocks"
#define WE_STATS_INIT_COL_EXTENT "InitColExtent"
#define WE_STATS_INIT_DCT_EXTENT "InitDctExtent"
#define WE_STATS_OPEN_DCT_FILE "OpenDctFile"
#define WE_STATS_PARSE_COL "ParseCol"
#define WE_STATS_PARSE_DCT "ParseDct"
#define WE_STATS_PARSE_DCT_SEEK_EXTENT_BLK "ParseDctSeekExtentBlk"
#define WE_STATS_READ_INTO_BUF "ReadIntoBuf"
#define WE_STATS_RESIZE_OUT_BUF "ResizeOutBuf"
#define WE_STATS_WAIT_FOR_INTERMEDIATE_FLUSH "WaitForIntermediateFlush"
#define WE_STATS_WAIT_FOR_READ_BUF "WaitForReadBuf"
#define WE_STATS_WAIT_TO_COMPLETE_PARSE "WaitCompleteParse"
#define WE_STATS_WAIT_TO_COMPLETE_READ "WaitCompleteRead"
#define WE_STATS_WAIT_TO_CREATE_COL_EXTENT "WaitCreateColExtent"
#define WE_STATS_WAIT_TO_CREATE_DCT_EXTENT "WaitCreateDctExtent"
#define WE_STATS_WAIT_TO_EXPAND_COL_EXTENT "WaitExpandColExtent"
#define WE_STATS_WAIT_TO_EXPAND_DCT_EXTENT "WaitExpandDctExtent"
#define WE_STATS_WAIT_TO_PARSE_DCT "WaitParseDct"
#define WE_STATS_WAIT_TO_RELEASE_OUT_BUF "WaitReleaseOutBuf"
#define WE_STATS_WAIT_TO_RESERVE_OUT_BUF "WaitReserveOutBuf"
#define WE_STATS_WAIT_TO_RESIZE_OUT_BUF "WaitResizeOutBuf"
#define WE_STATS_WAIT_TO_SELECT_COL "WaitSelectCol"
#define WE_STATS_WAIT_TO_SELECT_TBL "WaitSelectTbl"
#define WE_STATS_WRITE_COL "WriteCol"
#define WE_STATS_WRITE_DCT "WriteDct"
#define WE_STATS_COMPRESS_COL_INIT_BUF "CmpColInitBuf"
#define WE_STATS_COMPRESS_COL_COMPRESS "CmpColCompress"
#define WE_STATS_COMPRESS_COL_FINISH_EXTENT "CmpColFinishExtent"
#define WE_STATS_COMPRESS_DCT_INIT_BUF "CmpDctInitBuf"
#define WE_STATS_COMPRESS_DCT_COMPRESS "CmpDctCompress"
#define WE_STATS_COMPRESS_DCT_SEEKO_CHUNK "CmpDctSeekOutputChunk"
#define WE_STATS_COMPRESS_DCT_WRITE_CHUNK "CmpDctWriteChunk"
#define WE_STATS_COMPRESS_DCT_SEEKO_HDR "CmpDctSeekOutputHdr"
#define WE_STATS_COMPRESS_DCT_WRITE_HDR "CmpDctWriteHdr"
#define WE_STATS_COMPRESS_DCT_BACKUP_CHUNK "CmpDctBackupChunk"
#define WE_STATS_CREATE_COL_EXTENT "CreateColExtent"
#define WE_STATS_CREATE_DCT_EXTENT "CreateDctExtent"
#define WE_STATS_EXPAND_COL_EXTENT "ExpandColExtent"
#define WE_STATS_EXPAND_DCT_EXTENT "ExpandDctExtent"
#define WE_STATS_FLUSH_PRIMPROC_BLOCKS "FlushPrimProcBlocks"
#define WE_STATS_INIT_COL_EXTENT "InitColExtent"
#define WE_STATS_INIT_DCT_EXTENT "InitDctExtent"
#define WE_STATS_OPEN_DCT_FILE "OpenDctFile"
#define WE_STATS_PARSE_COL "ParseCol"
#define WE_STATS_PARSE_DCT "ParseDct"
#define WE_STATS_PARSE_DCT_SEEK_EXTENT_BLK "ParseDctSeekExtentBlk"
#define WE_STATS_READ_INTO_BUF "ReadIntoBuf"
#define WE_STATS_RESIZE_OUT_BUF "ResizeOutBuf"
#define WE_STATS_WAIT_FOR_INTERMEDIATE_FLUSH "WaitForIntermediateFlush"
#define WE_STATS_WAIT_FOR_READ_BUF "WaitForReadBuf"
#define WE_STATS_WAIT_TO_COMPLETE_PARSE "WaitCompleteParse"
#define WE_STATS_WAIT_TO_COMPLETE_READ "WaitCompleteRead"
#define WE_STATS_WAIT_TO_CREATE_COL_EXTENT "WaitCreateColExtent"
#define WE_STATS_WAIT_TO_CREATE_DCT_EXTENT "WaitCreateDctExtent"
#define WE_STATS_WAIT_TO_EXPAND_COL_EXTENT "WaitExpandColExtent"
#define WE_STATS_WAIT_TO_EXPAND_DCT_EXTENT "WaitExpandDctExtent"
#define WE_STATS_WAIT_TO_PARSE_DCT "WaitParseDct"
#define WE_STATS_WAIT_TO_RELEASE_OUT_BUF "WaitReleaseOutBuf"
#define WE_STATS_WAIT_TO_RESERVE_OUT_BUF "WaitReserveOutBuf"
#define WE_STATS_WAIT_TO_RESIZE_OUT_BUF "WaitResizeOutBuf"
#define WE_STATS_WAIT_TO_SELECT_COL "WaitSelectCol"
#define WE_STATS_WAIT_TO_SELECT_TBL "WaitSelectTbl"
#define WE_STATS_WRITE_COL "WriteCol"
#define WE_STATS_WRITE_DCT "WriteDct"
// Functions used to support performance profiling
static void enableProfiling(int nReadThreads, int nParseThreads);
static void registerReadProfThread ( );
static void registerParseProfThread( );
static void startReadEvent ( const std::string& eventString )
{
readEvent ( eventString, true );
}
static void stopReadEvent ( const std::string& eventString )
{
readEvent ( eventString, false );
}
static void startParseEvent( const std::string& eventString )
{
parseEvent( eventString, true );
}
static void stopParseEvent ( const std::string& eventString )
{
parseEvent( eventString, false );
}
static void printProfilingResults( );
// Functions used to support performance profiling
static void enableProfiling(int nReadThreads, int nParseThreads);
static void registerReadProfThread();
static void registerParseProfThread();
static void startReadEvent(const std::string& eventString)
{
readEvent(eventString, true);
}
static void stopReadEvent(const std::string& eventString)
{
readEvent(eventString, false);
}
static void startParseEvent(const std::string& eventString)
{
parseEvent(eventString, true);
}
static void stopParseEvent(const std::string& eventString)
{
parseEvent(eventString, false);
}
static void printProfilingResults();
#endif
private:
static bool m_bUseStats; // Use statistic flag
private:
static bool m_bUseStats; // Use statistic flag
#ifdef PROFILE
// Data members and functions used to support performance profiling
static bool fProfiling; // Is profiling enabled
// Data members and functions used to support performance profiling
static bool fProfiling; // Is profiling enabled
// Protect concurrent addition of Readers
static boost::mutex fRegisterReaderMutex;
// Protect concurrent addition of Readers
static boost::mutex fRegisterReaderMutex;
// Protect concurrent addition of Parsers
static boost::mutex fRegisterParseMutex;
// Protect concurrent addition of Parsers
static boost::mutex fRegisterParseMutex;
// Read threads to be profiled
static std::vector<pthread_t> fReadProfThreads;
// Read threads to be profiled
static std::vector<pthread_t> fReadProfThreads;
// Parse threads to be profiled
static std::vector<pthread_t> fParseProfThreads;
// Parse threads to be profiled
static std::vector<pthread_t> fParseProfThreads;
// Track/profile Read events
static std::vector<logging::StopWatch> fReadStopWatch;
// Track/profile Read events
static std::vector<logging::StopWatch> fReadStopWatch;
// Track/profile Parse events
static std::vector<logging::StopWatch> fParseStopWatch;
// Track/profile Parse events
static std::vector<logging::StopWatch> fParseStopWatch;
static void readEvent ( const std::string& eventString, bool start );
static void parseEvent ( const std::string& eventString, bool start );
static void readEvent(const std::string& eventString, bool start);
static void parseEvent(const std::string& eventString, bool start);
#endif
};
} //end of namespace
} // namespace WriteEngine

753
writeengine/shared/we_type.h
View File

@ -20,7 +20,6 @@
/** @file */
#pragma once
#undef EXPORT
@ -53,156 +52,169 @@ typedef idbdatafile::IDBDataFile IDBDataFile;
/************************************************************************
* Type definitions
************************************************************************/
typedef uint32_t OID; /** @brief Object ID */
typedef uint32_t FID; /** @brief File ID */
typedef uint64_t RID; /** @brief Row ID */
typedef uint32_t TxnID; /** @brief Transaction ID (New)*/
typedef uint32_t HWM; /** @brief high water mark */
typedef uint32_t OID; /** @brief Object ID */
typedef uint32_t FID; /** @brief File ID */
typedef uint64_t RID; /** @brief Row ID */
typedef uint32_t TxnID; /** @brief Transaction ID (New)*/
typedef uint32_t HWM; /** @brief high water mark */
/************************************************************************
* Type enumerations
************************************************************************/
enum DebugLevel /** @brief Debug level type */
enum DebugLevel /** @brief Debug level type */
{
DEBUG_0 = 0, /** @brief No debug info */
DEBUG_1 = 1, /** @brief Summary level debug */
DEBUG_2 = 2, /** @brief Moderate debug */
DEBUG_3 = 3, /** @brief Detail debug */
DEBUG_0 = 0, /** @brief No debug info */
DEBUG_1 = 1, /** @brief Summary level debug */
DEBUG_2 = 2, /** @brief Moderate debug */
DEBUG_3 = 3, /** @brief Detail debug */
};
// INFO2 only goes to log file unless '-i' cmd line arg is specified,
// in which case the msg will also get logged to the console.
// All other messages always get logged to the log file and the console.
enum MsgLevel /** @brief Message level */
enum MsgLevel /** @brief Message level */
{
MSGLVL_INFO1 = 0, /** @brief Basic Information level*/
MSGLVL_INFO2 = 1, /** @brief More Information level */
MSGLVL_WARNING = 2, /** @brief Warning level */
MSGLVL_ERROR = 3, /** @brief Error level */
MSGLVL_CRITICAL = 4, /** @brief Critical level */
MSGLVL_INFO1 = 0, /** @brief Basic Information level*/
MSGLVL_INFO2 = 1, /** @brief More Information level */
MSGLVL_WARNING = 2, /** @brief Warning level */
MSGLVL_ERROR = 3, /** @brief Error level */
MSGLVL_CRITICAL = 4, /** @brief Critical level */
};
enum OpType /** @brief Operation type */
enum OpType /** @brief Operation type */
{
NOOP = 0, /** @brief No oper */
INSERT = 1, /** @brief Insert */
UPDATE = 2, /** @brief Update */
DELETE = 4, /** @brief Delete */
QUERY = 8, /** @brief Query */
NOOP = 0, /** @brief No oper */
INSERT = 1, /** @brief Insert */
UPDATE = 2, /** @brief Update */
DELETE = 4, /** @brief Delete */
QUERY = 8, /** @brief Query */
};
enum ColType /** @brief Column type enumeration*/
enum ColType /** @brief Column type enumeration*/
{
// WR_BIT = 1, /** @brief Bit */
WR_BYTE = 2, /** @brief Byte */
WR_SHORT = 3, /** @brief Short */
WR_INT = 4, /** @brief Int */
// WR_LONG = 5, /** @brief Long */
WR_LONGLONG = 6, /** @brief Long long*/
WR_FLOAT = 7, /** @brief Float */
WR_DOUBLE = 8, /** @brief Double */
WR_CHAR = 9, /** @brief Char */
WR_TOKEN = 10, /** @brief Token */
WR_BLOB = 11, /** @brief BLOB */
WR_VARBINARY = 12, /** @brief VARBINARY */
WR_UBYTE = 13, /** @brief Unsigned Byte */
WR_USHORT = 14, /** @brief Unsigned Short */
WR_UINT = 15, /** @brief Unsigned Int */
WR_ULONGLONG = 16, /** @brief Unsigned Long long*/
WR_TEXT = 17, /** @brief TEXT */
WR_MEDINT = 18, /** @brief Medium Int */
WR_UMEDINT = 19, /** @brief Unsigned Medium Int */
WR_BINARY = 20 /** @brief BINARY */
// WIP
//WR_INT128
// WR_BIT = 1, /** @brief Bit */
WR_BYTE = 2, /** @brief Byte */
WR_SHORT = 3, /** @brief Short */
WR_INT = 4, /** @brief Int */
// WR_LONG = 5, /** @brief Long */
WR_LONGLONG = 6, /** @brief Long long*/
WR_FLOAT = 7, /** @brief Float */
WR_DOUBLE = 8, /** @brief Double */
WR_CHAR = 9, /** @brief Char */
WR_TOKEN = 10, /** @brief Token */
WR_BLOB = 11, /** @brief BLOB */
WR_VARBINARY = 12, /** @brief VARBINARY */
WR_UBYTE = 13, /** @brief Unsigned Byte */
WR_USHORT = 14, /** @brief Unsigned Short */
WR_UINT = 15, /** @brief Unsigned Int */
WR_ULONGLONG = 16, /** @brief Unsigned Long long*/
WR_TEXT = 17, /** @brief TEXT */
WR_MEDINT = 18, /** @brief Medium Int */
WR_UMEDINT = 19, /** @brief Unsigned Medium Int */
WR_BINARY = 20 /** @brief BINARY */
// WIP
// WR_INT128
};
// Describes relation of field to column for a bulk load
enum BulkFldColRel { BULK_FLDCOL_COLUMN_FIELD, // map input field to db col
BULK_FLDCOL_COLUMN_DEFAULT,// import def val to db col
BULK_FLDCOL_IGNORE_FIELD
};// ignore fld in import file
enum BulkFldColRel
{
BULK_FLDCOL_COLUMN_FIELD, // map input field to db col
BULK_FLDCOL_COLUMN_DEFAULT, // import def val to db col
BULK_FLDCOL_IGNORE_FIELD
}; // ignore fld in import file
// Bulk Load Mode (ex: local vs remote, single src vs multiple src files)
enum BulkModeType { BULK_MODE_REMOTE_SINGLE_SRC = 1,
BULK_MODE_REMOTE_MULTIPLE_SRC = 2,
BULK_MODE_LOCAL = 3
};
enum BulkModeType
{
BULK_MODE_REMOTE_SINGLE_SRC = 1,
BULK_MODE_REMOTE_MULTIPLE_SRC = 2,
BULK_MODE_LOCAL = 3
};
// Import Mode 0-text Import (default)
// 1-Binary Import with NULL values
// 2-Binary Import with saturated NULL values
enum ImportDataMode { IMPORT_DATA_TEXT = 0,
IMPORT_DATA_BIN_ACCEPT_NULL = 1,
IMPORT_DATA_BIN_SAT_NULL = 2
};
enum ImportDataMode
{
IMPORT_DATA_TEXT = 0,
IMPORT_DATA_BIN_ACCEPT_NULL = 1,
IMPORT_DATA_BIN_SAT_NULL = 2
};
/**
* the set of Calpont column data type names; MUST match ColDataType in
* calpontsystemcatalog.h.
*/
const char ColDataTypeStr[execplan::CalpontSystemCatalog::NUM_OF_COL_DATA_TYPE][20] =
const char ColDataTypeStr[execplan::CalpontSystemCatalog::NUM_OF_COL_DATA_TYPE][20] = {"bit",
"tinyint",
"char",
"smallint",
"decimal",
"medint",
"integer",
"float",
"date",
"bigint",
"double",
"datetime",
"varchar",
"varbinary",
"clob",
"blob",
"unsigned-tinyint",
"unsigned-smallint",
"unsigned-decimal",
"unsigned-med int",
"unsigned-int",
"unsigned-float",
"unsigned-bigint",
"unsigned-double",
"text",
"time",
"timestamp"};
enum FuncType
{
"bit",
"tinyint",
"char",
"smallint",
"decimal",
"medint",
"integer",
"float",
"date",
"bigint",
"double",
"datetime",
"varchar",
"varbinary",
"clob",
"blob",
"unsigned-tinyint",
"unsigned-smallint",
"unsigned-decimal",
"unsigned-med int",
"unsigned-int",
"unsigned-float",
"unsigned-bigint",
"unsigned-double",
"text",
"time",
"timestamp"
FUNC_WRITE_ENGINE,
FUNC_INDEX,
FUNC_DICTIONARY
};
enum FuncType { FUNC_WRITE_ENGINE, FUNC_INDEX, FUNC_DICTIONARY };
enum CacheListType { FREE_LIST, LRU_LIST, WRITE_LIST }; /** @brief List type */
enum CacheListType
{
FREE_LIST,
LRU_LIST,
WRITE_LIST
}; /** @brief List type */
/************************************************************************
* struct data block structure
************************************************************************/
struct DataBlock /** @brief Data block structure */
struct DataBlock /** @brief Data block structure */
{
long no; /** @brief block number */
uint64_t lbid; /** @brief lbid */
bool dirty; /** @brief block dirty flag */
int state; /** @brief initialized 0, read 1 , modified 2 */
unsigned char data[BYTE_PER_BLOCK];/** @brief data buffer */
DataBlock()
{
dirty = false; /** @brief constructor */
memset( data, 0, BYTE_PER_BLOCK );
}
long no; /** @brief block number */
uint64_t lbid; /** @brief lbid */
bool dirty; /** @brief block dirty flag */
int state; /** @brief initialized 0, read 1 , modified 2 */
unsigned char data[BYTE_PER_BLOCK]; /** @brief data buffer */
DataBlock()
{
dirty = false; /** @brief constructor */
memset(data, 0, BYTE_PER_BLOCK);
}
};
struct DataSubBlock /** @brief Data subblock structure*/
struct DataSubBlock /** @brief Data subblock structure*/
{
long no; /** @brief sub block number */
bool dirty; /** @brief block dirty flag */
unsigned char data[BYTE_PER_SUBBLOCK]; /** @brief data buffer */
DataSubBlock()
{
dirty = false; /** @brief constructor */
memset( data, 0, BYTE_PER_SUBBLOCK );
}
long no; /** @brief sub block number */
bool dirty; /** @brief block dirty flag */
unsigned char data[BYTE_PER_SUBBLOCK]; /** @brief data buffer */
DataSubBlock()
{
dirty = false; /** @brief constructor */
memset(data, 0, BYTE_PER_SUBBLOCK);
}
};
/************************************************************************
@ -212,303 +224,358 @@ struct DataSubBlock /** @brief Data subblock structure*/
* oid and fid replicate one another. oid mostly used by index, cache,
* and dictionary. fid mostly used by colop and bulk.
************************************************************************/
struct File /** @brief File structure */
struct File /** @brief File structure */
{
OID oid; /** @brief Oid */
FID fid; /** @brief File id */
HWM hwm; /** @brief High water mark */
IDBDataFile* pFile; /** @brief File handle */
uint32_t fPartition; /** @brief Partition for pFile*/
uint16_t fSegment; /** @brief Segment for pFile */
uint16_t fDbRoot; /** @brief DbRoot for pFile */
std::string fSegFileName; /** @brief Current seg file path */
File()
{
clear(); /** @brief constructor */
}
void clear()
{
pFile = NULL;
oid = fid = hwm = 0;
fPartition = fSegment = fDbRoot = 0;
fSegFileName.clear();
}
OID oid; /** @brief Oid */
FID fid; /** @brief File id */
HWM hwm; /** @brief High water mark */
IDBDataFile* pFile; /** @brief File handle */
uint32_t fPartition; /** @brief Partition for pFile*/
uint16_t fSegment; /** @brief Segment for pFile */
uint16_t fDbRoot; /** @brief DbRoot for pFile */
std::string fSegFileName; /** @brief Current seg file path */
File()
{
clear(); /** @brief constructor */
}
void clear()
{
pFile = NULL;
oid = fid = hwm = 0;
fPartition = fSegment = fDbRoot = 0;
fSegFileName.clear();
}
};
/************************************************************************
* @brief Internal communication block structure
************************************************************************/
struct CommBlock /** @brief Communication Block */
struct CommBlock /** @brief Communication Block */
{
File file; /** @brief File structure */
void clear()
{
file.clear();
}
File file; /** @brief File structure */
void clear()
{
file.clear();
}
};
/************************************************************************
* @brief column structure used to pass data in/out of we_colop functions
************************************************************************/
struct Column /** @brief Column structure */
struct Column /** @brief Column structure */
{
int colNo; /** @brief column number */
int colWidth; /** @brief column width */
ColType colType; /** @brief column type (internal use)*/
execplan::CalpontSystemCatalog::ColDataType colDataType; /** @brief column data type (from interface)*/
File dataFile; /** @brief column data file */
int compressionType; /** @brief column compression type*/
Column() : colNo(0), colWidth(0), colType(WR_INT),
colDataType(execplan::CalpontSystemCatalog::INT),
compressionType(idbdatafile::IDBPolicy::useHdfs() ? 2 : 0) { }
int colNo; /** @brief column number */
int colWidth; /** @brief column width */
ColType colType; /** @brief column type (internal use)*/
execplan::CalpontSystemCatalog::ColDataType colDataType; /** @brief column data type (from interface)*/
File dataFile; /** @brief column data file */
int compressionType; /** @brief column compression type*/
Column()
: colNo(0)
, colWidth(0)
, colType(WR_INT)
, colDataType(execplan::CalpontSystemCatalog::INT)
, compressionType(idbdatafile::IDBPolicy::useHdfs() ? 2 : 0)
{
}
};
/************************************************************************
* @brief dictionary related structures (Token struct is defined in
* we_typeext.h to facilitate its use in dbcon and utils/dataconvert).
************************************************************************/
typedef struct offset_ /** @brief Offset structure */
typedef struct offset_ /** @brief Offset structure */
{
int hdrLoc; /** @brief offset postion in hdr */
uint16_t offset; /** @brief offset in block */
int hdrLoc; /** @brief offset postion in hdr */
uint16_t offset; /** @brief offset in block */
} Offset;
/************************************************************************
* @brief interfaces with DDL/DML
************************************************************************/
typedef struct colTuple_struct /** @brief Column Tuple definition*/
typedef struct colTuple_struct /** @brief Column Tuple definition*/
{
boost::any data; /** @brief column value */
boost::any data; /** @brief column value */
} ColTuple;
typedef std::vector<ColTuple> ColTupleList; /** @brief column value list */
typedef std::vector<ColTuple> ColTupleList; /** @brief column value list */
struct ColStruct /** @brief Column Interface Struct*/
struct ColStruct /** @brief Column Interface Struct*/
{
OID dataOid; /** @brief column data file object id */
int colWidth; /** @brief column width */
bool tokenFlag; /** @brief column token flag, must be set to true if it is a token column */
execplan::CalpontSystemCatalog::ColDataType colDataType; /** @brief column data type (for interface)*/
ColType colType; /** @brief column type (internal use for write engine)*/
uint32_t fColPartition; /** @brief Partition for column file */
uint16_t fColSegment; /** @brief Segment for column file*/
uint16_t fColDbRoot; /** @brief DBRoot for column file */
int fCompressionType; /** @brief Compression tpye for column file */
ColStruct() : dataOid(0), colWidth(0), /** @brief constructor */
tokenFlag(false), colDataType(execplan::CalpontSystemCatalog::INT), colType(WR_INT),
fColPartition(0), fColSegment(0), fColDbRoot(0),
fCompressionType(idbdatafile::IDBPolicy::useHdfs() ? 2 : 0) { }
OID dataOid; /** @brief column data file object id */
int colWidth; /** @brief column width */
bool tokenFlag; /** @brief column token flag, must be set to true if it is a token column */
execplan::CalpontSystemCatalog::ColDataType colDataType; /** @brief column data type (for interface)*/
ColType colType; /** @brief column type (internal use for write engine)*/
uint32_t fColPartition; /** @brief Partition for column file */
uint16_t fColSegment; /** @brief Segment for column file*/
uint16_t fColDbRoot; /** @brief DBRoot for column file */
int fCompressionType; /** @brief Compression tpye for column file */
ColStruct()
: dataOid(0)
, colWidth(0)
, /** @brief constructor */
tokenFlag(false)
, colDataType(execplan::CalpontSystemCatalog::INT)
, colType(WR_INT)
, fColPartition(0)
, fColSegment(0)
, fColDbRoot(0)
, fCompressionType(idbdatafile::IDBPolicy::useHdfs() ? 2 : 0)
{
}
};
typedef std::vector<ColStruct> ColStructList; /** @brief column struct list */
typedef std::vector<ColTupleList> ColValueList; /** @brief column value list */
typedef std::vector<RID> RIDList; /** @brief RID list */
typedef std::vector<execplan::CalpontSystemCatalog::ColType> CSCTypesList; /** @brief CSC column types list */
typedef std::vector<ColStruct> ColStructList; /** @brief column struct list */
typedef std::vector<ColTupleList> ColValueList; /** @brief column value list */
typedef std::vector<RID> RIDList; /** @brief RID list */
typedef std::vector<execplan::CalpontSystemCatalog::ColType> CSCTypesList; /** @brief CSC column types list */
typedef std::vector<std::string> dictStr;
typedef std::vector<dictStr> DictStrList;
// dictionary
struct DctnryStruct /** @brief Dctnry Interface Struct*/
struct DctnryStruct /** @brief Dctnry Interface Struct*/
{
OID dctnryOid; /** @brief dictionary signature file */
OID columnOid; /** @brief corresponding column file */
int colWidth; /** @brief string width for the dictionary column*/
uint32_t fColPartition; /** @brief Partition for column file */
uint16_t fColSegment; /** @brief Segment for column file */
uint16_t fColDbRoot; /** @brief DBRoot for column file */
int fCompressionType; /** @brief Compression tpye for column file */
DctnryStruct() : dctnryOid(0), columnOid(0), /** @brief constructor */
colWidth(0),
fColPartition(0), fColSegment(0),
fColDbRoot(0), fCompressionType(idbdatafile::IDBPolicy::useHdfs() ? 2 : 0) { }
OID dctnryOid; /** @brief dictionary signature file */
OID columnOid; /** @brief corresponding column file */
int colWidth; /** @brief string width for the dictionary column*/
uint32_t fColPartition; /** @brief Partition for column file */
uint16_t fColSegment; /** @brief Segment for column file */
uint16_t fColDbRoot; /** @brief DBRoot for column file */
int fCompressionType; /** @brief Compression tpye for column file */
DctnryStruct()
: dctnryOid(0)
, columnOid(0)
, /** @brief constructor */
colWidth(0)
, fColPartition(0)
, fColSegment(0)
, fColDbRoot(0)
, fCompressionType(idbdatafile::IDBPolicy::useHdfs() ? 2 : 0)
{
}
};
struct DctnryTuple /** @brief Dictionary Tuple struct*/
struct DctnryTuple /** @brief Dictionary Tuple struct*/
{
unsigned char* sigValue; /** @brief dictionary signature value*/
int sigSize; /** @brief dictionary signature size */
Token token; /** @brief dictionary token */
bool isNull;
DctnryTuple() { }
~DctnryTuple() { }
unsigned char* sigValue; /** @brief dictionary signature value*/
int sigSize; /** @brief dictionary signature size */
Token token; /** @brief dictionary token */
bool isNull;
DctnryTuple()
{
}
~DctnryTuple()
{
}
};
typedef std::vector<DctnryTuple> DctColTupleList;
typedef std::vector<DctnryStruct> DctnryStructList; /** @brief column struct list */
typedef std::vector<DctColTupleList> DctnryValueList; /** @brief column value list */
typedef std::vector<DctnryStruct> DctnryStructList; /** @brief column struct list */
typedef std::vector<DctColTupleList> DctnryValueList; /** @brief column value list */
/************************************************************************
* @brief Used by Bulk Load to describe a column
************************************************************************/
struct JobColumn /** @brief Job Column Structure */
struct JobColumn /** @brief Job Column Structure */
{
std::string colName; /** @brief column name */
OID mapOid; /** @brief column OID */
execplan::CalpontSystemCatalog::ColDataType dataType; /** @brief column data type */
ColType weType; /** @brief write engine data type */
std::string typeName; /** @brief data type name */
const uint8_t* emptyVal; /** @brief default empty value */
int width; /** @brief column width; for a dictionary column, this is "eventually" the token width */
int definedWidth; /** @brief column width as defined in the table, used for non-dictionary strings */
int dctnryWidth; /** @brief dictionary width */
int precision; /** @brief precision of decimal */
int scale; /** @brief scale of decimal */
bool fNotNull; /** @brief not null flag */
BulkFldColRel fFldColRelation; /** @brief type of field/col relation*/
char colType; /** @brief column type, blank is regular, D is dictionary */
int compressionType; /** @brief compression type */
bool autoIncFlag; /** @brief auto increment flag */
DctnryStruct dctnry; /** @brief dictionary structure */
int64_t fMinIntSat; /** @brief For integer type, the min saturation value */
uint64_t fMaxIntSat; /** @brief For integer type, the max saturation value */
double fMinDblSat; /** @brief for float/double, the min saturation value */
double fMaxDblSat; /** @brief for float/double, the max saturation value */
bool fWithDefault; /** @brief With default */
long long fDefaultInt; /** @brief Integer column default */
unsigned long long fDefaultUInt; /** @brief UnsignedInt col default*/
double fDefaultDbl; /** @brief Dbl/Flt column default */
int128_t fDefaultWideDecimal; /** @brief Wide decimal column default */
std::string fDefaultChr; /** @brief Char column default */
JobColumn() : mapOid(0), dataType(execplan::CalpontSystemCatalog::INT), weType(WR_INT),
typeName("integer"), emptyVal(nullptr),
width(0), definedWidth(0), dctnryWidth(0),
precision(0), scale(0), fNotNull(false),
fFldColRelation(BULK_FLDCOL_COLUMN_FIELD), colType(' '),
compressionType(0), autoIncFlag(false),
fMinIntSat(0), fMaxIntSat(0),
fMinDblSat(0), fMaxDblSat(0), fWithDefault(false),
fDefaultInt(0), fDefaultUInt(0), fDefaultDbl(0.0),
fDefaultWideDecimal(0)
{ }
std::string colName; /** @brief column name */
OID mapOid; /** @brief column OID */
execplan::CalpontSystemCatalog::ColDataType dataType; /** @brief column data type */
ColType weType; /** @brief write engine data type */
std::string typeName; /** @brief data type name */
const uint8_t* emptyVal; /** @brief default empty value */
int width; /** @brief column width; for a dictionary column, this is "eventually" the token width */
int definedWidth; /** @brief column width as defined in the table, used for non-dictionary strings */
int dctnryWidth; /** @brief dictionary width */
int precision; /** @brief precision of decimal */
int scale; /** @brief scale of decimal */
bool fNotNull; /** @brief not null flag */
BulkFldColRel fFldColRelation; /** @brief type of field/col relation*/
char colType; /** @brief column type, blank is regular, D is dictionary */
int compressionType; /** @brief compression type */
bool autoIncFlag; /** @brief auto increment flag */
DctnryStruct dctnry; /** @brief dictionary structure */
int64_t fMinIntSat; /** @brief For integer type, the min saturation value */
uint64_t fMaxIntSat; /** @brief For integer type, the max saturation value */
double fMinDblSat; /** @brief for float/double, the min saturation value */
double fMaxDblSat; /** @brief for float/double, the max saturation value */
bool fWithDefault; /** @brief With default */
long long fDefaultInt; /** @brief Integer column default */
unsigned long long fDefaultUInt; /** @brief UnsignedInt col default*/
double fDefaultDbl; /** @brief Dbl/Flt column default */
int128_t fDefaultWideDecimal; /** @brief Wide decimal column default */
std::string fDefaultChr; /** @brief Char column default */
JobColumn()
: mapOid(0)
, dataType(execplan::CalpontSystemCatalog::INT)
, weType(WR_INT)
, typeName("integer")
, emptyVal(nullptr)
, width(0)
, definedWidth(0)
, dctnryWidth(0)
, precision(0)
, scale(0)
, fNotNull(false)
, fFldColRelation(BULK_FLDCOL_COLUMN_FIELD)
, colType(' ')
, compressionType(0)
, autoIncFlag(false)
, fMinIntSat(0)
, fMaxIntSat(0)
, fMinDblSat(0)
, fMaxDblSat(0)
, fWithDefault(false)
, fDefaultInt(0)
, fDefaultUInt(0)
, fDefaultDbl(0.0)
, fDefaultWideDecimal(0)
{
}
};
typedef std::vector<JobColumn> JobColList; /** @brief column value list */
typedef std::vector<JobColumn> JobColList; /** @brief column value list */
struct JobFieldRef // references field/column in JobTable
struct JobFieldRef // references field/column in JobTable
{
BulkFldColRel fFldColType; // type of field or column
unsigned fArrayIndex; // index into colList or fIgnoredFields
// in JobTable based on fFldColType.
JobFieldRef( ) : fFldColType(BULK_FLDCOL_COLUMN_FIELD), fArrayIndex(0) { }
JobFieldRef( BulkFldColRel fldColType, unsigned idx ) :
fFldColType( fldColType ), fArrayIndex( idx ) { }
BulkFldColRel fFldColType; // type of field or column
unsigned fArrayIndex; // index into colList or fIgnoredFields
// in JobTable based on fFldColType.
JobFieldRef() : fFldColType(BULK_FLDCOL_COLUMN_FIELD), fArrayIndex(0)
{
}
JobFieldRef(BulkFldColRel fldColType, unsigned idx) : fFldColType(fldColType), fArrayIndex(idx)
{
}
};
typedef std::vector<JobFieldRef> JobFieldRefList;
typedef std::vector<JobFieldRef> JobFieldRefList;
struct JobTable /** @brief Job Table Structure */
struct JobTable /** @brief Job Table Structure */
{
std::string tblName; /** @brief table name */
OID mapOid; /** @brief table OID */
std::string loadFileName; /** @brief table load file name */
uint64_t maxErrNum; /** @brief max number of error rows before abort */
JobColList colList; /** @brief list of columns to be loaded; followed by default columns to be loaded */
JobColList fIgnoredFields; /** @brief list of fields in input file to be ignored */
JobFieldRefList fFldRefs; /** @brief Combined list of refs to entries in colList and fIgnoredFields */
JobTable() : mapOid(0), maxErrNum(0) { }
std::string tblName; /** @brief table name */
OID mapOid; /** @brief table OID */
std::string loadFileName; /** @brief table load file name */
uint64_t maxErrNum; /** @brief max number of error rows before abort */
JobColList colList; /** @brief list of columns to be loaded; followed by default columns to be loaded */
JobColList fIgnoredFields; /** @brief list of fields in input file to be ignored */
JobFieldRefList fFldRefs; /** @brief Combined list of refs to entries in colList and fIgnoredFields */
JobTable() : mapOid(0), maxErrNum(0)
{
}
};
typedef std::vector<JobTable> JobTableList;/** @brief table list */
typedef std::vector<JobTable> JobTableList; /** @brief table list */
struct Job /** @brief Job Structure */
struct Job /** @brief Job Structure */
{
int id; /** @brief job id */
std::string schema; /** @brief database name */
std::string name; /** @brief job name */
std::string desc; /** @brief job description */
std::string userName; /** @brief user name */
JobTableList jobTableList; /** @brief job table list */
int id; /** @brief job id */
std::string schema; /** @brief database name */
std::string name; /** @brief job name */
std::string desc; /** @brief job description */
std::string userName; /** @brief user name */
JobTableList jobTableList; /** @brief job table list */
std::string createDate; /** @brief job create date */
std::string createTime; /** @brief job create time */
std::string createDate; /** @brief job create date */
std::string createTime; /** @brief job create time */
char fDelimiter;
char fEnclosedByChar;
char fEscapeChar;
int numberOfReadBuffers;
unsigned readBufferSize;
unsigned writeBufferSize;
Job() : id(0), fDelimiter('|'),
fEnclosedByChar('\0'), fEscapeChar('\0'),
numberOfReadBuffers(0), readBufferSize(0), writeBufferSize(0) { }
char fDelimiter;
char fEnclosedByChar;
char fEscapeChar;
int numberOfReadBuffers;
unsigned readBufferSize;
unsigned writeBufferSize;
Job()
: id(0)
, fDelimiter('|')
, fEnclosedByChar('\0')
, fEscapeChar('\0')
, numberOfReadBuffers(0)
, readBufferSize(0)
, writeBufferSize(0)
{
}
};
/************************************************************************
* @brief Cache memory
************************************************************************/
struct CacheBlock /** @brief Cache block structure */
struct CacheBlock /** @brief Cache block structure */
{
uint64_t fbo; /** @brief file fbo */
uint64_t lbid; /** @brief lbid */
bool dirty; /** @brief dirty flag */
int hitCount; /** @brief hit count */
unsigned char* data; /** @brief block buffer */
CacheBlock()
{
data = NULL; /** @brief constructor */
clear();
}
void clear()
{
fbo = lbid = hitCount = 0;
dirty = false;
uint64_t fbo; /** @brief file fbo */
uint64_t lbid; /** @brief lbid */
bool dirty; /** @brief dirty flag */
int hitCount; /** @brief hit count */
unsigned char* data; /** @brief block buffer */
CacheBlock()
{
data = NULL; /** @brief constructor */
clear();
}
void clear()
{
fbo = lbid = hitCount = 0;
dirty = false;
if ( data ) memset( data, 0, BYTE_PER_BLOCK);
} /** @brief clear, NOTE: buf must be free by caller first */
void init()
{
data = (unsigned char*)malloc(BYTE_PER_BLOCK);
}
void freeMem()
{
if ( data ) free( data );
}
if (data)
memset(data, 0, BYTE_PER_BLOCK);
} /** @brief clear, NOTE: buf must be free by caller first */
void init()
{
data = (unsigned char*)malloc(BYTE_PER_BLOCK);
}
void freeMem()
{
if (data)
free(data);
}
};
struct BlockBuffer /** @brief Block buffer */
struct BlockBuffer /** @brief Block buffer */
{
CommBlock cb; /** @brief Communication block structure */
CacheBlock block; /** @brief Cache block strucutre */
CacheListType listType; /** @brief List number, 0 - free, 1 - LRU, 2 - write */
BlockBuffer()
{
clear(); /** @brief constructor */
}
void init()
{
block.init();
}
void freeMem()
{
block.freeMem();
}
void clear()
{
cb.clear();
block.clear();
listType = FREE_LIST;
}
CommBlock cb; /** @brief Communication block structure */
CacheBlock block; /** @brief Cache block strucutre */
CacheListType listType; /** @brief List number, 0 - free, 1 - LRU, 2 - write */
BlockBuffer()
{
clear(); /** @brief constructor */
}
void init()
{
block.init();
}
void freeMem()
{
block.freeMem();
}
void clear()
{
cb.clear();
block.clear();
listType = FREE_LIST;
}
};
struct CacheControl /** @brief Cache control structure */
struct CacheControl /** @brief Cache control structure */
{
int totalBlock; /** @brief The toal number of allocated blocks */
int pctFree; /** @brief The percentage of free blocks when some blocks must be aged out */
int checkInterval; /** @brief A check point interval in seconds */
CacheControl()
{
totalBlock = pctFree = checkInterval = 0; /** @brief constructor */
}
int totalBlock; /** @brief The toal number of allocated blocks */
int pctFree; /** @brief The percentage of free blocks when some blocks must be aged out */
int checkInterval; /** @brief A check point interval in seconds */
CacheControl()
{
totalBlock = pctFree = checkInterval = 0; /** @brief constructor */
}
};
/************************************************************************
* @brief Bulk parse meta data describing data in a read buffer.
* An offset of COLPOSPAIR_NULL_TOKEN_OFFSET represents a null token.
************************************************************************/
struct ColPosPair /** @brief Column position pair structure */
struct ColPosPair /** @brief Column position pair structure */
{
int start; /** @brief start position */
int offset; /** @brief length of token*/
int start; /** @brief start position */
int offset; /** @brief length of token*/
};
/************************************************************************
@ -517,9 +584,10 @@ struct ColPosPair /** @brief Column position pair structure */
************************************************************************/
class SecondaryShutdownException : public std::runtime_error
{
public:
SecondaryShutdownException(const std::string& msg) :
std::runtime_error(msg) { }
public:
SecondaryShutdownException(const std::string& msg) : std::runtime_error(msg)
{
}
};
/************************************************************************
@ -528,20 +596,21 @@ public:
************************************************************************/
class WeException : public std::runtime_error
{
public:
WeException(const std::string& msg, int err = 0) :
std::runtime_error(msg), fErrorCode(err) { }
void errorCode(int code)
{
fErrorCode = code;
}
int errorCode() const
{
return fErrorCode;
}
private:
int fErrorCode;
public:
WeException(const std::string& msg, int err = 0) : std::runtime_error(msg), fErrorCode(err)
{
}
void errorCode(int code)
{
fErrorCode = code;
}
int errorCode() const
{
return fErrorCode;
}
private:
int fErrorCode;
};
} //end of namespace
} // namespace WriteEngine

84
writeengine/shared/we_typeext.h
View File

@ -34,79 +34,77 @@
#include <sstream>
#include "IDBFileSystem.h"
/** Namespace WriteEngine */
namespace WriteEngine
{
/************************************************************************
* Type definitions
************************************************************************/
typedef uint64_t RID; // Row ID
typedef uint64_t RID; // Row ID
/************************************************************************
* Dictionary related structure
************************************************************************/
struct Token
{
uint64_t op : 10; // ordinal position within a block
uint64_t fbo : 36; // file block number
uint64_t bc : 18; // block count
Token() // constructor, set to null value
{
op = 0x3FE;
fbo = 0xFFFFFFFFFLL;
bc = 0x3FFFF;
}
uint64_t op : 10; // ordinal position within a block
uint64_t fbo : 36; // file block number
uint64_t bc : 18; // block count
Token() // constructor, set to null value
{
op = 0x3FE;
fbo = 0xFFFFFFFFFLL;
bc = 0x3FFFF;
}
};
constexpr uid_t UID_NONE = (uid_t) -1;
constexpr gid_t GID_NONE = (gid_t) -1;
constexpr uid_t UID_NONE = (uid_t)-1;
constexpr gid_t GID_NONE = (gid_t)-1;
class WeUIDGID
{
public:
WeUIDGID(): uid(UID_NONE), gid(GID_NONE) {}
virtual ~WeUIDGID() {};
virtual void setUIDGID(const uid_t uid, const gid_t gid);
void setUIDGID(const WeUIDGID* id);
bool chownPath(std::ostringstream& error,
const std::string& fileName,
const idbdatafile::IDBFileSystem& fs) const;
;
public:
WeUIDGID() : uid(UID_NONE), gid(GID_NONE)
{
}
virtual ~WeUIDGID(){};
virtual void setUIDGID(const uid_t uid, const gid_t gid);
void setUIDGID(const WeUIDGID* id);
bool chownPath(std::ostringstream& error, const std::string& fileName,
const idbdatafile::IDBFileSystem& fs) const;
;
private:
uid_t uid;
gid_t gid;
private:
uid_t uid;
gid_t gid;
};
inline void WeUIDGID::setUIDGID(const uid_t p_uid, const gid_t p_gid)
{
uid = p_uid; gid = p_gid;
uid = p_uid;
gid = p_gid;
}
inline void WeUIDGID::setUIDGID(const WeUIDGID* id)
{
if (id->uid != UID_NONE)
*this = *id;
if (id->uid != UID_NONE)
*this = *id;
}
inline bool WeUIDGID::chownPath(std::ostringstream& error,
const std::string& fileName,
const idbdatafile::IDBFileSystem& fs) const
inline bool WeUIDGID::chownPath(std::ostringstream& error, const std::string& fileName,
const idbdatafile::IDBFileSystem& fs) const
{
if (uid != UID_NONE)
if (uid != UID_NONE)
{
int funcErrno = 0;
if (fs.chown(fileName.c_str(), uid, gid, funcErrno) == -1)
{
int funcErrno = 0;
if (fs.chown(fileName.c_str(), uid, gid, funcErrno) == -1)
{
error << "Error calling chown() with uid " << uid
<< " and gid " << gid << " with the file "
<< fileName << " with errno " << funcErrno;
return true;
}
error << "Error calling chown() with uid " << uid << " and gid " << gid << " with the file " << fileName
<< " with errno " << funcErrno;
return true;
}
return false;
}
return false;
}
} //end of namespace
} // namespace WriteEngine