You've already forked mariadb-columnstore-engine
mirror of
https://github.com/mariadb-corporation/mariadb-columnstore-engine.git
synced 2025-10-30 07:25:34 +03:00
285 lines
11 KiB
C++
285 lines
11 KiB
C++
/* Copyright (C) 2014 InfiniDB, Inc.
|
||
|
||
This program is free software; you can redistribute it and/or
|
||
modify it under the terms of the GNU General Public License
|
||
as published by the Free Software Foundation; version 2 of
|
||
the License.
|
||
|
||
This program is distributed in the hope that it will be useful,
|
||
but WITHOUT ANY WARRANTY; without even the implied warranty of
|
||
MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
|
||
GNU General Public License for more details.
|
||
|
||
You should have received a copy of the GNU General Public License
|
||
along with this program; if not, write to the Free Software
|
||
Foundation, Inc., 51 Franklin Street, Fifth Floor, Boston,
|
||
MA 02110-1301, USA. */
|
||
|
||
#include "we_bulkrollbackfilecompressedhdfs.h"
|
||
|
||
#include <sstream>
|
||
#include <boost/scoped_array.hpp>
|
||
#include <boost/filesystem/path.hpp>
|
||
#include <boost/filesystem/convenience.hpp>
|
||
|
||
#include "we_define.h"
|
||
#include "we_fileop.h"
|
||
#include "we_bulkrollbackmgr.h"
|
||
#include "we_confirmhdfsdbfile.h"
|
||
#include "we_convertor.h"
|
||
#include "messageids.h"
|
||
#include "IDBDataFile.h"
|
||
#include "IDBPolicy.h"
|
||
using namespace idbdatafile;
|
||
using namespace compress;
|
||
using namespace execplan;
|
||
|
||
namespace
|
||
{
|
||
const char* DATA_DIR_SUFFIX = "_data";
|
||
const char* OLD_FILE_SUFFIX = ".old_bulk";
|
||
}
|
||
|
||
namespace WriteEngine
|
||
{
|
||
|
||
//------------------------------------------------------------------------------
|
||
// BulkRollbackFileCompressedHdfs constructor
|
||
//------------------------------------------------------------------------------
|
||
BulkRollbackFileCompressedHdfs::BulkRollbackFileCompressedHdfs(
|
||
BulkRollbackMgr* mgr) :
|
||
BulkRollbackFile(mgr)
|
||
{
|
||
}
|
||
|
||
//------------------------------------------------------------------------------
|
||
// BulkRollbackFileCompressedHdfs destructor
|
||
//------------------------------------------------------------------------------
|
||
BulkRollbackFileCompressedHdfs::~BulkRollbackFileCompressedHdfs()
|
||
{
|
||
}
|
||
|
||
//------------------------------------------------------------------------------
|
||
// Truncate the specified database segment file to the extent specified by
|
||
// the given file offset. Also updates the header(s) as well.
|
||
//
|
||
// columnOID - OID of segment file to be truncated
|
||
// dbRoot - DBRoot of segment file to be truncated
|
||
// partNum - Partition number of segment file to be truncated
|
||
// segNum - Segment number of segment file to be truncated
|
||
// fileSizeBlocks - Number of raw data blocks to be left in the file.
|
||
// Remainder of file is to be truncated.
|
||
//------------------------------------------------------------------------------
|
||
void BulkRollbackFileCompressedHdfs::truncateSegmentFile(
|
||
OID columnOID,
|
||
uint32_t dbRoot,
|
||
uint32_t partNum,
|
||
uint32_t segNum,
|
||
long long fileSizeBlocks )
|
||
{
|
||
std::ostringstream msgText;
|
||
msgText << "Truncating compressed HDFS column file"
|
||
": dbRoot-" << dbRoot <<
|
||
"; part#-" << partNum <<
|
||
"; seg#-" << segNum <<
|
||
"; rawTotBlks-" << fileSizeBlocks;
|
||
fMgr->logAMessage( logging::LOG_TYPE_INFO,
|
||
logging::M0075, columnOID, msgText.str() );
|
||
|
||
restoreFromBackup( "column", columnOID, dbRoot, partNum, segNum );
|
||
}
|
||
|
||
//------------------------------------------------------------------------------
|
||
// Reinitialize a column segment extent (in the db file) to empty values,
|
||
// following the HWM. Remaining extents in the file are truncated.
|
||
// Also updates the header(s) as well.
|
||
//
|
||
// columnOID - OID of segment file to be reinitialized
|
||
// dbRoot - DBRoot of segment file to be reinitialized
|
||
// partNum - Partition number of segment file to be reinitialized
|
||
// segNum - Segment number of segment file to be reinitialized
|
||
// startOffsetBlk - File offset (after the HWM block), at which the file is
|
||
// to be reinitialized. Value is in raw data blocks.
|
||
// nBlocks - Number of blocks to be reinitialized
|
||
// colType - Data type of the applicable column
|
||
// colWidth - Width in bytes, of the applicable column
|
||
// restoreHwmChk - Specifies whether HWM chunk is to be restored.
|
||
//------------------------------------------------------------------------------
|
||
void BulkRollbackFileCompressedHdfs::reInitTruncColumnExtent(
|
||
OID columnOID,
|
||
uint32_t dbRoot,
|
||
uint32_t partNum,
|
||
uint32_t segNum,
|
||
long long startOffsetBlk,
|
||
int nBlocks,
|
||
CalpontSystemCatalog::ColDataType colType,
|
||
uint32_t colWidth,
|
||
bool restoreHwmChk )
|
||
{
|
||
long long startOffset = startOffsetBlk * BYTE_PER_BLOCK;
|
||
|
||
std::ostringstream msgText;
|
||
msgText << "Reinit HWM compressed column extent in HDFS db file" <<
|
||
": dbRoot-" << dbRoot <<
|
||
"; part#-" << partNum <<
|
||
"; seg#-" << segNum <<
|
||
"; rawOffset(bytes)-"<< startOffset <<
|
||
"; rawFreeBlks-" << nBlocks;
|
||
fMgr->logAMessage( logging::LOG_TYPE_INFO,
|
||
logging::M0075, columnOID, msgText.str() );
|
||
|
||
restoreFromBackup( "column", columnOID, dbRoot, partNum, segNum );
|
||
}
|
||
|
||
//------------------------------------------------------------------------------
|
||
// Reinitialize a dictionary segment extent (in the db file) to empty blocks,
|
||
// following the HWM. Remaining extents in the file are truncated.
|
||
// Also updates the header(s) as well.
|
||
//
|
||
// dStoreOID - OID of segment store file to be reinitialized
|
||
// dbRoot - DBRoot of segment file to be reinitialized
|
||
// partNum - Partition number of segment file to be reinitialized
|
||
// segNum - Segment number of segment file to be reinitialized
|
||
// startOffsetBlk - Starting block (after the HWM block), at which the file is
|
||
// to be reinitialized. Value is in raw data blocks.
|
||
// nBlocks - Number of blocks to be reinitialized
|
||
//------------------------------------------------------------------------------
|
||
void BulkRollbackFileCompressedHdfs::reInitTruncDctnryExtent(
|
||
OID dStoreOID,
|
||
uint32_t dbRoot,
|
||
uint32_t partNum,
|
||
uint32_t segNum,
|
||
long long startOffsetBlk,
|
||
int nBlocks )
|
||
{
|
||
long long startOffset = startOffsetBlk * BYTE_PER_BLOCK;
|
||
|
||
std::ostringstream msgText;
|
||
msgText << "Reinit HWM compressed dictionary store extent in HDFS db file"
|
||
": dbRoot-" << dbRoot <<
|
||
"; part#-" << partNum <<
|
||
"; seg#-" << segNum <<
|
||
"; rawOffset(bytes)-" << startOffset <<
|
||
"; rawFreeBlks-" << nBlocks;
|
||
fMgr->logAMessage( logging::LOG_TYPE_INFO,
|
||
logging::M0075, dStoreOID, msgText.str() );
|
||
|
||
restoreFromBackup( "dictionary store", dStoreOID, dbRoot, partNum, segNum );
|
||
}
|
||
|
||
//------------------------------------------------------------------------------
|
||
// For HDFS system, just always return true.
|
||
// Let ConfirmHdfsDbFile later determine when/if/how to restore from any
|
||
// existing backup file.
|
||
//------------------------------------------------------------------------------
|
||
bool BulkRollbackFileCompressedHdfs::doWeReInitExtent( OID columnOID,
|
||
uint32_t dbRoot,
|
||
uint32_t partNum,
|
||
uint32_t segNum) const
|
||
{
|
||
return true;
|
||
}
|
||
|
||
//------------------------------------------------------------------------------
|
||
// Replace the currently specified db file with it's corresponding backup file.
|
||
// The backup file is a complete backup, not just a backup of a single chunk.
|
||
//
|
||
// The initial implementation for this function restored from a NNN.pNNN.sNNN
|
||
// file stored under the meta file directory.
|
||
// The latest implementation for this function restores from a FILENNN.cdf.tmp
|
||
// or FILENNN.cdf.orig file stored in the same OID directory as the FILENNN.cdf
|
||
// file.
|
||
// However, this function still looks for the first backup file (NNN.pNNN.sNNN)
|
||
// in case the user did not upgrade cleanly, and we have to restore using an
|
||
// old leftover backup file.
|
||
//------------------------------------------------------------------------------
|
||
void BulkRollbackFileCompressedHdfs::restoreFromBackup(const char* colType,
|
||
OID columnOID,
|
||
uint32_t dbRoot,
|
||
uint32_t partNum,
|
||
uint32_t segNum)
|
||
{
|
||
// Construct file name for db file to be restored
|
||
char dbFileName[FILE_NAME_SIZE];
|
||
int rc = fDbFile.getFileName( columnOID, dbFileName,
|
||
dbRoot, partNum, segNum );
|
||
if (rc != NO_ERROR)
|
||
{
|
||
std::ostringstream oss;
|
||
oss << "Error restoring " << colType <<
|
||
" HDFS file for OID " << columnOID <<
|
||
"; Can't construct file name for DBRoot" << dbRoot <<
|
||
"; partition-" << partNum <<
|
||
"; segment-" << segNum;
|
||
throw WeException( oss.str(), rc );
|
||
}
|
||
|
||
// Construct file name for backup copy of db file
|
||
std::ostringstream ossFile;
|
||
ossFile << "/" << columnOID << ".p" << partNum << ".s" << segNum;
|
||
std::string backupFileName( fMgr->getMetaFileName() );
|
||
backupFileName += DATA_DIR_SUFFIX;
|
||
backupFileName += ossFile.str();
|
||
|
||
std::string dbFileNameTmp = dbFileName;
|
||
dbFileNameTmp += OLD_FILE_SUFFIX;
|
||
|
||
// For backwards compatibility...
|
||
// Restore from backup file used in initial HDFS release, in case the user
|
||
// upgraded without going down cleanly. In that case we might need to
|
||
// rollback using an old backup file left from previous release.
|
||
if ( IDBPolicy::exists(backupFileName.c_str()) )
|
||
{
|
||
// Rename current db file to make room for restored file
|
||
rc = IDBPolicy::rename( dbFileName, dbFileNameTmp.c_str() );
|
||
if (rc != 0)
|
||
{
|
||
std::ostringstream oss;
|
||
oss << "Error restoring " << colType <<
|
||
" HDFS file for OID " << columnOID <<
|
||
"; Can't move old file for DBRoot" << dbRoot <<
|
||
"; partition-" << partNum <<
|
||
"; segment-" << segNum;
|
||
throw WeException( oss.str(), ERR_COMP_RENAME_FILE );
|
||
}
|
||
|
||
// Rename backup file to replace current db file
|
||
rc = IDBPolicy::rename( backupFileName.c_str(), dbFileName );
|
||
if (rc != 0)
|
||
{
|
||
std::ostringstream oss;
|
||
oss << "Error restoring " << colType <<
|
||
" HDFS file for OID " << columnOID <<
|
||
"; Can't rename backup file for DBRoot" << dbRoot <<
|
||
"; partition-" << partNum <<
|
||
"; segment-" << segNum;
|
||
throw WeException( oss.str(), ERR_METADATABKUP_COMP_RENAME );
|
||
}
|
||
|
||
// Delete db file we just replaced with backup
|
||
IDBPolicy::remove( dbFileNameTmp.c_str() );
|
||
}
|
||
else // Restore from HDFS temp swap backup file; This is the normal case
|
||
{
|
||
std::string errMsg;
|
||
ConfirmHdfsDbFile confirmHdfs;
|
||
rc = confirmHdfs.endDbFileChange( std::string("tmp"),
|
||
dbFileName,
|
||
false,
|
||
errMsg);
|
||
if (rc != 0)
|
||
{
|
||
std::ostringstream oss;
|
||
oss << "Error restoring " << colType <<
|
||
" HDFS file for OID " << columnOID <<
|
||
"; DBRoot" << dbRoot <<
|
||
"; partition-" << partNum <<
|
||
"; segment-" << segNum <<
|
||
"; " << errMsg;
|
||
throw WeException( oss.str(), rc );
|
||
}
|
||
}
|
||
}
|
||
|
||
} //end of namespace
|