database/mariadb-columnstore-engine
1
0
Fork 0
mirror of https://github.com/mariadb-corporation/mariadb-columnstore-engine.git synced 2025-07-30 19:23:07 +03:00
Code Activity

Reformat all code to coding standard

Browse Source
This commit is contained in:
Andrew Hutchings
2017-10-26 17:18:17 +01:00
parent 4985f3456e
commit 01446d1e22
1296 changed files with 403852 additions and 353747 deletions
Download Patch File Download Diff File Expand all files Collapse all files

967
writeengine/bulk/cpimport.cpp
View File

File diff suppressed because it is too large Load Diff

271
writeengine/bulk/parsetest.cpp
View File

@ -22,7 +22,7 @@
#include <iostream>
#include <we_bulkload.h>
#define ENV_BULK_DIR "CP_BULK_DIR"
#define ENV_BULK_DIR "CP_BULK_DIR"
#include <string>
#include <boost/progress.hpp>
@ -34,183 +34,234 @@ string Lines[MAXSTRINGS];
typedef std::vector<std::string> LineFldList;
const int parseStr( const string& instr, LineFldList fields)
const int parseStr( const string& instr, LineFldList fields)
{
typedef boost::tokenizer<boost::char_separator<char> >
tokenizer;
typedef boost::tokenizer<boost::char_separator<char> >
tokenizer;
boost::char_separator<char> sep("|");
tokenizer tokens(instr, sep);
for (tokenizer::iterator tok_iter = tokens.begin(); tok_iter != tokens.end(); ++tok_iter){
for (tokenizer::iterator tok_iter = tokens.begin(); tok_iter != tokens.end(); ++tok_iter)
{
// std::cout << "<" << *tok_iter << "> ";
fields.push_back( *tok_iter );
}
//std::cout << "\n";
return EXIT_SUCCESS;
return EXIT_SUCCESS;
}
int strtok_test( const string& instr ){
char *fragment;
char *search = (char*)malloc( instr.length());
int strtok_test( const string& instr )
{
char* fragment;
char* search = (char*)malloc( instr.length());
memcpy( search, instr.c_str(), instr.length());
fragment = strtok(search, "|");
do {
// printf("Token: %s\n", fragment);
do
{
// printf("Token: %s\n", fragment);
fragment = strtok(NULL, "|");
} while (fragment);
}
while (fragment);
free( search );
return EXIT_SUCCESS;
}
int handrolled_test( const string& instr ){
char *search = (char*)malloc( instr.length());
char *pos;
int count=0;
int handrolled_test( const string& instr )
{
char* search = (char*)malloc( instr.length());
char* pos;
int count = 0;
int span;
string temp;
string results[10];
memcpy( search, instr.data(), instr.length());
if (search[0] == '|'){
pos = search+1;
} else {
if (search[0] == '|')
{
pos = search + 1;
}
else
{
pos = search;
}
while (pos) {
}
while (pos)
{
span = strcspn(pos, "|");
if (span){
if (span)
{
temp.assign(pos, span);
results[count++].assign(temp);
}
pos = index(pos+1, '|');
if (pos){ pos++ ;}
pos = index(pos + 1, '|');
if (pos)
{
pos++ ;
}
};
free( search );
//printf("\n%i dips", count);
return EXIT_SUCCESS;
}
int handrolled_test2( string& instr, string Fields[] ){
}
char *search = (char*)malloc( instr.length() +1 );
char *pos; // pos is used to step inside the search string
int count=0; // keeps track of fields found
int handrolled_test2( string& instr, string Fields[] )
{
char* search = (char*)malloc( instr.length() + 1 );
char* pos; // pos is used to step inside the search string
int count = 0; // keeps track of fields found
int charspan;
int num_bars;
strcpy( search, instr.c_str() );
pos = search;
if (search[0] == '|'){
pos = search+1;
if (search[0] == '|')
{
pos = search + 1;
Fields[count++].assign(""); // a leading bar indicates an opening blank
} else {
}
else
{
pos = search;
}
while (pos < search+instr.length()-1 ) {
}
while (pos < search + instr.length() - 1 )
{
charspan = strcspn(pos, "|");
if (charspan){
if (charspan)
{
Fields[count++].assign(pos, charspan);
pos += charspan + 1;
} else {
}
else
{
Fields[count++].assign("");
pos++;
}
num_bars = strspn(pos, "|");
pos += num_bars;
for( ; num_bars>0; num_bars--){
for ( ; num_bars > 0; num_bars--)
{
Fields[count++].assign("");
}
};
free( search );
return count;
}
int parseToken(){
}
int parseToken()
{
return 1;
}
int build_data(){
int build_data()
{
int idx;
for (idx=0; idx < MAXSTRINGS; idx++){
for (idx = 0; idx < MAXSTRINGS; idx++)
{
//tpch data files are of the form
// item|item|item and the line may end with |
// even though this may wrongly suggest a blank value at the end
// even though this may wrongly suggest a blank value at the end
Lines[idx] = "12345|abcdef|banana|banana|"; // 'item item item item'
}
//std::cout << Lines[idx-1] << endl;
return 0;
}
int main(int argc, char **argv)
int main(int argc, char** argv)
{
string sJobIdStr, sBulkDir = "", sDbDir = "", sFileName, sTmp;
int fcount;
string Fields[1000] ;
string search;
string searches[]= {
"", "|", "|||||||||||||||", "12345|abcdef|banana|", "123456789012345678901234567890",
"|12345678901234567890|12345678901234567890|12345678901234567890|12345678901234567890|12345678901234567890|12345678901234567890|12345678901234567890|12345678901234567890|12345678901234567890|12345678901234567890|12345678901234567890",
"|12345|abcdef|banana|bank123", "|123456789012345678901234567890", "12345|abcdef|banana|bank123",
"12345||abcdef||banana|bank", "|12345||abcdef|banana|bank", "|12345|abcdef|banana|bank|",
"|12345|abcdef|banana||", "|12345|abcdef|banana|||"
};
// 14 elements
printf("\nAccuracy:");
for (int test=0; test < 14; test++){
printf("\n\nSearch string %i: %s", test, searches[test].c_str());
fcount = handrolled_test2(searches[test], Fields);
for (int idx = 0; idx < fcount; idx++){
printf("\nString %i: %s$", idx, Fields[idx].c_str());
}
}
printf("\n\nSpeed:\n");
build_data();
boost::timer t;
LineFldList parseFields;
for (int idx=0; idx< MAXSTRINGS; idx++){
parseStr(Lines[idx], parseFields);
}
printf("Boost Parse Timer: %lf\n", t.elapsed());
t.restart();
for (int idx=0; idx< MAXSTRINGS; idx++){
strtok_test(Lines[idx]);
}
printf("Strtok Timer: %lf\n", t.elapsed());
t.restart();
for (int idx=0; idx< MAXSTRINGS; idx++){
handrolled_test(Lines[idx]);
}
printf("Handrolled Timer: %lf\n", t.elapsed());
string sJobIdStr, sBulkDir = "", sDbDir = "", sFileName, sTmp;
int fcount;
string Fields[1000] ;
string search;
string searches[] =
{
"", "|", "|||||||||||||||", "12345|abcdef|banana|", "123456789012345678901234567890",
"|12345678901234567890|12345678901234567890|12345678901234567890|12345678901234567890|12345678901234567890|12345678901234567890|12345678901234567890|12345678901234567890|12345678901234567890|12345678901234567890|12345678901234567890",
"|12345|abcdef|banana|bank123", "|123456789012345678901234567890", "12345|abcdef|banana|bank123",
"12345||abcdef||banana|bank", "|12345||abcdef|banana|bank", "|12345|abcdef|banana|bank|",
"|12345|abcdef|banana||", "|12345|abcdef|banana|||"
};
// 14 elements
printf("\nAccuracy:");
t.restart();
for (int idx=0; idx< MAXSTRINGS; idx++){
fcount = handrolled_test2(Lines[idx], Fields);
}
printf("Handrolled2 Timer: %lf\n", t.elapsed());
printf("\n");
return 0;
for (int test = 0; test < 14; test++)
{
printf("\n\nSearch string %i: %s", test, searches[test].c_str());
fcount = handrolled_test2(searches[test], Fields);
for (int idx = 0; idx < fcount; idx++)
{
printf("\nString %i: %s$", idx, Fields[idx].c_str());
}
}
printf("\n\nSpeed:\n");
build_data();
boost::timer t;
LineFldList parseFields;
for (int idx = 0; idx < MAXSTRINGS; idx++)
{
parseStr(Lines[idx], parseFields);
}
printf("Boost Parse Timer: %lf\n", t.elapsed());
t.restart();
for (int idx = 0; idx < MAXSTRINGS; idx++)
{
strtok_test(Lines[idx]);
}
printf("Strtok Timer: %lf\n", t.elapsed());
t.restart();
for (int idx = 0; idx < MAXSTRINGS; idx++)
{
handrolled_test(Lines[idx]);
}
printf("Handrolled Timer: %lf\n", t.elapsed());
t.restart();
for (int idx = 0; idx < MAXSTRINGS; idx++)
{
fcount = handrolled_test2(Lines[idx], Fields);
}
printf("Handrolled2 Timer: %lf\n", t.elapsed());
printf("\n");
return 0;
}

28
writeengine/bulk/resource.h
View File

@ -1,14 +1,14 @@
//{{NO_DEPENDENCIES}}
// Microsoft Visual C++ generated include file.
// Used by cpimport.rc
// Next default values for new objects
//
#ifdef APSTUDIO_INVOKED
#ifndef APSTUDIO_READONLY_SYMBOLS
#define _APS_NEXT_RESOURCE_VALUE 101
#define _APS_NEXT_COMMAND_VALUE 40001
#define _APS_NEXT_CONTROL_VALUE 1001
#define _APS_NEXT_SYMED_VALUE 101
#endif
#endif
//{{NO_DEPENDENCIES}}
// Microsoft Visual C++ generated include file.
// Used by cpimport.rc
// Next default values for new objects
//
#ifdef APSTUDIO_INVOKED
#ifndef APSTUDIO_READONLY_SYMBOLS
#define _APS_NEXT_RESOURCE_VALUE 101
#define _APS_NEXT_COMMAND_VALUE 40001
#define _APS_NEXT_CONTROL_VALUE 1001
#define _APS_NEXT_SYMED_VALUE 101
#endif
#endif

73
writeengine/bulk/we_brmreporter.cpp
View File

@ -35,7 +35,8 @@
#include "cacheutils.h"
#include "IDBPolicy.h"
namespace WriteEngine {
namespace WriteEngine
{
//------------------------------------------------------------------------------
// Constructor
@ -98,14 +99,16 @@ void BRMReporter::addToErrMsgEntry(const std::string& errCritMsg)
//------------------------------------------------------------------------------
void BRMReporter::sendErrMsgToFile(const std::string& rptFileName)
{
if((!rptFileName.empty())&&(fRptFileName.empty()))
fRptFileName=rptFileName;
if ((!fRptFileName.empty())&&(fCritErrMsgs.size()))
if ((!rptFileName.empty()) && (fRptFileName.empty()))
fRptFileName = rptFileName;
if ((!fRptFileName.empty()) && (fCritErrMsgs.size()))
{
fRptFile.open( fRptFileName.c_str(), std::ios_base::app);
if ( fRptFile.good() )
{
for (unsigned int i=0; i<fCritErrMsgs.size(); i++)
for (unsigned int i = 0; i < fCritErrMsgs.size(); i++)
{
fRptFile << "MERR: " << fCritErrMsgs[i] << std::endl;
//std::cout <<"**********" << fCritErrMsgs[i] << std::endl;
@ -125,8 +128,8 @@ void BRMReporter::sendErrMsgToFile(const std::string& rptFileName)
// sendBRMInfo(). Once PrimProc cache is flushed, we can send the BRM updates.
//------------------------------------------------------------------------------
int BRMReporter::sendBRMInfo(const std::string& rptFileName,
const std::vector<std::string>& errFiles,
const std::vector<std::string>& badFiles)
const std::vector<std::string>& errFiles,
const std::vector<std::string>& badFiles)
{
int rc = NO_ERROR;
@ -138,7 +141,7 @@ int BRMReporter::sendBRMInfo(const std::string& rptFileName,
if ( fFileInfo.size() > 0 )
{
for (unsigned k=0; k<fFileInfo.size(); k++)
for (unsigned k = 0; k < fFileInfo.size(); k++)
{
allFileInfo.push_back( fFileInfo[k] );
}
@ -146,9 +149,10 @@ int BRMReporter::sendBRMInfo(const std::string& rptFileName,
std::vector<BRM::OID_t> oidsToFlush;
std::set<BRM::OID_t> oidSet;
if (fDctnryFileInfo.size() > 0)
{
for (unsigned k=0; k<fDctnryFileInfo.size(); k++)
for (unsigned k = 0; k < fDctnryFileInfo.size(); k++)
{
allFileInfo.push_back( fDctnryFileInfo[k] );
oidSet.insert( fDctnryFileInfo[k].oid );
@ -157,9 +161,9 @@ int BRMReporter::sendBRMInfo(const std::string& rptFileName,
// Store dictionary oids in std::set first, to eliminate duplicates
if (oidSet.size() > 0)
{
for (std::set<BRM::OID_t>::const_iterator iter=oidSet.begin();
iter != oidSet.end();
++iter)
for (std::set<BRM::OID_t>::const_iterator iter = oidSet.begin();
iter != oidSet.end();
++iter)
{
oidsToFlush.push_back( *iter );
}
@ -170,7 +174,7 @@ int BRMReporter::sendBRMInfo(const std::string& rptFileName,
if (allFileInfo.size() > 0)
{
cacheutils::purgePrimProcFdCache(allFileInfo,
Config::getLocalModuleID());
Config::getLocalModuleID());
}
// Flush PrimProc block cache
@ -199,6 +203,7 @@ int BRMReporter::sendBRMInfo(const std::string& rptFileName,
fRptFileName = rptFileName;
rc = openRptFile( );
if (rc != NO_ERROR)
{
return rc;
@ -208,12 +213,12 @@ int BRMReporter::sendBRMInfo(const std::string& rptFileName,
sendHWMToFile( );
// Log the list of *.err and *.bad files
for (unsigned k=0; k<errFiles.size(); k++)
for (unsigned k = 0; k < errFiles.size(); k++)
{
fRptFile << "ERR: " << errFiles[k] << std::endl;
}
for (unsigned k=0; k<badFiles.size(); k++)
for (unsigned k = 0; k < badFiles.size(); k++)
{
fRptFile << "BAD: " << badFiles[k] << std::endl;
}
@ -233,7 +238,7 @@ int BRMReporter::sendHWMandCPToBRM( )
if (fHWMInfo.size() > 0)
{
std::ostringstream oss;
oss << "Committing " << fHWMInfo.size() << " HWM update(s) for table "<<
oss << "Committing " << fHWMInfo.size() << " HWM update(s) for table " <<
fTableName << " to BRM";
fLog->logMsg( oss.str(), MSGLVL_INFO2 );
}
@ -276,12 +281,12 @@ void BRMReporter::sendHWMToFile( )
fTableName << " to report file " << fRptFileName;
fLog->logMsg( oss.str(), MSGLVL_INFO2 );
for (unsigned int i=0; i<fHWMInfo.size(); i++)
for (unsigned int i = 0; i < fHWMInfo.size(); i++)
{
fRptFile << "HWM: " << fHWMInfo[i].oid << ' ' <<
fHWMInfo[i].partNum << ' ' <<
fHWMInfo[i].segNum << ' ' <<
fHWMInfo[i].hwm << std::endl;
fHWMInfo[i].partNum << ' ' <<
fHWMInfo[i].segNum << ' ' <<
fHWMInfo[i].hwm << std::endl;
}
}
}
@ -298,14 +303,14 @@ void BRMReporter::sendCPToFile( )
fTableName << " to report file " << fRptFileName;
fLog->logMsg( oss.str(), MSGLVL_INFO2 );
for (unsigned int i=0; i<fCPInfo.size(); i++)
for (unsigned int i = 0; i < fCPInfo.size(); i++)
{
fRptFile << "CP: " << fCPInfo[i].startLbid << ' ' <<
fCPInfo[i].max << ' ' <<
fCPInfo[i].min << ' ' <<
fCPInfo[i].seqNum << ' ' <<
fCPInfo[i].type << ' ' <<
fCPInfo[i].newExtent << std::endl;
fCPInfo[i].max << ' ' <<
fCPInfo[i].min << ' ' <<
fCPInfo[i].seqNum << ' ' <<
fCPInfo[i].type << ' ' <<
fCPInfo[i].newExtent << std::endl;
}
}
}
@ -321,12 +326,13 @@ void BRMReporter::reportTotals(
if (fRptFile.is_open())
{
fRptFile << "ROWS: " << totalReadRows << ' ' <<
totalInsertedRows << std::endl;
for (unsigned k=0; k<satCounts.size(); k++)
totalInsertedRows << std::endl;
for (unsigned k = 0; k < satCounts.size(); k++)
{
if (boost::get<0>(satCounts[k]) > 0)
fRptFile << "DATA: " << k << ' ' << boost::get<0>(satCounts[k]) << ' ' <<
boost::get<1>(satCounts[k]) << ' ' << boost::get<2>(satCounts[k]) << std::endl;
boost::get<1>(satCounts[k]) << ' ' << boost::get<2>(satCounts[k]) << std::endl;
}
closeRptFile();
@ -338,8 +344,8 @@ void BRMReporter::reportTotals(
// limit.
//------------------------------------------------------------------------------
void BRMReporter::rptMaxErrJob(const std::string& rptFileName,
const std::vector<std::string>& errFiles,
const std::vector<std::string>& badFiles )
const std::vector<std::string>& errFiles,
const std::vector<std::string>& badFiles )
{
// We only write out information if we are generating a report file.
if (!rptFileName.empty())
@ -356,12 +362,12 @@ void BRMReporter::rptMaxErrJob(const std::string& rptFileName,
}
// Log the list of *.err and *.bad files
for (unsigned k=0; k<errFiles.size(); k++)
for (unsigned k = 0; k < errFiles.size(); k++)
{
fRptFile << "ERR: " << errFiles[k] << std::endl;
}
for (unsigned k=0; k<badFiles.size(); k++)
for (unsigned k = 0; k < badFiles.size(); k++)
{
fRptFile << "BAD: " << badFiles[k] << std::endl;
}
@ -376,6 +382,7 @@ void BRMReporter::rptMaxErrJob(const std::string& rptFileName,
int BRMReporter::openRptFile( )
{
fRptFile.open( fRptFileName.c_str() );
if ( fRptFile.fail() )
{
int errRc = errno;

15
writeengine/bulk/we_brmreporter.h
View File

@ -36,7 +36,8 @@ using namespace execplan;
#ifndef _WE_BRMREPORTER_H_
#define _WE_BRMREPORTER_H_
namespace WriteEngine {
namespace WriteEngine
{
class Log;
/** @brief Accumulate update info that is later forwarded to BRM.
@ -47,7 +48,7 @@ class Log;
*/
class BRMReporter
{
public:
public:
/** @brief Constructor
* @param logger Object used for logging
@ -106,8 +107,8 @@ class BRMReporter
*/
void reportTotals(uint64_t totalReadRows,
uint64_t totalInsertedRows,
const std::vector<boost::tuple<CalpontSystemCatalog::ColDataType,
uint64_t,uint64_t> >& satCounts);
const std::vector<boost::tuple<CalpontSystemCatalog::ColDataType,
uint64_t, uint64_t> >& satCounts);
/** @brief Generate report for job that exceeds error limit
* @param rptFileName Name of file to save info, else info is dropped
@ -115,10 +116,10 @@ class BRMReporter
* @param badFiles List of *.bad filenames to record in report file
*/
void rptMaxErrJob(const std::string& rptFileName,
const std::vector<std::string>& errFiles,
const std::vector<std::string>& badFiles );
const std::vector<std::string>& errFiles,
const std::vector<std::string>& badFiles );
private:
private:
// Disable copy constructor and assignment operator by declaring and
// not defining.

506
writeengine/bulk/we_bulkload.cpp
View File

File diff suppressed because it is too large Load Diff

343
writeengine/bulk/we_bulkload.h
View File

@ -32,7 +32,7 @@
#include <sys/time.h>
#include <we_log.h>
#include <we_colop.h>
#include <we_colop.h>
#include <we_xmljob.h>
#include <we_convertor.h>
#include <writeengine.h>
@ -63,72 +63,75 @@ class BulkLoad : public FileOp
{
public:
/**
* @brief BulkLoad onstructor
*/
/**
* @brief BulkLoad onstructor
*/
EXPORT BulkLoad();
/**
* @brief BulkLoad destructor
*/
/**
* @brief BulkLoad destructor
*/
EXPORT ~BulkLoad();
/**
* @brief Load job information
*/
/**
* @brief Load job information
*/
EXPORT int loadJobInfo( const std::string& fullFileName,
bool bUseTempJobFile,
const std::string& systemLang,
int argc,
char** argv,
bool bLogInfo2ToConsole,
bool bValidateColumnList );
bool bUseTempJobFile,
const std::string& systemLang,
int argc,
char** argv,
bool bLogInfo2ToConsole,
bool bValidateColumnList );
/**
* @brief Pre process jobs to validate and assign values to the job structure
*/
/**
* @brief Pre process jobs to validate and assign values to the job structure
*/
int preProcess(Job& job, int tableNo, TableInfo* tableInfo);
/**
* @brief Print job information
*/
/**
* @brief Print job information
*/
void printJob( );
/**
* @brief Process job
*/
/**
* @brief Process job
*/
EXPORT int processJob( );
/**
* @brief Set Debug level for this BulkLoad object and any data members
*/
/**
* @brief Set Debug level for this BulkLoad object and any data members
*/
void setAllDebug( DebugLevel level );
/**
* @brief Update next autoincrement value for specified OID.
* @param columnOID oid of autoincrement column to be updated
* @param nextValue next autoincrement value to assign to tableOID
*/
/**
* @brief Update next autoincrement value for specified OID.
* @param columnOID oid of autoincrement column to be updated
* @param nextValue next autoincrement value to assign to tableOID
*/
static int updateNextValue(OID columnOID, uint64_t nextValue);
// Accessors and mutators
void addToCmdLineImportFileList(const std::string& importFile);
const std::string& getAlternateImportDir( ) const;
const std::string& getErrorDir ( ) const;
const std::string& getJobDir ( ) const;
const std::string& getJobDir ( ) const;
const std::string& getSchema ( ) const;
const std::string& getTempJobDir ( ) const;
bool getTruncationAsError ( ) const;
BulkModeType getBulkLoadMode ( ) const;
bool getContinue ( ) const;
boost::uuids::uuid getJobUUID ( ) const { return fUUID; }
boost::uuids::uuid getJobUUID ( ) const
{
return fUUID;
}
EXPORT int setAlternateImportDir( const std::string& loadDir,
std::string& errMsg);
std::string& errMsg);
void setImportDataMode ( ImportDataMode importMode );
void setColDelimiter ( char delim );
void setBulkLoadMode ( BulkModeType bulkMode,
const std::string& rptFileName );
const std::string& rptFileName );
void setEnclosedByChar ( char enChar);
void setEscapeChar ( char esChar);
void setKeepRbMetaFiles ( bool keepMeta );
@ -145,21 +148,25 @@ public:
void setVbufReadSize ( int vbufReadSize );
void setTruncationAsError ( bool bTruncationAsError );
void setJobUUID ( const std::string& jobUUID );
void setErrorDir ( const std::string& errorDir );
void setErrorDir ( const std::string& errorDir );
// Timer functions
void startTimer ( );
void stopTimer ( );
double getTotalRunTime ( ) const;
void disableTimeOut ( const bool disableTimeOut);
bool disableTimeOut ( ) const;
static void disableConsoleOutput ( const bool noConsoleOutput)
{ fNoConsoleOutput = noConsoleOutput; }
static bool disableConsoleOutput ( )
{ return fNoConsoleOutput; }
// Add error message into appropriate BRM updater
static void disableConsoleOutput ( const bool noConsoleOutput)
{
fNoConsoleOutput = noConsoleOutput;
}
static bool disableConsoleOutput ( )
{
return fNoConsoleOutput;
}
// Add error message into appropriate BRM updater
static bool addErrorMsg2BrmUpdater(const std::string& tablename, const std::ostringstream& oss);
void setDefaultJobUUID ( );
@ -185,7 +192,7 @@ private:
int fFileVbufSize; // Internal file system buffer size
long long fMaxErrors; // Max allowable errors per job
std::string fAlternateImportDir; // Alternate bulk import directory
std::string fErrorDir; // Opt. where error records record
std::string fErrorDir; // Opt. where error records record
std::string fProcessName; // Application process name
static boost::ptr_vector<TableInfo> fTableInfo;// Vector of Table information
int fNoOfParseThreads; // Number of parse threads
@ -193,9 +200,9 @@ private:
boost::thread_group fReadThreads; // Read thread group
boost::thread_group fParseThreads; // Parse thread group
boost::mutex fReadMutex; // Manages table selection by each
// read thread
// read thread
boost::mutex fParseMutex; // Manages table/buffer/column
// selection by each parsing thread
// selection by each parsing thread
BRM::TxnID fTxnID; // TransID acquired from SessionMgr
bool fKeepRbMetaFiles; // Keep/delete bulkRB metadata files
bool fNullStringMode; // Treat "NULL" as NULL value
@ -210,7 +217,7 @@ private:
bool fbTruncationAsError; // Treat string truncation as error
ImportDataMode fImportDataMode; // Importing text or binary data
bool fbContinue; // true when read and parse r running
//
//
static boost::mutex* fDDLMutex; // Insure only 1 DDL op at a time
EXPORT static const std::string DIR_BULK_JOB; // Bulk job directory
@ -237,15 +244,15 @@ private:
// Get column for parsing. Called by the parse thread.
// @bug 2099 - Temporary hack to diagnose deadlock. Added report parm below.
bool lockColumnForParse(int id, // thread id
int &tableId, // selected table id
int &columnId, // selected column id
int &myParseBuffer, // selected parse buffer
int& tableId, // selected table id
int& columnId, // selected column id
int& myParseBuffer, // selected parse buffer
bool report);
// Map specified DBRoot to it's first segment file number
int mapDBRootToFirstSegment(OID columnOid,
uint16_t dbRoot,
uint16_t& segment);
uint16_t dbRoot,
uint16_t& segment);
// The thread method for the read thread.
void read(int id);
@ -263,12 +270,12 @@ private:
// Determine starting HWM and LBID after block skipping added to HWM
int preProcessHwmLbid( const ColumnInfo* info,
int minWidth,
uint32_t partition,
uint16_t segment,
HWM& hwm,
BRM::LBID_t& lbid,
bool& bSkippedToNewExtent);
int minWidth,
uint32_t partition,
uint16_t segment,
HWM& hwm,
BRM::LBID_t& lbid,
bool& bSkippedToNewExtent);
// Rollback any tables that are left in a locked state at EOJ.
int rollbackLockedTables( );
@ -278,14 +285,14 @@ private:
// Save metadata info required for shared-nothing bulk rollback.
int saveBulkRollbackMetaData( Job& job, // current job
TableInfo* tableInfo, // TableInfo for table of interest
const std::vector<DBRootExtentInfo>& segFileInfo, //vector seg file info
const std::vector<BRM::EmDbRootHWMInfo_v>& dbRootHWMInfoPM);
TableInfo* tableInfo, // TableInfo for table of interest
const std::vector<DBRootExtentInfo>& segFileInfo, //vector seg file info
const std::vector<BRM::EmDbRootHWMInfo_v>& dbRootHWMInfoPM);
// Manage/validate the list of 1 or more import data files
int manageImportDataFileList(Job& job, // current job
int tableNo, // table number of current job
TableInfo* tableInfo); // TableInfo for table of interest
int tableNo, // table number of current job
TableInfo* tableInfo); // TableInfo for table of interest
// Break up list of file names into a vector of filename strings
int buildImportDataFileList(
@ -297,119 +304,187 @@ private:
//------------------------------------------------------------------------------
// Inline functions
//------------------------------------------------------------------------------
inline void BulkLoad::addToCmdLineImportFileList(const std::string& importFile){
fCmdLineImportFiles.push_back( importFile ); }
inline void BulkLoad::addToCmdLineImportFileList(const std::string& importFile)
{
fCmdLineImportFiles.push_back( importFile );
}
inline const std::string& BulkLoad::getAlternateImportDir( ) const {
return fAlternateImportDir; }
inline const std::string& BulkLoad::getAlternateImportDir( ) const
{
return fAlternateImportDir;
}
inline const std::string& BulkLoad::getErrorDir( ) const {
return fErrorDir; }
inline const std::string& BulkLoad::getErrorDir( ) const
{
return fErrorDir;
}
inline const std::string& BulkLoad::getJobDir( ) const {
return DIR_BULK_JOB; }
inline const std::string& BulkLoad::getJobDir( ) const
{
return DIR_BULK_JOB;
}
inline const std::string& BulkLoad::getSchema( ) const {
return fJobInfo.getJob().schema; }
inline const std::string& BulkLoad::getSchema( ) const
{
return fJobInfo.getJob().schema;
}
inline const std::string& BulkLoad::getTempJobDir( ) const {
return DIR_BULK_TEMP_JOB; }
inline const std::string& BulkLoad::getTempJobDir( ) const
{
return DIR_BULK_TEMP_JOB;
}
inline bool BulkLoad::getTruncationAsError ( ) const {
return fbTruncationAsError; }
inline bool BulkLoad::getTruncationAsError ( ) const
{
return fbTruncationAsError;
}
inline BulkModeType BulkLoad::getBulkLoadMode ( ) const {
return fBulkMode; }
inline BulkModeType BulkLoad::getBulkLoadMode ( ) const
{
return fBulkMode;
}
inline bool BulkLoad::getContinue ( ) const {
return fbContinue; }
inline bool BulkLoad::getContinue ( ) const
{
return fbContinue;
}
inline void BulkLoad::printJob() {
inline void BulkLoad::printJob()
{
if (isDebug(DEBUG_1))
fJobInfo.printJobInfo(fLog);
else
fJobInfo.printJobInfoBrief(fLog); }
fJobInfo.printJobInfoBrief(fLog);
}
inline void BulkLoad::setAllDebug( DebugLevel level ) {
inline void BulkLoad::setAllDebug( DebugLevel level )
{
setDebugLevel( level );
fLog.setDebugLevel( level ); }
fLog.setDebugLevel( level );
}
inline void BulkLoad::setColDelimiter( char delim ) {
fColDelim = delim; }
inline void BulkLoad::setColDelimiter( char delim )
{
fColDelim = delim;
}
inline void BulkLoad::setBulkLoadMode(
BulkModeType bulkMode,
const std::string& rptFileName ) {
const std::string& rptFileName )
{
fBulkMode = bulkMode;
fBRMRptFileName = rptFileName; }
fBRMRptFileName = rptFileName;
}
inline void BulkLoad::setEnclosedByChar( char enChar ) {
fEnclosedByChar = enChar; }
inline void BulkLoad::setEnclosedByChar( char enChar )
{
fEnclosedByChar = enChar;
}
inline void BulkLoad::setEscapeChar( char esChar ) {
fEscapeChar = esChar; }
inline void BulkLoad::setEscapeChar( char esChar )
{
fEscapeChar = esChar;
}
inline void BulkLoad::setImportDataMode(ImportDataMode importMode) {
fImportDataMode = importMode; }
inline void BulkLoad::setImportDataMode(ImportDataMode importMode)
{
fImportDataMode = importMode;
}
inline void BulkLoad::setKeepRbMetaFiles( bool keepMeta ) {
fKeepRbMetaFiles = keepMeta; }
inline void BulkLoad::setKeepRbMetaFiles( bool keepMeta )
{
fKeepRbMetaFiles = keepMeta;
}
// Mutator takes an unsigned int, but we store in a long long, because...
// TableInfo which eventually needs this attribute, takes an unsigned int,
// but we want to be able to init to -1, to indicate when it has not been set.
inline void BulkLoad::setMaxErrorCount( unsigned int maxErrors ) {
fMaxErrors = maxErrors; }
inline void BulkLoad::setMaxErrorCount( unsigned int maxErrors )
{
fMaxErrors = maxErrors;
}
inline void BulkLoad::setNoOfParseThreads(int parseThreads ) {
fNoOfParseThreads = parseThreads; }
inline void BulkLoad::setNoOfParseThreads(int parseThreads )
{
fNoOfParseThreads = parseThreads;
}
inline void BulkLoad::setNoOfReadThreads( int readThreads ) {
fNoOfReadThreads = readThreads; }
inline void BulkLoad::setNoOfReadThreads( int readThreads )
{
fNoOfReadThreads = readThreads;
}
inline void BulkLoad::setNullStringMode( bool bMode ) {
fNullStringMode = bMode; }
inline void BulkLoad::setNullStringMode( bool bMode )
{
fNullStringMode = bMode;
}
inline void BulkLoad::setParserNum( int parser ) {
fNumOfParser = parser; }
inline void BulkLoad::setParserNum( int parser )
{
fNumOfParser = parser;
}
inline void BulkLoad::setProcessName( const std::string& processName ) {
fProcessName = processName; }
inline void BulkLoad::setProcessName( const std::string& processName )
{
fProcessName = processName;
}
inline void BulkLoad::setReadBufferCount( int noOfReadBuffers ) {
fNoOfBuffers = noOfReadBuffers; }
inline void BulkLoad::setReadBufferCount( int noOfReadBuffers )
{
fNoOfBuffers = noOfReadBuffers;
}
inline void BulkLoad::setReadBufferSize( int readBufferSize ) {
fBufferSize = readBufferSize; }
inline void BulkLoad::setReadBufferSize( int readBufferSize )
{
fBufferSize = readBufferSize;
}
inline void BulkLoad::setTxnID( BRM::TxnID txnID ) {
fTxnID = txnID; }
inline void BulkLoad::setTxnID( BRM::TxnID txnID )
{
fTxnID = txnID;
}
inline void BulkLoad::setVbufReadSize( int vbufReadSize ) {
fFileVbufSize = vbufReadSize; }
inline void BulkLoad::setVbufReadSize( int vbufReadSize )
{
fFileVbufSize = vbufReadSize;
}
inline void BulkLoad::setTruncationAsError(bool bTruncationAsError) {
fbTruncationAsError = bTruncationAsError; }
inline void BulkLoad::setTruncationAsError(bool bTruncationAsError)
{
fbTruncationAsError = bTruncationAsError;
}
inline void BulkLoad::setErrorDir( const std::string& errorDir ) {
fErrorDir = errorDir; }
inline void BulkLoad::setErrorDir( const std::string& errorDir )
{
fErrorDir = errorDir;
}
inline void BulkLoad::startTimer( ) {
gettimeofday( &fStartTime, 0 ); }
inline void BulkLoad::startTimer( )
{
gettimeofday( &fStartTime, 0 );
}
inline void BulkLoad::stopTimer() {
inline void BulkLoad::stopTimer()
{
gettimeofday( &fEndTime, 0 );
fTotalTime = (fEndTime.tv_sec + (fEndTime.tv_usec / 1000000.0)) -
(fStartTime.tv_sec + (fStartTime.tv_usec / 1000000.0)); }
(fStartTime.tv_sec + (fStartTime.tv_usec / 1000000.0));
}
inline double BulkLoad::getTotalRunTime() const {
return fTotalTime; }
inline void BulkLoad::disableTimeOut( const bool disableTimeOut) {
fDisableTimeOut = disableTimeOut; }
inline double BulkLoad::getTotalRunTime() const
{
return fTotalTime;
}
inline bool BulkLoad::disableTimeOut() const {
return fDisableTimeOut; }
inline void BulkLoad::disableTimeOut( const bool disableTimeOut)
{
fDisableTimeOut = disableTimeOut;
}
inline bool BulkLoad::disableTimeOut() const
{
return fDisableTimeOut;
}
} // end of namespace

806
writeengine/bulk/we_bulkloadbuffer.cpp
View File

File diff suppressed because it is too large Load Diff

175
writeengine/bulk/we_bulkloadbuffer.h
View File

@ -31,14 +31,14 @@
#include "we_columninfo.h"
#include "calpontsystemcatalog.h"
namespace WriteEngine
namespace WriteEngine
{
class Log;
// Used to collect stats about a BulkLoadBuffer buffer that is being parsed
class BLBufferStats
{
public:
public:
int64_t minBufferVal;
int64_t maxBufferVal;
int64_t satCount;
@ -59,16 +59,16 @@ class BLBufferStats
class BulkLoadBuffer
{
private:
private:
//--------------------------------------------------------------------------
// Private Data Members
//--------------------------------------------------------------------------
char *fData; // Buffer with data read from tbl file
char *fDataParser; // for temporary use by parser
char* fData; // Buffer with data read from tbl file
char* fDataParser; // for temporary use by parser
char *fOverflowBuf; // Overflow data held for next buffer
char* fOverflowBuf; // Overflow data held for next buffer
unsigned fOverflowSize; // Current size of fOverflowBuf
// Information about the locker and status for each column in this buffer.
@ -88,38 +88,38 @@ private:
unsigned fParseComplete; // Num of columns that are parseComplete
unsigned fTotalRows; // Max rows this buffer can now hold;
// size of fTokens array
std::vector< std::pair<RID,std::string> > fRowStatus;//Status of bad rows
// size of fTokens array
std::vector< std::pair<RID, std::string> > fRowStatus; //Status of bad rows
std::vector<std::string> fErrRows; // Rejected rows to write to .bad file
uint32_t fTotalReadRows; // Total valid rows read into buffer;
// this count excludes rejected rows
// this count excludes rejected rows
uint32_t fTotalReadRowsParser; // for temporary use by parser
uint32_t fTotalReadRowsForLog; // Total rows read into this buffer
// including invalid rows
// including invalid rows
RID fStartRow; // Starting row id for rows in buffer,
// relative to start of job.
// Rejected rows are excluded.
// relative to start of job.
// Rejected rows are excluded.
RID fStartRowParser; // for temporary use by parser
RID fStartRowForLogging; // Starting row id for rows in buffer,
// relative to start of current input
// file. All rows are counted.
// relative to start of current input
// file. All rows are counted.
RID fStartRowForLoggingParser; // for temporary use by parser
uint32_t fAutoIncGenCount; // How many auto-increment values are
// to be generated for current buffer
// to be generated for current buffer
uint32_t fAutoIncGenCountParser; // for temporary use by parser
uint64_t fAutoIncNextValue; // Next auto-increment value assign to
// a row in this buffer
// a row in this buffer
unsigned fNumberOfColumns; // Number of ColumnInfo objs in table
ColPosPair **fTokens; // Vector of start and offsets for the
// column values read from tbl files
ColPosPair **fTokensParser; // for temporary use by parser
ColPosPair** fTokens; // Vector of start and offsets for the
// column values read from tbl files
ColPosPair** fTokensParser; // for temporary use by parser
char fColDelim; // Character to delimit columns in a row
unsigned fBufferSize; // Size of input read buffer (fData)
@ -127,16 +127,16 @@ private:
boost::mutex fSyncUpdatesBLB; // Mutex to synchronize updates
Log* fLog; // Logger object
bool fNullStringMode; // Indicates if "NULL" string is to be
// treated as a NULL value or not
// treated as a NULL value or not
char fEnclosedByChar; // Optional char to enclose col values
char fEscapeChar; // Used to escape enclosed character
int fBufferId; // Id for this read buffer
std::string fTableName; // Table assigned to this read buffer
JobFieldRefList fFieldList; // Complete list of cols and flds
unsigned int fNumFieldsInFile; // Number of fields in input file
// (including fields to be ignored)
// (including fields to be ignored)
unsigned int fNumColsInFile; // Number of flds in input file targeted
// for db cols (omits default cols)
// for db cols (omits default cols)
bool fbTruncationAsError; // Treat string truncation as error
ImportDataMode fImportDataMode; // Import data in text or binary mode
unsigned int fFixedBinaryRecLen; // Fixed rec len used in binary mode
@ -147,26 +147,26 @@ private:
/** @brief Copy constructor
*/
BulkLoadBuffer(const BulkLoadBuffer &buffer);
BulkLoadBuffer(const BulkLoadBuffer& buffer);
/** @brief Assignment operator
*/
BulkLoadBuffer & operator =(const BulkLoadBuffer & buffer);
BulkLoadBuffer& operator =(const BulkLoadBuffer& buffer);
/** @brief Convert the buffer data depending upon the data type
*/
void convert(char *field, int fieldLength,
bool nullFlag, unsigned char *output,
const JobColumn & column,
void convert(char* field, int fieldLength,
bool nullFlag, unsigned char* output,
const JobColumn& column,
BLBufferStats& bufStats);
/** @brief Copy the overflow data
*/
void copyOverflow(const BulkLoadBuffer & buffer);
void copyOverflow(const BulkLoadBuffer& buffer);
/** @brief Parse a Read buffer for a nonDictionary column
*/
int parseCol(ColumnInfo &columnInfo);
int parseCol(ColumnInfo& columnInfo);
/** @brief Parse a Read buffer for a nonDictionary column
*/
@ -177,10 +177,10 @@ private:
/** @brief Parse a Read buffer for a Dictionary column
*/
int parseDict(ColumnInfo &columnInfo);
int parseDict(ColumnInfo& columnInfo);
/** @brief Parse a Dictionary Read buffer into a ColumnBufferSection.
*
*
* Parses the Read buffer into a section up to the point at which
* the buffer crosses an extent boundary.
*
@ -191,7 +191,7 @@ private:
* @param totalReadRows Number of buffer rows ready to be parsed
* @param nRowsParsed Number of buffer rows that were parsed
*/
int parseDictSection(ColumnInfo &columnInfo, int tokenPos,
int parseDictSection(ColumnInfo& columnInfo, int tokenPos,
RID startRow, uint32_t totalReadRows,
uint32_t& nRowsParsed);
@ -207,13 +207,13 @@ private:
/** @brief Binary tokenization of the buffer, and fill up the token array.
*/
int tokenizeBinary(const boost::ptr_vector<ColumnInfo>& columnsInfo,
unsigned int allowedErrCntThisCall,
bool bEndOfData);
unsigned int allowedErrCntThisCall,
bool bEndOfData);
/** @brief Determine if specified value is NULL or not.
*/
bool isBinaryFieldNull(void* val, WriteEngine::ColType ct,
execplan::CalpontSystemCatalog::ColDataType dt);
execplan::CalpontSystemCatalog::ColDataType dt);
public:
@ -239,93 +239,128 @@ public:
void reset();
/** @brief Resets the column locks.
* TableInfo::fSyncUpdatesTI mutex should be locked when calling this
* TableInfo::fSyncUpdatesTI mutex should be locked when calling this
* function (see fColumnLocks discussion).
*/
void resetColumnLocks();
/** @brief Get the buffer status
*/
Status getStatusBLB() const {return fStatusBLB;}
Status getStatusBLB() const
{
return fStatusBLB;
}
/** @brief Set the buffer status
*/
void setStatusBLB(const Status & status){fStatusBLB = status;}
void setStatusBLB(const Status& status)
{
fStatusBLB = status;
}
/** @brief Try to lock a column for the buffer
* TableInfo::fSyncUpdatesTI mutex should be locked when calling this
* TableInfo::fSyncUpdatesTI mutex should be locked when calling this
* function (see fColumnLocks discussion).
*/
bool tryAndLockColumn(const int & columnId, const int & id);
bool tryAndLockColumn(const int& columnId, const int& id);
/** @brief Read the table data into the buffer
*/
int fillFromFile(const BulkLoadBuffer& overFlowBufIn,
FILE * handle, RID & totalRows, RID & correctTotalRows,
const boost::ptr_vector<ColumnInfo>& columnsInfo,
unsigned int allowedErrCntThisCall);
FILE* handle, RID& totalRows, RID& correctTotalRows,
const boost::ptr_vector<ColumnInfo>& columnsInfo,
unsigned int allowedErrCntThisCall);
/** @brief Get the overflow size
*/
int getOverFlowSize() const {return fOverflowSize;}
int getOverFlowSize() const
{
return fOverflowSize;
}
/** @brief Parse the buffer data
*/
int parse(ColumnInfo &columnInfo);
int parse(ColumnInfo& columnInfo);
/** @brief Set the delimiter used to delimit the columns within a row
*/
void setColDelimiter(const char & delim){fColDelim = delim;}
void setColDelimiter(const char& delim)
{
fColDelim = delim;
}
/** @brief Set mode to treat "NULL" string as NULL value or not.
*/
void setNullStringMode( bool bMode ) { fNullStringMode = bMode; }
void setNullStringMode( bool bMode )
{
fNullStringMode = bMode;
}
/** @brief Set character optionally used to enclose input column values.
*/
void setEnclosedByChar( char enChar ) { fEnclosedByChar = enChar; }
void setEnclosedByChar( char enChar )
{
fEnclosedByChar = enChar;
}
/** @brief Set escape char to use in conjunction with enclosed by char.
*/
void setEscapeChar ( char esChar ) { fEscapeChar = esChar; }
void setEscapeChar ( char esChar )
{
fEscapeChar = esChar;
}
/** @brief Get the column status
* TableInfo::fSyncUpdatesTI mutex should be locked when calling this
* TableInfo::fSyncUpdatesTI mutex should be locked when calling this
* function (see fColumnLocks discussion).
*/
Status getColumnStatus(const int & columnId) const
{ return fColumnLocks[columnId].status; }
Status getColumnStatus(const int& columnId) const
{
return fColumnLocks[columnId].status;
}
/** @brief Set the column status
* TableInfo::fSyncUpdatesTI mutex should be locked when calling this
* TableInfo::fSyncUpdatesTI mutex should be locked when calling this
* function (see fColumnLocks discussion).
* @returns TRUE if all columns in the buffer are complete.
*/
bool setColumnStatus(const int &columnId, const Status & status);
bool setColumnStatus(const int& columnId, const Status& status);
/** @brief Get the error row status's
*/
const std::vector< std::pair<RID,std::string> >& getErrorRows() const
{return fRowStatus;}
const std::vector< std::pair<RID, std::string> >& getErrorRows() const
{
return fRowStatus;
}
/** @brief Get the error rows
*/
const std::vector<std::string>& getExactErrorRows() const
{return fErrRows;}
void clearErrRows() {fRowStatus.clear();fErrRows.clear();}
{
return fErrRows;
}
void clearErrRows()
{
fRowStatus.clear();
fErrRows.clear();
}
/** @brief Get the column locker.
* TableInfo::fSyncUpdatesTI mutex should be locked when calling this
* TableInfo::fSyncUpdatesTI mutex should be locked when calling this
* function (see fColumnLocks discussion).
*/
const int getColumnLocker(const int & columnId) const
{ return fColumnLocks[columnId].locker; }
const int getColumnLocker(const int& columnId) const
{
return fColumnLocks[columnId].locker;
}
/** @brief set truncation as error for this import.
*/
void setTruncationAsError(bool bTruncationAsError)
{ fbTruncationAsError = bTruncationAsError; }
void setTruncationAsError(bool bTruncationAsError)
{
fbTruncationAsError = bTruncationAsError;
}
/** @brief retrieve the tuncation as error setting for this
* import. When set, this causes char and varchar strings
@ -333,15 +368,19 @@ public:
* as errors instead of warnings.
*/
bool getTruncationAsError() const
{ return fbTruncationAsError; }
{
return fbTruncationAsError;
}
/** @brief Set text vs binary import mode along with corresponding fixed
* record length that is used if the binary mode is set to TRUE.
*/
void setImportDataMode( ImportDataMode importMode,
unsigned int fixedBinaryRecLen )
{ fImportDataMode = importMode;
fFixedBinaryRecLen = fixedBinaryRecLen; }
unsigned int fixedBinaryRecLen )
{
fImportDataMode = importMode;
fFixedBinaryRecLen = fixedBinaryRecLen;
}
};
}

4
writeengine/bulk/we_bulkstatus.cpp
View File

@ -29,6 +29,6 @@
namespace WriteEngine
{
/*static*/
volatile int BulkStatus::fJobStatus = EXIT_SUCCESS;
/*static*/
volatile int BulkStatus::fJobStatus = EXIT_SUCCESS;
}

12
writeengine/bulk/we_bulkstatus.h
View File

@ -39,8 +39,14 @@ namespace WriteEngine
class BulkStatus
{
public:
static int getJobStatus() { return fJobStatus; }
static void setJobStatus(int jobStatus) { fJobStatus = jobStatus; }
static int getJobStatus()
{
return fJobStatus;
}
static void setJobStatus(int jobStatus)
{
fJobStatus = jobStatus;
}
private:
/* @brief Global job status flag.
@ -49,7 +55,7 @@ private:
* as a flag. Making the variable volatile should suffice, to make it
* work with multiple threads.
*/
static volatile int fJobStatus;
static volatile int fJobStatus;
};
} // end of namespace

15
writeengine/bulk/we_colbuf.cpp
View File

@ -33,7 +33,8 @@
#include "IDBDataFile.h"
using namespace idbdatafile;
namespace WriteEngine {
namespace WriteEngine
{
ColumnBuffer::ColumnBuffer(ColumnInfo* pColInfo, Log* logger) :
fBuffer(0), fBufSize(0), fFile(0), fColInfo(pColInfo), fLog(logger)
@ -72,16 +73,18 @@ int ColumnBuffer::setDbFile(IDBDataFile* f, HWM startHwm, const char* /*hdrs*/)
//------------------------------------------------------------------------------
void ColumnBuffer::resizeAndCopy(int newSize, int startOffset, int endOffset)
{
unsigned char *old_buffer = fBuffer;
unsigned char* old_buffer = fBuffer;
fBuffer = new unsigned char[newSize];
if (startOffset != -1) {
if (startOffset != -1)
{
int destBufferOffset = 0;
// If the data in "buffer" wraps around, then here's where we copy
// the data at the end of the buffer, before copying the data
// at the start of the bufffer.
if(endOffset < startOffset) {
if (endOffset < startOffset)
{
memcpy(fBuffer, old_buffer + startOffset, fBufSize - startOffset);
destBufferOffset = fBufSize - startOffset;
startOffset = 0;
@ -107,8 +110,10 @@ int ColumnBuffer::writeToFile(int startOffset, int writeSize)
Stats::startParseEvent(WE_STATS_WRITE_COL);
#endif
size_t nitems = fFile->write(fBuffer + startOffset, writeSize) / writeSize;
if (nitems != 1)
return ERR_FILE_WRITE;
#ifdef PROFILE
Stats::stopParseEvent(WE_STATS_WRITE_COL);
#endif
@ -119,7 +124,7 @@ int ColumnBuffer::writeToFile(int startOffset, int writeSize)
//------------------------------------------------------------------------------
// Add data to output buffer
//------------------------------------------------------------------------------
void ColumnBuffer::write(const void * const data, int startOffset, int bytes) const
void ColumnBuffer::write(const void* const data, int startOffset, int bytes) const
{
memcpy(fBuffer + startOffset, data, bytes);
}

21
writeengine/bulk/we_colbuf.h
View File

@ -31,7 +31,8 @@
#include "we_type.h"
namespace WriteEngine {
namespace WriteEngine
{
class Log;
struct ColumnInfo;
@ -41,9 +42,10 @@ struct ColumnInfo;
* writing it out to the intended destination(currently a file stream). The
* file stream should be initialized by the client of this class
*/
class ColumnBuffer {
class ColumnBuffer
{
public:
public:
/** @brief default Constructor
*/
@ -62,7 +64,10 @@ class ColumnBuffer {
/** @brief Returns size of the buffer
*/
int getSize() const { return fBufSize; }
int getSize() const
{
return fBufSize;
}
/** @brief Reset the ColBuf to-be-compressed buffer prior to importing the
* next extent. This is a no-op for uncompressed columns.
@ -71,12 +76,12 @@ class ColumnBuffer {
virtual int resetToBeCompressedColBuf( long long& startFileOffset );
/** @brief Set the IDBDataFile* destination for the applicable col segment file.
*
*
* @param The destination IDBDataFile stream to which buffer data will be written
* @param Starting HWM for cFile
* @param Headers with ptr information (only applies to compressed files)
*/
virtual int setDbFile(IDBDataFile * const cFile, HWM startHwm, const char* hdrs);
virtual int setDbFile(IDBDataFile* const cFile, HWM startHwm, const char* hdrs);
/** @brief Resize the buffer, also copying the section denoted by the
* offsets to the new buffer. If offsets are -1, then the buffer is
@ -96,7 +101,7 @@ class ColumnBuffer {
* @param startOffset
* @param bytes
*/
void write(const void * const data, int startOffset, int bytes) const;
void write(const void* const data, int startOffset, int bytes) const;
/** @brief Write data to FILE
*
@ -105,7 +110,7 @@ class ColumnBuffer {
*/
virtual int writeToFile(int startOffset, int writeSize);
protected:
protected:
// Disable copy constructor and assignment operator by declaring and
// not defining.

175
writeengine/bulk/we_colbufcompressed.cpp
View File

@ -48,20 +48,21 @@ using namespace idbdatafile;
#include "idbcompress.h"
using namespace compress;
namespace WriteEngine {
namespace WriteEngine
{
//------------------------------------------------------------------------------
// Constructor
//------------------------------------------------------------------------------
ColumnBufferCompressed::ColumnBufferCompressed( ColumnInfo* pColInfo,
Log* logger) :
ColumnBuffer(pColInfo, logger),
fToBeCompressedBuffer(0),
fToBeCompressedCapacity(0),
fNumBytes(0),
fCompressor(0),
fPreLoadHWMChunk(true),
fFlushedStartHwmChunk(false)
Log* logger) :
ColumnBuffer(pColInfo, logger),
fToBeCompressedBuffer(0),
fToBeCompressedCapacity(0),
fNumBytes(0),
fCompressor(0),
fPreLoadHWMChunk(true),
fFlushedStartHwmChunk(false)
{
fUserPaddingBytes = Config::getNumCompressedPadBlks() * BYTE_PER_BLOCK;
fCompressor = new compress::IDBCompressInterface( fUserPaddingBytes );
@ -74,6 +75,7 @@ ColumnBufferCompressed::~ColumnBufferCompressed()
{
if (fToBeCompressedBuffer)
delete []fToBeCompressedBuffer;
fToBeCompressedBuffer = 0;
fToBeCompressedCapacity = 0;
fNumBytes = 0;
@ -90,6 +92,7 @@ int ColumnBufferCompressed::setDbFile(IDBDataFile* f, HWM startHwm, const char*
fStartingHwm = startHwm;
IDBCompressInterface compressor;
if (compressor.getPtrList(hdrs, fChunkPtrs) != 0)
{
return ERR_COMP_PARSE_HDRS;
@ -99,10 +102,11 @@ int ColumnBufferCompressed::setDbFile(IDBDataFile* f, HWM startHwm, const char*
// rollback), that fall after the HWM, then drop those trailing ptrs.
unsigned int chunkIndex = 0;
unsigned int blockOffsetWithinChunk = 0;
fCompressor->locateBlock(fStartingHwm,chunkIndex,blockOffsetWithinChunk);
if ((chunkIndex+1) < fChunkPtrs.size())
fCompressor->locateBlock(fStartingHwm, chunkIndex, blockOffsetWithinChunk);
if ((chunkIndex + 1) < fChunkPtrs.size())
{
fChunkPtrs.resize(chunkIndex+1);
fChunkPtrs.resize(chunkIndex + 1);
}
return NO_ERROR;
@ -125,12 +129,14 @@ int ColumnBufferCompressed::resetToBeCompressedColBuf(
fToBeCompressedBuffer =
new unsigned char[IDBCompressInterface::UNCOMPRESSED_INBUF_LEN];
}
BlockOp::setEmptyBuf( fToBeCompressedBuffer,
IDBCompressInterface::UNCOMPRESSED_INBUF_LEN,
fColInfo->column.emptyVal,
fColInfo->column.width );
if (fLog->isDebug( DEBUG_2 )) {
if (fLog->isDebug( DEBUG_2 ))
{
std::ostringstream oss;
oss << "Initializing empty chunk for next extent: OID-" <<
fColInfo->curCol.dataFile.fid <<
@ -138,16 +144,17 @@ int ColumnBufferCompressed::resetToBeCompressedColBuf(
"; part-" << fColInfo->curCol.dataFile.fPartition <<
"; seg-" << fColInfo->curCol.dataFile.fSegment <<
"; hwm-" << fStartingHwm;
fLog->logMsg( oss.str(), MSGLVL_INFO2 );
fLog->logMsg( oss.str(), MSGLVL_INFO2 );
}
fToBeCompressedCapacity = IDBCompressInterface::UNCOMPRESSED_INBUF_LEN;
// Set file offset past end of last chunk
startFileOffset = IDBCompressInterface::HDR_BUF_LEN*2;
startFileOffset = IDBCompressInterface::HDR_BUF_LEN * 2;
if (fChunkPtrs.size() > 0)
startFileOffset = fChunkPtrs[ fChunkPtrs.size()-1 ].first +
fChunkPtrs[ fChunkPtrs.size()-1 ].second;
startFileOffset = fChunkPtrs[ fChunkPtrs.size() - 1 ].first +
fChunkPtrs[ fChunkPtrs.size() - 1 ].second;
// Positition ourselves to start of empty to-be-compressed buffer
fNumBytes = 0;
@ -174,6 +181,7 @@ int ColumnBufferCompressed::writeToFile(int startOffset, int writeSize)
#endif
long long startFileOffset;
int rc = initToBeCompressedBuffer( startFileOffset );
if (rc != NO_ERROR)
{
WErrorCodes ec;
@ -187,6 +195,7 @@ int ColumnBufferCompressed::writeToFile(int startOffset, int writeSize)
}
rc = fColInfo->colOp->setFileOffset(fFile, startFileOffset, SEEK_SET);
if (rc != NO_ERROR)
{
WErrorCodes ec;
@ -199,6 +208,7 @@ int ColumnBufferCompressed::writeToFile(int startOffset, int writeSize)
return rc;
}
#ifdef PROFILE
Stats::stopParseEvent(WE_STATS_COMPRESS_COL_INIT_BUF);
#endif
@ -208,8 +218,8 @@ int ColumnBufferCompressed::writeToFile(int startOffset, int writeSize)
// Expand the compression buffer size if working with an abbrev extent, and
// the bytes we are about to add will overflow the abbreviated extent.
if((fToBeCompressedCapacity<IDBCompressInterface::UNCOMPRESSED_INBUF_LEN) &&
((fNumBytes + writeSize) > fToBeCompressedCapacity) )
if ((fToBeCompressedCapacity < IDBCompressInterface::UNCOMPRESSED_INBUF_LEN) &&
((fNumBytes + writeSize) > fToBeCompressedCapacity) )
{
std::ostringstream oss;
oss << "Expanding abbrev to-be-compressed buffer for: OID-" <<
@ -223,7 +233,8 @@ int ColumnBufferCompressed::writeToFile(int startOffset, int writeSize)
if ((fNumBytes + writeSize) <= fToBeCompressedCapacity)
{
if (fLog->isDebug( DEBUG_2 )) {
if (fLog->isDebug( DEBUG_2 ))
{
std::ostringstream oss;
oss << "Buffering data to-be-compressed for: OID-" <<
fColInfo->curCol.dataFile.fid <<
@ -231,7 +242,7 @@ int ColumnBufferCompressed::writeToFile(int startOffset, int writeSize)
"; part-" << fColInfo->curCol.dataFile.fPartition <<
"; seg-" << fColInfo->curCol.dataFile.fSegment <<
"; addBytes-" << writeSize <<
"; totBytes-" << (fNumBytes+writeSize);
"; totBytes-" << (fNumBytes + writeSize);
fLog->logMsg( oss.str(), MSGLVL_INFO2 );
}
@ -251,11 +262,13 @@ int ColumnBufferCompressed::writeToFile(int startOffset, int writeSize)
idbassert( (fNumBytes <= fToBeCompressedCapacity) ); // DMC-temp debug
size_t writeSizeOut = 0;
if ((fNumBytes + writeSizeX) > fToBeCompressedCapacity)
{
writeSizeOut = fToBeCompressedCapacity - fNumBytes;
if (fLog->isDebug( DEBUG_2 )) {
if (fLog->isDebug( DEBUG_2 ))
{
std::ostringstream oss;
oss << "Buffering data (full) to-be-compressed for: OID-" <<
fColInfo->curCol.dataFile.fid <<
@ -278,6 +291,7 @@ int ColumnBufferCompressed::writeToFile(int startOffset, int writeSize)
// std::endl;
//std::cin >> resp;
int rc = compressAndFlush( false );
//std::cout << "dbg: after writeToFile->compressAndFlush" <<
// std::endl;
//std::cin >> resp;
@ -287,7 +301,7 @@ int ColumnBufferCompressed::writeToFile(int startOffset, int writeSize)
std::ostringstream oss;
oss << "writeToFile: error compressing and writing chunk "
"for OID " << fColInfo->curCol.dataFile.fid <<
"; " << ec.errorString(rc);
"; " << ec.errorString(rc);
fLog->logMsg( oss.str(), rc, MSGLVL_ERROR );
return rc;
@ -295,9 +309,9 @@ int ColumnBufferCompressed::writeToFile(int startOffset, int writeSize)
// Start over again loading a new to-be-compressed buffer
BlockOp::setEmptyBuf( fToBeCompressedBuffer,
IDBCompressInterface::UNCOMPRESSED_INBUF_LEN,
fColInfo->column.emptyVal,
fColInfo->column.width );
IDBCompressInterface::UNCOMPRESSED_INBUF_LEN,
fColInfo->column.emptyVal,
fColInfo->column.width );
fToBeCompressedCapacity =
IDBCompressInterface::UNCOMPRESSED_INBUF_LEN;
@ -309,7 +323,8 @@ int ColumnBufferCompressed::writeToFile(int startOffset, int writeSize)
{
writeSizeOut = writeSizeX;
if (fLog->isDebug( DEBUG_2 )) {
if (fLog->isDebug( DEBUG_2 ))
{
std::ostringstream oss;
oss << "Buffering data (new) to-be-compressed for: OID-" <<
fColInfo->curCol.dataFile.fid <<
@ -356,7 +371,7 @@ int ColumnBufferCompressed::writeToFile(int startOffset, int writeSize)
int ColumnBufferCompressed::compressAndFlush( bool bFinishingFile )
{
const int OUTPUT_BUFFER_SIZE = IDBCompressInterface::maxCompressedSize(fToBeCompressedCapacity) +
fUserPaddingBytes;
fUserPaddingBytes;
unsigned char* compressedOutBuf = new unsigned char[ OUTPUT_BUFFER_SIZE ];
boost::scoped_array<unsigned char> compressedOutBufPtr(compressedOutBuf);
unsigned int outputLen = OUTPUT_BUFFER_SIZE;
@ -366,10 +381,11 @@ int ColumnBufferCompressed::compressAndFlush( bool bFinishingFile )
#endif
int rc = fCompressor->compressBlock(
reinterpret_cast<char*>(fToBeCompressedBuffer),
fToBeCompressedCapacity,
compressedOutBuf,
outputLen );
reinterpret_cast<char*>(fToBeCompressedBuffer),
fToBeCompressedCapacity,
compressedOutBuf,
outputLen );
if (rc != 0)
{
return ERR_COMP_COMPRESS;
@ -377,12 +393,13 @@ int ColumnBufferCompressed::compressAndFlush( bool bFinishingFile )
// Round up the compressed chunk size
rc = fCompressor->padCompressedChunks( compressedOutBuf,
outputLen, OUTPUT_BUFFER_SIZE );
outputLen, OUTPUT_BUFFER_SIZE );
if (rc != 0)
{
return ERR_COMP_PAD_DATA;
}
#ifdef PROFILE
Stats::stopParseEvent(WE_STATS_COMPRESS_COL_COMPRESS);
Stats::startParseEvent(WE_STATS_WRITE_COL);
@ -390,13 +407,16 @@ int ColumnBufferCompressed::compressAndFlush( bool bFinishingFile )
off64_t fileOffset = fFile->tell();
size_t nitems = fFile->write(compressedOutBuf, outputLen) / outputLen;
if (nitems != 1)
return ERR_FILE_WRITE;
CompChunkPtr compChunk(
(uint64_t)fileOffset, (uint64_t)outputLen);
fChunkPtrs.push_back( compChunk );
if (fLog->isDebug( DEBUG_2 )) {
if (fLog->isDebug( DEBUG_2 ))
{
std::ostringstream oss;
oss << "Writing compressed data for: OID-" <<
fColInfo->curCol.dataFile.fid <<
@ -404,7 +424,7 @@ int ColumnBufferCompressed::compressAndFlush( bool bFinishingFile )
"; part-" << fColInfo->curCol.dataFile.fPartition <<
"; seg-" << fColInfo->curCol.dataFile.fSegment <<
"; bytes-" << outputLen <<
"; fileOffset-"<< fileOffset;
"; fileOffset-" << fileOffset;
fLog->logMsg( oss.str(), MSGLVL_INFO2 );
}
@ -424,8 +444,9 @@ int ColumnBufferCompressed::compressAndFlush( bool bFinishingFile )
//char resp;
//std::cout << "dbg: before fflush of hdrs" << std::endl;
//std::cin >> resp;
if (fFile->flush() != 0)
if (fFile->flush() != 0)
return ERR_FILE_FLUSH;
//std::cout << "dbg: after fflush of hdrs" << std::endl;
//std::cin >> resp;
fFlushedStartHwmChunk = true;
@ -437,7 +458,7 @@ int ColumnBufferCompressed::compressAndFlush( bool bFinishingFile )
if ( !bFinishingFile )
{
RETURN_ON_ERROR( fColInfo->colOp->setFileOffset(
fFile, fileOffset, SEEK_SET) );
fFile, fileOffset, SEEK_SET) );
}
}
@ -476,48 +497,50 @@ int ColumnBufferCompressed::finishFile(bool bTruncFile)
// Truncate file (if applicable) based on offset and size of last chunk
if (bTruncFile && (fChunkPtrs.size() > 0))
{
long long truncateFileSize = fChunkPtrs[fChunkPtrs.size()-1].first +
fChunkPtrs[fChunkPtrs.size()-1].second;
long long truncateFileSize = fChunkPtrs[fChunkPtrs.size() - 1].first +
fChunkPtrs[fChunkPtrs.size() - 1].second;
// @bug5769 Don't initialize extents or truncate db files on HDFS
if (idbdatafile::IDBPolicy::useHdfs())
{
std::ostringstream oss1;
oss1 << "Finished writing column file"
": OID-" << fColInfo->curCol.dataFile.fid <<
"; DBRoot-" << fColInfo->curCol.dataFile.fDbRoot <<
"; part-" << fColInfo->curCol.dataFile.fPartition <<
"; seg-" << fColInfo->curCol.dataFile.fSegment <<
"; size-" << truncateFileSize;
": OID-" << fColInfo->curCol.dataFile.fid <<
"; DBRoot-" << fColInfo->curCol.dataFile.fDbRoot <<
"; part-" << fColInfo->curCol.dataFile.fPartition <<
"; seg-" << fColInfo->curCol.dataFile.fSegment <<
"; size-" << truncateFileSize;
fLog->logMsg( oss1.str(), MSGLVL_INFO2 );
}
else
{
std::ostringstream oss1;
oss1 << "Truncating column file"
": OID-" << fColInfo->curCol.dataFile.fid <<
"; DBRoot-" << fColInfo->curCol.dataFile.fDbRoot <<
"; part-" << fColInfo->curCol.dataFile.fPartition <<
"; seg-" << fColInfo->curCol.dataFile.fSegment <<
"; size-" << truncateFileSize;
": OID-" << fColInfo->curCol.dataFile.fid <<
"; DBRoot-" << fColInfo->curCol.dataFile.fDbRoot <<
"; part-" << fColInfo->curCol.dataFile.fPartition <<
"; seg-" << fColInfo->curCol.dataFile.fSegment <<
"; size-" << truncateFileSize;
fLog->logMsg( oss1.str(), MSGLVL_INFO2 );
int rc = NO_ERROR;
if (truncateFileSize > 0)
rc = fColInfo->colOp->truncateFile( fFile, truncateFileSize );
else
rc = ERR_COMP_TRUNCATE_ZERO;//@bug3913-Catch truncate to 0 bytes
if (rc != NO_ERROR)
{
WErrorCodes ec;
std::ostringstream oss2;
oss2 << "finishFile: error truncating file for " <<
"OID " << fColInfo->curCol.dataFile.fid <<
"; DBRoot-" << fColInfo->curCol.dataFile.fDbRoot <<
"; part-" << fColInfo->curCol.dataFile.fPartition <<
"; seg-" << fColInfo->curCol.dataFile.fSegment <<
"; size-" << truncateFileSize <<
"; " << ec.errorString(rc);
"OID " << fColInfo->curCol.dataFile.fid <<
"; DBRoot-" << fColInfo->curCol.dataFile.fDbRoot <<
"; part-" << fColInfo->curCol.dataFile.fPartition <<
"; seg-" << fColInfo->curCol.dataFile.fSegment <<
"; size-" << truncateFileSize <<
"; " << ec.errorString(rc);
fLog->logMsg( oss2.str(), rc, MSGLVL_ERROR );
return rc;
@ -551,16 +574,18 @@ int ColumnBufferCompressed::finishFile(bool bTruncFile)
int ColumnBufferCompressed::saveCompressionHeaders( )
{
// Construct the header records
char hdrBuf[IDBCompressInterface::HDR_BUF_LEN*2];
char hdrBuf[IDBCompressInterface::HDR_BUF_LEN * 2];
fCompressor->initHdr( hdrBuf, fColInfo->column.compressionType );
fCompressor->setBlockCount(hdrBuf,
(fColInfo->getFileSize()/BYTE_PER_BLOCK) );
(fColInfo->getFileSize() / BYTE_PER_BLOCK) );
std::vector<uint64_t> ptrs;
for (unsigned i=0; i<fChunkPtrs.size(); i++)
for (unsigned i = 0; i < fChunkPtrs.size(); i++)
{
ptrs.push_back( fChunkPtrs[i].first );
}
unsigned lastIdx = fChunkPtrs.size() - 1;
ptrs.push_back( fChunkPtrs[lastIdx].first + fChunkPtrs[lastIdx].second );
fCompressor->storePtrs( ptrs, hdrBuf );
@ -607,12 +632,14 @@ int ColumnBufferCompressed::initToBeCompressedBuffer(long long& startFileOffset)
unsigned int chunkIndex = 0;
unsigned int blockOffsetWithinChunk = 0;
bool bSkipStartingBlks = false;
if (fPreLoadHWMChunk)
{
if (fChunkPtrs.size() > 0)
{
fCompressor->locateBlock(fStartingHwm,
chunkIndex, blockOffsetWithinChunk);
chunkIndex, blockOffsetWithinChunk);
if (chunkIndex < fChunkPtrs.size())
startFileOffset = fChunkPtrs[chunkIndex].first;
else
@ -647,11 +674,12 @@ int ColumnBufferCompressed::initToBeCompressedBuffer(long long& startFileOffset)
// Read the chunk
RETURN_ON_ERROR( fColInfo->colOp->setFileOffset(
fFile, startFileOffset, SEEK_SET) );
fFile, startFileOffset, SEEK_SET) );
char* compressedOutBuf = new char[ fChunkPtrs[chunkIndex].second ];
boost::scoped_array<char> compressedOutBufPtr(compressedOutBuf);
size_t itemsRead = fFile->read(compressedOutBuf, fChunkPtrs[chunkIndex].second) / fChunkPtrs[chunkIndex].second;
if (itemsRead != 1)
{
std::ostringstream oss;
@ -669,10 +697,11 @@ int ColumnBufferCompressed::initToBeCompressedBuffer(long long& startFileOffset)
// Uncompress the chunk into our 4MB buffer
unsigned int outLen = IDBCompressInterface::UNCOMPRESSED_INBUF_LEN;
int rc = fCompressor->uncompressBlock(
compressedOutBuf,
fChunkPtrs[chunkIndex].second,
fToBeCompressedBuffer,
outLen);
compressedOutBuf,
fChunkPtrs[chunkIndex].second,
fToBeCompressedBuffer,
outLen);
if (rc)
{
WErrorCodes ec;
@ -696,10 +725,10 @@ int ColumnBufferCompressed::initToBeCompressedBuffer(long long& startFileOffset)
// We are going to add data to, and thus re-add, the last chunk; so we
// drop it from our list.
fChunkPtrs.resize( fChunkPtrs.size()-1 );
fChunkPtrs.resize( fChunkPtrs.size() - 1 );
}
else // We have left the HWM chunk; just position file offset,
// without reading anything
// without reading anything
{
// If it's not a new buffer, we need to initialize, since we won't be
// reading in anything to overlay what's in the to-be-compressed buffer.
@ -711,7 +740,8 @@ int ColumnBufferCompressed::initToBeCompressedBuffer(long long& startFileOffset)
fColInfo->column.width );
}
if (fLog->isDebug( DEBUG_2 )) {
if (fLog->isDebug( DEBUG_2 ))
{
std::ostringstream oss;
oss << "Initializing new empty chunk: OID-" <<
fColInfo->curCol.dataFile.fid <<
@ -725,10 +755,11 @@ int ColumnBufferCompressed::initToBeCompressedBuffer(long long& startFileOffset)
fToBeCompressedCapacity = IDBCompressInterface::UNCOMPRESSED_INBUF_LEN;
// Set file offset to start after last current chunk
startFileOffset = IDBCompressInterface::HDR_BUF_LEN*2;
startFileOffset = IDBCompressInterface::HDR_BUF_LEN * 2;
if (fChunkPtrs.size() > 0)
startFileOffset = fChunkPtrs[ fChunkPtrs.size()-1 ].first +
fChunkPtrs[ fChunkPtrs.size()-1 ].second;
startFileOffset = fChunkPtrs[ fChunkPtrs.size() - 1 ].first +
fChunkPtrs[ fChunkPtrs.size() - 1 ].second;
// Position ourselves to start of empty to-be-compressed buffer.
// If we are starting the first extent on a PM, we may employ blk
@ -742,5 +773,5 @@ int ColumnBufferCompressed::initToBeCompressedBuffer(long long& startFileOffset)
return NO_ERROR;
}
}

24
writeengine/bulk/we_colbufcompressed.h
View File

@ -34,7 +34,8 @@
#include "idbcompress.h"
namespace WriteEngine {
namespace WriteEngine
{
class Log;
struct ColumnInfo;
@ -45,9 +46,10 @@ struct ColumnInfo;
* (currently a file stream). The file stream should be initialized by
* the client of this class
*/
class ColumnBufferCompressed : public ColumnBuffer {
class ColumnBufferCompressed : public ColumnBuffer
{
public:
public:
/** @brief default Constructor
*/
@ -75,7 +77,7 @@ class ColumnBufferCompressed : public ColumnBuffer {
* @param startHwm Starting HWM for cFile
* @param hdrs Headers with ptr information.
*/
virtual int setDbFile(IDBDataFile * const cFile, HWM startHwm, const char* hdrs);
virtual int setDbFile(IDBDataFile* const cFile, HWM startHwm, const char* hdrs);
/** @brief Write data to FILE
*
@ -84,7 +86,7 @@ class ColumnBufferCompressed : public ColumnBuffer {
*/
virtual int writeToFile(int startOffset, int writeSize);
private:
private:
// Disable copy constructor and assignment operator by declaring and
// not defining.
@ -94,22 +96,22 @@ class ColumnBufferCompressed : public ColumnBuffer {
// Compress and flush the to-be-compressed buffer; updates header if needed
int compressAndFlush(bool bFinishFile);
int initToBeCompressedBuffer( long long& startFileOffset);
// Initialize the to-be-compressed buffer
// Initialize the to-be-compressed buffer
int saveCompressionHeaders(); // Saves compression headers to the db file
unsigned char* fToBeCompressedBuffer; // data waiting to be compressed
size_t fToBeCompressedCapacity;//size of comp buffer;
// should always be 4MB, unless
// working with abbrev extent.
// should always be 4MB, unless
// working with abbrev extent.
size_t fNumBytes; // num Bytes in comp buffer
compress::IDBCompressInterface*
fCompressor; // data compression object
fCompressor; // data compression object
compress::CompChunkPtrList
fChunkPtrs; // col file header information
fChunkPtrs; // col file header information
bool fPreLoadHWMChunk; // preload 1st HWM chunk only
unsigned int fUserPaddingBytes; // compressed chunk padding
bool fFlushedStartHwmChunk; // have we rewritten the hdr
// for the starting HWM chunk
// for the starting HWM chunk
};
}

271
writeengine/bulk/we_colbufmgr.cpp
View File

@ -43,22 +43,24 @@
namespace
{
// Minimum time to wait for a condition, so as to periodically wake up and
// check the global job status, to see if the job needs to terminate.
const int COND_WAIT_SECONDS = 3;
// Minimum time to wait for a condition, so as to periodically wake up and
// check the global job status, to see if the job needs to terminate.
const int COND_WAIT_SECONDS = 3;
}
namespace WriteEngine {
namespace WriteEngine
{
//------------------------------------------------------------------------------
// ColumnBufferManger constructor
//------------------------------------------------------------------------------
ColumnBufferManager::ColumnBufferManager(ColumnInfo* pColInfo,
int width, Log* logger, int compressionType) :
int width, Log* logger, int compressionType) :
fBufWriteOffset(0), fBufFreeOffset(0), fResizePending(false),
fColWidth(width),
fMaxRowId(std::numeric_limits<WriteEngine::RID>::max()),
fColInfo(pColInfo), fLog(logger) {
fColInfo(pColInfo), fLog(logger)
{
if (compressionType)
fCBuf = new ColumnBufferCompressed(pColInfo, logger);
else
@ -68,7 +70,8 @@ ColumnBufferManager::ColumnBufferManager(ColumnInfo* pColInfo,
//------------------------------------------------------------------------------
// ColumnBufferManger destructor
//------------------------------------------------------------------------------
ColumnBufferManager::~ColumnBufferManager() {
ColumnBufferManager::~ColumnBufferManager()
{
if (fCBuf)
delete fCBuf;
}
@ -117,7 +120,8 @@ int ColumnBufferManager::reserveSection(
uint32_t nRowsIn,
uint32_t& secRowCnt,
ColumnBufferSection** cbs,
RID& lastInputRowInExtent) {
RID& lastInputRowInExtent)
{
#ifdef PROFILE
Stats::startParseEvent(WE_STATS_WAIT_TO_RESERVE_OUT_BUF);
#endif
@ -128,14 +132,17 @@ int ColumnBufferManager::reserveSection(
//..Ensure that ColumnBufferSection allocations are made in input row order
bool bWaitedForInSequence = false;
while (1) {
while (1)
{
RID startRowTest = (std::numeric_limits<WriteEngine::RID>::max() ==
fMaxRowId) ? 0 : fMaxRowId + 1;
if (startRowTest == startRowId)
if (startRowTest == startRowId)
break;
if (fLog->isDebug( DEBUG_3 )) {
if (fLog->isDebug( DEBUG_3 ))
{
bWaitedForInSequence = true;
std::ostringstream oss;
oss << "OID-" << fColInfo->curCol.dataFile.fid <<
@ -149,12 +156,14 @@ int ColumnBufferManager::reserveSection(
if (BulkStatus::getJobStatus() == EXIT_FAILURE)
{
throw SecondaryShutdownException( "ColumnBufferManager::"
"reserveSection(1) responding to job termination");
"reserveSection(1) responding to job termination");
}
}
if (fLog->isDebug( DEBUG_3 )) {
if (bWaitedForInSequence) {
if (fLog->isDebug( DEBUG_3 ))
{
if (bWaitedForInSequence)
{
std::ostringstream oss;
oss << "OID-" << fColInfo->curCol.dataFile.fid <<
"; Resume after waiting for in-sequence";
@ -164,12 +173,15 @@ int ColumnBufferManager::reserveSection(
//..Check/wait for any pending output buffer expansion to be completed
bool bWaitedForResize = false;
while (fResizePending) {
if (fLog->isDebug( DEBUG_3 )) {
while (fResizePending)
{
if (fLog->isDebug( DEBUG_3 ))
{
bWaitedForResize = true;
std::ostringstream oss;
oss << "OID-" << fColInfo->curCol.dataFile.fid <<
"; Waiting for pending resize";
"; Waiting for pending resize";
fLog->logMsg( oss.str(), MSGLVL_INFO2 );
}
@ -179,12 +191,14 @@ int ColumnBufferManager::reserveSection(
if (BulkStatus::getJobStatus() == EXIT_FAILURE)
{
throw SecondaryShutdownException( "ColumnBufferManager::"
"reserveSection(2) responding to job termination");
"reserveSection(2) responding to job termination");
}
}
if (fLog->isDebug( DEBUG_3 )) {
if (bWaitedForResize) {
if (fLog->isDebug( DEBUG_3 ))
{
if (bWaitedForResize)
{
std::ostringstream oss;
oss << "OID-" << fColInfo->curCol.dataFile.fid <<
"; Resume after waiting for pending resize";
@ -202,10 +216,11 @@ int ColumnBufferManager::reserveSection(
int bufferSize = fCBuf->getSize();
int remainingSpace = 0;
if(bufferSize > 0) {
if (bufferSize > 0)
{
//Calculate remaining space
remainingSpace = bufferSize -
(fBufFreeOffset + bufferSize - fBufWriteOffset) % bufferSize;
(fBufFreeOffset + bufferSize - fBufWriteOffset) % bufferSize;
}
//..Restrict the new section to the extent boundary if applicable.
@ -216,12 +231,14 @@ int ColumnBufferManager::reserveSection(
int spaceRequired = nRows * fColWidth;
if (nRows > 0) {
if (nRows > 0)
{
//..If not enough room to add nRows to output buffer, wait for pending
// sections to be released, so that we can flush and resize the buffer.
//..@bug 3456: compare to remainingSpace-1 and not remainingSpace.
// See note in function description that precedes this function.
if (spaceRequired > (remainingSpace-1)) {
if (spaceRequired > (remainingSpace - 1))
{
//#ifdef PROFILE
// Stats::startParseEvent(WE_STATS_WAIT_TO_RESIZE_OUT_BUF);
//#endif
@ -230,8 +247,10 @@ int ColumnBufferManager::reserveSection(
// Wait for all other threads to finish writing pending sections
// to the output buffer, before we resize the buffer
while(fSectionsInUse.size() > 0) {
if (fLog->isDebug( DEBUG_3 )) {
while (fSectionsInUse.size() > 0)
{
if (fLog->isDebug( DEBUG_3 ))
{
bWaitedForSectionsInUse = true;
std::ostringstream oss;
oss << "OID-" << fColInfo->curCol.dataFile.fid <<
@ -240,22 +259,26 @@ int ColumnBufferManager::reserveSection(
fSectionsInUse.size();
fLog->logMsg( oss.str(), MSGLVL_INFO2 );
}
fBufInUse.timed_wait(lock, wait_seconds);
// See if JobStatus has been set to quit by another thread
if (BulkStatus::getJobStatus() == EXIT_FAILURE)
{
throw SecondaryShutdownException( "ColumnBufferManager::"
"reserveSection(3) responding to job termination");
"reserveSection(3) responding to job termination");
}
}
//#ifdef PROFILE
// Stats::stopParseEvent(WE_STATS_WAIT_TO_RESIZE_OUT_BUF);
// Stats::startParseEvent(WE_STATS_RESIZE_OUT_BUF);
//#endif
if (fLog->isDebug( DEBUG_3 )) {
if (bWaitedForSectionsInUse) {
if (fLog->isDebug( DEBUG_3 ))
{
if (bWaitedForSectionsInUse)
{
std::ostringstream oss;
oss << "OID-" << fColInfo->curCol.dataFile.fid <<
"; Resume after waiting to resize output buffer";
@ -265,15 +288,16 @@ int ColumnBufferManager::reserveSection(
// @bug 1977 correct problem; writing extra blocks
// Flush remaining data blocks to disk "if" buffer contains data
if(bufferSize > 0) {
if (bufferSize > 0)
{
if (fBufFreeOffset != fBufWriteOffset)
RETURN_ON_ERROR( writeToFile(
(fBufFreeOffset + bufferSize - 1)%bufferSize) );
(fBufFreeOffset + bufferSize - 1) % bufferSize) );
}
resizeColumnBuffer(spaceRequired);
bufferSize = fCBuf->getSize(); // update bufferSize after resize-
// ColumnBuffer() expanded the buffer
// ColumnBuffer() expanded the buffer
fResizePending = false;
fResizeInProgress.notify_all();
//#ifdef PROFILE
@ -300,6 +324,7 @@ int ColumnBufferManager::reserveSection(
// extent. We do this even though we have not yet allocated the next
// extent from the extent map.
lastInputRowInExtent = fColInfo->lastInputRowInExtent( );
if ((startRowId + nRowsIn) > lastInputRowInExtent)
fColInfo->lastInputRowInExtentInc( );
@ -310,7 +335,8 @@ int ColumnBufferManager::reserveSection(
// Release the data in the specified ColumnBufferSection, meaning the data in
// that section is ready to be written to the database.
//------------------------------------------------------------------------------
int ColumnBufferManager::releaseSection(ColumnBufferSection* cbs) {
int ColumnBufferManager::releaseSection(ColumnBufferSection* cbs)
{
#ifdef PROFILE
Stats::startParseEvent(WE_STATS_WAIT_TO_RELEASE_OUT_BUF);
#endif
@ -319,13 +345,15 @@ int ColumnBufferManager::releaseSection(ColumnBufferSection* cbs) {
Stats::stopParseEvent(WE_STATS_WAIT_TO_RELEASE_OUT_BUF);
#endif
cbs->setStatus(WRITE_COMPLETE);
int lastWriteOffset = fBufWriteOffset;
std::list<ColumnBufferSection*>::iterator it = fSectionsInUse.begin();
if (it != fSectionsInUse.end())
{
ColumnBufferSection* cbs_temp = *it;
while (WRITE_COMPLETE == cbs_temp->getStatus())
{
lastWriteOffset = cbs_temp->getStartOffset() +
@ -333,11 +361,14 @@ int ColumnBufferManager::releaseSection(ColumnBufferSection* cbs) {
delete cbs_temp;
it = fSectionsInUse.erase(it);
if (it == fSectionsInUse.end())
break;
cbs_temp = *it;
}
}
fBufInUse.notify_all();
RETURN_ON_ERROR( writeToFile(lastWriteOffset) );
@ -349,53 +380,64 @@ int ColumnBufferManager::releaseSection(ColumnBufferSection* cbs) {
// Expand the output buffer for the column this ColumBufferManager is managing.
// "spaceRequired" indicates the number of additional bytes that are needed.
//------------------------------------------------------------------------------
void ColumnBufferManager::resizeColumnBuffer(int spaceRequired) {
void ColumnBufferManager::resizeColumnBuffer(int spaceRequired)
{
int bufferSize = fCBuf->getSize();
int bufferSizeOld = bufferSize;
int dataRemaining = (bufferSize > 0) ?
((fBufFreeOffset - fBufWriteOffset + bufferSize) % bufferSize) : 0;
((fBufFreeOffset - fBufWriteOffset + bufferSize) % bufferSize) : 0;
int resizeAction = 0;
if(0 == bufferSize) {
if (0 == bufferSize)
{
bufferSize = (int)(spaceRequired * 1.2); //Additional 20% to account
//for changes in number of rows
//because of varying line-widths
//for changes in number of rows
//because of varying line-widths
resizeAction = 1;
} else {
if(spaceRequired > bufferSize) {
}
else
{
if (spaceRequired > bufferSize)
{
bufferSize = spaceRequired * 2;
resizeAction = 2;
} else {
}
else
{
bufferSize *= 2; //Double the buffer size
resizeAction = 3;
}
}
//Round off the bufferSize to size of a disk block
if(bufferSize % BLOCK_SIZE > 0) {
bufferSize = (((int)(bufferSize/BLOCK_SIZE)) + 1) * BLOCK_SIZE;
if (bufferSize % BLOCK_SIZE > 0)
{
bufferSize = (((int)(bufferSize / BLOCK_SIZE)) + 1) * BLOCK_SIZE;
}
if (resizeAction > 0) {
if (fLog->isDebug( DEBUG_2 )) {
if (resizeAction > 0)
{
if (fLog->isDebug( DEBUG_2 ))
{
RID numRowsInBuffer = dataRemaining / fColWidth;
RID firstRid = fMaxRowId - numRowsInBuffer + 1;
std::ostringstream oss;
oss << "Resizing out buffer (case" <<
resizeAction << ") for OID-" <<
fColInfo->curCol.dataFile.fid <<
"; oldSize-" << bufferSizeOld <<
"; freeOff-" << fBufFreeOffset <<
"; writeOff-" << fBufWriteOffset<<
"; startRID-" << firstRid <<
"; rows-" << numRowsInBuffer<<
"; reqBytes-" << spaceRequired <<
"; newSize-" << bufferSize;
resizeAction << ") for OID-" <<
fColInfo->curCol.dataFile.fid <<
"; oldSize-" << bufferSizeOld <<
"; freeOff-" << fBufFreeOffset <<
"; writeOff-" << fBufWriteOffset <<
"; startRID-" << firstRid <<
"; rows-" << numRowsInBuffer <<
"; reqBytes-" << spaceRequired <<
"; newSize-" << bufferSize;
fLog->logMsg( oss.str(), MSGLVL_INFO2 );
}
}
// @bug 1977 correct problem; writing extra blocks
// If we have no data in buffer, we still call resizeAndCopy()
// to expand the buffer; we just pass -1 for the buffer offsets.
@ -408,6 +450,7 @@ void ColumnBufferManager::resizeColumnBuffer(int spaceRequired) {
int endOffset = (fBufFreeOffset + bufferSize - 1) % bufferSize;
fCBuf->resizeAndCopy(bufferSize, fBufWriteOffset, endOffset);
}
fBufFreeOffset = dataRemaining;
fBufWriteOffset = 0;
}
@ -420,14 +463,15 @@ void ColumnBufferManager::resizeColumnBuffer(int spaceRequired) {
// applicable bytes "wrap-around" in the buffer, then this function will
// perform 2 I/O operations to write the 2 noncontiguous chunks of data.
//------------------------------------------------------------------------------
int ColumnBufferManager::writeToFile(int endOffset) {
int ColumnBufferManager::writeToFile(int endOffset)
{
int bufferSize = fCBuf->getSize();
if (endOffset == fBufWriteOffset)
return NO_ERROR;
unsigned int writeSize =
(endOffset - fBufWriteOffset + bufferSize)%bufferSize + 1;
(endOffset - fBufWriteOffset + bufferSize) % bufferSize + 1;
// Don't bother writing anything if we don't at least have a BLOCK_SIZE
// set of bytes to write out; which means we need to be sure to flush
@ -436,10 +480,11 @@ int ColumnBufferManager::writeToFile(int endOffset) {
if (writeSize < BLOCK_SIZE)
return NO_ERROR;
writeSize = writeSize - writeSize%BLOCK_SIZE; //round down to mult of blksiz
writeSize = writeSize - writeSize % BLOCK_SIZE; //round down to mult of blksiz
endOffset = (fBufWriteOffset + writeSize - 1) % bufferSize;
if (fLog->isDebug( DEBUG_3 )) {
if (fLog->isDebug( DEBUG_3 ))
{
std::ostringstream oss;
oss << "Writing OID-" << fColInfo->curCol.dataFile.fid <<
"; bufWriteOff-" << fBufWriteOffset <<
@ -452,14 +497,16 @@ int ColumnBufferManager::writeToFile(int endOffset) {
// Account for circular buffer by making 2 calls to write the data,
// if we are wrapping around at the end of the buffer.
if(endOffset < fBufWriteOffset) {
if (endOffset < fBufWriteOffset)
{
RETURN_ON_ERROR( writeToFileExtentCheck(
fBufWriteOffset, bufferSize - fBufWriteOffset) );
fBufWriteOffset, bufferSize - fBufWriteOffset) );
fBufWriteOffset = 0;
}
RETURN_ON_ERROR( writeToFileExtentCheck(
fBufWriteOffset, endOffset - fBufWriteOffset + 1) );
fBufWriteOffset = (endOffset + 1)%bufferSize;
fBufWriteOffset, endOffset - fBufWriteOffset + 1) );
fBufWriteOffset = (endOffset + 1) % bufferSize;
return NO_ERROR;
}
@ -482,17 +529,19 @@ int ColumnBufferManager::writeToFile(int endOffset) {
// internal buffer, or if an abbreviated extent is expanded.
//------------------------------------------------------------------------------
int ColumnBufferManager::writeToFileExtentCheck(
uint32_t startOffset, uint32_t writeSize) {
uint32_t startOffset, uint32_t writeSize)
{
if (fLog->isDebug( DEBUG_3 )) {
if (fLog->isDebug( DEBUG_3 ))
{
std::ostringstream oss;
oss << "Col extent check: OID-" <<
fColInfo->curCol.dataFile.fid <<
"; DBRoot-" << fColInfo->curCol.dataFile.fDbRoot <<
"; part-" << fColInfo->curCol.dataFile.fPartition <<
"; seg-" << fColInfo->curCol.dataFile.fSegment <<
"; Wanting to write " << writeSize <<
" bytes, with avail space " << fColInfo->availFileSize;
fColInfo->curCol.dataFile.fid <<
"; DBRoot-" << fColInfo->curCol.dataFile.fDbRoot <<
"; part-" << fColInfo->curCol.dataFile.fPartition <<
"; seg-" << fColInfo->curCol.dataFile.fSegment <<
"; Wanting to write " << writeSize <<
" bytes, with avail space " << fColInfo->availFileSize;
fLog->logMsg( oss.str(), MSGLVL_INFO2 );
}
@ -502,13 +551,17 @@ int ColumnBufferManager::writeToFileExtentCheck(
// If extent out of space, see if this is an abbrev extent we can expand
long long availableFileSize = fColInfo->availFileSize;
if ((availableFileSize < writeSize) && (fColInfo->isAbbrevExtent())) {
if ((availableFileSize < writeSize) && (fColInfo->isAbbrevExtent()))
{
int rc = fColInfo->expandAbbrevExtent(true);
if (rc != NO_ERROR) {
if (rc != NO_ERROR)
{
WErrorCodes ec;
std::ostringstream oss;
oss << "writeToFileExtentCheck: expand extent failed: " <<
ec.errorString(rc);
ec.errorString(rc);
fLog->logMsg( oss.str(), rc, MSGLVL_ERROR );
return rc;
}
@ -516,59 +569,73 @@ int ColumnBufferManager::writeToFileExtentCheck(
availableFileSize = fColInfo->availFileSize;
}
if (availableFileSize >= writeSize) {
if (availableFileSize >= writeSize)
{
int rc = fCBuf->writeToFile(startOffset, writeSize);
if (rc != NO_ERROR) {
if (rc != NO_ERROR)
{
WErrorCodes ec;
std::ostringstream oss;
oss << "writeToFileExtentCheck: write1 extent failed: " <<
ec.errorString(rc);
ec.errorString(rc);
fLog->logMsg( oss.str(), rc, MSGLVL_ERROR );
return rc;
}
fColInfo->updateBytesWrittenCounts( writeSize );
}
else {
else
{
// We use ColumnInfo to help us add an extent to the "next"
// segment file, if needed.
// Current extent does not have enough room for buffer, so we
// have to break up the buffer into 2 extents; creating a new
// extent and switching the db column file "on-the-fly".
int writeSize1 = availableFileSize;
if (writeSize1 > 0)
{
int rc = fCBuf->writeToFile(startOffset, writeSize1);
if (rc != NO_ERROR) {
if (rc != NO_ERROR)
{
WErrorCodes ec;
std::ostringstream oss;
oss << "writeToFileExtentCheck: write2 extent failed: " <<
ec.errorString(rc);
ec.errorString(rc);
fLog->logMsg( oss.str(), rc, MSGLVL_ERROR );
return rc;
}
fColInfo->updateBytesWrittenCounts( writeSize1 );
}
int rc = fColInfo->extendColumn( true );
if (rc != NO_ERROR) {
if (rc != NO_ERROR)
{
WErrorCodes ec;
std::ostringstream oss;
oss << "writeToFileExtentCheck: extend column failed: " <<
ec.errorString(rc);
ec.errorString(rc);
fLog->logMsg( oss.str(), rc, MSGLVL_ERROR );
return rc;
}
int writeSize2 = writeSize - writeSize1;
fCBuf->writeToFile(startOffset+writeSize1, writeSize2);
if (rc != NO_ERROR) {
fCBuf->writeToFile(startOffset + writeSize1, writeSize2);
if (rc != NO_ERROR)
{
WErrorCodes ec;
std::ostringstream oss;
oss << "writeToFileExtentCheck: write3 extent failed: " <<
ec.errorString(rc);
ec.errorString(rc);
fLog->logMsg( oss.str(), rc, MSGLVL_ERROR );
return rc;
}
fColInfo->updateBytesWrittenCounts( writeSize2 );
}
@ -578,10 +645,13 @@ int ColumnBufferManager::writeToFileExtentCheck(
//------------------------------------------------------------------------------
// Flush the contents of internal fCBuf (column buffer) to disk.
//------------------------------------------------------------------------------
int ColumnBufferManager::flush( ) {
int ColumnBufferManager::flush( )
{
if(fBufFreeOffset == fBufWriteOffset) {
if (fLog->isDebug( DEBUG_2 )) {
if (fBufFreeOffset == fBufWriteOffset)
{
if (fLog->isDebug( DEBUG_2 ))
{
std::ostringstream oss;
oss << "Skipping write flush for: OID-" <<
fColInfo->curCol.dataFile.fid <<
@ -592,19 +662,23 @@ int ColumnBufferManager::flush( ) {
fBufFreeOffset;
fLog->logMsg( oss.str(), MSGLVL_INFO2 );
}
return NO_ERROR;
}
int bufferSize = fCBuf->getSize();
// Account for circular buffer by making 2 calls to write the data,
// if we are wrapping around at the end of the buffer.
if(fBufFreeOffset < fBufWriteOffset) {
if (fBufFreeOffset < fBufWriteOffset)
{
RETURN_ON_ERROR( writeToFileExtentCheck(
fBufWriteOffset, bufferSize - fBufWriteOffset) );
fBufWriteOffset, bufferSize - fBufWriteOffset) );
fBufWriteOffset = 0;
}
RETURN_ON_ERROR( writeToFileExtentCheck(
fBufWriteOffset, fBufFreeOffset - fBufWriteOffset) );
fBufWriteOffset, fBufFreeOffset - fBufWriteOffset) );
fBufWriteOffset = fBufFreeOffset;
return NO_ERROR;
@ -619,7 +693,8 @@ int ColumnBufferManager::flush( ) {
// flushed to the segment file containing the first extent. This function is
// currently only needed for Dictionary column processing.
//------------------------------------------------------------------------------
int ColumnBufferManager::intermediateFlush() {
int ColumnBufferManager::intermediateFlush()
{
boost::posix_time::seconds wait_seconds(COND_WAIT_SECONDS);
boost::mutex::scoped_lock lock(fColInfo->colMutex());
@ -628,16 +703,19 @@ int ColumnBufferManager::intermediateFlush() {
#ifdef PROFILE
Stats::startParseEvent(WE_STATS_WAIT_FOR_INTERMEDIATE_FLUSH);
#endif
while(fSectionsInUse.size() > 0) {
while (fSectionsInUse.size() > 0)
{
fBufInUse.timed_wait(lock, wait_seconds);
// See if JobStatus has been set to terminate by another thread
if (BulkStatus::getJobStatus() == EXIT_FAILURE)
{
throw SecondaryShutdownException( "ColumnBufferManager::"
"intermediateFlush() responding to job termination");
"intermediateFlush() responding to job termination");
}
}
#ifdef PROFILE
Stats::stopParseEvent(WE_STATS_WAIT_FOR_INTERMEDIATE_FLUSH);
#endif
@ -653,7 +731,8 @@ int ColumnBufferManager::intermediateFlush() {
// as sections from the output buffer are being copied to the column segment
// file(s).
//------------------------------------------------------------------------------
int ColumnBufferManager::rowsExtentCheck( int nRows, int& nRows2 ) {
int ColumnBufferManager::rowsExtentCheck( int nRows, int& nRows2 )
{
nRows2 = nRows;
return NO_ERROR;

32
writeengine/bulk/we_colbufmgr.h
View File

@ -36,8 +36,9 @@
#include "we_colbufsec.h"
#include "idbcompress.h"
namespace WriteEngine {
class Log;
namespace WriteEngine
{
class Log;
struct ColumnInfo;
@ -56,8 +57,9 @@ struct ColumnInfo;
* disk.
*/
class ColumnBufferManager {
public:
class ColumnBufferManager
{
public:
//-------------------------------------------------------------------------
// Public Functions
@ -85,7 +87,7 @@ class ColumnBufferManager {
* @param secRowCnt (out) Resulting size of the reserved section.
* May be less than nRows for the case where a group of
* dictionary rows crosses an extent boundary.
* @param ColumnBufferSection (out) ptr with info on reserved section
* @param ColumnBufferSection (out) ptr with info on reserved section
* @param lastRIDInExtent (out) last RID in relevant extent.
* @return success or fail status
*/
@ -110,7 +112,10 @@ class ColumnBufferManager {
* @param bTruncFile is file to be truncated
* @return NO_ERROR or success
*/
int finishFile(bool bTruncFile) { return fCBuf->finishFile(bTruncFile); }
int finishFile(bool bTruncFile)
{
return fCBuf->finishFile(bTruncFile);
}
/** @brief Method to ensure that all the data in the buffer has been
* written to the file (for uncompressed), or the to-be-compressed buffer
@ -136,19 +141,23 @@ class ColumnBufferManager {
* @param hdrs with ptr information (only applies to compressed files)
*/
int setDbFile(IDBDataFile* const cFile, HWM hwm, const char* hdrs)
{ return fCBuf->setDbFile(cFile, hwm, hdrs); }
{
return fCBuf->setDbFile(cFile, hwm, hdrs);
}
/** @brief Reset the ColBuf to-be-compressed buffer prior to importing the
* next extent.
*/
int resetToBeCompressedColBuf(long long& startFileOffset)
{ return fCBuf->resetToBeCompressedColBuf( startFileOffset ); }
{
return fCBuf->resetToBeCompressedColBuf( startFileOffset );
}
/** @brief Wrapper around extendColumn(), used for dictionary token columns.
*/
int extendTokenColumn( );
protected:
protected:
//-------------------------------------------------------------------------
// Protected Functions
@ -251,9 +260,10 @@ class ColumnBufferManager {
//------------------------------------------------------------------------------
// Specialization of ColumnBufferManager that is used for Dictionary columns.
//------------------------------------------------------------------------------
class ColumnBufferManagerDctnry : public ColumnBufferManager {
class ColumnBufferManagerDctnry : public ColumnBufferManager
{
public:
public:
ColumnBufferManagerDctnry(ColumnInfo* pColInfo, int colWidth,
Log* logger, int compressionType);
virtual ~ColumnBufferManagerDctnry();

64
writeengine/bulk/we_colbufmgrdctnry.cpp
View File

@ -26,7 +26,8 @@
#include <iostream>
#include <sstream>
namespace WriteEngine {
namespace WriteEngine
{
//------------------------------------------------------------------------------
// ColumnBufferManagerDctnry constructor that takes a ColumnInfo, colWidth, and
@ -62,22 +63,24 @@ int ColumnBufferManagerDctnry::writeToFileExtentCheck(
{
std::ostringstream oss;
oss << "Dctnry writeToFileExtentCheck"
": OID-" << fColInfo->curCol.dataFile.fid <<
"; DBRoot-" << fColInfo->curCol.dataFile.fDbRoot <<
"; part-" << fColInfo->curCol.dataFile.fPartition <<
"; seg-" << fColInfo->curCol.dataFile.fSegment <<
"; writeSize-" << writeSize <<
"; oldAvailFileSize-" << fColInfo->availFileSize <<
"; newAvailFileSize-" << (fColInfo->availFileSize - writeSize);
": OID-" << fColInfo->curCol.dataFile.fid <<
"; DBRoot-" << fColInfo->curCol.dataFile.fDbRoot <<
"; part-" << fColInfo->curCol.dataFile.fPartition <<
"; seg-" << fColInfo->curCol.dataFile.fSegment <<
"; writeSize-" << writeSize <<
"; oldAvailFileSize-" << fColInfo->availFileSize <<
"; newAvailFileSize-" << (fColInfo->availFileSize - writeSize);
fLog->logMsg( oss.str(), MSGLVL_INFO2 );
}
int rc = fCBuf->writeToFile(startOffset, writeSize);
if (rc != NO_ERROR) {
if (rc != NO_ERROR)
{
WErrorCodes ec;
std::ostringstream oss;
oss << "writeToFileExtentCheck: write token extent failed: " <<
ec.errorString(rc);
ec.errorString(rc);
fLog->logMsg( oss.str(), rc, MSGLVL_ERROR );
return rc;
}
@ -106,12 +109,13 @@ int ColumnBufferManagerDctnry::rowsExtentCheck( int nRows, int& nRows2 )
int bufferSize = fCBuf->getSize();
long long spaceRequired = nRows * fColWidth;
long dataInBuffer = 0;
if (bufferSize > 0)
dataInBuffer = (fBufFreeOffset - fBufWriteOffset + bufferSize) % bufferSize;
// if extent is out of space, see if this is an abbrev extent we can expand
if (((dataInBuffer + spaceRequired) > fColInfo->availFileSize) &&
(fColInfo->isAbbrevExtent()))
(fColInfo->isAbbrevExtent()))
{
RETURN_ON_ERROR( fColInfo->expandAbbrevExtent(true) )
}
@ -125,22 +129,22 @@ int ColumnBufferManagerDctnry::rowsExtentCheck( int nRows, int& nRows2 )
{
std::ostringstream oss1;
oss1 << "Dctnry rowsExtentCheck (filling extent): OID-" <<
fColInfo->curCol.dataFile.fid <<
"; DBRoot-" << fColInfo->curCol.dataFile.fDbRoot <<
"; part-" << fColInfo->curCol.dataFile.fPartition <<
"; seg-" << fColInfo->curCol.dataFile.fSegment <<
"; spaceRequired-" << spaceRequired <<
"; dataInBuffer-" << dataInBuffer <<
"; availSpace-" << fColInfo->availFileSize;
fColInfo->curCol.dataFile.fid <<
"; DBRoot-" << fColInfo->curCol.dataFile.fDbRoot <<
"; part-" << fColInfo->curCol.dataFile.fPartition <<
"; seg-" << fColInfo->curCol.dataFile.fSegment <<
"; spaceRequired-" << spaceRequired <<
"; dataInBuffer-" << dataInBuffer <<
"; availSpace-" << fColInfo->availFileSize;
fLog->logMsg( oss1.str(), MSGLVL_INFO2 );
std::ostringstream oss2;
oss2 << "Dctnry rowsExtentCheck: OID-" <<
fColInfo->curCol.dataFile.fid <<
"; DBRoot-" << fColInfo->curCol.dataFile.fDbRoot <<
"; part-" << fColInfo->curCol.dataFile.fPartition <<
"; seg-" << fColInfo->curCol.dataFile.fSegment <<
"; Changing nRows from " << nRows << " to " << nRows2;
fColInfo->curCol.dataFile.fid <<
"; DBRoot-" << fColInfo->curCol.dataFile.fDbRoot <<
"; part-" << fColInfo->curCol.dataFile.fPartition <<
"; seg-" << fColInfo->curCol.dataFile.fSegment <<
"; Changing nRows from " << nRows << " to " << nRows2;
fLog->logMsg( oss2.str(), MSGLVL_INFO2 );
}
}
@ -150,13 +154,13 @@ int ColumnBufferManagerDctnry::rowsExtentCheck( int nRows, int& nRows2 )
{
std::ostringstream oss;
oss << "Dctnry rowsExtentCheck: OID-" <<
fColInfo->curCol.dataFile.fid <<
"; DBRoot-" << fColInfo->curCol.dataFile.fDbRoot <<
"; part-" << fColInfo->curCol.dataFile.fPartition <<
"; seg-" << fColInfo->curCol.dataFile.fSegment <<
"; spaceRequired-" << spaceRequired <<
"; dataInBuffer-" << dataInBuffer <<
"; availSpace-" << fColInfo->availFileSize;
fColInfo->curCol.dataFile.fid <<
"; DBRoot-" << fColInfo->curCol.dataFile.fDbRoot <<
"; part-" << fColInfo->curCol.dataFile.fPartition <<
"; seg-" << fColInfo->curCol.dataFile.fSegment <<
"; spaceRequired-" << spaceRequired <<
"; dataInBuffer-" << dataInBuffer <<
"; availSpace-" << fColInfo->availFileSize;
fLog->logMsg( oss.str(), MSGLVL_INFO2 );
}
}

64
writeengine/bulk/we_colbufsec.cpp
View File

@ -26,61 +26,73 @@
#include "we_colbufsec.h"
namespace WriteEngine {
namespace WriteEngine
{
ColumnBufferSection::ColumnBufferSection(
ColumnBuffer * const cb,
ColumnBuffer* const cb,
RID sRowId,
RID eRowId,
int width,
int sOffset)
:fCBuf(cb),
fStartRowId(sRowId),
fEndRowId(eRowId),
fColWidth(width),
fBufStartOffset(sOffset),
fCurrRowId(sRowId),
fStatus(INIT_COMPLETE) {
}
ColumnBufferSection::~ColumnBufferSection() {
: fCBuf(cb),
fStartRowId(sRowId),
fEndRowId(eRowId),
fColWidth(width),
fBufStartOffset(sOffset),
fCurrRowId(sRowId),
fStatus(INIT_COMPLETE)
{
}
void ColumnBufferSection::write(const void * const data, int nRows) {
//Casting void * to unsigned char * without modifying the constness
const unsigned char * const tData =
static_cast<const unsigned char * const>(data);
ColumnBufferSection::~ColumnBufferSection()
{
}
if(fCurrRowId + nRows + 1> fEndRowId) {
//TODO: Handle error (old-dmc)
void ColumnBufferSection::write(const void* const data, int nRows)
{
//Casting void * to unsigned char * without modifying the constness
const unsigned char* const tData =
static_cast<const unsigned char* const>(data);
if (fCurrRowId + nRows + 1 > fEndRowId)
{
//TODO: Handle error (old-dmc)
}
int startOffset = (fBufStartOffset + (fCurrRowId-fStartRowId) *
fColWidth) % fCBuf->getSize();
int startOffset = (fBufStartOffset + (fCurrRowId - fStartRowId) *
fColWidth) % fCBuf->getSize();
int nBytes = nRows * fColWidth;
int bytesWritten = 0;
if((startOffset + nBytes) > fCBuf->getSize()) {
if ((startOffset + nBytes) > fCBuf->getSize())
{
fCBuf->write(tData, startOffset, fCBuf->getSize() - startOffset);
bytesWritten = fCBuf->getSize() - startOffset;
startOffset = 0;
}
fCBuf->write(tData + bytesWritten, startOffset, nBytes - bytesWritten);
fCurrRowId += nRows;
}
void ColumnBufferSection::setStatus(int s) {
void ColumnBufferSection::setStatus(int s)
{
fStatus = s;
}
int ColumnBufferSection::getStatus() const {
int ColumnBufferSection::getStatus() const
{
return fStatus;
}
int ColumnBufferSection::getStartOffset() const {
int ColumnBufferSection::getStartOffset() const
{
return fBufStartOffset;
}
int ColumnBufferSection::getSectionSize() const {
int ColumnBufferSection::getSectionSize() const
{
return (fEndRowId - fStartRowId + 1) * fColWidth;
}

29
writeengine/bulk/we_colbufsec.h
View File

@ -30,11 +30,13 @@
#include "we_type.h"
#include "we_colbuf.h"
namespace WriteEngine {
namespace WriteEngine
{
/* @brief Status codes for the ColumnBufferSection
*/
enum {
enum
{
INIT_COMPLETE,
WRITE_COMPLETE,
};
@ -43,19 +45,20 @@ enum {
* @brief ColumnBufferSection class represent a section of the ColumnBuffer and
* provides functionality for writing to the ColumnBuffer.
*/
class ColumnBufferSection {
class ColumnBufferSection
{
public:
public:
/* @brief Constructor
*
* @param cb Ptr to this section's corresponding CoulmnBuffer
* @param startRowId Starting row-id of the column
* @param endRowId Ending row-id of the column
* @param Width Width of the underlying column
* @param startOffset
* @param startOffset
*/
ColumnBufferSection(ColumnBuffer* const cb, RID startRowId,
RID endRowId, int colWidth, int startOffset);
RID endRowId, int colWidth, int startOffset);
/* @brief Default destructor
*/
@ -72,7 +75,7 @@ class ColumnBufferSection {
* @param data pointer to the data
* @param nRows Number of rows to be written, starting from data pointer
*/
void write(const void * const data, int nRows);
void write(const void* const data, int nRows);
/* @brief Returns the current status of the section
*/
@ -88,13 +91,19 @@ class ColumnBufferSection {
/* @brief Returns the ending row-id of this section
*/
RID endRowId() const { return fEndRowId; }
RID endRowId() const
{
return fEndRowId;
}
/* @brief Returns the starting row-id of this section
*/
RID startRowId() const { return fStartRowId; }
RID startRowId() const
{
return fStartRowId;
}
private:
private:
const ColumnBuffer* const fCBuf; //ColumnBuffer associated with this section
RID fStartRowId; // Starting row-id for this section

55
writeengine/bulk/we_colextinf.cpp
View File

@ -39,9 +39,9 @@
namespace
{
typedef std::tr1::unordered_map<
WriteEngine::RID,WriteEngine::ColExtInfEntry,WriteEngine::uint64Hasher>
RowExtMap;
typedef std::tr1::unordered_map <
WriteEngine::RID, WriteEngine::ColExtInfEntry, WriteEngine::uint64Hasher >
RowExtMap;
}
namespace WriteEngine
@ -78,6 +78,7 @@ void ColExtInf::addOrUpdateEntry( RID lastInputRow,
boost::mutex::scoped_lock lock(fMapMutex);
RowExtMap::iterator iter = fMap.find( lastInputRow );
if (iter == fMap.end()) // Add entry
{
ColExtInfEntry entry( minVal, maxVal );
@ -99,17 +100,19 @@ void ColExtInf::addOrUpdateEntry( RID lastInputRow,
{
if (isUnsigned(colDataType))
{
if (static_cast<uint64_t>(minVal)
< static_cast<uint64_t>(iter->second.fMinVal))
if (static_cast<uint64_t>(minVal)
< static_cast<uint64_t>(iter->second.fMinVal))
iter->second.fMinVal = minVal;
if (static_cast<uint64_t>(maxVal)
> static_cast<uint64_t>(iter->second.fMaxVal))
> static_cast<uint64_t>(iter->second.fMaxVal))
iter->second.fMaxVal = maxVal;
}
else
{
if (minVal < iter->second.fMinVal)
iter->second.fMinVal = minVal;
if (maxVal > iter->second.fMaxVal)
iter->second.fMaxVal = maxVal;
}
@ -130,9 +133,11 @@ int ColExtInf::updateEntryLbid( BRM::LBID_t startLbid )
// row order, so we get the "first" object in fPendingExtentRows, and
// that should be the extent corresponding to the LBID we just got.
std::set<RID>::iterator iterPendingExt = fPendingExtentRows.begin();
if (iterPendingExt != fPendingExtentRows.end())
{
RowExtMap::iterator iter = fMap.find( *iterPendingExt );
if (iter != fMap.end())
{
iter->second.fLbid = startLbid;
@ -158,11 +163,12 @@ int ColExtInf::updateEntryLbid( BRM::LBID_t startLbid )
void ColExtInf::getCPInfoForBRM( JobColumn column, BRMReporter& brmReporter )
{
bool bIsChar = ((column.weType == WriteEngine::WR_CHAR) &&
(column.colType != COL_TYPE_DICT));
(column.colType != COL_TYPE_DICT));
boost::mutex::scoped_lock lock(fMapMutex);
RowExtMap::const_iterator iter = fMap.begin();
while (iter != fMap.end())
{
// If/when we support NULL values, we could have an extent with initial
@ -172,6 +178,7 @@ void ColExtInf::getCPInfoForBRM( JobColumn column, BRMReporter& brmReporter )
// if applicable (indicating an extent with no non-NULL values).
int64_t minVal = iter->second.fMinVal;
int64_t maxVal = iter->second.fMaxVal;
if ( bIsChar )
{
// If we have added 1 or more rows, then we should have a valid
@ -180,12 +187,12 @@ void ColExtInf::getCPInfoForBRM( JobColumn column, BRMReporter& brmReporter )
// else we leave fMinVal & fMaxVal set to LLONG_MIN and send as-is,
// to let BRM know we added no rows.
if ((iter->second.fMinVal != iter->second.fMaxVal) ||
(iter->second.fMinVal != LLONG_MIN))
(iter->second.fMinVal != LLONG_MIN))
{
minVal = static_cast<int64_t>( uint64ToStr(
static_cast<uint64_t>(iter->second.fMinVal) ) );
static_cast<uint64_t>(iter->second.fMinVal) ) );
maxVal = static_cast<int64_t>( uint64ToStr(
static_cast<uint64_t>(iter->second.fMaxVal) ) );
static_cast<uint64_t>(iter->second.fMaxVal) ) );
}
}
@ -194,9 +201,10 @@ void ColExtInf::getCPInfoForBRM( JobColumn column, BRMReporter& brmReporter )
{
std::ostringstream oss;
oss << "Saving CP update for OID-" << fColOid <<
"; lbid-" << iter->second.fLbid <<
"; type-" << bIsChar <<
"; isNew-" << iter->second.fNewExtent;
"; lbid-" << iter->second.fLbid <<
"; type-" << bIsChar <<
"; isNew-" << iter->second.fNewExtent;
if (bIsChar)
{
char minValStr[sizeof(int64_t) + 1];
@ -211,12 +219,12 @@ void ColExtInf::getCPInfoForBRM( JobColumn column, BRMReporter& brmReporter )
else if (isUnsigned(column.dataType))
{
oss << "; min: " << static_cast<uint64_t>(minVal) <<
"; max: " << static_cast<uint64_t>(maxVal);
"; max: " << static_cast<uint64_t>(maxVal);
}
else
{
oss << "; min: " << minVal <<
"; max: " << maxVal;
"; max: " << maxVal;
}
fLog->logMsg( oss.str(), MSGLVL_INFO2 );
@ -233,6 +241,7 @@ void ColExtInf::getCPInfoForBRM( JobColumn column, BRMReporter& brmReporter )
++iter;
}
fMap.clear(); // don't need map anymore, so release memory
}
@ -247,21 +256,25 @@ void ColExtInf::print( const JobColumn& column )
std::ostringstream oss;
oss << "ColExtInf Map for OID: " << fColOid;
RowExtMap::const_iterator iter = fMap.begin();
while (iter != fMap.end())
{
oss << std::endl <<
" RowKey-" << iter->first <<
"; lbid-" << iter->second.fLbid;
if (iter->second.fLbid == (BRM::LBID_t)INVALID_LBID)
oss << " (unset)";
oss << "; newExt-" << iter->second.fNewExtent;
if ( bIsChar )
{
// Swap/restore byte order before printing character string
int64_t minVal = static_cast<int64_t>( uint64ToStr(
static_cast<uint64_t>(iter->second.fMinVal) ) );
static_cast<uint64_t>(iter->second.fMinVal) ) );
int64_t maxVal = static_cast<int64_t>( uint64ToStr(
static_cast<uint64_t>(iter->second.fMaxVal) ) );
static_cast<uint64_t>(iter->second.fMaxVal) ) );
char minValStr[sizeof(int64_t) + 1];
char maxValStr[sizeof(int64_t) + 1];
memcpy(minValStr, &minVal, sizeof(int64_t));
@ -274,18 +287,20 @@ void ColExtInf::print( const JobColumn& column )
else if (isUnsigned(column.dataType))
{
oss << "; min: " << static_cast<uint64_t>(iter->second.fMinVal) <<
"; max: " << static_cast<uint64_t>(iter->second.fMaxVal);
"; max: " << static_cast<uint64_t>(iter->second.fMaxVal);
}
else
{
oss << "; min: " << iter->second.fMinVal <<
"; max: " << iter->second.fMaxVal;
"; max: " << iter->second.fMaxVal;
}
++iter;
}
oss << std::endl << " ColExtInf Rows/Extents waiting LBIDs: ";
std::set<RID>::const_iterator iterPendingExt=fPendingExtentRows.begin();
std::set<RID>::const_iterator iterPendingExt = fPendingExtentRows.begin();
while (iterPendingExt != fPendingExtentRows.end())
{
oss << *iterPendingExt << ", ";

57
writeengine/bulk/we_colextinf.h
View File

@ -43,9 +43,9 @@
namespace WriteEngine
{
class Log;
class BRMReporter;
typedef execplan::CalpontSystemCatalog::ColDataType ColDataType;
class Log;
class BRMReporter;
typedef execplan::CalpontSystemCatalog::ColDataType ColDataType;
//------------------------------------------------------------------------------
/** @brief Class to store min/max and LBID information for an extent.
* For character data, the min and max values are maintained in reverse
@ -60,34 +60,34 @@ class ColExtInfEntry
public:
// Default constructor
ColExtInfEntry() : fLbid(INVALID_LBID),
fMinVal(LLONG_MIN),
fMaxVal(LLONG_MIN),
fNewExtent(true) { }
fMinVal(LLONG_MIN),
fMaxVal(LLONG_MIN),
fNewExtent(true) { }
// Used to create entry for an existing extent we are going to add data to.
ColExtInfEntry(BRM::LBID_t lbid, bool bIsNewExtent) :
fLbid(lbid),
fMinVal(LLONG_MIN),
fMaxVal(LLONG_MIN),
fNewExtent(bIsNewExtent) { }
fLbid(lbid),
fMinVal(LLONG_MIN),
fMaxVal(LLONG_MIN),
fNewExtent(bIsNewExtent) { }
// Used to create entry for a new extent, with LBID not yet allocated
ColExtInfEntry(int64_t minVal, int64_t maxVal) :
fLbid(INVALID_LBID),
fMinVal(minVal),
fMaxVal(maxVal),
fNewExtent(true) { }
fLbid(INVALID_LBID),
fMinVal(minVal),
fMaxVal(maxVal),
fNewExtent(true) { }
// Used to create entry for a new extent, with LBID not yet allocated
ColExtInfEntry(uint64_t minVal, uint64_t maxVal) :
fLbid(INVALID_LBID),
fMinVal(static_cast<int64_t>(minVal)),
fMaxVal(static_cast<int64_t>(maxVal)),
fNewExtent(true) { }
fLbid(INVALID_LBID),
fMinVal(static_cast<int64_t>(minVal)),
fMaxVal(static_cast<int64_t>(maxVal)),
fNewExtent(true) { }
BRM::LBID_t fLbid; // LBID for an extent; should be the starting LBID
int64_t fMinVal; // minimum value for extent associated with LBID
int64_t fMaxVal; // maximum value for extent associated with LBID
int64_t fMaxVal; // maximum value for extent associated with LBID
bool fNewExtent;// is this a new extent
};
@ -96,10 +96,12 @@ public:
* the last input Row number in the extent, as the key.
*/
//------------------------------------------------------------------------------
struct uint64Hasher : public std::unary_function<RID,std::size_t>
struct uint64Hasher : public std::unary_function<RID, std::size_t>
{
std::size_t operator()(RID val) const
{ return static_cast<std::size_t>(val); }
{
return static_cast<std::size_t>(val);
}
};
//------------------------------------------------------------------------------
@ -120,12 +122,15 @@ public:
virtual void addOrUpdateEntry( RID lastInputRow,
int64_t minVal,
int64_t maxVal,
ColDataType colDataType ){ }
ColDataType colDataType ) { }
virtual void getCPInfoForBRM ( JobColumn column,
BRMReporter& brmReporter){ }
BRMReporter& brmReporter) { }
virtual void print( const JobColumn& column ) { }
virtual int updateEntryLbid( BRM::LBID_t startLbid ) { return NO_ERROR; }
virtual int updateEntryLbid( BRM::LBID_t startLbid )
{
return NO_ERROR;
}
};
//------------------------------------------------------------------------------
@ -193,10 +198,10 @@ private:
Log* fLog; // Log used for debug logging
boost::mutex fMapMutex; // protects unordered map access
std::set<RID> fPendingExtentRows; // list of lastInputRow entries that
// are awaiting an LBID assignment.
// are awaiting an LBID assignment.
// unordered map where we collect the min/max values per extent
std::tr1::unordered_map<RID,ColExtInfEntry,uint64Hasher> fMap;
std::tr1::unordered_map<RID, ColExtInfEntry, uint64Hasher> fMap;
// disable copy constructor and assignment operator
ColExtInf(const ColExtInf&);

16
writeengine/bulk/we_colopbulk.cpp
View File

@ -63,16 +63,16 @@ ColumnOpBulk::~ColumnOpBulk()
//------------------------------------------------------------------------------
// @bug 5572 - HDFS usage: add *.tmp file backup flag
IDBDataFile* ColumnOpBulk::openFile(const WriteEngine::Column& column,
uint16_t dbRoot,
uint32_t partition,
uint16_t segment,
std::string& segFile,
bool useTmpSuffix,
const char* mode,
int ioBuffSize) const
uint16_t dbRoot,
uint32_t partition,
uint16_t segment,
std::string& segFile,
bool useTmpSuffix,
const char* mode,
int ioBuffSize) const
{
return FileOp::openFile(column.dataFile.fid, dbRoot, partition, segment,
segFile, mode, column.colWidth, useTmpSuffix);
segFile, mode, column.colWidth, useTmpSuffix);
}
//------------------------------------------------------------------------------

12
writeengine/bulk/we_colopbulk.h
View File

@ -42,17 +42,17 @@ class Log;
*/
class ColumnOpBulk : public ColumnOp
{
public:
ColumnOpBulk();
ColumnOpBulk(Log* logger, int compressionType);
public:
ColumnOpBulk();
ColumnOpBulk(Log* logger, int compressionType);
virtual ~ColumnOpBulk();
virtual bool abbreviatedExtent(IDBDataFile*, int) const;
virtual int blocksInFile(IDBDataFile*) const;
virtual IDBDataFile* openFile(const WriteEngine::Column& column,
uint16_t dbRoot, uint32_t partition, uint16_t segment,
std::string& segFile, bool useTmpSuffix, const char* mode = "r+b",
int ioBuffSize = DEFAULT_BUFSIZ) const;
uint16_t dbRoot, uint32_t partition, uint16_t segment,
std::string& segFile, bool useTmpSuffix, const char* mode = "r+b",
int ioBuffSize = DEFAULT_BUFSIZ) const;
virtual int readBlock(IDBDataFile*, unsigned char*, const uint64_t);
virtual int saveBlock(IDBDataFile*, const unsigned char*, const uint64_t);
};

81
writeengine/bulk/we_columnautoinc.cpp
View File

@ -64,26 +64,28 @@ ColumnAutoInc::~ColumnAutoInc( )
// colInfo - ColumnInfo associated with auto-increment column.
//------------------------------------------------------------------------------
int ColumnAutoInc::init( const std::string& fullTableName,
ColumnInfo* colInfo )
ColumnInfo* colInfo )
{
fMaxIntSat = colInfo->column.fMaxIntSat;
fTableName = fullTableName;
fColumnName= colInfo->column.colName;
fColumnName = colInfo->column.colName;
fColumnOID = colInfo->column.mapOid;
std::string::size_type periodIdx = fTableName.find('.');
if (periodIdx == std::string::npos)
{
std::ostringstream oss;
oss << "Error parsing full table name to get auto-increment value for "
<< fTableName;
fLog->logMsg( oss.str(), ERR_AUTOINC_TABLE_NAME, MSGLVL_ERROR );
BulkLoad::addErrorMsg2BrmUpdater(fTableName, oss);
BulkLoad::addErrorMsg2BrmUpdater(fTableName, oss);
return ERR_AUTOINC_TABLE_NAME;
}
uint64_t nextAuto = 0;
int rc = getNextValueFromSysCat( nextAuto );
if (rc != NO_ERROR)
{
return rc;
@ -91,21 +93,22 @@ int ColumnAutoInc::init( const std::string& fullTableName,
std::string errMsg;
rc = BRMWrapper::getInstance()->startAutoIncrementSequence(
fColumnOID, nextAuto, colInfo->column.width, colInfo->column.dataType, errMsg );
fColumnOID, nextAuto, colInfo->column.width, colInfo->column.dataType, errMsg );
if (rc != NO_ERROR)
{
std::ostringstream oss;
oss << "Unable to initialize auto-increment sequence for " <<
fTableName << "; " << errMsg;
fLog->logMsg( oss.str(), rc, MSGLVL_ERROR );
BulkLoad::addErrorMsg2BrmUpdater(fTableName, oss);
BulkLoad::addErrorMsg2BrmUpdater(fTableName, oss);
return rc;
}
std::ostringstream oss2;
oss2 << "Initializing next auto increment for table-" << fTableName <<
", column-" << fColumnName <<
"; autoincrement " << nextAuto;
", column-" << fColumnName <<
"; autoincrement " << nextAuto;
fLog->logMsg( oss2.str(), MSGLVL_INFO2 );
initNextAutoInc( nextAuto );
@ -119,11 +122,12 @@ int ColumnAutoInc::init( const std::string& fullTableName,
// Don't need to use fAutoIncMutex in this function as long as we call it from
// the main thread, during preprocessing. But we go ahead and use the mutex
// for completeness. Using the mutex should not affect performance, since this
// function is only called once per table.
// function is only called once per table.
//------------------------------------------------------------------------------
void ColumnAutoInc::initNextAutoInc( uint64_t nextValue )
{
boost::mutex::scoped_lock lock(fAutoIncMutex);
// nextValue is unusable if < 1; probably means we already reached max value
if (nextValue < 1)
fAutoIncLastValue = fMaxIntSat;
@ -149,7 +153,8 @@ int ColumnAutoInc::finish( )
// We grab AI lock in order to access/synchronize DBRM and the system
// catalog as a single operation, to avoid race condition between apps.
try {
try
{
dbrm.getAILock( fColumnOID );
}
catch (std::exception& ex)
@ -158,28 +163,31 @@ int ColumnAutoInc::finish( )
oss << "Error locking auto-increment nextValue lock for table " <<
fTableName << "; column " << fColumnName << "; " << ex.what();
fLog->logMsg( oss.str(), ERR_AUTOINC_GET_LOCK, MSGLVL_ERROR );
BulkLoad::addErrorMsg2BrmUpdater(fTableName, oss);
BulkLoad::addErrorMsg2BrmUpdater(fTableName, oss);
return ERR_AUTOINC_GET_LOCK;
}
uint64_t sysCatNextAuto = 0;
rc = getNextValueFromSysCat( sysCatNextAuto );
if (rc == NO_ERROR)
{
// Update system catalog if my latest AI nextValue is > the current
// syscat AI nextValue. max(uint64_t) denotes an AI column that has maxed out.
uint64_t myNextValue = getNextAutoIncToSave();
if ( (sysCatNextAuto != AUTOINCR_SATURATED) && // do not update if syscat already at max
((myNextValue > sysCatNextAuto) ||
(myNextValue == AUTOINCR_SATURATED)) )
((myNextValue > sysCatNextAuto) ||
(myNextValue == AUTOINCR_SATURATED)) )
{
std::ostringstream oss2;
oss2 << "Updating next auto increment for table-" << fTableName <<
", column-" << fColumnName <<
"; autoincrement " << myNextValue;
", column-" << fColumnName <<
"; autoincrement " << myNextValue;
fLog->logMsg( oss2.str(), MSGLVL_INFO2 );
rc = BulkLoad::updateNextValue( fColumnOID, myNextValue );
if (rc != NO_ERROR)
{
WErrorCodes ec;
@ -188,7 +196,7 @@ int ColumnAutoInc::finish( )
fTableName << "; column " << fColumnName << "; rc=" << rc <<
"; " << ec.errorString(ERR_AUTOINC_UPDATE);
fLog->logMsg( oss.str(), ERR_AUTOINC_UPDATE, MSGLVL_ERROR );
BulkLoad::addErrorMsg2BrmUpdater(fTableName, oss);
BulkLoad::addErrorMsg2BrmUpdater(fTableName, oss);
// Don't exit this function yet. We set return code and fall
// through to bottom of the function to release the AI lock.
rc = ERR_AUTOINC_UPDATE;
@ -197,15 +205,16 @@ int ColumnAutoInc::finish( )
else
{
std::ostringstream oss2;
oss2 << "Skip updating next auto increment for table-"<<fTableName<<
", column-" << fColumnName <<
"; autoincrement " << myNextValue <<
"; syscat AI already at " << sysCatNextAuto;
oss2 << "Skip updating next auto increment for table-" << fTableName <<
", column-" << fColumnName <<
"; autoincrement " << myNextValue <<
"; syscat AI already at " << sysCatNextAuto;
fLog->logMsg( oss2.str(), MSGLVL_INFO2 );
}
} // end of rc==NO_ERROR from getNextValueFromSysCat()
try {
try
{
dbrm.releaseAILock( fColumnOID );
}
catch (std::exception& ex)
@ -213,7 +222,7 @@ int ColumnAutoInc::finish( )
// If we have trouble releasing AI lock, we log it, but we don't
// consider it fatal to the job; so we don't return bad return code.
std::ostringstream oss;
oss << "Error releasing auto-increment nextValue lock for table "<<
oss << "Error releasing auto-increment nextValue lock for table " <<
fTableName << "; column " << fColumnName << "; " << ex.what();
fLog->logMsg( oss.str(), ERR_AUTOINC_REL_LOCK, MSGLVL_WARNING );
//return ERR_AUTOINC_REL_LOCK;
@ -232,6 +241,7 @@ uint64_t ColumnAutoInc::getNextAutoIncToSave( )
uint64_t nextValue = AUTOINCR_SATURATED;
boost::mutex::scoped_lock lock(fAutoIncMutex);
// nextValue is returned as -1 if we reached max value
if (fAutoIncLastValue < fMaxIntSat)
nextValue = fAutoIncLastValue + 1;
@ -249,10 +259,11 @@ int ColumnAutoInc::getNextValueFromSysCat( uint64_t& nextValue )
std::string sName;
std::string tName;
sName.assign(fTableName, 0, periodIdx);
tName.assign(fTableName, periodIdx+1,
fTableName.length() - (periodIdx+1));
execplan::CalpontSystemCatalog::TableName tbl(sName,tName);
tName.assign(fTableName, periodIdx + 1,
fTableName.length() - (periodIdx + 1));
execplan::CalpontSystemCatalog::TableName tbl(sName, tName);
uint64_t nextAuto = 0;
try
{
boost::shared_ptr<execplan::CalpontSystemCatalog> systemCatPtr =
@ -263,11 +274,14 @@ int ColumnAutoInc::getNextValueFromSysCat( uint64_t& nextValue )
// Handle bad return code or thrown exception from
// system catalog query.
nextAuto = systemCatPtr->nextAutoIncrValue( tbl );
if (nextAuto == 0) {
if (nextAuto == 0)
{
throw std::runtime_error(
"Not an auto-increment column, or column not found");
}
else if (nextAuto == AUTOINCR_SATURATED) {
else if (nextAuto == AUTOINCR_SATURATED)
{
throw std::runtime_error(
"auto-increment max value already reached");
}
@ -280,7 +294,7 @@ int ColumnAutoInc::getNextValueFromSysCat( uint64_t& nextValue )
oss << "Unable to get current auto-increment value for " <<
sName << "." << tName << "; " << ex.what();
fLog->logMsg( oss.str(), ERR_AUTOINC_INIT1, MSGLVL_ERROR );
BulkLoad::addErrorMsg2BrmUpdater(tName, oss);
BulkLoad::addErrorMsg2BrmUpdater(tName, oss);
return ERR_AUTOINC_INIT1;
}
catch (...)
@ -289,7 +303,7 @@ int ColumnAutoInc::getNextValueFromSysCat( uint64_t& nextValue )
oss << "Unable to get current auto-increment value for " <<
sName << "." << tName << "; unknown exception";
fLog->logMsg( oss.str(), ERR_AUTOINC_INIT2, MSGLVL_ERROR );
BulkLoad::addErrorMsg2BrmUpdater(tName, oss);
BulkLoad::addErrorMsg2BrmUpdater(tName, oss);
return ERR_AUTOINC_INIT2;
}
@ -321,6 +335,7 @@ int ColumnAutoIncJob::reserveNextRange(
uint64_t& nextValue )
{
boost::mutex::scoped_lock lock(fAutoIncMutex);
if ((fMaxIntSat - autoIncCount) < fAutoIncLastValue)
{
return ERR_AUTOINC_GEN_EXCEED_MAX;
@ -362,17 +377,20 @@ int ColumnAutoIncIncremental::reserveNextRange(
uint64_t nextValArg = 0;
std::string errMsg;
int rc = BRMWrapper::getInstance()->getAutoIncrementRange(
fColumnOID, countArg, nextValArg, errMsg );
fColumnOID, countArg, nextValArg, errMsg );
if (rc != NO_ERROR)
{
std::ostringstream oss;
oss << "Error reserving auto-increment range (" << countArg <<
" numbers) for table "<<
" numbers) for table " <<
fTableName << "; column " << fColumnName << "; " << errMsg;
if (rc == ERR_AUTOINC_GEN_EXCEED_MAX)
oss << " Max allowed value is " << fMaxIntSat << ".";
fLog->logMsg( oss.str(), rc, MSGLVL_ERROR );
BulkLoad::addErrorMsg2BrmUpdater(fTableName, oss);
BulkLoad::addErrorMsg2BrmUpdater(fTableName, oss);
return rc;
}
@ -385,6 +403,7 @@ int ColumnAutoIncIncremental::reserveNextRange(
// update fAutoIncLastValue. We only update it if the range in question
// exceeds the current value for fAutoIncLastValue.
boost::mutex::scoped_lock lock(fAutoIncMutex);
if (autoIncLastValue > fAutoIncLastValue)
fAutoIncLastValue = autoIncLastValue;

14
writeengine/bulk/we_columnautoinc.h
View File

@ -34,8 +34,8 @@
namespace WriteEngine
{
struct ColumnInfo;
class Log;
struct ColumnInfo;
class Log;
//------------------------------------------------------------------------------
/** @brief Abstract base class used for derived auto-increment classes.
@ -58,14 +58,14 @@ public:
* @param colInfo The auto-increment column.
*/
int init( const std::string& fullTableName,
ColumnInfo* colInfo );
ColumnInfo* colInfo );
/** @brief Reserve next range of auto-increment numbers.
* @param autoIncCount Number of auto-increment numbers to reserve
* @param nextValue Starting number of reserved range
*/
virtual int reserveNextRange(uint32_t autoIncCount,
uint64_t& nextValue) = 0;
uint64_t& nextValue) = 0;
/** @brief Finished with auto-incrementing; perform any applicable updates.
*/
@ -87,7 +87,7 @@ protected:
//------------------------------------------------------------------------------
/** @brief Auto-increment implementation that reserves auto-increment numbers
* once for the entire job.
* once for the entire job.
*
* Assumes a lock is applied to the table throughout the life of the job.
* This allows the auto-increment next value to be managed through 2 single
@ -103,7 +103,7 @@ public:
virtual ~ColumnAutoIncJob();
virtual int reserveNextRange(uint32_t autoIncCount,
uint64_t& nextValue);
uint64_t& nextValue);
};
//------------------------------------------------------------------------------
@ -122,7 +122,7 @@ public:
virtual ~ColumnAutoIncIncremental();
virtual int reserveNextRange(uint32_t autoIncCount,
uint64_t& nextValue);
uint64_t& nextValue);
};
} //end of namespace

589
writeengine/bulk/we_columninfo.cpp
View File

File diff suppressed because it is too large Load Diff

68
writeengine/bulk/we_columninfo.h
View File

@ -50,7 +50,7 @@ class BRMReporter;
class TableInfo;
struct DBRootExtentInfo;
enum Status
enum Status
{
PARSE_COMPLETE = 0,
READ_COMPLETE,
@ -129,7 +129,7 @@ struct ColumnInfo
// Public Data Members
//--------------------------------------------------------------------------
/** @brief Current column
/** @brief Current column
*/
Column curCol;
@ -157,7 +157,7 @@ struct ColumnInfo
/** @brief Instance of the write buffer manager.
*/
ColumnBufferManager *fColBufferMgr;
ColumnBufferManager* fColBufferMgr;
/** @brief Freespace (in bytes) at the end of the current db column file
* For compressed data files, this is the "raw" data byte count,
@ -207,7 +207,7 @@ struct ColumnInfo
* corresponding column token file.
*/
int updateDctnryStore(char* buf,
ColPosPair ** pos,
ColPosPair** pos,
const int totalRow,
char* tokenBuf);
@ -311,33 +311,33 @@ struct ColumnInfo
* @param bIsNewExtent Treat as new extent when updating CP min/max
*/
int setupInitialColumnExtent( uint16_t dbRoot,
uint32_t partition,
uint16_t segment,
const std::string& tblName,
BRM::LBID_t lbid,
HWM oldHwm,
HWM hwm,
bool bSkippedToNewExtent,
bool bIsNewExtent );
uint32_t partition,
uint16_t segment,
const std::string& tblName,
BRM::LBID_t lbid,
HWM oldHwm,
HWM hwm,
bool bSkippedToNewExtent,
bool bIsNewExtent );
/** @brief Setup a DB file to be created for starting extent only when needed
* @param dbRoot DBRoot of starting extent
* @param partition Partition number of starting extent
* @param segment Segment file number of starting extent
* @param hwm Starting HWM for new start extent
* @param bEmptyPM Are we setting up delayed file creation because a PM
* has no extents (or is the HWM extent just disabled)
*/
/** @brief Setup a DB file to be created for starting extent only when needed
* @param dbRoot DBRoot of starting extent
* @param partition Partition number of starting extent
* @param segment Segment file number of starting extent
* @param hwm Starting HWM for new start extent
* @param bEmptyPM Are we setting up delayed file creation because a PM
* has no extents (or is the HWM extent just disabled)
*/
void setupDelayedFileCreation(
uint16_t dbRoot,
uint32_t partition,
uint16_t segment,
HWM hwm,
bool bEmptyPM );
uint16_t dbRoot,
uint32_t partition,
uint16_t segment,
HWM hwm,
bool bEmptyPM );
/** @brief Belatedly create a starting DB file for a PM that has none.
* @param tableName Name of table for which this column belongs
*/
/** @brief Belatedly create a starting DB file for a PM that has none.
* @param tableName Name of table for which this column belongs
*/
int createDelayedFileIfNeeded( const std::string& tableName );
/** @brief Update how many bytes of data are in the column segment file and
@ -371,7 +371,7 @@ struct ColumnInfo
* @param sNum Segment number of relevant dictionary store segment file.
*/
virtual int truncateDctnryStore(OID dctnryOid,
uint16_t root, uint32_t pNum, uint16_t sNum) const;
uint16_t root, uint32_t pNum, uint16_t sNum) const;
/** @brief Increment saturated row count for this column in current import
* @param satIncCnt Increment count to add to the total saturation count.
@ -397,7 +397,7 @@ struct ColumnInfo
*/
unsigned rowsPerExtent( );
protected:
protected:
//--------------------------------------------------------------------------
// Protected Functions
@ -416,7 +416,7 @@ struct ColumnInfo
void lastInputRowInExtentInit( bool bIsNewExtent );
virtual int resetFileOffsetsNewExtent(const char* hdr);
// Reset file; start new extent
// Reset file; start new extent
void setFileSize( HWM hwm, int abbrevFlag ); // Set fileSize data member
// Prepare initial column segment file for importing of data.
@ -424,7 +424,7 @@ struct ColumnInfo
// used for abbreviated extents, to detect when block skipping has
// caused us to require a full expanded extent.
// newHWM - Starting point for adding data after initial blockskipping
virtual int setupInitialColumnFile( HWM oldHWM, // original HWM
virtual int setupInitialColumnFile( HWM oldHWM, // original HWM
HWM newHWM ); // new HWM to start from
virtual int saveDctnryStoreHWMChunk(bool& needBackup);//Backup Dct HWM Chunk
@ -499,7 +499,7 @@ struct ColumnInfo
int fColWidthFactor; // Wid factor relative to other cols
InitialDBFileStat fDelayedFileCreation; // Denotes when initial DB file is
// to be created after preprocessing
// to be created after preprocessing
unsigned fRowsPerExtent; // Number of rows per column extent
};
@ -524,7 +524,7 @@ inline int64_t ColumnInfo::getFileSize( ) const
inline void ColumnInfo::incSaturatedCnt( int64_t satIncCnt )
{
(void)atomicops::atomicAdd(&fSaturatedRowCnt, satIncCnt);
(void)atomicops::atomicAdd(&fSaturatedRowCnt, satIncCnt);
}
inline bool ColumnInfo::isAbbrevExtent( )

135
writeengine/bulk/we_columninfocompressed.cpp
View File

@ -44,11 +44,11 @@ namespace WriteEngine
// ColumnInfoCompressed constructor
//------------------------------------------------------------------------------
ColumnInfoCompressed::ColumnInfoCompressed(Log* logger,
int idIn,
const JobColumn& columnIn,
DBRootExtentTracker* pDBRootExtTrk,
TableInfo* pTableInfo):
//RBMetaWriter* rbMetaWriter) :
int idIn,
const JobColumn& columnIn,
DBRootExtentTracker* pDBRootExtTrk,
TableInfo* pTableInfo):
//RBMetaWriter* rbMetaWriter) :
ColumnInfo(logger, idIn, columnIn, pDBRootExtTrk, pTableInfo),
fRBMetaWriter(pTableInfo->rbMetaWriter())
{
@ -65,7 +65,7 @@ ColumnInfoCompressed::~ColumnInfoCompressed()
// Close the current compressed Column file after first compressing/flushing
// any remaining data, and re-writing the headers as well.
//------------------------------------------------------------------------------
int ColumnInfoCompressed::closeColumnFile(bool bCompletingExtent,bool bAbort)
int ColumnInfoCompressed::closeColumnFile(bool bCompletingExtent, bool bAbort)
{
int rc = NO_ERROR;
@ -80,6 +80,7 @@ int ColumnInfoCompressed::closeColumnFile(bool bCompletingExtent,bool bAbort)
if (fColBufferMgr)
{
rc = fColBufferMgr->finishFile( bCompletingExtent );
if (rc != NO_ERROR)
{
WErrorCodes ec;
@ -111,7 +112,8 @@ int ColumnInfoCompressed::setupInitialColumnFile( HWM oldHwm, HWM hwm )
RETURN_ON_ERROR( colOp->readHeaders(curCol.dataFile.pFile, hdr) );
// Initialize the output buffer manager for the column.
WriteEngine::ColumnBufferManager *mgr;
WriteEngine::ColumnBufferManager* mgr;
if (column.colType == COL_TYPE_DICT)
{
mgr = new ColumnBufferManagerDctnry(
@ -124,12 +126,13 @@ int ColumnInfoCompressed::setupInitialColumnFile( HWM oldHwm, HWM hwm )
this, column.width, fLog, column.compressionType);
RETURN_ON_ERROR( mgr->setDbFile(curCol.dataFile.pFile, hwm, hdr) );
}
fColBufferMgr = mgr;
IDBCompressInterface compressor;
int abbrevFlag =
( compressor.getBlockCount(hdr) ==
uint64_t(INITIAL_EXTENT_ROWS_TO_DISK*column.width/BYTE_PER_BLOCK) );
uint64_t(INITIAL_EXTENT_ROWS_TO_DISK * column.width / BYTE_PER_BLOCK) );
setFileSize( hwm, abbrevFlag );
// See if dealing with abbreviated extent that will need expanding.
@ -141,9 +144,10 @@ int ColumnInfoCompressed::setupInitialColumnFile( HWM oldHwm, HWM hwm )
if (isAbbrevExtent())
{
unsigned int numBlksForFirstExtent =
(INITIAL_EXTENT_ROWS_TO_DISK*column.width) / BYTE_PER_BLOCK;
if ( ((oldHwm+1) <= numBlksForFirstExtent) &&
((hwm+1 ) > numBlksForFirstExtent) )
(INITIAL_EXTENT_ROWS_TO_DISK * column.width) / BYTE_PER_BLOCK;
if ( ((oldHwm + 1) <= numBlksForFirstExtent) &&
((hwm + 1 ) > numBlksForFirstExtent) )
{
RETURN_ON_ERROR( expandAbbrevExtent(false) );
}
@ -164,13 +168,13 @@ int ColumnInfoCompressed::setupInitialColumnFile( HWM oldHwm, HWM hwm )
{
std::ostringstream oss;
oss << "Init raw data offsets in compressed column file OID-" <<
curCol.dataFile.fid <<
curCol.dataFile.fid <<
"; DBRoot-" << curCol.dataFile.fDbRoot <<
"; part-" << curCol.dataFile.fPartition <<
"; seg-" << curCol.dataFile.fSegment <<
"; abbrev-" << abbrevFlag <<
"; begByte-"<< fSizeWritten <<
"; endByte-"<< fileSize <<
"; begByte-" << fSizeWritten <<
"; endByte-" << fileSize <<
"; freeBytes-" << availFileSize;
fLog->logMsg( oss.str(), MSGLVL_INFO2 );
}
@ -202,11 +206,11 @@ int ColumnInfoCompressed::resetFileOffsetsNewExtent(const char* hdr)
// Reinitialize ColBuf for the next extent
long long startFileOffset;
RETURN_ON_ERROR( fColBufferMgr->resetToBeCompressedColBuf(
startFileOffset ) );
startFileOffset ) );
// Set the file offset to point to the chunk we are adding or updating
RETURN_ON_ERROR( colOp->setFileOffset(curCol.dataFile.pFile,
startFileOffset) );
startFileOffset) );
}
return NO_ERROR;
@ -224,23 +228,25 @@ int ColumnInfoCompressed::saveDctnryStoreHWMChunk(bool& needBackup)
#endif
needBackup = false;
int rc = NO_ERROR;
try
{
needBackup = fRBMetaWriter->backupDctnryHWMChunk(
column.dctnry.dctnryOid,
curCol.dataFile.fDbRoot,
curCol.dataFile.fPartition,
curCol.dataFile.fSegment );
column.dctnry.dctnryOid,
curCol.dataFile.fDbRoot,
curCol.dataFile.fPartition,
curCol.dataFile.fSegment );
}
catch (WeException& ex)
{
fLog->logMsg(ex.what(), ex.errorCode(), MSGLVL_ERROR);
rc = ex.errorCode();
}
#ifdef PROFILE
Stats::stopParseEvent(WE_STATS_COMPRESS_DCT_BACKUP_CHUNK);
#endif
return rc;
}
@ -273,10 +279,10 @@ int ColumnInfoCompressed::truncateDctnryStore(
{
std::ostringstream oss1;
oss1 << "Finished writing dictionary file"
": OID-" << dctnryOid <<
"; DBRoot-" << root <<
"; part-" << pNum <<
"; seg-" << sNum;
": OID-" << dctnryOid <<
"; DBRoot-" << root <<
"; part-" << pNum <<
"; seg-" << sNum;
// Have to rework this logging if we want to keep it.
// Filesize is not correct when adding data to an "existing" file,
@ -300,7 +306,8 @@ int ColumnInfoCompressed::truncateDctnryStore(
// (to the nearest extent)
std::string segFile;
IDBDataFile* dFile = fTruncateDctnryFileOp.openFile(dctnryOid,
root, pNum, sNum, segFile);
root, pNum, sNum, segFile);
if (dFile == 0)
{
rc = ERR_FILE_OPEN;
@ -319,7 +326,8 @@ int ColumnInfoCompressed::truncateDctnryStore(
char controlHdr[ IDBCompressInterface::HDR_BUF_LEN ];
rc = fTruncateDctnryFileOp.readFile( dFile,
(unsigned char*)controlHdr, IDBCompressInterface::HDR_BUF_LEN);
(unsigned char*)controlHdr, IDBCompressInterface::HDR_BUF_LEN);
if (rc != NO_ERROR)
{
WErrorCodes ec;
@ -339,6 +347,7 @@ int ColumnInfoCompressed::truncateDctnryStore(
IDBCompressInterface compressor;
int rc1 = compressor.verifyHdr( controlHdr );
if (rc1 != 0)
{
rc = ERR_COMP_VERIFY_HDRS;
@ -358,22 +367,23 @@ int ColumnInfoCompressed::truncateDctnryStore(
return rc;
}
// No need to perform file truncation if the dictionary file just contains
// a single abbreviated extent. Truncating up to the nearest extent would
// actually grow the file (something we don't want to do), because we have
// not yet reserved a full extent (on disk) for this dictionary store file.
// No need to perform file truncation if the dictionary file just contains
// a single abbreviated extent. Truncating up to the nearest extent would
// actually grow the file (something we don't want to do), because we have
// not yet reserved a full extent (on disk) for this dictionary store file.
const int PSEUDO_COL_WIDTH = 8;
uint64_t numBlocks = compressor.getBlockCount( controlHdr );
if ( numBlocks == uint64_t
(INITIAL_EXTENT_ROWS_TO_DISK*PSEUDO_COL_WIDTH/BYTE_PER_BLOCK) )
(INITIAL_EXTENT_ROWS_TO_DISK * PSEUDO_COL_WIDTH / BYTE_PER_BLOCK) )
{
std::ostringstream oss1;
oss1 << "Skip truncating abbreviated dictionary file"
": OID-" << dctnryOid <<
"; DBRoot-" << root <<
"; part-" << pNum <<
"; seg-" << sNum <<
"; blocks-" << numBlocks;
": OID-" << dctnryOid <<
"; DBRoot-" << root <<
"; part-" << pNum <<
"; seg-" << sNum <<
"; blocks-" << numBlocks;
fLog->logMsg( oss1.str(), MSGLVL_INFO2 );
fTruncateDctnryFileOp.closeFile( dFile );
@ -385,7 +395,8 @@ int ColumnInfoCompressed::truncateDctnryStore(
char* pointerHdr = new char[ptrHdrSize];
rc = fTruncateDctnryFileOp.readFile(dFile,
(unsigned char*)pointerHdr, ptrHdrSize);
(unsigned char*)pointerHdr, ptrHdrSize);
if (rc != NO_ERROR)
{
WErrorCodes ec;
@ -406,6 +417,7 @@ int ColumnInfoCompressed::truncateDctnryStore(
CompChunkPtrList chunkPtrs;
rc1 = compressor.getPtrList( pointerHdr, ptrHdrSize, chunkPtrs );
delete[] pointerHdr;
if (rc1 != 0)
{
rc = ERR_COMP_PARSE_HDRS;
@ -428,33 +440,36 @@ int ColumnInfoCompressed::truncateDctnryStore(
// Truncate the relevant dictionary store file to the nearest extent
if (chunkPtrs.size() > 0)
{
long long dataByteLength = chunkPtrs[chunkPtrs.size()-1].first +
chunkPtrs[chunkPtrs.size()-1].second -
hdrSize;
long long dataByteLength = chunkPtrs[chunkPtrs.size() - 1].first +
chunkPtrs[chunkPtrs.size() - 1].second -
hdrSize;
long long extentBytes =
fRowsPerExtent * PSEUDO_COL_WIDTH;
long long rem = dataByteLength % extentBytes;
if (rem > 0)
{
dataByteLength = dataByteLength - rem + extentBytes;
}
long long truncateFileSize = dataByteLength + hdrSize;
std::ostringstream oss1;
oss1 << "Truncating dictionary file"
": OID-" << dctnryOid <<
"; DBRoot-" << root <<
"; part-" << pNum <<
"; seg-" << sNum <<
"; size-" << truncateFileSize;
": OID-" << dctnryOid <<
"; DBRoot-" << root <<
"; part-" << pNum <<
"; seg-" << sNum <<
"; size-" << truncateFileSize;
fLog->logMsg( oss1.str(), MSGLVL_INFO2 );
if (truncateFileSize > 0)
rc = fTruncateDctnryFileOp.truncateFile(dFile,truncateFileSize);
rc = fTruncateDctnryFileOp.truncateFile(dFile, truncateFileSize);
else
rc = ERR_COMP_TRUNCATE_ZERO;//@bug3913-Catch truncate to 0 bytes
if (rc != NO_ERROR)
{
WErrorCodes ec;
@ -491,20 +506,21 @@ int ColumnInfoCompressed::extendColumnOldExtent(
HWM hwmNextIn )
{
const unsigned int BLKS_PER_EXTENT =
(fRowsPerExtent * column.width)/BYTE_PER_BLOCK;
(fRowsPerExtent * column.width) / BYTE_PER_BLOCK;
// Round up HWM to the end of the current extent
unsigned int nBlks = hwmNextIn + 1;
unsigned int nRem = nBlks % BLKS_PER_EXTENT;
HWM hwmNext = 0;
if (nRem > 0)
hwmNext = nBlks - nRem + BLKS_PER_EXTENT - 1;
else
hwmNext = nBlks - 1;
std::ostringstream oss;
std::ostringstream oss;
oss << "Padding compressed partial extent to extent boundary in OID-" <<
curCol.dataFile.fid <<
curCol.dataFile.fid <<
"; DBRoot-" << dbRootNext <<
"; part-" << partitionNext <<
"; seg-" << segmentNext <<
@ -521,15 +537,16 @@ int ColumnInfoCompressed::extendColumnOldExtent(
std::string segFileName;
std::string errTask;
int rc = colOp->fillCompColumnExtentEmptyChunks(
curCol.dataFile.fid,
curCol.colWidth,
column.emptyVal,
curCol.dataFile.fDbRoot,
curCol.dataFile.fPartition,
curCol.dataFile.fSegment,
curCol.dataFile.hwm,
segFileName,
errTask);
curCol.dataFile.fid,
curCol.colWidth,
column.emptyVal,
curCol.dataFile.fDbRoot,
curCol.dataFile.fPartition,
curCol.dataFile.fSegment,
curCol.dataFile.hwm,
segFileName,
errTask);
if (rc != NO_ERROR)
{
WErrorCodes ec;

16
writeengine/bulk/we_columninfocompressed.h
View File

@ -42,16 +42,16 @@ class RBMetaWriter;
*/
class ColumnInfoCompressed : public ColumnInfo
{
public:
public:
/** @brief Constructor.
*/
ColumnInfoCompressed(Log* logger,
int id,
const JobColumn& column,
DBRootExtentTracker* pDBRootExtTrk,
TableInfo* pTableInfo);
//RBMetaWriter* rbMetaWriter);
int id,
const JobColumn& column,
DBRootExtentTracker* pDBRootExtTrk,
TableInfo* pTableInfo);
//RBMetaWriter* rbMetaWriter);
/** @brief Destructor
*/
@ -71,9 +71,9 @@ class ColumnInfoCompressed : public ColumnInfo
* @param sNum Segment number of relevant dictionary store segment file.
*/
virtual int truncateDctnryStore(OID dctnryOid,
uint16_t root, uint32_t pNum, uint16_t sNum) const;
uint16_t root, uint32_t pNum, uint16_t sNum) const;
private:
private:
virtual int resetFileOffsetsNewExtent(const char* hdr);

96
writeengine/bulk/we_extentstripealloc.cpp
View File

@ -35,11 +35,11 @@
namespace
{
typedef std::tr1::unordered_multimap<WriteEngine::OID,
WriteEngine::AllocExtEntry,
WriteEngine::AllocExtHasher> AllocExtMap;
typedef AllocExtMap::iterator AllocExtMapIter;
typedef AllocExtMap::const_iterator ConstAllocExtMapIter;
typedef std::tr1::unordered_multimap<WriteEngine::OID,
WriteEngine::AllocExtEntry,
WriteEngine::AllocExtHasher> AllocExtMap;
typedef AllocExtMap::iterator AllocExtMapIter;
typedef AllocExtMap::const_iterator ConstAllocExtMapIter;
}
namespace WriteEngine
@ -49,7 +49,7 @@ namespace WriteEngine
// Constructor
//------------------------------------------------------------------------------
ExtentStripeAlloc::ExtentStripeAlloc( OID tableOID,
Log* logger ) : fTableOID(tableOID), fLog(logger), fStripeCount(0)
Log* logger ) : fTableOID(tableOID), fLog(logger), fStripeCount(0)
{
}
@ -80,7 +80,7 @@ void ExtentStripeAlloc::addColumn( OID colOID, int colWidth )
// number, segment number, etc associated with the allocated extent are
// returned in the output arguments.
//
// Note that a multimap is used for the internal extent collection, with the
// Note that a multimap is used for the internal extent collection, with the
// column OID as the key. We would typically expect that we would not have
// more than 1 extent entry for a given column OID in our map at the same
// time. If this were always true, then a map (and not a multimap) would
@ -98,20 +98,20 @@ void ExtentStripeAlloc::addColumn( OID colOID, int colWidth )
// one column extent with the same DBRoot.
//------------------------------------------------------------------------------
int ExtentStripeAlloc::allocateExtent( OID oid,
uint16_t dbRoot,
uint32_t& partNum, // used as input for empty DBRoot, else output only
uint16_t& segNum,
BRM::LBID_t& startLbid,
int& allocSize,
HWM& hwm,
std::string& errMsg )
uint16_t dbRoot,
uint32_t& partNum, // used as input for empty DBRoot, else output only
uint16_t& segNum,
BRM::LBID_t& startLbid,
int& allocSize,
HWM& hwm,
std::string& errMsg )
{
int retStatus = NO_ERROR;
bool bFound = false;
AllocExtMapIter extentEntryIter;
errMsg.clear();
std::pair<AllocExtMapIter,AllocExtMapIter> iters;
std::pair<AllocExtMapIter, AllocExtMapIter> iters;
boost::mutex::scoped_lock lock(fMapMutex);
@ -119,18 +119,20 @@ int ExtentStripeAlloc::allocateExtent( OID oid,
// We also filter by selecting the lowest stripe number. See
// function description that precedes this function for more detail.
iters = fMap.equal_range( oid );
if (iters.first != iters.second)
{
for (AllocExtMapIter it=iters.first; it!=iters.second; ++it)
for (AllocExtMapIter it = iters.first; it != iters.second; ++it)
{
if (it->second.fDbRoot == dbRoot)
{
if ((!bFound) ||
(it->second.fStripeKey <
extentEntryIter->second.fStripeKey))
(it->second.fStripeKey <
extentEntryIter->second.fStripeKey))
{
extentEntryIter = it;
}
bFound = true;
}
}
@ -155,13 +157,14 @@ int ExtentStripeAlloc::allocateExtent( OID oid,
std::ostringstream oss1;
oss1 << "Allocating next stripe(" << fStripeCount <<
") of column extents for table " << fTableOID <<
"; DBRoot-" << dbRoot;
") of column extents for table " << fTableOID <<
"; DBRoot-" << dbRoot;
fLog->logMsg( oss1.str(), MSGLVL_INFO2 );
std::vector<BRM::CreateStripeColumnExtentsArgIn> cols;
std::vector<BRM::CreateStripeColumnExtentsArgOut> extents;
for (unsigned int j=0; j<fColOIDs.size(); ++j)
for (unsigned int j = 0; j < fColOIDs.size(); ++j)
{
BRM::CreateStripeColumnExtentsArgIn colEntry;
colEntry.oid = fColOIDs[j];
@ -178,13 +181,13 @@ int ExtentStripeAlloc::allocateExtent( OID oid,
std::string allocStatusMsg;
int rc = BRMWrapper::getInstance()->allocateStripeColExtents(
cols, dbRoot, allocPartNum, allocSegNum, extents );
cols, dbRoot, allocPartNum, allocSegNum, extents );
// If allocation error occurs, we go ahead and store extent entries
// with error status, to satisfy subsequent allocations in same stripe.
if (rc != NO_ERROR)
{
for (unsigned int i=0; i<fColOIDs.size(); ++i)
for (unsigned int i = 0; i < fColOIDs.size(); ++i)
{
if (oid != fColOIDs[i])
{
@ -197,11 +200,11 @@ int ExtentStripeAlloc::allocateExtent( OID oid,
// For error case, just store 0 for part#,segnum, etc.
AllocExtEntry extentEntry(fColOIDs[i], fColWidths[i],
dbRoot, 0, 0, 0, 0, 0,
allocStatus, allocStatusMsg, fStripeCount );
dbRoot, 0, 0, 0, 0, 0,
allocStatus, allocStatusMsg, fStripeCount );
fMap.insert( AllocExtMap::value_type(fColOIDs[i],
extentEntry) );
extentEntry) );
}
}
@ -215,7 +218,7 @@ int ExtentStripeAlloc::allocateExtent( OID oid,
// Save allocated extents into fMap for later use. For the OID
// requested by this function call, we just return the extent info.
for (unsigned int i=0; i<fColOIDs.size(); ++i)
for (unsigned int i = 0; i < fColOIDs.size(); ++i)
{
allocStartLbid = extents[i].startLbid;
allocAllocSize = extents[i].allocSize;
@ -246,13 +249,14 @@ int ExtentStripeAlloc::allocateExtent( OID oid,
hwm = allocHwm;
}
else // Add all extents in "stripe" (other than requested column)
{ // to the collection of extents
{
// to the collection of extents
AllocExtEntry extentEntry(fColOIDs[i], fColWidths[i],
dbRoot, allocPartNum, allocSegNum,
allocStartLbid, allocAllocSize, allocHwm,
allocStatus, allocStatusMsg, fStripeCount );
dbRoot, allocPartNum, allocSegNum,
allocStartLbid, allocAllocSize, allocHwm,
allocStatus, allocStatusMsg, fStripeCount );
fMap.insert( AllocExtMap::value_type(fColOIDs[i],extentEntry) );
fMap.insert( AllocExtMap::value_type(fColOIDs[i], extentEntry) );
}
}
}
@ -273,22 +277,22 @@ void ExtentStripeAlloc::print( )
if (fMap.size() > 0)
{
for (ConstAllocExtMapIter iter=fMap.begin();
iter!= fMap.end();
++iter)
for (ConstAllocExtMapIter iter = fMap.begin();
iter != fMap.end();
++iter)
{
oss << std::endl;
oss << " oid: " << iter->second.fOid <<
"; wid: " << iter->second.fColWidth <<
"; root: " << iter->second.fDbRoot <<
"; part: " << iter->second.fPartNum <<
"; seg: " << iter->second.fSegNum <<
"; lbid: " << iter->second.fStartLbid <<
"; size: " << iter->second.fAllocSize <<
"; hwm: " << iter->second.fHwm <<
"; stripe: " << iter->second.fStripeKey <<
"; stat: " << iter->second.fStatus <<
"; msg: " << iter->second.fStatusMsg;
"; wid: " << iter->second.fColWidth <<
"; root: " << iter->second.fDbRoot <<
"; part: " << iter->second.fPartNum <<
"; seg: " << iter->second.fSegNum <<
"; lbid: " << iter->second.fStartLbid <<
"; size: " << iter->second.fAllocSize <<
"; hwm: " << iter->second.fHwm <<
"; stripe: " << iter->second.fStripeKey <<
"; stat: " << iter->second.fStatus <<
"; msg: " << iter->second.fStatusMsg;
}
}
else

64
writeengine/bulk/we_extentstripealloc.h
View File

@ -42,7 +42,7 @@
namespace WriteEngine
{
class Log;
class Log;
//------------------------------------------------------------------------------
/** @brief Represents an extent allocation entry that is part of a "stripe".
@ -53,15 +53,15 @@ class AllocExtEntry
public:
// Default constructor
AllocExtEntry() : fOid(0),
fColWidth(0),
fDbRoot(0),
fPartNum(0),
fSegNum(0),
fStartLbid(0),
fAllocSize(0),
fHwm(0),
fStatus(NO_ERROR),
fStripeKey(0) { }
fColWidth(0),
fDbRoot(0),
fPartNum(0),
fSegNum(0),
fStartLbid(0),
fAllocSize(0),
fHwm(0),
fStatus(NO_ERROR),
fStripeKey(0) { }
// Used to create entry for an existing extent we are going to add data to.
AllocExtEntry ( OID& oid, int colWidth,
@ -69,17 +69,17 @@ public:
BRM::LBID_t startLbid, int allocSize,
HWM hwm, int status, const std::string& statusMsg,
unsigned int stripeKey ) :
fOid(oid),
fColWidth(colWidth),
fDbRoot(dbRoot),
fPartNum(partNum),
fSegNum(segNum),
fStartLbid(startLbid),
fAllocSize(allocSize),
fHwm(hwm),
fStatus(status),
fStatusMsg(statusMsg),
fStripeKey(stripeKey) { }
fOid(oid),
fColWidth(colWidth),
fDbRoot(dbRoot),
fPartNum(partNum),
fSegNum(segNum),
fStartLbid(startLbid),
fAllocSize(allocSize),
fHwm(hwm),
fStatus(status),
fStatusMsg(statusMsg),
fStripeKey(stripeKey) { }
OID fOid; // column OID
int fColWidth; // colum width (in bytes)
@ -99,10 +99,12 @@ public:
* the corresponding column OID as the key.
*/
//------------------------------------------------------------------------------
struct AllocExtHasher : public std::unary_function<OID,std::size_t>
struct AllocExtHasher : public std::unary_function<OID, std::size_t>
{
std::size_t operator()(OID val) const
{ return static_cast<std::size_t>(val); }
{
return static_cast<std::size_t>(val);
}
};
//------------------------------------------------------------------------------
@ -149,13 +151,13 @@ public:
* @return NO_ERROR returned upon success
*/
int allocateExtent( OID oid,
uint16_t dbRoot,
uint32_t& partNum,
uint16_t& segNum,
BRM::LBID_t& startLbid,
int& allocSize,
HWM& hwm,
std::string& errMsg );
uint16_t dbRoot,
uint32_t& partNum,
uint16_t& segNum,
BRM::LBID_t& startLbid,
int& allocSize,
HWM& hwm,
std::string& errMsg );
/** @brief Debug print function.
*/
@ -170,7 +172,7 @@ private:
std::vector<int> fColWidths; // Widths associated with fColOIDs
// unordered map where we collect the allocated extents
std::tr1::unordered_multimap<OID,AllocExtEntry,AllocExtHasher> fMap;
std::tr1::unordered_multimap<OID, AllocExtEntry, AllocExtHasher> fMap;
// disable copy constructor and assignment operator
ExtentStripeAlloc(const ExtentStripeAlloc&);

793
writeengine/bulk/we_tableinfo.cpp
View File

File diff suppressed because it is too large Load Diff

301
writeengine/bulk/we_tableinfo.h
View File

@ -50,7 +50,7 @@
namespace WriteEngine
{
/* @brief Class which maintains the information for a table.
*/
class TableInfo
@ -64,36 +64,36 @@ private:
int fTableId; // Table id
int fBufferSize; // Size of buffer used by BulkLoadBuffer
int fFileBufSize; // Size of fFileBuffer passed to setvbuf
// to read import files. Comes from
// writeBufferSize tag in job xml file
// to read import files. Comes from
// writeBufferSize tag in job xml file
char fColDelim; // Used to delimit col values in a row
volatile Status fStatusTI; // Status of table. Made volatile to
// insure BulkLoad methods can access
// (thru getStatusTI()) correctly w/o
// having to go through a mutex lock.
// insure BulkLoad methods can access
// (thru getStatusTI()) correctly w/o
// having to go through a mutex lock.
int fReadBufCount; // Number of read buffers
// (size of fBuffers vector)
// (size of fBuffers vector)
unsigned fNumberOfColumns; // Number of ColumnInfo objs in this tbl
// (size of fColumns vector)
FILE * fHandle; // Handle to the input load file
// (size of fColumns vector)
FILE* fHandle; // Handle to the input load file
int fCurrentReadBuffer; // Id of current buffer being popu-
// lated by the read thread
// lated by the read thread
RID fTotalReadRows; // Total number of rows read
volatile unsigned fTotalErrRows; // Total error rows among all input
// for this table. Is volatile to
// insure parser & reader threads
// see the latest value.
// for this table. Is volatile to
// insure parser & reader threads
// see the latest value.
unsigned fMaxErrorRows; // Maximum error rows
int fLastBufferId; // Id of the last buffer
char* fFileBuffer; // File buffer passed to setvbuf()
int fCurrentParseBuffer; // Id of leading current buffer being
// parsed. There can be more than 1
// buffer being parsed concurrently.
// parsed. There can be more than 1
// buffer being parsed concurrently.
unsigned fNumberOfColsParsed; // Number of columns completed parsing
boost::ptr_vector<ColumnInfo> fColumns; // Columns of the table
boost::ptr_vector<BulkLoadBuffer> fBuffers; // Array of read buffers. Used
// to pass data from the read
// thread to the write thread(s)
// to pass data from the read
// thread to the write thread(s)
/* fSyncUpdatesTI is the mutex used to synchronize updates to TableInfo
* (excluding fErrorRows and fErrDataRows)
@ -101,7 +101,7 @@ private:
* This mutex is also used to coordinate access to fColumnLocks and
* fParseComplete (in BulkLoadBuffer) for the buffers within a table.
* See bulkloadBuffer.h for more information.
*
*
* As the controlling class, TableInfo is the one that is always
* getting/setting the status of the BulkLoadBuffer objects, so
* fSyncUpdatesTI is also used to set/get the BulkLoadBuffer status.
@ -109,7 +109,7 @@ private:
boost::mutex fSyncUpdatesTI;
boost::mutex fErrorRptInfoMutex; // Used to synhronize access to
// fRejectDataFile & fRejectErrFile
// fRejectDataFile & fRejectErrFile
int fLocker; // Read thread id reading this table
std::vector<std::string> fLoadFileList; // Load files
std::string fFileName; // Current load file
@ -121,11 +121,11 @@ private:
timeval fStartTime; // Time when reading and processing began for this table
const BRM::TxnID fTxnID; // Transaction id for the build load
RBMetaWriter fRBMetaWriter; // Manages the writing of bulk roll-
// back meta data for this table
// back meta data for this table
std::string fProcessName; // Name of application process used
// in db table locks
// in db table locks
bool fKeepRbMetaFile; // Keep or delete bulk rollback meta
// data file
// data file
bool fbTruncationAsError; // Treat string truncation as error
ImportDataMode fImportDataMode; // Import data in text or binary mode
@ -135,7 +135,7 @@ private:
bool fNullStringMode; // Treat "NULL" as a null value
char fEnclosedByChar; // Character to enclose col values
char fEscapeChar; // Escape character used in conjunc-
// tion with fEnclosedByChar
// tion with fEnclosedByChar
bool fProcessingBegun; // Has processing begun on this tbl
BulkModeType fBulkMode; // Distributed bulk mode (1,2, or 3)
std::string fBRMRptFileName; // Name of distributed mode rpt file
@ -143,7 +143,7 @@ private:
uint64_t fTableLockID; // Unique table lock ID
std::vector<uint16_t> fOrigDbRootIds; // List of DBRoots at start of job
std::string fErrorDir; // Opt dir for *.err and *.bad files
std::string fErrorDir; // Opt dir for *.err and *.bad files
std::vector<std::string> fErrFiles; // List of *.err files for this table
std::vector<std::string> fBadFiles; // List of *.bad files for this table
std::ofstream fRejectDataFile; // File containing rejected rows
@ -179,20 +179,20 @@ private:
// Write the list of errors for this table
void writeErrorList(const std::vector< std::pair<RID,
std::string> >* errorRows,
const std::vector<std::string>* errorDatRows,
bool bCloseFile);
std::string> >* errorRows,
const std::vector<std::string>* errorDatRows,
bool bCloseFile);
// Write out rejected rows, and corresponding error messages
void writeBadRows( const std::vector<std::string>* errorDatRows,
bool bCloseFile );
void writeErrReason( const std::vector< std::pair<RID,
std::string> >* errorRows,
std::string> >* errorRows,
bool bCloseFile );
// Disable copy constructor and assignment operator
TableInfo (const TableInfo &tableInfo); //
TableInfo & operator =(const TableInfo & info);
TableInfo (const TableInfo& tableInfo); //
TableInfo& operator =(const TableInfo& info);
public:
@ -237,13 +237,13 @@ public:
* @param errMsg (out) Error message
*/
int allocateBRMColumnExtent(OID columnOID,
uint16_t dbRoot,
uint32_t& partition,
uint16_t& segment,
BRM::LBID_t& startLbid,
int& allocSize,
HWM& hwm,
std::string& errMsg );
uint16_t dbRoot,
uint32_t& partition,
uint16_t& segment,
BRM::LBID_t& startLbid,
int& allocSize,
HWM& hwm,
std::string& errMsg );
/** @brief Delete the bulk rollback metadata file.
*/
@ -266,7 +266,7 @@ public:
* @param Buffer size
*/
void setFileBufferSize(const int fileBufSize);
/** @brief Set the delimiter used to delimit column values within a row
*/
void setColDelimiter(const char delim);
@ -279,7 +279,7 @@ public:
*/
int getCurrentParseBuffer() const;
/** @brief Get the number of columns
/** @brief Get the number of columns
*/
int getNumberOfColumns() const;
@ -328,7 +328,7 @@ public:
/** @brief set the table id
*/
void setTableId(const int & id);
void setTableId(const int& id);
/** @brief get the file name
*/
@ -338,18 +338,18 @@ public:
*/
OID getTableOID( );
/** @brief Set the directory for *.err and *.bad files. May be
* empty string, in which case we use current dir.
/** @brief Set the directory for *.err and *.bad files. May be
* empty string, in which case we use current dir.
*/
void setErrorDir(const std::string& errorDir);
/** @brief get the bulk rollback meta data writer object for this table
/** @brief get the bulk rollback meta data writer object for this table
*/
RBMetaWriter* rbMetaWriter();
/** @brief Add column information to the table
*/
void addColumn(ColumnInfo * info);
void addColumn(ColumnInfo* info);
/** @brief Initialize the buffer list
* @param noOfBuffers Number of buffers to create for this table
@ -367,13 +367,13 @@ public:
/** @brief parse method
*/
int parseColumn(const int &columnId, const int &bufferId,
int parseColumn(const int& columnId, const int& bufferId,
double& processingTime);
/** @brief update the buffer status for column
*/
int setParseComplete(const int &columnId,
const int & bufferId,
int setParseComplete(const int& columnId,
const int& bufferId,
double processingTime);
/** @brief update the status to reflect a parsing error
@ -382,12 +382,12 @@ public:
/** @brief Check if buffer ready for parsing.
*/
bool bufferReadyForParse(const int &bufferId, bool report) const;
bool bufferReadyForParse(const int& bufferId, bool report) const;
/** @brief Check if a column is available for parsing in the buffer
* and return the column id if available
*/
int getColumnForParse(const int & id,const int & bufferId,bool report);
int getColumnForParse(const int& id, const int& bufferId, bool report);
/** @brief Do we have a db lock with the session manager for this table.
*/
@ -395,7 +395,7 @@ public:
/** @brief Lock the table for reading
*/
bool lockForRead(const int & locker);
bool lockForRead(const int& locker);
/** @brief Rollback changes made to "this" table by the current import job
*/
@ -408,7 +408,7 @@ public:
/** @brief set job file name under process.
*/
void setJobFileName(const std::string & jobFileName);
void setJobFileName(const std::string& jobFileName);
/** @brief set job ID for this import.
*/
@ -447,8 +447,8 @@ public:
* on this PM.
*/
int saveBulkRollbackMetaData( Job& job,
const std::vector<DBRootExtentInfo>& segFileInfo,
const std::vector<BRM::EmDbRootHWMInfo_v>& dbRootHWMInfoColVec );
const std::vector<DBRootExtentInfo>& segFileInfo,
const std::vector<BRM::EmDbRootHWMInfo_v>& dbRootHWMInfoColVec );
/** @brief Mark table as complete
*/
@ -466,110 +466,175 @@ public:
//------------------------------------------------------------------------------
// Inline functions
//------------------------------------------------------------------------------
inline int TableInfo::getCurrentParseBuffer() const {
return fCurrentParseBuffer; }
inline int TableInfo::getCurrentParseBuffer() const
{
return fCurrentParseBuffer;
}
inline std::string TableInfo::getFileName() const {
return fFileName; }
inline std::string TableInfo::getFileName() const
{
return fFileName;
}
inline ImportDataMode TableInfo::getImportDataMode() const {
return fImportDataMode; }
inline ImportDataMode TableInfo::getImportDataMode() const
{
return fImportDataMode;
}
inline int TableInfo::getNumberOfBuffers() const {
return fReadBufCount; }
inline int TableInfo::getNumberOfBuffers() const
{
return fReadBufCount;
}
inline int TableInfo::getNumberOfColumns() const {
return fNumberOfColumns; }
inline int TableInfo::getNumberOfColumns() const
{
return fNumberOfColumns;
}
inline Status TableInfo::getStatusTI() const {
return fStatusTI; }
inline Status TableInfo::getStatusTI() const
{
return fStatusTI;
}
inline unsigned TableInfo::getMaxErrorRows() const {
return fMaxErrorRows; }
inline unsigned TableInfo::getMaxErrorRows() const
{
return fMaxErrorRows;
}
inline uint64_t TableInfo::getTableLockID() const {
return fTableLockID; }
inline uint64_t TableInfo::getTableLockID() const
{
return fTableLockID;
}
inline std::string TableInfo::getTableName() const {
return fTableName; }
inline std::string TableInfo::getTableName() const
{
return fTableName;
}
inline OID TableInfo::getTableOID( ) {
return fTableOID; }
inline OID TableInfo::getTableOID( )
{
return fTableOID;
}
inline bool TableInfo::getTruncationAsError() const {
return fbTruncationAsError; }
inline bool TableInfo::getTruncationAsError() const
{
return fbTruncationAsError;
}
inline bool TableInfo::hasProcessingBegun() {
return fProcessingBegun; }
inline bool TableInfo::hasProcessingBegun()
{
return fProcessingBegun;
}
inline bool TableInfo::isTableLocked() {
return fTableLocked; }
inline bool TableInfo::isTableLocked()
{
return fTableLocked;
}
inline void TableInfo::markTableComplete() {
inline void TableInfo::markTableComplete()
{
boost::mutex::scoped_lock lock(fSyncUpdatesTI);
fStatusTI = WriteEngine::PARSE_COMPLETE; }
fStatusTI = WriteEngine::PARSE_COMPLETE;
}
inline RBMetaWriter* TableInfo::rbMetaWriter() {
return &fRBMetaWriter; }
inline RBMetaWriter* TableInfo::rbMetaWriter()
{
return &fRBMetaWriter;
}
inline void TableInfo::setBufferSize(const int bufSize) {
fBufferSize = bufSize; }
inline void TableInfo::setBufferSize(const int bufSize)
{
fBufferSize = bufSize;
}
inline void TableInfo::setColDelimiter(const char delim) {
fColDelim = delim; }
inline void TableInfo::setColDelimiter(const char delim)
{
fColDelim = delim;
}
inline void TableInfo::setBulkLoadMode(
BulkModeType bulkMode,
const std::string& rptFileName ) {
const std::string& rptFileName )
{
fBulkMode = bulkMode,
fBRMRptFileName = rptFileName; }
fBRMRptFileName = rptFileName;
}
inline void TableInfo::setEnclosedByChar( char enChar ) {
fEnclosedByChar = enChar; }
inline void TableInfo::setEnclosedByChar( char enChar )
{
fEnclosedByChar = enChar;
}
inline void TableInfo::setEscapeChar ( char esChar ) {
fEscapeChar = esChar; }
inline void TableInfo::setEscapeChar ( char esChar )
{
fEscapeChar = esChar;
}
inline void TableInfo::setFileBufferSize(const int fileBufSize) {
fFileBufSize = fileBufSize; }
inline void TableInfo::setFileBufferSize(const int fileBufSize)
{
fFileBufSize = fileBufSize;
}
inline void TableInfo::setImportDataMode( ImportDataMode importMode ) {
fImportDataMode = importMode; }
inline void TableInfo::setImportDataMode( ImportDataMode importMode )
{
fImportDataMode = importMode;
}
inline void TableInfo::setJobFileName(const std::string & jobFileName) {
fjobFileName = jobFileName; }
inline void TableInfo::setJobFileName(const std::string& jobFileName)
{
fjobFileName = jobFileName;
}
inline void TableInfo::setJobId(int jobId) {
fJobId = jobId; }
inline void TableInfo::setJobId(int jobId)
{
fJobId = jobId;
}
inline void TableInfo::setLoadFilesInput(bool bReadFromStdin,
const std::vector<std::string>& files) {
const std::vector<std::string>& files)
{
fReadFromStdin = bReadFromStdin;
fLoadFileList = files; }
fLoadFileList = files;
}
inline void TableInfo::setMaxErrorRows(const unsigned int maxErrorRows) {
fMaxErrorRows = maxErrorRows; }
inline void TableInfo::setMaxErrorRows(const unsigned int maxErrorRows)
{
fMaxErrorRows = maxErrorRows;
}
inline void TableInfo::setNullStringMode( bool bMode ) {
fNullStringMode = bMode; }
inline void TableInfo::setNullStringMode( bool bMode )
{
fNullStringMode = bMode;
}
inline void TableInfo::setTableId(const int & id) {
fTableId = id; }
inline void TableInfo::setTableId(const int& id)
{
fTableId = id;
}
inline void TableInfo::setTruncationAsError(bool bTruncationAsError) {
fbTruncationAsError = bTruncationAsError; }
inline void TableInfo::setTruncationAsError(bool bTruncationAsError)
{
fbTruncationAsError = bTruncationAsError;
}
inline void TableInfo::setJobUUID(const boost::uuids::uuid& jobUUID) {
fJobUUID = jobUUID; }
inline void TableInfo::setJobUUID(const boost::uuids::uuid& jobUUID)
{
fJobUUID = jobUUID;
}
inline void TableInfo::setErrorDir( const std::string& errorDir ) {
fErrorDir = errorDir;
inline void TableInfo::setErrorDir( const std::string& errorDir )
{
fErrorDir = errorDir;
#ifdef _MSC_VER
if (fErrorDir.length() > 0 && *(--(fErrorDir.end())) != '/' && *(--(fErrorDir.end())) != '\\')
fErrorDir.push_back('\\'); }
if (fErrorDir.length() > 0 && *(--(fErrorDir.end())) != '/' && *(--(fErrorDir.end())) != '\\')
fErrorDir.push_back('\\');
}
#else
if (fErrorDir.length() > 0 && *(--(fErrorDir.end())) != '/')
fErrorDir.push_back('/'); }
fErrorDir.push_back('/');
}
#endif
}
#endif

5
writeengine/bulk/we_tempxmlgendata.cpp
View File

@ -34,8 +34,8 @@ namespace WriteEngine
// TempXMLGenData constructor
//------------------------------------------------------------------------------
TempXMLGenData::TempXMLGenData(const std::string& jobId,
const std::string& schema,
const std::string& table)
const std::string& schema,
const std::string& table)
{
fParms[ JOBID] = jobId; // add or override default value
fSchema = schema;
@ -57,6 +57,7 @@ TempXMLGenData::~TempXMLGenData( )
void TempXMLGenData::print(std::ostream& os) const
{
os << "Generating runtime job xml file for: schema-" << fSchema;
if (fTables.size() > 0)
os << ": table-" << fTables[0];
}

8
writeengine/bulk/we_tempxmlgendata.h
View File

@ -37,16 +37,16 @@ namespace WriteEngine
*/
class TempXMLGenData : public XMLGenData
{
public:
public:
TempXMLGenData(const std::string& jobId,
const std::string& schema,
const std::string& table );
const std::string& schema,
const std::string& table );
virtual ~TempXMLGenData( );
virtual void print(std::ostream& os) const;
private:
private:
TempXMLGenData(const TempXMLGenData&); //disable default copy ctor
TempXMLGenData& operator=(const TempXMLGenData&);//disable def assignment
};

161
writeengine/bulk/we_workers.cpp
View File

@ -40,20 +40,22 @@ void BulkLoad::sleepMS(long ms)
{
struct timespec rm_ts;
rm_ts.tv_sec = ms/1000;
rm_ts.tv_nsec = ms%1000 *1000000;
rm_ts.tv_sec = ms / 1000;
rm_ts.tv_nsec = ms % 1000 * 1000000;
#ifdef _MSC_VER
Sleep(ms);
Sleep(ms);
#else
struct timespec abs_ts;
do
{
abs_ts.tv_sec = rm_ts.tv_sec;
abs_ts.tv_sec = rm_ts.tv_sec;
abs_ts.tv_nsec = rm_ts.tv_nsec;
}
while(nanosleep(&abs_ts,&rm_ts) < 0);
#endif
}
while (nanosleep(&abs_ts, &rm_ts) < 0);
#endif
}
//------------------------------------------------------------------------------
// This is the main entry point method for each Read thread.
@ -64,7 +66,7 @@ void BulkLoad::read(int id)
#ifdef PROFILE
Stats::registerReadProfThread( );
#endif
// First get a table to work on.
// First get a table to work on.
// Acquire the read mutex
// Iterate over the table list
// if the table's status is new, set the locker = id
@ -76,27 +78,30 @@ void BulkLoad::read(int id)
//
// LOOP to select and read the next table
//
while(true)
while (true)
{
tableId = -1;
#ifdef PROFILE
Stats::startReadEvent(WE_STATS_WAIT_TO_SELECT_TBL);
#endif
if((tableId = lockTableForRead(id)) == -1)
if ((tableId = lockTableForRead(id)) == -1)
{
fLog.logMsg( "BulkLoad::ReadOperation No more tables "
"available for processing. Read thread "
+ Convertor::int2Str(id) + " exiting...",
MSGLVL_INFO2);
"available for processing. Read thread "
+ Convertor::int2Str(id) + " exiting...",
MSGLVL_INFO2);
#ifdef PROFILE
Stats::stopReadEvent(WE_STATS_WAIT_TO_SELECT_TBL);
#endif
return;
}
#ifdef PROFILE
Stats::stopReadEvent(WE_STATS_WAIT_TO_SELECT_TBL);
#endif
int rc = fTableInfo[tableId].readTableData( );
if (rc != NO_ERROR)
{
// Error occurred while reading the data, break out of loop.
@ -116,6 +121,7 @@ void BulkLoad::read(int id)
{
// We are bailing out because another thread set bad job status
ostringstream oss;
if (tableId != -1)
oss << "Bulkload Read (thread " << id <<
") Stopped reading Table " <<
@ -123,12 +129,14 @@ void BulkLoad::read(int id)
else
oss << "Bulkload Read (thread " << id <<
") Stopped reading Tables. " << ex.what();
fLog.logMsg( oss.str(), MSGLVL_INFO1 );
}
catch (exception& ex)
{
BulkStatus::setJobStatus( EXIT_FAILURE );
ostringstream oss;
if (tableId != -1)
oss << "Bulkload Read (thread " << id <<
") Failed for Table " <<
@ -138,13 +146,16 @@ void BulkLoad::read(int id)
oss << "Bulkload Read (thread " << id <<
") Failed for Table. " << ex.what() <<
". Terminating this job.";
if(tableId != -1) fTableInfo[tableId].fBRMReporter.addToErrMsgEntry(oss.str());
fLog.logMsg( oss.str(), ERR_UNKNOWN, MSGLVL_CRITICAL );
if (tableId != -1) fTableInfo[tableId].fBRMReporter.addToErrMsgEntry(oss.str());
fLog.logMsg( oss.str(), ERR_UNKNOWN, MSGLVL_CRITICAL );
}
catch (...)
{
BulkStatus::setJobStatus( EXIT_FAILURE );
ostringstream oss;
if (tableId != -1)
oss << "Bulkload Read (thread " << id <<
") Failed for Table " <<
@ -153,8 +164,10 @@ void BulkLoad::read(int id)
else
oss << "Bulkload Read (thread " << id <<
") Failed for Table. Terminating this job.";
if(tableId != -1) fTableInfo[tableId].fBRMReporter.addToErrMsgEntry(oss.str());
fLog.logMsg( oss.str(), ERR_UNKNOWN, MSGLVL_CRITICAL );
if (tableId != -1) fTableInfo[tableId].fBRMReporter.addToErrMsgEntry(oss.str());
fLog.logMsg( oss.str(), ERR_UNKNOWN, MSGLVL_CRITICAL );
}
}
@ -167,14 +180,14 @@ void BulkLoad::read(int id)
int BulkLoad::lockTableForRead(int id)
{
boost::mutex::scoped_lock lock(fReadMutex);
for(unsigned i=0; i<fTableInfo.size(); ++i)
for (unsigned i = 0; i < fTableInfo.size(); ++i)
{
if(fTableInfo[i].lockForRead(id))
return i;
if (fTableInfo[i].lockForRead(id))
return i;
}
return -1;
return -1;
}
//------------------------------------------------------------------------------
@ -197,13 +210,13 @@ void BulkLoad::parse(int id)
//
// LOOP to parse BulkLoadBuffers as they're loaded by Read thread(s)
//
while(true)
while (true)
{
#ifdef PROFILE
Stats::startParseEvent(WE_STATS_WAIT_TO_SELECT_COL);
#endif
// @bug 3271: Conditionally compile the thread deadlock debug logging
// @bug 3271: Conditionally compile the thread deadlock debug logging
#ifdef DEADLOCK_DEBUG
// @bug2099+ Temporary hack.
struct timeval tvStart;
@ -219,7 +232,7 @@ void BulkLoad::parse(int id)
// LOOP to wait and select table/column/buffers
// (BulkLoadBuffers) as they are loaded by the Read buffer.
//
while(true)
while (true)
{
tableId = -1;
columnId = -1;
@ -229,10 +242,10 @@ void BulkLoad::parse(int id)
if (BulkStatus::getJobStatus() == EXIT_FAILURE)
{
throw SecondaryShutdownException( "BulkLoad::"
"parse() responding to job termination");
"parse() responding to job termination");
}
if(allTablesDone(WriteEngine::PARSE_COMPLETE))
if (allTablesDone(WriteEngine::PARSE_COMPLETE))
{
#ifdef PROFILE
Stats::stopParseEvent(WE_STATS_WAIT_TO_SELECT_COL);
@ -242,25 +255,28 @@ void BulkLoad::parse(int id)
return;
}
if(lockColumnForParse(id, tableId, columnId,
myParseBuffer, report))
if (lockColumnForParse(id, tableId, columnId,
myParseBuffer, report))
break;
// Sleep and check the condition again.
sleepMS(1);
#ifdef DEADLOCK_DEBUG
// @bug2099+
if(report) report = false; // report one time.
if(!reported)
if (report) report = false; // report one time.
if (!reported)
{
struct timeval tvNow;
gettimeofday(&tvNow, 0);
if((tvNow.tv_sec - tvStart.tv_sec) >= 100)
if ((tvNow.tv_sec - tvStart.tv_sec) >= 100)
{
time_t t = time(0);
char timeString[50];
ctime_r(&t, timeString);
timeString[ strlen(timeString)-1 ] = '\0';
timeString[ strlen(timeString) - 1 ] = '\0';
ostringstream oss;
oss << endl << endl << timeString <<
": BulkLoad::parse(" << id << "); " <<
@ -278,18 +294,21 @@ void BulkLoad::parse(int id)
reported = true;
}
}
// @bug2099-
#endif
}
#ifdef PROFILE
Stats::stopParseEvent(WE_STATS_WAIT_TO_SELECT_COL);
#endif
// Have obtained the table and column for parsing.
// Start parsing the column data.
double processingTime;
int rc = fTableInfo[tableId].parseColumn(columnId,myParseBuffer,
processingTime);
if(rc != NO_ERROR)
int rc = fTableInfo[tableId].parseColumn(columnId, myParseBuffer,
processingTime);
if (rc != NO_ERROR)
{
// Error occurred while parsing the data, break out of loop.
BulkStatus::setJobStatus( EXIT_FAILURE );
@ -307,7 +326,7 @@ void BulkLoad::parse(int id)
// Parsing is complete. Acquire the mutex and increment
// the parsingComplete value for the buffer
if(fTableInfo[tableId].getStatusTI() != WriteEngine::ERR)
if (fTableInfo[tableId].getStatusTI() != WriteEngine::ERR)
{
#ifdef PROFILE
Stats::startParseEvent(WE_STATS_WAIT_TO_COMPLETE_PARSE);
@ -318,8 +337,9 @@ void BulkLoad::parse(int id)
Stats::startParseEvent(WE_STATS_COMPLETING_PARSE);
#endif
rc = fTableInfo[tableId].setParseComplete(columnId,
myParseBuffer,
processingTime);
myParseBuffer,
processingTime);
if (rc != NO_ERROR)
{
BulkStatus::setJobStatus( EXIT_FAILURE );
@ -334,6 +354,7 @@ void BulkLoad::parse(int id)
setParseErrorOnTable( tableId, false );
return;
}
#ifdef PROFILE
Stats::stopParseEvent(WE_STATS_COMPLETING_PARSE);
#endif
@ -344,24 +365,30 @@ void BulkLoad::parse(int id)
{
// We are bailing out because another thread set bad job status
ostringstream oss;
if (tableId != -1) {
if (tableId != -1)
{
oss << "Bulkload Parse (thread " << id <<
") Stopped parsing Table " <<
fTableInfo[tableId].getTableName() << ". " << ex.what();
setParseErrorOnTable( tableId, true );
}
else {
else
{
oss << "Bulkload Parse (thread " << id <<
") Stopped parsing Tables. " << ex.what();
}
fLog.logMsg( oss.str(), MSGLVL_INFO1 );
}
catch (exception& ex)
{
BulkStatus::setJobStatus( EXIT_FAILURE );
ostringstream oss;
if (tableId != -1) {
if (tableId != -1)
{
oss << "Bulkload Parse (thread " << id <<
") Failed for Table " <<
fTableInfo[tableId].getTableName() << ". " << ex.what() <<
@ -370,18 +397,22 @@ void BulkLoad::parse(int id)
setParseErrorOnTable( tableId, true );
fTableInfo[tableId].fBRMReporter.addToErrMsgEntry(oss.str());
}
else {
else
{
oss << "Bulkload Parse (thread " << id <<
") Failed for Table. " << ex.what() <<
". Terminating this job.";
}
fLog.logMsg( oss.str(), ERR_UNKNOWN, MSGLVL_CRITICAL );
fLog.logMsg( oss.str(), ERR_UNKNOWN, MSGLVL_CRITICAL );
}
catch (...)
{
BulkStatus::setJobStatus( EXIT_FAILURE );
ostringstream oss;
if (tableId != -1) {
if (tableId != -1)
{
oss << "Bulkload Parse (thread " << id <<
") Failed for Table " <<
fTableInfo[tableId].getTableName() <<
@ -390,11 +421,13 @@ void BulkLoad::parse(int id)
setParseErrorOnTable( tableId, true );
fTableInfo[tableId].fBRMReporter.addToErrMsgEntry(oss.str());
}
else {
else
{
oss << "Bulkload Parse (thread " << id <<
") Failed for Table. Terminating this job.";
}
fLog.logMsg( oss.str(), ERR_UNKNOWN, MSGLVL_CRITICAL );
fLog.logMsg( oss.str(), ERR_UNKNOWN, MSGLVL_CRITICAL );
}
}
@ -406,11 +439,11 @@ void BulkLoad::parse(int id)
//------------------------------------------------------------------------------
// @bug2099 - Temp hack to diagnose deadlock. Added report parm and couts below.
bool BulkLoad::lockColumnForParse(
int thrdId,
int & tableId,
int & columnId,
int & myParseBuffer,
bool report)
int thrdId,
int& tableId,
int& columnId,
int& myParseBuffer,
bool report)
{
// Acquire mutex
// Iterate on the table list
@ -419,17 +452,18 @@ bool BulkLoad::lockColumnForParse(
// else, go to the next table for checking if a column is available
boost::mutex::scoped_lock lock(fParseMutex);
for(unsigned i=0; i<fTableInfo.size(); ++i)
for (unsigned i = 0; i < fTableInfo.size(); ++i)
{
if (fTableInfo[i].getStatusTI() == WriteEngine::PARSE_COMPLETE)
continue;
int currentParseBuffer = fTableInfo[i].getCurrentParseBuffer();
myParseBuffer = currentParseBuffer;
do
{
// @bug2099+
if(report)
if (report)
{
ostringstream oss;
std::string bufStatusStr;
@ -443,27 +477,31 @@ bool BulkLoad::lockColumnForParse(
#endif
":fTableInfo[" << i << "]" << bufStatusStr << " (" <<
stat << ")";
if ( stat != WriteEngine::PARSE_COMPLETE )
{
oss << "; fCurrentParseBuffer is " << myParseBuffer;
}
oss << endl;
cout << oss.str();
cout.flush();
}
// @bug2099-
// get a buffer and column to parse if available.
if((columnId = fTableInfo[i].getColumnForParse(
thrdId,myParseBuffer,report )) != -1)
if ((columnId = fTableInfo[i].getColumnForParse(
thrdId, myParseBuffer, report )) != -1)
{
tableId = i;
return true;
}
myParseBuffer = (myParseBuffer + 1) %
fTableInfo[i].getNumberOfBuffers();
}
while(myParseBuffer != currentParseBuffer);
}
while (myParseBuffer != currentParseBuffer);
}
return false;
@ -481,11 +519,12 @@ bool BulkLoad::lockColumnForParse(
//------------------------------------------------------------------------------
bool BulkLoad::allTablesDone(Status status)
{
for(unsigned i=0; i<fTableInfo.size(); ++i)
for (unsigned i = 0; i < fTableInfo.size(); ++i)
{
if(fTableInfo[i].getStatusTI() == WriteEngine::ERR)
if (fTableInfo[i].getStatusTI() == WriteEngine::ERR)
return true;
if(fTableInfo[i].getStatusTI() != status)
if (fTableInfo[i].getStatusTI() != status)
return false;
}