database/mariadb-columnstore-engine
1
0
Fork 0
mirror of https://github.com/mariadb-corporation/mariadb-columnstore-engine.git synced 2025-07-30 19:23:07 +03:00
Code Activity

MCOL-265 Add support for TIMESTAMP data type

Browse Source
This commit is contained in:
Gagan Goel
2019-03-17 14:14:03 -04:00
parent 8a7ccd7d93
commit e89d1ac3cf
167 changed files with 4346 additions and 250 deletions
Download Patch File Download Diff File Expand all files Collapse all files

27
writeengine/bulk/cpimport.cpp
View File

@ -51,6 +51,7 @@
#include "utils_utf8.h"
#include "IDBPolicy.h"
#include "MonitorProcMem.h"
#include "dataconvert.h"
using namespace std;
using namespace WriteEngine;
@ -104,7 +105,7 @@ void printUsage()
" [-c readBufSize] [-e maxErrs] [-B libBufSize] [-n NullOption] " << endl <<
" [-E encloseChar] [-C escapeChar] [-I binaryOpt] [-S] "
"[-d debugLevel] [-i] " << endl <<
" [-D] [-N] [-L rejectDir]" << endl;
" [-D] [-N] [-L rejectDir] [-T timeZone]" << endl;
cout << endl << "Traditional usage without positional parameters "
"(XML job file required):" << endl <<
@ -114,7 +115,7 @@ void printUsage()
" [-E encloseChar] [-C escapeChar] [-I binaryOpt] [-S] "
"[-d debugLevel] [-i] " << endl <<
" [-p path] [-l loadFile]" << endl <<
" [-D] [-N] [-L rejectDir]" << endl << endl;
" [-D] [-N] [-L rejectDir] [-T timeZone]" << endl << endl;
cout << " Positional parameters:" << endl <<
" dbName Name of database to load" << endl <<
@ -162,7 +163,10 @@ void printUsage()
" -S Treat string truncations as errors" << endl <<
" -D Disable timeout when waiting for table lock" << endl <<
" -N Disable console output" << endl <<
" -L send *.err and *.bad (reject) files here" << endl << endl;
" -L send *.err and *.bad (reject) files here" << endl <<
" -T Timezone used for TIMESTAMP datatype" << endl <<
" Possible values: \"SYSTEM\" (default)" << endl <<
" : Offset in the form +/-HH:MM" << endl << endl;
cout << " Example1:" << endl <<
" cpimport.bin -j 1234" << endl <<
@ -313,7 +317,7 @@ void parseCmdLineArgs(
std::string jobUUID;
while ( (option = getopt(
argc, argv, "b:c:d:e:f:hij:kl:m:n:p:r:s:u:w:B:C:DE:I:P:R:SX:NL:")) != EOF )
argc, argv, "b:c:d:e:f:hij:kl:m:n:p:r:s:u:w:B:C:DE:I:P:R:ST:X:NL:")) != EOF )
{
switch (option)
{
@ -668,6 +672,21 @@ void parseCmdLineArgs(
break;
}
case 'T':
{
std::string timeZone = optarg;
long offset;
if (timeZone != "SYSTEM" && dataconvert::timeZoneToOffset(timeZone.c_str(), timeZone.size(), &offset))
{
startupError ( std::string(
"Value for option -T is invalid"), true );
}
curJob.setTimeZone( timeZone );
break;
}
case 'X': // Hidden extra options
{
if (!strcmp(optarg, "AllowMissingColumn"))

5
writeengine/bulk/we_bulkload.cpp
View File

@ -156,7 +156,8 @@ BulkLoad::BulkLoad() :
fImportDataMode(IMPORT_DATA_TEXT),
fbContinue(false),
fDisableTimeOut(false),
fUUID(boost::uuids::nil_generator()())
fUUID(boost::uuids::nil_generator()()),
fTimeZone("SYSTEM")
{
fTableInfo.clear();
setDebugLevel( DEBUG_0 );
@ -251,6 +252,7 @@ int BulkLoad::loadJobInfo(
{
fJobFileName = fullName;
fRootDir = Config::getBulkRoot();
fJobInfo.setTimeZone(fTimeZone);
if ( !exists( fullName.c_str() ) )
{
@ -486,6 +488,7 @@ int BulkLoad::preProcess( Job& job, int tableNo,
tableInfo->setEnclosedByChar(fEnclosedByChar);
tableInfo->setEscapeChar(fEscapeChar);
tableInfo->setImportDataMode(fImportDataMode);
tableInfo->setTimeZone(fTimeZone);
tableInfo->setJobUUID(fUUID);
if (fMaxErrors != -1)

13
writeengine/bulk/we_bulkload.h
View File

@ -115,6 +115,7 @@ public:
void addToCmdLineImportFileList(const std::string& importFile);
const std::string& getAlternateImportDir( ) const;
const std::string& getErrorDir ( ) const;
const std::string& getTimeZone ( ) const;
const std::string& getJobDir ( ) const;
const std::string& getSchema ( ) const;
const std::string& getTempJobDir ( ) const;
@ -149,6 +150,7 @@ public:
void setTruncationAsError ( bool bTruncationAsError );
void setJobUUID ( const std::string& jobUUID );
void setErrorDir ( const std::string& errorDir );
void setTimeZone ( const std::string& timeZone );
// Timer functions
void startTimer ( );
void stopTimer ( );
@ -227,6 +229,7 @@ private:
bool fDisableTimeOut; // disable timeout when waiting for table lock
boost::uuids::uuid fUUID; // job UUID
static bool fNoConsoleOutput; // disable output to console
std::string fTimeZone; // Timezone to use for TIMESTAMP data type
//--------------------------------------------------------------------------
// Private Functions
@ -319,6 +322,11 @@ inline const std::string& BulkLoad::getErrorDir( ) const
return fErrorDir;
}
inline const std::string& BulkLoad::getTimeZone( ) const
{
return fTimeZone;
}
inline const std::string& BulkLoad::getJobDir( ) const
{
return DIR_BULK_JOB;
@ -459,6 +467,11 @@ inline void BulkLoad::setErrorDir( const std::string& errorDir )
fErrorDir = errorDir;
}
inline void BulkLoad::setTimeZone( const std::string& timeZone )
{
fTimeZone = timeZone;
}
inline void BulkLoad::startTimer( )
{
gettimeofday( &fStartTime, 0 );

60
writeengine/bulk/we_bulkloadbuffer.cpp
View File

@ -139,6 +139,7 @@ BulkLoadBuffer::BulkLoadBuffer(
fEnclosedByChar('\0'), fEscapeChar('\\'),
fBufferId(bufferId), fTableName(tableName),
fbTruncationAsError(false), fImportDataMode(IMPORT_DATA_TEXT),
fTimeZone("SYSTEM"),
fFixedBinaryRecLen(0)
{
fData = new char[bufferSize];
@ -939,6 +940,7 @@ void BulkLoadBuffer::convert(char* field, int fieldLength,
bool bSatVal = false;
if ( column.dataType != CalpontSystemCatalog::DATETIME &&
column.dataType != CalpontSystemCatalog::TIMESTAMP &&
column.dataType != CalpontSystemCatalog::TIME )
{
if (nullFlag)
@ -1072,6 +1074,59 @@ void BulkLoadBuffer::convert(char* field, int fieldLength,
pVal = &llDate;
}
else if (column.dataType == CalpontSystemCatalog::TIMESTAMP)
{
// timestamp conversion
int rc = 0;
if (nullFlag)
{
if (column.fWithDefault)
{
llDate = column.fDefaultInt;
// fall through to update saturation and min/max
}
else
{
llDate = joblist::TIMESTAMPNULL;
pVal = &llDate;
break;
}
}
else
{
if (fImportDataMode != IMPORT_DATA_TEXT)
{
memcpy(&llDate, field, sizeof(llDate));
if (!dataconvert::DataConvert::isColumnTimeStampValid(
llDate))
rc = -1;
}
else
{
llDate = dataconvert::DataConvert::convertColumnTimestamp(
field, dataconvert::CALPONTDATETIME_ENUM,
rc, fieldLength, fTimeZone );
}
}
if (rc == 0)
{
if (llDate < bufStats.minBufferVal)
bufStats.minBufferVal = llDate;
if (llDate > bufStats.maxBufferVal)
bufStats.maxBufferVal = llDate;
}
else
{
llDate = 0;
bufStats.satCount++;
}
pVal = &llDate;
}
else
{
// datetime conversion
@ -3089,6 +3144,11 @@ bool BulkLoadBuffer::isBinaryFieldNull(void* val,
if ((*(uint64_t*)val) == joblist::DATETIMENULL)
isNullFlag = true;
}
else if (dt == execplan::CalpontSystemCatalog::TIMESTAMP)
{
if ((*(uint64_t*)val) == joblist::TIMESTAMPNULL)
isNullFlag = true;
}
else if (dt == execplan::CalpontSystemCatalog::TIME)
{
if ((*(uint64_t*)val) == joblist::TIMENULL)

8
writeengine/bulk/we_bulkloadbuffer.h
View File

@ -139,6 +139,7 @@ private:
// for db cols (omits default cols)
bool fbTruncationAsError; // Treat string truncation as error
ImportDataMode fImportDataMode; // Import data in text or binary mode
std::string fTimeZone; // Timezone used by TIMESTAMP datatype
unsigned int fFixedBinaryRecLen; // Fixed rec len used in binary mode
//--------------------------------------------------------------------------
@ -381,6 +382,13 @@ public:
fImportDataMode = importMode;
fFixedBinaryRecLen = fixedBinaryRecLen;
}
/** @brief set timezone.
*/
void setTimeZone(const std::string& timeZone)
{
fTimeZone = timeZone;
}
};
inline bool isTrueWord(const char *field, int fieldLength)

7
writeengine/bulk/we_tableinfo.cpp
View File

@ -114,6 +114,7 @@ TableInfo::TableInfo(Log* logger, const BRM::TxnID txnID,
fKeepRbMetaFile(bKeepRbMetaFile),
fbTruncationAsError(false),
fImportDataMode(IMPORT_DATA_TEXT),
fTimeZone("SYSTEM"),
fTableLocked(false),
fReadFromStdin(false),
fNullStringMode(false),
@ -989,6 +990,11 @@ void TableInfo::reportTotals(double elapsedTime)
ossSatCnt <<
"invalid date/times replaced with zero value : ";
}
else if (fColumns[i].column.dataType == CalpontSystemCatalog::TIMESTAMP)
{
ossSatCnt <<
"invalid timestamps replaced with zero value : ";
}
else if (fColumns[i].column.dataType == CalpontSystemCatalog::TIME)
{
ossSatCnt <<
@ -1246,6 +1252,7 @@ void TableInfo::initializeBuffers(int noOfBuffers,
buffer->setTruncationAsError(getTruncationAsError());
buffer->setImportDataMode(fImportDataMode,
fixedBinaryRecLen);
buffer->setTimeZone(fTimeZone);
fBuffers.push_back(buffer);
}
}

19
writeengine/bulk/we_tableinfo.h
View File

@ -128,6 +128,7 @@ private:
// data file
bool fbTruncationAsError; // Treat string truncation as error
ImportDataMode fImportDataMode; // Import data in text or binary mode
std::string fTimeZone; // Timezone used by TIMESTAMP data type
volatile bool fTableLocked; // Do we have db table lock
@ -253,6 +254,10 @@ public:
*/
ImportDataMode getImportDataMode( ) const;
/** @brief Get timezone.
*/
const std::string& getTimeZone( ) const;
/** @brief Get number of buffers
*/
int getNumberOfBuffers() const;
@ -310,6 +315,10 @@ public:
*/
void setImportDataMode( ImportDataMode importMode );
/** @brief Set timezone.
*/
void setTimeZone( const std::string& timeZone );
/** @brief Enable distributed mode, saving BRM updates in rptFileName
*/
void setBulkLoadMode(BulkModeType bulkMode, const std::string& rptFileName);
@ -481,6 +490,11 @@ inline ImportDataMode TableInfo::getImportDataMode() const
return fImportDataMode;
}
inline const std::string& TableInfo::getTimeZone() const
{
return fTimeZone;
}
inline int TableInfo::getNumberOfBuffers() const
{
return fReadBufCount;
@ -580,6 +594,11 @@ inline void TableInfo::setImportDataMode( ImportDataMode importMode )
fImportDataMode = importMode;
}
inline void TableInfo::setTimeZone( const std::string& timeZone )
{
fTimeZone = timeZone;
}
inline void TableInfo::setJobFileName(const std::string& jobFileName)
{
fjobFileName = jobFileName;

4
writeengine/server/we_ddlcommandproc.cpp
View File

@ -3537,6 +3537,7 @@ uint8_t WE_DDLCommandProc::fillNewColumn(ByteStream& bs, std::string& err)
int dataWidth, scale, precision, compressionType, refColWidth, refCompressionType;
string defaultValStr;
ColTuple defaultVal;
string timeZone;
bs >> tmp32;
txnID = tmp32;
@ -3565,6 +3566,7 @@ uint8_t WE_DDLCommandProc::fillNewColumn(ByteStream& bs, std::string& err)
refColWidth = tmp32;
bs >> tmp8;
refCompressionType = tmp8;
bs >> timeZone;
//Find the fill in value
bool isNULL = false;
@ -3577,7 +3579,7 @@ uint8_t WE_DDLCommandProc::fillNewColumn(ByteStream& bs, std::string& err)
colType.scale = scale;
colType.precision = precision;
bool pushWarning = false;
defaultVal.data = DataConvert::convertColumnData(colType, defaultValStr, pushWarning, isNULL);
defaultVal.data = DataConvert::convertColumnData(colType, defaultValStr, pushWarning, timeZone, isNULL, false, false);
fWEWrapper.setTransId(txnID);
fWEWrapper.setIsInsert(true);
fWEWrapper.setBulkFlag(true);

11
writeengine/server/we_ddlcommon.h
View File

@ -281,6 +281,13 @@ inline boost::any getNullValueForType(const execplan::CalpontSystemCatalog::ColT
}
break;
case execplan::CalpontSystemCatalog::TIMESTAMP:
{
long long d = joblist::TIMESTAMPNULL;
value = d;
}
break;
case execplan::CalpontSystemCatalog::CHAR:
{
std::string charnull;
@ -444,6 +451,10 @@ inline int convertDataType(int dataType)
calpontDataType = CalpontSystemCatalog::TIME;
break;
case ddlpackage::DDL_TIMESTAMP:
calpontDataType = CalpontSystemCatalog::TIMESTAMP;
break;
case ddlpackage::DDL_CLOB:
calpontDataType = CalpontSystemCatalog::CLOB;
break;

34
writeengine/server/we_dmlcommandproc.cpp
View File

@ -356,7 +356,7 @@ uint8_t WE_DMLCommandProc::processSingleInsert(messageqcpp::ByteStream& bs, std:
try
{
datavalue = DataConvert::convertColumnData(colType, indata, pushWarning, isNULL);
datavalue = DataConvert::convertColumnData(colType, indata, pushWarning, insertPkg.get_TimeZone(), isNULL, false, false);
}
catch (exception&)
{
@ -1235,7 +1235,7 @@ uint8_t WE_DMLCommandProc::processBatchInsert(messageqcpp::ByteStream& bs, std::
try
{
datavalue = DataConvert::convertColumnData(colType, indata, pushWarning, isNULL);
datavalue = DataConvert::convertColumnData(colType, indata, pushWarning, insertPkg.get_TimeZone(), isNULL, false, false);
}
catch (exception&)
{
@ -1782,6 +1782,7 @@ uint8_t WE_DMLCommandProc::processBatchInsertBinary(messageqcpp::ByteStream& bs,
case execplan::CalpontSystemCatalog::BIGINT:
case execplan::CalpontSystemCatalog::DATETIME:
case execplan::CalpontSystemCatalog::TIME:
case execplan::CalpontSystemCatalog::TIMESTAMP:
case execplan::CalpontSystemCatalog::UBIGINT:
bs >> val64;
@ -2689,6 +2690,8 @@ uint8_t WE_DMLCommandProc::processUpdate(messageqcpp::ByteStream& bs,
CalpontSystemCatalog::OID oid = 0;
CalpontSystemCatalog::ROPair tableRO;
std::string timeZone = cpackages[txnId].get_TimeZone();
try
{
tableRO = systemCatalogPtr->tableRID(tableName);
@ -2984,6 +2987,13 @@ uint8_t WE_DMLCommandProc::processUpdate(messageqcpp::ByteStream& bs,
break;
}
case CalpontSystemCatalog::TIMESTAMP:
{
intColVal = row.getUintField<8>(fetchColPos);
value = DataConvert::timestampToString(intColVal, timeZone, colType.precision);
break;
}
case CalpontSystemCatalog::TIME:
{
intColVal = row.getIntField<8>(fetchColPos);
@ -3321,6 +3331,13 @@ uint8_t WE_DMLCommandProc::processUpdate(messageqcpp::ByteStream& bs,
break;
}
case CalpontSystemCatalog::TIMESTAMP:
{
intColVal = row.getUintField<8>(fetchColPos);
value = DataConvert::timestampToString(intColVal, timeZone, colType.precision);
break;
}
case CalpontSystemCatalog::TIME:
{
intColVal = row.getIntField<8>(fetchColPos);
@ -3495,7 +3512,7 @@ uint8_t WE_DMLCommandProc::processUpdate(messageqcpp::ByteStream& bs,
try
{
datavalue = DataConvert::convertColumnData(colType, colType.defaultValue, pushWarn, isNull);
datavalue = DataConvert::convertColumnData(colType, colType.defaultValue, pushWarn, timeZone, isNull, false, false);
}
catch (exception&)
{
@ -3527,7 +3544,7 @@ uint8_t WE_DMLCommandProc::processUpdate(messageqcpp::ByteStream& bs,
{
try
{
datavalue = DataConvert::convertColumnData(colType, value, pushWarn, isNull);
datavalue = DataConvert::convertColumnData(colType, value, pushWarn, timeZone, isNull, false, false);
}
catch (exception&)
{
@ -3568,7 +3585,8 @@ uint8_t WE_DMLCommandProc::processUpdate(messageqcpp::ByteStream& bs,
string inData (columnsUpdated[j]->get_Data());
if (((colType.colDataType == execplan::CalpontSystemCatalog::DATE) && (inData == "0000-00-00")) ||
((colType.colDataType == execplan::CalpontSystemCatalog::DATETIME) && (inData == "0000-00-00 00:00:00")))
((colType.colDataType == execplan::CalpontSystemCatalog::DATETIME) && (inData == "0000-00-00 00:00:00")) ||
((colType.colDataType == execplan::CalpontSystemCatalog::TIMESTAMP) && (inData == "0000-00-00 00:00:00")))
{
isNull = true;
}
@ -3623,7 +3641,7 @@ uint8_t WE_DMLCommandProc::processUpdate(messageqcpp::ByteStream& bs,
try
{
datavalue = DataConvert::convertColumnData(colType, inData, pushWarn, isNull);
datavalue = DataConvert::convertColumnData(colType, inData, pushWarn, timeZone, isNull, false, false);
}
catch (exception&)
{
@ -3669,7 +3687,7 @@ uint8_t WE_DMLCommandProc::processUpdate(messageqcpp::ByteStream& bs,
{
try
{
datavalue = DataConvert::convertColumnData(colType, colType.defaultValue, pushWarn, isNull);
datavalue = DataConvert::convertColumnData(colType, colType.defaultValue, pushWarn, timeZone, isNull, false, false);
}
catch (exception&)
{
@ -3702,7 +3720,7 @@ uint8_t WE_DMLCommandProc::processUpdate(messageqcpp::ByteStream& bs,
{
try
{
datavalue = DataConvert::convertColumnData(colType, inData, pushWarn, isNull, false, true);
datavalue = DataConvert::convertColumnData(colType, inData, pushWarn, timeZone, isNull, false, true);
}
catch (exception& ex)
{

1
writeengine/shared/we_blockop.cpp
View File

@ -164,6 +164,7 @@ uint64_t BlockOp::getEmptyRowValue(
case CalpontSystemCatalog::VARCHAR :
case CalpontSystemCatalog::DATE :
case CalpontSystemCatalog::DATETIME :
case CalpontSystemCatalog::TIMESTAMP :
default:
offset = ( colDataType == CalpontSystemCatalog::VARCHAR ) ? -1 : 0;
emptyVal = joblist::CHAR1EMPTYROW;

13
writeengine/shared/we_convertor.cpp
View File

@ -367,9 +367,10 @@ void Convertor::convertColType(CalpontSystemCatalog::ColDataType dataType,
internalType = WriteEngine::WR_FLOAT;
break;
// Map BIGINT and DATETIME to WR_LONGLONG
// Map BIGINT, DATETIME, TIMESTAMP, and TIME to WR_LONGLONG
case CalpontSystemCatalog::BIGINT :
case CalpontSystemCatalog::DATETIME :
case CalpontSystemCatalog::TIMESTAMP :
case CalpontSystemCatalog::TIME :
internalType = WriteEngine::WR_LONGLONG;
break;
@ -429,7 +430,7 @@ void Convertor::convertColType(CalpontSystemCatalog::ColDataType dataType,
internalType = WriteEngine::WR_UINT;
break;
// Map UBIGINT to WR_ULONGLONG
// Map UBIGINT, TIMESTAMP to WR_ULONGLONG
case CalpontSystemCatalog::UBIGINT:
internalType = WriteEngine::WR_ULONGLONG;
break;
@ -481,7 +482,7 @@ void Convertor::convertWEColType(ColType internalType,
dataType = CalpontSystemCatalog::FLOAT;
break;
// Map BIGINT and DATETIME to WR_LONGLONG
// Map BIGINT, DATETIME, TIME, and TIMESTAMP to WR_LONGLONG
case WriteEngine::WR_LONGLONG :
dataType = CalpontSystemCatalog::BIGINT;
break;
@ -531,7 +532,7 @@ void Convertor::convertWEColType(ColType internalType,
dataType = CalpontSystemCatalog::UINT;
break;
// Map UBIGINT to WR_ULONGLONG
// Map UBIGINT and TIMESTAMP to WR_ULONGLONG
case WriteEngine::WR_ULONGLONG:
dataType = CalpontSystemCatalog::UBIGINT;
break;
@ -595,9 +596,10 @@ void Convertor::convertColType(ColStruct* curStruct)
*internalType = WriteEngine::WR_FLOAT;
break;
// Map BIGINT and DATETIME to WR_LONGLONG
// Map BIGINT, DATETIME, TIME, and TIMESTAMP to WR_LONGLONG
case CalpontSystemCatalog::BIGINT :
case CalpontSystemCatalog::DATETIME :
case CalpontSystemCatalog::TIMESTAMP :
case CalpontSystemCatalog::TIME :
*internalType = WriteEngine::WR_LONGLONG;
break;
@ -767,6 +769,7 @@ int Convertor::getCorrectRowWidth(CalpontSystemCatalog::ColDataType dataType, in
case CalpontSystemCatalog::DATETIME:
case CalpontSystemCatalog::TIME:
case CalpontSystemCatalog::TIMESTAMP:
newWidth = 8;
break;

3
writeengine/shared/we_type.h
View File

@ -164,7 +164,8 @@ const char ColDataTypeStr[execplan::CalpontSystemCatalog::NUM_OF_COL_DATA_TYPE]
"unsigned-bigint",
"unsigned-double",
"text",
"time"
"time",
"timestamp"
};
enum FuncType { FUNC_WRITE_ENGINE, FUNC_INDEX, FUNC_DICTIONARY };

31
writeengine/splitter/we_cmdargs.cpp
View File

@ -36,6 +36,7 @@ using namespace std;
#include <boost/uuid/uuid_io.hpp>
#include <boost/filesystem.hpp>
#include "dataconvert.h"
#include "liboamcpp.h"
using namespace oam;
@ -74,7 +75,8 @@ WECmdArgs::WECmdArgs(int argc, char** argv) :
fBlockMode3(false),
fbTruncationAsError(false),
fUUID(boost::uuids::nil_generator()()),
fConsoleOutput(true)
fConsoleOutput(true),
fTimeZone("SYSTEM")
{
try
{
@ -210,6 +212,7 @@ std::string WECmdArgs::getCpImportCmdLine()
}
aSS << " -P " << getModuleID();
aSS << " -T " << fTimeZone;
if (fbTruncationAsError)
aSS << " -S ";
@ -482,7 +485,7 @@ void WECmdArgs::usage()
cout << "\t\t [-r readers] [-j JobID] [-e maxErrs] [-B libBufSize] [-w parsers]\n";
cout << "\t\t [-s c] [-E enclosedChar] [-C escapeChar] [-n NullOption]\n";
cout << "\t\t [-q batchQty] [-p jobPath] [-P list of PMs] [-S] [-i] [-v verbose]\n";
cout << "\t\t [-I binaryOpt]\n";
cout << "\t\t [-I binaryOpt] [-T timeZone]\n";
cout << "Traditional usage without positional parameters (XML job file required):\n";
@ -491,7 +494,7 @@ void WECmdArgs::usage()
cout << "\t\t [-b readBufs] [-p path] [-c readBufSize] [-e maxErrs] [-B libBufSize]\n";
cout << "\t\t [-n NullOption] [-E encloseChar] [-C escapeChar] [-i] [-v verbose]\n";
cout << "\t\t [-d debugLevel] [-q batchQty] [-l loadFile] [-P list of PMs] [-S]\n";
cout << "\t\t [-I binaryOpt]\n";
cout << "\t\t [-I binaryOpt] [-T timeZone]\n";
cout << "\n\nPositional parameters:\n";
cout << "\tdbName Name of the database to load\n";
@ -535,7 +538,10 @@ void WECmdArgs::usage()
<< "\t-m\tmode\n"
<< "\t\t\t1 - rows will be loaded in a distributed manner across PMs.\n"
<< "\t\t\t2 - PM based input files loaded onto their respective PM.\n"
<< "\t\t\t3 - input files will be loaded on the local PM.\n";
<< "\t\t\t3 - input files will be loaded on the local PM.\n"
<< "\t-T\tTimezone used for TIMESTAMP datatype.\n"
<< "\t\tPossible values: \"SYSTEM\" (default)\n"
<< "\t\t : Offset in the form +/-HH:MM\n";
cout << "\nExample1: Traditional usage\n"
<< "\tcpimport -j 1234";
@ -574,7 +580,7 @@ void WECmdArgs::parseCmdLineArgs(int argc, char** argv)
// fPrgmName = "/home/bpaul/genii/export/bin/cpimport";
while ((aCh = getopt(argc, argv,
"d:j:w:s:v:l:r:b:e:B:f:q:ihm:E:C:P:I:n:p:c:SN"))
"d:j:w:s:v:l:r:b:e:B:f:q:ihm:E:C:P:I:n:p:c:ST:N"))
!= EOF)
{
switch (aCh)
@ -842,6 +848,21 @@ void WECmdArgs::parseCmdLineArgs(int argc, char** argv)
break;
}
case 'T':
{
std::string timeZone = optarg;
long offset;
if (timeZone != "SYSTEM" && dataconvert::timeZoneToOffset(timeZone.c_str(), timeZone.size(), &offset))
{
throw (runtime_error(
"Value for option -T is invalid"));
}
fTimeZone = timeZone;
break;
}
case 'q': // -q: batch quantity - default value is 10000
{
errno = 0;

5
writeengine/splitter/we_cmdargs.h
View File

@ -190,6 +190,10 @@ public:
{
return fConsoleOutput;
}
const std::string& getTimeZone() const
{
return fTimeZone;
}
private: // variables for SplitterApp
@ -294,6 +298,7 @@ private: // variables for SplitterApp
bool fbTruncationAsError; // Treat string truncation as error
boost::uuids::uuid fUUID;
bool fConsoleOutput; // If false, no output to console.
std::string fTimeZone; // Timezone to use for TIMESTAMP datatype
};
//----------------------------------------------------------------------

4
writeengine/splitter/we_sdhandler.cpp
View File

@ -1910,6 +1910,10 @@ void WESDHandler::onCleanupResult(int PmId, messageqcpp::SBS& Sbs)
ossSatCnt << "invalid date/times replaced with zero value: ";
break;
case CalpontSystemCatalog::TIMESTAMP:
ossSatCnt << "invalid timestamps replaced with zero value: ";
break;
case CalpontSystemCatalog::TIME:
ossSatCnt << "invalid times replaced with zero value: ";
break;

19
writeengine/xml/we_xmljob.cpp
View File

@ -76,7 +76,8 @@ const long long infinidb_precision[19] =
//------------------------------------------------------------------------------
XMLJob::XMLJob( ) : fDebugLevel( DEBUG_0 ),
fDeleteTempFile(false),
fValidateColList(true)
fValidateColList(true),
fTimeZone("SYSTEM")
{
}
@ -1128,6 +1129,22 @@ void XMLJob::fillInXMLDataNotNullDefault(
break;
}
case execplan::CalpontSystemCatalog::TIMESTAMP:
{
int convertStatus;
int64_t dt =
dataconvert::DataConvert::convertColumnTimestamp(
col_defaultValue.c_str(),
dataconvert::CALPONTDATETIME_ENUM, convertStatus,
col_defaultValue.length(), fTimeZone );
if (convertStatus != 0)
bDefaultConvertError = true;
col.fDefaultInt = dt;
break;
}
case execplan::CalpontSystemCatalog::TIME:
{
int convertStatus;

9
writeengine/xml/we_xmljob.h
View File

@ -123,6 +123,14 @@ public:
*/
EXPORT bool processNode( xmlNode* pParentNode );
/**
* @brief Set timezone
*/
void setTimeZone(const std::string& timeZone)
{
fTimeZone = timeZone;
}
private:
void setJobData( xmlNode* pNode,
const xmlTag tag,
@ -154,6 +162,7 @@ private:
JobColList fDefaultColumns; // temporary list of default cols
// for table node being processed
bool fValidateColList; // Validate all cols have XML tag
std::string fTimeZone; // Timezone used for TIMESTAMP datatype
};
} //end of namespace