diff --git a/mysql-test/columnstore/basic/r/MCOL-5164-max-errors-and-report.result b/mysql-test/columnstore/basic/r/MCOL-5164-max-errors-and-report.result new file mode 100644 index 000000000..904096a42 --- /dev/null +++ b/mysql-test/columnstore/basic/r/MCOL-5164-max-errors-and-report.result @@ -0,0 +1,51 @@ +DROP DATABASE IF EXISTS mcol5164; +CREATE DATABASE mcol5164; +USE mcol5164; +CREATE TABLE t1(col1 INT, col2 VARCHAR(64)) ENGINE=Columnstore; +SELECT * FROM t1; +col1 col2 +Rejected rows: +0,test0,wrong +1,test1,wrong +2,test2,wrong +3,test3,wrong +4,test4,wrong +5,test5,wrong +6,test6,wrong +7,test7,wrong +8,test8,wrong +9,test9,wrong +10,test10,wrong +TRUNCATE t1; +SELECT * FROM t1; +col1 col2 +11 test11-good +Rejected rows: +0,test0,wrong +1,test1,wrong +2,test2,wrong +3,test3,wrong +4,test4,wrong +5,test5,wrong +6,test6,wrong +7,test7,wrong +8,test8,wrong +9,test9,wrong +10,test10,wrong +TRUNCATE t1; +SELECT * FROM t1; +col1 col2 +11 test11-good +Rejected rows: +0,test0,wrong +1,test1,wrong +2,test2,wrong +3,test3,wrong +4,test4,wrong +5,test5,wrong +6,test6,wrong +7,test7,wrong +8,test8,wrong +9,test9,wrong +10,test10,wrong +DROP DATABASE mcol5164; diff --git a/mysql-test/columnstore/basic/t/MCOL-5164-max-errors-and-report.test b/mysql-test/columnstore/basic/t/MCOL-5164-max-errors-and-report.test new file mode 100644 index 000000000..68c6cfe80 --- /dev/null +++ b/mysql-test/columnstore/basic/t/MCOL-5164-max-errors-and-report.test @@ -0,0 +1,52 @@ +if (!$MYSQL_TEST_ROOT){ + skip Should be run by root to execute cpimport; +} + +-- source ../include/have_columnstore.inc + +--disable_warnings +DROP DATABASE IF EXISTS mcol5164; +--enable_warnings + +CREATE DATABASE mcol5164; +USE mcol5164; + +CREATE TABLE t1(col1 INT, col2 VARCHAR(64)) ENGINE=Columnstore; + +--exec mkdir -p /tmp/mtr-mcol5164 +--exec awk 'BEGIN { for (i = 0; i < 11; i++) { printf "%d,test%d,wrong\n", i, i; }; printf "%d,test%d-good", i, i; }' > /tmp/mtr-mcol5164/mcol5164.csv + +--disable_result_log +--error 1 # exceeds default max-errors +--exec $MCS_CPIMPORT -s , -L /tmp/mtr-mcol5164 mcol5164 t1 /tmp/mtr-mcol5164/mcol5164.csv +--enable_result_log +SELECT * FROM t1; +--exec echo Rejected rows: +--exec cat /tmp/mtr-mcol5164/mcol5164.csv*.bad +--exec rm -f /tmp/mtr-mcol5164/*.err +--exec rm -f /tmp/mtr-mcol5164/*.bad +TRUNCATE t1; + +# implicitly set max-errors +--disable_result_log +--exec $MCS_CPIMPORT -s , -e 11 -L /tmp/mtr-mcol5164 mcol5164 t1 /tmp/mtr-mcol5164/mcol5164.csv +--enable_result_log +SELECT * FROM t1; +--exec echo Rejected rows: +--exec cat /tmp/mtr-mcol5164/mcol5164.csv*.bad +--exec rm -f /tmp/mtr-mcol5164/*.err +--exec rm -f /tmp/mtr-mcol5164/*.bad +TRUNCATE t1; + +# max-errors = all +--disable_result_log +--exec $MCS_CPIMPORT -s , -e all -L /tmp/mtr-mcol5164 mcol5164 t1 /tmp/mtr-mcol5164/mcol5164.csv +--enable_result_log +SELECT * FROM t1; +--exec echo Rejected rows: +--exec cat /tmp/mtr-mcol5164/mcol5164.csv*.bad + + +# Clean UP +--exec rm -rf /tmp/mtr-mcol5164 +DROP DATABASE mcol5164; diff --git a/writeengine/bulk/we_bulkload.cpp b/writeengine/bulk/we_bulkload.cpp index 2b4f1ca1f..9e2b22099 100644 --- a/writeengine/bulk/we_bulkload.cpp +++ b/writeengine/bulk/we_bulkload.cpp @@ -530,14 +530,14 @@ int BulkLoad::preProcess(Job& job, int tableNo, std::shared_ptr& tabl if (pwd) tableInfo->setUIDGID(pwd->pw_uid, pwd->pw_gid); - if (fMaxErrors != -1) + if (fMaxErrors != MAX_ERRORS_DEFAULT) tableInfo->setMaxErrorRows(fMaxErrors); else tableInfo->setMaxErrorRows(job.jobTableList[tableNo].maxErrNum); // @bug 3929: cpimport.bin error messaging using up too much memory. // Validate that max allowed error count is within valid range - long long maxErrNum = tableInfo->getMaxErrorRows(); + int maxErrNum = tableInfo->getMaxErrorRows(); if (maxErrNum > MAX_ALLOW_ERROR_COUNT) { diff --git a/writeengine/bulk/we_bulkload.h b/writeengine/bulk/we_bulkload.h index f9ab26ffd..34585b4f2 100644 --- a/writeengine/bulk/we_bulkload.h +++ b/writeengine/bulk/we_bulkload.h @@ -129,7 +129,7 @@ class BulkLoad : public FileOp void setEscapeChar(char esChar); void setSkipRows(size_t skipRows); void setKeepRbMetaFiles(bool keepMeta); - void setMaxErrorCount(unsigned int maxErrors); + void setMaxErrorCount(int maxErrors); void setNoOfParseThreads(int parseThreads); void setNoOfReadThreads(int readThreads); void setNullStringMode(bool bMode); @@ -184,13 +184,13 @@ class BulkLoad : public FileOp Log fLog; // logger - int fNumOfParser; // total number of parser + int fNumOfParser{0}; // total number of parser char fColDelim{0}; // delimits col values within a row int fNoOfBuffers{-1}; // Number of read buffers int fBufferSize{-1}; // Read buffer size int fFileVbufSize{-1}; // Internal file system buffer size - long long fMaxErrors{-1}; // Max allowable errors per job + long long fMaxErrors{MAX_ERRORS_DEFAULT}; // Max allowable errors per job std::string fAlternateImportDir; // Alternate bulk import directory std::string fErrorDir; // Opt. where error records record std::string fProcessName; // Application process name @@ -429,10 +429,7 @@ inline void BulkLoad::setKeepRbMetaFiles(bool keepMeta) fKeepRbMetaFiles = keepMeta; } -// Mutator takes an unsigned int, but we store in a long long, because... -// TableInfo which eventually needs this attribute, takes an unsigned int, -// but we want to be able to init to -1, to indicate when it has not been set. -inline void BulkLoad::setMaxErrorCount(unsigned int maxErrors) +inline void BulkLoad::setMaxErrorCount(int maxErrors) { fMaxErrors = maxErrors; } diff --git a/writeengine/bulk/we_bulkloadbuffer.cpp b/writeengine/bulk/we_bulkloadbuffer.cpp index cd43d6cd0..5d65b3215 100644 --- a/writeengine/bulk/we_bulkloadbuffer.cpp +++ b/writeengine/bulk/we_bulkloadbuffer.cpp @@ -2049,7 +2049,7 @@ int BulkLoadBuffer::parseDictSection(ColumnInfo& columnInfo, int tokenPos, RID s int BulkLoadBuffer::fillFromMemory(const BulkLoadBuffer& overFlowBufIn, const char* input, size_t length, size_t* parse_length, size_t& skipRows, RID& totalReadRows, RID& correctTotalRows, const boost::ptr_vector& columnsInfo, - unsigned int allowedErrCntThisCall) + int allowedErrCntThisCall) { boost::mutex::scoped_lock lock(fSyncUpdatesBLB); reset(); @@ -2153,7 +2153,7 @@ int BulkLoadBuffer::fillFromMemory(const BulkLoadBuffer& overFlowBufIn, const ch int BulkLoadBuffer::fillFromFile(const BulkLoadBuffer& overFlowBufIn, FILE* handle, size_t& skipRows, RID& totalReadRows, RID& correctTotalRows, const boost::ptr_vector& columnsInfo, - unsigned int allowedErrCntThisCall) + int allowedErrCntThisCall) { boost::mutex::scoped_lock lock(fSyncUpdatesBLB); reset(); @@ -2277,7 +2277,7 @@ int BulkLoadBuffer::fillFromFile(const BulkLoadBuffer& overFlowBufIn, FILE* hand // depending on whether the user has enabled the "enclosed by" feature. //------------------------------------------------------------------------------ void BulkLoadBuffer::tokenize(const boost::ptr_vector& columnsInfo, - unsigned int allowedErrCntThisCall, size_t& skipRows) + int allowedErrCntThisCall, size_t& skipRows) { unsigned offset = 0; // length of field unsigned curCol = 0; // dest db column counter within a row @@ -2789,7 +2789,7 @@ void BulkLoadBuffer::tokenize(const boost::ptr_vector& columnsInfo, // Quit if we exceed max allowable errors for this call. // We set lastRowHead = p, so that the code that follows this // loop won't try to save any data in fOverflowBuf. - if (errorCount > allowedErrCntThisCall) + if (allowedErrCntThisCall != MAX_ERRORS_ALL && errorCount > static_cast(allowedErrCntThisCall)) { lastRowHead = p + 1; p++; @@ -2928,7 +2928,7 @@ void BulkLoadBuffer::resizeTokenArray() // then tokenize() will stop reading data and exit. //------------------------------------------------------------------------------ int BulkLoadBuffer::tokenizeBinary(const boost::ptr_vector& columnsInfo, - unsigned int allowedErrCntThisCall, bool bEndOfData) + int allowedErrCntThisCall, bool bEndOfData) { unsigned curCol = 0; // dest db column counter within a row unsigned curRowNum = 0; // "total" number of rows read during this call @@ -3082,7 +3082,7 @@ int BulkLoadBuffer::tokenizeBinary(const boost::ptr_vector& columnsI errorCount++; // Quit if we exceed max allowable errors for this call - if (errorCount > allowedErrCntThisCall) + if (allowedErrCntThisCall != MAX_ERRORS_ALL && errorCount > static_cast(allowedErrCntThisCall)) break; } diff --git a/writeengine/bulk/we_bulkloadbuffer.h b/writeengine/bulk/we_bulkloadbuffer.h index 475c98663..fafc330ee 100644 --- a/writeengine/bulk/we_bulkloadbuffer.h +++ b/writeengine/bulk/we_bulkloadbuffer.h @@ -215,12 +215,12 @@ class BulkLoadBuffer /** @brief tokenize the buffer contents and fill up the token array. */ - void tokenize(const boost::ptr_vector& columnsInfo, unsigned int allowedErrCntThisCall, + void tokenize(const boost::ptr_vector& columnsInfo, int allowedErrCntThisCall, size_t& skipRows); /** @brief Binary tokenization of the buffer, and fill up the token array. */ - int tokenizeBinary(const boost::ptr_vector& columnsInfo, unsigned int allowedErrCntThisCall, + int tokenizeBinary(const boost::ptr_vector& columnsInfo, int allowedErrCntThisCall, bool bEndOfData); /** @brief Determine if specified value is NULL or not. @@ -275,13 +275,13 @@ class BulkLoadBuffer int fillFromMemory(const BulkLoadBuffer& overFlowBufIn, const char* input, size_t length, size_t* parse_length, size_t& skipRows, RID& totalReadRows, RID& correctTotalRows, - const boost::ptr_vector& columnsInfo, unsigned int allowedErrCntThisCall); + const boost::ptr_vector& columnsInfo, int allowedErrCntThisCall); /** @brief Read the table data into the buffer */ int fillFromFile(const BulkLoadBuffer& overFlowBufIn, FILE* handle, size_t& skipRows, RID& totalRows, RID& correctTotalRows, const boost::ptr_vector& columnsInfo, - unsigned int allowedErrCntThisCall); + int allowedErrCntThisCall); /** @brief Get the overflow size */ diff --git a/writeengine/bulk/we_cmdargs.cpp b/writeengine/bulk/we_cmdargs.cpp index 132771002..640e42867 100644 --- a/writeengine/bulk/we_cmdargs.cpp +++ b/writeengine/bulk/we_cmdargs.cpp @@ -70,8 +70,7 @@ WECmdArgs::WECmdArgs(int argc, char** argv) DECLARE_INT_ARG("read-buffer-size,c", fReadBufSize, 1, INT_MAX, "Application read buffer size (in bytes)") DECLARE_INT_ARG("debug,d", fDebugLvl, 1, 3, "Print different level(1-3) debug message") - DECLARE_INT_ARG("max-errors,e", fMaxErrors, 0, INT_MAX, - "Maximum number of allowable error per table per PM") + ("max-errors,e", po::value(), "Maximum number (or 'all') of allowable error per table per PM") ("file-path,f", po::value(&fPmFilePath), "Data file directory path. Default is current working directory.\n" "\tIn Mode 1, represents the local input file path.\n" @@ -304,6 +303,24 @@ void WECmdArgs::parseCmdLineArgs(int argc, char** argv) fAllowMissingColumn = true; } } + if (vm.contains("max-errors")) + { + auto optarg= vm["max-errors"].as(); + if (optarg == "all") + { + fMaxErrors = MAX_ERRORS_ALL; + } + else + { + errno = 0; + long lValue = strtol(optarg.c_str(), nullptr, 10); + if (errno != 0 || lValue < 0 || lValue > INT_MAX) + { + startupError("Option --max-errors/-e is invalid or out of range"); + } + fMaxErrors = lValue; + } + } if (fArgMode != -1) fMode = fArgMode; // BUG 4210 @@ -337,10 +354,7 @@ void WECmdArgs::fillParams(BulkLoad& curJob, std::string& sJobIdStr, std::string curJob.setReadBufferCount(fIOReadBufSize); curJob.setReadBufferSize(fReadBufSize); - if (fMaxErrors >= 0) - { - curJob.setMaxErrorCount(fMaxErrors); - } + curJob.setMaxErrorCount(fMaxErrors); if (!fPmFilePath.empty()) { importPath = fPmFilePath; diff --git a/writeengine/bulk/we_cmdargs.h b/writeengine/bulk/we_cmdargs.h index 5446fdc61..6eb272f32 100644 --- a/writeengine/bulk/we_cmdargs.h +++ b/writeengine/bulk/we_cmdargs.h @@ -91,7 +91,7 @@ private: int fNoOfReadThrds{1}; // No. of read buffers int fDebugLvl{0}; // Debug level - int fMaxErrors{-1}; // Max allowable errors + int fMaxErrors{MAX_ERRORS_DEFAULT}; // Max allowable errors int fReadBufSize{-1}; // Read buffer size int fIOReadBufSize{-1}; // I/O read buffer size int fSetBufSize{0}; // Buff size w/setvbuf diff --git a/writeengine/bulk/we_tableinfo.cpp b/writeengine/bulk/we_tableinfo.cpp index 1df5b752b..d9a2c5888 100644 --- a/writeengine/bulk/we_tableinfo.cpp +++ b/writeengine/bulk/we_tableinfo.cpp @@ -412,7 +412,11 @@ int TableInfo::readTableData() // We keep a running total of read errors; fMaxErrorRows specifies // the error limit. Here's where we see how many more errors we // still have below the limit, and we pass this to fillFromFile(). - unsigned allowedErrCntThisCall = ((fMaxErrorRows > fTotalErrRows) ? (fMaxErrorRows - fTotalErrRows) : 0); + int allowedErrCntThisCall; + if (fMaxErrorRows == MAX_ERRORS_ALL) + allowedErrCntThisCall = MAX_ERRORS_ALL; + else + allowedErrCntThisCall = static_cast(fMaxErrorRows) > fTotalErrRows ? fMaxErrorRows - fTotalErrRows : 0; // Fill in the specified buffer. // fTotalReadRowsPerInputFile is ongoing total number of rows read, @@ -485,7 +489,7 @@ int TableInfo::readTableData() writeErrorList(&fBuffers[readBufNo].getErrorRows(), &fBuffers[readBufNo].getExactErrorRows(), false); fBuffers[readBufNo].clearErrRows(); - if (fTotalErrRows > fMaxErrorRows) + if (fMaxErrorRows != MAX_ERRORS_ALL && fTotalErrRows > static_cast(fMaxErrorRows)) { // flush the reject data file and output the rejected rows // flush err file and output the rejected row id and the reason. diff --git a/writeengine/bulk/we_tableinfo.h b/writeengine/bulk/we_tableinfo.h index 3d4e836a8..3cbfe204e 100644 --- a/writeengine/bulk/we_tableinfo.h +++ b/writeengine/bulk/we_tableinfo.h @@ -85,7 +85,7 @@ class TableInfo : public WeUIDGID // for this table. Is volatile to // insure parser & reader threads // see the latest value. - unsigned fMaxErrorRows; // Maximum error rows + int fMaxErrorRows; // Maximum error rows int fLastBufferId; // Id of the last buffer char* fFileBuffer; // File buffer passed to setvbuf() int fCurrentParseBuffer; // Id of leading current buffer being @@ -298,7 +298,7 @@ class TableInfo : public WeUIDGID /** @brief Get the number of maximum allowed error rows */ - unsigned getMaxErrorRows() const; + int getMaxErrorRows() const; /** @brief retrieve the tuncation as error setting for this * import. When set, this causes char and varchar strings @@ -309,7 +309,7 @@ class TableInfo : public WeUIDGID /** @brief set the maximum number of error rows allowed */ - void setMaxErrorRows(const unsigned int maxErrorRows); + void setMaxErrorRows(int maxErrorRows); /** @brief Set mode to treat "NULL" string as NULL value or not. */ @@ -513,7 +513,7 @@ inline Status TableInfo::getStatusTI() const return fStatusTI; } -inline unsigned TableInfo::getMaxErrorRows() const +inline int TableInfo::getMaxErrorRows() const { return fMaxErrorRows; } @@ -630,7 +630,7 @@ inline void TableInfo::setLoadFilesInput(bool bReadFromStdin, bool bReadFromS3, fS3Region = s3region; } -inline void TableInfo::setMaxErrorRows(const unsigned int maxErrorRows) +inline void TableInfo::setMaxErrorRows(int maxErrorRows) { fMaxErrorRows = maxErrorRows; } diff --git a/writeengine/shared/we_type.h b/writeengine/shared/we_type.h index faef43018..7772351ab 100644 --- a/writeengine/shared/we_type.h +++ b/writeengine/shared/we_type.h @@ -144,6 +144,13 @@ enum ImportDataMode IMPORT_DATA_BIN_SAT_NULL = 2 }; +// Max number of ignored errors +enum MaxErrors +{ + MAX_ERRORS_DEFAULT = -1, // default value + MAX_ERRORS_ALL = -2 // special case: ignore all errors +}; + /** * the set of Calpont column data type names; MUST match ColDataType in * calpontsystemcatalog.h. diff --git a/writeengine/splitter/we_cmdargs.cpp b/writeengine/splitter/we_cmdargs.cpp index 816e2a9f6..87ed30e1d 100644 --- a/writeengine/splitter/we_cmdargs.cpp +++ b/writeengine/splitter/we_cmdargs.cpp @@ -71,8 +71,7 @@ WECmdArgs::WECmdArgs(int argc, char** argv) DECLARE_INT_ARG("debug,d", fDebugLvl, 1, 3, "Print different level(1-3) debug message") ("verbose,v", po::value()) ("silent,N", po::bool_switch()) - DECLARE_INT_ARG("max-errors,e", fMaxErrors, 0, INT_MAX, - "Maximum number of allowable error per table per PM") + ("max-errors,e", po::value(), "Maximum number (or 'all') of allowable error per table per PM") ("file-path,f", po::value(&fPmFilePath), "Data file directory path. Default is current working directory.\n" "\tIn Mode 1, represents the local input file path.\n" @@ -230,7 +229,9 @@ std::string WECmdArgs::getCpImportCmdLine(bool skipRows) if (fNoOfWriteThrds > 0) aSS << " -w " << fNoOfWriteThrds; - if (fMaxErrors >= 0) + if (fMaxErrors == MAX_ERRORS_ALL) + aSS << " -e all "; + else if (fMaxErrors != MAX_ERRORS_DEFAULT) aSS << " -e " << fMaxErrors; // BUG 5088 @@ -446,7 +447,7 @@ bool WECmdArgs::checkForCornerCases() cout << "Invalid option -b with Mode 0" << endl; throw(runtime_error("Mismatched options.")); } - else if (fMaxErrors >= 0) + else if (fMaxErrors >= 0 || fMaxErrors == MAX_ERRORS_ALL) { cout << "Invalid option -e with Mode 0" << endl; throw(runtime_error("Mismatched options.")); @@ -735,6 +736,24 @@ void WECmdArgs::parseCmdLineArgs(int argc, char** argv) fBatchQty = 10000; } } + if (vm.contains("max-errors")) + { + auto optarg = vm["max-errors"].as(); + if (optarg == "all") + { + fMaxErrors = MAX_ERRORS_ALL; + } + else + { + errno = 0; + long lValue = strtol(optarg.c_str(), nullptr, 10); + if (errno != 0 || lValue < 0 || lValue > INT_MAX) + { + throw runtime_error("Option --max-errors/-e is invalid or out of range"); + } + fMaxErrors = lValue; + } + } if (fArgMode != -1) fMode = fArgMode; // BUG 4210 diff --git a/writeengine/splitter/we_cmdargs.h b/writeengine/splitter/we_cmdargs.h index 64d17b598..96b2aa1c3 100644 --- a/writeengine/splitter/we_cmdargs.h +++ b/writeengine/splitter/we_cmdargs.h @@ -323,7 +323,7 @@ class WECmdArgs int fBatchQty{10000}; // No. of batch Qty. int fNoOfReadThrds{0}; // No. of read buffers int fDebugLvl{0}; // Debug level - int fMaxErrors{-1}; // Max allowable errors + int fMaxErrors{MAX_ERRORS_DEFAULT}; // Max allowable errors int fReadBufSize{0}; // Read buffer size int fIOReadBufSize{0}; // I/O read buffer size int fSetBufSize{0}; // Buff size w/setvbuf diff --git a/writeengine/xml/we_xmlgenproc.cpp b/writeengine/xml/we_xmlgenproc.cpp index c21251261..de4d5771a 100644 --- a/writeengine/xml/we_xmlgenproc.cpp +++ b/writeengine/xml/we_xmlgenproc.cpp @@ -230,8 +230,16 @@ void XMLGenProc::makeTableData(const CalpontSystemCatalog::TableName& table, con } xmlTextWriterWriteAttribute(fWriter, BAD_CAST xmlTagTable[TAG_LOAD_NAME], BAD_CAST tmp.c_str()); - xmlTextWriterWriteFormatAttribute(fWriter, BAD_CAST xmlTagTable[TAG_MAX_ERR_ROW], "%d", - atoi(fInputMgr->getParm(XMLGenData::MAXERROR).c_str())); + auto sMaxErrors = fInputMgr->getParm(XMLGenData::MAXERROR); + if (sMaxErrors == "all") + { + xmlTextWriterWriteAttribute(fWriter, BAD_CAST xmlTagTable[TAG_MAX_ERR_ROW], BAD_CAST sMaxErrors.c_str()); + } + else + { + xmlTextWriterWriteFormatAttribute(fWriter, BAD_CAST xmlTagTable[TAG_MAX_ERR_ROW], "%d", + atoi(sMaxErrors.c_str())); + } } kount++; diff --git a/writeengine/xml/we_xmljob.cpp b/writeengine/xml/we_xmljob.cpp index 34b50e026..fd7bd85d6 100644 --- a/writeengine/xml/we_xmljob.cpp +++ b/writeengine/xml/we_xmljob.cpp @@ -49,9 +49,7 @@ namespace WriteEngine // Constructor //------------------------------------------------------------------------------ XMLJob::XMLJob() - : fDeleteTempFile(false) - , fValidateColList(true) - , fTimeZone(dataconvert::systemTimeZoneOffset()) + : fDeleteTempFile(false), fValidateColList(true), fTimeZone(dataconvert::systemTimeZoneOffset()) { } @@ -197,7 +195,7 @@ void XMLJob::printJobInfo(Log& logger) const logger.logMsg(oss3.str(), MSGLVL_INFO2); } // end of loop through columns in a table - } // end of loop through tables + } // end of loop through tables } //------------------------------------------------------------------------------ @@ -473,8 +471,13 @@ void XMLJob::setJobDataTable(xmlNode* pNode) if (getNodeAttributeStr(pNode, xmlTagTable[TAG_LOAD_NAME], bufString)) curTable.loadFileName = bufString; - if (getNodeAttribute(pNode, xmlTagTable[TAG_MAX_ERR_ROW], &intVal, TYPE_INT)) - curTable.maxErrNum = intVal; + if (getNodeAttributeStr(pNode, xmlTagTable[TAG_MAX_ERR_ROW], bufString)) + { + if (bufString == "all") + curTable.maxErrNum = MAX_ERRORS_ALL; + else + curTable.maxErrNum = atoi(bufString.c_str()); + } fJob.jobTableList.push_back(curTable); } @@ -683,7 +686,6 @@ void XMLJob::initSatLimits(JobColumn& curColumn) const { curColumn.fMaxIntSat = dataconvert::decimalRangeUp(curColumn.precision); curColumn.fMinIntSat = -curColumn.fMaxIntSat; - } else if (curColumn.typeName == ColDataTypeStr[CalpontSystemCatalog::UDECIMAL]) { @@ -987,12 +989,13 @@ void XMLJob::fillInXMLDataNotNullDefault(const std::string& fullTblName, { if (LIKELY(colType.colWidth == datatypes::MAXDECIMALWIDTH)) { - col.fDefaultWideDecimal = colType.decimal128FromString(col_defaultValue.safeString(), &bDefaultConvertError); + col.fDefaultWideDecimal = + colType.decimal128FromString(col_defaultValue.safeString(), &bDefaultConvertError); } else { - col.fDefaultInt = Convertor::convertDecimalString(col_defaultValue.str(), - col_defaultValue.length(), colType.scale); + col.fDefaultInt = Convertor::convertDecimalString(col_defaultValue.str(), col_defaultValue.length(), + colType.scale); if (errno == ERANGE) bDefaultConvertError = true; @@ -1004,9 +1007,8 @@ void XMLJob::fillInXMLDataNotNullDefault(const std::string& fullTblName, case execplan::CalpontSystemCatalog::DATE: { int convertStatus; - int32_t dt = dataconvert::DataConvert::convertColumnDate(col_defaultValue.str(), - dataconvert::CALPONTDATE_ENUM, convertStatus, - col_defaultValue.length()); + int32_t dt = dataconvert::DataConvert::convertColumnDate( + col_defaultValue.str(), dataconvert::CALPONTDATE_ENUM, convertStatus, col_defaultValue.length()); if (convertStatus != 0) bDefaultConvertError = true; @@ -1046,9 +1048,8 @@ void XMLJob::fillInXMLDataNotNullDefault(const std::string& fullTblName, case execplan::CalpontSystemCatalog::TIME: { int convertStatus; - int64_t dt = dataconvert::DataConvert::convertColumnTime(col_defaultValue.str(), - dataconvert::CALPONTTIME_ENUM, convertStatus, - col_defaultValue.length()); + int64_t dt = dataconvert::DataConvert::convertColumnTime( + col_defaultValue.str(), dataconvert::CALPONTTIME_ENUM, convertStatus, col_defaultValue.length()); if (convertStatus != 0) bDefaultConvertError = true;