database/mariadb-columnstore-engine
1
0
Fork 0
mirror of https://github.com/mariadb-corporation/mariadb-columnstore-engine.git synced 2025-07-29 08:21:15 +03:00
Code Activity

MCOL-4483: Fix and consolidate log files and cpimport logging.

Browse Source
This commit is contained in:
benthompson15
2021-01-20 12:04:52 -06:00
parent ceae2118a2
commit afa88866bb
32 changed files with 172 additions and 287 deletions
Download Patch File Download Diff File Expand all files Collapse all files

4
writeengine/bulk/cpimport.cpp
View File

@ -52,6 +52,7 @@
#include "IDBPolicy.h"
#include "MonitorProcMem.h"
#include "dataconvert.h"
#include "mcsconfig.h"
using namespace std;
using namespace WriteEngine;
@ -1149,7 +1150,8 @@ int main(int argc, char** argv)
int rc = NO_ERROR;
std::string exceptionMsg;
TASK task; // track tasks being performed
// set this upfront
curJob.setErrorDir(string(MCSLOGDIR) + "/cpimport/");
try
{
//--------------------------------------------------------------------------

8
writeengine/bulk/we_bulkload.cpp
View File

@ -53,6 +53,7 @@
#include "idbcompress.h"
#include "calpontsystemcatalog.h"
#include "we_ddlcommandclient.h"
#include "mcsconfig.h"
using namespace std;
@ -76,7 +77,6 @@ namespace WriteEngine
/* static */ const std::string BulkLoad::DIR_BULK_JOB("job");
/* static */ const std::string BulkLoad::DIR_BULK_TEMP_JOB("tmpjob");
/* static */ const std::string BulkLoad::DIR_BULK_IMPORT("/data/import/");
/* static */ const std::string BulkLoad::DIR_BULK_LOG("/log/");
/* static */ bool BulkLoad::fNoConsoleOutput = false;
//------------------------------------------------------------------------------
@ -275,9 +275,9 @@ int BulkLoad::loadJobInfo(
const Job& curJob = fJobInfo.getJob();
string logFile, errlogFile;
logFile = fRootDir + DIR_BULK_LOG + "Job_" +
logFile = std::string(MCSLOGDIR) + "/cpimport/" + "Job_" +
Convertor::int2Str( curJob.id ) + LOG_SUFFIX;
errlogFile = fRootDir + DIR_BULK_LOG + "Job_" +
errlogFile = std::string(MCSLOGDIR) + "/cpimport/" + "Job_" +
Convertor::int2Str( curJob.id ) + ERR_LOG_SUFFIX;
if (disableConsoleOutput())
@ -1055,7 +1055,7 @@ int BulkLoad::processJob( )
(fBulkMode == BULK_MODE_REMOTE_MULTIPLE_SRC))
tableInfo->setBulkLoadMode( fBulkMode, fBRMRptFileName );
tableInfo->setErrorDir(getErrorDir());
tableInfo->setErrorDir(string(getErrorDir()));
tableInfo->setTruncationAsError(getTruncationAsError());
rc = manageImportDataFileList( curJob, i, tableInfo );

12
writeengine/bulk/we_tableinfo.cpp
View File

@ -1577,9 +1577,9 @@ void TableInfo::writeBadRows( const std::vector<std::string>* errorDatRows,
rejectFileNameToLog = fRejectDataFileName;
}
oss << "Number of rows with errors = " << fRejectDataCnt <<
". Exact error rows are listed in file " <<
rejectFileNameToLog;
oss << "Number of rows with bad data = " << fRejectDataCnt <<
". Exact rows are listed in file located here: " <<
fErrorDir;
fLog->logMsg(oss.str(), MSGLVL_INFO1);
fRejectDataCnt = 0;
@ -1691,9 +1691,9 @@ void TableInfo::writeErrReason( const std::vector< std::pair<RID,
errFileNameToLog = fRejectErrFileName;
}
oss << "Number of rows with errors = " << fRejectErrCnt <<
". Row numbers with error reasons are listed in file " <<
errFileNameToLog;
oss << "Number of rows with errors = " << fRejectDataCnt <<
". Exact rows are listed in file located here: " <<
fErrorDir;
fLog->logMsg(oss.str(), MSGLVL_INFO1);
fRejectErrCnt = 0;

19
writeengine/splitter/we_cmdargs.cpp
View File

@ -77,7 +77,8 @@ WECmdArgs::WECmdArgs(int argc, char** argv) :
fbTruncationAsError(false),
fUUID(boost::uuids::nil_generator()()),
fConsoleOutput(true),
fTimeZone("SYSTEM")
fTimeZone("SYSTEM"),
fErrorDir(string(MCSLOGDIR)+"/cpimport/")
{
try
{
@ -146,6 +147,9 @@ std::string WECmdArgs::getCpImportCmdLine()
aSS << " -f " << fPmFilePath;
}
if (fErrorDir.length() > 0)
aSS << " -L " << fErrorDir;
if (fUsername.length() > 0)
aSS << " -U " << fUsername;
@ -567,7 +571,10 @@ void WECmdArgs::usage()
<< "\t-K\tS3 Authentication Secret (for S3 imports)\n"
<< "\t-t\tS3 Bucket (for S3 imports)\n"
<< "\t-H\tS3 Hostname (for S3 imports, Amazon's S3 default)\n"
<< "\t-g\tS3 Region (for S3 imports)\n";
<< "\t-g\tS3 Region (for S3 imports)\n"
<< "\t-L\tDirectory for the output .err and .bad files.\n"
<< "\t\tDefault is " << string(MCSLOGDIR);
cout << "\nExample1: Traditional usage\n"
<< "\tcpimport -j 1234";
@ -602,7 +609,7 @@ void WECmdArgs::parseCmdLineArgs(int argc, char** argv)
fPrgmName = string(MCSBINDIR) + "/" + "cpimport.bin"; //argv[0] is splitter but we need cpimport
while ((aCh = getopt(argc, argv,
"d:j:w:s:v:l:r:b:e:B:f:q:ihm:E:C:P:I:n:p:c:ST:Ny:K:t:H:g:U:"))
"d:j:w:s:v:l:r:b:e:B:f:q:ihm:E:C:P:I:n:p:c:ST:Ny:K:t:H:g:U:L:"))
!= EOF)
{
switch (aCh)
@ -942,6 +949,12 @@ void WECmdArgs::parseCmdLineArgs(int argc, char** argv)
break;
}
case 'L': // -L set the output location of .bad/.err files
{
fErrorDir = optarg;
break;
}
default:
{
std::string aErr = "Unknown command line option " + aCh;

9
writeengine/splitter/we_cmdargs.h
View File

@ -268,6 +268,14 @@ public:
{
return fS3Region;
}
std::string getErrorDir() const
{
return fErrorDir;
}
void setErrorDir(std::string fErrorDir)
{
this->fErrorDir = fErrorDir;
}
std::string& getUsername();
std::string PrepMode2ListOfFiles(std::string& FileName); // Bug 4342
void getColumnList( std::set<std::string>& columnList ) const;
@ -329,6 +337,7 @@ private: // variables for SplitterApp
bool fConsoleOutput; // If false, no output to console.
std::string fTimeZone; // Timezone to use for TIMESTAMP datatype
std::string fUsername; // Username of the data files owner
std::string fErrorDir;
};
//----------------------------------------------------------------------

12
writeengine/splitter/we_sdhandler.cpp
View File

@ -474,20 +474,20 @@ void WESDHandler::setup()
std::string aErrLogName;
if (fRef.fCmdArgs.isJobLogOnly())
aLogName = fRef.fCmdArgs.getBulkRootDir() + "/log/" +
aLogName = std::string(MCSLOGDIR) + "/cpimport/" +
"cpimport_Job_" + fRef.fCmdArgs.getJobId() + ".log";
else
aLogName = fRef.fCmdArgs.getBulkRootDir() + "/log/" +
aLogName = std::string(MCSLOGDIR) + "/cpimport/" +
"cpimport_" + aTimeStamp + "_" + aPid.str() + ".log";
if (getDebugLvl() > 1) cout << "LogName : " << aLogName << endl;
if (fRef.fCmdArgs.isJobLogOnly())
aErrLogName = fRef.fCmdArgs.getBulkRootDir() + "/log/" +
aErrLogName = std::string(MCSLOGDIR) + "/cpimport/" +
"cpimport_Job_" + fRef.fCmdArgs.getJobId() + ".err";
else
aErrLogName = fRef.fCmdArgs.getBulkRootDir() + "/log/" +
aErrLogName = std::string(MCSLOGDIR) + "/cpimport/" +
"cpimport_" + aTimeStamp + "_" + aPid.str() + ".err";
@ -1529,7 +1529,7 @@ void WESDHandler::onErrorFile(int PmId, messageqcpp::SBS& Sbs)
{
std::string aFile = aTmpFileName.substr(aPos + 1); //+1 to pass '/'
std::string aInFile = fRef.getLocFile(); // input file
std::string aInFile = fRef.fCmdArgs.getErrorDir();
if (aInFile != "/dev/stdin")
{
@ -1627,7 +1627,7 @@ void WESDHandler::onBadFile(int PmId, messageqcpp::SBS& Sbs)
{
std::string aFile = aTmpFileName.substr(aPos + 1); //+1 to pass '/'
std::string aInFile = fRef.getLocFile(); // input file
std::string aInFile = fRef.fCmdArgs.getErrorDir();
if (aInFile != "/dev/stdin")
{

6
writeengine/xml/we_xmlgenproc.cpp
View File

@ -29,6 +29,7 @@
#include "we_config.h"
#include "we_xmltag.h"
#include "we_xmlgendata.h"
#include "mcsconfig.h"
#include <boost/date_time/posix_time/posix_time.hpp>
#include <boost/filesystem/path.hpp>
@ -39,7 +40,6 @@ namespace
const char* DICT_TYPE("D");
const char* ENCODING("UTF-8");
const char* LOGNAME("Jobxml_");
const std::string LOGDIR("/log/");
const char* JOBNAME("Job_");
}
@ -60,9 +60,9 @@ XMLGenProc::XMLGenProc(XMLGenData* mgr, bool bUseXmlLogFile, bool bSysCatRpt) :
fSysCatRpt(bSysCatRpt),
fUseXmlLogFile(bUseXmlLogFile)
{
std::string logFile(Config::getBulkRoot() + std::string(LOGDIR) + LOGNAME +
std::string logFile(std::string(MCSLOGDIR) + "/cpimport/" + LOGNAME +
fInputMgr->getParm(XMLGenData::JOBID) + ".log" );
std::string errFile(Config::getBulkRoot() + std::string(LOGDIR) + LOGNAME +
std::string errFile(std::string(MCSLOGDIR) + "/cpimport/" + LOGNAME +
fInputMgr->getParm(XMLGenData::JOBID) + ".err" );
fErrorString.append(errFile + "\n");