database/mariadb-columnstore-engine
1
0
Fork 0
mirror of https://github.com/mariadb-corporation/mariadb-columnstore-engine.git synced 2025-08-08 14:22:09 +03:00
Code Activity

MCOL-4483: Fix and consolidate log files and cpimport logging.

Browse Source
This commit is contained in:
benthompson15
2021-01-20 12:04:52 -06:00
parent ceae2118a2
commit afa88866bb
32 changed files with 172 additions and 287 deletions
Download Patch File Download Diff File Expand all files Collapse all files

19
writeengine/splitter/we_cmdargs.cpp
View File

@@ -77,7 +77,8 @@ WECmdArgs::WECmdArgs(int argc, char** argv) :
fbTruncationAsError(false),
fUUID(boost::uuids::nil_generator()()),
fConsoleOutput(true),
fTimeZone("SYSTEM")
fTimeZone("SYSTEM"),
fErrorDir(string(MCSLOGDIR)+"/cpimport/")
{
try
{
@@ -146,6 +147,9 @@ std::string WECmdArgs::getCpImportCmdLine()
aSS << " -f " << fPmFilePath;
}
if (fErrorDir.length() > 0)
aSS << " -L " << fErrorDir;
if (fUsername.length() > 0)
aSS << " -U " << fUsername;
@@ -567,7 +571,10 @@ void WECmdArgs::usage()
<< "\t-K\tS3 Authentication Secret (for S3 imports)\n"
<< "\t-t\tS3 Bucket (for S3 imports)\n"
<< "\t-H\tS3 Hostname (for S3 imports, Amazon's S3 default)\n"
<< "\t-g\tS3 Region (for S3 imports)\n";
<< "\t-g\tS3 Region (for S3 imports)\n"
<< "\t-L\tDirectory for the output .err and .bad files.\n"
<< "\t\tDefault is " << string(MCSLOGDIR);
cout << "\nExample1: Traditional usage\n"
<< "\tcpimport -j 1234";
@@ -602,7 +609,7 @@ void WECmdArgs::parseCmdLineArgs(int argc, char** argv)
fPrgmName = string(MCSBINDIR) + "/" + "cpimport.bin"; //argv[0] is splitter but we need cpimport
while ((aCh = getopt(argc, argv,
"d:j:w:s:v:l:r:b:e:B:f:q:ihm:E:C:P:I:n:p:c:ST:Ny:K:t:H:g:U:"))
"d:j:w:s:v:l:r:b:e:B:f:q:ihm:E:C:P:I:n:p:c:ST:Ny:K:t:H:g:U:L:"))
!= EOF)
{
switch (aCh)
@@ -942,6 +949,12 @@ void WECmdArgs::parseCmdLineArgs(int argc, char** argv)
break;
}
case 'L': // -L set the output location of .bad/.err files
{
fErrorDir = optarg;
break;
}
default:
{
std::string aErr = "Unknown command line option " + aCh;

9
writeengine/splitter/we_cmdargs.h
View File

@@ -268,6 +268,14 @@ public:
{
return fS3Region;
}
std::string getErrorDir() const
{
return fErrorDir;
}
void setErrorDir(std::string fErrorDir)
{
this->fErrorDir = fErrorDir;
}
std::string& getUsername();
std::string PrepMode2ListOfFiles(std::string& FileName); // Bug 4342
void getColumnList( std::set<std::string>& columnList ) const;
@@ -329,6 +337,7 @@ private: // variables for SplitterApp
bool fConsoleOutput; // If false, no output to console.
std::string fTimeZone; // Timezone to use for TIMESTAMP datatype
std::string fUsername; // Username of the data files owner
std::string fErrorDir;
};
//----------------------------------------------------------------------

12
writeengine/splitter/we_sdhandler.cpp
View File

@@ -474,20 +474,20 @@ void WESDHandler::setup()
std::string aErrLogName;
if (fRef.fCmdArgs.isJobLogOnly())
aLogName = fRef.fCmdArgs.getBulkRootDir() + "/log/" +
aLogName = std::string(MCSLOGDIR) + "/cpimport/" +
"cpimport_Job_" + fRef.fCmdArgs.getJobId() + ".log";
else
aLogName = fRef.fCmdArgs.getBulkRootDir() + "/log/" +
aLogName = std::string(MCSLOGDIR) + "/cpimport/" +
"cpimport_" + aTimeStamp + "_" + aPid.str() + ".log";
if (getDebugLvl() > 1) cout << "LogName : " << aLogName << endl;
if (fRef.fCmdArgs.isJobLogOnly())
aErrLogName = fRef.fCmdArgs.getBulkRootDir() + "/log/" +
aErrLogName = std::string(MCSLOGDIR) + "/cpimport/" +
"cpimport_Job_" + fRef.fCmdArgs.getJobId() + ".err";
else
aErrLogName = fRef.fCmdArgs.getBulkRootDir() + "/log/" +
aErrLogName = std::string(MCSLOGDIR) + "/cpimport/" +
"cpimport_" + aTimeStamp + "_" + aPid.str() + ".err";
@@ -1529,7 +1529,7 @@ void WESDHandler::onErrorFile(int PmId, messageqcpp::SBS& Sbs)
{
std::string aFile = aTmpFileName.substr(aPos + 1); //+1 to pass '/'
std::string aInFile = fRef.getLocFile(); // input file
std::string aInFile = fRef.fCmdArgs.getErrorDir();
if (aInFile != "/dev/stdin")
{
@@ -1627,7 +1627,7 @@ void WESDHandler::onBadFile(int PmId, messageqcpp::SBS& Sbs)
{
std::string aFile = aTmpFileName.substr(aPos + 1); //+1 to pass '/'
std::string aInFile = fRef.getLocFile(); // input file
std::string aInFile = fRef.fCmdArgs.getErrorDir();
if (aInFile != "/dev/stdin")
{