1
0
mirror of https://github.com/mariadb-corporation/mariadb-columnstore-engine.git synced 2025-07-30 19:23:07 +03:00

Merge pull request #1031 from pleblanc1976/we-splitter-read-from-s3

MCOL-3520: Fix importing files from S3 for mode 1 imports.
This commit is contained in:
Gagan Goel
2020-02-10 11:57:33 -05:00
committed by Patrick LeBlanc
parent f71158601e
commit 3faa1600c3
7 changed files with 186 additions and 79 deletions

View File

@ -37,7 +37,10 @@ class WECmdArgs
public:
WECmdArgs(int argc, char** argv);
virtual ~WECmdArgs() {}
public:
typedef std::vector<unsigned int> VecInts;
typedef std::vector<std::string> VecArgs;
void appTestFunction();
void parseCmdLineArgs(int argc, char** argv);
std::string getCpImportCmdLine();
@ -54,7 +57,6 @@ public:
char* pBuff, int FileIdx);
void updateWithJobFile(int Idx);
public:
std::string getJobFileName();
std::string getBrmRptFileName();
std::string getTmpFileDir();
@ -195,19 +197,6 @@ public:
return fTimeZone;
}
private: // variables for SplitterApp
typedef std::vector<std::string> VecArgs;
VecArgs fVecArgs;
typedef std::vector<unsigned int> VecInts;
VecInts fPmVec;
VecArgs fVecJobFiles; //JobFiles splitter from master JobFile
int fMultiTableCount; //MultiTable count
VecArgs fColFldsFromJobFile;//List of columns from any job file, that
// represent fields in the import data
public:
bool getPmStatus(int Id);
bool str2PmList(std::string& PmList, VecInts& V);
int getPmVecSize()
@ -252,11 +241,44 @@ public:
{
fMultiTableCount = Count;
}
bool isS3Import() const
{
return !fS3Key.empty();
}
std::string getS3Key() const
{
return fS3Key;
}
std::string getS3Bucket() const
{
return fS3Bucket;
}
std::string getS3Host() const
{
return fS3Host;
}
std::string getS3Secret() const
{
return fS3Secret;
}
std::string getS3Region() const
{
return fS3Region;
}
std::string PrepMode2ListOfFiles(std::string& FileName); // Bug 4342
void getColumnList( std::set<std::string>& columnList ) const;
private: // variables for SplitterApp
VecArgs fVecArgs;
VecInts fPmVec;
VecArgs fVecJobFiles; //JobFiles splitter from master JobFile
int fMultiTableCount; //MultiTable count
VecArgs fColFldsFromJobFile;//List of columns from any job file, that
// represent fields in the import data
std::string fJobId; // JobID
std::string fOrigJobId; // Original JobID, in case we have to split it
bool fJobLogOnly; // Job number is only for log filename only