database/mariadb-columnstore-engine
1
0
Fork 0
mirror of https://github.com/mariadb-corporation/mariadb-columnstore-engine.git synced 2025-07-30 19:23:07 +03:00
Code Activity

Merge pull request #1031 from pleblanc1976/we-splitter-read-from-s3

Browse Source 
MCOL-3520: Fix importing files from S3 for mode 1 imports.
This commit is contained in:
Gagan Goel
2020-02-10 11:57:33 -05:00
committed by Patrick LeBlanc
parent f71158601e
commit 3faa1600c3
7 changed files with 186 additions and 79 deletions
Download Patch File Download Diff File Expand all files Collapse all files

34
writeengine/splitter/we_sdhandler.cpp
View File

@ -501,14 +501,9 @@ void WESDHandler::setup()
//fLog.setLogFileName(aLogName.c_str(), aErrLogName.c_str(), false);
fLog.setLogFileName(aLogName.c_str(), aErrLogName.c_str(), getConsoleLog());
// In mode 0 and Mode 1, we need to check for local file availability
if ((0 == fRef.fCmdArgs.getMode()) || (1 == fRef.fCmdArgs.getMode()))
{
if (!check4InputFile(fRef.getLocFile()))
{
throw (runtime_error("Could not open Input file " + fRef.getLocFile()));
}
}
// In mode 0 and Mode 1, we need to construct the input file list and check availability
if (0 == fRef.fCmdArgs.getMode() || 1 == fRef.fCmdArgs.getMode())
setInputFileList(fRef.getLocFile());
fImportRslt.startTimer();
@ -2726,28 +2721,9 @@ std::string WESDHandler::getTime2Str() const
//------------------------------------------------------------------------------
bool WESDHandler::check4InputFile(std::string InFileName)
void WESDHandler::setInputFileList(std::string InFileName)
{
bool aRet = false;
if ((0 == InFileName.compare("STDIN")) || (0 == InFileName.compare("stdin")))
{
fFileReadThread.add2InputDataFileList(InFileName);
return true;
}
else
{
//BUG 4342 - Need to support "list of infiles"
fFileReadThread.chkForListOfFiles(InFileName);
std::string aFileName = fFileReadThread.getNextInputDataFile();
std::ifstream aFile(aFileName.c_str());
aRet = (aFile.good()) ? true : false;
// add back to list, which we pop_front for checking the file.
if (aRet) fFileReadThread.add2InputDataFileList(aFileName);
}
return aRet;
fFileReadThread.chkForListOfFiles(InFileName);
}
//------------------------------------------------------------------------------