database/mariadb-columnstore-engine
1
0
Fork 0
mirror of https://github.com/mariadb-corporation/mariadb-columnstore-engine.git synced 2025-07-29 08:21:15 +03:00
Code Activity

Feature/mcol 4882 cpimport skip rows (#3594)

Browse Source 
* feat(cpimport): MCOL-4882 add a parameter to skip header rows

* chore(cpimport): MCOL-4882 Use boost::program_options to arguments parsing

* feat(cpimport.bin): MCOL-4882 Add missing changes

* add test

* fix clang

* add missing cmdline argument

* fix bug

* Fix double lines skipping

* Fix incorrect --silent (-N) parsing

* fix default --max-errors processing

* fix overwriting default username

* move initialization to members declaration
This commit is contained in:
Alexey Antipovsky
2025-07-11 21:35:43 +02:00
committed by GitHub
parent 1c8d5ec04e
commit 78c1b5034d
30 changed files with 1379 additions and 1469 deletions
Download Patch File Download Diff File Expand all files Collapse all files

44
writeengine/xml/we_xmlgendata.cpp
View File

@ -35,18 +35,19 @@ namespace WriteEngine
{
/* static */ const std::string XMLGenData::DELIMITER("-d");
/* static */ const std::string XMLGenData::DESCRIPTION("-s");
/* static */ const std::string XMLGenData::ENCLOSED_BY_CHAR("-E");
/* static */ const std::string XMLGenData::ESCAPE_CHAR("-C");
/* static */ const std::string XMLGenData::JOBID("-j");
/* static */ const std::string XMLGenData::ENCLOSED_BY_CHAR("-E");
/* static */ const std::string XMLGenData::ESCAPE_CHAR("-C");
/* static */ const std::string XMLGenData::JOBID("-j");
/* static */ const std::string XMLGenData::MAXERROR("-e");
/* static */ const std::string XMLGenData::NAME("-n");
/* static */ const std::string XMLGenData::PATH("-p");
/* static */ const std::string XMLGenData::RPT_DEBUG("-b");
/* static */ const std::string XMLGenData::RPT_DEBUG("-b");
/* static */ const std::string XMLGenData::USER("-u");
/* static */ const std::string XMLGenData::NO_OF_READ_BUFFER("-r");
/* static */ const std::string XMLGenData::READ_BUFFER_CAPACITY("-c");
/* static */ const std::string XMLGenData::WRITE_BUFFER_SIZE("-w");
/* static */ const std::string XMLGenData::EXT("-x");
/* static */ const std::string XMLGenData::SKIP_ROWS("-O");
//------------------------------------------------------------------------------
// XMLGenData constructor
@ -54,39 +55,38 @@ namespace WriteEngine
//------------------------------------------------------------------------------
XMLGenData::XMLGenData()
{
fParms.insert(ParmList::value_type(DELIMITER, std::string("|")));
fParms.insert(ParmList::value_type(DESCRIPTION, std::string()));
fParms.insert(ParmList::value_type(ENCLOSED_BY_CHAR, std::string("")));
fParms.insert(ParmList::value_type(ESCAPE_CHAR, std::string("\\")));
fParms.insert(ParmList::value_type(JOBID, std::string("299")));
fParms.insert(ParmList::value_type(MAXERROR, std::string("10")));
fParms.insert(ParmList::value_type(NAME, std::string()));
fParms.emplace(DELIMITER, "|");
fParms.emplace(DESCRIPTION, "");
fParms.emplace(ENCLOSED_BY_CHAR, "");
fParms.emplace(ESCAPE_CHAR, "\\");
fParms.emplace(JOBID, "299");
fParms.emplace(MAXERROR, "10");
fParms.emplace(NAME, "");
boost::filesystem::path p{std::string(Config::getBulkRoot())};
p /= JOBDIR;
fParms.insert(ParmList::value_type(PATH, p.string()));
fParms.emplace(PATH, p.string());
fParms.insert(ParmList::value_type(RPT_DEBUG, std::string("0")));
fParms.insert(ParmList::value_type(USER, std::string()));
fParms.insert(ParmList::value_type(NO_OF_READ_BUFFER, std::string("5")));
fParms.insert(ParmList::value_type(READ_BUFFER_CAPACITY, std::string("1048576")));
fParms.insert(ParmList::value_type(WRITE_BUFFER_SIZE, std::string("10485760")));
fParms.insert(ParmList::value_type(EXT, std::string("tbl")));
fParms.emplace(RPT_DEBUG, "0");
fParms.emplace(USER, "");
fParms.emplace(NO_OF_READ_BUFFER, "5");
fParms.emplace(READ_BUFFER_CAPACITY, "1048576");
fParms.emplace(WRITE_BUFFER_SIZE, "10485760");
fParms.emplace(EXT, "tbl");
fParms.emplace(SKIP_ROWS, "0");
}
//------------------------------------------------------------------------------
// XMLGenData destructor
//------------------------------------------------------------------------------
/* virtual */
XMLGenData::~XMLGenData()
{
}
XMLGenData::~XMLGenData() = default;
//------------------------------------------------------------------------------
// Return value for the specified parm.
//------------------------------------------------------------------------------
std::string XMLGenData::getParm(const std::string& key) const
{
ParmList::const_iterator p = fParms.find(key);
auto p = fParms.find(key);
if (fParms.end() != p)
return p->second;

7
writeengine/xml/we_xmlgendata.h
View File

@ -60,10 +60,13 @@ class XMLGenData
EXPORT const static std::string READ_BUFFER_CAPACITY;
EXPORT const static std::string WRITE_BUFFER_SIZE;
EXPORT const static std::string EXT;
EXPORT const static std::string SKIP_ROWS;
/** @brief XMLGenData constructor
*/
EXPORT XMLGenData();
XMLGenData(const XMLGenData&) = delete;
XMLGenData& operator=(const XMLGenData&) = delete;
/** @brief XMLGenData destructor
*/
@ -92,10 +95,6 @@ class XMLGenData
ParmList fParms;
std::string fSchema;
LoadNames fLoadNames;
private:
XMLGenData(const XMLGenData&); // disable default copy ctor
XMLGenData& operator=(const XMLGenData&); // disable default assignment
};
} // namespace WriteEngine

5
writeengine/xml/we_xmlgenproc.cpp
View File

@ -147,6 +147,11 @@ void XMLGenProc::startXMLFile()
xmlTextWriterWriteElement(fWriter, BAD_CAST xmlTagTable[TAG_ESCAPE_CHAR],
BAD_CAST fInputMgr->getParm(XMLGenData::ESCAPE_CHAR).c_str());
if (auto skipRows = fInputMgr->getParm(XMLGenData::SKIP_ROWS); !skipRows.empty())
{
xmlTextWriterWriteElement(fWriter, BAD_CAST xmlTagTable[TAG_SKIP_ROWS], BAD_CAST skipRows.c_str());
}
// Added new tags for configurable parameters
xmlTextWriterStartElement(fWriter, BAD_CAST xmlTagTable[TAG_READ_BUFFERS]);
xmlTextWriterWriteFormatAttribute(fWriter, BAD_CAST xmlTagTable[TAG_NO_OF_READ_BUFFERS], "%d",

12
writeengine/xml/we_xmljob.cpp
View File

@ -130,6 +130,7 @@ void XMLJob::printJobInfo(Log& logger) const
oss1 << "Read Buffers: " << job.numberOfReadBuffers << endl;
oss1 << "Read Buffer Size: " << job.readBufferSize << endl;
oss1 << "setvbuf Size: " << job.writeBufferSize << endl;
oss1 << "Header rows : " << job.fSkipRows << endl;
oss1 << "Create Date : " << job.createDate << endl;
oss1 << "Create Time : " << job.createTime << endl;
oss1 << "Schema Name : " << job.schema << endl;
@ -223,7 +224,8 @@ void XMLJob::printJobInfoBrief(Log& logger) const
oss1 << "n/a";
oss1 << "); ReadBufs(" << job.numberOfReadBuffers << "); ReadBufSize(" << job.readBufferSize
<< "); setvbufSize(" << job.writeBufferSize << ')';
<< "); setvbufSize(" << job.writeBufferSize << "); "
<< "SkipRows(" << job.fSkipRows << ")";
logger.logMsg(oss1.str(), MSGLVL_INFO2);
for (unsigned int i = 0; i < job.jobTableList.size(); i++)
@ -316,6 +318,8 @@ bool XMLJob::processNode(xmlNode* pNode)
setJobData(pNode, TAG_ENCLOSED_BY_CHAR, true, TYPE_CHAR);
else if (isTag(pNode, TAG_ESCAPE_CHAR))
setJobData(pNode, TAG_ESCAPE_CHAR, true, TYPE_CHAR);
else if (isTag(pNode, TAG_SKIP_ROWS))
setJobData(pNode, TAG_SKIP_ROWS, true, TYPE_INT);
else
{
ostringstream oss;
@ -432,6 +436,12 @@ void XMLJob::setJobData(xmlNode* pNode, const xmlTag tag, bool bExpectContent, X
break;
}
case TAG_SKIP_ROWS:
{
fJob.fSkipRows = intVal;
break;
}
default: break;
}
}

4
writeengine/xml/we_xmltag.h
View File

@ -73,6 +73,7 @@ enum xmlTag
TAG_TBL_OID,
TAG_WIDTH,
TAG_SCHEMA_NAME,
TAG_SKIP_ROWS,
NUM_OF_XML_TAGS
};
@ -93,6 +94,7 @@ const char xmlTagTable[NUM_OF_XML_TAGS + 1][MAX_XML_TAG_NAME_SIZE] = {
"origName", //@bug 3599: deprecated; kept for backwards compatibility
"precision", "scale",
"tblName", //@bug 3599: replaces origName
"tblOid", "width", "Name"};
"tblOid", "width", "Name",
"skipRows"};
} // namespace WriteEngine