From d010397be647a540189687434c57084bed1e7938 Mon Sep 17 00:00:00 2001
From: Andrew Hutchings <andrew@linuxjedi.co.uk>
Date: Wed, 9 May 2018 14:52:42 +0100
Subject: [PATCH] MCOL-1197 Make -c work in cpimport

It turns out -c wasn't actually connected to anything and now with have
BLOB/TEXT it is pretty useful. If -c is set to < 1MB then 1MB is used,
otherwise it will use the selected buffer size.
---
 writeengine/splitter/we_cmdargs.h          |  1 +
 writeengine/splitter/we_filereadthread.cpp | 18 ++++++++++++++----
 writeengine/splitter/we_filereadthread.h   |  5 +++--
 writeengine/splitter/we_sdhandler.cpp      |  7 +++++++
 writeengine/splitter/we_sdhandler.h        |  1 +
 5 files changed, 26 insertions(+), 6 deletions(-)

diff --git a/writeengine/splitter/we_cmdargs.h b/writeengine/splitter/we_cmdargs.h
index 96e06a4bc..803f77c54 100644
--- a/writeengine/splitter/we_cmdargs.h
+++ b/writeengine/splitter/we_cmdargs.h
@@ -77,6 +77,7 @@ class WECmdArgs
 		char getDelimChar() { return fColDelim; }
 		ImportDataMode getImportDataMode() const { return fImportDataMode; }
 		bool getConsoleLog() { return fConsoleLog; }
+        int getReadBufSize() { return fReadBufSize; }
 
 		bool isCpimportInvokeMode(){return (fBlockMode3)? false : fCpiInvoke;}
 		bool isQuiteMode() const {	return fQuiteMode;	}
diff --git a/writeengine/splitter/we_filereadthread.cpp b/writeengine/splitter/we_filereadthread.cpp
index 6e4fcb4a7..185557965 100644
--- a/writeengine/splitter/we_filereadthread.cpp
+++ b/writeengine/splitter/we_filereadthread.cpp
@@ -87,6 +87,15 @@ WEFileReadThread::WEFileReadThread(WESDHandler& aSdh):fSdh(aSdh),
 {
 	//TODO batch qty to get from config
 	fBatchQty = 10000;
+    if (fSdh.getReadBufSize() < DEFAULTBUFFSIZE)
+    {
+        fBuffSize = DEFAULTBUFFSIZE;
+    }
+    else
+    {
+        fBuffSize = fSdh.getReadBufSize();
+    }
+    fBuff = new char [fBuffSize];
 
 }
 
@@ -106,6 +115,7 @@ WEFileReadThread::~WEFileReadThread()
 		delete fpThread;
 	}
 	fpThread=0;
+    delete []fBuff;
 	//cout << "WEFileReadThread destructor called" << endl;
 }
 
@@ -330,16 +340,16 @@ unsigned int WEFileReadThread::readDataFile(messageqcpp::SBS& Sbs)
 			if(fEnclEsc)
 			{
 				//pStart = aBuff;
-				aLen = getNextRow(fInFile, fBuff, sizeof(fBuff)-1);
+				aLen = getNextRow(fInFile, fBuff, fBuffSize-1);
 			}
 			else
 			{
-				fInFile.getline(fBuff, sizeof(fBuff)-1);
+				fInFile.getline(fBuff, fBuffSize-1);
 				aLen=fInFile.gcount();
 			}
 			////aLen chars incl \n, Therefore aLen-1; '<<' oper won't go past it
 			//cout << "Data Length " << aLen <<endl;
-			if((aLen < (sizeof(fBuff)-2)) && (aLen>0))
+			if((aLen < (fBuffSize-2)) && (aLen>0))
 			{
 				fBuff[aLen-1] = '\n';
 				fBuff[aLen]=0;
@@ -348,7 +358,7 @@ unsigned int WEFileReadThread::readDataFile(messageqcpp::SBS& Sbs)
 				aIdx++;
 				if(fSdh.getDebugLvl()>2) cout << "File data line = " << aIdx <<endl;
 			}
-			else if(aLen>=sizeof(fBuff)-2)	//Didn't hit delim; BIG ROW
+			else if(aLen>=fBuffSize-2)	//Didn't hit delim; BIG ROW
 			{
 				cout <<"Bad Row data " << endl;
 				cout << fBuff << endl;
diff --git a/writeengine/splitter/we_filereadthread.h b/writeengine/splitter/we_filereadthread.h
index 623184e8d..f9a486d5c 100644
--- a/writeengine/splitter/we_filereadthread.h
+++ b/writeengine/splitter/we_filereadthread.h
@@ -98,7 +98,7 @@ public:
     void add2InputDataFileList(std::string& FileName);
 
 private:
-    enum { MAXBUFFSIZE=1024*1024 };
+    enum { DEFAULTBUFFSIZE=1024*1024 };
 
     // don't allow anyone else to set
     void setTgtPmId(unsigned int fTgtPmId) { this->fTgtPmId = fTgtPmId; }
@@ -120,7 +120,8 @@ private:
     char fEncl;							// Encl char
     char fEsc;							// Esc char
     char fDelim;						// Column Delimit char
-    char fBuff[MAXBUFFSIZE];			// main data buffer
+    char* fBuff;			// main data buffer
+    int fBuffSize;
 };
 
 } /* namespace WriteEngine */
diff --git a/writeengine/splitter/we_sdhandler.cpp b/writeengine/splitter/we_sdhandler.cpp
index 56ace80dc..78019b338 100644
--- a/writeengine/splitter/we_sdhandler.cpp
+++ b/writeengine/splitter/we_sdhandler.cpp
@@ -2301,6 +2301,13 @@ char WESDHandler::getEscChar()
 
 //------------------------------------------------------------------------------
 
+int WESDHandler::getReadBufSize()
+{
+    return fRef.fCmdArgs.getReadBufSize();
+}
+
+//------------------------------------------------------------------------------
+
 char WESDHandler::getDelimChar()
 {
 	return fRef.fCmdArgs.getDelimChar();
diff --git a/writeengine/splitter/we_sdhandler.h b/writeengine/splitter/we_sdhandler.h
index 2d4b20cc2..ff6bc829e 100644
--- a/writeengine/splitter/we_sdhandler.h
+++ b/writeengine/splitter/we_sdhandler.h
@@ -149,6 +149,7 @@ public:
     char getEscChar();
     char getDelimChar();
 	bool getConsoleLog();
+    int getReadBufSize();
     ImportDataMode getImportDataMode() const;
     void sysLog(const logging::Message::Args& msgArgs,
     		logging::LOG_TYPE logType, logging::Message::MessageID msgId);