MCOL-4576: remove S3 options from cpimport. (#2307)

writeengine/bulk/cpimport.cpp

@@ -172,12 +172,11 @@ void printUsage()
        << "        -T Timezone used for TIMESTAMP datatype" << endl
        << "           Possible values: \"SYSTEM\" (default)" << endl
        << "                          : Offset in the form +/-HH:MM" << endl
-       << endl
-       << "        -y S3 Authentication Key (for S3 imports)" << endl
-       << "        -K S3 Authentication Secret (for S3 imports)" << endl
-       << "        -t S3 Bucket (for S3 imports)" << endl
-       << "        -H S3 Hostname (for S3 imports, Amazon's S3 default)" << endl
-       << "        -g S3 Regions (for S3 imports)" << endl
+//       << "        -y S3 Authentication Key (for S3 imports)" << endl
+//       << "        -K S3 Authentication Secret (for S3 imports)" << endl
+//       << "        -t S3 Bucket (for S3 imports)" << endl
+//       << "        -H S3 Hostname (for S3 imports, Amazon's S3 default)" << endl
+//       << "        -g S3 Regions (for S3 imports)" << endl
        << "        -U username of new data files owner. Default is mysql" << endl;
 
   cout << "    Example1:" << endl
@@ -310,7 +309,7 @@ void parseCmdLineArgs(int argc, char** argv, BulkLoad& curJob, std::string& sJob
   BulkModeType bulkMode = BULK_MODE_LOCAL;
   std::string jobUUID;
 
-  while ((option = getopt(argc, argv, "b:c:d:e:f:hij:kl:m:n:p:r:s:u:w:B:C:DE:I:P:R:ST:X:NL:y:K:t:H:g:U:")) !=
+  while ((option = getopt(argc, argv, "b:c:d:e:f:hij:kl:m:n:p:r:s:u:w:B:C:DE:I:P:R:ST:X:NL:U:")) !=
          EOF)
   {
     switch (option)
@@ -677,7 +676,7 @@ void parseCmdLineArgs(int argc, char** argv, BulkLoad& curJob, std::string& sJob
         break;
       }
 
-      case 'y':
+/*      case 'y':
       {
         curJob.setS3Key(optarg);
         break;
@@ -706,7 +705,7 @@ void parseCmdLineArgs(int argc, char** argv, BulkLoad& curJob, std::string& sJob
         curJob.setS3Region(optarg);
         break;
       }
-
+*/
       case 'U':
       {
         curJob.setUsername(optarg);

writeengine/splitter/we_cmdargs.cpp

@@ -561,11 +561,6 @@ void WECmdArgs::usage()
        << "\t-T\tTimezone used for TIMESTAMP datatype.\n"
        << "\t\tPossible values: \"SYSTEM\" (default)\n"
        << "\t\t               : Offset in the form +/-HH:MM\n"
-       << "\t-y\tS3 Authentication Key (for S3 imports)\n"
-       << "\t-K\tS3 Authentication Secret (for S3 imports)\n"
-       << "\t-t\tS3 Bucket (for S3 imports)\n"
-       << "\t-H\tS3 Hostname (for S3 imports, Amazon's S3 default)\n"
-       << "\t-g\tS3 Region (for S3 imports)\n"
        << "\t-L\tDirectory for the output .err and .bad files.\n"
        << "\t\tDefault is " << string(MCSLOGDIR);
 
@@ -598,7 +593,7 @@ void WECmdArgs::parseCmdLineArgs(int argc, char** argv)
   if (argc > 0)
     fPrgmName = string(MCSBINDIR) + "/" + "cpimport.bin";  // argv[0] is splitter but we need cpimport
 
-  while ((aCh = getopt(argc, argv, "d:j:w:s:v:l:r:b:e:B:f:q:ihm:E:C:P:I:n:p:c:ST:Ny:K:t:H:g:U:L:")) != EOF)
+  while ((aCh = getopt(argc, argv, "d:j:w:s:v:l:r:b:e:B:f:q:ihm:E:C:P:I:n:p:c:ST:N:U:L:")) != EOF)
   {
     switch (aCh)
     {
@@ -906,7 +901,7 @@ void WECmdArgs::parseCmdLineArgs(int argc, char** argv)
         fConsoleOutput = false;
         break;
       }
-
+/*
       case 'y':  //-y S3 Key
       {
         fS3Key = optarg;
@@ -936,7 +931,7 @@ void WECmdArgs::parseCmdLineArgs(int argc, char** argv)
         fS3Region = optarg;
         break;
       }
-
+*/
       case 'U':  //-U username of the files owner
       {
         fUsername = optarg;

writeengine/splitter/we_filereadthread.cpp

@@ -481,6 +481,15 @@ void WEFileReadThread::openInFile()
         use ms3 lib to d/l data into mem
         use boost::iostreams to wrap the mem in a stream interface
         point infile's stream buffer to it.
+        MCOL-4576: The options to setup S3 with cpimport have been removed and this
+        code is unreachable. However we may need to resurrect it at some point in some form.
+        Performance issues with extremely large data files as well as the fact files larger
+        than system memory will cause an OOM error. Multipart downloads/uploads need to be
+        implemented or more likely a different streaming solution developed with external API tools
+
+        MCOL-4576 work around is to use 3rd party CLI tools and pipe data file from S3 bucket
+        into cpimport stdin. 3rd party tooling for large object downloads will be more efficient.
+
     */
 
     if (fSdh.getDebugLvl())