database/mariadb-columnstore-engine
1
0
Fork 0
mirror of https://github.com/mariadb-corporation/mariadb-columnstore-engine.git synced 2025-07-29 08:21:15 +03:00
Code Activity

MCOL-5013: Load Data from S3 into Columnstore

Browse Source 
Introduced UDF and stored prodecure.
usage:

set columnstore_s3_key='<s3_key>';
set columnstore_s3_secret='<s3_secret>';
set columnstore_s3_region='region';

and then use UDF
select columnstore_dataload("<tablename>", "<filename>", "<bucket>", "<db_name>");
for UDF db_name can be ommited, then current connection db will be used

or stored function
call calpontsys.columnstore_load_from_s3("<tablename>", "<filename>", "<bucket>", "<db_name>");
This commit is contained in:
Leonid Fedorov
2022-03-28 14:26:02 +00:00
parent 7d955a0f85
commit f5b2a6885f
9 changed files with 356 additions and 29 deletions
Download Patch File Download Diff File Expand all files Collapse all files

16
writeengine/splitter/we_cmdargs.cpp
View File

@ -561,11 +561,11 @@ void WECmdArgs::usage()
<< "\t-T\tTimezone used for TIMESTAMP datatype.\n"
<< "\t\tPossible values: \"SYSTEM\" (default)\n"
<< "\t\t : Offset in the form +/-HH:MM\n"
// << "\t-y\tS3 Authentication Key (for S3 imports)\n"
// << "\t-K\tS3 Authentication Secret (for S3 imports)\n"
// << "\t-t\tS3 Bucket (for S3 imports)\n"
// << "\t-H\tS3 Hostname (for S3 imports, Amazon's S3 default)\n"
// << "\t-g\tS3 Region (for S3 imports)\n"
<< "\t-y\tS3 Authentication Key (for S3 imports)\n"
<< "\t-K\tS3 Authentication Secret (for S3 imports)\n"
<< "\t-t\tS3 Bucket (for S3 imports)\n"
<< "\t-H\tS3 Hostname (for S3 imports, Amazon's S3 default)\n"
<< "\t-g\tS3 Region (for S3 imports)\n"
<< "\t-L\tDirectory for the output .err and .bad files.\n"
<< "\t\tDefault is " << string(MCSLOGDIR);
@ -598,7 +598,7 @@ void WECmdArgs::parseCmdLineArgs(int argc, char** argv)
if (argc > 0)
fPrgmName = string(MCSBINDIR) + "/" + "cpimport.bin"; // argv[0] is splitter but we need cpimport
while ((aCh = getopt(argc, argv, "d:j:w:s:v:l:r:b:e:B:f:q:ihm:E:C:P:I:n:p:c:ST:NU:L:")) != EOF)
while ((aCh = getopt(argc, argv, "d:j:w:s:v:l:r:b:e:B:f:q:ihm:E:C:P:I:n:p:c:ST:Ny:K:t:H:g:U:L:")) != EOF)
{
switch (aCh)
{
@ -906,7 +906,7 @@ void WECmdArgs::parseCmdLineArgs(int argc, char** argv)
fConsoleOutput = false;
break;
}
/*
case 'y': //-y S3 Key
{
fS3Key = optarg;
@ -936,7 +936,7 @@ void WECmdArgs::parseCmdLineArgs(int argc, char** argv)
fS3Region = optarg;
break;
}
*/
case 'U': //-U username of the files owner
{
fUsername = optarg;