database/mariadb-columnstore-engine
1
0
Fork 0
mirror of https://github.com/mariadb-corporation/mariadb-columnstore-engine.git synced 2025-07-30 19:23:07 +03:00
Code Activity

MCOL-5013: Load Data from S3 into Columnstore

Browse Source 
Introduced UDF and stored prodecure.
usage:

set columnstore_s3_key='<s3_key>';
set columnstore_s3_secret='<s3_secret>';
set columnstore_s3_region='region';

and then use UDF
select columnstore_dataload("<tablename>", "<filename>", "<bucket>", "<db_name>");
for UDF db_name can be ommited, then current connection db will be used

or stored function
call calpontsys.columnstore_load_from_s3("<tablename>", "<filename>", "<bucket>", "<db_name>");
This commit is contained in:
Leonid Fedorov
2022-03-28 14:26:02 +00:00
parent 7d955a0f85
commit f5b2a6885f
9 changed files with 356 additions and 29 deletions
Download Patch File Download Diff File Expand all files Collapse all files

93
dbcon/mysql/ha_mcs_sysvars.cpp
View File

@ -188,6 +188,22 @@ static MYSQL_THDVAR_ULONGLONG(cache_flush_threshold, PLUGIN_VAR_RQCMDARG,
"Threshold on the number of rows in the cache to trigger a flush", NULL, NULL,
500000, 1, 1000000000, 1);
static MYSQL_THDVAR_STR(cmapi_host, PLUGIN_VAR_NOCMDOPT|PLUGIN_VAR_MEMALLOC, "CMAPI host", NULL, NULL,
"https://localhost");
static MYSQL_THDVAR_STR(cmapi_version, PLUGIN_VAR_NOCMDOPT|PLUGIN_VAR_MEMALLOC, "CMAPI version", NULL, NULL,
"0.4.0");
static MYSQL_THDVAR_STR(cmapi_key, PLUGIN_VAR_NOCMDOPT|PLUGIN_VAR_MEMALLOC, "CMAPI key", NULL, NULL,
"");
static MYSQL_THDVAR_ULONGLONG(cmapi_port, PLUGIN_VAR_NOCMDOPT, "CMAPI port", NULL,
NULL, 8640, 100, 65356, 1);
static MYSQL_THDVAR_STR(s3_key, PLUGIN_VAR_NOCMDOPT|PLUGIN_VAR_MEMALLOC, "S3 Authentication Key ", NULL, NULL, "");
static MYSQL_THDVAR_STR(s3_secret, PLUGIN_VAR_NOCMDOPT|PLUGIN_VAR_MEMALLOC, "S3 Authentication Secret", NULL, NULL, "");
static MYSQL_THDVAR_STR(s3_region, PLUGIN_VAR_NOCMDOPT|PLUGIN_VAR_MEMALLOC, "S3 region", NULL, NULL, "");
st_mysql_sys_var* mcs_system_variables[] = {MYSQL_SYSVAR(compression_type),
MYSQL_SYSVAR(fe_conn_info_ptr),
MYSQL_SYSVAR(original_optimizer_flags),
@ -217,6 +233,13 @@ st_mysql_sys_var* mcs_system_variables[] = {MYSQL_SYSVAR(compression_type),
MYSQL_SYSVAR(cache_inserts),
MYSQL_SYSVAR(cache_use_import),
MYSQL_SYSVAR(cache_flush_threshold),
MYSQL_SYSVAR(cmapi_host),
MYSQL_SYSVAR(cmapi_port),
MYSQL_SYSVAR(cmapi_version),
MYSQL_SYSVAR(cmapi_key),
MYSQL_SYSVAR(s3_key),
MYSQL_SYSVAR(s3_secret),
MYSQL_SYSVAR(s3_region),
NULL};
st_mysql_show_var mcs_status_variables[] = {{"columnstore_version", (char*)&cs_version, SHOW_CHAR},
@ -505,3 +528,73 @@ void set_cache_flush_threshold(THD* thd, ulonglong value)
{
THDVAR(thd, cache_flush_threshold) = value;
}
ulong get_cmapi_port(THD* thd)
{
return (thd == NULL) ? 5000 : THDVAR(thd, cmapi_port);
}
void set_cmapi_port(THD* thd, ulong value)
{
THDVAR(thd, cmapi_port) = value;
}
const char* get_cmapi_host(THD* thd)
{
return (thd == NULL) ? "127.0.0.1" : THDVAR(thd, cmapi_host);
}
void set_cmapi_host(THD* thd, char* value)
{
THDVAR(thd, cmapi_host) = value;
}
const char* get_cmapi_version(THD* thd)
{
return (thd == NULL) ? "0.4.0" : THDVAR(thd, cmapi_version);
}
void set_cmapi_version(THD* thd, char* value)
{
THDVAR(thd, cmapi_version) = value;
}
const char* get_cmapi_key(THD* thd)
{
return (thd == NULL) ? "" : THDVAR(thd, cmapi_key);
}
void set_cmapi_key(THD* thd, char* value)
{
THDVAR(thd, cmapi_key) = value;
}
const char* get_s3_key(THD* thd)
{
return THDVAR(thd, s3_key);
}
void set_s3_key(THD* thd, char* value)
{
THDVAR(thd, s3_key) = value;
}
const char* get_s3_secret(THD* thd)
{
return THDVAR(thd, s3_secret);
}
void set_s3_secret(THD* thd, char* value)
{
THDVAR(thd, s3_secret) = value;
}
const char* get_s3_region(THD* thd)
{
return THDVAR(thd, s3_region);
}
void set_s3_region(THD* thd, char* value)
{
THDVAR(thd, s3_region) = value;
}