database/mariadb
1
0
Fork 0
mirror of https://github.com/MariaDB/server.git synced 2025-08-08 11:22:35 +03:00
Code Activity

MDEV-23662: Make S3 Storage Engine usable with MinIO

Browse Source 
Add @@s3_port and @@s3_use_http. One can set these to access the
local MinIO, for example.
This commit is contained in:
Sergei Petrunia
2020-09-04 00:15:22 +03:00
parent c5517cd864
commit e38ac39104
6 changed files with 49 additions and 0 deletions
Download Patch File Download Diff File Expand all files Collapse all files

2
mysql-test/suite/s3/basic.result
View File

@@ -94,11 +94,13 @@ s3_pagecache_age_threshold X
s3_pagecache_buffer_size X s3_pagecache_buffer_size X
s3_pagecache_division_limit X s3_pagecache_division_limit X
s3_pagecache_file_hash_size X s3_pagecache_file_hash_size X
s3_port X
s3_protocol_version X s3_protocol_version X
s3_region X s3_region X
s3_replicate_alter_as_create_select X s3_replicate_alter_as_create_select X
s3_secret_key X s3_secret_key X
s3_slave_ignore_updates X s3_slave_ignore_updates X
s3_use_http X
show variables like "s3_slave%"; show variables like "s3_slave%";
Variable_name Value Variable_name Value
s3_slave_ignore_updates OFF s3_slave_ignore_updates OFF

12
mysql-test/suite/s3/my.cnf
View File

@@ -10,3 +10,15 @@ s3=ON
#s3-access-key=... #s3-access-key=...
#s3-secret-key=... #s3-secret-key=...
#s3-region=eu-north-1 #s3-region=eu-north-1
##
## Configuration for local MinIO
##
s3-host-name="127.0.0.1"
# Note: s3-host-name="localhost" doesn't work. It causes
# libmarias3 to use the wrong variant of the protocol.
s3-bucket=storage-engine
s3-access-key=minioadmin
s3-secret-key=minioadmin
s3-port=9000
s3-use-http=ON

12
mysql-test/suite/s3/slave.cnf
View File

@@ -6,3 +6,15 @@ s3=ON
#s3-access-key=... #s3-access-key=...
#s3-secret-key=... #s3-secret-key=...
#s3-region=eu-north-1 #s3-region=eu-north-1
##
## Configuration for local MinIO
##
s3-host-name="127.0.0.1"
# Note: s3-host-name="localhost" doesn't work. It causes
# libmarias3 to use the wrong variant of the protocol.
s3-bucket=storage-engine
s3-access-key=minioadmin
s3-secret-key=minioadmin
s3-port=9000
s3-use-http=ON

15
storage/maria/ha_s3.cc
View File

@@ -80,6 +80,8 @@ static ulong s3_pagecache_file_hash_size;
static ulonglong s3_pagecache_buffer_size; static ulonglong s3_pagecache_buffer_size;
static char *s3_bucket, *s3_access_key=0, *s3_secret_key=0, *s3_region; static char *s3_bucket, *s3_access_key=0, *s3_secret_key=0, *s3_region;
static char *s3_host_name; static char *s3_host_name;
static int s3_port;
static my_bool s3_use_http;
static char *s3_tmp_access_key=0, *s3_tmp_secret_key=0; static char *s3_tmp_access_key=0, *s3_tmp_secret_key=0;
static my_bool s3_debug= 0, s3_slave_ignore_updates= 0; static my_bool s3_debug= 0, s3_slave_ignore_updates= 0;
static my_bool s3_replicate_alter_as_create_select= 0; static my_bool s3_replicate_alter_as_create_select= 0;
@@ -181,6 +183,15 @@ static MYSQL_SYSVAR_STR(host_name, s3_host_name,
PLUGIN_VAR_RQCMDARG | PLUGIN_VAR_READONLY, PLUGIN_VAR_RQCMDARG | PLUGIN_VAR_READONLY,
"AWS host name", "AWS host name",
0, 0, DEFAULT_AWS_HOST_NAME); 0, 0, DEFAULT_AWS_HOST_NAME);
static MYSQL_SYSVAR_INT(port, s3_port,
PLUGIN_VAR_RQCMDARG | PLUGIN_VAR_READONLY,
"Port number to connect to (0 means use default)",
NULL /*check*/, NULL /*update*/, 0 /*default*/,
0 /*min*/, 65535 /*max*/, 1 /*blk*/);
static MYSQL_SYSVAR_BOOL(use_http, s3_use_http,
PLUGIN_VAR_RQCMDARG | PLUGIN_VAR_READONLY,
"If true, force use of HTTP protocol",
NULL /*check*/, NULL /*update*/, 0 /*default*/);
static MYSQL_SYSVAR_STR(access_key, s3_tmp_access_key, static MYSQL_SYSVAR_STR(access_key, s3_tmp_access_key,
PLUGIN_VAR_RQCMDARG | PLUGIN_VAR_READONLY | PLUGIN_VAR_MEMALLOC, PLUGIN_VAR_RQCMDARG | PLUGIN_VAR_READONLY | PLUGIN_VAR_MEMALLOC,
"AWS access key", "AWS access key",
@@ -276,6 +287,8 @@ static my_bool s3_info_init(S3_INFO *info)
return 1; return 1;
info->protocol_version= (uint8_t) s3_protocol_version; info->protocol_version= (uint8_t) s3_protocol_version;
lex_string_set(&info->host_name, s3_host_name); lex_string_set(&info->host_name, s3_host_name);
info->port= s3_port;
info->use_http= s3_use_http;
lex_string_set(&info->access_key, s3_access_key); lex_string_set(&info->access_key, s3_access_key);
lex_string_set(&info->secret_key, s3_secret_key); lex_string_set(&info->secret_key, s3_secret_key);
lex_string_set(&info->region, s3_region); lex_string_set(&info->region, s3_region);
@@ -1050,6 +1063,8 @@ static struct st_mysql_sys_var* system_variables[]= {
MYSQL_SYSVAR(pagecache_division_limit), MYSQL_SYSVAR(pagecache_division_limit),
MYSQL_SYSVAR(pagecache_file_hash_size), MYSQL_SYSVAR(pagecache_file_hash_size),
MYSQL_SYSVAR(host_name), MYSQL_SYSVAR(host_name),
MYSQL_SYSVAR(port),
MYSQL_SYSVAR(use_http),
MYSQL_SYSVAR(bucket), MYSQL_SYSVAR(bucket),
MYSQL_SYSVAR(access_key), MYSQL_SYSVAR(access_key),
MYSQL_SYSVAR(secret_key), MYSQL_SYSVAR(secret_key),

6
storage/maria/s3_func.c
View File

@@ -157,6 +157,12 @@ ms3_st *s3_open_connection(S3_INFO *s3)
if (s3->protocol_version) if (s3->protocol_version)
ms3_set_option(s3_client, MS3_OPT_FORCE_PROTOCOL_VERSION, ms3_set_option(s3_client, MS3_OPT_FORCE_PROTOCOL_VERSION,
&s3->protocol_version); &s3->protocol_version);
if (s3->port)
ms3_set_option(s3_client, MS3_OPT_PORT_NUMBER, &s3->port);
if (s3->use_http)
ms3_set_option(s3_client, MS3_OPT_USE_HTTP, NULL);
return s3_client; return s3_client;
} }

2
storage/maria/s3_func.h
View File

@@ -45,6 +45,8 @@ typedef struct s3_info
{ {
/* Connection strings */ /* Connection strings */
LEX_CSTRING access_key, secret_key, region, bucket, host_name; LEX_CSTRING access_key, secret_key, region, bucket, host_name;
int port; // 0 means 'Use default'
my_bool use_http;
/* Will be set by caller or by ma_open() */ /* Will be set by caller or by ma_open() */
LEX_CSTRING database, table; LEX_CSTRING database, table;