diff --git a/mysql-test/suite/s3/basic.result b/mysql-test/suite/s3/basic.result index 790806ee43c..47c35d8cfbe 100644 --- a/mysql-test/suite/s3/basic.result +++ b/mysql-test/suite/s3/basic.result @@ -130,16 +130,19 @@ s3_block_size X s3_bucket X s3_debug X s3_host_name X +s3_no_content_type X s3_pagecache_age_threshold X s3_pagecache_buffer_size X s3_pagecache_division_limit X s3_pagecache_file_hash_size X s3_port X s3_protocol_version X +s3_provider X s3_region X s3_replicate_alter_as_create_select X s3_secret_key X s3_slave_ignore_updates X +s3_ssl_no_verify X s3_use_http X show variables like "s3_slave%"; Variable_name Value diff --git a/mysql-test/suite/s3/my.cnf b/mysql-test/suite/s3/my.cnf index f851aa18d6d..4f4049cf89b 100644 --- a/mysql-test/suite/s3/my.cnf +++ b/mysql-test/suite/s3/my.cnf @@ -12,6 +12,8 @@ s3-secret-key=@ENV.S3_SECRET_KEY s3-region=@ENV.S3_REGION s3-port=@ENV.S3_PORT s3-use-http=@ENV.S3_USE_HTTP +s3-ssl-no-verify=@ENV.S3_SSL_NO_VERIFY +s3-provider=@ENV.S3_PROVIDER #s3-host-name=s3.amazonaws.com #s3-protocol-version=Amazon diff --git a/mysql-test/suite/s3/slave.cnf b/mysql-test/suite/s3/slave.cnf index 4f4d3d39ac7..729a48c49eb 100644 --- a/mysql-test/suite/s3/slave.cnf +++ b/mysql-test/suite/s3/slave.cnf @@ -10,6 +10,8 @@ s3-secret-key=@ENV.S3_SECRET_KEY s3-region=@ENV.S3_REGION s3-port=@ENV.S3_PORT s3-use-http=@ENV.S3_USE_HTTP +s3-ssl-no-verify=@ENV.S3_SSL_NO_VERIFY +s3-provider=@ENV.S3_PROVIDER # You can change the following when running the tests against # your own S3 setup diff --git a/mysql-test/suite/s3/suite.pm b/mysql-test/suite/s3/suite.pm index cdefbc5e323..383bf4f7798 100644 --- a/mysql-test/suite/s3/suite.pm +++ b/mysql-test/suite/s3/suite.pm @@ -20,6 +20,8 @@ if(connect(SOCK, $paddr)) $ENV{'S3_REGION'} = ""; $ENV{'S3_PROTOCOL_VERSION'} = "Auto"; $ENV{'S3_USE_HTTP'} = "ON"; + $ENV{'S3_SSL_NO_VERIFY'} = "OFF"; + $ENV{'S3_PROVIDER'} = "Default"; } else { @@ -62,6 +64,17 @@ else { $ENV{'S3_USE_HTTP'} = "OFF"; } + + if (!$ENV{'S3_SSL_NO_VERIFY'}) + { + $ENV{'S3_SSL_NO_VERIFY'} = "OFF"; + } + + if (!$ENV{'S3_PROVIDER'}) + { + $ENV{'S3_PROVIDER'} = "Default"; + } + } bless { }; diff --git a/storage/maria/aria_s3_copy.cc b/storage/maria/aria_s3_copy.cc index b8a0f5b7921..1b0063c21f8 100644 --- a/storage/maria/aria_s3_copy.cc +++ b/storage/maria/aria_s3_copy.cc @@ -41,7 +41,10 @@ static const char *opt_database; static const char *opt_s3_bucket="MariaDB"; static my_bool opt_compression, opt_verbose, opt_force, opt_s3_debug; static my_bool opt_s3_use_http; +static my_bool opt_s3_ssl_no_verify; +static my_bool opt_s3_no_content_type; static ulong opt_operation= OP_IMPOSSIBLE, opt_protocol_version= 1; +static ulong opt_provider= 0; static ulong opt_block_size; static ulong opt_s3_port; static char **default_argv=0; @@ -73,6 +76,13 @@ static struct my_option my_long_options[] = {"s3_use_http", 'P', "If true, force use of HTTP protocol", (char**) &opt_s3_use_http, (char**) &opt_s3_use_http, 0, GET_BOOL, NO_ARG, 0, 0, 0, 0, 0, 0}, + {"s3_ssl_no_verify", 's', "If true, verification of the S3 endpoint SSL " + "certificate is disabled", + (char**) &opt_s3_ssl_no_verify, (char**) &opt_s3_ssl_no_verify, + 0, GET_BOOL, NO_ARG, 0, 0, 0, 0, 0, 0}, + {"s3_no_content_type", 'n', "If true, disables the Content-Type header", + (char**) &opt_s3_no_content_type, (char**) &opt_s3_no_content_type, + 0, GET_BOOL, NO_ARG, 0, 0, 0, 0, 0, 0}, {"compress", 'c', "Use compression", &opt_compression, &opt_compression, 0, GET_BOOL, NO_ARG, 0, 0, 0, 0, 0, 0}, {"op", 'o', "Operation to execute. One of 'from_s3', 'to_s3' or " @@ -92,6 +102,10 @@ static struct my_option my_long_options[] = "Note: \"Legacy\", \"Original\" and \"Amazon\" are deprecated.", &opt_protocol_version, &opt_protocol_version, &s3_protocol_typelib, GET_ENUM, REQUIRED_ARG, 0, 0, 0, 0, 0, 0}, + {"s3_provider", 'R', "Enable S3 provider specific compatibility tweaks " + "\"Default\", \"Amazon\", or \"Huawei\".", + &opt_provider, &opt_provider, &s3_provider_typelib, + GET_ENUM, REQUIRED_ARG, 0, 0, 0, 0, 0, 0}, {"force", 'f', "Force copy even if target exists", &opt_force, &opt_force, 0, GET_BOOL, NO_ARG, 0, 0, 0, 0, 0, 0}, {"verbose", 'v', "Write more information", &opt_verbose, &opt_verbose, @@ -220,6 +234,19 @@ int main(int argc, char** argv) ms3_set_option(global_s3_client, MS3_OPT_BUFFER_CHUNK_SIZE, &block_size); + /* Provider specific overrides */ + switch (opt_provider) + { + case 0: /* Default */ + break; + case 1: /* Amazon */ + opt_protocol_version = 5; + break; + case 2: /* Huawei */ + opt_s3_no_content_type = 1; + break; + } + if (opt_protocol_version > 2) { uint8_t protocol_version; @@ -245,6 +272,11 @@ int main(int argc, char** argv) if (opt_s3_use_http) ms3_set_option(global_s3_client, MS3_OPT_USE_HTTP, NULL); + if (opt_s3_ssl_no_verify) + ms3_set_option(global_s3_client, MS3_OPT_DISABLE_SSL_VERIFY, NULL); + + if (opt_s3_no_content_type) + ms3_set_option(global_s3_client, MS3_OPT_NO_CONTENT_TYPE, NULL); for (; *argv ; argv++) { diff --git a/storage/maria/ha_s3.cc b/storage/maria/ha_s3.cc index 276703cfaea..e911bdbe242 100644 --- a/storage/maria/ha_s3.cc +++ b/storage/maria/ha_s3.cc @@ -78,7 +78,7 @@ #define DEFAULT_AWS_HOST_NAME "s3.amazonaws.com" static PAGECACHE s3_pagecache; -static ulong s3_block_size, s3_protocol_version; +static ulong s3_block_size, s3_protocol_version, s3_provider; static ulong s3_pagecache_division_limit, s3_pagecache_age_threshold; static ulong s3_pagecache_file_hash_size; static ulonglong s3_pagecache_buffer_size; @@ -86,6 +86,8 @@ static char *s3_bucket, *s3_access_key=0, *s3_secret_key=0, *s3_region; static char *s3_host_name; static int s3_port; static my_bool s3_use_http; +static my_bool s3_ssl_no_verify; +static my_bool s3_no_content_type; static char *s3_tmp_access_key=0, *s3_tmp_secret_key=0; static my_bool s3_debug= 0, s3_slave_ignore_updates= 0; static my_bool s3_replicate_alter_as_create_select= 0; @@ -222,6 +224,10 @@ static MYSQL_SYSVAR_BOOL(use_http, s3_use_http, PLUGIN_VAR_RQCMDARG | PLUGIN_VAR_READONLY, "If true, force use of HTTP protocol", NULL /*check*/, NULL /*update*/, 0 /*default*/); +static MYSQL_SYSVAR_BOOL(ssl_no_verify, s3_ssl_no_verify, + PLUGIN_VAR_RQCMDARG | PLUGIN_VAR_READONLY, + "If true, SSL certificate verifiction for the S3 endpoint is disabled", + NULL, NULL, 0); static MYSQL_SYSVAR_STR(access_key, s3_tmp_access_key, PLUGIN_VAR_RQCMDARG | PLUGIN_VAR_READONLY | PLUGIN_VAR_MEMALLOC, "AWS access key", @@ -234,6 +240,15 @@ static MYSQL_SYSVAR_STR(region, s3_region, PLUGIN_VAR_RQCMDARG | PLUGIN_VAR_READONLY, "AWS region", 0, 0, ""); +static MYSQL_SYSVAR_BOOL(no_content_type, s3_no_content_type, + PLUGIN_VAR_RQCMDARG | PLUGIN_VAR_READONLY, + "If true, disables the Content-Type header, required for some providers", + NULL, NULL, 0); +static MYSQL_SYSVAR_ENUM(provider, s3_provider, + PLUGIN_VAR_RQCMDARG, + "Enable S3 provider specific compatibility tweaks " + "\"Default\", \"Amazon\", or \"Huawei\". ", + NULL, NULL, 0, &s3_provider_typelib); ha_create_table_option s3_table_option_list[]= { @@ -319,6 +334,9 @@ static my_bool s3_info_init(S3_INFO *info) lex_string_set(&info->host_name, s3_host_name); info->port= s3_port; info->use_http= s3_use_http; + info->ssl_no_verify= s3_ssl_no_verify; + info->no_content_type = s3_no_content_type; + info->provider= s3_provider; lex_string_set(&info->access_key, s3_access_key); lex_string_set(&info->secret_key, s3_secret_key); lex_string_set(&info->region, s3_region); @@ -1120,12 +1138,15 @@ static struct st_mysql_sys_var* system_variables[]= { MYSQL_SYSVAR(host_name), MYSQL_SYSVAR(port), MYSQL_SYSVAR(use_http), + MYSQL_SYSVAR(ssl_no_verify), MYSQL_SYSVAR(bucket), MYSQL_SYSVAR(access_key), MYSQL_SYSVAR(secret_key), MYSQL_SYSVAR(region), MYSQL_SYSVAR(slave_ignore_updates), MYSQL_SYSVAR(replicate_alter_as_create_select), + MYSQL_SYSVAR(no_content_type), + MYSQL_SYSVAR(provider), NULL }; diff --git a/storage/maria/libmarias3 b/storage/maria/libmarias3 index a81724ab07b..5e6aa32f96e 160000 --- a/storage/maria/libmarias3 +++ b/storage/maria/libmarias3 @@ -1 +1 @@ -Subproject commit a81724ab07bd28e16bf431419c24b6362d5894fc +Subproject commit 5e6aa32f96ebdbcaf32b90d6182685156f8198b5 diff --git a/storage/maria/s3_func.c b/storage/maria/s3_func.c index d85dc8a0b40..b8918c48d4b 100644 --- a/storage/maria/s3_func.c +++ b/storage/maria/s3_func.c @@ -43,6 +43,9 @@ static const char *protocol_types[]= {"Auto", "Original", "Amazon", "Legacy", "P TYPELIB s3_protocol_typelib= {array_elements(protocol_types)-1,"", protocol_types, NULL}; +static const char *providers[]= {"Default", "Amazon", "Huawei", NullS}; +TYPELIB s3_provider_typelib = {array_elements(providers)-1,"",providers, NULL}; + /****************************************************************************** Allocations handler for libmarias3 To be removed when we do the init allocation in mysqld.cc @@ -154,6 +157,20 @@ ms3_st *s3_open_connection(S3_INFO *s3) errno, ms3_error(errno)); my_errno= HA_ERR_NO_SUCH_TABLE; } + + /* Provider specific overrides */ + switch (s3->provider) + { + case 0: /* Default */ + break; + case 1: /* Amazon */ + s3->protocol_version = 5; + break; + case 2: /* Huawei */ + s3->no_content_type = 1; + break; + } + if (s3->protocol_version > 2) { uint8_t protocol_version; @@ -177,6 +194,12 @@ ms3_st *s3_open_connection(S3_INFO *s3) if (s3->use_http) ms3_set_option(s3_client, MS3_OPT_USE_HTTP, NULL); + if (s3->ssl_no_verify) + ms3_set_option(s3_client, MS3_OPT_DISABLE_SSL_VERIFY, NULL); + + if (s3->no_content_type) + ms3_set_option(s3_client, MS3_OPT_NO_CONTENT_TYPE, NULL); + return s3_client; } diff --git a/storage/maria/s3_func.h b/storage/maria/s3_func.h index f73a95dea24..9c0831b158d 100644 --- a/storage/maria/s3_func.h +++ b/storage/maria/s3_func.h @@ -38,6 +38,7 @@ extern struct s3_func { } s3f; extern TYPELIB s3_protocol_typelib; +extern TYPELIB s3_provider_typelib; /* Store information about a s3 connection */ @@ -47,6 +48,8 @@ struct s3_info LEX_CSTRING access_key, secret_key, region, bucket, host_name; int port; // 0 means 'Use default' my_bool use_http; + my_bool ssl_no_verify; + my_bool no_content_type; /* Will be set by caller or by ma_open() */ LEX_CSTRING database, table; @@ -63,6 +66,8 @@ struct s3_info /* Protocol for the list bucket API call. 1 for Amazon, 2 for some others */ uint8_t protocol_version; + + uint8_t provider; };