From c51f615bf575480b13ee542fdb3b7fe644e21098 Mon Sep 17 00:00:00 2001 From: Monty Date: Sat, 1 Jun 2019 13:36:52 +0300 Subject: [PATCH] Added options --s3-protocol-version and --s3-host-name --- mysql-test/suite/s3/amazon.result | 8 ++++++++ mysql-test/suite/s3/amazon.test | 28 ++++++++++++++++++++++++++++ mysql-test/suite/s3/basic.result | 2 ++ mysql-test/suite/s3/my.cnf | 6 ++++++ storage/maria/aria_s3_copy.cc | 30 ++++++++++++++++++++++-------- storage/maria/ha_s3.cc | 20 ++++++++++++++++++-- storage/maria/s3_func.c | 20 +++++++++++++++++--- storage/maria/s3_func.h | 9 ++++++++- 8 files changed, 109 insertions(+), 14 deletions(-) create mode 100644 mysql-test/suite/s3/amazon.result create mode 100644 mysql-test/suite/s3/amazon.test diff --git a/mysql-test/suite/s3/amazon.result b/mysql-test/suite/s3/amazon.result new file mode 100644 index 00000000000..7a02fbdfadd --- /dev/null +++ b/mysql-test/suite/s3/amazon.result @@ -0,0 +1,8 @@ +set @save_s3_protocol_version=@@global.s3_protocol_version; +set @@global.s3_protocol_version="Original"; +create table t1 (pk int primary key, a int); +insert into t1 values (1,1),(2,2),(3,3),(4,4); +alter table t1 engine=S3; +ERROR HY000: Got error from put_object(database/t1/frm): 8 Access Denied +drop table t1; +set @@global.s3_protocol_version=@save_s3_protocol_version; diff --git a/mysql-test/suite/s3/amazon.test b/mysql-test/suite/s3/amazon.test new file mode 100644 index 00000000000..4cb968831cb --- /dev/null +++ b/mysql-test/suite/s3/amazon.test @@ -0,0 +1,28 @@ +--source include/have_s3.inc + +if (`SELECT @@s3_host_name <> "s3.amazonaws.com"`) +{ + skip Not connected to AWS; +} + +--source create_database.inc + +# +# Check options against amazon +# + +set @save_s3_protocol_version=@@global.s3_protocol_version; +set @@global.s3_protocol_version="Original"; + +create table t1 (pk int primary key, a int); +insert into t1 values (1,1),(2,2),(3,3),(4,4); +--replace_result $database database +--error 3 +alter table t1 engine=S3; +drop table t1; + +# +# clean up +# +set @@global.s3_protocol_version=@save_s3_protocol_version; +--source drop_database.inc diff --git a/mysql-test/suite/s3/basic.result b/mysql-test/suite/s3/basic.result index b491c32d75c..d6e6f86f598 100644 --- a/mysql-test/suite/s3/basic.result +++ b/mysql-test/suite/s3/basic.result @@ -88,10 +88,12 @@ Variable_name Value s3_access_key X s3_block_size X s3_bucket X +s3_host_name X s3_pagecache_age_threshold X s3_pagecache_buffer_size X s3_pagecache_division_limit X s3_pagecache_file_hash_size X +s3_protocol_version X s3_region X s3_secret_key X show status like "s3%"; diff --git a/mysql-test/suite/s3/my.cnf b/mysql-test/suite/s3/my.cnf index d4e748dc488..1958a04343f 100644 --- a/mysql-test/suite/s3/my.cnf +++ b/mysql-test/suite/s3/my.cnf @@ -3,3 +3,9 @@ [mysqld.1] s3=ON +#s3-host-name=s3.amazonaws.com +#s3-protocol-version=Amazon +#s3-bucket=MariaDB +#s3-access-key=... +#s3-secret-key=... +#s3-region=eu-north-1 diff --git a/storage/maria/aria_s3_copy.cc b/storage/maria/aria_s3_copy.cc index 28e8933d118..9e8a83a1602 100644 --- a/storage/maria/aria_s3_copy.cc +++ b/storage/maria/aria_s3_copy.cc @@ -27,19 +27,23 @@ #include #include "s3_func.h" +static const char *op_types[]= {"to_s3", "from_s3", "delete_from_s3", NullS}; +static TYPELIB op_typelib= {array_elements(op_types)-1,"", op_types, NULL}; +#define OP_IMPOSSIBLE array_elements(op_types) + static const char *load_default_groups[]= { "aria_s3_copy", 0 }; static const char *opt_s3_access_key, *opt_s3_secret_key; static const char *opt_s3_region="eu-north-1"; +static const char *opt_s3_host_name= DEFAULT_AWS_HOST_NAME; static const char *opt_database; static const char *opt_s3_bucket="MariaDB"; static my_bool opt_compression, opt_verbose, opt_force, opt_s3_debug; -static int opt_operation= -1; +static ulong opt_operation= OP_IMPOSSIBLE, opt_protocol_version= 1; static ulong opt_block_size; static char **default_argv=0; -static const char *op_types[]= {"to_s3", "from_s3", "delete_from_s3", NullS}; -static TYPELIB op_typelib= {array_elements(op_types)-1,"", op_types, NULL}; static ms3_st *global_s3_client= 0; + static struct my_option my_long_options[] = { {"help", '?', "Display this help and exit.", 0, 0, 0, GET_NO_ARG, NO_ARG, 0, @@ -56,12 +60,15 @@ static struct my_option my_long_options[] = {"s3_bucket", 'b', "AWS prefix for tables", (char**) &opt_s3_bucket, (char**) &opt_s3_bucket, 0, GET_STR, REQUIRED_ARG, 0, 0, 0, 0, 0, 0}, + {"s3_host_name", 'h', "Host name to S3 provider", + (char**) &opt_s3_host_name, (char**) &opt_s3_host_name, 0, + GET_STR, REQUIRED_ARG, 0, 0, 0, 0, 0, 0}, {"compress", 'c', "Use compression", &opt_compression, &opt_compression, 0, GET_BOOL, NO_ARG, 0, 0, 0, 0, 0, 0}, {"op", 'o', "Operation to excecute. One of 'from_s3', 'to_s3' or " "'delete_from_s3'", &opt_operation, &opt_operation, &op_typelib, - GET_ENUM, REQUIRED_ARG, -1, 0, 0, 0, 0, 0}, + GET_ENUM, REQUIRED_ARG, OP_IMPOSSIBLE, 0, 0, 0, 0, 0}, {"database", 'd', "Database for copied table (second prefix). " "If not given, the directory of the table file is used", @@ -69,6 +76,10 @@ static struct my_option my_long_options[] = {"s3_block_size", 'B', "Block size for data/index blocks in s3", &opt_block_size, &opt_block_size, 0, GET_ULONG, REQUIRED_ARG, 4*1024*1024, 64*1024, 16*1024*1024, MALLOC_OVERHEAD, 1024, 0 }, + {"s3_protocol_version", 'L', + "Protocol used to communication with S3. One of \"Amazon\" or \"Original\".", + &opt_protocol_version, &opt_protocol_version, &s3_protocol_typelib, + GET_ENUM, REQUIRED_ARG, 1, 0, 0, 0, 0, 0}, {"force", 'f', "Force copy even if target exists", &opt_force, &opt_force, 0, GET_BOOL, NO_ARG, 0, 0, 0, 0, 0, 0}, {"verbose", 'v', "Write more information", &opt_verbose, &opt_verbose, @@ -101,7 +112,7 @@ static void usage(void) " see the PUBLIC for details.\n"); puts("Copy an Aria table to and from s3"); printf("Usage: %s --aws-access-key=# --aws-secret-access-key=# --aws-region # " - "--op=(from|to) [OPTIONS] tables[.MAI]\n", + "--op=(from_s3 | to_s3 | delete_from_s3) [OPTIONS] tables[.MAI]\n", my_progname_short); print_defaults("my", load_default_groups); puts(""); @@ -170,9 +181,9 @@ static void get_options(register int *argc,register char ***argv) fprintf(stderr, "--aws-secret-access-key was not given\n"); my_exit(-1); } - if ((int) opt_operation == -1) + if (opt_operation == OP_IMPOSSIBLE) { - fprintf(stderr, "You must specify an operation with --op=[from|to]\n"); + fprintf(stderr, "You must specify an operation with --op=[from_s3|to_s3|delete_from_s3]\n"); my_exit(-1); } if (opt_s3_debug) @@ -189,7 +200,7 @@ int main(int argc, char** argv) s3_init_library(); if (!(global_s3_client= ms3_init(opt_s3_access_key, opt_s3_secret_key, - opt_s3_region, NULL))) + opt_s3_region, opt_s3_host_name))) { fprintf(stderr, "Can't open connection to S3, error: %d %s", errno, ms3_error(errno)); @@ -198,7 +209,10 @@ int main(int argc, char** argv) { size_t block_size= opt_block_size; + uint8_t protocol_version= (uint8_t) opt_protocol_version+1; ms3_set_option(global_s3_client, MS3_OPT_BUFFER_CHUNK_SIZE, &block_size); + ms3_set_option(global_s3_client, MS3_OPT_FORCE_PROTOCOL_VERSION, + &protocol_version); } for (; *argv ; argv++) diff --git a/storage/maria/ha_s3.cc b/storage/maria/ha_s3.cc index 98a2d834d85..2eb627a698c 100644 --- a/storage/maria/ha_s3.cc +++ b/storage/maria/ha_s3.cc @@ -68,16 +68,18 @@ #include "s3_func.h" #include "aria_backup.h" +#define DEFAULT_AWS_HOST_NAME "s3.amazonaws.com" + static PAGECACHE s3_pagecache; -static ulong s3_block_size; +static ulong s3_block_size, s3_protocol_version; static ulong s3_pagecache_division_limit, s3_pagecache_age_threshold; static ulong s3_pagecache_file_hash_size; static ulonglong s3_pagecache_buffer_size; static char *s3_bucket, *s3_access_key=0, *s3_secret_key=0, *s3_region; +static char *s3_host_name; static char *s3_tmp_access_key=0, *s3_tmp_secret_key=0; handlerton *s3_hton= 0; - /* Don't show access or secret keys to users if they exists */ static void update_access_key(MYSQL_THD thd, @@ -115,6 +117,12 @@ static MYSQL_SYSVAR_ULONG(block_size, s3_block_size, "Block size for S3", 0, 0, 4*1024*1024, 65536, 16*1024*1024, 8192); +static MYSQL_SYSVAR_ENUM(protocol_version, s3_protocol_version, + PLUGIN_VAR_RQCMDARG, + "Protocol used to communication with S3. One of " + "\"Amazon\" or \"Original\".", + NULL, NULL, 1, &s3_protocol_typelib); + static MYSQL_SYSVAR_ULONG(pagecache_age_threshold, s3_pagecache_age_threshold, PLUGIN_VAR_RQCMDARG, "This characterizes the number of hits a hot block has to be untouched " @@ -148,6 +156,10 @@ static MYSQL_SYSVAR_STR(bucket, s3_bucket, PLUGIN_VAR_RQCMDARG | PLUGIN_VAR_READONLY, "AWS bucket", 0, 0, "MariaDB"); +static MYSQL_SYSVAR_STR(host_name, s3_host_name, + PLUGIN_VAR_RQCMDARG | PLUGIN_VAR_READONLY, + "AWS bucket", + 0, 0, DEFAULT_AWS_HOST_NAME); static MYSQL_SYSVAR_STR(access_key, s3_tmp_access_key, PLUGIN_VAR_RQCMDARG | PLUGIN_VAR_READONLY | PLUGIN_VAR_MEMALLOC, "AWS access key", @@ -241,6 +253,8 @@ static my_bool s3_info_init(S3_INFO *info) { if (!s3_usable()) return 1; + info->protocol_version= (uint8_t) s3_protocol_version+1; + lex_string_set(&info->host_name, s3_host_name); lex_string_set(&info->access_key, s3_access_key); lex_string_set(&info->secret_key, s3_secret_key); lex_string_set(&info->region, s3_region); @@ -702,10 +716,12 @@ static SHOW_VAR status_variables[]= { static struct st_mysql_sys_var* system_variables[]= { MYSQL_SYSVAR(block_size), + MYSQL_SYSVAR(protocol_version), MYSQL_SYSVAR(pagecache_age_threshold), MYSQL_SYSVAR(pagecache_buffer_size), MYSQL_SYSVAR(pagecache_division_limit), MYSQL_SYSVAR(pagecache_file_hash_size), + MYSQL_SYSVAR(host_name), MYSQL_SYSVAR(bucket), MYSQL_SYSVAR(access_key), MYSQL_SYSVAR(secret_key), diff --git a/storage/maria/s3_func.c b/storage/maria/s3_func.c index 7b8bf8d9e1f..b8031fea16b 100644 --- a/storage/maria/s3_func.c +++ b/storage/maria/s3_func.c @@ -37,6 +37,11 @@ static void convert_frm_to_disk_format(uchar *header); static int s3_read_frm_from_disk(const char *filename, uchar **to, size_t *to_size); +/* Used by ha_s3.cc and tools to define different protocol options */ + +static const char *protocol_types[]= {"Original", "Amazon", NullS}; +TYPELIB s3_protocol_typelib= {array_elements(protocol_types)-1,"", + protocol_types, NULL}; /****************************************************************************** Allocations handler for libmarias3 @@ -137,13 +142,15 @@ ms3_st *s3_open_connection(S3_INFO *s3) if (!(s3_client= ms3_init(s3->access_key.str, s3->secret_key.str, s3->region.str, - NULL))) + s3->host_name.str))) { my_printf_error(HA_ERR_NO_SUCH_TABLE, "Can't open connection to S3, error: %d %s", MYF(0), errno, ms3_error(errno)); my_errno= HA_ERR_NO_SUCH_TABLE; } + ms3_set_option(s3_client, MS3_OPT_FORCE_PROTOCOL_VERSION, + &s3->protocol_version); return s3_client; } @@ -282,11 +289,12 @@ int aria_copy_to_s3(ms3_st *s3_client, const char *aws_bucket, char filename[FN_REFLEN]; char *aws_path_end, *end; uchar *alloc_block= 0, *block; + ms3_status_st status; File file= -1; my_off_t file_size; size_t frm_length; int error; - ms3_status_st status; + my_bool frm_created= 0; DBUG_ENTER("aria_copy_to_s3"); aws_path_end= strxmov(aws_path, database, "/", table_name, NullS); @@ -324,6 +332,7 @@ int aria_copy_to_s3(ms3_st *s3_client, const char *aws_bucket, 0)) goto err; + frm_created= 1; my_free(alloc_block); alloc_block= 0; } @@ -347,7 +356,7 @@ int aria_copy_to_s3(ms3_st *s3_client, const char *aws_bucket, { fprintf(stderr, "Aria table %s doesn't match criteria to be copied to S3.\n" - "It should be non-transactional and should have row_format page", + "It should be non-transactional and should have row_format page\n", path); goto err; } @@ -432,6 +441,11 @@ int aria_copy_to_s3(ms3_st *s3_client, const char *aws_bucket, DBUG_RETURN(0); err: + if (frm_created) + { + end= strmov(aws_path_end,"/frm"); + (void) s3_delete_object(s3_client, aws_bucket, aws_path, 0); + } if (file >= 0) my_close(file, MYF(0)); my_free(alloc_block); diff --git a/storage/maria/s3_func.h b/storage/maria/s3_func.h index a99d9fe423f..d679e0790db 100644 --- a/storage/maria/s3_func.h +++ b/storage/maria/s3_func.h @@ -23,17 +23,24 @@ C_MODE_START #include +#define DEFAULT_AWS_HOST_NAME "s3.amazonaws.com" + +extern TYPELIB s3_protocol_typelib; + /* Store information about a s3 connection */ typedef struct s3_info { - LEX_CSTRING access_key, secret_key, region, bucket; + LEX_CSTRING access_key, secret_key, region, bucket, host_name; /* The following will be filled in by maria_open() */ LEX_CSTRING database, table; /* Sent to open to verify version */ LEX_CUSTRING tabledef_version; + + /* Protocol for the list bucket API call. 1 for Amazon, 2 for some others */ + uint8_t protocol_version; } S3_INFO;