Added options --s3-protocol-version and --s3-host-name

mysql-test/suite/s3/amazon.result

@@ -0,0 +1,8 @@
+set @save_s3_protocol_version=@@global.s3_protocol_version;
+set @@global.s3_protocol_version="Original";
+create table t1 (pk int primary key, a int);
+insert into t1 values (1,1),(2,2),(3,3),(4,4);
+alter table t1 engine=S3;
+ERROR HY000: Got error from put_object(database/t1/frm): 8 Access Denied
+drop table t1;
+set @@global.s3_protocol_version=@save_s3_protocol_version;

mysql-test/suite/s3/amazon.test

@@ -0,0 +1,28 @@
+--source include/have_s3.inc
+
+if (`SELECT @@s3_host_name <> "s3.amazonaws.com"`)
+{
+  skip Not connected to AWS;
+}
+
+--source create_database.inc
+
+#
+# Check options against amazon
+#
+
+set @save_s3_protocol_version=@@global.s3_protocol_version;
+set @@global.s3_protocol_version="Original";
+
+create table t1 (pk int primary key, a int);
+insert into t1 values (1,1),(2,2),(3,3),(4,4);
+--replace_result $database database
+--error 3
+alter table t1 engine=S3;
+drop table t1;
+
+#
+# clean up
+#
+set @@global.s3_protocol_version=@save_s3_protocol_version;
+--source drop_database.inc

mysql-test/suite/s3/basic.result

@@ -88,10 +88,12 @@ Variable_name	Value
 s3_access_key	X
 s3_block_size	X
 s3_bucket	X
+s3_host_name	X
 s3_pagecache_age_threshold	X
 s3_pagecache_buffer_size	X
 s3_pagecache_division_limit	X
 s3_pagecache_file_hash_size	X
+s3_protocol_version	X
 s3_region	X
 s3_secret_key	X
 show status like "s3%";

mysql-test/suite/s3/my.cnf

@@ -3,3 +3,9 @@
 
 [mysqld.1]
 s3=ON
+#s3-host-name=s3.amazonaws.com
+#s3-protocol-version=Amazon
+#s3-bucket=MariaDB
+#s3-access-key=...
+#s3-secret-key=...
+#s3-region=eu-north-1

storage/maria/aria_s3_copy.cc

@@ -27,19 +27,23 @@
 #include <libmarias3/marias3.h>
 #include "s3_func.h"
 
+static const char *op_types[]= {"to_s3", "from_s3", "delete_from_s3", NullS};
+static TYPELIB op_typelib= {array_elements(op_types)-1,"", op_types, NULL};
+#define OP_IMPOSSIBLE array_elements(op_types)
+
 static const char *load_default_groups[]= { "aria_s3_copy", 0 };
 static const char *opt_s3_access_key, *opt_s3_secret_key;
 static const char *opt_s3_region="eu-north-1";
+static const char *opt_s3_host_name= DEFAULT_AWS_HOST_NAME;
 static const char *opt_database;
 static const char *opt_s3_bucket="MariaDB";
 static my_bool opt_compression, opt_verbose, opt_force, opt_s3_debug;
-static int opt_operation= -1;
+static ulong opt_operation= OP_IMPOSSIBLE, opt_protocol_version= 1;
 static ulong opt_block_size;
 static char **default_argv=0;
-static const char *op_types[]= {"to_s3", "from_s3", "delete_from_s3", NullS};
-static TYPELIB op_typelib= {array_elements(op_types)-1,"", op_types, NULL};
 static ms3_st *global_s3_client= 0;
 
+
 static struct my_option my_long_options[] =
 {
   {"help", '?', "Display this help and exit.", 0, 0, 0, GET_NO_ARG, NO_ARG, 0,
@@ -56,12 +60,15 @@ static struct my_option my_long_options[] =
   {"s3_bucket", 'b', "AWS prefix for tables",
    (char**) &opt_s3_bucket, (char**) &opt_s3_bucket, 0,
     GET_STR, REQUIRED_ARG, 0, 0, 0, 0, 0, 0},
+  {"s3_host_name", 'h', "Host name to S3 provider",
+   (char**) &opt_s3_host_name, (char**) &opt_s3_host_name, 0,
+    GET_STR, REQUIRED_ARG, 0, 0, 0, 0, 0, 0},
   {"compress", 'c', "Use compression", &opt_compression, &opt_compression,
    0, GET_BOOL, NO_ARG, 0, 0, 0, 0, 0, 0},
   {"op", 'o', "Operation to excecute. One of 'from_s3', 'to_s3' or "
    "'delete_from_s3'",
    &opt_operation, &opt_operation, &op_typelib,
-   GET_ENUM, REQUIRED_ARG, -1, 0, 0, 0, 0, 0},
+   GET_ENUM, REQUIRED_ARG, OP_IMPOSSIBLE, 0, 0, 0, 0, 0},
   {"database", 'd',
    "Database for copied table (second prefix). "
    "If not given, the directory of the table file is used",
@@ -69,6 +76,10 @@ static struct my_option my_long_options[] =
   {"s3_block_size", 'B', "Block size for data/index blocks in s3",
    &opt_block_size, &opt_block_size, 0, GET_ULONG, REQUIRED_ARG,
    4*1024*1024, 64*1024, 16*1024*1024, MALLOC_OVERHEAD, 1024, 0 },
+  {"s3_protocol_version", 'L',
+   "Protocol used to communication with S3. One of \"Amazon\" or \"Original\".",
+   &opt_protocol_version, &opt_protocol_version, &s3_protocol_typelib,
+   GET_ENUM, REQUIRED_ARG, 1, 0, 0, 0, 0, 0},
   {"force", 'f', "Force copy even if target exists",
    &opt_force, &opt_force, 0, GET_BOOL, NO_ARG, 0, 0, 0, 0, 0, 0},
   {"verbose", 'v', "Write more information", &opt_verbose, &opt_verbose,
@@ -101,7 +112,7 @@ static void usage(void)
        " see the PUBLIC for details.\n");
   puts("Copy an Aria table to and from s3");
   printf("Usage: %s --aws-access-key=# --aws-secret-access-key=# --aws-region # "
-         "--op=(from|to) [OPTIONS] tables[.MAI]\n",
+         "--op=(from_s3 | to_s3 | delete_from_s3) [OPTIONS] tables[.MAI]\n",
          my_progname_short);
   print_defaults("my", load_default_groups);
   puts("");
@@ -170,9 +181,9 @@ static void get_options(register int *argc,register char ***argv)
     fprintf(stderr, "--aws-secret-access-key was not given\n");
     my_exit(-1);
   }
-  if ((int) opt_operation == -1)
+  if (opt_operation == OP_IMPOSSIBLE)
   {
-    fprintf(stderr, "You must specify an operation with --op=[from|to]\n");
+    fprintf(stderr, "You must specify an operation with --op=[from_s3|to_s3|delete_from_s3]\n");
     my_exit(-1);
   }
   if (opt_s3_debug)
@@ -189,7 +200,7 @@ int main(int argc, char** argv)
   s3_init_library();
   if (!(global_s3_client= ms3_init(opt_s3_access_key,
                                    opt_s3_secret_key,
-                                   opt_s3_region, NULL)))
+                                   opt_s3_region, opt_s3_host_name)))
   {
     fprintf(stderr, "Can't open connection to S3, error: %d %s", errno,
             ms3_error(errno));
@@ -198,7 +209,10 @@ int main(int argc, char** argv)
 
   {
     size_t block_size= opt_block_size;
+    uint8_t protocol_version= (uint8_t) opt_protocol_version+1;
     ms3_set_option(global_s3_client, MS3_OPT_BUFFER_CHUNK_SIZE, &block_size);
+    ms3_set_option(global_s3_client, MS3_OPT_FORCE_PROTOCOL_VERSION,
+                   &protocol_version);
   }
 
   for (; *argv ; argv++)

storage/maria/ha_s3.cc

@@ -68,16 +68,18 @@
 #include "s3_func.h"
 #include "aria_backup.h"
 
+#define DEFAULT_AWS_HOST_NAME "s3.amazonaws.com"
+
 static PAGECACHE s3_pagecache;
-static ulong s3_block_size;
+static ulong s3_block_size, s3_protocol_version;
 static ulong s3_pagecache_division_limit, s3_pagecache_age_threshold;
 static ulong s3_pagecache_file_hash_size;
 static ulonglong s3_pagecache_buffer_size;
 static char *s3_bucket, *s3_access_key=0, *s3_secret_key=0, *s3_region;
+static char *s3_host_name;
 static char *s3_tmp_access_key=0, *s3_tmp_secret_key=0;
 handlerton *s3_hton= 0;
 
-
 /* Don't show access or secret keys to users if they exists */
 
 static void update_access_key(MYSQL_THD thd,
@@ -115,6 +117,12 @@ static MYSQL_SYSVAR_ULONG(block_size, s3_block_size,
        "Block size for S3", 0, 0,
        4*1024*1024, 65536, 16*1024*1024, 8192);
 
+static MYSQL_SYSVAR_ENUM(protocol_version, s3_protocol_version,
+                         PLUGIN_VAR_RQCMDARG,
+                         "Protocol used to communication with S3. One of "
+                         "\"Amazon\" or \"Original\".",
+                         NULL, NULL, 1, &s3_protocol_typelib);
+
 static MYSQL_SYSVAR_ULONG(pagecache_age_threshold,
        s3_pagecache_age_threshold, PLUGIN_VAR_RQCMDARG,
        "This characterizes the number of hits a hot block has to be untouched "
@@ -148,6 +156,10 @@ static MYSQL_SYSVAR_STR(bucket, s3_bucket,
        PLUGIN_VAR_RQCMDARG | PLUGIN_VAR_READONLY,
       "AWS bucket",
        0, 0, "MariaDB");
+static MYSQL_SYSVAR_STR(host_name, s3_host_name,
+       PLUGIN_VAR_RQCMDARG | PLUGIN_VAR_READONLY,
+      "AWS bucket",
+       0, 0, DEFAULT_AWS_HOST_NAME);
 static MYSQL_SYSVAR_STR(access_key, s3_tmp_access_key,
        PLUGIN_VAR_RQCMDARG | PLUGIN_VAR_READONLY | PLUGIN_VAR_MEMALLOC,
       "AWS access key",
@@ -241,6 +253,8 @@ static my_bool s3_info_init(S3_INFO *info)
 {
   if (!s3_usable())
     return 1;
+  info->protocol_version= (uint8_t) s3_protocol_version+1;
+  lex_string_set(&info->host_name,  s3_host_name);
   lex_string_set(&info->access_key, s3_access_key);
   lex_string_set(&info->secret_key, s3_secret_key);
   lex_string_set(&info->region,     s3_region);
@@ -702,10 +716,12 @@ static SHOW_VAR status_variables[]= {
 
 static struct st_mysql_sys_var* system_variables[]= {
   MYSQL_SYSVAR(block_size),
+  MYSQL_SYSVAR(protocol_version),
   MYSQL_SYSVAR(pagecache_age_threshold),
   MYSQL_SYSVAR(pagecache_buffer_size),
   MYSQL_SYSVAR(pagecache_division_limit),
   MYSQL_SYSVAR(pagecache_file_hash_size),
+  MYSQL_SYSVAR(host_name),
   MYSQL_SYSVAR(bucket),
   MYSQL_SYSVAR(access_key),
   MYSQL_SYSVAR(secret_key),

storage/maria/s3_func.c

@@ -37,6 +37,11 @@ static void convert_frm_to_disk_format(uchar *header);
 static int s3_read_frm_from_disk(const char *filename, uchar **to,
                                  size_t *to_size);
 
+/* Used by ha_s3.cc and tools to define different protocol options */
+
+static const char *protocol_types[]= {"Original", "Amazon", NullS};
+TYPELIB s3_protocol_typelib= {array_elements(protocol_types)-1,"",
+                              protocol_types, NULL};
 
 /******************************************************************************
  Allocations handler for libmarias3
@@ -137,13 +142,15 @@ ms3_st *s3_open_connection(S3_INFO *s3)
   if (!(s3_client= ms3_init(s3->access_key.str,
                             s3->secret_key.str,
                             s3->region.str,
-                            NULL)))
+                            s3->host_name.str)))
   {
     my_printf_error(HA_ERR_NO_SUCH_TABLE,
                     "Can't open connection to S3, error: %d %s", MYF(0),
                     errno, ms3_error(errno));
     my_errno= HA_ERR_NO_SUCH_TABLE;
   }
+  ms3_set_option(s3_client, MS3_OPT_FORCE_PROTOCOL_VERSION,
+                 &s3->protocol_version);
   return s3_client;
 }
 
@@ -282,11 +289,12 @@ int aria_copy_to_s3(ms3_st *s3_client, const char *aws_bucket,
   char filename[FN_REFLEN];
   char *aws_path_end, *end;
   uchar *alloc_block= 0, *block;
+  ms3_status_st status;
   File file= -1;
   my_off_t file_size;
   size_t frm_length;
   int error;
-  ms3_status_st status;
+  my_bool frm_created= 0;
   DBUG_ENTER("aria_copy_to_s3");
 
   aws_path_end= strxmov(aws_path, database, "/", table_name, NullS);
@@ -324,6 +332,7 @@ int aria_copy_to_s3(ms3_st *s3_client, const char *aws_bucket,
                       0))
       goto err;
 
+    frm_created= 1;
     my_free(alloc_block);
     alloc_block= 0;
   }
@@ -347,7 +356,7 @@ int aria_copy_to_s3(ms3_st *s3_client, const char *aws_bucket,
   {
     fprintf(stderr,
             "Aria table %s doesn't match criteria to be copied to S3.\n"
-            "It should be non-transactional and should have row_format page",
+            "It should be non-transactional and should have row_format page\n",
             path);
     goto err;
   }
@@ -432,6 +441,11 @@ int aria_copy_to_s3(ms3_st *s3_client, const char *aws_bucket,
   DBUG_RETURN(0);
 
 err:
+  if (frm_created)
+  {
+    end= strmov(aws_path_end,"/frm");
+    (void) s3_delete_object(s3_client, aws_bucket, aws_path, 0);
+  }
   if (file >= 0)
     my_close(file, MYF(0));
   my_free(alloc_block);

storage/maria/s3_func.h

@@ -23,17 +23,24 @@
 C_MODE_START
 #include <libmarias3/marias3.h>
 
+#define DEFAULT_AWS_HOST_NAME "s3.amazonaws.com"
+
+extern TYPELIB s3_protocol_typelib;
+
 /* Store information about a s3 connection */
 
 typedef struct s3_info
 {
-  LEX_CSTRING access_key, secret_key, region, bucket;
+  LEX_CSTRING access_key, secret_key, region, bucket, host_name;
 
   /* The following will be filled in by maria_open() */
   LEX_CSTRING database, table;
 
   /* Sent to open to verify version */
   LEX_CUSTRING tabledef_version;
+
+  /* Protocol for the list bucket API call. 1 for Amazon, 2 for some others */
+  uint8_t protocol_version;
 } S3_INFO;