database/mariadb
1
0
Fork 0
mirror of https://github.com/MariaDB/server.git synced 2025-08-07 00:04:31 +03:00
Code Activity

Added options --s3-protocol-version and --s3-host-name

Browse Source
This commit is contained in:
Monty
2019-06-01 13:36:52 +03:00
parent 3e83034ee6
commit c51f615bf5
8 changed files with 109 additions and 14 deletions
Download Patch File Download Diff File Expand all files Collapse all files

8
mysql-test/suite/s3/amazon.result Normal file
View File

@@ -0,0 +1,8 @@
set @save_s3_protocol_version=@@global.s3_protocol_version;
set @@global.s3_protocol_version="Original";
create table t1 (pk int primary key, a int);
insert into t1 values (1,1),(2,2),(3,3),(4,4);
alter table t1 engine=S3;
ERROR HY000: Got error from put_object(database/t1/frm): 8 Access Denied
drop table t1;
set @@global.s3_protocol_version=@save_s3_protocol_version;

28
mysql-test/suite/s3/amazon.test Normal file
View File

@@ -0,0 +1,28 @@
--source include/have_s3.inc
if (`SELECT @@s3_host_name <> "s3.amazonaws.com"`)
{
skip Not connected to AWS;
}
--source create_database.inc
#
# Check options against amazon
#
set @save_s3_protocol_version=@@global.s3_protocol_version;
set @@global.s3_protocol_version="Original";
create table t1 (pk int primary key, a int);
insert into t1 values (1,1),(2,2),(3,3),(4,4);
--replace_result $database database
--error 3
alter table t1 engine=S3;
drop table t1;
#
# clean up
#
set @@global.s3_protocol_version=@save_s3_protocol_version;
--source drop_database.inc

2
mysql-test/suite/s3/basic.result
View File

@@ -88,10 +88,12 @@ Variable_name Value
s3_access_key X
s3_block_size X
s3_bucket X
s3_host_name X
s3_pagecache_age_threshold X
s3_pagecache_buffer_size X
s3_pagecache_division_limit X
s3_pagecache_file_hash_size X
s3_protocol_version X
s3_region X
s3_secret_key X
show status like "s3%";

6
mysql-test/suite/s3/my.cnf
View File

@@ -3,3 +3,9 @@
[mysqld.1]
s3=ON
#s3-host-name=s3.amazonaws.com
#s3-protocol-version=Amazon
#s3-bucket=MariaDB
#s3-access-key=...
#s3-secret-key=...
#s3-region=eu-north-1

30
storage/maria/aria_s3_copy.cc
View File

@@ -27,19 +27,23 @@
#include <libmarias3/marias3.h>
#include "s3_func.h"
static const char *op_types[]= {"to_s3", "from_s3", "delete_from_s3", NullS};
static TYPELIB op_typelib= {array_elements(op_types)-1,"", op_types, NULL};
#define OP_IMPOSSIBLE array_elements(op_types)
static const char *load_default_groups[]= { "aria_s3_copy", 0 };
static const char *opt_s3_access_key, *opt_s3_secret_key;
static const char *opt_s3_region="eu-north-1";
static const char *opt_s3_host_name= DEFAULT_AWS_HOST_NAME;
static const char *opt_database;
static const char *opt_s3_bucket="MariaDB";
static my_bool opt_compression, opt_verbose, opt_force, opt_s3_debug;
static int opt_operation= -1;
static ulong opt_operation= OP_IMPOSSIBLE, opt_protocol_version= 1;
static ulong opt_block_size;
static char **default_argv=0;
static const char *op_types[]= {"to_s3", "from_s3", "delete_from_s3", NullS};
static TYPELIB op_typelib= {array_elements(op_types)-1,"", op_types, NULL};
static ms3_st *global_s3_client= 0;
static struct my_option my_long_options[] =
{
{"help", '?', "Display this help and exit.", 0, 0, 0, GET_NO_ARG, NO_ARG, 0,
@@ -56,12 +60,15 @@ static struct my_option my_long_options[] =
{"s3_bucket", 'b', "AWS prefix for tables",
(char**) &opt_s3_bucket, (char**) &opt_s3_bucket, 0,
GET_STR, REQUIRED_ARG, 0, 0, 0, 0, 0, 0},
{"s3_host_name", 'h', "Host name to S3 provider",
(char**) &opt_s3_host_name, (char**) &opt_s3_host_name, 0,
GET_STR, REQUIRED_ARG, 0, 0, 0, 0, 0, 0},
{"compress", 'c', "Use compression", &opt_compression, &opt_compression,
0, GET_BOOL, NO_ARG, 0, 0, 0, 0, 0, 0},
{"op", 'o', "Operation to excecute. One of 'from_s3', 'to_s3' or "
"'delete_from_s3'",
&opt_operation, &opt_operation, &op_typelib,
GET_ENUM, REQUIRED_ARG, -1, 0, 0, 0, 0, 0},
GET_ENUM, REQUIRED_ARG, OP_IMPOSSIBLE, 0, 0, 0, 0, 0},
{"database", 'd',
"Database for copied table (second prefix). "
"If not given, the directory of the table file is used",
@@ -69,6 +76,10 @@ static struct my_option my_long_options[] =
{"s3_block_size", 'B', "Block size for data/index blocks in s3",
&opt_block_size, &opt_block_size, 0, GET_ULONG, REQUIRED_ARG,
4*1024*1024, 64*1024, 16*1024*1024, MALLOC_OVERHEAD, 1024, 0 },
{"s3_protocol_version", 'L',
"Protocol used to communication with S3. One of \"Amazon\" or \"Original\".",
&opt_protocol_version, &opt_protocol_version, &s3_protocol_typelib,
GET_ENUM, REQUIRED_ARG, 1, 0, 0, 0, 0, 0},
{"force", 'f', "Force copy even if target exists",
&opt_force, &opt_force, 0, GET_BOOL, NO_ARG, 0, 0, 0, 0, 0, 0},
{"verbose", 'v', "Write more information", &opt_verbose, &opt_verbose,
@@ -101,7 +112,7 @@ static void usage(void)
" see the PUBLIC for details.\n");
puts("Copy an Aria table to and from s3");
printf("Usage: %s --aws-access-key=# --aws-secret-access-key=# --aws-region # "
"--op=(from|to) [OPTIONS] tables[.MAI]\n",
"--op=(from_s3 | to_s3 | delete_from_s3) [OPTIONS] tables[.MAI]\n",
my_progname_short);
print_defaults("my", load_default_groups);
puts("");
@@ -170,9 +181,9 @@ static void get_options(register int *argc,register char ***argv)
fprintf(stderr, "--aws-secret-access-key was not given\n");
my_exit(-1);
}
if ((int) opt_operation == -1)
if (opt_operation == OP_IMPOSSIBLE)
{
fprintf(stderr, "You must specify an operation with --op=[from|to]\n");
fprintf(stderr, "You must specify an operation with --op=[from_s3|to_s3|delete_from_s3]\n");
my_exit(-1);
}
if (opt_s3_debug)
@@ -189,7 +200,7 @@ int main(int argc, char** argv)
s3_init_library();
if (!(global_s3_client= ms3_init(opt_s3_access_key,
opt_s3_secret_key,
opt_s3_region, NULL)))
opt_s3_region, opt_s3_host_name)))
{
fprintf(stderr, "Can't open connection to S3, error: %d %s", errno,
ms3_error(errno));
@@ -198,7 +209,10 @@ int main(int argc, char** argv)
{
size_t block_size= opt_block_size;
uint8_t protocol_version= (uint8_t) opt_protocol_version+1;
ms3_set_option(global_s3_client, MS3_OPT_BUFFER_CHUNK_SIZE, &block_size);
ms3_set_option(global_s3_client, MS3_OPT_FORCE_PROTOCOL_VERSION,
&protocol_version);
}
for (; *argv ; argv++)

20
storage/maria/ha_s3.cc
View File

@@ -68,16 +68,18 @@
#include "s3_func.h"
#include "aria_backup.h"
#define DEFAULT_AWS_HOST_NAME "s3.amazonaws.com"
static PAGECACHE s3_pagecache;
static ulong s3_block_size;
static ulong s3_block_size, s3_protocol_version;
static ulong s3_pagecache_division_limit, s3_pagecache_age_threshold;
static ulong s3_pagecache_file_hash_size;
static ulonglong s3_pagecache_buffer_size;
static char *s3_bucket, *s3_access_key=0, *s3_secret_key=0, *s3_region;
static char *s3_host_name;
static char *s3_tmp_access_key=0, *s3_tmp_secret_key=0;
handlerton *s3_hton= 0;
/* Don't show access or secret keys to users if they exists */
static void update_access_key(MYSQL_THD thd,
@@ -115,6 +117,12 @@ static MYSQL_SYSVAR_ULONG(block_size, s3_block_size,
"Block size for S3", 0, 0,
4*1024*1024, 65536, 16*1024*1024, 8192);
static MYSQL_SYSVAR_ENUM(protocol_version, s3_protocol_version,
PLUGIN_VAR_RQCMDARG,
"Protocol used to communication with S3. One of "
"\"Amazon\" or \"Original\".",
NULL, NULL, 1, &s3_protocol_typelib);
static MYSQL_SYSVAR_ULONG(pagecache_age_threshold,
s3_pagecache_age_threshold, PLUGIN_VAR_RQCMDARG,
"This characterizes the number of hits a hot block has to be untouched "
@@ -148,6 +156,10 @@ static MYSQL_SYSVAR_STR(bucket, s3_bucket,
PLUGIN_VAR_RQCMDARG | PLUGIN_VAR_READONLY,
"AWS bucket",
0, 0, "MariaDB");
static MYSQL_SYSVAR_STR(host_name, s3_host_name,
PLUGIN_VAR_RQCMDARG | PLUGIN_VAR_READONLY,
"AWS bucket",
0, 0, DEFAULT_AWS_HOST_NAME);
static MYSQL_SYSVAR_STR(access_key, s3_tmp_access_key,
PLUGIN_VAR_RQCMDARG | PLUGIN_VAR_READONLY | PLUGIN_VAR_MEMALLOC,
"AWS access key",
@@ -241,6 +253,8 @@ static my_bool s3_info_init(S3_INFO *info)
{
if (!s3_usable())
return 1;
info->protocol_version= (uint8_t) s3_protocol_version+1;
lex_string_set(&info->host_name, s3_host_name);
lex_string_set(&info->access_key, s3_access_key);
lex_string_set(&info->secret_key, s3_secret_key);
lex_string_set(&info->region, s3_region);
@@ -702,10 +716,12 @@ static SHOW_VAR status_variables[]= {
static struct st_mysql_sys_var* system_variables[]= {
MYSQL_SYSVAR(block_size),
MYSQL_SYSVAR(protocol_version),
MYSQL_SYSVAR(pagecache_age_threshold),
MYSQL_SYSVAR(pagecache_buffer_size),
MYSQL_SYSVAR(pagecache_division_limit),
MYSQL_SYSVAR(pagecache_file_hash_size),
MYSQL_SYSVAR(host_name),
MYSQL_SYSVAR(bucket),
MYSQL_SYSVAR(access_key),
MYSQL_SYSVAR(secret_key),

20
storage/maria/s3_func.c
View File

@@ -37,6 +37,11 @@ static void convert_frm_to_disk_format(uchar *header);
static int s3_read_frm_from_disk(const char *filename, uchar **to,
size_t *to_size);
/* Used by ha_s3.cc and tools to define different protocol options */
static const char *protocol_types[]= {"Original", "Amazon", NullS};
TYPELIB s3_protocol_typelib= {array_elements(protocol_types)-1,"",
protocol_types, NULL};
/******************************************************************************
Allocations handler for libmarias3
@@ -137,13 +142,15 @@ ms3_st *s3_open_connection(S3_INFO *s3)
if (!(s3_client= ms3_init(s3->access_key.str,
s3->secret_key.str,
s3->region.str,
NULL)))
s3->host_name.str)))
{
my_printf_error(HA_ERR_NO_SUCH_TABLE,
"Can't open connection to S3, error: %d %s", MYF(0),
errno, ms3_error(errno));
my_errno= HA_ERR_NO_SUCH_TABLE;
}
ms3_set_option(s3_client, MS3_OPT_FORCE_PROTOCOL_VERSION,
&s3->protocol_version);
return s3_client;
}
@@ -282,11 +289,12 @@ int aria_copy_to_s3(ms3_st *s3_client, const char *aws_bucket,
char filename[FN_REFLEN];
char *aws_path_end, *end;
uchar *alloc_block= 0, *block;
ms3_status_st status;
File file= -1;
my_off_t file_size;
size_t frm_length;
int error;
ms3_status_st status;
my_bool frm_created= 0;
DBUG_ENTER("aria_copy_to_s3");
aws_path_end= strxmov(aws_path, database, "/", table_name, NullS);
@@ -324,6 +332,7 @@ int aria_copy_to_s3(ms3_st *s3_client, const char *aws_bucket,
0))
goto err;
frm_created= 1;
my_free(alloc_block);
alloc_block= 0;
}
@@ -347,7 +356,7 @@ int aria_copy_to_s3(ms3_st *s3_client, const char *aws_bucket,
{
fprintf(stderr,
"Aria table %s doesn't match criteria to be copied to S3.\n"
"It should be non-transactional and should have row_format page",
"It should be non-transactional and should have row_format page\n",
path);
goto err;
}
@@ -432,6 +441,11 @@ int aria_copy_to_s3(ms3_st *s3_client, const char *aws_bucket,
DBUG_RETURN(0);
err:
if (frm_created)
{
end= strmov(aws_path_end,"/frm");
(void) s3_delete_object(s3_client, aws_bucket, aws_path, 0);
}
if (file >= 0)
my_close(file, MYF(0));
my_free(alloc_block);

9
storage/maria/s3_func.h
View File

@@ -23,17 +23,24 @@
C_MODE_START
#include <libmarias3/marias3.h>
#define DEFAULT_AWS_HOST_NAME "s3.amazonaws.com"
extern TYPELIB s3_protocol_typelib;
/* Store information about a s3 connection */
typedef struct s3_info
{
LEX_CSTRING access_key, secret_key, region, bucket;
LEX_CSTRING access_key, secret_key, region, bucket, host_name;
/* The following will be filled in by maria_open() */
LEX_CSTRING database, table;
/* Sent to open to verify version */
LEX_CUSTRING tabledef_version;
/* Protocol for the list bucket API call. 1 for Amazon, 2 for some others */
uint8_t protocol_version;
} S3_INFO;