From 2c3f5787896f1278e240095c49539ad7baf77d8b Mon Sep 17 00:00:00 2001 From: Sergei Golubchik Date: Thu, 27 Apr 2017 17:40:44 +0200 Subject: [PATCH 01/28] don't generate wsrep_sst_common in-place rename the source to wsrep_sst_common.sh --- scripts/CMakeLists.txt | 2 -- scripts/{wsrep_sst_common => wsrep_sst_common.sh} | 0 2 files changed, 2 deletions(-) rename scripts/{wsrep_sst_common => wsrep_sst_common.sh} (100%) diff --git a/scripts/CMakeLists.txt b/scripts/CMakeLists.txt index b758e9c0d05..de612c2e312 100644 --- a/scripts/CMakeLists.txt +++ b/scripts/CMakeLists.txt @@ -296,8 +296,6 @@ ELSE() wsrep_sst_xtrabackup wsrep_sst_xtrabackup-v2 ) - INSTALL(FILES ${CMAKE_CURRENT_SOURCE_DIR}/wsrep_sst_common - DESTINATION ${INSTALL_BINDIR} COMPONENT Server) ENDIF() IF (NOT WITHOUT_SERVER) SET(SERVER_SCRIPTS diff --git a/scripts/wsrep_sst_common b/scripts/wsrep_sst_common.sh similarity index 100% rename from scripts/wsrep_sst_common rename to scripts/wsrep_sst_common.sh From 99e1294c1e2ddd0bbd81129f1c0902be31a38f48 Mon Sep 17 00:00:00 2001 From: Sergei Golubchik Date: Mon, 24 Apr 2017 15:39:47 +0200 Subject: [PATCH 02/28] bugfix: federated/replication did not increment bytes_received status variable because mysql->net.thd was reset to NULL in mysql_real_connect() and thd_increment_bytes_received() didn't do anything. Fix: * set mysql->net.thd to current_thd instread. * remove the test for non-null THD from a very often used function thd_increment_bytes_received(). --- include/my_pthread.h | 20 ++++++++++++++++++++ sql-common/client.c | 2 +- sql/mysqld.cc | 2 +- sql/mysqld.h | 14 -------------- sql/sql_class.cc | 14 -------------- 5 files changed, 22 insertions(+), 30 deletions(-) diff --git a/include/my_pthread.h b/include/my_pthread.h index 37576ac3cb4..6b830ca36d2 100644 --- a/include/my_pthread.h +++ b/include/my_pthread.h @@ -367,6 +367,26 @@ int my_pthread_mutex_trylock(pthread_mutex_t *mutex); } while(0) #endif /* !set_timespec_time_nsec */ +#ifdef MYSQL_CLIENT +#define _current_thd() NULL +#elif defined(_WIN32) +#ifdef __cplusplus +extern "C" +#endif +MYSQL_THD _current_thd_noinline(); +#define _current_thd() _current_thd_noinline() +#else +/* + THR_THD is a key which will be used to set/get THD* for a thread, + using my_pthread_setspecific_ptr()/my_thread_getspecific_ptr(). +*/ +extern pthread_key(MYSQL_THD, THR_THD); +static inline MYSQL_THD _current_thd(void) +{ + return my_pthread_getspecific_ptr(MYSQL_THD,THR_THD); +} +#endif + /* safe_mutex adds checking to mutex for easier debugging */ struct st_hash; typedef struct st_safe_mutex_t diff --git a/sql-common/client.c b/sql-common/client.c index 86c7dea5a58..42b6667b1bf 100644 --- a/sql-common/client.c +++ b/sql-common/client.c @@ -3408,7 +3408,7 @@ CLI_MYSQL_REAL_CONNECT(MYSQL *mysql,const char *host, const char *user, if (mysql->options.extension && mysql->options.extension->async_context) net->vio->async_context= mysql->options.extension->async_context; - if (my_net_init(net, net->vio, 0, MYF(0))) + if (my_net_init(net, net->vio, _current_thd(), MYF(0))) { vio_delete(net->vio); net->vio = 0; diff --git a/sql/mysqld.cc b/sql/mysqld.cc index 4f83e19f905..83829841b39 100644 --- a/sql/mysqld.cc +++ b/sql/mysqld.cc @@ -6187,7 +6187,7 @@ static void bootstrap(MYSQL_FILE *file) thd->variables.wsrep_on= 0; #endif thd->bootstrap=1; - my_net_init(&thd->net,(st_vio*) 0, (void*) 0, MYF(0)); + my_net_init(&thd->net,(st_vio*) 0, thd, MYF(0)); thd->max_client_packet_length= thd->net.max_packet; thd->security_ctx->master_access= ~(ulong)0; thd->thread_id= thd->variables.pseudo_thread_id= thread_id++; diff --git a/sql/mysqld.h b/sql/mysqld.h index 0e21b0f020a..c9ce982ec6f 100644 --- a/sql/mysqld.h +++ b/sql/mysqld.h @@ -761,20 +761,6 @@ inline void dec_thread_running() void set_server_version(void); -#if defined(MYSQL_DYNAMIC_PLUGIN) && defined(_WIN32) -extern "C" THD *_current_thd_noinline(); -#define _current_thd() _current_thd_noinline() -#else -/* - THR_THD is a key which will be used to set/get THD* for a thread, - using my_pthread_setspecific_ptr()/my_thread_getspecific_ptr(). -*/ -extern pthread_key(THD*, THR_THD); -inline THD *_current_thd(void) -{ - return my_pthread_getspecific_ptr(THD*,THR_THD); -} -#endif #define current_thd _current_thd() inline int set_current_thd(THD *thd) { diff --git a/sql/sql_class.cc b/sql/sql_class.cc index 5dba389115a..8ecf0511d06 100644 --- a/sql/sql_class.cc +++ b/sql/sql_class.cc @@ -369,16 +369,6 @@ void thd_close_connection(THD *thd) vio_close(thd->net.vio); } -/** - Get current THD object from thread local data - - @retval The THD object for the thread, NULL if not connection thread -*/ -THD *thd_get_current_thd() -{ - return current_thd; -} - /** Lock data that needs protection in THD object @@ -4074,16 +4064,12 @@ my_bool thd_net_is_killed() void thd_increment_bytes_received(void *thd, ulong length) { - if (unlikely(!thd)) // Called from federatedx - thd= current_thd; ((THD*) thd)->status_var.bytes_received+= length; } void thd_increment_net_big_packet_count(void *thd, ulong length) { - if (unlikely(!thd)) // Called from federatedx - thd= current_thd; ((THD*) thd)->status_var.net_big_packet_count+= length; } From 175dd3ad5933e1ad4afb676251f323fe5527a7f1 Mon Sep 17 00:00:00 2001 From: Sergei Golubchik Date: Sun, 23 Apr 2017 20:05:55 +0200 Subject: [PATCH 03/28] cleanup: don't include *.h files into SQL_SOURCES --- sql/CMakeLists.txt | 28 ++++++++++++++-------------- 1 file changed, 14 insertions(+), 14 deletions(-) diff --git a/sql/CMakeLists.txt b/sql/CMakeLists.txt index f7d9c6ac3a8..7a42742a181 100644 --- a/sql/CMakeLists.txt +++ b/sql/CMakeLists.txt @@ -82,30 +82,30 @@ SET (SQL_SOURCE filesort_utils.cc filesort.cc gstream.cc signal_handler.cc - handler.cc hash_filo.h + handler.cc hostname.cc init.cc item.cc item_buff.cc item_cmpfunc.cc item_create.cc item_func.cc item_geofunc.cc item_row.cc item_strfunc.cc item_subselect.cc item_sum.cc item_timefunc.cc key.cc log.cc lock.cc log_event.cc rpl_record.cc rpl_reporting.cc log_event_old.cc rpl_record_old.cc - message.h mf_iocache.cc my_decimal.cc ../sql-common/my_time.c + mf_iocache.cc my_decimal.cc ../sql-common/my_time.c mysqld.cc net_serv.cc keycaches.cc ../sql-common/client_plugin.c - opt_range.cc opt_range.h opt_sum.cc + opt_range.cc opt_sum.cc ../sql-common/pack.c parse_file.cc password.c procedure.cc protocol.cc records.cc repl_failsafe.cc rpl_filter.cc set_var.cc slave.cc sp.cc sp_cache.cc sp_head.cc sp_pcontext.cc sp_rcontext.cc spatial.cc sql_acl.cc sql_analyse.cc sql_base.cc - sql_cache.cc sql_class.cc sql_client.cc sql_crypt.cc sql_crypt.h + sql_cache.cc sql_class.cc sql_client.cc sql_crypt.cc sql_cursor.cc sql_db.cc sql_delete.cc sql_derived.cc sql_digest.cc sql_do.cc sql_error.cc sql_handler.cc sql_get_diagnostics.cc sql_help.cc sql_insert.cc sql_lex.cc sql_list.cc sql_load.cc sql_manager.cc - sql_parse.cc sql_bootstrap.cc sql_bootstrap.h + sql_parse.cc sql_bootstrap.cc sql_partition.cc sql_plugin.cc sql_prepare.cc sql_rename.cc - debug_sync.cc debug_sync.h + debug_sync.cc sql_repl.cc sql_select.cc sql_show.cc sql_state.c group_by_handler.cc sql_statistics.cc sql_string.cc @@ -121,21 +121,21 @@ SET (SQL_SOURCE sql_profile.cc event_parse_data.cc sql_alter.cc sql_signal.cc rpl_handler.cc mdl.cc sql_admin.cc transaction.cc sys_vars.cc sql_truncate.cc datadict.cc - sql_reload.cc sql_cmd.h item_inetfunc.cc + sql_reload.cc item_inetfunc.cc # added in MariaDB: - sql_explain.h sql_explain.cc - sql_analyze_stmt.h sql_analyze_stmt.cc - sql_lifo_buffer.h sql_join_cache.h sql_join_cache.cc + sql_explain.cc + sql_analyze_stmt.cc + sql_join_cache.cc create_options.cc multi_range_read.cc opt_index_cond_pushdown.cc opt_subselect.cc opt_table_elimination.cc sql_expression_cache.cc gcalc_slicescan.cc gcalc_tools.cc threadpool_common.cc ../sql-common/mysql_async.c - my_apc.cc my_apc.h mf_iocache_encr.cc - my_json_writer.cc my_json_writer.h + my_apc.cc mf_iocache_encr.cc + my_json_writer.cc rpl_gtid.cc rpl_parallel.cc - sql_type.cc sql_type.h + sql_type.cc ${WSREP_SOURCES} table_cache.cc encryption.cc ${CMAKE_CURRENT_BINARY_DIR}/sql_builtin.cc @@ -171,7 +171,7 @@ TARGET_LINK_LIBRARIES(sql ${MYSQLD_STATIC_PLUGIN_LIBS} ${LIBSYSTEMD}) IF(WIN32) - SET(MYSQLD_SOURCE main.cc nt_servc.cc nt_servc.h message.rc) + SET(MYSQLD_SOURCE main.cc nt_servc.cc message.rc) TARGET_LINK_LIBRARIES(sql psapi) ELSE() SET(MYSQLD_SOURCE main.cc ${DTRACE_PROBES_ALL}) From db3910741347e7d741f4a854075c248e9081d722 Mon Sep 17 00:00:00 2001 From: Vladislav Vaintroub Date: Tue, 18 Apr 2017 16:37:57 +0000 Subject: [PATCH 04/28] MDEV-11663 Create services for functionality used by plugins Added service for - encryption (AES) - error reporting, e.g my_printf_error() --- include/my_crypt.h | 71 +---------- include/my_sys.h | 23 +--- include/mysql/plugin.h | 2 +- include/mysql/plugin_audit.h.pp | 37 ++++++ include/mysql/plugin_auth.h.pp | 37 ++++++ include/mysql/plugin_encryption.h.pp | 37 ++++++ include/mysql/plugin_ftparser.h.pp | 37 ++++++ include/mysql/plugin_password_validation.h.pp | 37 ++++++ include/mysql/service_my_crypt.h | 120 ++++++++++++++++++ include/mysql/service_my_print_error.h | 64 ++++++++++ include/mysql/services.h | 2 + include/service_versions.h | 2 + libservices/CMakeLists.txt | 4 +- libservices/my_crypt_service.c | 2 + libservices/my_print_error_service.c | 17 +++ mysql-test/r/handlersocket.result | 2 +- mysql-test/r/plugin.result | 6 +- .../suite/plugins/r/auth_ed25519.result | 2 +- .../plugins/r/cracklib_password_check.result | 2 +- .../suite/plugins/r/show_all_plugins.result | 4 +- .../plugins/r/simple_password_check.result | 2 +- mysys_ssl/my_crypt.cc | 26 ++++ plugin/auth_gssapi/gssapi_server.cc | 17 ++- plugin/auth_gssapi/sspi_server.cc | 4 +- .../aws_key_management_plugin.cc | 22 ++-- .../example_key_management_plugin.cc | 24 +++- .../file_key_management_plugin.cc | 26 +++- plugin/file_key_management/parser.cc | 12 +- sql/innodb_priv.h | 1 + sql/sql_plugin_services.ic | 26 +++- sql/unireg.h | 4 +- storage/innobase/log/log0crypt.cc | 4 +- storage/xtradb/log/log0crypt.cc | 5 +- 33 files changed, 537 insertions(+), 144 deletions(-) create mode 100644 include/mysql/service_my_crypt.h create mode 100644 include/mysql/service_my_print_error.h create mode 100644 libservices/my_crypt_service.c create mode 100644 libservices/my_print_error_service.c diff --git a/include/my_crypt.h b/include/my_crypt.h index e1e94c9bd9d..719e349bfb9 100644 --- a/include/my_crypt.h +++ b/include/my_crypt.h @@ -18,74 +18,7 @@ #ifndef MY_CRYPT_INCLUDED #define MY_CRYPT_INCLUDED -#include - -#ifdef __cplusplus -extern "C" { -#endif - -/* return values from my_aes_encrypt/my_aes_decrypt functions */ -#define MY_AES_OK 0 -#define MY_AES_BAD_DATA -100 -#define MY_AES_OPENSSL_ERROR -101 -#define MY_AES_BAD_KEYSIZE -102 - -/* The block size for all supported algorithms */ -#define MY_AES_BLOCK_SIZE 16 - -/* The max key length of all supported algorithms */ -#define MY_AES_MAX_KEY_LENGTH 32 - -#define MY_AES_CTX_SIZE 512 - -enum my_aes_mode { - MY_AES_ECB, MY_AES_CBC -#ifdef HAVE_EncryptAes128Ctr - , MY_AES_CTR -#endif -#ifdef HAVE_EncryptAes128Gcm - , MY_AES_GCM -#endif -}; - -int my_aes_crypt_init(void *ctx, enum my_aes_mode mode, int flags, - const unsigned char* key, unsigned int klen, - const unsigned char* iv, unsigned int ivlen); -int my_aes_crypt_update(void *ctx, const uchar *src, uint slen, - uchar *dst, uint *dlen); -int my_aes_crypt_finish(void *ctx, uchar *dst, uint *dlen); -int my_aes_crypt(enum my_aes_mode mode, int flags, - const uchar *src, uint slen, uchar *dst, uint *dlen, - const uchar *key, uint klen, const uchar *iv, uint ivlen); - -/* - calculate the length of the cyphertext from the length of the plaintext - for different AES encryption modes with padding enabled. - Without padding (ENCRYPTION_FLAG_NOPAD) cyphertext has the same length - as the plaintext -*/ -static inline uint my_aes_get_size(enum my_aes_mode mode __attribute__((unused)), uint source_length) -{ -#ifdef HAVE_EncryptAes128Ctr - if (mode == MY_AES_CTR) - return source_length; -#ifdef HAVE_EncryptAes128Gcm - if (mode == MY_AES_GCM) - return source_length + MY_AES_BLOCK_SIZE; -#endif -#endif - return (source_length / MY_AES_BLOCK_SIZE + 1) * MY_AES_BLOCK_SIZE; -} - -static inline uint my_aes_ctx_size(enum my_aes_mode mode __attribute__((unused))) -{ - return MY_AES_CTX_SIZE; -} - -int my_random_bytes(uchar* buf, int num); - -#ifdef __cplusplus -} -#endif +#include /* HAVE_EncryptAes128{Ctr,Gcm} */ +#include #endif /* MY_CRYPT_INCLUDED */ diff --git a/include/my_sys.h b/include/my_sys.h index 7480514dc08..ca933ed574e 100644 --- a/include/my_sys.h +++ b/include/my_sys.h @@ -42,6 +42,7 @@ typedef struct my_aio_result { #include /*for alloca*/ #endif #include +#include #define MY_INIT(name) { my_progname= name; my_init(); } @@ -104,18 +105,10 @@ typedef struct my_aio_result { #define MY_GIVE_INFO 2 /* Give time info about process*/ #define MY_DONT_FREE_DBUG 4 /* Do not call DBUG_END() in my_end() */ -#define ME_HIGHBYTE 8 /* Shift for colours */ -#define ME_NOCUR 1 /* Don't use curses message */ -#define ME_OLDWIN 2 /* Use old window */ -#define ME_BELL 4 /* Ring bell then printing message */ -#define ME_HOLDTANG 8 /* Don't delete last keys */ -#define ME_WAITTOT 16 /* Wait for errtime secs of for a action */ -#define ME_WAITTANG 32 /* Wait for a user action */ -#define ME_NOREFRESH 64 /* Write the error message to error log */ -#define ME_NOINPUT 128 /* Dont use the input libary */ -#define ME_COLOUR1 ((1 << ME_HIGHBYTE)) /* Possibly error-colours */ -#define ME_COLOUR2 ((2 << ME_HIGHBYTE)) -#define ME_COLOUR3 ((3 << ME_HIGHBYTE)) +#define ME_BELL 4 /* Ring bell then printing message */ +#define ME_WAITTANG 0 /* Wait for a user action */ +#define ME_NOREFRESH 64 /* Write the error message to error log */ +#define ME_NOINPUT 0 /* Dont use the input libary */ #define ME_JUST_INFO 1024 /**< not error but just info */ #define ME_JUST_WARNING 2048 /**< not error but just warning */ #define ME_FATALERROR 4096 /* Fatal statement error */ @@ -715,12 +708,6 @@ extern int my_sync(File fd, myf my_flags); extern int my_sync_dir(const char *dir_name, myf my_flags); extern int my_sync_dir_by_file(const char *file_name, myf my_flags); extern const char *my_get_err_msg(uint nr); -extern void my_error(uint nr,myf MyFlags, ...); -extern void my_printf_error(uint my_err, const char *format, - myf MyFlags, ...) - ATTRIBUTE_FORMAT(printf, 2, 4); -extern void my_printv_error(uint error, const char *format, myf MyFlags, - va_list ap); extern int my_error_register(const char** (*get_errmsgs) (void), uint first, uint last); extern const char **my_error_unregister(uint first, uint last); diff --git a/include/mysql/plugin.h b/include/mysql/plugin.h index a5bfa1bbc9e..61be12057e8 100644 --- a/include/mysql/plugin.h +++ b/include/mysql/plugin.h @@ -75,7 +75,7 @@ typedef struct st_mysql_xid MYSQL_XID; #define MYSQL_PLUGIN_INTERFACE_VERSION 0x0104 /* MariaDB plugin interface version */ -#define MARIA_PLUGIN_INTERFACE_VERSION 0x010c +#define MARIA_PLUGIN_INTERFACE_VERSION 0x010d /* The allowable types of plugins diff --git a/include/mysql/plugin_audit.h.pp b/include/mysql/plugin_audit.h.pp index eb5369c0377..d014edfedb9 100644 --- a/include/mysql/plugin_audit.h.pp +++ b/include/mysql/plugin_audit.h.pp @@ -137,6 +137,43 @@ size_t my_md5_context_size(); void my_md5_init(void *context); void my_md5_input(void *context, const unsigned char *buf, size_t len); void my_md5_result(void *context, unsigned char *digest); +enum my_aes_mode { + MY_AES_ECB, MY_AES_CBC +}; +extern struct my_crypt_service_st { + int (*my_aes_crypt_init)(void *ctx, enum my_aes_mode mode, int flags, + const unsigned char* key, unsigned int klen, + const unsigned char* iv, unsigned int ivlen); + int (*my_aes_crypt_update)(void *ctx, const unsigned char *src, unsigned int slen, + unsigned char *dst, unsigned int *dlen); + int (*my_aes_crypt_finish)(void *ctx, unsigned char *dst, unsigned int *dlen); + int (*my_aes_crypt)(enum my_aes_mode mode, int flags, + const unsigned char *src, unsigned int slen, unsigned char *dst, unsigned int *dlen, + const unsigned char *key, unsigned int klen, const unsigned char *iv, unsigned int ivlen); + unsigned int (*my_aes_get_size)(enum my_aes_mode mode, unsigned int source_length); + unsigned int (*my_aes_ctx_size)(enum my_aes_mode mode); + int (*my_random_bytes)(unsigned char* buf, int num); +} *my_crypt_service; +int my_aes_crypt_init(void *ctx, enum my_aes_mode mode, int flags, + const unsigned char* key, unsigned int klen, + const unsigned char* iv, unsigned int ivlen); +int my_aes_crypt_update(void *ctx, const unsigned char *src, unsigned int slen, + unsigned char *dst, unsigned int *dlen); +int my_aes_crypt_finish(void *ctx, unsigned char *dst, unsigned int *dlen); +int my_aes_crypt(enum my_aes_mode mode, int flags, + const unsigned char *src, unsigned int slen, unsigned char *dst, unsigned int *dlen, + const unsigned char *key, unsigned int klen, const unsigned char *iv, unsigned int ivlen); +int my_random_bytes(unsigned char* buf, int num); +unsigned int my_aes_get_size(enum my_aes_mode mode, unsigned int source_length); +unsigned int my_aes_ctx_size(enum my_aes_mode mode); +extern struct my_print_error_service_st { + void(*my_error_func)(unsigned int nr, unsigned long MyFlags, ...); + void(*my_printf_error_func)(unsigned int nr, const char *fmt, unsigned long MyFlags,...); + void(*my_printv_error_func)(unsigned int error, const char *format, unsigned long MyFlags, va_list ap); +} *my_print_error_service; +extern void my_error(unsigned int nr, unsigned long MyFlags, ...); +extern void my_printf_error(unsigned int my_err, const char *format, unsigned long MyFlags, ...); +extern void my_printv_error(unsigned int error, const char *format, unsigned long MyFlags,va_list ap); extern struct my_snprintf_service_st { size_t (*my_snprintf_type)(char*, size_t, const char*, ...); size_t (*my_vsnprintf_type)(char *, size_t, const char*, va_list); diff --git a/include/mysql/plugin_auth.h.pp b/include/mysql/plugin_auth.h.pp index 766682fb44f..9dd3deba8de 100644 --- a/include/mysql/plugin_auth.h.pp +++ b/include/mysql/plugin_auth.h.pp @@ -137,6 +137,43 @@ size_t my_md5_context_size(); void my_md5_init(void *context); void my_md5_input(void *context, const unsigned char *buf, size_t len); void my_md5_result(void *context, unsigned char *digest); +enum my_aes_mode { + MY_AES_ECB, MY_AES_CBC +}; +extern struct my_crypt_service_st { + int (*my_aes_crypt_init)(void *ctx, enum my_aes_mode mode, int flags, + const unsigned char* key, unsigned int klen, + const unsigned char* iv, unsigned int ivlen); + int (*my_aes_crypt_update)(void *ctx, const unsigned char *src, unsigned int slen, + unsigned char *dst, unsigned int *dlen); + int (*my_aes_crypt_finish)(void *ctx, unsigned char *dst, unsigned int *dlen); + int (*my_aes_crypt)(enum my_aes_mode mode, int flags, + const unsigned char *src, unsigned int slen, unsigned char *dst, unsigned int *dlen, + const unsigned char *key, unsigned int klen, const unsigned char *iv, unsigned int ivlen); + unsigned int (*my_aes_get_size)(enum my_aes_mode mode, unsigned int source_length); + unsigned int (*my_aes_ctx_size)(enum my_aes_mode mode); + int (*my_random_bytes)(unsigned char* buf, int num); +} *my_crypt_service; +int my_aes_crypt_init(void *ctx, enum my_aes_mode mode, int flags, + const unsigned char* key, unsigned int klen, + const unsigned char* iv, unsigned int ivlen); +int my_aes_crypt_update(void *ctx, const unsigned char *src, unsigned int slen, + unsigned char *dst, unsigned int *dlen); +int my_aes_crypt_finish(void *ctx, unsigned char *dst, unsigned int *dlen); +int my_aes_crypt(enum my_aes_mode mode, int flags, + const unsigned char *src, unsigned int slen, unsigned char *dst, unsigned int *dlen, + const unsigned char *key, unsigned int klen, const unsigned char *iv, unsigned int ivlen); +int my_random_bytes(unsigned char* buf, int num); +unsigned int my_aes_get_size(enum my_aes_mode mode, unsigned int source_length); +unsigned int my_aes_ctx_size(enum my_aes_mode mode); +extern struct my_print_error_service_st { + void(*my_error_func)(unsigned int nr, unsigned long MyFlags, ...); + void(*my_printf_error_func)(unsigned int nr, const char *fmt, unsigned long MyFlags,...); + void(*my_printv_error_func)(unsigned int error, const char *format, unsigned long MyFlags, va_list ap); +} *my_print_error_service; +extern void my_error(unsigned int nr, unsigned long MyFlags, ...); +extern void my_printf_error(unsigned int my_err, const char *format, unsigned long MyFlags, ...); +extern void my_printv_error(unsigned int error, const char *format, unsigned long MyFlags,va_list ap); extern struct my_snprintf_service_st { size_t (*my_snprintf_type)(char*, size_t, const char*, ...); size_t (*my_vsnprintf_type)(char *, size_t, const char*, va_list); diff --git a/include/mysql/plugin_encryption.h.pp b/include/mysql/plugin_encryption.h.pp index 6a66e90234a..7bb81614971 100644 --- a/include/mysql/plugin_encryption.h.pp +++ b/include/mysql/plugin_encryption.h.pp @@ -137,6 +137,43 @@ size_t my_md5_context_size(); void my_md5_init(void *context); void my_md5_input(void *context, const unsigned char *buf, size_t len); void my_md5_result(void *context, unsigned char *digest); +enum my_aes_mode { + MY_AES_ECB, MY_AES_CBC +}; +extern struct my_crypt_service_st { + int (*my_aes_crypt_init)(void *ctx, enum my_aes_mode mode, int flags, + const unsigned char* key, unsigned int klen, + const unsigned char* iv, unsigned int ivlen); + int (*my_aes_crypt_update)(void *ctx, const unsigned char *src, unsigned int slen, + unsigned char *dst, unsigned int *dlen); + int (*my_aes_crypt_finish)(void *ctx, unsigned char *dst, unsigned int *dlen); + int (*my_aes_crypt)(enum my_aes_mode mode, int flags, + const unsigned char *src, unsigned int slen, unsigned char *dst, unsigned int *dlen, + const unsigned char *key, unsigned int klen, const unsigned char *iv, unsigned int ivlen); + unsigned int (*my_aes_get_size)(enum my_aes_mode mode, unsigned int source_length); + unsigned int (*my_aes_ctx_size)(enum my_aes_mode mode); + int (*my_random_bytes)(unsigned char* buf, int num); +} *my_crypt_service; +int my_aes_crypt_init(void *ctx, enum my_aes_mode mode, int flags, + const unsigned char* key, unsigned int klen, + const unsigned char* iv, unsigned int ivlen); +int my_aes_crypt_update(void *ctx, const unsigned char *src, unsigned int slen, + unsigned char *dst, unsigned int *dlen); +int my_aes_crypt_finish(void *ctx, unsigned char *dst, unsigned int *dlen); +int my_aes_crypt(enum my_aes_mode mode, int flags, + const unsigned char *src, unsigned int slen, unsigned char *dst, unsigned int *dlen, + const unsigned char *key, unsigned int klen, const unsigned char *iv, unsigned int ivlen); +int my_random_bytes(unsigned char* buf, int num); +unsigned int my_aes_get_size(enum my_aes_mode mode, unsigned int source_length); +unsigned int my_aes_ctx_size(enum my_aes_mode mode); +extern struct my_print_error_service_st { + void(*my_error_func)(unsigned int nr, unsigned long MyFlags, ...); + void(*my_printf_error_func)(unsigned int nr, const char *fmt, unsigned long MyFlags,...); + void(*my_printv_error_func)(unsigned int error, const char *format, unsigned long MyFlags, va_list ap); +} *my_print_error_service; +extern void my_error(unsigned int nr, unsigned long MyFlags, ...); +extern void my_printf_error(unsigned int my_err, const char *format, unsigned long MyFlags, ...); +extern void my_printv_error(unsigned int error, const char *format, unsigned long MyFlags,va_list ap); extern struct my_snprintf_service_st { size_t (*my_snprintf_type)(char*, size_t, const char*, ...); size_t (*my_vsnprintf_type)(char *, size_t, const char*, va_list); diff --git a/include/mysql/plugin_ftparser.h.pp b/include/mysql/plugin_ftparser.h.pp index 6004f4b61aa..43c02059874 100644 --- a/include/mysql/plugin_ftparser.h.pp +++ b/include/mysql/plugin_ftparser.h.pp @@ -137,6 +137,43 @@ size_t my_md5_context_size(); void my_md5_init(void *context); void my_md5_input(void *context, const unsigned char *buf, size_t len); void my_md5_result(void *context, unsigned char *digest); +enum my_aes_mode { + MY_AES_ECB, MY_AES_CBC +}; +extern struct my_crypt_service_st { + int (*my_aes_crypt_init)(void *ctx, enum my_aes_mode mode, int flags, + const unsigned char* key, unsigned int klen, + const unsigned char* iv, unsigned int ivlen); + int (*my_aes_crypt_update)(void *ctx, const unsigned char *src, unsigned int slen, + unsigned char *dst, unsigned int *dlen); + int (*my_aes_crypt_finish)(void *ctx, unsigned char *dst, unsigned int *dlen); + int (*my_aes_crypt)(enum my_aes_mode mode, int flags, + const unsigned char *src, unsigned int slen, unsigned char *dst, unsigned int *dlen, + const unsigned char *key, unsigned int klen, const unsigned char *iv, unsigned int ivlen); + unsigned int (*my_aes_get_size)(enum my_aes_mode mode, unsigned int source_length); + unsigned int (*my_aes_ctx_size)(enum my_aes_mode mode); + int (*my_random_bytes)(unsigned char* buf, int num); +} *my_crypt_service; +int my_aes_crypt_init(void *ctx, enum my_aes_mode mode, int flags, + const unsigned char* key, unsigned int klen, + const unsigned char* iv, unsigned int ivlen); +int my_aes_crypt_update(void *ctx, const unsigned char *src, unsigned int slen, + unsigned char *dst, unsigned int *dlen); +int my_aes_crypt_finish(void *ctx, unsigned char *dst, unsigned int *dlen); +int my_aes_crypt(enum my_aes_mode mode, int flags, + const unsigned char *src, unsigned int slen, unsigned char *dst, unsigned int *dlen, + const unsigned char *key, unsigned int klen, const unsigned char *iv, unsigned int ivlen); +int my_random_bytes(unsigned char* buf, int num); +unsigned int my_aes_get_size(enum my_aes_mode mode, unsigned int source_length); +unsigned int my_aes_ctx_size(enum my_aes_mode mode); +extern struct my_print_error_service_st { + void(*my_error_func)(unsigned int nr, unsigned long MyFlags, ...); + void(*my_printf_error_func)(unsigned int nr, const char *fmt, unsigned long MyFlags,...); + void(*my_printv_error_func)(unsigned int error, const char *format, unsigned long MyFlags, va_list ap); +} *my_print_error_service; +extern void my_error(unsigned int nr, unsigned long MyFlags, ...); +extern void my_printf_error(unsigned int my_err, const char *format, unsigned long MyFlags, ...); +extern void my_printv_error(unsigned int error, const char *format, unsigned long MyFlags,va_list ap); extern struct my_snprintf_service_st { size_t (*my_snprintf_type)(char*, size_t, const char*, ...); size_t (*my_vsnprintf_type)(char *, size_t, const char*, va_list); diff --git a/include/mysql/plugin_password_validation.h.pp b/include/mysql/plugin_password_validation.h.pp index 966d92ae5ad..d2fe8fa8e23 100644 --- a/include/mysql/plugin_password_validation.h.pp +++ b/include/mysql/plugin_password_validation.h.pp @@ -137,6 +137,43 @@ size_t my_md5_context_size(); void my_md5_init(void *context); void my_md5_input(void *context, const unsigned char *buf, size_t len); void my_md5_result(void *context, unsigned char *digest); +enum my_aes_mode { + MY_AES_ECB, MY_AES_CBC +}; +extern struct my_crypt_service_st { + int (*my_aes_crypt_init)(void *ctx, enum my_aes_mode mode, int flags, + const unsigned char* key, unsigned int klen, + const unsigned char* iv, unsigned int ivlen); + int (*my_aes_crypt_update)(void *ctx, const unsigned char *src, unsigned int slen, + unsigned char *dst, unsigned int *dlen); + int (*my_aes_crypt_finish)(void *ctx, unsigned char *dst, unsigned int *dlen); + int (*my_aes_crypt)(enum my_aes_mode mode, int flags, + const unsigned char *src, unsigned int slen, unsigned char *dst, unsigned int *dlen, + const unsigned char *key, unsigned int klen, const unsigned char *iv, unsigned int ivlen); + unsigned int (*my_aes_get_size)(enum my_aes_mode mode, unsigned int source_length); + unsigned int (*my_aes_ctx_size)(enum my_aes_mode mode); + int (*my_random_bytes)(unsigned char* buf, int num); +} *my_crypt_service; +int my_aes_crypt_init(void *ctx, enum my_aes_mode mode, int flags, + const unsigned char* key, unsigned int klen, + const unsigned char* iv, unsigned int ivlen); +int my_aes_crypt_update(void *ctx, const unsigned char *src, unsigned int slen, + unsigned char *dst, unsigned int *dlen); +int my_aes_crypt_finish(void *ctx, unsigned char *dst, unsigned int *dlen); +int my_aes_crypt(enum my_aes_mode mode, int flags, + const unsigned char *src, unsigned int slen, unsigned char *dst, unsigned int *dlen, + const unsigned char *key, unsigned int klen, const unsigned char *iv, unsigned int ivlen); +int my_random_bytes(unsigned char* buf, int num); +unsigned int my_aes_get_size(enum my_aes_mode mode, unsigned int source_length); +unsigned int my_aes_ctx_size(enum my_aes_mode mode); +extern struct my_print_error_service_st { + void(*my_error_func)(unsigned int nr, unsigned long MyFlags, ...); + void(*my_printf_error_func)(unsigned int nr, const char *fmt, unsigned long MyFlags,...); + void(*my_printv_error_func)(unsigned int error, const char *format, unsigned long MyFlags, va_list ap); +} *my_print_error_service; +extern void my_error(unsigned int nr, unsigned long MyFlags, ...); +extern void my_printf_error(unsigned int my_err, const char *format, unsigned long MyFlags, ...); +extern void my_printv_error(unsigned int error, const char *format, unsigned long MyFlags,va_list ap); extern struct my_snprintf_service_st { size_t (*my_snprintf_type)(char*, size_t, const char*, ...); size_t (*my_vsnprintf_type)(char *, size_t, const char*, va_list); diff --git a/include/mysql/service_my_crypt.h b/include/mysql/service_my_crypt.h new file mode 100644 index 00000000000..83de0378e4a --- /dev/null +++ b/include/mysql/service_my_crypt.h @@ -0,0 +1,120 @@ +#ifndef MYSQL_SERVICE_MY_CRYPT_INCLUDED +#define MYSQL_SERVICE_MY_CRYPT_INCLUDED + +/* + Copyright (c) 2014 Google Inc. + Copyright (c) 2014, 2015 MariaDB Corporation + + This program is free software; you can redistribute it and/or modify + it under the terms of the GNU General Public License as published by + the Free Software Foundation; version 2 of the License. + + This program is distributed in the hope that it will be useful, + but WITHOUT ANY WARRANTY; without even the implied warranty of + MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the + GNU General Public License for more details. + + You should have received a copy of the GNU General Public License + along with this program; if not, write to the Free Software + Foundation, Inc., 51 Franklin St, Fifth Floor, Boston, MA 02110-1301 USA */ + +/** + @file + my crypt service + + AES encryption functions, and a function to generate random bytes. + + Include my_config.h before this file to use CTR and GCM modes + (they only work if server was compiled with openssl). +*/ + + +#ifdef __cplusplus +extern "C" { +#endif + +/* return values from my_aes_encrypt/my_aes_decrypt functions */ +#define MY_AES_OK 0 +#define MY_AES_BAD_DATA -100 +#define MY_AES_OPENSSL_ERROR -101 +#define MY_AES_BAD_KEYSIZE -102 + +/* The block size for all supported algorithms */ +#define MY_AES_BLOCK_SIZE 16 + +/* The max key length of all supported algorithms */ +#define MY_AES_MAX_KEY_LENGTH 32 + +#define MY_AES_CTX_SIZE 512 + +enum my_aes_mode { + MY_AES_ECB, MY_AES_CBC +#ifdef HAVE_EncryptAes128Ctr + , MY_AES_CTR +#endif +#ifdef HAVE_EncryptAes128Gcm + , MY_AES_GCM +#endif +}; + +extern struct my_crypt_service_st { + int (*my_aes_crypt_init)(void *ctx, enum my_aes_mode mode, int flags, + const unsigned char* key, unsigned int klen, + const unsigned char* iv, unsigned int ivlen); + int (*my_aes_crypt_update)(void *ctx, const unsigned char *src, unsigned int slen, + unsigned char *dst, unsigned int *dlen); + int (*my_aes_crypt_finish)(void *ctx, unsigned char *dst, unsigned int *dlen); + int (*my_aes_crypt)(enum my_aes_mode mode, int flags, + const unsigned char *src, unsigned int slen, unsigned char *dst, unsigned int *dlen, + const unsigned char *key, unsigned int klen, const unsigned char *iv, unsigned int ivlen); + unsigned int (*my_aes_get_size)(enum my_aes_mode mode, unsigned int source_length); + unsigned int (*my_aes_ctx_size)(enum my_aes_mode mode); + int (*my_random_bytes)(unsigned char* buf, int num); +} *my_crypt_service; + +#ifdef MYSQL_DYNAMIC_PLUGIN + +#define my_aes_crypt_init(A,B,C,D,E,F,G) \ + my_crypt_service->my_aes_crypt_init(A,B,C,D,E,F,G) + +#define my_aes_crypt_update(A,B,C,D,E) \ + my_crypt_service->my_aes_crypt_update(A,B,C,D,E) + +#define my_aes_crypt_finish(A,B,C) \ + my_crypt_service->my_aes_crypt_finish(A,B,C) + +#define my_aes_crypt(A,B,C,D,E,F,G,H,I,J) \ + my_crypt_service->my_aes_crypt(A,B,C,D,E,F,G,H,I,J) + +#define my_aes_get_size(A,B)\ + my_crypt_service->my_aes_get_size(A,B) + +#define my_aes_ctx_size(A)\ + my_crypt_service->my_aes_ctx_size(A) + +#define my_random_bytes(A,B)\ + my_crypt_service->my_random_bytes(A,B) + +#else + +int my_aes_crypt_init(void *ctx, enum my_aes_mode mode, int flags, + const unsigned char* key, unsigned int klen, + const unsigned char* iv, unsigned int ivlen); +int my_aes_crypt_update(void *ctx, const unsigned char *src, unsigned int slen, + unsigned char *dst, unsigned int *dlen); +int my_aes_crypt_finish(void *ctx, unsigned char *dst, unsigned int *dlen); +int my_aes_crypt(enum my_aes_mode mode, int flags, + const unsigned char *src, unsigned int slen, unsigned char *dst, unsigned int *dlen, + const unsigned char *key, unsigned int klen, const unsigned char *iv, unsigned int ivlen); + +int my_random_bytes(unsigned char* buf, int num); +unsigned int my_aes_get_size(enum my_aes_mode mode, unsigned int source_length); +unsigned int my_aes_ctx_size(enum my_aes_mode mode); +#endif + + +#ifdef __cplusplus +} +#endif + +#endif /* MYSQL_SERVICE_MY_CRYPT_INCLUDED */ diff --git a/include/mysql/service_my_print_error.h b/include/mysql/service_my_print_error.h new file mode 100644 index 00000000000..636151655e5 --- /dev/null +++ b/include/mysql/service_my_print_error.h @@ -0,0 +1,64 @@ +/* Copyright (c) 2016, MariaDB + +This program is free software; you can redistribute it and/or modify +it under the terms of the GNU General Public License as published by +the Free Software Foundation; version 2 of the License. + +This program is distributed in the hope that it will be useful, +but WITHOUT ANY WARRANTY; without even the implied warranty of +MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the +GNU General Public License for more details. + +You should have received a copy of the GNU General Public License +along with this program; if not, write to the Free Software +Foundation, Inc., 51 Franklin St, Fifth Floor, Boston, MA 02110-1301 USA */ + +#ifndef MYSQL_SERVICE_MY_PRINT_ERROR_INCLUDED +#define MYSQL_SERVICE_MY_PRINT_ERROR_INCLUDED + +/** + @file include/mysql/service_my_print_error.h + + This service provides functions for plugins to report + errors to client (without client, the errors are written to the error log). + +*/ +#ifdef __cplusplus +extern "C" { +#endif + +#ifndef MYSQL_ABI_CHECK +#include +#include +#endif + +#define ME_ERROR_LOG 64 /* Write the message to the error log */ +#define ME_NOTE 1024 /* Not an error, just a note */ +#define ME_WARNING 2048 /* Not an error, just a warning */ +#define ME_FATAL 4096 /* Fatal statement error */ + +extern struct my_print_error_service_st { + void (*my_error_func)(unsigned int nr, unsigned long MyFlags, ...); + void (*my_printf_error_func)(unsigned int nr, const char *fmt, unsigned long MyFlags,...); + void (*my_printv_error_func)(unsigned int error, const char *format, unsigned long MyFlags, va_list ap); +} *my_print_error_service; + +#ifdef MYSQL_DYNAMIC_PLUGIN + +#define my_error my_print_error_service->my_error_func +#define my_printf_error my_print_error_service->my_printf_error_func +#define my_printv_error(A,B,C,D) my_print_error_service->my_printv_error_func(A,B,C,D) + +#else + +extern void my_error(unsigned int nr, unsigned long MyFlags, ...); +extern void my_printf_error(unsigned int my_err, const char *format, unsigned long MyFlags, ...); +extern void my_printv_error(unsigned int error, const char *format, unsigned long MyFlags,va_list ap); +#endif + +#ifdef __cplusplus +} +#endif + +#endif + diff --git a/include/mysql/services.h b/include/mysql/services.h index 420f2430a36..6168c5ed8dc 100644 --- a/include/mysql/services.h +++ b/include/mysql/services.h @@ -26,6 +26,8 @@ extern "C" { #include #include #include +#include +#include #include #include #include diff --git a/include/service_versions.h b/include/service_versions.h index d79474f1d36..ddc780a44b9 100644 --- a/include/service_versions.h +++ b/include/service_versions.h @@ -27,7 +27,9 @@ #define VERSION_encryption 0x0300 #define VERSION_encryption_scheme 0x0100 #define VERSION_logger 0x0100 +#define VERSION_my_crypt 0x0100 #define VERSION_my_md5 0x0100 +#define VERSION_my_print_error 0x0100 #define VERSION_my_sha1 0x0101 #define VERSION_my_sha2 0x0100 #define VERSION_my_snprintf 0x0100 diff --git a/libservices/CMakeLists.txt b/libservices/CMakeLists.txt index 0b68a156077..e20be6d7a7c 100644 --- a/libservices/CMakeLists.txt +++ b/libservices/CMakeLists.txt @@ -22,7 +22,9 @@ SET(MYSQLSERVICES_SOURCES encryption_service.c kill_statement_service.c logger_service.c + my_crypt_service.c my_md5_service.c + my_print_error_service.c my_sha1_service.c my_sha2_service.c my_snprintf_service.c @@ -35,7 +37,7 @@ SET(MYSQLSERVICES_SOURCES thd_timezone_service.c thd_wait_service.c wsrep_service.c -) + ) ADD_CONVENIENCE_LIBRARY(mysqlservices ${MYSQLSERVICES_SOURCES}) INSTALL(TARGETS mysqlservices DESTINATION ${INSTALL_LIBDIR} COMPONENT Development) diff --git a/libservices/my_crypt_service.c b/libservices/my_crypt_service.c new file mode 100644 index 00000000000..e6b9e273094 --- /dev/null +++ b/libservices/my_crypt_service.c @@ -0,0 +1,2 @@ +#include +SERVICE_VERSION my_crypt_service= (void*)VERSION_my_crypt; diff --git a/libservices/my_print_error_service.c b/libservices/my_print_error_service.c new file mode 100644 index 00000000000..7642668d470 --- /dev/null +++ b/libservices/my_print_error_service.c @@ -0,0 +1,17 @@ +/* Copyright (c) 2016 MariaDB + + This program is free software; you can redistribute it and/or modify + it under the terms of the GNU General Public License as published by + the Free Software Foundation; version 2 of the License. + + This program is distributed in the hope that it will be useful, + but WITHOUT ANY WARRANTY; without even the implied warranty of + MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the + GNU General Public License for more details. + + You should have received a copy of the GNU General Public License + along with this program; if not, write to the Free Software + Foundation, Inc., 51 Franklin St, Fifth Floor, Boston, MA 02110-1301 USA */ + +#include +SERVICE_VERSION my_print_error_service= (void*)VERSION_my_print_error; \ No newline at end of file diff --git a/mysql-test/r/handlersocket.result b/mysql-test/r/handlersocket.result index 26c77813b26..1b3fc573548 100644 --- a/mysql-test/r/handlersocket.result +++ b/mysql-test/r/handlersocket.result @@ -5,7 +5,7 @@ plugin_version 1.0 plugin_status ACTIVE plugin_type DAEMON plugin_library handlersocket.so -plugin_library_version 1.12 +plugin_library_version 1.13 plugin_author higuchi dot akira at dena dot jp plugin_description Direct access into InnoDB plugin_license BSD diff --git a/mysql-test/r/plugin.result b/mysql-test/r/plugin.result index f278724cc9a..3a141a25b5c 100644 --- a/mysql-test/r/plugin.result +++ b/mysql-test/r/plugin.result @@ -12,7 +12,7 @@ PLUGIN_STATUS ACTIVE PLUGIN_TYPE STORAGE ENGINE PLUGIN_TYPE_VERSION # PLUGIN_LIBRARY ha_example.so -PLUGIN_LIBRARY_VERSION 1.12 +PLUGIN_LIBRARY_VERSION 1.13 PLUGIN_AUTHOR Brian Aker, MySQL AB PLUGIN_DESCRIPTION Example storage engine PLUGIN_LICENSE GPL @@ -25,7 +25,7 @@ PLUGIN_STATUS ACTIVE PLUGIN_TYPE DAEMON PLUGIN_TYPE_VERSION # PLUGIN_LIBRARY ha_example.so -PLUGIN_LIBRARY_VERSION 1.12 +PLUGIN_LIBRARY_VERSION 1.13 PLUGIN_AUTHOR Sergei Golubchik PLUGIN_DESCRIPTION Unusable Daemon PLUGIN_LICENSE GPL @@ -64,7 +64,7 @@ PLUGIN_STATUS DELETED PLUGIN_TYPE STORAGE ENGINE PLUGIN_TYPE_VERSION # PLUGIN_LIBRARY ha_example.so -PLUGIN_LIBRARY_VERSION 1.12 +PLUGIN_LIBRARY_VERSION 1.13 PLUGIN_AUTHOR Brian Aker, MySQL AB PLUGIN_DESCRIPTION Example storage engine PLUGIN_LICENSE GPL diff --git a/mysql-test/suite/plugins/r/auth_ed25519.result b/mysql-test/suite/plugins/r/auth_ed25519.result index 719dd462763..a3b85a11dea 100644 --- a/mysql-test/suite/plugins/r/auth_ed25519.result +++ b/mysql-test/suite/plugins/r/auth_ed25519.result @@ -27,7 +27,7 @@ PLUGIN_STATUS ACTIVE PLUGIN_TYPE AUTHENTICATION PLUGIN_TYPE_VERSION 2.1 PLUGIN_LIBRARY auth_ed25519.so -PLUGIN_LIBRARY_VERSION 1.12 +PLUGIN_LIBRARY_VERSION 1.13 PLUGIN_AUTHOR Sergei Golubchik PLUGIN_DESCRIPTION Elliptic curve ED25519 based authentication PLUGIN_LICENSE GPL diff --git a/mysql-test/suite/plugins/r/cracklib_password_check.result b/mysql-test/suite/plugins/r/cracklib_password_check.result index 479b4b00698..6b4e30b3d81 100644 --- a/mysql-test/suite/plugins/r/cracklib_password_check.result +++ b/mysql-test/suite/plugins/r/cracklib_password_check.result @@ -6,7 +6,7 @@ PLUGIN_STATUS ACTIVE PLUGIN_TYPE PASSWORD VALIDATION PLUGIN_TYPE_VERSION 1.0 PLUGIN_LIBRARY cracklib_password_check.so -PLUGIN_LIBRARY_VERSION 1.12 +PLUGIN_LIBRARY_VERSION 1.13 PLUGIN_AUTHOR Sergei Golubchik PLUGIN_DESCRIPTION Password validation via CrackLib PLUGIN_LICENSE GPL diff --git a/mysql-test/suite/plugins/r/show_all_plugins.result b/mysql-test/suite/plugins/r/show_all_plugins.result index c91a360d1d7..dd6cbfce4c4 100644 --- a/mysql-test/suite/plugins/r/show_all_plugins.result +++ b/mysql-test/suite/plugins/r/show_all_plugins.result @@ -4,8 +4,8 @@ Variable_name Value Opened_plugin_libraries 0 select * from information_schema.all_plugins where plugin_library='ha_example.so'; PLUGIN_NAME PLUGIN_VERSION PLUGIN_STATUS PLUGIN_TYPE PLUGIN_TYPE_VERSION PLUGIN_LIBRARY PLUGIN_LIBRARY_VERSION PLUGIN_AUTHOR PLUGIN_DESCRIPTION PLUGIN_LICENSE LOAD_OPTION PLUGIN_MATURITY PLUGIN_AUTH_VERSION -EXAMPLE 0.1 NOT INSTALLED STORAGE ENGINE MYSQL_VERSION_ID ha_example.so 1.12 Brian Aker, MySQL AB Example storage engine GPL OFF Experimental 0.1 -UNUSABLE 3.14 NOT INSTALLED DAEMON MYSQL_VERSION_ID ha_example.so 1.12 Sergei Golubchik Unusable Daemon GPL OFF Experimental 3.14.15.926 +EXAMPLE 0.1 NOT INSTALLED STORAGE ENGINE MYSQL_VERSION_ID ha_example.so 1.13 Brian Aker, MySQL AB Example storage engine GPL OFF Experimental 0.1 +UNUSABLE 3.14 NOT INSTALLED DAEMON MYSQL_VERSION_ID ha_example.so 1.13 Sergei Golubchik Unusable Daemon GPL OFF Experimental 3.14.15.926 show status like '%libraries%'; Variable_name Value Opened_plugin_libraries 1 diff --git a/mysql-test/suite/plugins/r/simple_password_check.result b/mysql-test/suite/plugins/r/simple_password_check.result index 11385bd6b01..672d0107492 100644 --- a/mysql-test/suite/plugins/r/simple_password_check.result +++ b/mysql-test/suite/plugins/r/simple_password_check.result @@ -6,7 +6,7 @@ PLUGIN_STATUS ACTIVE PLUGIN_TYPE PASSWORD VALIDATION PLUGIN_TYPE_VERSION 1.0 PLUGIN_LIBRARY simple_password_check.so -PLUGIN_LIBRARY_VERSION 1.12 +PLUGIN_LIBRARY_VERSION 1.13 PLUGIN_AUTHOR Sergei Golubchik PLUGIN_DESCRIPTION Simple password strength checks PLUGIN_LICENSE GPL diff --git a/mysys_ssl/my_crypt.cc b/mysys_ssl/my_crypt.cc index 49bd9af3f60..a0937a83e17 100644 --- a/mysys_ssl/my_crypt.cc +++ b/mysys_ssl/my_crypt.cc @@ -266,6 +266,32 @@ int my_aes_crypt(enum my_aes_mode mode, int flags, return res1 ? res1 : res2; } + +/* + calculate the length of the cyphertext from the length of the plaintext + for different AES encryption modes with padding enabled. + Without padding (ENCRYPTION_FLAG_NOPAD) cyphertext has the same length + as the plaintext +*/ +unsigned int my_aes_get_size(enum my_aes_mode mode __attribute__((unused)), unsigned int source_length) +{ +#ifdef HAVE_EncryptAes128Ctr + if (mode == MY_AES_CTR) + return source_length; +#ifdef HAVE_EncryptAes128Gcm + if (mode == MY_AES_GCM) + return source_length + MY_AES_BLOCK_SIZE; +#endif +#endif + return (source_length / MY_AES_BLOCK_SIZE + 1) * MY_AES_BLOCK_SIZE; +} + + +unsigned int my_aes_ctx_size(enum my_aes_mode) +{ + return MY_AES_CTX_SIZE; +} + #ifdef HAVE_YASSL #include int my_random_bytes(uchar* buf, int num) diff --git a/plugin/auth_gssapi/gssapi_server.cc b/plugin/auth_gssapi/gssapi_server.cc index ac75a4f1593..50c34ecc573 100644 --- a/plugin/auth_gssapi/gssapi_server.cc +++ b/plugin/auth_gssapi/gssapi_server.cc @@ -44,26 +44,30 @@ static char* get_default_principal_name() if(krb5_init_context(&context)) { - sql_print_warning("GSSAPI plugin : krb5_init_context failed"); + my_printf_error(0, "GSSAPI plugin : krb5_init_context failed", + ME_ERROR_LOG | ME_WARNING); goto cleanup; } if (krb5_sname_to_principal(context, NULL, "mariadb", KRB5_NT_SRV_HST, &principal)) { - sql_print_warning("GSSAPI plugin : krb5_sname_to_principal failed"); + my_printf_error(0, "GSSAPI plugin : krb5_sname_to_principal failed", + ME_ERROR_LOG | ME_WARNING); goto cleanup; } if (krb5_unparse_name(context, principal, &unparsed_name)) { - sql_print_warning("GSSAPI plugin : krb5_unparse_name failed"); + my_printf_error(0, "GSSAPI plugin : krb5_unparse_name failed", + ME_ERROR_LOG | ME_WARNING); goto cleanup; } /* Check for entry in keytab */ if (krb5_kt_read_service_key(context, NULL, principal, 0, (krb5_enctype)0, &key)) { - sql_print_warning("GSSAPI plugin : default principal '%s' not found in keytab", unparsed_name); + my_printf_error(0, "GSSAPI plugin : default principal '%s' not found in keytab", + ME_ERROR_LOG | ME_WARNING, unparsed_name); goto cleanup; } @@ -100,7 +104,8 @@ int plugin_init() /* import service principal from plain text */ if(srv_principal_name && srv_principal_name[0]) { - sql_print_information("GSSAPI plugin : using principal name '%s'", srv_principal_name); + my_printf_error(0, "GSSAPI plugin : using principal name '%s'", + ME_ERROR_LOG | ME_NOTE, srv_principal_name); principal_name_buf.length= strlen(srv_principal_name); principal_name_buf.value= srv_principal_name; major= gss_import_name(&minor, &principal_name_buf, GSS_C_NT_USER_NAME, &service_name); @@ -115,8 +120,6 @@ int plugin_init() service_name= GSS_C_NO_NAME; } - - /* Check if SPN configuration is OK */ major= gss_acquire_cred(&minor, service_name, GSS_C_INDEFINITE, GSS_C_NO_OID_SET, GSS_C_ACCEPT, &cred, NULL, diff --git a/plugin/auth_gssapi/sspi_server.cc b/plugin/auth_gssapi/sspi_server.cc index 1dfd2986aaa..d2c2ae7e4b9 100644 --- a/plugin/auth_gssapi/sspi_server.cc +++ b/plugin/auth_gssapi/sspi_server.cc @@ -284,8 +284,8 @@ int plugin_init() { srv_principal_name= get_default_principal_name(); } - sql_print_information("SSPI: using principal name '%s', mech '%s'", - srv_principal_name, srv_mech_name); + my_printf_error(0, "SSPI: using principal name '%s', mech '%s'", + ME_ERROR_LOG | ME_NOTE, srv_principal_name, srv_mech_name); ret = AcquireCredentialsHandle( srv_principal_name, diff --git a/plugin/aws_key_management/aws_key_management_plugin.cc b/plugin/aws_key_management/aws_key_management_plugin.cc index 83966b97c17..e6a14150ccb 100644 --- a/plugin/aws_key_management/aws_key_management_plugin.cc +++ b/plugin/aws_key_management/aws_key_management_plugin.cc @@ -175,7 +175,7 @@ static int plugin_init(void *p) client = new KMSClient(clientConfiguration); if (!client) { - sql_print_error("Can not initialize KMS client"); + my_printf_error(ER_UNKNOWN_ERROR, "Can not initialize KMS client", ME_ERROR_LOG,); DBUG_RETURN(-1); } @@ -254,12 +254,12 @@ static int load_key(KEY_INFO *info) if (!ret) { - sql_print_information("AWS KMS plugin: loaded key %u, version %u, key length %u bit", + my_printf_error(ER_UNKNOWN_ERROR, "AWS KMS plugin: loaded key %u, version %u, key length %u bit", ME_ERROR_LOG | ER_NOTE, info->key_id, info->key_version,(uint)info->length*8); } else { - sql_print_warning("AWS KMS plugin: key %u, version %u could not be decrypted", + my_printf_error(ER_UNKNOWN_ERROR, "AWS KMS plugin: key %u, version %u could not be decrypted", ME_ERROR_LOG | ER_WARNING, info->key_id, info->key_version); } DBUG_RETURN(ret); @@ -344,13 +344,13 @@ static int aws_decrypt_key(const char *path, KEY_INFO *info) ifstream ifs(path, ios::binary | ios::ate); if (!ifs.good()) { - sql_print_error("can't open file %s", path); + my_printf_error(ER_UNKNOWN_ERROR, "can't open file %s", ME_ERROR_LOG, path); DBUG_RETURN(-1); } size_t pos = (size_t)ifs.tellg(); if (!pos || pos == SIZE_T_MAX) { - sql_print_error("invalid key file %s", path); + my_printf_error(ER_UNKNOWN_ERROR, "invalid key file %s", ME_ERROR_LOG, path); DBUG_RETURN(-1); } std::vector contents(pos); @@ -364,7 +364,7 @@ static int aws_decrypt_key(const char *path, KEY_INFO *info) DecryptOutcome outcome = client->Decrypt(request); if (!outcome.IsSuccess()) { - sql_print_error("AWS KMS plugin: Decrypt failed for %s : %s", path, + my_printf_error(ER_UNKNOWN_ERROR, "AWS KMS plugin: Decrypt failed for %s : %s", ME_ERROR_LOG, path, outcome.GetError().GetMessage().c_str()); DBUG_RETURN(-1); } @@ -373,7 +373,7 @@ static int aws_decrypt_key(const char *path, KEY_INFO *info) if (len > (int)sizeof(info->data)) { - sql_print_error("AWS KMS plugin: encoding key too large for %s", path); + my_printf_error(ER_UNKNOWN_ERROR, "AWS KMS plugin: encoding key too large for %s", ME_ERROR_LOG, path); DBUG_RETURN(ENCRYPTION_KEY_BUFFER_TOO_SMALL); } memcpy(info->data, plaintext.GetUnderlyingData(), len); @@ -395,7 +395,7 @@ static int aws_generate_datakey(uint keyid, uint version) outcome= client->GenerateDataKeyWithoutPlaintext(request); if (!outcome.IsSuccess()) { - sql_print_error("AWS KMS plugin : GenerateDataKeyWithoutPlaintext failed : %s - %s", + my_printf_error(ER_UNKNOWN_ERROR, "AWS KMS plugin : GenerateDataKeyWithoutPlaintext failed : %s - %s", ME_ERROR_LOG, outcome.GetError().GetExceptionName().c_str(), outcome.GetError().GetMessage().c_str()); DBUG_RETURN(-1); @@ -409,19 +409,19 @@ static int aws_generate_datakey(uint keyid, uint version) int fd= my_open(filename, O_RDWR | O_CREAT, 0); if (fd < 0) { - sql_print_error("AWS KMS plugin: Can't create file %s", filename); + my_printf_error(ER_UNKNOWN_ERROR, "AWS KMS plugin: Can't create file %s", ME_ERROR_LOG, filename); DBUG_RETURN(-1); } size_t len= byteBuffer.GetLength(); if (my_write(fd, byteBuffer.GetUnderlyingData(), len, 0) != len) { - sql_print_error("AWS KMS plugin: can't write to %s", filename); + my_printf_error(ER_UNKNOWN_ERROR, "AWS KMS plugin: can't write to %s", ME_ERROR_LOG, filename); my_close(fd, 0); my_delete(filename, 0); DBUG_RETURN(-1); } my_close(fd, 0); - sql_print_information("AWS KMS plugin: generated encrypted datakey for key id=%u, version=%u", + my_printf_error(ER_UNKNOWN_ERROR, "AWS KMS plugin: generated encrypted datakey for key id=%u, version=%u", ME_ERROR_LOG | ER_NOTE, keyid, version); DBUG_RETURN(0); } diff --git a/plugin/example_key_management/example_key_management_plugin.cc b/plugin/example_key_management/example_key_management_plugin.cc index 2b417866406..dc1e2038dd2 100644 --- a/plugin/example_key_management/example_key_management_plugin.cc +++ b/plugin/example_key_management/example_key_management_plugin.cc @@ -114,14 +114,32 @@ static int example_key_management_plugin_deinit(void *p) return 0; } + +static int ctx_update(void *ctx, const unsigned char *src, unsigned int slen, + unsigned char *dst, unsigned int *dlen) +{ + return my_aes_crypt_update(ctx, src, slen, dst, dlen); +} + + +int ctx_finish(void *ctx, unsigned char *dst, unsigned int *dlen) +{ + return my_aes_crypt_finish(ctx, dst, dlen); +} + +static uint ctx_size(unsigned int , unsigned int key_version) +{ + return my_aes_ctx_size(mode(key_version)); +} + struct st_mariadb_encryption example_key_management_plugin= { MariaDB_ENCRYPTION_INTERFACE_VERSION, get_latest_key_version, get_key, - (uint (*)(unsigned int, unsigned int))my_aes_ctx_size, + ctx_size, ctx_init, - my_aes_crypt_update, - my_aes_crypt_finish, + ctx_update, + ctx_finish, get_length }; diff --git a/plugin/file_key_management/file_key_management_plugin.cc b/plugin/file_key_management/file_key_management_plugin.cc index 5872a070813..2c3e22c02d2 100644 --- a/plugin/file_key_management/file_key_management_plugin.cc +++ b/plugin/file_key_management/file_key_management_plugin.cc @@ -13,7 +13,8 @@ along with this program; if not, write to the Free Software Foundation, Inc., 51 Franklin St, Fifth Floor, Boston, MA 02110-1301 USA */ - +#include +#include #include "parser.h" #include #include @@ -146,20 +147,37 @@ static int ctx_init(void *ctx, const unsigned char* key, unsigned int klen, return my_aes_crypt_init(ctx, mode(flags), flags, key, klen, iv, ivlen); } +static int ctx_update(void *ctx, const unsigned char *src, unsigned int slen, + unsigned char *dst, unsigned int *dlen) +{ + return my_aes_crypt_update(ctx, src, slen, dst, dlen); +} + + +static int ctx_finish(void *ctx, unsigned char *dst, unsigned int *dlen) +{ + return my_aes_crypt_finish(ctx, dst, dlen); +} + static unsigned int get_length(unsigned int slen, unsigned int key_id, unsigned int key_version) { return my_aes_get_size(mode(0), slen); } +static uint ctx_size(uint, uint) +{ + return my_aes_ctx_size(mode(0)); +} + struct st_mariadb_encryption file_key_management_plugin= { MariaDB_ENCRYPTION_INTERFACE_VERSION, get_latest_version, get_key_from_key_file, - (uint (*)(unsigned int, unsigned int))my_aes_ctx_size, + ctx_size, ctx_init, - my_aes_crypt_update, - my_aes_crypt_finish, + ctx_update, + ctx_finish, get_length }; diff --git a/plugin/file_key_management/parser.cc b/plugin/file_key_management/parser.cc index 294ccc9ff79..facea9fad25 100644 --- a/plugin/file_key_management/parser.cc +++ b/plugin/file_key_management/parser.cc @@ -232,7 +232,7 @@ bool Parser::parse_file(Dynamic_array *keys, const char *secret) void Parser::report_error(const char *reason, uint position) { my_printf_error(EE_READ, "%s at %s line %u, column %u", - MYF(ME_NOREFRESH), reason, filename, line_number, position + 1); + ME_ERROR_LOG, reason, filename, line_number, position + 1); } /* @@ -300,9 +300,8 @@ char* Parser::read_and_decrypt_file(const char *secret) { if (!filename || !filename[0]) { - my_printf_error(EE_CANT_OPEN_STREAM, - "file-key-management-filename is not set", - MYF(ME_NOREFRESH)); + my_printf_error(EE_CANT_OPEN_STREAM, "file-key-management-filename is not set", + ME_ERROR_LOG); goto err0; } @@ -351,7 +350,7 @@ char* Parser::read_and_decrypt_file(const char *secret) iv, OpenSSL_iv_len)) { - my_printf_error(EE_READ, "Cannot decrypt %s. Wrong key?", MYF(ME_NOREFRESH), filename); + my_printf_error(EE_READ, "Cannot decrypt %s. Wrong key?", ME_ERROR_LOG, filename); goto err3; } @@ -361,7 +360,7 @@ char* Parser::read_and_decrypt_file(const char *secret) } else if (*secret) { - my_printf_error(EE_READ, "Cannot decrypt %s. Not encrypted", MYF(ME_NOREFRESH), filename); + my_printf_error(EE_READ, "Cannot decrypt %s. Not encrypted", ME_ERROR_LOG, filename); goto err2; } @@ -378,4 +377,3 @@ err1: err0: return NULL; } - diff --git a/sql/innodb_priv.h b/sql/innodb_priv.h index ec85aa352f8..27aa9ac8645 100644 --- a/sql/innodb_priv.h +++ b/sql/innodb_priv.h @@ -28,6 +28,7 @@ void localtime_to_TIME(MYSQL_TIME *to, struct tm *from); uint strconvert(CHARSET_INFO *from_cs, const char *from, uint from_length, CHARSET_INFO *to_cs, char *to, uint to_length, uint *errors); + void sql_print_error(const char *format, ...); #define thd_binlog_pos(X, Y, Z) mysql_bin_log_commit_pos(X, Z, Y) diff --git a/sql/sql_plugin_services.ic b/sql/sql_plugin_services.ic index 8e651d64ba1..572ceacbd7e 100644 --- a/sql/sql_plugin_services.ic +++ b/sql/sql_plugin_services.ic @@ -133,7 +133,7 @@ static struct base64_service_st base64_handler= { base64_decode }; -static struct thd_error_context_service_st thd_error_conext_handler= { +static struct thd_error_context_service_st thd_error_context_handler= { thd_get_error_message, thd_get_error_number, thd_get_error_row, @@ -196,6 +196,24 @@ static struct encryption_scheme_service_st encryption_scheme_handler= encryption_scheme_decrypt }; +static struct my_crypt_service_st crypt_handler= +{ + my_aes_crypt_init, + my_aes_crypt_update, + my_aes_crypt_finish, + my_aes_crypt, + my_aes_get_size, + my_aes_ctx_size, + my_random_bytes +}; + +static struct my_print_error_service_st my_print_error_handler= +{ + my_error, + my_printf_error, + my_printv_error +}; + static struct st_service_ref list_of_services[]= { { "base64_service", VERSION_base64, &base64_handler }, @@ -203,19 +221,21 @@ static struct st_service_ref list_of_services[]= { "encryption_scheme_service", VERSION_encryption_scheme, &encryption_scheme_handler }, { "encryption_service", VERSION_encryption, &encryption_handler }, { "logger_service", VERSION_logger, &logger_service_handler }, + { "my_crypt_service", VERSION_my_crypt, &crypt_handler}, { "my_md5_service", VERSION_my_md5, &my_md5_handler}, + { "my_print_error_service", VERSION_my_print_error, &my_print_error_handler}, { "my_sha1_service", VERSION_my_sha1, &my_sha1_handler}, { "my_sha2_service", VERSION_my_sha2, &my_sha2_handler}, { "my_snprintf_service", VERSION_my_snprintf, &my_snprintf_handler }, { "progress_report_service", VERSION_progress_report, &progress_report_handler }, { "thd_alloc_service", VERSION_thd_alloc, &thd_alloc_handler }, { "thd_autoinc_service", VERSION_thd_autoinc, &thd_autoinc_handler }, - { "thd_error_context_service", VERSION_thd_error_context, &thd_error_conext_handler }, + { "thd_error_context_service", VERSION_thd_error_context, &thd_error_context_handler }, { "thd_kill_statement_service", VERSION_kill_statement, &thd_kill_statement_handler }, { "thd_rnd_service", VERSION_thd_rnd, &thd_rnd_handler }, { "thd_specifics_service", VERSION_thd_specifics, &thd_specifics_handler }, { "thd_timezone_service", VERSION_thd_timezone, &thd_timezone_handler }, { "thd_wait_service", VERSION_thd_wait, &thd_wait_handler }, - { "wsrep_service", VERSION_wsrep, &wsrep_handler }, + { "wsrep_service", VERSION_wsrep, &wsrep_handler } }; diff --git a/sql/unireg.h b/sql/unireg.h index 10751b6ec93..e1947d8d3cd 100644 --- a/sql/unireg.h +++ b/sql/unireg.h @@ -53,8 +53,8 @@ #define ER_THD(thd,X) ((thd)->variables.errmsgs[(X) - ER_ERROR_FIRST]) #define ER_THD_OR_DEFAULT(thd,X) ((thd) ? ER_THD(thd, X) : ER_DEFAULT(X)) -#define ME_INFO (ME_HOLDTANG+ME_OLDWIN+ME_NOREFRESH) -#define ME_ERROR (ME_BELL+ME_OLDWIN+ME_NOREFRESH) +#define ME_INFO (ME_HOLDTANG | ME_NOREFRESH) +#define ME_ERROR (ME_BELL | ME_NOREFRESH) #define MYF_RW MYF(MY_WME+MY_NABP) /* Vid my_read & my_write */ #define SPECIAL_USE_LOCKS 1 /* Lock used databases */ diff --git a/storage/innobase/log/log0crypt.cc b/storage/innobase/log/log0crypt.cc index e6b5c845757..af9b2349187 100644 --- a/storage/innobase/log/log0crypt.cc +++ b/storage/innobase/log/log0crypt.cc @@ -25,8 +25,7 @@ Modified Jan Lindström jan.lindstrom@mariadb.com *******************************************************/ #include "m_string.h" #include "log0crypt.h" -#include -#include +#include #include "log0log.h" #include "srv0start.h" // for srv_start_lsn @@ -34,7 +33,6 @@ Modified Jan Lindström jan.lindstrom@mariadb.com #include "ha_prototypes.h" // IB_LOG_ -#include "my_crypt.h" /* Used for debugging */ // #define DEBUG_CRYPT 1 diff --git a/storage/xtradb/log/log0crypt.cc b/storage/xtradb/log/log0crypt.cc index e6b5c845757..f6c1416d81a 100644 --- a/storage/xtradb/log/log0crypt.cc +++ b/storage/xtradb/log/log0crypt.cc @@ -25,8 +25,7 @@ Modified Jan Lindström jan.lindstrom@mariadb.com *******************************************************/ #include "m_string.h" #include "log0crypt.h" -#include -#include +#include #include "log0log.h" #include "srv0start.h" // for srv_start_lsn @@ -34,8 +33,6 @@ Modified Jan Lindström jan.lindstrom@mariadb.com #include "ha_prototypes.h" // IB_LOG_ -#include "my_crypt.h" - /* Used for debugging */ // #define DEBUG_CRYPT 1 #define UNENCRYPTED_KEY_VER 0 From 7bf409593e624deb00265e21e0d4579ce9fa7999 Mon Sep 17 00:00:00 2001 From: Vladislav Vaintroub Date: Tue, 25 Apr 2017 15:44:18 +0200 Subject: [PATCH 05/28] MDEV-11660 Make encryption plugins "pure" Do not exporting mysqld entry points directly. This is needed for mariabackup, to load encryption plugins on Windows. All plugins are "pure" by default. To mark plugin "impure" it should use RECOMPILE_FOR_EMBEDDED or STORAGE_ENGINE keyword. --- cmake/plugin.cmake | 4 +- .../suite/encryption/r/filekeys_syntax.result | 2 +- plugin/auth_dialog/CMakeLists.txt | 5 +- .../aws_key_management_plugin.cc | 145 +++++++++--------- plugin/daemon_example/CMakeLists.txt | 5 +- .../example_key_management_plugin.cc | 13 +- .../file_key_management_plugin.cc | 18 +-- plugin/file_key_management/parser.cc | 106 ++++++++----- plugin/file_key_management/parser.h | 6 +- plugin/handler_socket/CMakeLists.txt | 2 +- plugin/locale_info/CMakeLists.txt | 2 +- plugin/metadata_lock_info/CMakeLists.txt | 3 +- plugin/qc_info/CMakeLists.txt | 2 +- plugin/query_response_time/CMakeLists.txt | 3 +- plugin/semisync/CMakeLists.txt | 6 +- plugin/server_audit/CMakeLists.txt | 5 +- plugin/wsrep_info/CMakeLists.txt | 2 +- 17 files changed, 182 insertions(+), 147 deletions(-) diff --git a/cmake/plugin.cmake b/cmake/plugin.cmake index 8615d6ed4b2..3e7c5804f77 100644 --- a/cmake/plugin.cmake +++ b/cmake/plugin.cmake @@ -201,13 +201,15 @@ MACRO(MYSQL_ADD_PLUGIN) # executable to the linker command line (it would result into link error). # Thus we skip TARGET_LINK_LIBRARIES on Linux, as it would only generate # an additional dependency. - IF(NOT ARG_CLIENT) + IF(ARG_RECOMPILE_FOR_EMBEDDED OR ARG_STORAGE_ENGINE) IF(MSVC) ADD_DEPENDENCIES(${target} gen_mysqld_lib) TARGET_LINK_LIBRARIES(${target} mysqld_import_lib) ELSEIF(NOT CMAKE_SYSTEM_NAME STREQUAL "Linux") TARGET_LINK_LIBRARIES (${target} mysqld) ENDIF() + ELSEIF(CMAKE_SYSTEM_NAME STREQUAL "Linux") + TARGET_LINK_LIBRARIES (${target} "-Wl,--no-undefined") ENDIF() IF(ARG_LINK_LIBRARIES) diff --git a/mysql-test/suite/encryption/r/filekeys_syntax.result b/mysql-test/suite/encryption/r/filekeys_syntax.result index eb8119bc4f5..a64d21eedbe 100644 --- a/mysql-test/suite/encryption/r/filekeys_syntax.result +++ b/mysql-test/suite/encryption/r/filekeys_syntax.result @@ -28,7 +28,7 @@ select plugin_status from information_schema.plugins where plugin_name = 'file_key_management'; plugin_status install soname 'file_key_management'; -ERROR HY000: Invalid key id at MYSQL_TMP_DIR/keys.txt line 2, column 11 +ERROR HY000: Invalid key id at MYSQL_TMP_DIR/keys.txt line 2, column 10 call mtr.add_suppression("Invalid key id"); call mtr.add_suppression("Plugin 'file_key_management' init function returned error"); call mtr.add_suppression("Plugin 'file_key_management' registration.*failed"); diff --git a/plugin/auth_dialog/CMakeLists.txt b/plugin/auth_dialog/CMakeLists.txt index 771bc615bd5..d16007405dc 100644 --- a/plugin/auth_dialog/CMakeLists.txt +++ b/plugin/auth_dialog/CMakeLists.txt @@ -15,5 +15,8 @@ # Foundation, Inc., 51 Franklin St, Fifth Floor, Boston, MA 02110-1301 USA ADD_DEFINITIONS(-DNO_GET_TTY_PASSWORD) +IF(LIBDL) + SET(LLDL LINK_LIBRARIES dl) +ENDIF() MYSQL_ADD_PLUGIN(dialog dialog.c ${CMAKE_SOURCE_DIR}/libmysql/get_password.c - MODULE_ONLY CLIENT COMPONENT ClientPlugins) + MODULE_ONLY CLIENT ${LLDL} COMPONENT ClientPlugins) diff --git a/plugin/aws_key_management/aws_key_management_plugin.cc b/plugin/aws_key_management/aws_key_management_plugin.cc index e6a14150ccb..dd0c7021fca 100644 --- a/plugin/aws_key_management/aws_key_management_plugin.cc +++ b/plugin/aws_key_management/aws_key_management_plugin.cc @@ -16,9 +16,7 @@ #include -#include -#include -#include +#include #include #include #include @@ -33,6 +31,10 @@ #include #include +#ifndef _WIN32 +#include +#endif + #include #include #include @@ -48,9 +50,6 @@ using namespace std; using namespace Aws::KMS; using namespace Aws::KMS::Model; using namespace Aws::Utils::Logging; -extern void sql_print_error(const char *format, ...); -extern void sql_print_warning(const char *format, ...); -extern void sql_print_information(const char *format, ...); /* Plaintext key info struct */ @@ -90,14 +89,8 @@ static int extract_id_and_version(const char *name, uint *id, uint *ver); static unsigned int get_latest_key_version(unsigned int key_id); static unsigned int get_latest_key_version_nolock(unsigned int key_id); static int load_key(KEY_INFO *info); +static std::mutex mtx; -/* Mutex to serialize access to caches */ -static mysql_mutex_t mtx; - -#ifdef HAVE_PSI_INTERFACE -static uint mtx_key; -static PSI_mutex_info mtx_info = {&mtx_key, "mtx", 0}; -#endif static Aws::KMS::KMSClient *client; @@ -140,6 +133,33 @@ protected: } }; +/* Get list of files in current directory */ +static vector traverse_current_directory() +{ + vector v; +#ifdef _WIN32 + WIN32_FIND_DATA find_data; + HANDLE h= FindFirstFile("*.*", &find_data); + if (h == INVALID_HANDLE_VALUE) + return v; + do + { + v.push_back(find_data.cFileName); + } + while (FindNextFile(h, &find_data)); + FindClose(h); +#else + DIR *dir = opendir("."); + if (!dir) + return v; + struct dirent *e; + while ((e= readdir(dir)) + v.push_back(e->d_name); + closedir(dir); +#endif + return v; +} + Aws::SDKOptions sdkOptions; /* @@ -150,7 +170,6 @@ Aws::SDKOptions sdkOptions; */ static int plugin_init(void *p) { - DBUG_ENTER("plugin_init"); #ifdef HAVE_YASSL sdkOptions.cryptoOptions.initAndCleanupOpenSSL = true; @@ -176,46 +195,33 @@ static int plugin_init(void *p) if (!client) { my_printf_error(ER_UNKNOWN_ERROR, "Can not initialize KMS client", ME_ERROR_LOG,); - DBUG_RETURN(-1); + return -1; } -#ifdef HAVE_PSI_INTERFACE - mysql_mutex_register("aws_key_management", &mtx_info, 1); -#endif - mysql_mutex_init(mtx_key, &mtx, NULL); - - MY_DIR *dirp = my_dir(".", MYF(0)); - if (!dirp) - { - sql_print_error("Can't scan current directory"); - DBUG_RETURN(-1); - } - for (unsigned int i=0; i < dirp->number_of_files; i++) + vector files= traverse_current_directory(); + for (size_t i=0; i < files.size(); i++) { KEY_INFO info; - if (extract_id_and_version(dirp->dir_entry[i].name, &info.key_id, &info.key_version) == 0) + if (extract_id_and_version(files[i].c_str(), &info.key_id, &info.key_version) == 0) { key_info_cache[KEY_ID_AND_VERSION(info.key_id, info.key_version)]= info; latest_version_cache[info.key_id]= max(info.key_version, latest_version_cache[info.key_id]); } } - my_dirend(dirp); - DBUG_RETURN(0); + return 0; } static int plugin_deinit(void *p) { - DBUG_ENTER("plugin_deinit"); latest_version_cache.clear(); key_info_cache.clear(); - mysql_mutex_destroy(&mtx); delete client; ShutdownAWSLogging(); Aws::ShutdownAPI(sdkOptions); - DBUG_RETURN(0); + return 0; } /* Generate filename to store the ciphered key */ @@ -242,8 +248,7 @@ static int load_key(KEY_INFO *info) { int ret; char path[256]; - DBUG_ENTER("load_key"); - DBUG_PRINT("enter", ("id=%u,ver=%u", info->key_id, info->key_version)); + format_keyfile_name(path, sizeof(path), info->key_id, info->key_version); ret= aws_decrypt_key(path, info); if (ret) @@ -262,7 +267,7 @@ static int load_key(KEY_INFO *info) my_printf_error(ER_UNKNOWN_ERROR, "AWS KMS plugin: key %u, version %u could not be decrypted", ME_ERROR_LOG | ER_WARNING, info->key_id, info->key_version); } - DBUG_RETURN(ret); + return ret; } @@ -281,19 +286,17 @@ static int load_key(KEY_INFO *info) static unsigned int get_latest_key_version(unsigned int key_id) { unsigned int ret; - DBUG_ENTER("get_latest_key_version"); - mysql_mutex_lock(&mtx); + mtx.lock(); ret= get_latest_key_version_nolock(key_id); - mysql_mutex_unlock(&mtx); - DBUG_PRINT("info", ("key=%u,ret=%u", key_id, ret)); - DBUG_RETURN(ret); + mtx.unlock(); + return ret; } static unsigned int get_latest_key_version_nolock(unsigned int key_id) { KEY_INFO info; uint ver; - DBUG_ENTER("get_latest_key_version_nolock"); + ver= latest_version_cache[key_id]; if (ver > 0) { @@ -302,13 +305,13 @@ static unsigned int get_latest_key_version_nolock(unsigned int key_id) if (info.load_failed) { /* Decryption failed previously, don't retry */ - DBUG_RETURN(ENCRYPTION_KEY_VERSION_INVALID); + return(ENCRYPTION_KEY_VERSION_INVALID); } else if (ver > 0) { /* Key exists already, return it*/ if (info.length > 0) - DBUG_RETURN(ver); + return(ver); } else // (ver == 0) { @@ -318,18 +321,18 @@ static unsigned int get_latest_key_version_nolock(unsigned int key_id) my_printf_error(ER_UNKNOWN_ERROR, "Can't generate encryption key %u, because 'aws_key_management_master_key_id' parameter is not set", MYF(0), key_id); - DBUG_RETURN(ENCRYPTION_KEY_VERSION_INVALID); + return(ENCRYPTION_KEY_VERSION_INVALID); } if (aws_generate_datakey(key_id, 1) != 0) - DBUG_RETURN(ENCRYPTION_KEY_VERSION_INVALID); + return(ENCRYPTION_KEY_VERSION_INVALID); info.key_id= key_id; info.key_version= 1; info.length= 0; } if (load_key(&info)) - DBUG_RETURN(ENCRYPTION_KEY_VERSION_INVALID); - DBUG_RETURN(info.key_version); + return(ENCRYPTION_KEY_VERSION_INVALID); + return(info.key_version); } @@ -338,20 +341,19 @@ static unsigned int get_latest_key_version_nolock(unsigned int key_id) */ static int aws_decrypt_key(const char *path, KEY_INFO *info) { - DBUG_ENTER("aws_decrypt_key"); /* Read file content into memory */ ifstream ifs(path, ios::binary | ios::ate); if (!ifs.good()) { my_printf_error(ER_UNKNOWN_ERROR, "can't open file %s", ME_ERROR_LOG, path); - DBUG_RETURN(-1); + return(-1); } size_t pos = (size_t)ifs.tellg(); if (!pos || pos == SIZE_T_MAX) { my_printf_error(ER_UNKNOWN_ERROR, "invalid key file %s", ME_ERROR_LOG, path); - DBUG_RETURN(-1); + return(-1); } std::vector contents(pos); ifs.seekg(0, ios::beg); @@ -366,7 +368,7 @@ static int aws_decrypt_key(const char *path, KEY_INFO *info) { my_printf_error(ER_UNKNOWN_ERROR, "AWS KMS plugin: Decrypt failed for %s : %s", ME_ERROR_LOG, path, outcome.GetError().GetMessage().c_str()); - DBUG_RETURN(-1); + return(-1); } Aws::Utils::ByteBuffer plaintext = outcome.GetResult().GetPlaintext(); size_t len = plaintext.GetLength(); @@ -374,19 +376,17 @@ static int aws_decrypt_key(const char *path, KEY_INFO *info) if (len > (int)sizeof(info->data)) { my_printf_error(ER_UNKNOWN_ERROR, "AWS KMS plugin: encoding key too large for %s", ME_ERROR_LOG, path); - DBUG_RETURN(ENCRYPTION_KEY_BUFFER_TOO_SMALL); + return(ENCRYPTION_KEY_BUFFER_TOO_SMALL); } memcpy(info->data, plaintext.GetUnderlyingData(), len); info->length= len; - DBUG_RETURN(0); + return(0); } /* Generate a new datakey and store it a file */ static int aws_generate_datakey(uint keyid, uint version) { - - DBUG_ENTER("aws_generate_datakey"); GenerateDataKeyWithoutPlaintextRequest request; request.SetKeyId(master_key_id); request.SetKeySpec(DataKeySpecMapper::GetDataKeySpecForName(key_spec_names[key_spec])); @@ -398,7 +398,7 @@ static int aws_generate_datakey(uint keyid, uint version) my_printf_error(ER_UNKNOWN_ERROR, "AWS KMS plugin : GenerateDataKeyWithoutPlaintext failed : %s - %s", ME_ERROR_LOG, outcome.GetError().GetExceptionName().c_str(), outcome.GetError().GetMessage().c_str()); - DBUG_RETURN(-1); + return(-1); } string out; @@ -406,24 +406,24 @@ static int aws_generate_datakey(uint keyid, uint version) Aws::Utils::ByteBuffer byteBuffer = outcome.GetResult().GetCiphertextBlob(); format_keyfile_name(filename, sizeof(filename), keyid, version); - int fd= my_open(filename, O_RDWR | O_CREAT, 0); + int fd= open(filename, O_WRONLY |O_CREAT|O_BINARY, IF_WIN(_S_IREAD, S_IRUSR| S_IRGRP| S_IROTH)); if (fd < 0) { my_printf_error(ER_UNKNOWN_ERROR, "AWS KMS plugin: Can't create file %s", ME_ERROR_LOG, filename); - DBUG_RETURN(-1); + return(-1); } size_t len= byteBuffer.GetLength(); - if (my_write(fd, byteBuffer.GetUnderlyingData(), len, 0) != len) + if (write(fd, byteBuffer.GetUnderlyingData(), len) != len) { my_printf_error(ER_UNKNOWN_ERROR, "AWS KMS plugin: can't write to %s", ME_ERROR_LOG, filename); - my_close(fd, 0); - my_delete(filename, 0); - DBUG_RETURN(-1); + close(fd); + unlink(filename); + return(-1); } - my_close(fd, 0); + close(fd); my_printf_error(ER_UNKNOWN_ERROR, "AWS KMS plugin: generated encrypted datakey for key id=%u, version=%u", ME_ERROR_LOG | ER_NOTE, keyid, version); - DBUG_RETURN(0); + return(0); } /* Key rotation for a single key */ @@ -479,7 +479,7 @@ static void update_rotate(MYSQL_THD, struct st_mysql_sys_var *, void *, const vo "aws_key_management_master_key_id must be set to generate new data keys", MYF(ME_JUST_WARNING)); return; } - mysql_mutex_lock(&mtx); + mtx.lock(); rotate_key= *(int *)val; switch (rotate_key) { @@ -493,7 +493,7 @@ static void update_rotate(MYSQL_THD, struct st_mysql_sys_var *, void *, const vo break; } rotate_key= 0; - mysql_mutex_unlock(&mtx); + mtx.unlock(); } static unsigned int get_key( @@ -504,8 +504,7 @@ static unsigned int get_key( { KEY_INFO info; - DBUG_ENTER("get_key"); - mysql_mutex_lock(&mtx); + mtx.lock(); info= key_info_cache[KEY_ID_AND_VERSION(key_id, version)]; if (info.length == 0 && !info.load_failed) { @@ -513,17 +512,17 @@ static unsigned int get_key( info.key_version= version; load_key(&info); } - mysql_mutex_unlock(&mtx); + mtx.unlock(); if (info.load_failed) - DBUG_RETURN(ENCRYPTION_KEY_VERSION_INVALID); + return(ENCRYPTION_KEY_VERSION_INVALID); if (*buflen < info.length) { *buflen= info.length; - DBUG_RETURN(ENCRYPTION_KEY_BUFFER_TOO_SMALL); + return(ENCRYPTION_KEY_BUFFER_TOO_SMALL); } *buflen= info.length; memcpy(dstbuf, info.data, info.length); - DBUG_RETURN(0); + return(0); } diff --git a/plugin/daemon_example/CMakeLists.txt b/plugin/daemon_example/CMakeLists.txt index 3d674c4ef3e..28fbff78302 100644 --- a/plugin/daemon_example/CMakeLists.txt +++ b/plugin/daemon_example/CMakeLists.txt @@ -13,7 +13,8 @@ # along with this program; if not, write to the Free Software # Foundation, Inc., 51 Franklin St, Fifth Floor, Boston, MA 02110-1301 USA -MYSQL_ADD_PLUGIN(daemon_example daemon_example.cc +MYSQL_ADD_PLUGIN(daemon_example daemon_example.cc RECOMPILE_FOR_EMBEDDED MODULE_ONLY MODULE_OUTPUT_NAME "libdaemon_example" COMPONENT Test) -INSTALL(FILES daemon_example.ini DESTINATION ${INSTALL_PLUGINDIR} COMPONENT Test) +INSTALL(FILES daemon_example.ini DESTINATION ${INSTALL_PLUGINDIR} + COMPONENT Test) diff --git a/plugin/example_key_management/example_key_management_plugin.cc b/plugin/example_key_management/example_key_management_plugin.cc index dc1e2038dd2..2b43729e3f7 100644 --- a/plugin/example_key_management/example_key_management_plugin.cc +++ b/plugin/example_key_management/example_key_management_plugin.cc @@ -28,18 +28,24 @@ #include #include #include -#include #include /* rotate key randomly between 45 and 90 seconds */ #define KEY_ROTATION_MIN 45 #define KEY_ROTATION_MAX 90 -static struct my_rnd_struct seed; static time_t key_version = 0; static time_t next_key_version = 0; static pthread_mutex_t mutex; + +/* Random double value in 0..1 range */ +static double double_rnd() +{ + return ((double)rand()) / RAND_MAX; +} + + static unsigned int get_latest_key_version(unsigned int key_id) { @@ -50,7 +56,7 @@ get_latest_key_version(unsigned int key_id) key_version = now; unsigned int interval = KEY_ROTATION_MAX - KEY_ROTATION_MIN; next_key_version = (time_t) (now + KEY_ROTATION_MIN + - my_rnd(&seed) * interval); + double_rnd() * interval); } pthread_mutex_unlock(&mutex); @@ -101,7 +107,6 @@ static unsigned int get_length(unsigned int slen, unsigned int key_id, static int example_key_management_plugin_init(void *p) { /* init */ - my_rnd_init(&seed, time(0), 0); pthread_mutex_init(&mutex, NULL); get_latest_key_version(1); diff --git a/plugin/file_key_management/file_key_management_plugin.cc b/plugin/file_key_management/file_key_management_plugin.cc index 2c3e22c02d2..f6f7083d5f7 100644 --- a/plugin/file_key_management/file_key_management_plugin.cc +++ b/plugin/file_key_management/file_key_management_plugin.cc @@ -66,22 +66,14 @@ static struct st_mysql_sys_var* settings[] = { NULL }; -Dynamic_array keys(static_cast(0)); +std::map keys; static keyentry *get_key(unsigned int key_id) { - keyentry *a= keys.front(), *b= keys.back() + 1, *c; - while (b - a > 1) - { - c= a + (b - a)/2; - if (c->id == key_id) - return c; - else if (c->id < key_id) - a= c; - else - b= c; - } - return a->id == key_id ? a : 0; + keyentry &key= keys[key_id]; + if (key.id == 0) + return 0; + return &key; } /* the version is always the same, no automatic key rotation */ diff --git a/plugin/file_key_management/parser.cc b/plugin/file_key_management/parser.cc index facea9fad25..ac78186a488 100644 --- a/plugin/file_key_management/parser.cc +++ b/plugin/file_key_management/parser.cc @@ -143,13 +143,13 @@ void Parser::bytes_to_key(const unsigned char *salt, const char *input, } -bool Parser::parse(Dynamic_array *keys) +bool Parser::parse(std::map *keys) { const char *secret= filekey; char buf[MAX_SECRET_SIZE + 1]; //If secret starts with FILE: interpret the secret as a filename. - if (is_prefix(filekey, FILE_PREFIX)) + if (strncmp(filekey, FILE_PREFIX,sizeof(FILE_PREFIX) -1) == 0) { if (read_filekey(filekey + sizeof(FILE_PREFIX) - 1, buf)) return 1; @@ -166,22 +166,26 @@ bool Parser::parse(Dynamic_array *keys) bool Parser::read_filekey(const char *filekey, char *secret) { - int f= my_open(filekey, O_RDONLY, MYF(MY_WME)); + int f= open(filekey, O_RDONLY|O_BINARY); if (f == -1) + { + my_error(EE_FILENOTFOUND,ME_ERROR_LOG, filekey, errno); return 1; - int len= my_read(f, (uchar*)secret, MAX_SECRET_SIZE, MYF(MY_WME)); - my_close(f, MYF(MY_WME)); + } + + int len= read(f, secret, MAX_SECRET_SIZE); if (len <= 0) + { + my_error(EE_READ,ME_ERROR_LOG, filekey, errno); + close(f); return 1; + } + close(f); while (secret[len - 1] == '\r' || secret[len - 1] == '\n') len--; secret[len]= '\0'; return 0; } -static int sort_keys(const keyentry *k1, const keyentry *k2) -{ - return k1->id < k2->id ? -1 : k1->id > k2->id; -} /** Get the keys from the key file and decrypt it with the @@ -191,7 +195,7 @@ static int sort_keys(const keyentry *k1, const keyentry *k2) @return 0 when ok, 1 for an error */ -bool Parser::parse_file(Dynamic_array *keys, const char *secret) +bool Parser::parse_file(std::map *keys, const char *secret) { char *buffer= read_and_decrypt_file(secret); @@ -208,19 +212,16 @@ bool Parser::parse_file(Dynamic_array *keys, const char *secret) case 1: // comment break; case -1: // error - my_free(buffer); + free(buffer); return 1; case 0: - if (keys->push(key)) - return 1; + (*keys)[key.id] = key; break; } } - keys->sort(sort_keys); - my_free(buffer); - - if (keys->elements() == 0 || keys->at(0).id != 1) + free(buffer); + if (keys->size() == 0 || (*keys)[1].id == 0) { report_error("System key id 1 is missing", 0); return 1; @@ -247,16 +248,25 @@ int Parser::parse_line(char **line_ptr, keyentry *key) while (isspace(*p) && *p != '\n') p++; if (*p != '#' && *p != '\n') { - int error; - p+= 100; // the number will surely end here (on a non-digit or with an overflow) - longlong id= my_strtoll10(p - 100, &p, &error); - if (error) + if (!isdigit(*p)) { report_error("Syntax error", p - *line_ptr); return -1; } - if (id < 1 || id > UINT_MAX32) + longlong id = 0; + while (isdigit(*p)) + { + id = id * 10 + *p - '0'; + if (id > UINT_MAX32) + { + report_error("Invalid key id", p - *line_ptr); + return -1; + } + p++; + } + + if (id < 1) { report_error("Invalid key id", p - *line_ptr); return -1; @@ -269,7 +279,7 @@ int Parser::parse_line(char **line_ptr, keyentry *key) } p++; - key->id= id; + key->id= (unsigned int)id; key->length=0; while (isxdigit(p[0]) && isxdigit(p[1]) && key->length < sizeof(key->key)) { @@ -295,9 +305,13 @@ int Parser::parse_line(char **line_ptr, keyentry *key) 'secret'. Store the content of the decrypted file in 'buffer'. The buffer has to be freed in the calling function. */ +#ifdef _WIN32 +#define lseek _lseeki64 +#endif char* Parser::read_and_decrypt_file(const char *secret) { + int f; if (!filename || !filename[0]) { my_printf_error(EE_CANT_OPEN_STREAM, "file-key-management-filename is not set", @@ -305,15 +319,21 @@ char* Parser::read_and_decrypt_file(const char *secret) goto err0; } - int f; - if ((f= my_open(filename, O_RDONLY, MYF(MY_WME))) < 0) + f= open(filename, O_RDONLY|O_BINARY, 0); + if (f < 0) + { + my_error(EE_FILENOTFOUND, ME_ERROR_LOG, filename, errno); goto err0; + } my_off_t file_size; - file_size= my_seek(f, 0, SEEK_END, MYF(MY_WME)); + file_size= lseek(f, 0, SEEK_END); - if (file_size == MY_FILEPOS_ERROR) + if (file_size == MY_FILEPOS_ERROR || (my_off_t)lseek(f, 0, SEEK_SET) == MY_FILEPOS_ERROR) + { + my_error(EE_CANT_SEEK, MYF(0), filename, errno); goto err1; + } if (file_size > MAX_KEY_FILE_SIZE) { @@ -323,29 +343,39 @@ char* Parser::read_and_decrypt_file(const char *secret) //Read file into buffer uchar *buffer; - buffer= (uchar*)my_malloc(file_size + 1, MYF(MY_WME)); + buffer= (uchar*)malloc((size_t)file_size + 1); if (!buffer) + { + my_error(EE_OUTOFMEMORY, ME_ERROR_LOG| ME_FATAL, file_size); goto err1; + } - if (my_pread(f, buffer, file_size, 0, MYF(MY_WME)) != file_size) + if (read(f, buffer, (int)file_size) != (int)file_size) + { + my_printf_error(EE_READ, + "read from %s failed, errno %d", + MYF(ME_ERROR_LOG|ME_FATAL), filename, errno); goto err2; + } // Check for file encryption uchar *decrypted; - if (file_size > OpenSSL_prefix_len && is_prefix((char*)buffer, OpenSSL_prefix)) + if (file_size > OpenSSL_prefix_len && strncmp((char*)buffer, OpenSSL_prefix, OpenSSL_prefix_len) == 0) { uchar key[OpenSSL_key_len]; uchar iv[OpenSSL_iv_len]; - decrypted= (uchar*)my_malloc(file_size, MYF(MY_WME)); + decrypted= (uchar*)malloc((size_t)file_size); if (!decrypted) + { + my_error(EE_OUTOFMEMORY, ME_ERROR_LOG | ME_FATAL, file_size); goto err2; - + } bytes_to_key(buffer + OpenSSL_prefix_len, secret, key, iv); uint32 d_size; if (my_aes_crypt(MY_AES_CBC, ENCRYPTION_FLAG_DECRYPT, buffer + OpenSSL_prefix_len + OpenSSL_salt_len, - file_size - OpenSSL_prefix_len - OpenSSL_salt_len, + (unsigned int)file_size - OpenSSL_prefix_len - OpenSSL_salt_len, decrypted, &d_size, key, OpenSSL_key_len, iv, OpenSSL_iv_len)) @@ -354,7 +384,7 @@ char* Parser::read_and_decrypt_file(const char *secret) goto err3; } - my_free(buffer); + free(buffer); buffer= decrypted; file_size= d_size; } @@ -365,15 +395,15 @@ char* Parser::read_and_decrypt_file(const char *secret) } buffer[file_size]= '\0'; - my_close(f, MYF(MY_WME)); + close(f); return (char*) buffer; err3: - my_free(decrypted); + free(decrypted); err2: - my_free(buffer); + free(buffer); err1: - my_close(f, MYF(MY_WME)); + close(f); err0: return NULL; } diff --git a/plugin/file_key_management/parser.h b/plugin/file_key_management/parser.h index c8349db70a0..627b7fd84a6 100644 --- a/plugin/file_key_management/parser.h +++ b/plugin/file_key_management/parser.h @@ -22,7 +22,7 @@ Created 09/15/2014 #include #include -#include +#include struct keyentry { unsigned int id; @@ -42,7 +42,7 @@ class Parser void bytes_to_key(const unsigned char *salt, const char *secret, unsigned char *key, unsigned char *iv); bool read_filekey(const char *filekey, char *secret); - bool parse_file(Dynamic_array *keys, const char *secret); + bool parse_file(std::map *keys, const char *secret); void report_error(const char *reason, unsigned int position); int parse_line(char **line_ptr, keyentry *key); char* read_and_decrypt_file(const char *secret); @@ -50,5 +50,5 @@ class Parser public: Parser(const char* fn, const char *fk) : filename(fn), filekey(fk), line_number(0) { } - bool parse(Dynamic_array *keys); + bool parse(std::map *keys); }; diff --git a/plugin/handler_socket/CMakeLists.txt b/plugin/handler_socket/CMakeLists.txt index 2e7caa80897..a10743210e9 100644 --- a/plugin/handler_socket/CMakeLists.txt +++ b/plugin/handler_socket/CMakeLists.txt @@ -34,6 +34,6 @@ SET(HANDLERSOCKET_SOURCES MYSQL_ADD_PLUGIN(handlersocket ${HANDLERSOCKET_SOURCES} MODULE_ONLY COMPONENT Server - LINK_LIBRARIES hsclient + LINK_LIBRARIES hsclient RECOMPILE_FOR_EMBEDDED ) diff --git a/plugin/locale_info/CMakeLists.txt b/plugin/locale_info/CMakeLists.txt index 1ace6619041..8f1dfa0d715 100644 --- a/plugin/locale_info/CMakeLists.txt +++ b/plugin/locale_info/CMakeLists.txt @@ -1,5 +1,5 @@ INCLUDE_DIRECTORIES(${CMAKE_SOURCE_DIR}/sql ${CMAKE_SOURCE_DIR}/regex ${CMAKE_SOURCE_DIR}/extra/yassl/include) -MYSQL_ADD_PLUGIN(LOCALES locale_info.cc) +MYSQL_ADD_PLUGIN(LOCALES locale_info.cc RECOMPILE_FOR_EMBEDDED) diff --git a/plugin/metadata_lock_info/CMakeLists.txt b/plugin/metadata_lock_info/CMakeLists.txt index 44393c09eb6..6b1f5108bf1 100644 --- a/plugin/metadata_lock_info/CMakeLists.txt +++ b/plugin/metadata_lock_info/CMakeLists.txt @@ -1,2 +1,3 @@ SET(METADATA_LOCK_INFO_SOURCES metadata_lock_info.cc) -MYSQL_ADD_PLUGIN(metadata_lock_info ${METADATA_LOCK_INFO_SOURCES} MODULE_OUTPUT_NAME "metadata_lock_info") +MYSQL_ADD_PLUGIN(metadata_lock_info ${METADATA_LOCK_INFO_SOURCES} + RECOMPILE_FOR_EMBEDDED) diff --git a/plugin/qc_info/CMakeLists.txt b/plugin/qc_info/CMakeLists.txt index d10f4547227..821ffb79225 100644 --- a/plugin/qc_info/CMakeLists.txt +++ b/plugin/qc_info/CMakeLists.txt @@ -2,4 +2,4 @@ INCLUDE_DIRECTORIES(${CMAKE_SOURCE_DIR}/sql ${PCRE_INCLUDES} ${CMAKE_SOURCE_DIR}/extra/yassl/include) -MYSQL_ADD_PLUGIN(QUERY_CACHE_INFO qc_info.cc) +MYSQL_ADD_PLUGIN(QUERY_CACHE_INFO qc_info.cc RECOMPILE_FOR_EMBEDDED) diff --git a/plugin/query_response_time/CMakeLists.txt b/plugin/query_response_time/CMakeLists.txt index f008d0256fc..112d72e429a 100644 --- a/plugin/query_response_time/CMakeLists.txt +++ b/plugin/query_response_time/CMakeLists.txt @@ -1,2 +1,3 @@ ADD_DEFINITIONS(-DHAVE_RESPONSE_TIME_DISTRIBUTION) -MYSQL_ADD_PLUGIN(QUERY_RESPONSE_TIME query_response_time.cc plugin.cc) +MYSQL_ADD_PLUGIN(QUERY_RESPONSE_TIME query_response_time.cc plugin.cc + RECOMPILE_FOR_EMBEDDED) diff --git a/plugin/semisync/CMakeLists.txt b/plugin/semisync/CMakeLists.txt index 33c0895e5e1..88998fb3093 100644 --- a/plugin/semisync/CMakeLists.txt +++ b/plugin/semisync/CMakeLists.txt @@ -17,10 +17,12 @@ SET(SEMISYNC_MASTER_SOURCES semisync.cc semisync_master.cc semisync_master_plugin.cc semisync.h semisync_master.h) -MYSQL_ADD_PLUGIN(semisync_master ${SEMISYNC_MASTER_SOURCES}) +MYSQL_ADD_PLUGIN(semisync_master ${SEMISYNC_MASTER_SOURCES} + RECOMPILE_FOR_EMBEDDED) SET(SEMISYNC_SLAVE_SOURCES semisync.cc semisync_slave.cc semisync_slave_plugin.cc semisync.h semisync_slave.h ) -MYSQL_ADD_PLUGIN(semisync_slave ${SEMISYNC_SLAVE_SOURCES}) +MYSQL_ADD_PLUGIN(semisync_slave ${SEMISYNC_SLAVE_SOURCES} + RECOMPILE_FOR_EMBEDDED) diff --git a/plugin/server_audit/CMakeLists.txt b/plugin/server_audit/CMakeLists.txt index 2c9964543bf..056a11f3753 100644 --- a/plugin/server_audit/CMakeLists.txt +++ b/plugin/server_audit/CMakeLists.txt @@ -13,7 +13,6 @@ # along with this program; if not, write to the Free Software # Foundation, Inc., 51 Franklin Street, Fifth Floor, Boston, MA 02111-1301 USA -SET(SERVER_AUDIT_SOURCES - server_audit.c test_audit_v4.c plugin_audit_v4.h) +SET(SOURCES server_audit.c test_audit_v4.c plugin_audit_v4.h) - MYSQL_ADD_PLUGIN(server_audit ${SERVER_AUDIT_SOURCES} MODULE_ONLY) +MYSQL_ADD_PLUGIN(server_audit ${SOURCES} MODULE_ONLY RECOMPILE_FOR_EMBEDDED) diff --git a/plugin/wsrep_info/CMakeLists.txt b/plugin/wsrep_info/CMakeLists.txt index 4dee10c34c1..34aee9fba2c 100644 --- a/plugin/wsrep_info/CMakeLists.txt +++ b/plugin/wsrep_info/CMakeLists.txt @@ -1,5 +1,5 @@ IF (WITH_WSREP) INCLUDE_DIRECTORIES(${CMAKE_SOURCE_DIR}/sql ${CMAKE_SOURCE_DIR}/wsrep) - MYSQL_ADD_PLUGIN(WSREP_INFO plugin.cc MODULE_ONLY) + MYSQL_ADD_PLUGIN(WSREP_INFO plugin.cc MODULE_ONLY RECOMPILE_FOR_EMBEDDED) ENDIF() From ec68f764f6f9dc4028a0af93b78449e76e972b9a Mon Sep 17 00:00:00 2001 From: Vladislav Vaintroub Date: Tue, 18 Apr 2017 17:09:28 +0000 Subject: [PATCH 06/28] MDEV-9566 prepare mysqltest for mariabackup - Do not throw output of exec command, if disable_result_log is set save and dump it if exec fails. Need tha to meaningfully analyze errors from mariabackup. - rmdir now removes the entire tree. need that because xtrabackup tests clean the whole directory. - all filesystem modifying commands now require the argument to be under MYSQLTEST_VARDIR or MYSQL_TMP_DIR. --- client/mysqltest.cc | 135 ++++++- mysql-test/include/write_var_to_file.inc | 3 +- mysql-test/r/mysql_plugin.result | 132 ------- mysql-test/t/mysql_plugin.test | 368 ------------------ mysql-test/t/mysqltest.test | 24 +- .../connect/t/secure_file_priv.test | 2 +- 6 files changed, 131 insertions(+), 533 deletions(-) delete mode 100644 mysql-test/r/mysql_plugin.result delete mode 100644 mysql-test/t/mysql_plugin.test diff --git a/client/mysqltest.cc b/client/mysqltest.cc index 2673db0e469..492411d49ca 100644 --- a/client/mysqltest.cc +++ b/client/mysqltest.cc @@ -3373,6 +3373,12 @@ void do_exec(struct st_command *command) #endif #endif + if (disable_result_log) + { + /* Collect stderr output as well, for the case app. crashes or returns error.*/ + dynstr_append(&ds_cmd, " 2>&1"); + } + DBUG_PRINT("info", ("Executing '%s' as '%s'", command->first_argument, ds_cmd.str)); @@ -3408,16 +3414,7 @@ void do_exec(struct st_command *command) len--; } #endif - if (disable_result_log) - { - if (len) - buf[len-1] = 0; - DBUG_PRINT("exec_result",("%s", buf)); - } - else - { - replace_dynstr_append_mem(ds_result, buf, len); - } + replace_dynstr_append_mem(ds_result, buf, len); } error= pclose(res_file); @@ -3427,7 +3424,7 @@ void do_exec(struct st_command *command) dynstr_free(&ds_sorted); } - if (error > 0) + if (error) { uint status= WEXITSTATUS(error); int i; @@ -3473,6 +3470,12 @@ void do_exec(struct st_command *command) } dynstr_free(&ds_cmd); + + if (disable_result_log) + { + /* Disable output in case of successful exit.*/ + dynstr_set(&ds_res,""); + } DBUG_VOID_RETURN; } @@ -3610,6 +3613,37 @@ void do_system(struct st_command *command) } +/* returns TRUE if path is inside a sandbox */ +bool is_sub_path(const char *path, size_t plen, const char *sandbox) +{ + size_t len= strlen(sandbox); + if (!sandbox || !len || plen <= len || memcmp(path, sandbox, len - 1) + || path[len] != '/') + return false; + return true; +} + + +/* returns TRUE if path cannot be modified */ +bool bad_path(const char *path) +{ + size_t plen= strlen(path); + + const char *vardir= getenv("MYSQLTEST_VARDIR"); + if (is_sub_path(path, plen, vardir)) + return false; + + const char *tmpdir= getenv("MYSQL_TMP_DIR"); + if (is_sub_path(path, plen, tmpdir)) + return false; + + report_or_die("Path '%s' is not a subdirectory of MYSQLTEST_VARDIR '%s'" + "or MYSQL_TMP_DIR '%s'", + path, vardir, tmpdir); + return true; +} + + /* SYNOPSIS set_wild_chars @@ -3668,6 +3702,9 @@ void do_remove_file(struct st_command *command) rm_args, sizeof(rm_args)/sizeof(struct command_arg), ' '); + if (bad_path(ds_filename.str)) + DBUG_VOID_RETURN; + DBUG_PRINT("info", ("removing file: %s", ds_filename.str)); error= my_delete(ds_filename.str, MYF(disable_warnings ? 0 : MY_WME)) != 0; handle_command_error(command, error, my_errno); @@ -3711,6 +3748,9 @@ void do_remove_files_wildcard(struct st_command *command) ' '); fn_format(dirname, ds_directory.str, "", "", MY_UNPACK_FILENAME); + if (bad_path(ds_directory.str)) + DBUG_VOID_RETURN; + DBUG_PRINT("info", ("listing directory: %s", dirname)); if (!(dir_info= my_dir(dirname, MYF(MY_DONT_SORT | MY_WANT_STAT | MY_WME)))) { @@ -3785,6 +3825,9 @@ void do_copy_file(struct st_command *command) sizeof(copy_file_args)/sizeof(struct command_arg), ' '); + if (bad_path(ds_to_file.str)) + DBUG_VOID_RETURN; + DBUG_PRINT("info", ("Copy %s to %s", ds_from_file.str, ds_to_file.str)); /* MY_HOLD_ORIGINAL_MODES prevents attempts to chown the file */ error= (my_copy(ds_from_file.str, ds_to_file.str, @@ -3822,6 +3865,9 @@ void do_move_file(struct st_command *command) sizeof(move_file_args)/sizeof(struct command_arg), ' '); + if (bad_path(ds_to_file.str)) + DBUG_VOID_RETURN; + DBUG_PRINT("info", ("Move %s to %s", ds_from_file.str, ds_to_file.str)); error= (my_rename(ds_from_file.str, ds_to_file.str, MYF(disable_warnings ? 0 : MY_WME)) != 0); @@ -3860,6 +3906,9 @@ void do_chmod_file(struct st_command *command) sizeof(chmod_file_args)/sizeof(struct command_arg), ' '); + if (bad_path(ds_file.str)) + DBUG_VOID_RETURN; + /* Parse what mode to set */ if (ds_mode.length != 4 || str2int(ds_mode.str, 8, 0, INT_MAX, &mode) == NullS) @@ -3931,6 +3980,9 @@ void do_mkdir(struct st_command *command) mkdir_args, sizeof(mkdir_args)/sizeof(struct command_arg), ' '); + if (bad_path(ds_dirname.str)) + DBUG_VOID_RETURN; + DBUG_PRINT("info", ("creating directory: %s", ds_dirname.str)); error= my_mkdir(ds_dirname.str, 0777, MYF(MY_WME)) != 0; handle_command_error(command, error, my_errno); @@ -3938,6 +3990,47 @@ void do_mkdir(struct st_command *command) DBUG_VOID_RETURN; } + +/* + Remove directory recursively. +*/ +static int rmtree(const char *dir) +{ + char path[FN_REFLEN]; + char sep[]={ FN_LIBCHAR, 0 }; + int err=0; + + MY_DIR *dir_info= my_dir(dir, MYF(MY_DONT_SORT | MY_WANT_STAT)); + if (!dir_info) + return 1; + + for (uint i= 0; i < dir_info->number_of_files; i++) + { + FILEINFO *file= dir_info->dir_entry + i; + /* Skip "." and ".." */ + if (!strcmp(file->name, ".") || !strcmp(file->name, "..")) + continue; + + strxnmov(path, sizeof(path), dir, sep, file->name, NULL); + + if (!MY_S_ISDIR(file->mystat->st_mode)) + err= my_delete(path, 0); + else + err= rmtree(path); + + if(err) + break; + } + + my_dirend(dir_info); + + if (!err) + err= rmdir(dir); + + return err; +} + + /* SYNOPSIS do_rmdir @@ -3945,12 +4038,11 @@ void do_mkdir(struct st_command *command) DESCRIPTION rmdir - Remove the empty directory + Remove the directory tree */ void do_rmdir(struct st_command *command) { - int error; static DYNAMIC_STRING ds_dirname; const struct command_arg rmdir_args[] = { { "dirname", ARG_STRING, TRUE, &ds_dirname, "Directory to remove" } @@ -3961,9 +4053,13 @@ void do_rmdir(struct st_command *command) rmdir_args, sizeof(rmdir_args)/sizeof(struct command_arg), ' '); + if (bad_path(ds_dirname.str)) + DBUG_VOID_RETURN; + DBUG_PRINT("info", ("removing directory: %s", ds_dirname.str)); - error= rmdir(ds_dirname.str) != 0; - handle_command_error(command, error, errno); + if (rmtree(ds_dirname.str)) + handle_command_error(command, 1, errno); + dynstr_free(&ds_dirname); DBUG_VOID_RETURN; } @@ -4076,6 +4172,9 @@ static void do_list_files_write_file_command(struct st_command *command, list_files_args, sizeof(list_files_args)/sizeof(struct command_arg), ' '); + if (bad_path(ds_filename.str)) + DBUG_VOID_RETURN; + init_dynamic_string(&ds_content, "", 1024, 1024); error= get_list_files(&ds_content, &ds_dirname, &ds_wild); handle_command_error(command, error, my_errno); @@ -4127,7 +4226,8 @@ void read_until_delimiter(DYNAMIC_STRING *ds, while (1) { c= my_getc(cur_file->file); - + if (c == '\r') + c= my_getc(cur_file->file); if (c == '\n') { cur_file->lineno++; @@ -4178,6 +4278,9 @@ void do_write_file_command(struct st_command *command, my_bool append) sizeof(write_file_args)/sizeof(struct command_arg), ' '); + if (bad_path(ds_filename.str)) + DBUG_VOID_RETURN; + if (!append && access(ds_filename.str, F_OK) == 0) { /* The file should not be overwritten */ diff --git a/mysql-test/include/write_var_to_file.inc b/mysql-test/include/write_var_to_file.inc index 08de195ccbb..7982c6fab31 100644 --- a/mysql-test/include/write_var_to_file.inc +++ b/mysql-test/include/write_var_to_file.inc @@ -43,9 +43,8 @@ if ($write_to_file == 'GENERATE') if (`SELECT LENGTH(@@secure_file_priv) > 0`) { - --let $_wvtf_secure_file_priv= `SELECT @@secure_file_priv` --let $_wvtf_suffix= `SELECT UUID()` - --let $_wvtf_tmp_file= $_wvtf_secure_file_priv/_wvtf_$_wvtf_suffix + --let $_wvtf_tmp_file= $MYSQLTEST_VARDIR/_wvtf_$_wvtf_suffix --eval SELECT '$write_var' INTO DUMPFILE '$_wvtf_tmp_file' --copy_file $_wvtf_tmp_file $write_to_file diff --git a/mysql-test/r/mysql_plugin.result b/mysql-test/r/mysql_plugin.result deleted file mode 100644 index 0bcb47e4a10..00000000000 --- a/mysql-test/r/mysql_plugin.result +++ /dev/null @@ -1,132 +0,0 @@ -# -# Ensure the plugin isn't loaded. -# -SELECT * FROM mysql.plugin WHERE dl like 'libdaemon%' ORDER BY name; -name dl -# -# Enable the plugin... -# -# -# Simulate loading a plugin libary with multiple entry points. -# This will test the DISABLE to ensure all rows are removed. -# -INSERT INTO mysql.plugin VALUES ('wicky', 'libdaemon_example.so'); -INSERT INTO mysql.plugin VALUES ('wacky', 'libdaemon_example.so'); -INSERT INTO mysql.plugin VALUES ('wonky', 'libdaemon_example.so'); -# -# Ensure the plugin is now loaded. -# -SELECT * FROM mysql.plugin WHERE dl like 'libdaemon%' ORDER BY name; -name dl -daemon_example libdaemon_example.so -wacky libdaemon_example.so -wicky libdaemon_example.so -wonky libdaemon_example.so -# -# Ensure the plugin is loaded. -# -SELECT * FROM mysql.plugin WHERE dl like '%libdaemon%' ORDER BY name; -name dl -daemon_example libdaemon_example.so -# -# Ensure the plugin is replaced. -# -SELECT * FROM mysql.plugin WHERE dl like '%libdaemon%' ORDER BY name; -name dl -daemon_example liblibdaemon_example.so -# -# Disable the plugin... -# -# -# Ensure the plugin isn't loaded. -# -SELECT * FROM mysql.plugin WHERE dl like '%libdaemon%' ORDER BY name; -name dl -# -# Attempt to load non-existant plugin -# -ERROR: Cannot read plugin config file NOT_THERE_AT_ALL. File does not exist. -# -# Attempt to use non-existant plugin.ini file -# -ERROR: Cannot read plugin config file daemon_example. File does not exist. -# -# Attempt to omit the plugin -# -ERROR: No plugin specified. -# -# Attempt to omit DISABLE|ENABLE -# -ERROR: missing operation. Please specify either ' ENABLE' or ' DISABLE'. -# -# Attempt to use bad paths - datadir -# -ERROR: Cannot access datadir at '/data_not_there/'. -# -# Attempt to use bad paths - basedir -# -ERROR: Cannot access basedir at '/basedir_not_there/'. -# -# Attempt to use bad paths - plugin_dir -# -ERROR: Cannot read plugin config file daemon_example. File does not exist. -# -# Attempt to use bad paths - mysqld -# -ERROR: Cannot access mysqld path '/mysqld_not_there/'. -# -# Attempt to use bad paths - my_print_defaults -# -ERROR: Cannot access my-print-defaults path '/my_print_defaults_not_there/'. -# -# Missing library -# -ERROR: The plugin library is missing or in a different location. -# -# Bad format for config file -# -ERROR: Cannot read plugin config file daemon_example. Bad format in plugin configuration file. -# -# Missing base_dir option -# -ERROR: Missing --basedir option. -# -# Missing data_dir option -# -ERROR: Missing --datadir option. -# -# Missing plugin_dir option -# -ERROR: Missing --plugin_dir option. -# -# Show the help. -# -mysql_plugin Ver V.V.VV Distrib XX.XX.XX -Copyright (c) 2011, 2015, Oracle and/or its affiliates. All rights reserved. - -Enable or disable plugins. - -Usage: mysql_plugin [options] ENABLE|DISABLE - -Options: - -?, --help Display this help and exit. - -b, --basedir=name The basedir for the server. - -d, --datadir=name The datadir for the server. - -p, --plugin-dir=name - The plugin dir for the server. - -i, --plugin-ini=name - Read plugin information from configuration file specified - instead of from /.ini. - -n, --no-defaults Do not read values from configuration file. - -P, --print-defaults - Show default values from configuration file. - -m, --mysqld=name Path to mysqld executable. Example: /sbin/temp1/mysql/bin - -f, --my-print-defaults=name - Path to my_print_defaults executable. Example: - /source/temp11/extra - -v, --verbose More verbose output; you can use this multiple times to - get even more verbose output. - -V, --version Output version information and exit. - - -mysql_plugin Ver V.V.VV Distrib XX.XX.XX diff --git a/mysql-test/t/mysql_plugin.test b/mysql-test/t/mysql_plugin.test deleted file mode 100644 index 10bc03e0f06..00000000000 --- a/mysql-test/t/mysql_plugin.test +++ /dev/null @@ -1,368 +0,0 @@ -# -# Test mysql_plugin tool -# -# This test contains test cases for testing the mysql_plugin client with -# the daemon_example plugin. Test cases include tests for: -# -# - successful enable/disable -# - incorrect paths -# - missing paths/options -# -# Implementation Notes -# -# The mysql_plugin tool now accepts --mysqld the path to mysqld server. The -# mysqld path is extracted from MYSQLD_BOOTSTRAP_CMD line. We also extract -# the path of MYSQLD_BASEDIR (where mysql exists) and use it for the errmsg -# file. The directories differ between Windows and Unix but the Perl script -# included below will pick as per platform. -# -# The test is also designed to issue the --skip directive if the location of -# the mysqld, my_print_defaults, or daemon_example.ini files cannot be found. -# - ---source include/not_embedded.inc - -# Add the datadir, basedir, plugin_dir to the bootstrap command -let $MYSQLD_DATADIR= `select @@datadir`; -let $MYSQL_BASEDIR= `select @@basedir`; -let $MYSQL_ERRMSG_BASEDIR=`select @@lc_messages_dir`; -let $PLUGIN_DIR=`select @@plugin_dir`; - ---disable_abort_on_error - -# Perl script to extract the location of the basedir from environment -# variables. This is needed to ensure the test will run on the PB machines -# designed to test release as well as debug builds. It also checks for the -# location of the my_print_defaults and daemon_example.ini files. - -perl; -use File::Basename; - my ($mysqld)= split " ", $ENV{MYSQLD_BOOTSTRAP_CMD}; - my $mysqld_basedir=dirname($mysqld); - my $my_print_defaults= $ENV{MYSQL_MY_PRINT_DEFAULTS}; - my $my_print_defaults_basedir=dirname($my_print_defaults); - my $daemonexample_ini= "$ENV{DAEMONEXAMPLE_DIR}/daemon_example.ini"; - my $plugindir_ini= "$ENV{PLUGIN_DIR}/daemon_example.ini"; - my $notfound= ""; - open(FILE, ">", "$ENV{MYSQL_TMP_DIR}/mysqld.inc") or die; - print FILE "let \$MYSQLD_BASEDIR= $mysqld_basedir;\n"; - print FILE "let \$MYSQL_MY_PRINT_DEFAULTS_BASEDIR= $my_print_defaults_basedir;\n"; - if ((!-e $daemonexample_ini) || (!-r $daemonexample_ini)) - { - print FILE "let \$DAEMONEXAMPLE_DIR= $not_found;\n"; - } - if ((!-e $plugindir_ini) || (!-r $plugindir_ini)) - { - print FILE "let \$PLUGIN_DIR= $not_found;\n"; - } - close FILE; -EOF - - -source $MYSQL_TMP_DIR/mysqld.inc; -remove_file $MYSQL_TMP_DIR/mysqld.inc; - -# The mysql_plugin tool expects a directory structure like in the installed -# mysql version, so errmsg.sys will be copied to "basedir/share", we create -# and remove this structure. - ---mkdir $MYSQLD_BASEDIR/share ---mkdir $MYSQLD_BASEDIR/share/mysql ---copy_file $MYSQL_ERRMSG_BASEDIR/english/errmsg.sys $MYSQLD_BASEDIR/share/errmsg.sys ---copy_file $MYSQL_ERRMSG_BASEDIR/english/errmsg.sys $MYSQLD_BASEDIR/share/mysql/errmsg.sys - -# The mysql_plugin tool now accepts --my-print-defaults which points to the -# executable my_print_defaults.exe we can get this path from the variable -# $MYSQL_MY_PRINT_DEFAULTS. - -# Check for my_print_defaults location. Skip if not found. -if ($MYSQL_MY_PRINT_DEFAULTS_BASEDIR == '') -{ - --skip Test requires known location of my_print_defaults executable. -} - -# Check for mysqld location. Skip if not found. -if ($MYSQLD == '') -{ - --skip Test requires known location of mysqld executable. -} - -# Check for daemon_example.ini location. Skip if not found in either -# the plugin_dir path or the daemon_example_dir path. -if ($PLUGIN_DIR == '') -{ - if ($DAEMONEXAMPLE_DIR == '') - { - --skip Test requires known location of daemon_example.ini file. - } - let $PLUGIN_DIR = $DAEMONEXAMPLE_DIR; -} - -# Build client command for reuse. - -let $MYSQL_PLUGIN_CMD= $MYSQL_PLUGIN --datadir=$MYSQLD_DATADIR --basedir=$MYSQLD_BASEDIR --plugin-dir=$PLUGIN_DIR --mysqld=$MYSQLD_BASEDIR --my-print-defaults=$MYSQL_MY_PRINT_DEFAULTS_BASEDIR; - ---echo # ---echo # Ensure the plugin isn't loaded. ---echo # -SELECT * FROM mysql.plugin WHERE dl like 'libdaemon%' ORDER BY name; - ---echo # ---echo # Enable the plugin... ---echo # - ---exec echo "wait" > $MYSQLTEST_VARDIR/tmp/mysqld.1.expect ---shutdown_server 10 ---source include/wait_until_disconnected.inc - -# -# Enable the plugin -# ---exec $MYSQL_PLUGIN_CMD ENABLE daemon_example - -# -# Ensure enabling an enabled plugin doesn't fail ---exec $MYSQL_PLUGIN_CMD ENABLE daemon_example - -# -# Restart the server -# - ---exec echo "restart" > $MYSQLTEST_VARDIR/tmp/mysqld.1.expect ---enable_reconnect ---source include/wait_until_connected_again.inc - ---echo # ---echo # Simulate loading a plugin libary with multiple entry points. ---echo # This will test the DISABLE to ensure all rows are removed. ---echo # ---replace_regex /\.dll/.so/ -eval INSERT INTO mysql.plugin VALUES ('wicky', '$DAEMONEXAMPLE'); ---replace_regex /\.dll/.so/ -eval INSERT INTO mysql.plugin VALUES ('wacky', '$DAEMONEXAMPLE'); ---replace_regex /\.dll/.so/ -eval INSERT INTO mysql.plugin VALUES ('wonky', '$DAEMONEXAMPLE'); - ---echo # ---echo # Ensure the plugin is now loaded. ---echo # ---replace_regex /\.dll/.so/ -SELECT * FROM mysql.plugin WHERE dl like 'libdaemon%' ORDER BY name; - ---exec echo "wait" > $MYSQLTEST_VARDIR/tmp/mysqld.1.expect ---shutdown_server 10 ---source include/wait_until_disconnected.inc - -# -# Disable the plugin - to remove winky, wonky entries -# ---exec $MYSQL_PLUGIN_CMD DISABLE daemon_example - -# -# Enable the plugin again -# ---exec $MYSQL_PLUGIN_CMD ENABLE daemon_example - -# -# Restart the server -# ---exec echo "restart" > $MYSQLTEST_VARDIR/tmp/mysqld.1.expect ---enable_reconnect ---source include/wait_until_connected_again.inc - ---echo # ---echo # Ensure the plugin is loaded. ---echo # ---replace_regex /\.dll/.so/ -SELECT * FROM mysql.plugin WHERE dl like '%libdaemon%' ORDER BY name; - ---exec echo "wait" > $MYSQLTEST_VARDIR/tmp/mysqld.1.expect ---shutdown_server 10 ---source include/wait_until_disconnected.inc - -# To test the case where the same plugin is reloaded with a different soname, -# we must copy the example daemon to a new location renaming it. - -let $DAEMON_RELOAD = lib$DAEMONEXAMPLE; ---copy_file $PLUGIN_DIR/$DAEMONEXAMPLE $PLUGIN_DIR/$DAEMON_RELOAD ---copy_file include/libdaemon_example.ini $PLUGIN_DIR/libdaemon_example.ini - -# Now reload it and see that it is a different name. ---exec $MYSQL_PLUGIN_CMD ENABLE libdaemon_example - -# -# Restart the server -# ---exec echo "restart" > $MYSQLTEST_VARDIR/tmp/mysqld.1.expect ---enable_reconnect ---source include/wait_until_connected_again.inc - ---echo # ---echo # Ensure the plugin is replaced. ---echo # ---replace_regex /\.dll/.so/ -SELECT * FROM mysql.plugin WHERE dl like '%libdaemon%' ORDER BY name; - ---echo # ---echo # Disable the plugin... ---echo # - ---exec echo "wait" > $MYSQLTEST_VARDIR/tmp/mysqld.1.expect ---shutdown_server 10 ---source include/wait_until_disconnected.inc - -# -# Disable the plugin -# ---exec $MYSQL_PLUGIN_CMD DISABLE libdaemon_example - -# Remove files for last test case. - ---remove_file $PLUGIN_DIR/$DAEMON_RELOAD ---remove_file $DAEMONEXAMPLE_DIR/libdaemon_example.ini - -# -# Restart the server -# ---exec echo "restart" > $MYSQLTEST_VARDIR/tmp/mysqld.1.expect ---enable_reconnect ---source include/wait_until_connected_again.inc - ---echo # ---echo # Ensure the plugin isn't loaded. ---echo # -SELECT * FROM mysql.plugin WHERE dl like '%libdaemon%' ORDER BY name; - -# -# Stop the server for error conditions -# - ---exec echo "wait" > $MYSQLTEST_VARDIR/tmp/mysqld.1.expect ---shutdown_server 10 ---source include/wait_until_disconnected.inc - ---echo # ---echo # Attempt to load non-existant plugin ---echo # ---error 1,2,256 ---exec $MYSQL_PLUGIN_CMD DISABLE NOT_THERE_AT_ALL 2>&1 - ---echo # ---echo # Attempt to use non-existant plugin.ini file ---echo # ---error 1,2,7,256 ---exec $MYSQL_PLUGIN_CMD DISABLE daemon_example --plugin-ini=/NOT/THERE/pi.ini 2>&1 - ---echo # ---echo # Attempt to omit the plugin ---echo # ---error 1,2,256 ---exec $MYSQL_PLUGIN_CMD DISABLE 2>&1 - ---echo # ---echo # Attempt to omit DISABLE|ENABLE ---echo # ---error 1,2,256 ---exec $MYSQL_PLUGIN_CMD daemon_example 2>&1 - ---echo # ---echo # Attempt to use bad paths - datadir ---echo # -let $MYSQL_PLUGIN_CMD= $MYSQL_PLUGIN -n --datadir=/data_not_there/ --basedir=$MYSQL_BASEDIR --plugin-dir=$PLUGIN_DIR --mysqld=$MYSQLD_BASEDIR --my-print-defaults=$MYSQL_MY_PRINT_DEFAULTS_BASEDIR; ---error 1,2,256 ---exec $MYSQL_PLUGIN_CMD DISABLE daemon_example 2>&1 - ---echo # ---echo # Attempt to use bad paths - basedir ---echo # -let $MYSQL_PLUGIN_CMD= $MYSQL_PLUGIN -n --datadir=$MYSQLD_DATADIR --basedir=/basedir_not_there/ --plugin-dir=$PLUGIN_DIR --mysqld=$MYSQLD_BASEDIR --my-print-defaults=$MYSQL_MY_PRINT_DEFAULTS_BASEDIR; -replace_result "/basedir_not_there//" "/basedir_not_there/"; ---error 1,2,256 ---exec $MYSQL_PLUGIN_CMD DISABLE daemon_example 2>&1 - ---echo # ---echo # Attempt to use bad paths - plugin_dir ---echo # -let $MYSQL_PLUGIN_CMD= $MYSQL_PLUGIN -n --datadir=$MYSQLD_DATADIR --basedir=$MYSQL_BASEDIR --plugin-dir=/plugin_not_there/ --mysqld=$MYSQLD_BASEDIR --my-print-defaults=$MYSQL_MY_PRINT_DEFAULTS_BASEDIR; ---error 1,2,256 ---exec $MYSQL_PLUGIN_CMD DISABLE daemon_example 2>&1 - ---echo # ---echo # Attempt to use bad paths - mysqld ---echo # -let $MYSQL_PLUGIN_CMD= $MYSQL_PLUGIN -n --datadir=$MYSQLD_DATADIR --basedir=$MYSQL_BASEDIR --plugin-dir=$PLUGIN_DIR --mysqld=/mysqld_not_there/ --my-print-defaults=$MYSQL_MY_PRINT_DEFAULTS_BASEDIR; ---error 1,2,256 ---exec $MYSQL_PLUGIN_CMD DISABLE daemon_example 2>&1 - ---echo # ---echo # Attempt to use bad paths - my_print_defaults ---echo # -let $MYSQL_PLUGIN_CMD= $MYSQL_PLUGIN -n --datadir=$MYSQLD_DATADIR --basedir=$MYSQL_BASEDIR --plugin-dir=$PLUGIN_DIR --mysqld=$MYSQLD_BASEDIR --my-print-defaults=/my_print_defaults_not_there/; ---error 1,2,256 ---exec $MYSQL_PLUGIN_CMD DISABLE daemon_example 2>&1 - - ---echo # ---echo # Missing library ---echo # -let $MYSQL_PLUGIN_CMD= $MYSQL_PLUGIN -n --datadir=$MYSQLD_DATADIR --basedir=$MYSQL_BASEDIR --plugin-dir=$PLUGIN_DIR --plugin-ini=$MYSQL_TEST_DIR/include/daemon_example_bad_soname.ini --mysqld=$MYSQLD_BASEDIR --my-print-defaults=$MYSQL_MY_PRINT_DEFAULTS_BASEDIR; ---error 1,2,256 ---exec $MYSQL_PLUGIN_CMD DISABLE daemon_example 2>&1 - ---echo # ---echo # Bad format for config file ---echo # -let $MYSQL_PLUGIN_CMD= $MYSQL_PLUGIN -n --datadir=$MYSQLD_DATADIR --basedir=$MYSQL_BASEDIR --plugin-dir=$PLUGIN_DIR --plugin-ini=$MYSQL_TEST_DIR/include/daemon_example_bad_format.ini --mysqld=$MYSQLD_BASEDIR --my-print-defaults=$MYSQL_MY_PRINT_DEFAULTS_BASEDIR; ---error 1,2,256 ---exec $MYSQL_PLUGIN_CMD DISABLE daemon_example 2>&1 - ---echo # ---echo # Missing base_dir option ---echo # -let $MYSQL_PLUGIN_CMD= $MYSQL_PLUGIN -n --datadir=$MYSQLD_DATADIR --plugin-dir=$PLUGIN_DIR --mysqld=$MYSQLD_BASEDIR --my-print-defaults=$MYSQL_MY_PRINT_DEFAULTS_BASEDIR; ---error 1,2,139,256 ---exec $MYSQL_PLUGIN_CMD DISABLE daemon_example 2>&1 - ---echo # ---echo # Missing data_dir option ---echo # -let $MYSQL_PLUGIN_CMD= $MYSQL_PLUGIN -n --basedir=$MYSQL_BASEDIR --plugin-dir=$PLUGIN_DIR --mysqld=$MYSQLD_BASEDIR --my-print-defaults=$MYSQL_MY_PRINT_DEFAULTS_BASEDIR; ---error 1,2,139,256 ---exec $MYSQL_PLUGIN_CMD DISABLE daemon_example 2>&1 - ---echo # ---echo # Missing plugin_dir option ---echo # -let $MYSQL_PLUGIN_CMD= $MYSQL_PLUGIN -n --datadir=$MYSQL_DATADIR --basedir=$MYSQL_BASEDIR --mysqld=$MYSQLD_BASEDIR --my-print-defaults=$MYSQL_MY_PRINT_DEFAULTS_BASEDIR; ---error 1,2,139,256 ---exec $MYSQL_PLUGIN_CMD DISABLE daemon_example 2>&1 - ---echo # ---echo # Show the help. ---echo # -replace_result $MYSQL_PLUGIN mysql_plugin; ---replace_regex /Ver [0-9.]+ Distrib [0-9.]+/Ver V.V.VV Distrib XX.XX.XX/ /XX-m[0-9]+/XX/ /XX[a-z]/XX/ ---exec $MYSQL_PLUGIN --help - -replace_result $MYSQL_PLUGIN mysql_plugin; ---replace_regex /Ver [0-9.]+ Distrib [0-9.]+/Ver V.V.VV Distrib XX.XX.XX/ /XX-m[0-9]+/XX/ /XX[a-z]/XX/ ---exec $MYSQL_PLUGIN --version - -# -# Restart the server -# ---exec echo "restart" > $MYSQLTEST_VARDIR/tmp/mysqld.1.expect ---enable_reconnect ---source include/wait_until_connected_again.inc - -# -# Cleanup - ---remove_file $MYSQLTEST_VARDIR/tmp/mysqld.1.expect - -# Cleanup the share folder in the binary path. ---remove_file $MYSQLD_BASEDIR/share/errmsg.sys ---rmdir $MYSQLD_BASEDIR/share/mysql ---rmdir $MYSQLD_BASEDIR/share - ---enable_abort_on_error diff --git a/mysql-test/t/mysqltest.test b/mysql-test/t/mysqltest.test index e85d793b628..1e7d8a6a8ca 100644 --- a/mysql-test/t/mysqltest.test +++ b/mysql-test/t/mysqltest.test @@ -406,7 +406,7 @@ select 3 from t1 ; --disable_abort_on_error ONCE garbage; --disable_abort_on_error ONCE ---remove_file DoesNotExist +--remove_file $MYSQLTEST_VARDIR/DoesNotExist --disable_result_log select 2; @@ -1940,8 +1940,6 @@ remove_file $MYSQLTEST_VARDIR/tmp/zero_length_file.result; remove_file $MYSQLTEST_VARDIR/tmp/zero_length_file.reject; --error 0,1 remove_file $MYSQLTEST_VARDIR/tmp/zero_length_file.log; ---error 0,1 -remove_file $MYSQL_TEST_DIR/r/zero_length_file.reject; --enable_warnings # @@ -2194,7 +2192,7 @@ drop table t1; --exec echo "remove_file ;" | $MYSQL_TEST 2>&1 --error 1 -remove_file non_existing_file; +remove_file $MYSQLTEST_VARDIR/non_existing_file; --enable_warnings # ---------------------------------------------------------------------------- @@ -2205,10 +2203,10 @@ remove_file non_existing_file; --exec echo "remove_files_wildcard ;" | $MYSQL_TEST 2>&1 --error 1 -remove_files_wildcard non_existing_dir; +remove_files_wildcard $MYSQLTEST_VARDIR/non_existing_dir; --error 1 -remove_files_wildcard non_existing_dir non_existing_file; +remove_files_wildcard $MYSQLTEST_VARDIR/non_existing_dir non_existing_file; # ---------------------------------------------------------------------------- # test for write_file @@ -2217,7 +2215,7 @@ remove_files_wildcard non_existing_dir non_existing_file; --exec echo "write_file ;" | $MYSQL_TEST 2>&1 --error 1 ---exec echo "write_file filename ;" | $MYSQL_TEST 2>&1 +--exec echo "write_file $MYSQLTEST_VARDIR/filename ;" | $MYSQL_TEST 2>&1 # Comment out this test as it confuses cmd.exe with unmatched " #--error 1 @@ -2463,19 +2461,19 @@ remove_file $MYSQLTEST_VARDIR/tmp/file1.tmp; --exec echo "chmod ;" | $MYSQL_TEST 2>&1 --error 1 ---exec echo "chmod 0 from_file;" | $MYSQL_TEST 2>&1 +--exec echo "chmod 0 $MYSQLTEST_VARDIR/from_file;" | $MYSQL_TEST 2>&1 --error 1 ---exec echo "chmod 08 from_file;" | $MYSQL_TEST 2>&1 +--exec echo "chmod 08 $MYSQLTEST_VARDIR/from_file;" | $MYSQL_TEST 2>&1 --error 1 ---exec echo "chmod from_file;" | $MYSQL_TEST 2>&1 +--exec echo "chmod $MYSQLTEST_VARDIR/from_file;" | $MYSQL_TEST 2>&1 --error 1 ---exec echo "chmod ABZD from_file;" | $MYSQL_TEST 2>&1 +--exec echo "chmod ABZD $MYSQLTEST_VARDIR/from_file;" | $MYSQL_TEST 2>&1 --error 1 ---exec echo "chmod 06789 from_file;" | $MYSQL_TEST 2>&1 +--exec echo "chmod 06789 $MYSQLTEST_VARDIR/from_file;" | $MYSQL_TEST 2>&1 # ---------------------------------------------------------------------------- @@ -2876,8 +2874,6 @@ list_files_append_file $MYSQLTEST_VARDIR/tmp/testdir/file2.txt $MYSQLTEST_VARDIR list_files_write_file $MYSQLTEST_VARDIR/tmp/testdir/file2.txt $MYSQLTEST_VARDIR/tmp/testdir file?.txt; list_files_append_file $MYSQLTEST_VARDIR/tmp/testdir/file3.txt $MYSQLTEST_VARDIR/tmp/testdir file*.txt; diff_files $MYSQLTEST_VARDIR/tmp/testdir/file2.txt $MYSQLTEST_VARDIR/tmp/testdir/file3.txt; ---error 1 -rmdir $MYSQLTEST_VARDIR/tmp/testdir; cat_file $MYSQLTEST_VARDIR/tmp/testdir/file3.txt; diff --git a/storage/connect/mysql-test/connect/t/secure_file_priv.test b/storage/connect/mysql-test/connect/t/secure_file_priv.test index 46633502034..f7792536892 100644 --- a/storage/connect/mysql-test/connect/t/secure_file_priv.test +++ b/storage/connect/mysql-test/connect/t/secure_file_priv.test @@ -10,4 +10,4 @@ let $SECUREDIR= `select @@secure_file_priv`; INSERT INTO t1 VALUES (10); SELECT * FROM t1; DROP TABLE t1; ---remove_file $SECUREDIR/t1.dbf +--remove_file $MYSQL_TMP_DIR/t1.dbf From f06ab0fc99daa9049f8b9f8992d403047efd41e1 Mon Sep 17 00:00:00 2001 From: Vladislav Vaintroub Date: Tue, 18 Apr 2017 17:20:55 +0000 Subject: [PATCH 07/28] MDEV-9566 Server code changes in preparation for mariabackup - Backup will load encryption plugins outside of mysqld. Thus, do not force loading MyISAM plugin in plugin_load. - init_signals() will be used in backup, make it global, not static. --- sql/mysqld.cc | 4 ++-- sql/sql_plugin.cc | 32 +++++++++++++++++++------------- 2 files changed, 21 insertions(+), 15 deletions(-) diff --git a/sql/mysqld.cc b/sql/mysqld.cc index 83829841b39..0bf57d9543b 100644 --- a/sql/mysqld.cc +++ b/sql/mysqld.cc @@ -3100,7 +3100,7 @@ LONG WINAPI my_unhandler_exception_filter(EXCEPTION_POINTERS *ex_pointers) } -static void init_signals(void) +void init_signals(void) { if(opt_console) SetConsoleCtrlHandler(console_event_handler,TRUE); @@ -3231,7 +3231,7 @@ static size_t my_setstacksize(pthread_attr_t *attr, size_t stacksize) #ifndef EMBEDDED_LIBRARY -static void init_signals(void) +void init_signals(void) { sigset_t set; struct sigaction sa; diff --git a/sql/sql_plugin.cc b/sql/sql_plugin.cc index eacf39098b4..469e96c5fee 100644 --- a/sql/sql_plugin.cc +++ b/sql/sql_plugin.cc @@ -1606,22 +1606,28 @@ int plugin_init(int *argc, char **argv, int flags) } } - /* First, we initialize only MyISAM - that should always succeed */ - plugin_ptr= plugin_find_internal(&MyISAM, MYSQL_STORAGE_ENGINE_PLUGIN); - DBUG_ASSERT(plugin_ptr); - DBUG_ASSERT(plugin_ptr->load_option == PLUGIN_FORCE); - - if (plugin_initialize(&tmp_root, plugin_ptr, argc, argv, false)) - goto err_unlock; - /* - set the global default storage engine variable so that it will - not be null in any child thread. + First, we initialize only MyISAM - that should almost always succeed + (almost always, because plugins can be loaded outside of the server, too). */ - global_system_variables.table_plugin= - intern_plugin_lock(NULL, plugin_int_to_ref(plugin_ptr)); - DBUG_ASSERT(plugin_ptr->ref_count == 1); + plugin_ptr= plugin_find_internal(&MyISAM, MYSQL_STORAGE_ENGINE_PLUGIN); + DBUG_ASSERT(plugin_ptr || !mysql_mandatory_plugins[0]); + if (plugin_ptr) + { + DBUG_ASSERT(plugin_ptr->load_option == PLUGIN_FORCE); + if (plugin_initialize(&tmp_root, plugin_ptr, argc, argv, false)) + goto err_unlock; + + /* + set the global default storage engine variable so that it will + not be null in any child thread. + */ + global_system_variables.table_plugin = + intern_plugin_lock(NULL, plugin_int_to_ref(plugin_ptr)); + DBUG_ASSERT(plugin_ptr->ref_count == 1); + + } mysql_mutex_unlock(&LOCK_plugin); /* Register (not initialize!) all dynamic plugins */ From 9c4b7cad279fde927c192a8928aa43cb751b2116 Mon Sep 17 00:00:00 2001 From: Vladislav Vaintroub Date: Tue, 18 Apr 2017 17:57:07 +0000 Subject: [PATCH 08/28] MDEV-9566 Prepare xtradb for xtrabackup These changes are comparable to Percona's modifications in innodb in the Percona Xtrabackup repository. - If functions are used in backup as well as in innodb, make them non-static. - Define IS_XTRABACKUP() macro for special handling of innodb running inside backup. - Extend some functions for backup. fil_space_for_table_exists_in_mem() gets additional parameter 'remove_from_data_dict_if_does_not_exist', for partial backups fil_load_single_table_tablespaces() gets an optional parameter predicate which tells whether to load tablespace based on database or table name, also for partial backups. srv_undo_tablespaces_init() gets an optional parameter 'backup_mode' - Allow single redo log file (for backup "prepare") - Do not read doublewrite buffer pages in backup, they are outdated - Add function fil_remove_invalid_table_from_data_dict(), to remove non-existing tables from data dictionary in case of partial backups. - On Windows, fix file share modes when opening tablespaces, to allow mariabackup to read tablespaces while server is online. - Avoid access to THDVARs in backup, because innodb plugin is not loaded, and THDVAR would crash in this case. --- storage/xtradb/btr/btr0btr.cc | 2 - storage/xtradb/buf/buf0flu.cc | 2 +- storage/xtradb/buf/buf0rea.cc | 7 +- storage/xtradb/dict/dict0load.cc | 28 ++- storage/xtradb/fil/fil0fil.cc | 321 +++++++++++++++++++++++++--- storage/xtradb/handler/ha_innodb.cc | 58 +++-- storage/xtradb/include/fil0fil.h | 6 +- storage/xtradb/include/srv0srv.h | 12 +- storage/xtradb/include/trx0sys.h | 5 +- storage/xtradb/include/univ.i | 2 +- storage/xtradb/log/log0log.cc | 2 +- storage/xtradb/log/log0recv.cc | 100 ++++++++- storage/xtradb/os/os0file.cc | 14 +- storage/xtradb/row/row0mysql.cc | 3 +- storage/xtradb/srv/srv0srv.cc | 12 +- storage/xtradb/srv/srv0start.cc | 40 +++- storage/xtradb/trx/trx0sys.cc | 27 ++- storage/xtradb/trx/trx0trx.cc | 24 ++- 18 files changed, 556 insertions(+), 109 deletions(-) diff --git a/storage/xtradb/btr/btr0btr.cc b/storage/xtradb/btr/btr0btr.cc index c94b539c2c7..d84c93f8b3e 100644 --- a/storage/xtradb/btr/btr0btr.cc +++ b/storage/xtradb/btr/btr0btr.cc @@ -722,7 +722,6 @@ btr_root_fseg_validate( /**************************************************************//** Gets the root node of a tree and x- or s-latches it. @return root page, x- or s-latched */ -static buf_block_t* btr_root_block_get( /*===============*/ @@ -1531,7 +1530,6 @@ btr_node_ptr_set_child_page_no( /************************************************************//** Returns the child page of a node pointer and x-latches it. @return child page, x-latched */ -static buf_block_t* btr_node_ptr_get_child( /*===================*/ diff --git a/storage/xtradb/buf/buf0flu.cc b/storage/xtradb/buf/buf0flu.cc index dc47f6f486d..9c9823bbc28 100644 --- a/storage/xtradb/buf/buf0flu.cc +++ b/storage/xtradb/buf/buf0flu.cc @@ -2864,7 +2864,7 @@ DECLARE_THREAD(buf_flush_page_cleaner_thread)( success = buf_flush_list(PCT_IO(100), LSN_MAX, &n_flushed); buf_flush_wait_batch_end(NULL, BUF_FLUSH_LIST); - } while (!success || n_flushed > 0); + } while (!success || n_flushed > 0 || (IS_XTRABACKUP() && buf_get_n_pending_read_ios() > 0)); /* Some sanity checks */ ut_a(srv_get_active_thread_type() == SRV_NONE); diff --git a/storage/xtradb/buf/buf0rea.cc b/storage/xtradb/buf/buf0rea.cc index e275eead4cc..85b04d37a08 100644 --- a/storage/xtradb/buf/buf0rea.cc +++ b/storage/xtradb/buf/buf0rea.cc @@ -955,11 +955,8 @@ buf_read_ibuf_merge_pages( tablespace_deleted: /* We have deleted or are deleting the single-table - tablespace: remove the entries for that page */ - - ibuf_merge_or_delete_for_page(NULL, space_ids[i], - page_nos[i], - zip_size, FALSE); + tablespace: remove the entries for tablespace. */ + ibuf_delete_for_discarded_space(space_ids[i]); break; case DB_DECRYPTION_FAILED: ib_logf(IB_LOG_LEVEL_ERROR, diff --git a/storage/xtradb/dict/dict0load.cc b/storage/xtradb/dict/dict0load.cc index 83f7bdafb35..dc3f9c85bba 100644 --- a/storage/xtradb/dict/dict0load.cc +++ b/storage/xtradb/dict/dict0load.cc @@ -945,6 +945,10 @@ dict_insert_tablespace_and_filepath( return(err); } +/* Set by Xtrabackup */ +my_bool (*dict_check_if_skip_table)(const char* name) = 0; + + /********************************************************************//** This function looks at each table defined in SYS_TABLES. It checks the tablespace for any table with a space_id > 0. It looks up the tablespace @@ -1064,6 +1068,9 @@ loop: bool is_temp = false; bool discarded = false; + bool print_error_if_does_not_exist; + bool remove_from_data_dict_if_does_not_exist; + ib_uint32_t flags2 = static_cast( mach_read_from_4(field)); @@ -1089,6 +1096,19 @@ loop: goto loop; } + + ut_a(!IS_XTRABACKUP() || dict_check_if_skip_table); + + if (is_temp || discarded || + (IS_XTRABACKUP() && dict_check_if_skip_table(name))) { + print_error_if_does_not_exist = false; + } + else { + print_error_if_does_not_exist = true; + } + + remove_from_data_dict_if_does_not_exist = IS_XTRABACKUP() && !(is_temp || discarded); + mtr_commit(&mtr); switch (dict_check) { @@ -1096,8 +1116,8 @@ loop: /* All tablespaces should have been found in fil_load_single_table_tablespaces(). */ if (fil_space_for_table_exists_in_mem( - space_id, name, !(is_temp || discarded), - false, NULL, 0, flags) + space_id, name, print_error_if_does_not_exist, + remove_from_data_dict_if_does_not_exist , false, NULL, 0, flags) && !(is_temp || discarded)) { /* If user changes the path of .ibd files in *.isl files before doing crash recovery , @@ -1130,7 +1150,7 @@ loop: trx_resurrect_table_locks(). */ if (fil_space_for_table_exists_in_mem( space_id, name, false, - false, NULL, 0, flags)) { + false, false, NULL, 0, flags)) { break; } /* fall through */ @@ -2383,7 +2403,7 @@ err_exit: table->file_unreadable = true; } else if (!fil_space_for_table_exists_in_mem( - table->space, name, false, true, heap, + table->space, name, false, IS_XTRABACKUP(), true, heap, table->id, table->flags)) { if (DICT_TF2_FLAG_IS_SET(table, DICT_TF2_TEMPORARY)) { diff --git a/storage/xtradb/fil/fil0fil.cc b/storage/xtradb/fil/fil0fil.cc index 7ac08cc0e97..216a36e2e82 100644 --- a/storage/xtradb/fil/fil0fil.cc +++ b/storage/xtradb/fil/fil0fil.cc @@ -67,6 +67,7 @@ static ulint srv_data_read, srv_data_written; #include #endif #include "row0mysql.h" +#include "trx0purge.h" MYSQL_PLUGIN_IMPORT extern my_bool lower_case_file_system; @@ -369,7 +370,6 @@ fil_node_get_space_id( /*******************************************************************//** Returns the table space by a given name, NULL if not found. */ -UNIV_INLINE fil_space_t* fil_space_get_by_name( /*==================*/ @@ -1574,12 +1574,13 @@ fil_space_create( if (!fil_system->space_id_reuse_warned) { fil_system->space_id_reuse_warned = TRUE; - - ib_logf(IB_LOG_LEVEL_WARN, - "Allocated tablespace %lu, old maximum " - "was %lu", - (ulong) id, - (ulong) fil_system->max_assigned_id); + if (!IS_XTRABACKUP()) { + ib_logf(IB_LOG_LEVEL_WARN, + "Allocated tablespace %lu, old maximum " + "was %lu", + (ulong)id, + (ulong)fil_system->max_assigned_id); + } } fil_system->max_assigned_id = id; @@ -2413,6 +2414,19 @@ fil_read_first_page( const char* check_msg = NULL; fil_space_crypt_t* cdata; + if (IS_XTRABACKUP() && srv_backup_mode) { + /* Files smaller than page size may occur + in xtrabackup, when server creates new file + but has not yet written into it, or wrote only + partially. Checks size here, to avoid exit in os_file_read. + This file will be skipped by xtrabackup if it is too small. + */ + os_offset_t file_size; + file_size = os_file_get_size(data_file); + if (file_size < FIL_IBD_FILE_INITIAL_SIZE*UNIV_PAGE_SIZE) { + return "File size is less than minimum"; + } + } buf = static_cast(ut_malloc(2 * UNIV_PAGE_SIZE)); /* Align the memory for a possible read from a raw device */ @@ -2443,7 +2457,9 @@ fil_read_first_page( } } - check_msg = fil_check_first_page(page, *space_id, *flags); + if (!(IS_XTRABACKUP() && srv_backup_mode)) { + check_msg = fil_check_first_page(page, *space_id, *flags); + } } flushed_lsn = mach_read_from_8(page + @@ -3137,7 +3153,7 @@ fil_delete_tablespace( err = DB_IO_ERROR; } - if (err == DB_SUCCESS) { + if (err == DB_SUCCESS && !IS_XTRABACKUP()) { #ifndef UNIV_HOTBACKUP /* Write a log record about the deletion of the .ibd file, so that mysqlbackup can replay it in the @@ -3536,7 +3552,7 @@ skip_second_rename: mutex_exit(&fil_system->mutex); #ifndef UNIV_HOTBACKUP - if (success && !recv_recovery_on) { + if (success && !recv_recovery_on && !IS_XTRABACKUP()) { mtr_t mtr; mtr_start(&mtr); @@ -3782,7 +3798,18 @@ fil_create_new_single_table_tablespace( ibool success; /* TRUE if a table is created with CREATE TEMPORARY TABLE */ bool is_temp = !!(flags2 & DICT_TF2_TEMPORARY); - bool has_data_dir = FSP_FLAGS_HAS_DATA_DIR(flags) != 0; + + + /* For XtraBackup recovery we force remote tablespaces to be local, + i.e. never execute the code path corresponding to has_data_dir == true. + We don't create .isl files either, because we rely on innobackupex to + copy them under a global lock, and use them to copy remote tablespaces + to their proper locations on --copy-back. + + See also MySQL bug #72022: dir_path is always NULL for remote + tablespaces when a MLOG_FILE_CREATE* log record is replayed (the remote + directory is not available from MLOG_FILE_CREATE*). */ + bool has_data_dir = FSP_FLAGS_HAS_DATA_DIR(flags) != 0 && !IS_XTRABACKUP(); ulint atomic_writes = FSP_FLAGS_GET_ATOMIC_WRITES(flags); fil_space_crypt_t *crypt_data = NULL; @@ -3964,6 +3991,7 @@ fil_create_new_single_table_tablespace( } #ifndef UNIV_HOTBACKUP + if (!IS_XTRABACKUP()) { mtr_t mtr; ulint mlog_file_flag = 0; @@ -4004,6 +4032,138 @@ error_exit_3: return(err); } +#include "pars0pars.h" +#include "que0que.h" +#include "dict0priv.h" +static +void +fil_remove_invalid_table_from_data_dict(const char *name) +{ + trx_t* trx; + pars_info_t* info = NULL; + + trx = trx_allocate_for_mysql(); + trx_start_for_ddl(trx, TRX_DICT_OP_TABLE); + + ut_ad(mutex_own(&dict_sys->mutex)); + + trx->op_info = "removing invalid table from data dictionary"; + + info = pars_info_create(); + + pars_info_add_str_literal(info, "table_name", name); + + que_eval_sql(info, + "PROCEDURE DROP_TABLE_PROC () IS\n" + "sys_foreign_id CHAR;\n" + "table_id CHAR;\n" + "index_id CHAR;\n" + "foreign_id CHAR;\n" + "found INT;\n" + + "DECLARE CURSOR cur_fk IS\n" + "SELECT ID FROM SYS_FOREIGN\n" + "WHERE FOR_NAME = :table_name\n" + "AND TO_BINARY(FOR_NAME)\n" + " = TO_BINARY(:table_name)\n" + "LOCK IN SHARE MODE;\n" + + "DECLARE CURSOR cur_idx IS\n" + "SELECT ID FROM SYS_INDEXES\n" + "WHERE TABLE_ID = table_id\n" + "LOCK IN SHARE MODE;\n" + + "BEGIN\n" + "SELECT ID INTO table_id\n" + "FROM SYS_TABLES\n" + "WHERE NAME = :table_name\n" + "LOCK IN SHARE MODE;\n" + "IF (SQL % NOTFOUND) THEN\n" + " RETURN;\n" + "END IF;\n" + "found := 1;\n" + "SELECT ID INTO sys_foreign_id\n" + "FROM SYS_TABLES\n" + "WHERE NAME = 'SYS_FOREIGN'\n" + "LOCK IN SHARE MODE;\n" + "IF (SQL % NOTFOUND) THEN\n" + " found := 0;\n" + "END IF;\n" + "IF (:table_name = 'SYS_FOREIGN') THEN\n" + " found := 0;\n" + "END IF;\n" + "IF (:table_name = 'SYS_FOREIGN_COLS') THEN\n" + " found := 0;\n" + "END IF;\n" + "OPEN cur_fk;\n" + "WHILE found = 1 LOOP\n" + " FETCH cur_fk INTO foreign_id;\n" + " IF (SQL % NOTFOUND) THEN\n" + " found := 0;\n" + " ELSE\n" + " DELETE FROM SYS_FOREIGN_COLS\n" + " WHERE ID = foreign_id;\n" + " DELETE FROM SYS_FOREIGN\n" + " WHERE ID = foreign_id;\n" + " END IF;\n" + "END LOOP;\n" + "CLOSE cur_fk;\n" + "found := 1;\n" + "OPEN cur_idx;\n" + "WHILE found = 1 LOOP\n" + " FETCH cur_idx INTO index_id;\n" + " IF (SQL % NOTFOUND) THEN\n" + " found := 0;\n" + " ELSE\n" + " DELETE FROM SYS_FIELDS\n" + " WHERE INDEX_ID = index_id;\n" + " DELETE FROM SYS_INDEXES\n" + " WHERE ID = index_id\n" + " AND TABLE_ID = table_id;\n" + " END IF;\n" + "END LOOP;\n" + "CLOSE cur_idx;\n" + "DELETE FROM SYS_COLUMNS\n" + "WHERE TABLE_ID = table_id;\n" + "DELETE FROM SYS_TABLES\n" + "WHERE NAME = :table_name;\n" + "END;\n" + , FALSE, trx); + + /* SYS_DATAFILES and SYS_TABLESPACES do not necessarily exist + on XtraBackup recovery. See comments around + dict_create_or_check_foreign_constraint_tables() in + innobase_start_or_create_for_mysql(). */ + if (dict_table_get_low("SYS_DATAFILES") != NULL) { + info = pars_info_create(); + + pars_info_add_str_literal(info, "table_name", name); + + que_eval_sql(info, + "PROCEDURE DROP_TABLE_PROC () IS\n" + "space_id INT;\n" + + "BEGIN\n" + "SELECT SPACE INTO space_id\n" + "FROM SYS_TABLES\n" + "WHERE NAME = :table_name;\n" + "IF (SQL % NOTFOUND) THEN\n" + " RETURN;\n" + "END IF;\n" + "DELETE FROM SYS_TABLESPACES\n" + "WHERE SPACE = space_id;\n" + "DELETE FROM SYS_DATAFILES\n" + "WHERE SPACE = space_id;\n" + "END;\n" + , FALSE, trx); + } + + trx_commit_for_mysql(trx); + + trx_free_for_mysql(trx); +} + + #ifndef UNIV_HOTBACKUP /********************************************************************//** Report information about a bad tablespace. */ @@ -4144,8 +4304,10 @@ fil_open_single_table_tablespace( in the default location. If it is remote, it should not be here. */ def.filepath = fil_make_ibd_name(tablename, false); - /* The path_in was read from SYS_DATAFILES. */ - if (path_in) { + /* The path_in was read from SYS_DATAFILES. + We skip SYS_DATAFILES validation and remote tablespaces discovery for + XtraBackup, as all tablespaces are local for XtraBackup recovery. */ + if (path_in && !IS_XTRABACKUP()) { if (strcmp(def.filepath, path_in)) { dict.filepath = mem_strdup(path_in); /* possibility of multiple files. */ @@ -4287,12 +4449,19 @@ fil_open_single_table_tablespace( /* The following call prints an error message */ os_file_get_last_error(true); - ib_logf(IB_LOG_LEVEL_ERROR, + ib_logf(IS_XTRABACKUP() ? IB_LOG_LEVEL_WARN : IB_LOG_LEVEL_ERROR, "Could not find a valid tablespace file for '%s'. " "See " REFMAN "innodb-troubleshooting-datadict.html " "for how to resolve the issue.", tablename); + if (IS_XTRABACKUP() && fix_dict) { + ib_logf(IB_LOG_LEVEL_WARN, + "It will be removed from the data dictionary."); + if (purge_sys) { + fil_remove_invalid_table_from_data_dict(tablename); + } + } err = DB_CORRUPTION; goto cleanup_and_exit; @@ -4717,6 +4886,11 @@ check_first_page: } if (!fsp->success) { + if (IS_XTRABACKUP()) { + /* Do not attempt restore from doublewrite buffer + in Xtrabackup, this does not work.*/ + return; + } if (!restore_attempted) { if (!fil_user_tablespace_find_space_id(fsp)) { return; @@ -4784,6 +4958,10 @@ fil_load_single_table_tablespace( os_offset_t size; fil_space_t* space; + fsp_open_info* fsp; + ulong minimum_size; + ibool file_space_create_success; + memset(&def, 0, sizeof(def)); memset(&remote, 0, sizeof(remote)); @@ -4839,6 +5017,7 @@ fil_load_single_table_tablespace( # endif /* !UNIV_HOTBACKUP */ #endif + /* Check for a link file which locates a remote tablespace. */ remote.success = fil_open_linked_file( tablename, &remote.filepath, &remote.file, FALSE); @@ -4849,6 +5028,17 @@ fil_load_single_table_tablespace( if (!remote.success) { os_file_close(remote.file); mem_free(remote.filepath); + + if (srv_backup_mode && (remote.id == ULINT_UNDEFINED + || remote.id == 0)) { + + /* Ignore files that have uninitialized space + IDs on the backup stage. This means that a + tablespace has just been created and we will + replay the corresponding log records on + prepare. */ + goto func_exit_after_close; + } } } @@ -4863,6 +5053,18 @@ fil_load_single_table_tablespace( fil_validate_single_table_tablespace(tablename, &def); if (!def.success) { os_file_close(def.file); + + if (IS_XTRABACKUP() && srv_backup_mode && (def.id == ULINT_UNDEFINED + || def.id == 0)) { + + /* Ignore files that have uninitialized space + IDs on the backup stage. This means that a + tablespace has just been created and we will + replay the corresponding log records on + prepare. */ + + goto func_exit_after_close; + } } } @@ -4948,7 +5150,7 @@ will_not_choose: /* At this point, only one tablespace is open */ ut_a(def.success == !remote.success); - fsp_open_info* fsp = def.success ? &def : &remote; + fsp = def.success ? &def : &remote; /* Get and test the file size. */ size = os_file_get_size(fsp->file); @@ -4967,19 +5169,14 @@ will_not_choose: /* Every .ibd file is created >= 4 pages in size. Smaller files cannot be ok. */ - ulong minimum_size = FIL_IBD_FILE_INITIAL_SIZE * UNIV_PAGE_SIZE; + minimum_size = FIL_IBD_FILE_INITIAL_SIZE * UNIV_PAGE_SIZE; if (size < minimum_size) { -#ifndef UNIV_HOTBACKUP ib_logf(IB_LOG_LEVEL_ERROR, "The size of single-table tablespace file %s " "is only " UINT64PF ", should be at least %lu!", fsp->filepath, size, minimum_size); os_file_close(fsp->file); goto no_good_file; -#else - fsp->id = ULINT_UNDEFINED; - fsp->flags = 0; -#endif /* !UNIV_HOTBACKUP */ } #ifdef UNIV_HOTBACKUP @@ -5050,6 +5247,7 @@ will_not_choose: } mutex_exit(&fil_system->mutex); #endif /* UNIV_HOTBACKUP */ + /* Adjust the memory-based flags that would normally be set by dict_tf_to_fsp_flags(). In recovery, we have no data dictionary. */ if (FSP_FLAGS_HAS_PAGE_COMPRESSION(fsp->flags)) { @@ -5060,7 +5258,7 @@ will_not_choose: /* We will leave atomic_writes at ATOMIC_WRITES_DEFAULT. That will be adjusted in fil_space_for_table_exists_in_mem(). */ - ibool file_space_create_success = fil_space_create( + file_space_create_success = fil_space_create( tablename, fsp->id, fsp->flags, FIL_TABLESPACE, fsp->crypt_data, false); @@ -5088,13 +5286,56 @@ will_not_choose: } func_exit: - os_file_close(fsp->file); + /* We reuse file handles on the backup stage in XtraBackup to avoid + inconsistencies between the file name and the actual tablespace contents + if a DDL occurs between a fil_load_single_table_tablespaces() call and + the actual copy operation. */ + if (IS_XTRABACKUP() && srv_backup_mode && !srv_close_files) { + + fil_node_t* node; + fil_space_t* space; + + mutex_enter(&fil_system->mutex); + + space = fil_space_get_by_id(fsp->id); + + if (space) { + node = UT_LIST_GET_LAST(space->chain); + + /* The handle will be closed by xtrabackup in + xtrabackup_copy_datafile(). We set node->open to TRUE to + make sure no one calls fil_node_open_file() + (i.e. attempts to reopen the tablespace by name) during + the backup stage. */ + + node->open = TRUE; + node->handle = fsp->file; + + /* The following is copied from fil_node_open_file() to + pass fil_system validaty checks. We cannot use + fil_node_open_file() directly, as that would re-open the + file by name and create another file handle. */ + + fil_system->n_open++; + fil_n_file_opened++; + + if (fil_space_belongs_in_lru(space)) { + + /* Put the node to the LRU list */ + UT_LIST_ADD_FIRST(LRU, fil_system->LRU, node); + } + } + + mutex_exit(&fil_system->mutex); + } + else { + os_file_close(fsp->file); + } + -#ifdef UNIV_HOTBACKUP func_exit_after_close: -#else ut_ad(!mutex_own(&fil_system->mutex)); -#endif + mem_free(tablename); if (remote.success) { mem_free(remote.filepath); @@ -5108,7 +5349,7 @@ directory. We retry 100 times if os_file_readdir_next_file() returns -1. The idea is to read as much good data as we can and jump over bad data. @return 0 if ok, -1 if error even after the retries, 1 if at the end of the directory */ -static +UNIV_INTERN int fil_file_readdir_next_file( /*=======================*/ @@ -5149,7 +5390,7 @@ space id is != 0. @return DB_SUCCESS or error number */ UNIV_INTERN dberr_t -fil_load_single_table_tablespaces(void) +fil_load_single_table_tablespaces(ibool (*pred)(const char*, const char*)) /*===================================*/ { int ret; @@ -5224,14 +5465,20 @@ fil_load_single_table_tablespaces(void) goto next_file_item; } - /* We found a symlink or a file */ + /* We found a symlink or a file + + Ignore .isl files on XtraBackup + recovery, all tablespaces must be local. */ if (strlen(fileinfo.name) > 4 && (0 == strcmp(fileinfo.name + strlen(fileinfo.name) - 4, ".ibd") - || 0 == strcmp(fileinfo.name - + strlen(fileinfo.name) - 4, - ".isl"))) { + || ((!IS_XTRABACKUP() || srv_backup_mode) + && 0 == strcmp(fileinfo.name + + strlen(fileinfo.name) - 4, + ".isl"))) + && (!pred || + pred(dbinfo.name, fileinfo.name))) { /* The name ends in .ibd or .isl; try opening the file */ fil_load_single_table_tablespace( @@ -5387,6 +5634,9 @@ fil_space_for_table_exists_in_mem( information to the .err log if a matching tablespace is not found from memory */ + bool remove_from_data_dict_if_does_not_exist, + /*!< in: remove from the data dictionary + if tablespace does not exist */ bool adjust_space, /*!< in: whether to adjust space id when find table space mismatch */ mem_heap_t* heap, /*!< in: heap memory */ @@ -5457,6 +5707,11 @@ fil_space_for_table_exists_in_mem( if (fnamespace == NULL) { if (print_error_if_does_not_exist) { fil_report_missing_tablespace(name, id); + if (IS_XTRABACKUP() && remove_from_data_dict_if_does_not_exist) { + ib_logf(IB_LOG_LEVEL_WARN, + "It will be removed from " + "the data dictionary."); + } } } else { ut_print_timestamp(stderr); @@ -6119,7 +6374,7 @@ _fil_io( /* Check that at least the start offset is within the bounds of a single-table tablespace, including rollback tablespaces. */ if (UNIV_UNLIKELY(node->size <= block_offset) - && space->id != 0 && space->purpose == FIL_TABLESPACE) { + && space->id != 0 && space->purpose == FIL_TABLESPACE) { fil_report_invalid_page_access( block_offset, space_id, space->name, byte_offset, diff --git a/storage/xtradb/handler/ha_innodb.cc b/storage/xtradb/handler/ha_innodb.cc index 433ba46956f..18ecee4aee9 100644 --- a/storage/xtradb/handler/ha_innodb.cc +++ b/storage/xtradb/handler/ha_innodb.cc @@ -284,7 +284,8 @@ static TYPELIB innodb_stats_method_typelib = { /** Possible values for system variables "innodb_checksum_algorithm" and "innodb_log_checksum_algorithm". */ -static const char* innodb_checksum_algorithm_names[] = { +UNIV_INTERN +const char* innodb_checksum_algorithm_names[] = { "CRC32", "STRICT_CRC32", "INNODB", @@ -296,7 +297,8 @@ static const char* innodb_checksum_algorithm_names[] = { /** Used to define an enumerate type of the system variables innodb_checksum_algorithm and innodb_log_checksum_algorithm. */ -static TYPELIB innodb_checksum_algorithm_typelib = { +UNIV_INTERN +TYPELIB innodb_checksum_algorithm_typelib = { array_elements(innodb_checksum_algorithm_names) - 1, "innodb_checksum_algorithm_typelib", innodb_checksum_algorithm_names, @@ -2013,7 +2015,9 @@ thd_supports_xa( THD* thd) /*!< in: thread handle, or NULL to query the global innodb_supports_xa */ { - return(THDVAR(thd, support_xa)); + /* THDVAR cannot be used in xtrabackup, + plugin variables for innodb are not loaded. */ + return (thd || !IS_XTRABACKUP())? THDVAR(thd, support_xa): FALSE; } /** Get the value of innodb_tmpdir. @@ -2046,7 +2050,9 @@ thd_fake_changes( THD* thd) /*!< in: thread handle, or NULL to query the global innodb_supports_xa */ { - return(THDVAR((THD*) thd, fake_changes)); + /* THDVAR cannot be used in xtrabackup, + plugin variables for innodb are not loaded */ + return (thd || !IS_XTRABACKUP())? THDVAR((THD*) thd, fake_changes) : FALSE ; } /******************************************************************//** @@ -2086,7 +2092,10 @@ thd_flush_log_at_trx_commit( /*================================*/ void* thd) { - return(THDVAR((THD*) thd, flush_log_at_trx_commit)); + /* THDVAR cannot be used in xtrabackup, + plugin variables for innodb are not loaded, + this makes xtrabackup crash when trying to use them. */ + return (thd || !IS_XTRABACKUP())? THDVAR((THD*)thd, flush_log_at_trx_commit) : FALSE; } /********************************************************************//** @@ -3053,7 +3062,7 @@ trx_is_started( /****************************************************************//** Update log_checksum_algorithm_ptr with a pointer to the function corresponding to a given checksum algorithm. */ -static + void innodb_log_checksum_func_update( /*============================*/ @@ -21951,22 +21960,27 @@ ib_logf( str = static_cast(malloc(BUFSIZ)); my_vsnprintf(str, BUFSIZ, format, args); #endif /* __WIN__ */ - - switch(level) { - case IB_LOG_LEVEL_INFO: - sql_print_information("InnoDB: %s", str); - break; - case IB_LOG_LEVEL_WARN: - sql_print_warning("InnoDB: %s", str); - break; - case IB_LOG_LEVEL_ERROR: - sql_print_error("InnoDB: %s", str); - sd_notifyf(0, "STATUS=InnoDB: Error: %s", str); - break; - case IB_LOG_LEVEL_FATAL: - sql_print_error("InnoDB: %s", str); - sd_notifyf(0, "STATUS=InnoDB: Fatal: %s", str); - break; + if (!IS_XTRABACKUP()) { + switch (level) { + case IB_LOG_LEVEL_INFO: + sql_print_information("InnoDB: %s", str); + break; + case IB_LOG_LEVEL_WARN: + sql_print_warning("InnoDB: %s", str); + break; + case IB_LOG_LEVEL_ERROR: + sql_print_error("InnoDB: %s", str); + sd_notifyf(0, "STATUS=InnoDB: Error: %s", str); + break; + case IB_LOG_LEVEL_FATAL: + sql_print_error("InnoDB: %s", str); + sd_notifyf(0, "STATUS=InnoDB: Fatal: %s", str); + break; + } + } + else { + /* Don't use server logger for XtraBackup, just print to stderr. */ + fprintf(stderr, "InnoDB: %s\n", str); } va_end(args); diff --git a/storage/xtradb/include/fil0fil.h b/storage/xtradb/include/fil0fil.h index 0f7526a8e5e..37269db1960 100644 --- a/storage/xtradb/include/fil0fil.h +++ b/storage/xtradb/include/fil0fil.h @@ -136,6 +136,7 @@ extern fil_addr_t fil_addr_null; #define FIL_PAGE_DATA 38 /*!< start of the data on the page */ /* Following are used when page compression is used */ + #define FIL_PAGE_COMPRESSED_SIZE 2 /*!< Number of bytes used to store actual payload data size on compressed pages. */ @@ -1042,7 +1043,7 @@ space id is != 0. @return DB_SUCCESS or error number */ UNIV_INTERN dberr_t -fil_load_single_table_tablespaces(void); +fil_load_single_table_tablespaces(ibool (*pred)(const char*, const char*)=0); /*===================================*/ /*******************************************************************//** Returns TRUE if a single-table tablespace does not exist in the memory cache, @@ -1081,6 +1082,9 @@ fil_space_for_table_exists_in_mem( information to the .err log if a matching tablespace is not found from memory */ + bool remove_from_data_dict_if_does_not_exist, + /*!< in: remove from the data dictionary + if tablespace does not exist */ bool adjust_space, /*!< in: whether to adjust space id when find table space mismatch */ mem_heap_t* heap, /*!< in: heap memory */ diff --git a/storage/xtradb/include/srv0srv.h b/storage/xtradb/include/srv0srv.h index bb6be232517..6a962262f69 100644 --- a/storage/xtradb/include/srv0srv.h +++ b/storage/xtradb/include/srv0srv.h @@ -499,7 +499,9 @@ as enum type because the configure option takes unsigned integer type. */ extern ulong srv_innodb_stats_method; #ifdef UNIV_LOG_ARCHIVE -extern ibool srv_log_archive_on; +extern bool srv_log_archive_on; +extern bool srv_archive_recovery; +extern ib_uint64_t srv_archive_recovery_limit_lsn; #endif /* UNIV_LOG_ARCHIVE */ extern char* srv_file_flush_method_str; @@ -550,6 +552,14 @@ extern ulong srv_pass_corrupt_table; extern ulong srv_log_checksum_algorithm; +extern bool srv_apply_log_only; + +extern bool srv_backup_mode; +extern bool srv_close_files; +extern bool srv_xtrabackup; + +#define IS_XTRABACKUP() (srv_xtrabackup) + extern my_bool srv_force_primary_key; /* Helper macro to support srv_pass_corrupt_table checks. If 'cond' is FALSE, diff --git a/storage/xtradb/include/trx0sys.h b/storage/xtradb/include/trx0sys.h index e4956c6a822..274a8eb9304 100644 --- a/storage/xtradb/include/trx0sys.h +++ b/storage/xtradb/include/trx0sys.h @@ -336,8 +336,9 @@ trx_sys_update_wsrep_checkpoint( trx_sysf_t* sys_header, /*!< in: sys_header */ mtr_t* mtr); /*!< in: mtr */ -void -/** Read WSREP checkpoint XID from sys header. */ +/** Read WSREP checkpoint XID from sys header. +@return true on success, false on error. */ +bool trx_sys_read_wsrep_checkpoint( XID* xid); /*!< out: WSREP XID */ #endif /* WITH_WSREP */ diff --git a/storage/xtradb/include/univ.i b/storage/xtradb/include/univ.i index e698f08f15b..310053b9145 100644 --- a/storage/xtradb/include/univ.i +++ b/storage/xtradb/include/univ.i @@ -635,7 +635,7 @@ functions. */ #ifdef __WIN__ #define usleep(a) Sleep((a)/1000) -typedef ulint os_thread_ret_t; +typedef DWORD os_thread_ret_t; #define OS_THREAD_DUMMY_RETURN return(0) #else typedef void* os_thread_ret_t; diff --git a/storage/xtradb/log/log0log.cc b/storage/xtradb/log/log0log.cc index 25c8ed06981..309de7daaf8 100644 --- a/storage/xtradb/log/log0log.cc +++ b/storage/xtradb/log/log0log.cc @@ -2629,7 +2629,7 @@ loop: start_lsn += len; buf += len; - if (recv_sys->report(ut_time())) { + if (recv_sys && recv_sys->report(ut_time())) { ib_logf(IB_LOG_LEVEL_INFO, "Read redo log up to LSN=" LSN_PF, start_lsn); sd_notifyf(0, "STATUS=Read redo log up to LSN=" LSN_PF, diff --git a/storage/xtradb/log/log0recv.cc b/storage/xtradb/log/log0recv.cc index d6d31b0c572..978e6051711 100644 --- a/storage/xtradb/log/log0recv.cc +++ b/storage/xtradb/log/log0recv.cc @@ -692,7 +692,6 @@ recv_synchronize_groups( /***********************************************************************//** Checks the consistency of the checkpoint info @return TRUE if ok */ -static ibool recv_check_cp_is_consistent( /*========================*/ @@ -722,7 +721,7 @@ recv_check_cp_is_consistent( /********************************************************//** Looks for the maximum consistent checkpoint from the log groups. @return error code or DB_SUCCESS */ -static MY_ATTRIBUTE((nonnull, warn_unused_result)) +MY_ATTRIBUTE((nonnull, warn_unused_result)) dberr_t recv_find_max_checkpoint( /*=====================*/ @@ -3474,6 +3473,7 @@ recv_recovery_from_checkpoint_finish(void) #ifdef __WIN__ if (recv_writer_thread_handle) { CloseHandle(recv_writer_thread_handle); + recv_writer_thread_handle = 0; } #endif /* __WIN__ */ @@ -3700,6 +3700,102 @@ recv_reset_log_files_for_backup( } #endif /* UNIV_HOTBACKUP */ +/******************************************************//** +Checks the 4-byte checksum to the trailer checksum field of a log +block. We also accept a log block in the old format before +InnoDB-3.23.52 where the checksum field contains the log block number. +@return TRUE if ok, or if the log block may be in the format of InnoDB +version predating 3.23.52 */ +UNIV_INTERN +ibool +log_block_checksum_is_ok_or_old_format( +/*===================================*/ + const byte* block) /*!< in: pointer to a log block */ +{ +#ifdef UNIV_LOG_DEBUG + return(TRUE); +#endif /* UNIV_LOG_DEBUG */ + + ulint block_checksum = log_block_get_checksum(block); + + if (UNIV_LIKELY(srv_log_checksum_algorithm == + SRV_CHECKSUM_ALGORITHM_NONE || + log_block_calc_checksum(block) == block_checksum)) { + + return(TRUE); + } + + if (srv_log_checksum_algorithm == SRV_CHECKSUM_ALGORITHM_STRICT_CRC32 || + srv_log_checksum_algorithm == SRV_CHECKSUM_ALGORITHM_STRICT_INNODB || + srv_log_checksum_algorithm == SRV_CHECKSUM_ALGORITHM_STRICT_NONE) { + + const char* algo = NULL; + + ib_logf(IB_LOG_LEVEL_ERROR, + "log block checksum mismatch: expected " ULINTPF ", " + "calculated checksum " ULINTPF, + block_checksum, + log_block_calc_checksum(block)); + + if (block_checksum == LOG_NO_CHECKSUM_MAGIC) { + + algo = "none"; + } else if (block_checksum == + log_block_calc_checksum_crc32(block)) { + + algo = "crc32"; + } else if (block_checksum == + log_block_calc_checksum_innodb(block)) { + + algo = "innodb"; + } + + if (algo) { + + const char* current_algo; + + current_algo = buf_checksum_algorithm_name( + (srv_checksum_algorithm_t) + srv_log_checksum_algorithm); + + ib_logf(IB_LOG_LEVEL_ERROR, + "current InnoDB log checksum type: %s, " + "detected log checksum type: %s", + current_algo, + algo); + } + + ib_logf(IB_LOG_LEVEL_FATAL, + "STRICT method was specified for innodb_log_checksum, " + "so we intentionally assert here."); + } + + ut_ad(srv_log_checksum_algorithm == SRV_CHECKSUM_ALGORITHM_CRC32 || + srv_log_checksum_algorithm == SRV_CHECKSUM_ALGORITHM_INNODB); + + if (block_checksum == LOG_NO_CHECKSUM_MAGIC || + block_checksum == log_block_calc_checksum_crc32(block) || + block_checksum == log_block_calc_checksum_innodb(block)) { + + return(TRUE); + } + + if (log_block_get_hdr_no(block) == block_checksum) { + + /* We assume the log block is in the format of + InnoDB version < 3.23.52 and the block is ok */ +#if 0 + fprintf(stderr, + "InnoDB: Scanned old format < InnoDB-3.23.52" + " log block number %lu\n", + log_block_get_hdr_no(block)); +#endif + return(TRUE); + } + + return(FALSE); +} + void recv_dblwr_t::add(byte* page) { pages.push_back(page); diff --git a/storage/xtradb/os/os0file.cc b/storage/xtradb/os/os0file.cc index 03200fee80b..4f219b18428 100644 --- a/storage/xtradb/os/os0file.cc +++ b/storage/xtradb/os/os0file.cc @@ -130,7 +130,7 @@ UNIV_INTERN os_ib_mutex_t os_file_seek_mutexes[OS_FILE_N_SEEK_MUTEXES]; #define OS_AIO_MERGE_N_CONSECUTIVE 64 #ifdef WITH_INNODB_DISALLOW_WRITES -#define WAIT_ALLOW_WRITES() os_event_wait(srv_allow_writes_event) +#define WAIT_ALLOW_WRITES() if (!IS_XTRABACKUP()) os_event_wait(srv_allow_writes_event) #else #define WAIT_ALLOW_WRITES() do { } while (0) #endif /* WITH_INNODB_DISALLOW_WRITES */ @@ -1001,7 +1001,6 @@ os_file_lock( #ifndef UNIV_HOTBACKUP /****************************************************************//** Creates the seek mutexes used in positioned reads and writes. */ -static void os_io_init_simple(void) /*===================*/ @@ -1640,6 +1639,10 @@ os_file_create_simple_no_error_handling_func( return((os_file_t) -1); } + if (IS_XTRABACKUP()) { + share_mode |= FILE_SHARE_DELETE | FILE_SHARE_WRITE; + } + file = CreateFile((LPCTSTR) name, access, share_mode, @@ -1921,7 +1924,10 @@ os_file_create_func( create_mode &= ~OS_FILE_ON_ERROR_NO_EXIT; create_mode &= ~OS_FILE_ON_ERROR_SILENT; - + if (srv_backup_mode){ + /* Permit others to write, while I'm reading. */ + share_mode |= FILE_SHARE_WRITE; + } if (create_mode == OS_FILE_OPEN_RAW) { ut_a(!srv_read_only_mode); @@ -3525,7 +3531,7 @@ os_file_get_status( fh = CreateFile( (LPCTSTR) path, // File to open access, - 0, // No sharing + FILE_SHARE_READ|FILE_SHARE_WRITE, NULL, // Default security OPEN_EXISTING, // Existing file only FILE_ATTRIBUTE_NORMAL, // Normal file diff --git a/storage/xtradb/row/row0mysql.cc b/storage/xtradb/row/row0mysql.cc index a4141c36eda..b4d359d5009 100644 --- a/storage/xtradb/row/row0mysql.cc +++ b/storage/xtradb/row/row0mysql.cc @@ -4483,8 +4483,9 @@ row_drop_table_for_mysql( if (!is_temp && !fil_space_for_table_exists_in_mem( space_id, tablename, - print_msg, false, NULL, 0, + print_msg, IS_XTRABACKUP() && print_msg, false, NULL, 0, table_flags)) { + /* This might happen if we are dropping a discarded tablespace */ err = DB_SUCCESS; diff --git a/storage/xtradb/srv/srv0srv.cc b/storage/xtradb/srv/srv0srv.cc index 59c780ffa47..47928a343de 100644 --- a/storage/xtradb/srv/srv0srv.cc +++ b/storage/xtradb/srv/srv0srv.cc @@ -203,7 +203,7 @@ performance killer causing calling thread to context switch. Besides, Innodb is preallocating large number (often millions) of os_events. With kernel event objects it takes a big chunk out of non-paged pool, which is better suited for tasks like IO than for storing idle event objects. */ -UNIV_INTERN ibool srv_use_native_conditions = FALSE; +UNIV_INTERN ibool srv_use_native_conditions = TRUE; #endif /* __WIN__ */ UNIV_INTERN ulint srv_n_data_files = 0; @@ -366,7 +366,9 @@ readahead request. */ UNIV_INTERN ulong srv_read_ahead_threshold = 56; #ifdef UNIV_LOG_ARCHIVE -UNIV_INTERN ibool srv_log_archive_on = FALSE; +UNIV_INTERN bool srv_log_archive_on; +UNIV_INTERN bool srv_archive_recovery; +UNIV_INTERN ib_uint64_t srv_archive_recovery_limit_lsn; #endif /* UNIV_LOG_ARCHIVE */ /* This parameter is used to throttle the number of insert buffers that are @@ -522,6 +524,12 @@ UNIV_INTERN ulong srv_doublewrite_batch_size = 120; UNIV_INTERN ulong srv_replication_delay = 0; +UNIV_INTERN bool srv_apply_log_only; + +UNIV_INTERN bool srv_backup_mode; +UNIV_INTERN bool srv_close_files; +UNIV_INTERN bool srv_xtrabackup; + UNIV_INTERN ulong srv_pass_corrupt_table = 0; /* 0:disable 1:enable */ UNIV_INTERN ulong srv_log_checksum_algorithm = diff --git a/storage/xtradb/srv/srv0start.cc b/storage/xtradb/srv/srv0start.cc index bc1d5acfd4d..c383121de55 100644 --- a/storage/xtradb/srv/srv0start.cc +++ b/storage/xtradb/srv/srv0start.cc @@ -140,7 +140,7 @@ SRV_SHUTDOWN_CLEANUP and then to SRV_SHUTDOWN_LAST_PHASE, and so on */ UNIV_INTERN enum srv_shutdown_state srv_shutdown_state = SRV_SHUTDOWN_NONE; /** Files comprising the system tablespace */ -static os_file_t files[1000]; +os_file_t files[1000]; /** io_handler_thread parameters for thread identification */ static ulint n[SRV_MAX_N_IO_THREADS]; @@ -826,7 +826,7 @@ open_log_file( /*********************************************************************//** Creates or opens database data files and closes them. @return DB_SUCCESS or error code */ -static MY_ATTRIBUTE((nonnull, warn_unused_result)) +MY_ATTRIBUTE((nonnull, warn_unused_result)) dberr_t open_or_create_data_files( /*======================*/ @@ -1080,8 +1080,10 @@ skip_size_check: /* This is the earliest location where we can load the double write buffer. */ if (i == 0) { + /* XtraBackup never loads corrupted pages from + the doublewrite buffer */ buf_dblwr_init_or_load_pages( - files[i], srv_data_file_names[i], true); + files[i], srv_data_file_names[i], !IS_XTRABACKUP()); } bool retry = true; @@ -1365,12 +1367,15 @@ srv_undo_tablespace_open( /******************************************************************** Opens the configured number of undo tablespaces. @return DB_SUCCESS or error code */ -static dberr_t srv_undo_tablespaces_init( /*======================*/ ibool create_new_db, /*!< in: TRUE if new db being created */ + ibool backup_mode, /*!< in: TRUE disables reading + the system tablespace (used in + XtraBackup), FALSE is passed on + recovery. */ const ulint n_conf_tablespaces, /*!< in: configured undo tablespaces */ ulint* n_opened) /*!< out: number of UNDO @@ -1424,7 +1429,7 @@ srv_undo_tablespaces_init( we build the undo_tablespace_ids ourselves since they don't already exist. */ - if (!create_new_db) { + if (!create_new_db && !backup_mode) { n_undo_tablespaces = trx_rseg_get_n_undo_tablespaces( undo_tablespace_ids); } else { @@ -2318,11 +2323,11 @@ innobase_start_or_create_for_mysql(void) max_flushed_lsn = min_flushed_lsn = log_get_lsn(); goto files_checked; - } else if (i < 2) { - /* must have at least 2 log files */ - ib_logf(IB_LOG_LEVEL_ERROR, - "Only one log file found."); - return(err); + } else if (i < 2 && !IS_XTRABACKUP()) { + /* must have at least 2 log files */ + ib_logf(IB_LOG_LEVEL_ERROR, + "Only one log file found."); + return(err); } /* opened all files */ @@ -2416,6 +2421,7 @@ files_checked: err = srv_undo_tablespaces_init( create_new_db, + FALSE, srv_undo_tablespaces, &srv_undo_tablespaces_open); @@ -2689,6 +2695,17 @@ files_checked: dict_check_tablespaces_and_store_max_id(dict_check); } + if (IS_XTRABACKUP() + && !srv_backup_mode + && srv_read_only_mode + && srv_log_file_size_requested != srv_log_file_size) { + + ib_logf(IB_LOG_LEVEL_WARN, + "Log files size mismatch, ignored in readonly mode"); + srv_log_file_size_requested = srv_log_file_size; + } + + if (!srv_force_recovery && !recv_sys->found_corrupt_log && (srv_log_file_size_requested != srv_log_file_size @@ -3323,7 +3340,8 @@ innobase_shutdown_for_mysql(void) srv_was_started = FALSE; srv_start_has_been_called = FALSE; - + /* reset io_tid_i, in case current process does second innodb start (xtrabackup might do that).*/ + io_tid_i = 0; return(DB_SUCCESS); } #endif /* !UNIV_HOTBACKUP */ diff --git a/storage/xtradb/trx/trx0sys.cc b/storage/xtradb/trx/trx0sys.cc index 1b4d4f00074..ba45e67322c 100644 --- a/storage/xtradb/trx/trx0sys.cc +++ b/storage/xtradb/trx/trx0sys.cc @@ -397,7 +397,7 @@ trx_sys_update_wsrep_checkpoint( } -void +bool trx_sys_read_wsrep_checkpoint(XID* xid) /*===================================*/ { @@ -418,7 +418,7 @@ trx_sys_read_wsrep_checkpoint(XID* xid) xid->formatID = -1; trx_sys_update_wsrep_checkpoint(xid, sys_header, &mtr); mtr_commit(&mtr); - return; + return false; } xid->formatID = (int)mach_read_from_4( @@ -435,6 +435,7 @@ trx_sys_read_wsrep_checkpoint(XID* xid) XIDDATASIZE); mtr_commit(&mtr); + return true; } #endif /* WITH_WSREP */ @@ -1330,14 +1331,17 @@ trx_sys_close(void) trx_purge_sys_close(); /* Free the double write data structures. */ - buf_dblwr_free(); + if (buf_dblwr) { + buf_dblwr_free(); + } - ut_a(UT_LIST_GET_LEN(trx_sys->ro_trx_list) == 0); /* Only prepared transactions may be left in the system. Free them. */ ut_a(UT_LIST_GET_LEN(trx_sys->rw_trx_list) == trx_sys->n_prepared_trx || srv_read_only_mode - || srv_force_recovery >= SRV_FORCE_NO_TRX_UNDO); + || srv_force_recovery >= SRV_FORCE_NO_TRX_UNDO + || (IS_XTRABACKUP() && srv_apply_log_only)); + while ((trx = UT_LIST_GET_FIRST(trx_sys->rw_trx_list)) != NULL) { trx_free_prepared(trx); @@ -1368,10 +1372,12 @@ trx_sys_close(void) UT_LIST_REMOVE(view_list, trx_sys->view_list, prev_view); } - ut_a(UT_LIST_GET_LEN(trx_sys->view_list) == 0); - ut_a(UT_LIST_GET_LEN(trx_sys->ro_trx_list) == 0); - ut_a(UT_LIST_GET_LEN(trx_sys->rw_trx_list) == 0); - ut_a(UT_LIST_GET_LEN(trx_sys->mysql_trx_list) == 0); + if (!IS_XTRABACKUP() || !srv_apply_log_only) { + ut_a(UT_LIST_GET_LEN(trx_sys->view_list) == 0); + ut_a(UT_LIST_GET_LEN(trx_sys->ro_trx_list) == 0); + ut_a(UT_LIST_GET_LEN(trx_sys->rw_trx_list) == 0); + ut_a(UT_LIST_GET_LEN(trx_sys->mysql_trx_list) == 0); + } mutex_free(&trx_sys->mutex); @@ -1418,6 +1424,9 @@ ulint trx_sys_any_active_transactions(void) /*=================================*/ { + if (IS_XTRABACKUP() && srv_apply_log_only) { + return(0); + } mutex_enter(&trx_sys->mutex); ulint total_trx = UT_LIST_GET_LEN(trx_sys->mysql_trx_list); diff --git a/storage/xtradb/trx/trx0trx.cc b/storage/xtradb/trx/trx0trx.cc index ed80bafa6c6..d0cb4a883cc 100644 --- a/storage/xtradb/trx/trx0trx.cc +++ b/storage/xtradb/trx/trx0trx.cc @@ -721,9 +721,16 @@ trx_resurrect_insert( if (srv_force_recovery == 0) { - trx->state = TRX_STATE_PREPARED; - trx_sys->n_prepared_trx++; - trx_sys->n_prepared_recovered_trx++; + /* XtraBackup should rollback prepared XA + transactions */ + if (IS_XTRABACKUP()) { + trx->state = TRX_STATE_ACTIVE; + } + else { + trx->state = TRX_STATE_PREPARED; + trx_sys->n_prepared_trx++; + trx_sys->n_prepared_recovered_trx++; + } } else { fprintf(stderr, "InnoDB: Since innodb_force_recovery" @@ -790,13 +797,16 @@ trx_resurrect_update_in_prepared_state( if (srv_force_recovery == 0) { if (trx_state_eq(trx, TRX_STATE_NOT_STARTED)) { - trx_sys->n_prepared_trx++; - trx_sys->n_prepared_recovered_trx++; + if (!IS_XTRABACKUP()) { + trx_sys->n_prepared_trx++; + trx_sys->n_prepared_recovered_trx++; + } } else { ut_ad(trx_state_eq(trx, TRX_STATE_PREPARED)); } - - trx->state = TRX_STATE_PREPARED; + /* XtraBackup should rollback prepared XA + transactions */ + trx->state = IS_XTRABACKUP()?TRX_STATE_ACTIVE: TRX_STATE_PREPARED; } else { fprintf(stderr, "InnoDB: Since innodb_force_recovery" From d7714308e014a7335e1c0f60dc0afbf5eba5ff99 Mon Sep 17 00:00:00 2001 From: Vladislav Vaintroub Date: Tue, 18 Apr 2017 18:43:20 +0000 Subject: [PATCH 09/28] MDEV-9566 Add Percona Xtrabackup 2.3.7 --- extra/mariabackup/CMakeLists.txt | 173 + extra/mariabackup/backup_copy.cc | 2047 ++++++ extra/mariabackup/backup_copy.h | 51 + extra/mariabackup/backup_mysql.cc | 1756 +++++ extra/mariabackup/backup_mysql.h | 92 + extra/mariabackup/changed_page_bitmap.cc | 1018 +++ extra/mariabackup/changed_page_bitmap.h | 85 + extra/mariabackup/common.h | 134 + extra/mariabackup/compact.cc | 1059 +++ extra/mariabackup/compact.h | 44 + extra/mariabackup/datasink.c | 130 + extra/mariabackup/datasink.h | 98 + extra/mariabackup/ds_archive.c | 275 + extra/mariabackup/ds_archive.h | 28 + extra/mariabackup/ds_buffer.c | 189 + extra/mariabackup/ds_buffer.h | 39 + extra/mariabackup/ds_compress.c | 462 ++ extra/mariabackup/ds_compress.h | 28 + extra/mariabackup/ds_encrypt.c | 617 ++ extra/mariabackup/ds_encrypt.h | 28 + extra/mariabackup/ds_local.c | 151 + extra/mariabackup/ds_local.h | 28 + extra/mariabackup/ds_stdout.c | 121 + extra/mariabackup/ds_stdout.h | 28 + extra/mariabackup/ds_tmpfile.c | 248 + extra/mariabackup/ds_tmpfile.h | 30 + extra/mariabackup/ds_xbstream.c | 223 + extra/mariabackup/ds_xbstream.h | 28 + extra/mariabackup/fil_cur.cc | 402 ++ extra/mariabackup/fil_cur.h | 123 + extra/mariabackup/innobackupex.cc | 1153 ++++ extra/mariabackup/innobackupex.h | 45 + extra/mariabackup/quicklz/quicklz.c | 848 +++ extra/mariabackup/quicklz/quicklz.h | 144 + extra/mariabackup/read_filt.cc | 206 + extra/mariabackup/read_filt.h | 62 + extra/mariabackup/version_check.pl | 1373 ++++ extra/mariabackup/write_filt.cc | 219 + extra/mariabackup/write_filt.h | 61 + extra/mariabackup/wsrep.cc | 219 + extra/mariabackup/wsrep.h | 32 + extra/mariabackup/xb_regex.h | 71 + extra/mariabackup/xbcloud.cc | 2721 ++++++++ extra/mariabackup/xbcrypt.c | 694 ++ extra/mariabackup/xbcrypt.h | 84 + extra/mariabackup/xbcrypt_common.c | 60 + extra/mariabackup/xbcrypt_read.c | 251 + extra/mariabackup/xbcrypt_write.c | 104 + extra/mariabackup/xbstream.c | 456 ++ extra/mariabackup/xbstream.h | 103 + extra/mariabackup/xbstream_read.c | 227 + extra/mariabackup/xbstream_write.c | 280 + extra/mariabackup/xtrabackup.cc | 7220 +++++++++++++++++++++ extra/mariabackup/xtrabackup.h | 232 + extra/mariabackup/xtrabackup_version.h.in | 27 + 55 files changed, 26627 insertions(+) create mode 100644 extra/mariabackup/CMakeLists.txt create mode 100644 extra/mariabackup/backup_copy.cc create mode 100644 extra/mariabackup/backup_copy.h create mode 100644 extra/mariabackup/backup_mysql.cc create mode 100644 extra/mariabackup/backup_mysql.h create mode 100644 extra/mariabackup/changed_page_bitmap.cc create mode 100644 extra/mariabackup/changed_page_bitmap.h create mode 100644 extra/mariabackup/common.h create mode 100644 extra/mariabackup/compact.cc create mode 100644 extra/mariabackup/compact.h create mode 100644 extra/mariabackup/datasink.c create mode 100644 extra/mariabackup/datasink.h create mode 100644 extra/mariabackup/ds_archive.c create mode 100644 extra/mariabackup/ds_archive.h create mode 100644 extra/mariabackup/ds_buffer.c create mode 100644 extra/mariabackup/ds_buffer.h create mode 100644 extra/mariabackup/ds_compress.c create mode 100644 extra/mariabackup/ds_compress.h create mode 100644 extra/mariabackup/ds_encrypt.c create mode 100644 extra/mariabackup/ds_encrypt.h create mode 100644 extra/mariabackup/ds_local.c create mode 100644 extra/mariabackup/ds_local.h create mode 100644 extra/mariabackup/ds_stdout.c create mode 100644 extra/mariabackup/ds_stdout.h create mode 100644 extra/mariabackup/ds_tmpfile.c create mode 100644 extra/mariabackup/ds_tmpfile.h create mode 100644 extra/mariabackup/ds_xbstream.c create mode 100644 extra/mariabackup/ds_xbstream.h create mode 100644 extra/mariabackup/fil_cur.cc create mode 100644 extra/mariabackup/fil_cur.h create mode 100644 extra/mariabackup/innobackupex.cc create mode 100644 extra/mariabackup/innobackupex.h create mode 100644 extra/mariabackup/quicklz/quicklz.c create mode 100644 extra/mariabackup/quicklz/quicklz.h create mode 100644 extra/mariabackup/read_filt.cc create mode 100644 extra/mariabackup/read_filt.h create mode 100644 extra/mariabackup/version_check.pl create mode 100644 extra/mariabackup/write_filt.cc create mode 100644 extra/mariabackup/write_filt.h create mode 100644 extra/mariabackup/wsrep.cc create mode 100644 extra/mariabackup/wsrep.h create mode 100644 extra/mariabackup/xb_regex.h create mode 100644 extra/mariabackup/xbcloud.cc create mode 100644 extra/mariabackup/xbcrypt.c create mode 100644 extra/mariabackup/xbcrypt.h create mode 100644 extra/mariabackup/xbcrypt_common.c create mode 100644 extra/mariabackup/xbcrypt_read.c create mode 100644 extra/mariabackup/xbcrypt_write.c create mode 100644 extra/mariabackup/xbstream.c create mode 100644 extra/mariabackup/xbstream.h create mode 100644 extra/mariabackup/xbstream_read.c create mode 100644 extra/mariabackup/xbstream_write.c create mode 100644 extra/mariabackup/xtrabackup.cc create mode 100644 extra/mariabackup/xtrabackup.h create mode 100644 extra/mariabackup/xtrabackup_version.h.in diff --git a/extra/mariabackup/CMakeLists.txt b/extra/mariabackup/CMakeLists.txt new file mode 100644 index 00000000000..a84ac304486 --- /dev/null +++ b/extra/mariabackup/CMakeLists.txt @@ -0,0 +1,173 @@ +# Copyright (c) 2013 Percona LLC and/or its affiliates. +# +# This program is free software; you can redistribute it and/or modify +# it under the terms of the GNU General Public License as published by +# the Free Software Foundation; version 2 of the License. + +# This program is distributed in the hope that it will be useful, +# but WITHOUT ANY WARRANTY; without even the implied warranty of +# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the +# GNU General Public License for more details. + +# You should have received a copy of the GNU General Public License +# along with this program; if not, write to the Free Software +# Foundation, Inc., 51 Franklin Street, Fifth Floor, Boston, MA 02110-1301, USA + +INCLUDE(gcrypt) +INCLUDE(curl) +INCLUDE(libev) + +ADD_SUBDIRECTORY(libarchive) +ADD_SUBDIRECTORY(jsmn) + +FIND_GCRYPT() +FIND_CURL() +FIND_EV() + +# xxd is needed to embed version_check script +FIND_PROGRAM(XXD_PATH xxd) + +IF(NOT XXD_PATH) + MESSAGE(FATAL_ERROR "xxd not found. Try to install vim-common.") +ENDIF(NOT XXD_PATH) + +INCLUDE_DIRECTORIES( + ${CMAKE_SOURCE_DIR}/include + ${CMAKE_SOURCE_DIR}/storage/innobase/include + ${CMAKE_SOURCE_DIR}/sql + ${CMAKE_SOURCE_DIR}/storage/innobase/xtrabackup/src/libarchive/libarchive + ${CMAKE_SOURCE_DIR}/storage/innobase/xtrabackup/src/quicklz + ${CMAKE_SOURCE_DIR}/storage/innobase/xtrabackup/src/jsmn + ${GCRYPT_INCLUDE_DIR} + ${CURL_INCLUDE_DIRS} + ${LIBEV_INCLUDE_DIRS} + ${CMAKE_CURRENT_BINARY_DIR} + ) + +ADD_DEFINITIONS(${SSL_DEFINES}) + +######################################################################## +# xtrabackup binary +######################################################################## +CONFIGURE_FILE(${CMAKE_CURRENT_SOURCE_DIR}/xtrabackup_version.h.in + ${CMAKE_CURRENT_BINARY_DIR}/xtrabackup_version.h ) + +ADD_CUSTOM_COMMAND(OUTPUT ${CMAKE_CURRENT_BINARY_DIR}/version_check_pl.h + COMMAND ${XXD_PATH} --include version_check.pl + ${CMAKE_CURRENT_BINARY_DIR}/version_check_pl.h + WORKING_DIRECTORY ${CMAKE_CURRENT_SOURCE_DIR}) + +ADD_CUSTOM_TARGET(GenVersionCheck + DEPENDS ${CMAKE_CURRENT_BINARY_DIR}/version_check_pl.h) + +SET_SOURCE_FILES_PROPERTIES( + xtrabackup.cc + backup_mysql.cc + PROPERTIES COMPILE_FLAGS -DMYSQL_CLIENT) + +MYSQL_ADD_EXECUTABLE(xtrabackup + xtrabackup.cc + innobackupex.cc + changed_page_bitmap.cc + compact.cc + datasink.c + ds_archive.c + ds_buffer.c + ds_compress.c + ds_encrypt.c + ds_local.c + ds_stdout.c + ds_tmpfile.c + ds_xbstream.c + fil_cur.cc + quicklz/quicklz.c + read_filt.cc + write_filt.cc + wsrep.cc + xbcrypt_common.c + xbcrypt_write.c + xbstream_write.c + backup_mysql.cc + backup_copy.cc + ../../../../sql-common/client_authentication.cc + ) + +SET_TARGET_PROPERTIES(xtrabackup PROPERTIES ENABLE_EXPORTS TRUE) + +TARGET_LINK_LIBRARIES(xtrabackup + mysqlserver + ${GCRYPT_LIBS} + archive_static + ) + +ADD_DEPENDENCIES(xtrabackup GenVersionCheck) + +######################################################################## +# innobackupex symlink +######################################################################## +ADD_CUSTOM_COMMAND(TARGET xtrabackup + COMMAND ${CMAKE_COMMAND} ARGS -E create_symlink + xtrabackup innobackupex) +INSTALL(FILES ${CMAKE_CURRENT_BINARY_DIR}/innobackupex DESTINATION bin) + +######################################################################## +# xbstream binary +######################################################################## +MYSQL_ADD_EXECUTABLE(xbstream + ds_buffer.c + ds_local.c + ds_stdout.c + datasink.c + xbstream.c + xbstream_read.c + xbstream_write.c + ) + +SET_TARGET_PROPERTIES(xbstream + PROPERTIES LINKER_LANGUAGE CXX + ) + +TARGET_LINK_LIBRARIES(xbstream + mysys + mysys_ssl + ) + +######################################################################## +# xbcrypt binary +######################################################################## +MYSQL_ADD_EXECUTABLE(xbcrypt + xbcrypt.c + xbcrypt_common.c + xbcrypt_read.c + xbcrypt_write.c + ) + +SET_TARGET_PROPERTIES(xbcrypt + PROPERTIES LINKER_LANGUAGE CXX + ) + +TARGET_LINK_LIBRARIES(xbcrypt + ${GCRYPT_LIBS} + mysys + mysys_ssl + ) + +######################################################################## +# xbcloud binary +######################################################################## +MYSQL_ADD_EXECUTABLE(xbcloud + xbcloud.cc + ) + +SET_TARGET_PROPERTIES(xbcloud + PROPERTIES LINKER_LANGUAGE CXX + ) + +TARGET_LINK_LIBRARIES(xbcloud + ${GCRYPT_LIBS} + ${LIBEV_LIBRARIES} + ${CURL_LIBRARIES} + mysys + mysys_ssl + jsmn + ) diff --git a/extra/mariabackup/backup_copy.cc b/extra/mariabackup/backup_copy.cc new file mode 100644 index 00000000000..a051c64ee3b --- /dev/null +++ b/extra/mariabackup/backup_copy.cc @@ -0,0 +1,2047 @@ +/****************************************************** +hot backup tool for InnoDB +(c) 2009-2015 Percona LLC and/or its affiliates +Originally Created 3/3/2009 Yasufumi Kinoshita +Written by Alexey Kopytov, Aleksandr Kuzminsky, Stewart Smith, Vadim Tkachenko, +Yasufumi Kinoshita, Ignacio Nin and Baron Schwartz. + +This program is free software; you can redistribute it and/or modify +it under the terms of the GNU General Public License as published by +the Free Software Foundation; version 2 of the License. + +This program is distributed in the hope that it will be useful, +but WITHOUT ANY WARRANTY; without even the implied warranty of +MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the +GNU General Public License for more details. + +You should have received a copy of the GNU General Public License +along with this program; if not, write to the Free Software +Foundation, Inc., 51 Franklin Street, Fifth Floor, Boston, MA 02110-1301, USA + +******************************************************* + +This file incorporates work covered by the following copyright and +permission notice: + +Copyright (c) 2000, 2011, MySQL AB & Innobase Oy. All Rights Reserved. + +This program is free software; you can redistribute it and/or modify it under +the terms of the GNU General Public License as published by the Free Software +Foundation; version 2 of the License. + +This program is distributed in the hope that it will be useful, but WITHOUT +ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or FITNESS +FOR A PARTICULAR PURPOSE. See the GNU General Public License for more details. + +You should have received a copy of the GNU General Public License along with +this program; if not, write to the Free Software Foundation, Inc., 59 Temple +Place, Suite 330, Boston, MA 02111-1307 USA + +*******************************************************/ + +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include "fil_cur.h" +#include "xtrabackup.h" +#include "common.h" +#include "backup_copy.h" +#include "backup_mysql.h" + + +/* list of files to sync for --rsync mode */ +static std::set rsync_list; +/* locations of tablespaces read from .isl files */ +static std::map tablespace_locations; + +/* Whether LOCK BINLOG FOR BACKUP has been issued during backup */ +bool binlog_locked; + +/************************************************************************ +Struct represents file or directory. */ +struct datadir_node_t { + ulint dbpath_len; + char *filepath; + ulint filepath_len; + char *filepath_rel; + ulint filepath_rel_len; + bool is_empty_dir; + bool is_file; +}; + +/************************************************************************ +Holds the state needed to enumerate files in MySQL data directory. */ +struct datadir_iter_t { + char *datadir_path; + char *dbpath; + ulint dbpath_len; + char *filepath; + ulint filepath_len; + char *filepath_rel; + ulint filepath_rel_len; + os_ib_mutex_t mutex; + os_file_dir_t dir; + os_file_dir_t dbdir; + os_file_stat_t dbinfo; + os_file_stat_t fileinfo; + dberr_t err; + bool is_empty_dir; + bool is_file; + bool skip_first_level; +}; + + +/************************************************************************ +Represents the context of the thread processing MySQL data directory. */ +struct datadir_thread_ctxt_t { + datadir_iter_t *it; + uint n_thread; + uint *count; + os_ib_mutex_t count_mutex; + os_thread_id_t id; + bool ret; +}; + + +/************************************************************************ +Retirn true if character if file separator */ +bool +is_path_separator(char c) +{ + return(c == FN_LIBCHAR || c == FN_LIBCHAR2); +} + + +/************************************************************************ +Fill the node struct. Memory for node need to be allocated and freed by +the caller. It is caller responsibility to initialize node with +datadir_node_init and cleanup the memory with datadir_node_free. +Node can not be shared between threads. */ +static +void +datadir_node_fill(datadir_node_t *node, datadir_iter_t *it) +{ + if (node->filepath_len < it->filepath_len) { + free(node->filepath); + node->filepath = (char*)(ut_malloc(it->filepath_len)); + node->filepath_len = it->filepath_len; + } + if (node->filepath_rel_len < it->filepath_rel_len) { + free(node->filepath_rel); + node->filepath_rel = (char*)(ut_malloc(it->filepath_rel_len)); + node->filepath_rel_len = it->filepath_rel_len; + } + + strcpy(node->filepath, it->filepath); + strcpy(node->filepath_rel, it->filepath_rel); + node->is_empty_dir = it->is_empty_dir; + node->is_file = it->is_file; +} + +static +void +datadir_node_free(datadir_node_t *node) +{ + ut_free(node->filepath); + ut_free(node->filepath_rel); + memset(node, 0, sizeof(datadir_node_t)); +} + +static +void +datadir_node_init(datadir_node_t *node) +{ + memset(node, 0, sizeof(datadir_node_t)); +} + + +/************************************************************************ +Create the MySQL data directory iterator. Memory needs to be released +with datadir_iter_free. Position should be advanced with +datadir_iter_next_file. Iterator can be shared between multiple +threads. It is guaranteed that each thread receives unique file from +data directory into its local node struct. */ +static +datadir_iter_t * +datadir_iter_new(const char *path, bool skip_first_level = true) +{ + datadir_iter_t *it; + + it = static_cast(ut_malloc(sizeof(datadir_iter_t))); + memset(it, 0, sizeof(datadir_iter_t)); + + it->mutex = os_mutex_create(); + it->datadir_path = strdup(path); + + it->dir = os_file_opendir(it->datadir_path, TRUE); + + if (it->dir == NULL) { + + goto error; + } + + it->err = DB_SUCCESS; + + it->dbpath_len = FN_REFLEN; + it->dbpath = static_cast(ut_malloc(it->dbpath_len)); + + it->filepath_len = FN_REFLEN; + it->filepath = static_cast(ut_malloc(it->filepath_len)); + + it->filepath_rel_len = FN_REFLEN; + it->filepath_rel = static_cast(ut_malloc(it->filepath_rel_len)); + + it->skip_first_level = skip_first_level; + + return(it); + +error: + ut_free(it); + + return(NULL); +} + +static +bool +datadir_iter_next_database(datadir_iter_t *it) +{ + if (it->dbdir != NULL) { + if (os_file_closedir(it->dbdir) != 0) { + + msg("Warning: could not" + " close database directory %s\n", it->dbpath); + + it->err = DB_ERROR; + + } + it->dbdir = NULL; + } + + while (fil_file_readdir_next_file(&it->err, it->datadir_path, + it->dir, &it->dbinfo) == 0) { + ulint len; + + if ((it->dbinfo.type == OS_FILE_TYPE_FILE + && it->skip_first_level) + || it->dbinfo.type == OS_FILE_TYPE_UNKNOWN) { + + continue; + } + + /* We found a symlink or a directory; try opening it to see + if a symlink is a directory */ + + len = strlen(it->datadir_path) + + strlen (it->dbinfo.name) + 2; + if (len > it->dbpath_len) { + it->dbpath_len = len; + + if (it->dbpath) { + + ut_free(it->dbpath); + } + + it->dbpath = static_cast + (ut_malloc(it->dbpath_len)); + } + ut_snprintf(it->dbpath, it->dbpath_len, + "%s/%s", it->datadir_path, + it->dbinfo.name); + srv_normalize_path_for_win(it->dbpath); + + if (it->dbinfo.type == OS_FILE_TYPE_FILE) { + it->is_file = true; + return(true); + } + + /* We want wrong directory permissions to be a fatal error for + XtraBackup. */ + it->dbdir = os_file_opendir(it->dbpath, TRUE); + + if (it->dbdir != NULL) { + + it->is_file = false; + return(true); + } + + } + + return(false); +} + +/************************************************************************ +Concatenate n parts into single path */ +static +void +make_path_n(int n, char **path, ulint *path_len, ...) +{ + ulint len_needed = n + 1; + char *p; + int i; + va_list vl; + + ut_ad(n > 0); + + va_start(vl, path_len); + for (i = 0; i < n; i++) { + p = va_arg(vl, char*); + len_needed += strlen(p); + } + va_end(vl); + + if (len_needed < *path_len) { + ut_free(*path); + *path = static_cast(ut_malloc(len_needed)); + } + + va_start(vl, path_len); + p = va_arg(vl, char*); + strcpy(*path, p); + for (i = 1; i < n; i++) { + size_t plen; + p = va_arg(vl, char*); + plen = strlen(*path); + if (!is_path_separator((*path)[plen - 1])) { + (*path)[plen] = FN_LIBCHAR; + (*path)[plen + 1] = 0; + } + strcat(*path + plen, p); + } + va_end(vl); +} + +static +bool +datadir_iter_next_file(datadir_iter_t *it) +{ + if (it->is_file && it->dbpath) { + make_path_n(2, &it->filepath, &it->filepath_len, + it->datadir_path, it->dbinfo.name); + + make_path_n(1, &it->filepath_rel, &it->filepath_rel_len, + it->dbinfo.name); + + it->is_empty_dir = false; + it->is_file = false; + + return(true); + } + + if (!it->dbpath || !it->dbdir) { + + return(false); + } + + while (fil_file_readdir_next_file(&it->err, it->dbpath, it->dbdir, + &it->fileinfo) == 0) { + + if (it->fileinfo.type == OS_FILE_TYPE_DIR) { + + continue; + } + + /* We found a symlink or a file */ + make_path_n(3, &it->filepath, &it->filepath_len, + it->datadir_path, it->dbinfo.name, + it->fileinfo.name); + + make_path_n(2, &it->filepath_rel, &it->filepath_rel_len, + it->dbinfo.name, it->fileinfo.name); + + it->is_empty_dir = false; + + return(true); + } + + return(false); +} + +static +bool +datadir_iter_next(datadir_iter_t *it, datadir_node_t *node) +{ + bool ret = true; + + os_mutex_enter(it->mutex); + + if (datadir_iter_next_file(it)) { + + datadir_node_fill(node, it); + + goto done; + } + + while (datadir_iter_next_database(it)) { + + if (datadir_iter_next_file(it)) { + + datadir_node_fill(node, it); + + goto done; + } + + make_path_n(2, &it->filepath, &it->filepath_len, + it->datadir_path, it->dbinfo.name); + + make_path_n(1, &it->filepath_rel, &it->filepath_rel_len, + it->dbinfo.name); + + it->is_empty_dir = true; + + datadir_node_fill(node, it); + + goto done; + } + + /* nothing found */ + ret = false; + +done: + os_mutex_exit(it->mutex); + + return(ret); +} + +/************************************************************************ +Interface to read MySQL data file sequentially. One should open file +with datafile_open to get cursor and close the cursor with +datafile_close. Cursor can not be shared between multiple +threads. */ +static +void +datadir_iter_free(datadir_iter_t *it) +{ + os_mutex_free(it->mutex); + + if (it->dbdir) { + + os_file_closedir(it->dbdir); + } + + if (it->dir) { + + os_file_closedir(it->dir); + } + + ut_free(it->dbpath); + ut_free(it->filepath); + ut_free(it->filepath_rel); + free(it->datadir_path); + ut_free(it); +} + + +/************************************************************************ +Holds the state needed to copy single data file. */ +struct datafile_cur_t { + os_file_t file; + char rel_path[FN_REFLEN]; + char abs_path[FN_REFLEN]; + MY_STAT statinfo; + uint thread_n; + byte* orig_buf; + byte* buf; + ib_int64_t buf_size; + ib_int64_t buf_read; + ib_int64_t buf_offset; +}; + +static +void +datafile_close(datafile_cur_t *cursor) +{ + if (cursor->file != 0) { + os_file_close(cursor->file); + } + ut_free(cursor->buf); +} + +static +bool +datafile_open(const char *file, datafile_cur_t *cursor, uint thread_n) +{ + ulint success; + + memset(cursor, 0, sizeof(datafile_cur_t)); + + strncpy(cursor->abs_path, file, sizeof(cursor->abs_path)); + + /* Get the relative path for the destination tablespace name, i.e. the + one that can be appended to the backup root directory. Non-system + tablespaces may have absolute paths for remote tablespaces in MySQL + 5.6+. We want to make "local" copies for the backup. */ + strncpy(cursor->rel_path, + xb_get_relative_path(cursor->abs_path, FALSE), + sizeof(cursor->rel_path)); + + cursor->file = os_file_create_simple_no_error_handling(0, + cursor->abs_path, + OS_FILE_OPEN, + OS_FILE_READ_ONLY, + &success); + if (!success) { + /* The following call prints an error message */ + os_file_get_last_error(TRUE); + + msg("[%02u] error: cannot open " + "file %s\n", + thread_n, cursor->abs_path); + + return(false); + } + + if (my_fstat(cursor->file, &cursor->statinfo, MYF(MY_WME))) { + msg("[%02u] error: cannot stat %s\n", + thread_n, cursor->abs_path); + + datafile_close(cursor); + + return(false); + } + + posix_fadvise(cursor->file, 0, 0, POSIX_FADV_SEQUENTIAL); + + cursor->buf_size = 10 * 1024 * 1024; + cursor->buf = static_cast(ut_malloc(cursor->buf_size)); + + return(true); +} + + +static +xb_fil_cur_result_t +datafile_read(datafile_cur_t *cursor) +{ + ulint success; + ulint to_read; + + xtrabackup_io_throttling(); + + to_read = min(cursor->statinfo.st_size - cursor->buf_offset, + cursor->buf_size); + + if (to_read == 0) { + return(XB_FIL_CUR_EOF); + } + + success = os_file_read(cursor->file, cursor->buf, cursor->buf_offset, + to_read); + if (!success) { + return(XB_FIL_CUR_ERROR); + } + + posix_fadvise(cursor->file, cursor->buf_offset, to_read, + POSIX_FADV_DONTNEED); + + cursor->buf_read = to_read; + cursor->buf_offset += to_read; + + return(XB_FIL_CUR_SUCCESS); +} + + + +/************************************************************************ +Check to see if a file exists. +Takes name of the file to check. +@return true if file exists. */ +static +bool +file_exists(const char *filename) +{ + MY_STAT stat_arg; + + if (!my_stat(filename, &stat_arg, MYF(0))) { + + return(false); + } + + return(true); +} + +/************************************************************************ +Trim leading slashes from absolute path so it becomes relative */ +static +const char * +trim_dotslash(const char *path) +{ + while (*path) { + if (is_path_separator(*path)) { + ++path; + continue; + } + if (*path == '.' && is_path_separator(path[1])) { + path += 2; + continue; + } + break; + } + + return(path); +} + + + +/************************************************************************ +Check if string ends with given suffix. +@return true if string ends with given suffix. */ +static +bool +ends_with(const char *str, const char *suffix) +{ + size_t suffix_len = strlen(suffix); + size_t str_len = strlen(str); + return(str_len >= suffix_len + && strcmp(str + str_len - suffix_len, suffix) == 0); +} + +/************************************************************************ +Create directories recursively. +@return 0 if directories created successfully. */ +static +int +mkdirp(const char *pathname, int Flags, myf MyFlags) +{ + char parent[PATH_MAX], *p; + + /* make a parent directory path */ + strncpy(parent, pathname, sizeof(parent)); + parent[sizeof(parent) - 1] = 0; + + for (p = parent + strlen(parent); + !is_path_separator(*p) && p != parent; p--); + + *p = 0; + + /* try to make parent directory */ + if (p != parent && mkdirp(parent, Flags, MyFlags) != 0) { + return(-1); + } + + /* make this one if parent has been made */ + if (my_mkdir(pathname, Flags, MyFlags) == 0) { + return(0); + } + + /* if it already exists that is fine */ + if (errno == EEXIST) { + return(0); + } + + return(-1); +} + +/************************************************************************ +Return true if first and second arguments are the same path. */ +bool +equal_paths(const char *first, const char *second) +{ + char real_first[PATH_MAX]; + char real_second[PATH_MAX]; + + if (realpath(first, real_first) == NULL) { + return false; + } + if (realpath(second, real_second) == NULL) { + return false; + } + + return (strcmp(real_first, real_second) == 0); +} + +/************************************************************************ +Check if directory exists. Optionally create directory if doesn't +exist. +@return true if directory exists and if it was created successfully. */ +bool +directory_exists(const char *dir, bool create) +{ + os_file_dir_t os_dir; + MY_STAT stat_arg; + char errbuf[MYSYS_STRERROR_SIZE]; + + if (my_stat(dir, &stat_arg, MYF(0)) == NULL) { + + if (!create) { + return(false); + } + + if (mkdirp(dir, 0777, MYF(0)) < 0) { + + msg("Can not create directory %s: %s\n", dir, + my_strerror(errbuf, sizeof(errbuf), my_errno)); + + return(false); + + } + } + + /* could be symlink */ + os_dir = os_file_opendir(dir, FALSE); + + if (os_dir == NULL) { + + msg("Can not open directory %s: %s\n", dir, + my_strerror(errbuf, sizeof(errbuf), my_errno)); + + return(false); + } + + os_file_closedir(os_dir); + + return(true); +} + +/************************************************************************ +Check that directory exists and it is empty. */ +static +bool +directory_exists_and_empty(const char *dir, const char *comment) +{ + os_file_dir_t os_dir; + dberr_t err; + os_file_stat_t info; + bool empty; + + if (!directory_exists(dir, true)) { + return(false); + } + + os_dir = os_file_opendir(dir, FALSE); + + if (os_dir == NULL) { + msg("%s can not open directory %s\n", comment, dir); + return(false); + } + + empty = (fil_file_readdir_next_file(&err, dir, os_dir, &info) != 0); + + os_file_closedir(os_dir); + + if (!empty) { + msg("%s directory %s is not empty!\n", comment, dir); + } + + return(empty); +} + + +/************************************************************************ +Check if file name ends with given set of suffixes. +@return true if it does. */ +static +bool +filename_matches(const char *filename, const char **ext_list) +{ + const char **ext; + + for (ext = ext_list; *ext; ext++) { + if (ends_with(filename, *ext)) { + return(true); + } + } + + return(false); +} + + +/************************************************************************ +Copy data file for backup. Also check if it is allowed to copy by +comparing its name to the list of known data file types and checking +if passes the rules for partial backup. +@return true if file backed up or skipped successfully. */ +static +bool +datafile_copy_backup(const char *filepath, uint thread_n) +{ + const char *ext_list[] = {"frm", "isl", "MYD", "MYI", "MAD", "MAI", + "MRG", "TRG", "TRN", "ARM", "ARZ", "CSM", "CSV", "opt", "par", + NULL}; + + /* Get the name and the path for the tablespace. node->name always + contains the path (which may be absolute for remote tablespaces in + 5.6+). space->name contains the tablespace name in the form + "./database/table.ibd" (in 5.5-) or "database/table" (in 5.6+). For a + multi-node shared tablespace, space->name contains the name of the first + node, but that's irrelevant, since we only need node_name to match them + against filters, and the shared tablespace is always copied regardless + of the filters value. */ + + if (check_if_skip_table(filepath)) { + msg_ts("[%02u] Skipping %s.\n", thread_n, filepath); + return(true); + } + + if (filename_matches(filepath, ext_list)) { + return copy_file(ds_data, filepath, filepath, thread_n); + } + + return(true); +} + + +/************************************************************************ +Same as datafile_copy_backup, but put file name into the list for +rsync command. */ +static +bool +datafile_rsync_backup(const char *filepath, bool save_to_list, FILE *f) +{ + const char *ext_list[] = {"frm", "isl", "MYD", "MYI", "MAD", "MAI", + "MRG", "TRG", "TRN", "ARM", "ARZ", "CSM", "CSV", "opt", "par", + NULL}; + + /* Get the name and the path for the tablespace. node->name always + contains the path (which may be absolute for remote tablespaces in + 5.6+). space->name contains the tablespace name in the form + "./database/table.ibd" (in 5.5-) or "database/table" (in 5.6+). For a + multi-node shared tablespace, space->name contains the name of the first + node, but that's irrelevant, since we only need node_name to match them + against filters, and the shared tablespace is always copied regardless + of the filters value. */ + + if (check_if_skip_table(filepath)) { + return(true); + } + + if (filename_matches(filepath, ext_list)) { + fprintf(f, "%s\n", filepath); + if (save_to_list) { + rsync_list.insert(filepath); + } + } + + return(true); +} + + +static +bool +backup_file_vprintf(const char *filename, const char *fmt, va_list ap) +{ + ds_file_t *dstfile = NULL; + MY_STAT stat; /* unused for now */ + char *buf = 0; + int buf_len; + const char *action; + + memset(&stat, 0, sizeof(stat)); + + buf_len = vasprintf(&buf, fmt, ap); + + stat.st_size = buf_len; + stat.st_mtime = my_time(0); + + dstfile = ds_open(ds_data, filename, &stat); + if (dstfile == NULL) { + msg("[%02u] error: " + "cannot open the destination stream for %s\n", + 0, filename); + goto error; + } + + action = xb_get_copy_action("Writing"); + msg_ts("[%02u] %s %s\n", 0, action, filename); + + if (buf_len == -1) { + goto error; + } + + if (ds_write(dstfile, buf, buf_len)) { + goto error; + } + + /* close */ + msg_ts("[%02u] ...done\n", 0); + free(buf); + + if (ds_close(dstfile)) { + goto error_close; + } + + return(true); + +error: + free(buf); + if (dstfile != NULL) { + ds_close(dstfile); + } + +error_close: + msg("[%02u] Error: backup file failed.\n", 0); + return(false); /*ERROR*/ +} + + +bool +backup_file_printf(const char *filename, const char *fmt, ...) +{ + bool result; + va_list ap; + + va_start(ap, fmt); + + result = backup_file_vprintf(filename, fmt, ap); + + va_end(ap); + + return(result); +} + +static +bool +run_data_threads(datadir_iter_t *it, os_thread_func_t func, uint n) +{ + datadir_thread_ctxt_t *data_threads; + uint i, count; + os_ib_mutex_t count_mutex; + bool ret; + + data_threads = (datadir_thread_ctxt_t*) + (ut_malloc(sizeof(datadir_thread_ctxt_t) * n)); + + count_mutex = os_mutex_create(); + count = n; + + for (i = 0; i < n; i++) { + data_threads[i].it = it; + data_threads[i].n_thread = i + 1; + data_threads[i].count = &count; + data_threads[i].count_mutex = count_mutex; + os_thread_create(func, data_threads + i, &data_threads[i].id); + } + + /* Wait for threads to exit */ + while (1) { + os_thread_sleep(100000); + os_mutex_enter(count_mutex); + if (count == 0) { + os_mutex_exit(count_mutex); + break; + } + os_mutex_exit(count_mutex); + } + + os_mutex_free(count_mutex); + + ret = true; + for (i = 0; i < n; i++) { + ret = data_threads[i].ret && ret; + if (!data_threads[i].ret) { + msg("Error: thread %u failed.\n", i); + } + } + + ut_free(data_threads); + + return(ret); +} + + +/************************************************************************ +Copy file for backup/restore. +@return true in case of success. */ +bool +copy_file(ds_ctxt_t *datasink, + const char *src_file_path, + const char *dst_file_path, + uint thread_n) +{ + char dst_name[FN_REFLEN]; + ds_file_t *dstfile = NULL; + datafile_cur_t cursor; + xb_fil_cur_result_t res; + const char *action; + + if (!datafile_open(src_file_path, &cursor, thread_n)) { + goto error; + } + + strncpy(dst_name, cursor.rel_path, sizeof(dst_name)); + + dstfile = ds_open(datasink, trim_dotslash(dst_file_path), + &cursor.statinfo); + if (dstfile == NULL) { + msg("[%02u] error: " + "cannot open the destination stream for %s\n", + thread_n, dst_name); + goto error; + } + + action = xb_get_copy_action(); + msg_ts("[%02u] %s %s to %s\n", + thread_n, action, src_file_path, dstfile->path); + + /* The main copy loop */ + while ((res = datafile_read(&cursor)) == XB_FIL_CUR_SUCCESS) { + + if (ds_write(dstfile, cursor.buf, cursor.buf_read)) { + goto error; + } + } + + if (res == XB_FIL_CUR_ERROR) { + goto error; + } + + /* close */ + msg_ts("[%02u] ...done\n", thread_n); + datafile_close(&cursor); + if (ds_close(dstfile)) { + goto error_close; + } + return(true); + +error: + datafile_close(&cursor); + if (dstfile != NULL) { + ds_close(dstfile); + } + +error_close: + msg("[%02u] Error: copy_file() failed.\n", thread_n); + return(false); /*ERROR*/ +} + + +/************************************************************************ +Try to move file by renaming it. If source and destination are on +different devices fall back to copy and unlink. +@return true in case of success. */ +static +bool +move_file(ds_ctxt_t *datasink, + const char *src_file_path, + const char *dst_file_path, + const char *dst_dir, uint thread_n) +{ + char errbuf[MYSYS_STRERROR_SIZE]; + char dst_file_path_abs[FN_REFLEN]; + char dst_dir_abs[FN_REFLEN]; + size_t dirname_length; + + ut_snprintf(dst_file_path_abs, sizeof(dst_file_path_abs), + "%s/%s", dst_dir, dst_file_path); + + dirname_part(dst_dir_abs, dst_file_path_abs, &dirname_length); + + if (!directory_exists(dst_dir_abs, true)) { + return(false); + } + + if (file_exists(dst_file_path_abs)) { + msg("Error: Move file %s to %s failed: Destination " + "file exists\n", + src_file_path, dst_file_path_abs); + return(false); + } + + msg_ts("[%02u] Moving %s to %s\n", + thread_n, src_file_path, dst_file_path_abs); + + if (my_rename(src_file_path, dst_file_path_abs, MYF(0)) != 0) { + if (my_errno == EXDEV) { + bool ret; + ret = copy_file(datasink, src_file_path, + dst_file_path, thread_n); + msg_ts("[%02u] Removing %s\n", thread_n, src_file_path); + if (unlink(src_file_path) != 0) { + msg("Error: unlink %s failed: %s\n", + src_file_path, + my_strerror(errbuf, + sizeof(errbuf), errno)); + } + return(ret); + } + msg("Can not move file %s to %s: %s\n", + src_file_path, dst_file_path_abs, + my_strerror(errbuf, sizeof(errbuf), my_errno)); + return(false); + } + + msg_ts("[%02u] ...done\n", thread_n); + + return(true); +} + + +/************************************************************************ +Read link from .isl file if any and store it in the global map associated +with given tablespace. */ +static +void +read_link_file(const char *ibd_filepath, const char *link_filepath) +{ + char *filepath= NULL; + + FILE *file = fopen(link_filepath, "r+b"); + if (file) { + filepath = static_cast(malloc(OS_FILE_MAX_PATH)); + + os_file_read_string(file, filepath, OS_FILE_MAX_PATH); + fclose(file); + + if (strlen(filepath)) { + /* Trim whitespace from end of filepath */ + ulint lastch = strlen(filepath) - 1; + while (lastch > 4 && filepath[lastch] <= 0x20) { + filepath[lastch--] = 0x00; + } + srv_normalize_path_for_win(filepath); + } + + tablespace_locations[ibd_filepath] = filepath; + } + free(filepath); +} + + +/************************************************************************ +Return the location of given .ibd if it was previously read +from .isl file. +@return NULL or destination .ibd file path. */ +static +const char * +tablespace_filepath(const char *ibd_filepath) +{ + std::map::iterator it; + + it = tablespace_locations.find(ibd_filepath); + + if (it != tablespace_locations.end()) { + return it->second.c_str(); + } + + return NULL; +} + + +/************************************************************************ +Copy or move file depending on current mode. +@return true in case of success. */ +static +bool +copy_or_move_file(const char *src_file_path, + const char *dst_file_path, + const char *dst_dir, + uint thread_n) +{ + ds_ctxt_t *datasink = ds_data; /* copy to datadir by default */ + char filedir[FN_REFLEN]; + size_t filedir_len; + bool ret; + + /* read the link from .isl file */ + if (ends_with(src_file_path, ".isl")) { + char *ibd_filepath; + + ibd_filepath = strdup(src_file_path); + strcpy(ibd_filepath + strlen(ibd_filepath) - 3, "ibd"); + + read_link_file(ibd_filepath, src_file_path); + + free(ibd_filepath); + } + + /* check if there is .isl file */ + if (ends_with(src_file_path, ".ibd")) { + char *link_filepath; + const char *filepath; + + link_filepath = strdup(src_file_path); + strcpy(link_filepath + strlen(link_filepath) - 3, "isl"); + + read_link_file(src_file_path, link_filepath); + + filepath = tablespace_filepath(src_file_path); + + if (filepath != NULL) { + dirname_part(filedir, filepath, &filedir_len); + + dst_file_path = filepath + filedir_len; + dst_dir = filedir; + + if (!directory_exists(dst_dir, true)) { + ret = false; + goto cleanup; + } + + datasink = ds_create(dst_dir, DS_TYPE_LOCAL); + } + + free(link_filepath); + } + + ret = (xtrabackup_copy_back ? + copy_file(datasink, src_file_path, dst_file_path, thread_n) : + move_file(datasink, src_file_path, dst_file_path, + dst_dir, thread_n)); + +cleanup: + + if (datasink != ds_data) { + ds_destroy(datasink); + } + + return(ret); +} + + + + +bool +backup_files(const char *from, bool prep_mode) +{ + char rsync_tmpfile_name[FN_REFLEN]; + FILE *rsync_tmpfile = NULL; + datadir_iter_t *it; + datadir_node_t node; + bool ret = true; + + if (prep_mode && !opt_rsync) { + return(true); + } + + if (opt_rsync) { + snprintf(rsync_tmpfile_name, sizeof(rsync_tmpfile_name), + "%s/%s%d", opt_mysql_tmpdir, + "xtrabackup_rsyncfiles_pass", + prep_mode ? 1 : 2); + rsync_tmpfile = fopen(rsync_tmpfile_name, "w"); + if (rsync_tmpfile == NULL) { + msg("Error: can't create file %s\n", + rsync_tmpfile_name); + return(false); + } + } + + msg_ts("Starting %s non-InnoDB tables and files\n", + prep_mode ? "prep copy of" : "to backup"); + + datadir_node_init(&node); + it = datadir_iter_new(from); + + while (datadir_iter_next(it, &node)) { + + if (!node.is_empty_dir) { + if (opt_rsync) { + ret = datafile_rsync_backup(node.filepath, + !prep_mode, rsync_tmpfile); + } else { + ret = datafile_copy_backup(node.filepath, 1); + } + if (!ret) { + msg("Failed to copy file %s\n", node.filepath); + goto out; + } + } else if (!prep_mode) { + /* backup fake file into empty directory */ + char path[FN_REFLEN]; + ut_snprintf(path, sizeof(path), + "%s/db.opt", node.filepath); + if (!(ret = backup_file_printf( + trim_dotslash(path), "%s", ""))) { + msg("Failed to create file %s\n", path); + goto out; + } + } + } + + if (opt_rsync) { + std::stringstream cmd; + int err; + + if (buffer_pool_filename && file_exists(buffer_pool_filename)) { + fprintf(rsync_tmpfile, "%s\n", buffer_pool_filename); + rsync_list.insert(buffer_pool_filename); + } + if (file_exists("ib_lru_dump")) { + fprintf(rsync_tmpfile, "%s\n", "ib_lru_dump"); + rsync_list.insert("ib_lru_dump"); + } + + fclose(rsync_tmpfile); + rsync_tmpfile = NULL; + + cmd << "rsync -t . --files-from=" << rsync_tmpfile_name + << " " << xtrabackup_target_dir; + + msg_ts("Starting rsync as: %s\n", cmd.str().c_str()); + if ((err = system(cmd.str().c_str()) && !prep_mode) != 0) { + msg_ts("Error: rsync failed with error code %d\n", err); + ret = false; + goto out; + } + msg_ts("rsync finished successfully.\n"); + + if (!prep_mode && !opt_no_lock) { + char path[FN_REFLEN]; + char dst_path[FN_REFLEN]; + char *newline; + + /* Remove files that have been removed between first and + second passes. Cannot use "rsync --delete" because it + does not work with --files-from. */ + snprintf(rsync_tmpfile_name, sizeof(rsync_tmpfile_name), + "%s/%s", opt_mysql_tmpdir, + "xtrabackup_rsyncfiles_pass1"); + + rsync_tmpfile = fopen(rsync_tmpfile_name, "r"); + if (rsync_tmpfile == NULL) { + msg("Error: can't open file %s\n", + rsync_tmpfile_name); + return(false); + } + + while (fgets(path, sizeof(path), rsync_tmpfile)) { + + newline = strchr(path, '\n'); + if (newline) { + *newline = 0; + } + if (rsync_list.count(path) < 1) { + snprintf(dst_path, sizeof(dst_path), + "%s/%s", xtrabackup_target_dir, + path); + msg_ts("Removing %s\n", dst_path); + unlink(dst_path); + } + } + + fclose(rsync_tmpfile); + rsync_tmpfile = NULL; + } + } + + msg_ts("Finished %s non-InnoDB tables and files\n", + prep_mode ? "a prep copy of" : "backing up"); + +out: + datadir_iter_free(it); + datadir_node_free(&node); + + if (rsync_tmpfile != NULL) { + fclose(rsync_tmpfile); + } + + return(ret); +} + +bool +backup_start() +{ + if (!opt_no_lock) { + if (opt_safe_slave_backup) { + if (!wait_for_safe_slave(mysql_connection)) { + return(false); + } + } + + if (!backup_files(fil_path_to_mysql_datadir, true)) { + return(false); + } + + history_lock_time = time(NULL); + + if (!lock_tables(mysql_connection)) { + return(false); + } + } + + if (!backup_files(fil_path_to_mysql_datadir, false)) { + return(false); + } + + // There is no need to stop slave thread before coping non-Innodb data when + // --no-lock option is used because --no-lock option requires that no DDL or + // DML to non-transaction tables can occur. + if (opt_no_lock) { + if (opt_safe_slave_backup) { + if (!wait_for_safe_slave(mysql_connection)) { + return(false); + } + } + } + + if (opt_slave_info) { + lock_binlog_maybe(mysql_connection); + + if (!write_slave_info(mysql_connection)) { + return(false); + } + } + + /* The only reason why Galera/binlog info is written before + wait_for_ibbackup_log_copy_finish() is that after that call the xtrabackup + binary will start streamig a temporary copy of REDO log to stdout and + thus, any streaming from innobackupex would interfere. The only way to + avoid that is to have a single process, i.e. merge innobackupex and + xtrabackup. */ + if (opt_galera_info) { + if (!write_galera_info(mysql_connection)) { + return(false); + } + write_current_binlog_file(mysql_connection); + } + + if (opt_binlog_info == BINLOG_INFO_ON) { + + lock_binlog_maybe(mysql_connection); + write_binlog_info(mysql_connection); + } + + if (have_flush_engine_logs) { + msg_ts("Executing FLUSH NO_WRITE_TO_BINLOG ENGINE LOGS...\n"); + xb_mysql_query(mysql_connection, + "FLUSH NO_WRITE_TO_BINLOG ENGINE LOGS", false); + } + + return(true); +} + + +bool +backup_finish() +{ + /* release all locks */ + if (!opt_no_lock) { + unlock_all(mysql_connection); + history_lock_time = 0; + } else { + history_lock_time = time(NULL) - history_lock_time; + } + + if (opt_safe_slave_backup && sql_thread_started) { + msg("Starting slave SQL thread\n"); + xb_mysql_query(mysql_connection, + "START SLAVE SQL_THREAD", false); + } + + /* Copy buffer pool dump or LRU dump */ + if (!opt_rsync) { + if (buffer_pool_filename && file_exists(buffer_pool_filename)) { + const char *dst_name; + + dst_name = trim_dotslash(buffer_pool_filename); + copy_file(ds_data, buffer_pool_filename, dst_name, 0); + } + if (file_exists("ib_lru_dump")) { + copy_file(ds_data, "ib_lru_dump", "ib_lru_dump", 0); + } + } + + msg_ts("Backup created in directory '%s'\n", xtrabackup_target_dir); + if (mysql_binlog_position != NULL) { + msg("MySQL binlog position: %s\n", mysql_binlog_position); + } + if (mysql_slave_position && opt_slave_info) { + msg("MySQL slave binlog position: %s\n", + mysql_slave_position); + } + + if (!write_backup_config_file()) { + return(false); + } + + if (!write_xtrabackup_info(mysql_connection)) { + return(false); + } + + + + return(true); +} + +bool +ibx_copy_incremental_over_full() +{ + const char *ext_list[] = {"frm", "isl", "MYD", "MYI", "MAD", "MAI", + "MRG", "TRG", "TRN", "ARM", "ARZ", "CSM", "CSV", "opt", "par", + NULL}; + const char *sup_files[] = {"xtrabackup_binlog_info", + "xtrabackup_galera_info", + "xtrabackup_slave_info", + "xtrabackup_info", + "ib_lru_dump", + NULL}; + datadir_iter_t *it = NULL; + datadir_node_t node; + bool ret = true; + char path[FN_REFLEN]; + int i; + + datadir_node_init(&node); + + /* If we were applying an incremental change set, we need to make + sure non-InnoDB files and xtrabackup_* metainfo files are copied + to the full backup directory. */ + + if (xtrabackup_incremental) { + + ds_data = ds_create(xtrabackup_target_dir, DS_TYPE_LOCAL); + + it = datadir_iter_new(xtrabackup_incremental_dir); + + while (datadir_iter_next(it, &node)) { + + /* copy only non-innodb files */ + + if (node.is_empty_dir + || !filename_matches(node.filepath, ext_list)) { + continue; + } + + if (file_exists(node.filepath_rel)) { + unlink(node.filepath_rel); + } + + if (!(ret = copy_file(ds_data, node.filepath, + node.filepath_rel, 1))) { + msg("Failed to copy file %s\n", + node.filepath); + goto cleanup; + } + } + + /* copy buffer pool dump */ + if (innobase_buffer_pool_filename) { + const char *src_name; + + src_name = trim_dotslash(innobase_buffer_pool_filename); + + snprintf(path, sizeof(path), "%s/%s", + xtrabackup_incremental_dir, + src_name); + + if (file_exists(path)) { + copy_file(ds_data, path, + innobase_buffer_pool_filename, 0); + } + } + + /* copy supplementary files */ + + for (i = 0; sup_files[i]; i++) { + snprintf(path, sizeof(path), "%s/%s", + xtrabackup_incremental_dir, + sup_files[i]); + + if (file_exists(path)) + { + if (file_exists(sup_files[i])) { + unlink(sup_files[i]); + } + copy_file(ds_data, path, sup_files[i], 0); + } + } + + } + +cleanup: + if (it != NULL) { + datadir_iter_free(it); + } + + if (ds_data != NULL) { + ds_destroy(ds_data); + } + + datadir_node_free(&node); + + return(ret); +} + +bool +ibx_cleanup_full_backup() +{ + const char *ext_list[] = {"delta", "meta", "ibd", NULL}; + datadir_iter_t *it = NULL; + datadir_node_t node; + bool ret = true; + + datadir_node_init(&node); + + /* If we are applying an incremental change set, we need to make + sure non-InnoDB files are cleaned up from full backup dir before + we copy files from incremental dir. */ + + it = datadir_iter_new(xtrabackup_target_dir); + + while (datadir_iter_next(it, &node)) { + + if (node.is_empty_dir) { + rmdir(node.filepath); + } + + if (xtrabackup_incremental && !node.is_empty_dir + && !filename_matches(node.filepath, ext_list)) { + unlink(node.filepath); + } + } + + datadir_iter_free(it); + + datadir_node_free(&node); + + return(ret); +} + +bool +apply_log_finish() +{ + if (!ibx_cleanup_full_backup() + || !ibx_copy_incremental_over_full()) { + return(false); + } + + return(true); +} + +bool +copy_back() +{ + char *innobase_data_file_path_copy; + ulint i; + bool ret; + datadir_iter_t *it = NULL; + datadir_node_t node; + char *dst_dir; + + memset(&node, 0, sizeof(node)); + + if (!opt_force_non_empty_dirs) { + if (!directory_exists_and_empty(mysql_data_home, + "Original data")) { + return(false); + } + } else { + if (!directory_exists(mysql_data_home, true)) { + return(false); + } + } + if (srv_undo_dir && *srv_undo_dir + && !directory_exists(srv_undo_dir, true)) { + return(false); + } + if (innobase_data_home_dir && *innobase_data_home_dir + && !directory_exists(innobase_data_home_dir, true)) { + return(false); + } + if (srv_log_group_home_dir && *srv_log_group_home_dir + && !directory_exists(srv_log_group_home_dir, true)) { + return(false); + } + + /* cd to backup directory */ + if (my_setwd(xtrabackup_target_dir, MYF(MY_WME))) + { + msg("cannot my_setwd %s\n", xtrabackup_target_dir); + return(false); + } + + /* parse data file path */ + + if (!innobase_data_file_path) { + innobase_data_file_path = (char*) "ibdata1:10M:autoextend"; + } + innobase_data_file_path_copy = strdup(innobase_data_file_path); + + if (!(ret = srv_parse_data_file_paths_and_sizes( + innobase_data_file_path_copy))) { + msg("syntax error in innodb_data_file_path\n"); + return(false); + } + + srv_max_n_threads = 1000; + os_sync_mutex = NULL; + ut_mem_init(); + /* temporally dummy value to avoid crash */ + srv_page_size_shift = 14; + srv_page_size = (1 << srv_page_size_shift); + os_sync_init(); + sync_init(); + os_io_init_simple(); + mem_init(srv_mem_pool_size); + ut_crc32_init(); + + /* copy undo tablespaces */ + if (srv_undo_tablespaces > 0) { + + dst_dir = (srv_undo_dir && *srv_undo_dir) + ? srv_undo_dir : mysql_data_home; + + ds_data = ds_create(dst_dir, DS_TYPE_LOCAL); + + for (i = 1; i <= srv_undo_tablespaces; i++) { + char filename[20]; + sprintf(filename, "undo%03lu", i); + if (!(ret = copy_or_move_file(filename, filename, + dst_dir, 1))) { + goto cleanup; + } + } + + ds_destroy(ds_data); + ds_data = NULL; + } + + /* copy redo logs */ + + dst_dir = (srv_log_group_home_dir && *srv_log_group_home_dir) + ? srv_log_group_home_dir : mysql_data_home; + + ds_data = ds_create(dst_dir, DS_TYPE_LOCAL); + + for (i = 0; i < (ulong)innobase_log_files_in_group; i++) { + char filename[20]; + sprintf(filename, "ib_logfile%lu", i); + + if (!file_exists(filename)) { + continue; + } + + if (!(ret = copy_or_move_file(filename, filename, + dst_dir, 1))) { + goto cleanup; + } + } + + ds_destroy(ds_data); + ds_data = NULL; + + /* copy innodb system tablespace(s) */ + + dst_dir = (innobase_data_home_dir && *innobase_data_home_dir) + ? innobase_data_home_dir : mysql_data_home; + + ds_data = ds_create(dst_dir, DS_TYPE_LOCAL); + + for (i = 0; i < srv_n_data_files; i++) { + const char *filename = base_name(srv_data_file_names[i]); + + if (!(ret = copy_or_move_file(filename, srv_data_file_names[i], + dst_dir, 1))) { + goto cleanup; + } + } + + ds_destroy(ds_data); + ds_data = NULL; + + /* copy the rest of tablespaces */ + ds_data = ds_create(mysql_data_home, DS_TYPE_LOCAL); + + it = datadir_iter_new(".", false); + + datadir_node_init(&node); + + while (datadir_iter_next(it, &node)) { + const char *ext_list[] = {"backup-my.cnf", "xtrabackup_logfile", + "xtrabackup_binary", "xtrabackup_binlog_info", + "xtrabackup_checkpoints", ".qp", ".pmap", ".tmp", + ".xbcrypt", NULL}; + const char *filename; + char c_tmp; + int i_tmp; + bool is_ibdata_file; + + /* create empty directories */ + if (node.is_empty_dir) { + char path[FN_REFLEN]; + + snprintf(path, sizeof(path), "%s/%s", + mysql_data_home, node.filepath_rel); + + msg_ts("[%02u] Creating directory %s\n", 1, path); + + if (mkdirp(path, 0777, MYF(0)) < 0) { + char errbuf[MYSYS_STRERROR_SIZE]; + + msg("Can not create directory %s: %s\n", + path, my_strerror(errbuf, + sizeof(errbuf), my_errno)); + ret = false; + + goto cleanup; + + } + + msg_ts("[%02u] ...done.", 1); + + continue; + } + + filename = base_name(node.filepath); + + /* skip .qp and .xbcrypt files */ + if (filename_matches(filename, ext_list)) { + continue; + } + + /* skip undo tablespaces */ + if (sscanf(filename, "undo%d%c", &i_tmp, &c_tmp) == 1) { + continue; + } + + /* skip redo logs */ + if (sscanf(filename, "ib_logfile%d%c", &i_tmp, &c_tmp) == 1) { + continue; + } + + /* skip innodb data files */ + is_ibdata_file = false; + for (i = 0; i < srv_n_data_files; i++) { + const char *ibfile; + + ibfile = base_name(srv_data_file_names[i]); + + if (strcmp(ibfile, filename) == 0) { + is_ibdata_file = true; + continue; + } + } + if (is_ibdata_file) { + continue; + } + + if (!(ret = copy_or_move_file(node.filepath, node.filepath_rel, + mysql_data_home, 1))) { + goto cleanup; + } + } + + /* copy buufer pool dump */ + + if (innobase_buffer_pool_filename) { + const char *src_name; + char path[FN_REFLEN]; + + src_name = trim_dotslash(innobase_buffer_pool_filename); + + snprintf(path, sizeof(path), "%s/%s", + mysql_data_home, + src_name); + + /* could be already copied with other files + from data directory */ + if (file_exists(src_name) && + !file_exists(innobase_buffer_pool_filename)) { + copy_or_move_file(src_name, + innobase_buffer_pool_filename, + mysql_data_home, 0); + } + } + +cleanup: + if (it != NULL) { + datadir_iter_free(it); + } + + datadir_node_free(&node); + + free(innobase_data_file_path_copy); + + if (ds_data != NULL) { + ds_destroy(ds_data); + } + + ds_data = NULL; + + sync_close(); + sync_initialized = FALSE; + os_sync_free(); + mem_close(); + os_sync_mutex = NULL; + ut_free_all_mem(); + + return(ret); +} + +bool +decrypt_decompress_file(const char *filepath, uint thread_n) +{ + std::stringstream cmd, message; + char *dest_filepath = strdup(filepath); + bool needs_action = false; + + cmd << "cat " << filepath; + + if (ends_with(filepath, ".xbcrypt") && opt_decrypt) { + cmd << " | xbcrypt --decrypt --encrypt-algo=" + << xtrabackup_encrypt_algo_names[opt_decrypt_algo]; + if (xtrabackup_encrypt_key) { + cmd << " --encrypt-key=" << xtrabackup_encrypt_key; + } else { + cmd << " --encrypt-key-file=" + << xtrabackup_encrypt_key_file; + } + dest_filepath[strlen(dest_filepath) - 8] = 0; + message << "decrypting"; + needs_action = true; + } + + if (opt_decompress + && (ends_with(filepath, ".qp") + || (ends_with(filepath, ".qp.xbcrypt") + && opt_decrypt))) { + cmd << " | qpress -dio "; + dest_filepath[strlen(dest_filepath) - 3] = 0; + if (needs_action) { + message << " and "; + } + message << "decompressing"; + needs_action = true; + } + + cmd << " > " << dest_filepath; + message << " " << filepath; + + free(dest_filepath); + + if (needs_action) { + + msg_ts("[%02u] %s\n", thread_n, message.str().c_str()); + + if (system(cmd.str().c_str()) != 0) { + return(false); + } + + if (opt_remove_original) { + msg_ts("[%02u] removing %s\n", thread_n, filepath); + if (my_delete(filepath, MYF(MY_WME)) != 0) { + return(false); + } + } + } + + return(true); +} + +static +os_thread_ret_t +decrypt_decompress_thread_func(void *arg) +{ + bool ret = true; + datadir_node_t node; + datadir_thread_ctxt_t *ctxt = (datadir_thread_ctxt_t *)(arg); + + datadir_node_init(&node); + + while (datadir_iter_next(ctxt->it, &node)) { + + /* skip empty directories in backup */ + if (node.is_empty_dir) { + continue; + } + + if (!ends_with(node.filepath, ".qp") + && !ends_with(node.filepath, ".xbcrypt")) { + continue; + } + + if (!(ret = decrypt_decompress_file(node.filepath, + ctxt->n_thread))) { + goto cleanup; + } + } + +cleanup: + + datadir_node_free(&node); + + os_mutex_enter(ctxt->count_mutex); + --(*ctxt->count); + os_mutex_exit(ctxt->count_mutex); + + ctxt->ret = ret; + + os_thread_exit(NULL); + OS_THREAD_DUMMY_RETURN; +} + +bool +decrypt_decompress() +{ + bool ret; + datadir_iter_t *it = NULL; + + srv_max_n_threads = 1000; + os_sync_mutex = NULL; + ut_mem_init(); + os_sync_init(); + sync_init(); + + /* cd to backup directory */ + if (my_setwd(xtrabackup_target_dir, MYF(MY_WME))) + { + msg("cannot my_setwd %s\n", xtrabackup_target_dir); + return(false); + } + + /* copy the rest of tablespaces */ + ds_data = ds_create(".", DS_TYPE_LOCAL); + + it = datadir_iter_new(".", false); + + ut_a(xtrabackup_parallel >= 0); + + ret = run_data_threads(it, decrypt_decompress_thread_func, + xtrabackup_parallel ? xtrabackup_parallel : 1); + + if (it != NULL) { + datadir_iter_free(it); + } + + if (ds_data != NULL) { + ds_destroy(ds_data); + } + + ds_data = NULL; + + sync_close(); + sync_initialized = FALSE; + os_sync_free(); + os_sync_mutex = NULL; + ut_free_all_mem(); + + return(ret); +} + +void +version_check() +{ + if (opt_password != NULL) { + setenv("option_mysql_password", opt_password, 1); + } + if (opt_user != NULL) { + setenv("option_mysql_user", opt_user, 1); + } + if (opt_host != NULL) { + setenv("option_mysql_host", opt_host, 1); + } + if (opt_socket != NULL) { + setenv("option_mysql_socket", opt_socket, 1); + } + if (opt_port != 0) { + char port[20]; + snprintf(port, sizeof(port), "%u", opt_port); + setenv("option_mysql_port", port, 1); + } + + FILE *pipe = popen("perl", "w"); + if (pipe == NULL) { + return; + } + + fputs((const char *)version_check_pl, pipe); + + pclose(pipe); +} diff --git a/extra/mariabackup/backup_copy.h b/extra/mariabackup/backup_copy.h new file mode 100644 index 00000000000..c8fc5fc8ba9 --- /dev/null +++ b/extra/mariabackup/backup_copy.h @@ -0,0 +1,51 @@ + +#ifndef XTRABACKUP_BACKUP_COPY_H +#define XTRABACKUP_BACKUP_COPY_H + +#include +#include "datasink.h" + +/* special files */ +#define XTRABACKUP_SLAVE_INFO "xtrabackup_slave_info" +#define XTRABACKUP_GALERA_INFO "xtrabackup_galera_info" +#define XTRABACKUP_BINLOG_INFO "xtrabackup_binlog_info" +#define XTRABACKUP_INFO "xtrabackup_info" + +extern bool binlog_locked; + +bool +backup_file_printf(const char *filename, const char *fmt, ...) + ATTRIBUTE_FORMAT(printf, 2, 0); + +/************************************************************************ +Return true if first and second arguments are the same path. */ +bool +equal_paths(const char *first, const char *second); + +/************************************************************************ +Copy file for backup/restore. +@return true in case of success. */ +bool +copy_file(ds_ctxt_t *datasink, + const char *src_file_path, + const char *dst_file_path, + uint thread_n); + +bool +backup_start(); +bool +backup_finish(); +bool +apply_log_finish(); +bool +copy_back(); +bool +decrypt_decompress(); +void +version_check(); +bool +is_path_separator(char); +bool +directory_exists(const char *dir, bool create); + +#endif diff --git a/extra/mariabackup/backup_mysql.cc b/extra/mariabackup/backup_mysql.cc new file mode 100644 index 00000000000..1ae8d10053e --- /dev/null +++ b/extra/mariabackup/backup_mysql.cc @@ -0,0 +1,1756 @@ +/****************************************************** +hot backup tool for InnoDB +(c) 2009-2015 Percona LLC and/or its affiliates +Originally Created 3/3/2009 Yasufumi Kinoshita +Written by Alexey Kopytov, Aleksandr Kuzminsky, Stewart Smith, Vadim Tkachenko, +Yasufumi Kinoshita, Ignacio Nin and Baron Schwartz. + +This program is free software; you can redistribute it and/or modify +it under the terms of the GNU General Public License as published by +the Free Software Foundation; version 2 of the License. + +This program is distributed in the hope that it will be useful, +but WITHOUT ANY WARRANTY; without even the implied warranty of +MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the +GNU General Public License for more details. + +You should have received a copy of the GNU General Public License +along with this program; if not, write to the Free Software +Foundation, Inc., 51 Franklin Street, Fifth Floor, Boston, MA 02110-1301, USA + +******************************************************* + +This file incorporates work covered by the following copyright and +permission notice: + +Copyright (c) 2000, 2011, MySQL AB & Innobase Oy. All Rights Reserved. + +This program is free software; you can redistribute it and/or modify it under +the terms of the GNU General Public License as published by the Free Software +Foundation; version 2 of the License. + +This program is distributed in the hope that it will be useful, but WITHOUT +ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or FITNESS +FOR A PARTICULAR PURPOSE. See the GNU General Public License for more details. + +You should have received a copy of the GNU General Public License along with +this program; if not, write to the Free Software Foundation, Inc., 59 Temple +Place, Suite 330, Boston, MA 02111-1307 USA + +*******************************************************/ + +#include +#include +#include +#include +#include +#include +#include "common.h" +#include "xtrabackup.h" +#include "xtrabackup_version.h" +#include "backup_copy.h" +#include "backup_mysql.h" +#include "mysqld.h" + + +char *tool_name; +char tool_args[2048]; + +/* mysql flavor and version */ +mysql_flavor_t server_flavor = FLAVOR_UNKNOWN; +unsigned long mysql_server_version = 0; + +/* server capabilities */ +bool have_changed_page_bitmaps = false; +bool have_backup_locks = false; +bool have_backup_safe_binlog_info = false; +bool have_lock_wait_timeout = false; +bool have_galera_enabled = false; +bool have_flush_engine_logs = false; +bool have_multi_threaded_slave = false; +bool have_gtid_slave = false; + +/* Kill long selects */ +os_thread_id_t kill_query_thread_id; +os_event_t kill_query_thread_started; +os_event_t kill_query_thread_stopped; +os_event_t kill_query_thread_stop; + +bool sql_thread_started = false; +char *mysql_slave_position = NULL; +char *mysql_binlog_position = NULL; +char *buffer_pool_filename = NULL; + +/* History on server */ +time_t history_start_time; +time_t history_end_time; +time_t history_lock_time; + +MYSQL *mysql_connection; + +extern "C" { +MYSQL * STDCALL +cli_mysql_real_connect(MYSQL *mysql,const char *host, const char *user, + const char *passwd, const char *db, + uint port, const char *unix_socket,ulong client_flag); +} + +#define mysql_real_connect cli_mysql_real_connect + + +MYSQL * +xb_mysql_connect() +{ + MYSQL *connection = mysql_init(NULL); + char mysql_port_str[std::numeric_limits::digits10 + 3]; + + sprintf(mysql_port_str, "%d", opt_port); + + if (connection == NULL) { + msg("Failed to init MySQL struct: %s.\n", + mysql_error(connection)); + return(NULL); + } + + if (!opt_secure_auth) { + mysql_options(connection, MYSQL_SECURE_AUTH, + (char *) &opt_secure_auth); + } + + msg_ts("Connecting to MySQL server host: %s, user: %s, password: %s, " + "port: %s, socket: %s\n", opt_host ? opt_host : "localhost", + opt_user ? opt_user : "not set", + opt_password ? "set" : "not set", + opt_port != 0 ? mysql_port_str : "not set", + opt_socket ? opt_socket : "not set"); + +#ifdef HAVE_OPENSSL + if (opt_use_ssl) + { + mysql_ssl_set(connection, opt_ssl_key, opt_ssl_cert, + opt_ssl_ca, opt_ssl_capath, + opt_ssl_cipher); + mysql_options(connection, MYSQL_OPT_SSL_CRL, opt_ssl_crl); + mysql_options(connection, MYSQL_OPT_SSL_CRLPATH, + opt_ssl_crlpath); + } + mysql_options(connection,MYSQL_OPT_SSL_VERIFY_SERVER_CERT, + (char*)&opt_ssl_verify_server_cert); +#if !defined(HAVE_YASSL) + if (opt_server_public_key && *opt_server_public_key) + mysql_options(connection, MYSQL_SERVER_PUBLIC_KEY, + opt_server_public_key); +#endif +#endif + + if (!mysql_real_connect(connection, + opt_host ? opt_host : "localhost", + opt_user, + opt_password, + "" /*database*/, opt_port, + opt_socket, 0)) { + msg("Failed to connect to MySQL server: %s.\n", + mysql_error(connection)); + mysql_close(connection); + return(NULL); + } + + xb_mysql_query(connection, "SET SESSION wait_timeout=2147483", + false, true); + + return(connection); +} + +/*********************************************************************//** +Execute mysql query. */ +MYSQL_RES * +xb_mysql_query(MYSQL *connection, const char *query, bool use_result, + bool die_on_error) +{ + MYSQL_RES *mysql_result = NULL; + + if (mysql_query(connection, query)) { + msg("Error: failed to execute query %s: %s\n", query, + mysql_error(connection)); + if (die_on_error) { + exit(EXIT_FAILURE); + } + return(NULL); + } + + /* store result set on client if there is a result */ + if (mysql_field_count(connection) > 0) { + if ((mysql_result = mysql_store_result(connection)) == NULL) { + msg("Error: failed to fetch query result %s: %s\n", + query, mysql_error(connection)); + exit(EXIT_FAILURE); + } + + if (!use_result) { + mysql_free_result(mysql_result); + } + } + + return mysql_result; +} + + +struct mysql_variable { + const char *name; + char **value; +}; + + +static +void +read_mysql_variables(MYSQL *connection, const char *query, mysql_variable *vars, + bool vertical_result) +{ + MYSQL_RES *mysql_result; + MYSQL_ROW row; + mysql_variable *var; + + mysql_result = xb_mysql_query(connection, query, true); + + ut_ad(!vertical_result || mysql_num_fields(mysql_result) == 2); + + if (vertical_result) { + while ((row = mysql_fetch_row(mysql_result))) { + char *name = row[0]; + char *value = row[1]; + for (var = vars; var->name; var++) { + if (strcmp(var->name, name) == 0 + && value != NULL) { + *(var->value) = strdup(value); + } + } + } + } else { + MYSQL_FIELD *field; + + if ((row = mysql_fetch_row(mysql_result)) != NULL) { + int i = 0; + while ((field = mysql_fetch_field(mysql_result)) + != NULL) { + char *name = field->name; + char *value = row[i]; + for (var = vars; var->name; var++) { + if (strcmp(var->name, name) == 0 + && value != NULL) { + *(var->value) = strdup(value); + } + } + ++i; + } + } + } + + mysql_free_result(mysql_result); +} + + +static +void +free_mysql_variables(mysql_variable *vars) +{ + mysql_variable *var; + + for (var = vars; var->name; var++) { + free(*(var->value)); + } +} + + +static +char * +read_mysql_one_value(MYSQL *connection, const char *query) +{ + MYSQL_RES *mysql_result; + MYSQL_ROW row; + char *result = NULL; + + mysql_result = xb_mysql_query(connection, query, true); + + ut_ad(mysql_num_fields(mysql_result) == 1); + + if ((row = mysql_fetch_row(mysql_result))) { + result = strdup(row[0]); + } + + mysql_free_result(mysql_result); + + return(result); +} + +static +bool +check_server_version(unsigned long version_number, + const char *version_string, + const char *version_comment, + const char *innodb_version) +{ + bool version_supported = false; + bool mysql51 = false; + + mysql_server_version = version_number; + + server_flavor = FLAVOR_UNKNOWN; + if (strstr(version_comment, "Percona") != NULL) { + server_flavor = FLAVOR_PERCONA_SERVER; + } else if (strstr(version_comment, "MariaDB") != NULL || + strstr(version_string, "MariaDB") != NULL) { + server_flavor = FLAVOR_MARIADB; + } else if (strstr(version_comment, "MySQL") != NULL) { + server_flavor = FLAVOR_MYSQL; + } + + mysql51 = version_number > 50100 && version_number < 50500; + version_supported = version_supported + || (mysql51 && innodb_version != NULL); + version_supported = version_supported + || (version_number > 50500 && version_number < 50700); + version_supported = version_supported + || ((version_number > 100000 && version_number < 100300) + && server_flavor == FLAVOR_MARIADB); + + if (mysql51 && innodb_version == NULL) { + msg("Error: Built-in InnoDB in MySQL 5.1 is not " + "supported in this release. You can either use " + "Percona XtraBackup 2.0, or upgrade to InnoDB " + "plugin.\n"); + } else if (!version_supported) { + msg("Error: Unsupported server version: '%s'. Please " + "report a bug at " + "https://bugs.launchpad.net/percona-xtrabackup\n", + version_string); + } + + return(version_supported); +} + +/*********************************************************************//** +Receive options important for XtraBackup from MySQL server. +@return true on success. */ +bool +get_mysql_vars(MYSQL *connection) +{ + char *gtid_mode_var = NULL; + char *version_var = NULL; + char *version_comment_var = NULL; + char *innodb_version_var = NULL; + char *have_backup_locks_var = NULL; + char *have_backup_safe_binlog_info_var = NULL; + char *log_bin_var = NULL; + char *lock_wait_timeout_var= NULL; + char *wsrep_on_var = NULL; + char *slave_parallel_workers_var = NULL; + char *gtid_slave_pos_var = NULL; + char *innodb_buffer_pool_filename_var = NULL; + char *datadir_var = NULL; + char *innodb_log_group_home_dir_var = NULL; + char *innodb_log_file_size_var = NULL; + char *innodb_log_files_in_group_var = NULL; + char *innodb_data_file_path_var = NULL; + char *innodb_data_home_dir_var = NULL; + char *innodb_undo_directory_var = NULL; + char *innodb_page_size_var = NULL; + + unsigned long server_version = mysql_get_server_version(connection); + + bool ret = true; + + mysql_variable mysql_vars[] = { + {"have_backup_locks", &have_backup_locks_var}, + {"have_backup_safe_binlog_info", + &have_backup_safe_binlog_info_var}, + {"log_bin", &log_bin_var}, + {"lock_wait_timeout", &lock_wait_timeout_var}, + {"gtid_mode", >id_mode_var}, + {"version", &version_var}, + {"version_comment", &version_comment_var}, + {"innodb_version", &innodb_version_var}, + {"wsrep_on", &wsrep_on_var}, + {"slave_parallel_workers", &slave_parallel_workers_var}, + {"gtid_slave_pos", >id_slave_pos_var}, + {"innodb_buffer_pool_filename", + &innodb_buffer_pool_filename_var}, + {"datadir", &datadir_var}, + {"innodb_log_group_home_dir", &innodb_log_group_home_dir_var}, + {"innodb_log_file_size", &innodb_log_file_size_var}, + {"innodb_log_files_in_group", &innodb_log_files_in_group_var}, + {"innodb_data_file_path", &innodb_data_file_path_var}, + {"innodb_data_home_dir", &innodb_data_home_dir_var}, + {"innodb_undo_directory", &innodb_undo_directory_var}, + {"innodb_page_size", &innodb_page_size_var}, + {NULL, NULL} + }; + + read_mysql_variables(connection, "SHOW VARIABLES", + mysql_vars, true); + + if (have_backup_locks_var != NULL && !opt_no_backup_locks) { + have_backup_locks = true; + } + + if (opt_binlog_info == BINLOG_INFO_AUTO) { + + if (have_backup_safe_binlog_info_var != NULL) + opt_binlog_info = BINLOG_INFO_LOCKLESS; + else if (log_bin_var != NULL && !strcmp(log_bin_var, "ON")) + opt_binlog_info = BINLOG_INFO_ON; + else + opt_binlog_info = BINLOG_INFO_OFF; + } + + if (have_backup_safe_binlog_info_var == NULL && + opt_binlog_info == BINLOG_INFO_LOCKLESS) { + + msg("Error: --binlog-info=LOCKLESS is not supported by the " + "server\n"); + return(false); + } + + if (lock_wait_timeout_var != NULL) { + have_lock_wait_timeout = true; + } + + if (wsrep_on_var != NULL) { + have_galera_enabled = true; + } + + /* Check server version compatibility and detect server flavor */ + + if (!(ret = check_server_version(server_version, version_var, + version_comment_var, + innodb_version_var))) { + goto out; + } + + if (server_version > 50500) { + have_flush_engine_logs = true; + } + + if (slave_parallel_workers_var != NULL + && atoi(slave_parallel_workers_var) > 0) { + have_multi_threaded_slave = true; + } + + if (innodb_buffer_pool_filename_var != NULL) { + buffer_pool_filename = strdup(innodb_buffer_pool_filename_var); + } + + if ((gtid_mode_var && strcmp(gtid_mode_var, "ON") == 0) || + (gtid_slave_pos_var && *gtid_slave_pos_var)) { + have_gtid_slave = true; + } + + msg("Using server version %s\n", version_var); + + if (!(ret = detect_mysql_capabilities_for_backup())) { + goto out; + } + + /* make sure datadir value is the same in configuration file */ + if (check_if_param_set("datadir")) { + if (!directory_exists(mysql_data_home, false)) { + msg("Warning: option 'datadir' points to " + "nonexistent directory '%s'\n", mysql_data_home); + } + if (!directory_exists(datadir_var, false)) { + msg("Warning: MySQL variable 'datadir' points to " + "nonexistent directory '%s'\n", datadir_var); + } + if (!equal_paths(mysql_data_home, datadir_var)) { + msg("Warning: option 'datadir' has different " + "values:\n" + " '%s' in defaults file\n" + " '%s' in SHOW VARIABLES\n", + mysql_data_home, datadir_var); + } + } + + /* get some default values is they are missing from my.cnf */ + if (!check_if_param_set("datadir") && datadir_var && *datadir_var) { + strmake(mysql_real_data_home, datadir_var, FN_REFLEN - 1); + mysql_data_home= mysql_real_data_home; + } + + if (!check_if_param_set("innodb_data_file_path") + && innodb_data_file_path_var && *innodb_data_file_path_var) { + innobase_data_file_path = my_strdup( + innodb_data_file_path_var, MYF(MY_FAE)); + } + + if (!check_if_param_set("innodb_data_home_dir") + && innodb_data_home_dir_var && *innodb_data_home_dir_var) { + innobase_data_home_dir = my_strdup( + innodb_data_home_dir_var, MYF(MY_FAE)); + } + + if (!check_if_param_set("innodb_log_group_home_dir") + && innodb_log_group_home_dir_var + && *innodb_log_group_home_dir_var) { + srv_log_group_home_dir = my_strdup( + innodb_log_group_home_dir_var, MYF(MY_FAE)); + } + + if (!check_if_param_set("innodb_undo_directory") + && innodb_undo_directory_var && *innodb_undo_directory_var) { + srv_undo_dir = my_strdup( + innodb_undo_directory_var, MYF(MY_FAE)); + } + + if (!check_if_param_set("innodb_log_files_in_group") + && innodb_log_files_in_group_var) { + char *endptr; + + innobase_log_files_in_group = strtol( + innodb_log_files_in_group_var, &endptr, 10); + ut_ad(*endptr == 0); + } + + if (!check_if_param_set("innodb_log_file_size") + && innodb_log_file_size_var) { + char *endptr; + + innobase_log_file_size = strtoll( + innodb_log_file_size_var, &endptr, 10); + ut_ad(*endptr == 0); + } + + if (!check_if_param_set("innodb_page_size") && innodb_page_size_var) { + char *endptr; + + innobase_page_size = strtoll( + innodb_page_size_var, &endptr, 10); + ut_ad(*endptr == 0); + } + +out: + free_mysql_variables(mysql_vars); + + return(ret); +} + +/*********************************************************************//** +Query the server to find out what backup capabilities it supports. +@return true on success. */ +bool +detect_mysql_capabilities_for_backup() +{ + const char *query = "SELECT 'INNODB_CHANGED_PAGES', COUNT(*) FROM " + "INFORMATION_SCHEMA.PLUGINS " + "WHERE PLUGIN_NAME LIKE 'INNODB_CHANGED_PAGES'"; + char *innodb_changed_pages = NULL; + mysql_variable vars[] = { + {"INNODB_CHANGED_PAGES", &innodb_changed_pages}, {NULL, NULL}}; + + if (xtrabackup_incremental) { + + read_mysql_variables(mysql_connection, query, vars, true); + + ut_ad(innodb_changed_pages != NULL); + + have_changed_page_bitmaps = (atoi(innodb_changed_pages) == 1); + + /* INNODB_CHANGED_PAGES are listed in + INFORMATION_SCHEMA.PLUGINS in MariaDB, but + FLUSH NO_WRITE_TO_BINLOG CHANGED_PAGE_BITMAPS + is not supported for versions below 10.1.6 + (see MDEV-7472) */ + if (server_flavor == FLAVOR_MARIADB && + mysql_server_version < 100106) { + have_changed_page_bitmaps = false; + } + + free_mysql_variables(vars); + } + + /* do some sanity checks */ + if (opt_galera_info && !have_galera_enabled) { + msg("--galera-info is specified on the command " + "line, but the server does not support Galera " + "replication. Ignoring the option.\n"); + opt_galera_info = false; + } + + if (opt_slave_info && have_multi_threaded_slave && + !have_gtid_slave) { + msg("The --slave-info option requires GTID enabled for a " + "multi-threaded slave.\n"); + return(false); + } + + return(true); +} + +static +bool +select_incremental_lsn_from_history(lsn_t *incremental_lsn) +{ + MYSQL_RES *mysql_result; + MYSQL_ROW row; + char query[1000]; + char buf[100]; + + if (opt_incremental_history_name) { + mysql_real_escape_string(mysql_connection, buf, + opt_incremental_history_name, + strlen(opt_incremental_history_name)); + ut_snprintf(query, sizeof(query), + "SELECT innodb_to_lsn " + "FROM PERCONA_SCHEMA.xtrabackup_history " + "WHERE name = '%s' " + "AND innodb_to_lsn IS NOT NULL " + "ORDER BY innodb_to_lsn DESC LIMIT 1", + buf); + } + + if (opt_incremental_history_uuid) { + mysql_real_escape_string(mysql_connection, buf, + opt_incremental_history_uuid, + strlen(opt_incremental_history_uuid)); + ut_snprintf(query, sizeof(query), + "SELECT innodb_to_lsn " + "FROM PERCONA_SCHEMA.xtrabackup_history " + "WHERE uuid = '%s' " + "AND innodb_to_lsn IS NOT NULL " + "ORDER BY innodb_to_lsn DESC LIMIT 1", + buf); + } + + mysql_result = xb_mysql_query(mysql_connection, query, true); + + ut_ad(mysql_num_fields(mysql_result) == 1); + if (!(row = mysql_fetch_row(mysql_result))) { + msg("Error while attempting to find history record " + "for %s %s\n", + opt_incremental_history_uuid ? "uuid" : "name", + opt_incremental_history_uuid ? + opt_incremental_history_uuid : + opt_incremental_history_name); + return(false); + } + + *incremental_lsn = strtoull(row[0], NULL, 10); + + mysql_free_result(mysql_result); + + msg("Found and using lsn: " LSN_PF " for %s %s\n", *incremental_lsn, + opt_incremental_history_uuid ? "uuid" : "name", + opt_incremental_history_uuid ? + opt_incremental_history_uuid : + opt_incremental_history_name); + + return(true); +} + +static +const char * +eat_sql_whitespace(const char *query) +{ + bool comment = false; + + while (*query) { + if (comment) { + if (query[0] == '*' && query[1] == '/') { + query += 2; + comment = false; + continue; + } + ++query; + continue; + } + if (query[0] == '/' && query[1] == '*') { + query += 2; + comment = true; + continue; + } + if (strchr("\t\n\r (", query[0])) { + ++query; + continue; + } + break; + } + + return(query); +} + +static +bool +is_query_from_list(const char *query, const char **list) +{ + const char **item; + + query = eat_sql_whitespace(query); + + item = list; + while (*item) { + if (strncasecmp(query, *item, strlen(*item)) == 0) { + return(true); + } + ++item; + } + + return(false); +} + +static +bool +is_query(const char *query) +{ + const char *query_list[] = {"insert", "update", "delete", "replace", + "alter", "load", "select", "do", "handler", "call", "execute", + "begin", NULL}; + + return is_query_from_list(query, query_list); +} + +static +bool +is_select_query(const char *query) +{ + const char *query_list[] = {"select", NULL}; + + return is_query_from_list(query, query_list); +} + +static +bool +is_update_query(const char *query) +{ + const char *query_list[] = {"insert", "update", "delete", "replace", + "alter", "load", NULL}; + + return is_query_from_list(query, query_list); +} + +static +bool +have_queries_to_wait_for(MYSQL *connection, uint threshold) +{ + MYSQL_RES *result; + MYSQL_ROW row; + bool all_queries; + + result = xb_mysql_query(connection, "SHOW FULL PROCESSLIST", true); + + all_queries = (opt_lock_wait_query_type == QUERY_TYPE_ALL); + while ((row = mysql_fetch_row(result)) != NULL) { + const char *info = row[7]; + int duration = atoi(row[5]); + char *id = row[0]; + + if (info != NULL + && duration >= (int)threshold + && ((all_queries && is_query(info)) + || is_update_query(info))) { + msg_ts("Waiting for query %s (duration %d sec): %s", + id, duration, info); + return(true); + } + } + + return(false); +} + +static +void +kill_long_queries(MYSQL *connection, uint timeout) +{ + MYSQL_RES *result; + MYSQL_ROW row; + bool all_queries; + char kill_stmt[100]; + + result = xb_mysql_query(connection, "SHOW FULL PROCESSLIST", true); + + all_queries = (opt_kill_long_query_type == QUERY_TYPE_ALL); + while ((row = mysql_fetch_row(result)) != NULL) { + const char *info = row[7]; + int duration = atoi(row[5]); + char *id = row[0]; + + if (info != NULL && + duration >= (int)timeout && + ((all_queries && is_query(info)) || + is_select_query(info))) { + msg_ts("Killing query %s (duration %d sec): %s\n", + id, duration, info); + ut_snprintf(kill_stmt, sizeof(kill_stmt), + "KILL %s", id); + xb_mysql_query(connection, kill_stmt, false, false); + } + } +} + +static +bool +wait_for_no_updates(MYSQL *connection, uint timeout, uint threshold) +{ + time_t start_time; + + start_time = time(NULL); + + msg_ts("Waiting %u seconds for queries running longer than %u seconds " + "to finish\n", timeout, threshold); + + while (time(NULL) <= (time_t)(start_time + timeout)) { + if (!have_queries_to_wait_for(connection, threshold)) { + return(true); + } + os_thread_sleep(1000000); + } + + msg_ts("Unable to obtain lock. Please try again later."); + + return(false); +} + +static +os_thread_ret_t +kill_query_thread( +/*===============*/ + void *arg __attribute__((unused))) +{ + MYSQL *mysql; + time_t start_time; + + start_time = time(NULL); + + os_event_set(kill_query_thread_started); + + msg_ts("Kill query timeout %d seconds.\n", + opt_kill_long_queries_timeout); + + while (time(NULL) - start_time < + (time_t)opt_kill_long_queries_timeout) { + if (os_event_wait_time(kill_query_thread_stop, 1000) != + OS_SYNC_TIME_EXCEEDED) { + goto stop_thread; + } + } + + if ((mysql = xb_mysql_connect()) == NULL) { + msg("Error: kill query thread failed\n"); + goto stop_thread; + } + + while (true) { + kill_long_queries(mysql, time(NULL) - start_time); + if (os_event_wait_time(kill_query_thread_stop, 1000) != + OS_SYNC_TIME_EXCEEDED) { + break; + } + } + + mysql_close(mysql); + +stop_thread: + msg_ts("Kill query thread stopped\n"); + + os_event_set(kill_query_thread_stopped); + + os_thread_exit(NULL); + OS_THREAD_DUMMY_RETURN; +} + + +static +void +start_query_killer() +{ + kill_query_thread_stop = os_event_create(); + kill_query_thread_started = os_event_create(); + kill_query_thread_stopped = os_event_create(); + + os_thread_create(kill_query_thread, NULL, &kill_query_thread_id); + + os_event_wait(kill_query_thread_started); +} + +static +void +stop_query_killer() +{ + os_event_set(kill_query_thread_stop); + os_event_wait_time(kill_query_thread_stopped, 60000); +} + +/*********************************************************************//** +Function acquires either a backup tables lock, if supported +by the server, or a global read lock (FLUSH TABLES WITH READ LOCK) +otherwise. +@returns true if lock acquired */ +bool +lock_tables(MYSQL *connection) +{ + if (have_lock_wait_timeout) { + /* Set the maximum supported session value for + lock_wait_timeout to prevent unnecessary timeouts when the + global value is changed from the default */ + xb_mysql_query(connection, + "SET SESSION lock_wait_timeout=31536000", false); + } + + if (have_backup_locks) { + msg_ts("Executing LOCK TABLES FOR BACKUP...\n"); + xb_mysql_query(connection, "LOCK TABLES FOR BACKUP", false); + return(true); + } + + if (!opt_lock_wait_timeout && !opt_kill_long_queries_timeout) { + + /* We do first a FLUSH TABLES. If a long update is running, the + FLUSH TABLES will wait but will not stall the whole mysqld, and + when the long update is done the FLUSH TABLES WITH READ LOCK + will start and succeed quickly. So, FLUSH TABLES is to lower + the probability of a stage where both mysqldump and most client + connections are stalled. Of course, if a second long update + starts between the two FLUSHes, we have that bad stall. + + Option lock_wait_timeout serve the same purpose and is not + compatible with this trick. + */ + + msg_ts("Executing FLUSH NO_WRITE_TO_BINLOG TABLES...\n"); + + xb_mysql_query(connection, + "FLUSH NO_WRITE_TO_BINLOG TABLES", false); + } + + if (opt_lock_wait_timeout) { + if (!wait_for_no_updates(connection, opt_lock_wait_timeout, + opt_lock_wait_threshold)) { + return(false); + } + } + + msg_ts("Executing FLUSH TABLES WITH READ LOCK...\n"); + + if (opt_kill_long_queries_timeout) { + start_query_killer(); + } + + if (have_galera_enabled) { + xb_mysql_query(connection, + "SET SESSION wsrep_causal_reads=0", false); + } + + xb_mysql_query(connection, "FLUSH TABLES WITH READ LOCK", false); + + if (opt_kill_long_queries_timeout) { + stop_query_killer(); + } + + return(true); +} + + +/*********************************************************************//** +If backup locks are used, execute LOCK BINLOG FOR BACKUP provided that we are +not in the --no-lock mode and the lock has not been acquired already. +@returns true if lock acquired */ +bool +lock_binlog_maybe(MYSQL *connection) +{ + if (have_backup_locks && !opt_no_lock && !binlog_locked) { + msg_ts("Executing LOCK BINLOG FOR BACKUP...\n"); + xb_mysql_query(connection, "LOCK BINLOG FOR BACKUP", false); + binlog_locked = true; + + return(true); + } + + return(false); +} + + +/*********************************************************************//** +Releases either global read lock acquired with FTWRL and the binlog +lock acquired with LOCK BINLOG FOR BACKUP, depending on +the locking strategy being used */ +void +unlock_all(MYSQL *connection) +{ + if (opt_debug_sleep_before_unlock) { + msg_ts("Debug sleep for %u seconds\n", + opt_debug_sleep_before_unlock); + os_thread_sleep(opt_debug_sleep_before_unlock * 1000); + } + + if (binlog_locked) { + msg_ts("Executing UNLOCK BINLOG\n"); + xb_mysql_query(connection, "UNLOCK BINLOG", false); + } + + msg_ts("Executing UNLOCK TABLES\n"); + xb_mysql_query(connection, "UNLOCK TABLES", false); + + msg_ts("All tables unlocked\n"); +} + + +static +int +get_open_temp_tables(MYSQL *connection) +{ + char *slave_open_temp_tables = NULL; + mysql_variable status[] = { + {"Slave_open_temp_tables", &slave_open_temp_tables}, + {NULL, NULL} + }; + int result = false; + + read_mysql_variables(connection, + "SHOW STATUS LIKE 'slave_open_temp_tables'", status, true); + + result = slave_open_temp_tables ? atoi(slave_open_temp_tables) : 0; + + free_mysql_variables(status); + + return(result); +} + +/*********************************************************************//** +Wait until it's safe to backup a slave. Returns immediately if +the host isn't a slave. Currently there's only one check: +Slave_open_temp_tables has to be zero. Dies on timeout. */ +bool +wait_for_safe_slave(MYSQL *connection) +{ + char *read_master_log_pos = NULL; + char *slave_sql_running = NULL; + int n_attempts = 1; + const int sleep_time = 3; + int open_temp_tables = 0; + bool result = true; + + mysql_variable status[] = { + {"Read_Master_Log_Pos", &read_master_log_pos}, + {"Slave_SQL_Running", &slave_sql_running}, + {NULL, NULL} + }; + + sql_thread_started = false; + + read_mysql_variables(connection, "SHOW SLAVE STATUS", status, false); + + if (!(read_master_log_pos && slave_sql_running)) { + msg("Not checking slave open temp tables for " + "--safe-slave-backup because host is not a slave\n"); + goto cleanup; + } + + if (strcmp(slave_sql_running, "Yes") == 0) { + sql_thread_started = true; + xb_mysql_query(connection, "STOP SLAVE SQL_THREAD", false); + } + + if (opt_safe_slave_backup_timeout > 0) { + n_attempts = opt_safe_slave_backup_timeout / sleep_time; + } + + open_temp_tables = get_open_temp_tables(connection); + msg_ts("Slave open temp tables: %d\n", open_temp_tables); + + while (open_temp_tables && n_attempts--) { + msg_ts("Starting slave SQL thread, waiting %d seconds, then " + "checking Slave_open_temp_tables again (%d attempts " + "remaining)...\n", sleep_time, n_attempts); + + xb_mysql_query(connection, "START SLAVE SQL_THREAD", false); + os_thread_sleep(sleep_time * 1000000); + xb_mysql_query(connection, "STOP SLAVE SQL_THREAD", false); + + open_temp_tables = get_open_temp_tables(connection); + msg_ts("Slave open temp tables: %d\n", open_temp_tables); + } + + /* Restart the slave if it was running at start */ + if (open_temp_tables == 0) { + msg_ts("Slave is safe to backup\n"); + goto cleanup; + } + + result = false; + + if (sql_thread_started) { + msg_ts("Restarting slave SQL thread.\n"); + xb_mysql_query(connection, "START SLAVE SQL_THREAD", false); + } + + msg_ts("Slave_open_temp_tables did not become zero after " + "%d seconds\n", opt_safe_slave_backup_timeout); + +cleanup: + free_mysql_variables(status); + + return(result); +} + + +/*********************************************************************//** +Retrieves MySQL binlog position of the master server in a replication +setup and saves it in a file. It also saves it in mysql_slave_position +variable. */ +bool +write_slave_info(MYSQL *connection) +{ + char *master = NULL; + char *filename = NULL; + char *gtid_executed = NULL; + char *position = NULL; + char *gtid_slave_pos = NULL; + char *ptr; + bool result = false; + + mysql_variable status[] = { + {"Master_Host", &master}, + {"Relay_Master_Log_File", &filename}, + {"Exec_Master_Log_Pos", &position}, + {"Executed_Gtid_Set", >id_executed}, + {NULL, NULL} + }; + + mysql_variable variables[] = { + {"gtid_slave_pos", >id_slave_pos}, + {NULL, NULL} + }; + + read_mysql_variables(connection, "SHOW SLAVE STATUS", status, false); + read_mysql_variables(connection, "SHOW VARIABLES", variables, true); + + if (master == NULL || filename == NULL || position == NULL) { + msg("Failed to get master binlog coordinates " + "from SHOW SLAVE STATUS\n"); + msg("This means that the server is not a " + "replication slave. Ignoring the --slave-info " + "option\n"); + /* we still want to continue the backup */ + result = true; + goto cleanup; + } + + /* Print slave status to a file. + If GTID mode is used, construct a CHANGE MASTER statement with + MASTER_AUTO_POSITION and correct a gtid_purged value. */ + if (gtid_executed != NULL && *gtid_executed) { + /* MySQL >= 5.6 with GTID enabled */ + + for (ptr = strchr(gtid_executed, '\n'); + ptr; + ptr = strchr(ptr, '\n')) { + *ptr = ' '; + } + + result = backup_file_printf(XTRABACKUP_SLAVE_INFO, + "SET GLOBAL gtid_purged='%s';\n" + "CHANGE MASTER TO MASTER_AUTO_POSITION=1\n", + gtid_executed); + + ut_a(asprintf(&mysql_slave_position, + "master host '%s', purge list '%s'", + master, gtid_executed) != -1); + } else if (gtid_slave_pos && *gtid_slave_pos) { + /* MariaDB >= 10.0 with GTID enabled */ + result = backup_file_printf(XTRABACKUP_SLAVE_INFO, + "SET GLOBAL gtid_slave_pos = '%s';\n" + "CHANGE MASTER TO master_use_gtid = slave_pos\n", + gtid_slave_pos); + ut_a(asprintf(&mysql_slave_position, + "master host '%s', gtid_slave_pos %s", + master, gtid_slave_pos) != -1); + } else { + result = backup_file_printf(XTRABACKUP_SLAVE_INFO, + "CHANGE MASTER TO MASTER_LOG_FILE='%s', " + "MASTER_LOG_POS=%s\n", filename, position); + ut_a(asprintf(&mysql_slave_position, + "master host '%s', filename '%s', position '%s'", + master, filename, position) != -1); + } + +cleanup: + free_mysql_variables(status); + free_mysql_variables(variables); + + return(result); +} + + +/*********************************************************************//** +Retrieves MySQL Galera and +saves it in a file. It also prints it to stdout. */ +bool +write_galera_info(MYSQL *connection) +{ + char *state_uuid = NULL, *state_uuid55 = NULL; + char *last_committed = NULL, *last_committed55 = NULL; + bool result; + + mysql_variable status[] = { + {"Wsrep_local_state_uuid", &state_uuid}, + {"wsrep_local_state_uuid", &state_uuid55}, + {"Wsrep_last_committed", &last_committed}, + {"wsrep_last_committed", &last_committed55}, + {NULL, NULL} + }; + + /* When backup locks are supported by the server, we should skip + creating xtrabackup_galera_info file on the backup stage, because + wsrep_local_state_uuid and wsrep_last_committed will be inconsistent + without blocking commits. The state file will be created on the prepare + stage using the WSREP recovery procedure. */ + if (have_backup_locks) { + return(true); + } + + read_mysql_variables(connection, "SHOW STATUS", status, true); + + if ((state_uuid == NULL && state_uuid55 == NULL) + || (last_committed == NULL && last_committed55 == NULL)) { + msg("Failed to get master wsrep state from SHOW STATUS.\n"); + result = false; + goto cleanup; + } + + result = backup_file_printf(XTRABACKUP_GALERA_INFO, + "%s:%s\n", state_uuid ? state_uuid : state_uuid55, + last_committed ? last_committed : last_committed55); + +cleanup: + free_mysql_variables(status); + + return(result); +} + + +/*********************************************************************//** +Flush and copy the current binary log file into the backup, +if GTID is enabled */ +bool +write_current_binlog_file(MYSQL *connection) +{ + char *executed_gtid_set = NULL; + char *gtid_binlog_state = NULL; + char *log_bin_file = NULL; + char *log_bin_dir = NULL; + bool gtid_exists; + bool result = true; + char filepath[FN_REFLEN]; + + mysql_variable status[] = { + {"Executed_Gtid_Set", &executed_gtid_set}, + {NULL, NULL} + }; + + mysql_variable status_after_flush[] = { + {"File", &log_bin_file}, + {NULL, NULL} + }; + + mysql_variable vars[] = { + {"gtid_binlog_state", >id_binlog_state}, + {"log_bin_basename", &log_bin_dir}, + {NULL, NULL} + }; + + read_mysql_variables(connection, "SHOW MASTER STATUS", status, false); + read_mysql_variables(connection, "SHOW VARIABLES", vars, true); + + gtid_exists = (executed_gtid_set && *executed_gtid_set) + || (gtid_binlog_state && *gtid_binlog_state); + + if (gtid_exists) { + size_t log_bin_dir_length; + + lock_binlog_maybe(connection); + + xb_mysql_query(connection, "FLUSH BINARY LOGS", false); + + read_mysql_variables(connection, "SHOW MASTER STATUS", + status_after_flush, false); + + if (opt_log_bin != NULL && strchr(opt_log_bin, FN_LIBCHAR)) { + /* If log_bin is set, it has priority */ + if (log_bin_dir) { + free(log_bin_dir); + } + log_bin_dir = strdup(opt_log_bin); + } else if (log_bin_dir == NULL) { + /* Default location is MySQL datadir */ + log_bin_dir = strdup("./"); + } + + dirname_part(log_bin_dir, log_bin_dir, &log_bin_dir_length); + + /* strip final slash if it is not the only path component */ + if (log_bin_dir_length > 1 && + log_bin_dir[log_bin_dir_length - 1] == FN_LIBCHAR) { + log_bin_dir[log_bin_dir_length - 1] = 0; + } + + if (log_bin_dir == NULL || log_bin_file == NULL) { + msg("Failed to get master binlog coordinates from " + "SHOW MASTER STATUS"); + result = false; + goto cleanup; + } + + ut_snprintf(filepath, sizeof(filepath), "%s%c%s", + log_bin_dir, FN_LIBCHAR, log_bin_file); + result = copy_file(ds_data, filepath, log_bin_file, 0); + } + +cleanup: + free_mysql_variables(status_after_flush); + free_mysql_variables(status); + free_mysql_variables(vars); + + return(result); +} + + +/*********************************************************************//** +Retrieves MySQL binlog position and +saves it in a file. It also prints it to stdout. */ +bool +write_binlog_info(MYSQL *connection) +{ + char *filename = NULL; + char *position = NULL; + char *gtid_mode = NULL; + char *gtid_current_pos = NULL; + char *gtid_executed = NULL; + char *gtid = NULL; + bool result; + bool mysql_gtid; + bool mariadb_gtid; + + mysql_variable status[] = { + {"File", &filename}, + {"Position", &position}, + {"Executed_Gtid_Set", >id_executed}, + {NULL, NULL} + }; + + mysql_variable vars[] = { + {"gtid_mode", >id_mode}, + {"gtid_current_pos", >id_current_pos}, + {NULL, NULL} + }; + + read_mysql_variables(connection, "SHOW MASTER STATUS", status, false); + read_mysql_variables(connection, "SHOW VARIABLES", vars, true); + + if (filename == NULL || position == NULL) { + /* Do not create xtrabackup_binlog_info if binary + log is disabled */ + result = true; + goto cleanup; + } + + mysql_gtid = ((gtid_mode != NULL) && (strcmp(gtid_mode, "ON") == 0)); + mariadb_gtid = (gtid_current_pos != NULL); + + gtid = (gtid_executed != NULL ? gtid_executed : gtid_current_pos); + + if (mariadb_gtid || mysql_gtid) { + ut_a(asprintf(&mysql_binlog_position, + "filename '%s', position '%s', " + "GTID of the last change '%s'", + filename, position, gtid) != -1); + result = backup_file_printf(XTRABACKUP_BINLOG_INFO, + "%s\t%s\t%s\n", filename, position, + gtid); + } else { + ut_a(asprintf(&mysql_binlog_position, + "filename '%s', position '%s'", + filename, position) != -1); + result = backup_file_printf(XTRABACKUP_BINLOG_INFO, + "%s\t%s\n", filename, position); + } + +cleanup: + free_mysql_variables(status); + free_mysql_variables(vars); + + return(result); +} + + + +/*********************************************************************//** +Writes xtrabackup_info file and if backup_history is enable creates +PERCONA_SCHEMA.xtrabackup_history and writes a new history record to the +table containing all the history info particular to the just completed +backup. */ +bool +write_xtrabackup_info(MYSQL *connection) +{ + MYSQL_STMT *stmt; + MYSQL_BIND bind[19]; + char *uuid = NULL; + char *server_version = NULL; + char buf_start_time[100]; + char buf_end_time[100]; + int idx; + tm tm; + my_bool null = TRUE; + + const char *xb_stream_name[] = {"file", "tar", "xbstream"}; + const char *ins_query = "insert into PERCONA_SCHEMA.xtrabackup_history(" + "uuid, name, tool_name, tool_command, tool_version, " + "ibbackup_version, server_version, start_time, end_time, " + "lock_time, binlog_pos, innodb_from_lsn, innodb_to_lsn, " + "partial, incremental, format, compact, compressed, " + "encrypted) " + "values(?,?,?,?,?,?,?,from_unixtime(?),from_unixtime(?)," + "?,?,?,?,?,?,?,?,?,?)"; + + ut_ad(xtrabackup_stream_fmt < 3); + + uuid = read_mysql_one_value(connection, "SELECT UUID()"); + server_version = read_mysql_one_value(connection, "SELECT VERSION()"); + localtime_r(&history_start_time, &tm); + strftime(buf_start_time, sizeof(buf_start_time), + "%Y-%m-%d %H:%M:%S", &tm); + history_end_time = time(NULL); + localtime_r(&history_end_time, &tm); + strftime(buf_end_time, sizeof(buf_end_time), + "%Y-%m-%d %H:%M:%S", &tm); + backup_file_printf(XTRABACKUP_INFO, + "uuid = %s\n" + "name = %s\n" + "tool_name = %s\n" + "tool_command = %s\n" + "tool_version = %s\n" + "ibbackup_version = %s\n" + "server_version = %s\n" + "start_time = %s\n" + "end_time = %s\n" + "lock_time = %d\n" + "binlog_pos = %s\n" + "innodb_from_lsn = %llu\n" + "innodb_to_lsn = %llu\n" + "partial = %s\n" + "incremental = %s\n" + "format = %s\n" + "compact = %s\n" + "compressed = %s\n" + "encrypted = %s\n", + uuid, /* uuid */ + opt_history ? opt_history : "", /* name */ + tool_name, /* tool_name */ + tool_args, /* tool_command */ + XTRABACKUP_VERSION, /* tool_version */ + XTRABACKUP_VERSION, /* ibbackup_version */ + server_version, /* server_version */ + buf_start_time, /* start_time */ + buf_end_time, /* end_time */ + history_lock_time, /* lock_time */ + mysql_binlog_position ? + mysql_binlog_position : "", /* binlog_pos */ + incremental_lsn, /* innodb_from_lsn */ + metadata_to_lsn, /* innodb_to_lsn */ + (xtrabackup_tables /* partial */ + || xtrabackup_tables_file + || xtrabackup_databases + || xtrabackup_databases_file) ? "Y" : "N", + xtrabackup_incremental ? "Y" : "N", /* incremental */ + xb_stream_name[xtrabackup_stream_fmt], /* format */ + xtrabackup_compact ? "Y" : "N", /* compact */ + xtrabackup_compress ? "compressed" : "N", /* compressed */ + xtrabackup_encrypt ? "Y" : "N"); /* encrypted */ + + if (!opt_history) { + goto cleanup; + } + + xb_mysql_query(connection, + "CREATE DATABASE IF NOT EXISTS PERCONA_SCHEMA", false); + xb_mysql_query(connection, + "CREATE TABLE IF NOT EXISTS PERCONA_SCHEMA.xtrabackup_history(" + "uuid VARCHAR(40) NOT NULL PRIMARY KEY," + "name VARCHAR(255) DEFAULT NULL," + "tool_name VARCHAR(255) DEFAULT NULL," + "tool_command TEXT DEFAULT NULL," + "tool_version VARCHAR(255) DEFAULT NULL," + "ibbackup_version VARCHAR(255) DEFAULT NULL," + "server_version VARCHAR(255) DEFAULT NULL," + "start_time TIMESTAMP NULL DEFAULT NULL," + "end_time TIMESTAMP NULL DEFAULT NULL," + "lock_time BIGINT UNSIGNED DEFAULT NULL," + "binlog_pos VARCHAR(128) DEFAULT NULL," + "innodb_from_lsn BIGINT UNSIGNED DEFAULT NULL," + "innodb_to_lsn BIGINT UNSIGNED DEFAULT NULL," + "partial ENUM('Y', 'N') DEFAULT NULL," + "incremental ENUM('Y', 'N') DEFAULT NULL," + "format ENUM('file', 'tar', 'xbstream') DEFAULT NULL," + "compact ENUM('Y', 'N') DEFAULT NULL," + "compressed ENUM('Y', 'N') DEFAULT NULL," + "encrypted ENUM('Y', 'N') DEFAULT NULL" + ") CHARACTER SET utf8 ENGINE=innodb", false); + + stmt = mysql_stmt_init(connection); + + mysql_stmt_prepare(stmt, ins_query, strlen(ins_query)); + + memset(bind, 0, sizeof(bind)); + idx = 0; + + /* uuid */ + bind[idx].buffer_type = MYSQL_TYPE_STRING; + bind[idx].buffer = uuid; + bind[idx].buffer_length = strlen(uuid); + ++idx; + + /* name */ + bind[idx].buffer_type = MYSQL_TYPE_STRING; + bind[idx].buffer = (char*)(opt_history); + bind[idx].buffer_length = strlen(opt_history); + if (!(opt_history && *opt_history)) { + bind[idx].is_null = &null; + } + ++idx; + + /* tool_name */ + bind[idx].buffer_type = MYSQL_TYPE_STRING; + bind[idx].buffer = tool_name; + bind[idx].buffer_length = strlen(tool_name); + ++idx; + + /* tool_command */ + bind[idx].buffer_type = MYSQL_TYPE_STRING; + bind[idx].buffer = tool_args; + bind[idx].buffer_length = strlen(tool_args); + ++idx; + + /* tool_version */ + bind[idx].buffer_type = MYSQL_TYPE_STRING; + bind[idx].buffer = (char*)(XTRABACKUP_VERSION); + bind[idx].buffer_length = strlen(XTRABACKUP_VERSION); + ++idx; + + /* ibbackup_version */ + bind[idx].buffer_type = MYSQL_TYPE_STRING; + bind[idx].buffer = (char*)(XTRABACKUP_VERSION); + bind[idx].buffer_length = strlen(XTRABACKUP_VERSION); + ++idx; + + /* server_version */ + bind[idx].buffer_type = MYSQL_TYPE_STRING; + bind[idx].buffer = server_version; + bind[idx].buffer_length = strlen(server_version); + ++idx; + + /* start_time */ + bind[idx].buffer_type = MYSQL_TYPE_LONG; + bind[idx].buffer = &history_start_time; + ++idx; + + /* end_time */ + bind[idx].buffer_type = MYSQL_TYPE_LONG; + bind[idx].buffer = &history_end_time; + ++idx; + + /* lock_time */ + bind[idx].buffer_type = MYSQL_TYPE_LONG; + bind[idx].buffer = &history_lock_time; + ++idx; + + /* binlog_pos */ + bind[idx].buffer_type = MYSQL_TYPE_STRING; + bind[idx].buffer = mysql_binlog_position; + if (mysql_binlog_position != NULL) { + bind[idx].buffer_length = strlen(mysql_binlog_position); + } else { + bind[idx].is_null = &null; + } + ++idx; + + /* innodb_from_lsn */ + bind[idx].buffer_type = MYSQL_TYPE_LONGLONG; + bind[idx].buffer = (char*)(&incremental_lsn); + ++idx; + + /* innodb_to_lsn */ + bind[idx].buffer_type = MYSQL_TYPE_LONGLONG; + bind[idx].buffer = (char*)(&metadata_to_lsn); + ++idx; + + /* partial (Y | N) */ + bind[idx].buffer_type = MYSQL_TYPE_STRING; + bind[idx].buffer = (char*)((xtrabackup_tables + || xtrabackup_tables_file + || xtrabackup_databases + || xtrabackup_databases_file) ? "Y" : "N"); + bind[idx].buffer_length = 1; + ++idx; + + /* incremental (Y | N) */ + bind[idx].buffer_type = MYSQL_TYPE_STRING; + bind[idx].buffer = (char*)( + (xtrabackup_incremental + || xtrabackup_incremental_basedir + || opt_incremental_history_name + || opt_incremental_history_uuid) ? "Y" : "N"); + bind[idx].buffer_length = 1; + ++idx; + + /* format (file | tar | xbstream) */ + bind[idx].buffer_type = MYSQL_TYPE_STRING; + bind[idx].buffer = (char*)(xb_stream_name[xtrabackup_stream_fmt]); + bind[idx].buffer_length = strlen(xb_stream_name[xtrabackup_stream_fmt]); + ++idx; + + /* compact (Y | N) */ + bind[idx].buffer_type = MYSQL_TYPE_STRING; + bind[idx].buffer = (char*)(xtrabackup_compact ? "Y" : "N"); + bind[idx].buffer_length = 1; + ++idx; + + /* compressed (Y | N) */ + bind[idx].buffer_type = MYSQL_TYPE_STRING; + bind[idx].buffer = (char*)(xtrabackup_compress ? "Y" : "N"); + bind[idx].buffer_length = 1; + ++idx; + + /* encrypted (Y | N) */ + bind[idx].buffer_type = MYSQL_TYPE_STRING; + bind[idx].buffer = (char*)(xtrabackup_encrypt ? "Y" : "N"); + bind[idx].buffer_length = 1; + ++idx; + + ut_ad(idx == 19); + + mysql_stmt_bind_param(stmt, bind); + + mysql_stmt_execute(stmt); + mysql_stmt_close(stmt); + +cleanup: + + free(uuid); + free(server_version); + + return(true); +} + +bool +write_backup_config_file() +{ + return backup_file_printf("backup-my.cnf", + "# This MySQL options file was generated by innobackupex.\n\n" + "# The MySQL server\n" + "[mysqld]\n" + "innodb_checksum_algorithm=%s\n" + "innodb_log_checksum_algorithm=%s\n" + "innodb_data_file_path=%s\n" + "innodb_log_files_in_group=%lu\n" + "innodb_log_file_size=%lld\n" + "innodb_fast_checksum=%s\n" + "innodb_page_size=%lu\n" + "innodb_log_block_size=%lu\n" + "innodb_undo_directory=%s\n" + "innodb_undo_tablespaces=%lu\n" + "%s%s\n" + "%s%s\n", + innodb_checksum_algorithm_names[srv_checksum_algorithm], + innodb_checksum_algorithm_names[srv_log_checksum_algorithm], + innobase_data_file_path, + srv_n_log_files, + innobase_log_file_size, + srv_fast_checksum ? "true" : "false", + srv_page_size, + srv_log_block_size, + srv_undo_dir, + srv_undo_tablespaces, + innobase_doublewrite_file ? "innodb_doublewrite_file=" : "", + innobase_doublewrite_file ? innobase_doublewrite_file : "", + innobase_buffer_pool_filename ? + "innodb_buffer_pool_filename=" : "", + innobase_buffer_pool_filename ? + innobase_buffer_pool_filename : ""); +} + + +static +char *make_argv(char *buf, size_t len, int argc, char **argv) +{ + size_t left= len; + const char *arg; + + buf[0]= 0; + ++argv; --argc; + while (argc > 0 && left > 0) + { + arg = *argv; + if (strncmp(*argv, "--password", strlen("--password")) == 0) { + arg = "--password=..."; + } + if (strncmp(*argv, "--encrypt-key", + strlen("--encrypt-key")) == 0) { + arg = "--encrypt-key=..."; + } + if (strncmp(*argv, "--encrypt_key", + strlen("--encrypt_key")) == 0) { + arg = "--encrypt_key=..."; + } + left-= ut_snprintf(buf + len - left, left, + "%s%c", arg, argc > 1 ? ' ' : 0); + ++argv; --argc; + } + + return buf; +} + +void +capture_tool_command(int argc, char **argv) +{ + /* capture tool name tool args */ + tool_name = strrchr(argv[0], '/'); + tool_name = tool_name ? tool_name + 1 : argv[0]; + + make_argv(tool_args, sizeof(tool_args), argc, argv); +} + + +bool +select_history() +{ + if (opt_incremental_history_name || opt_incremental_history_uuid) { + if (!select_incremental_lsn_from_history( + &incremental_lsn)) { + return(false); + } + } + return(true); +} + +bool +flush_changed_page_bitmaps() +{ + if (xtrabackup_incremental && have_changed_page_bitmaps && + !xtrabackup_incremental_force_scan) { + xb_mysql_query(mysql_connection, + "FLUSH NO_WRITE_TO_BINLOG CHANGED_PAGE_BITMAPS", false); + } + return(true); +} + + +/*********************************************************************//** +Deallocate memory, disconnect from MySQL server, etc. +@return true on success. */ +void +backup_cleanup() +{ + free(mysql_slave_position); + free(mysql_binlog_position); + free(buffer_pool_filename); + + if (mysql_connection) { + mysql_close(mysql_connection); + } +} diff --git a/extra/mariabackup/backup_mysql.h b/extra/mariabackup/backup_mysql.h new file mode 100644 index 00000000000..3ccd7bdb613 --- /dev/null +++ b/extra/mariabackup/backup_mysql.h @@ -0,0 +1,92 @@ +#ifndef XTRABACKUP_BACKUP_MYSQL_H +#define XTRABACKUP_BACKUP_MYSQL_H + +#include + +/* mysql flavor and version */ +enum mysql_flavor_t { FLAVOR_UNKNOWN, FLAVOR_MYSQL, + FLAVOR_PERCONA_SERVER, FLAVOR_MARIADB }; +extern mysql_flavor_t server_flavor; +extern unsigned long mysql_server_version; + +/* server capabilities */ +extern bool have_changed_page_bitmaps; +extern bool have_backup_locks; +extern bool have_lock_wait_timeout; +extern bool have_galera_enabled; +extern bool have_flush_engine_logs; +extern bool have_multi_threaded_slave; +extern bool have_gtid_slave; + + +/* History on server */ +extern time_t history_start_time; +extern time_t history_end_time; +extern time_t history_lock_time; + + +extern bool sql_thread_started; +extern char *mysql_slave_position; +extern char *mysql_binlog_position; +extern char *buffer_pool_filename; + +/** connection to mysql server */ +extern MYSQL *mysql_connection; + +void +capture_tool_command(int argc, char **argv); + +bool +select_history(); + +bool +flush_changed_page_bitmaps(); + +void +backup_cleanup(); + +bool +get_mysql_vars(MYSQL *connection); + +bool +detect_mysql_capabilities_for_backup(); + +MYSQL * +xb_mysql_connect(); + +MYSQL_RES * +xb_mysql_query(MYSQL *connection, const char *query, bool use_result, + bool die_on_error = true); + +void +unlock_all(MYSQL *connection); + +bool +write_current_binlog_file(MYSQL *connection); + +bool +write_binlog_info(MYSQL *connection); + +bool +write_xtrabackup_info(MYSQL *connection); + +bool +write_backup_config_file(); + +bool +lock_binlog_maybe(MYSQL *connection); + +bool +lock_tables(MYSQL *connection); + +bool +wait_for_safe_slave(MYSQL *connection); + +bool +write_galera_info(MYSQL *connection); + +bool +write_slave_info(MYSQL *connection); + + +#endif diff --git a/extra/mariabackup/changed_page_bitmap.cc b/extra/mariabackup/changed_page_bitmap.cc new file mode 100644 index 00000000000..e385474c7aa --- /dev/null +++ b/extra/mariabackup/changed_page_bitmap.cc @@ -0,0 +1,1018 @@ +/****************************************************** +XtraBackup: hot backup tool for InnoDB +(c) 2009-2012 Percona Inc. +Originally Created 3/3/2009 Yasufumi Kinoshita +Written by Alexey Kopytov, Aleksandr Kuzminsky, Stewart Smith, Vadim Tkachenko, +Yasufumi Kinoshita, Ignacio Nin and Baron Schwartz. + +This program is free software; you can redistribute it and/or modify +it under the terms of the GNU General Public License as published by +the Free Software Foundation; version 2 of the License. + +This program is distributed in the hope that it will be useful, +but WITHOUT ANY WARRANTY; without even the implied warranty of +MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the +GNU General Public License for more details. + +You should have received a copy of the GNU General Public License +along with this program; if not, write to the Free Software +Foundation, Inc., 51 Franklin Street, Fifth Floor, Boston, MA 02110-1301, USA + +*******************************************************/ + +/* Changed page bitmap implementation */ + +#include "changed_page_bitmap.h" + +#include "common.h" +#include "xtrabackup.h" + +/* TODO: copy-pasted shared definitions from the XtraDB bitmap write code. +Remove these on the first opportunity, i.e. single-binary XtraBackup. */ + +/* log0online.h */ + +/** Single bitmap file information */ +struct log_online_bitmap_file_t { + char name[FN_REFLEN]; /*!< Name with full path */ + os_file_t file; /*!< Handle to opened file */ + ib_uint64_t size; /*!< Size of the file */ + ib_uint64_t offset; /*!< Offset of the next read, + or count of already-read bytes + */ +}; + +/** A set of bitmap files containing some LSN range */ +struct log_online_bitmap_file_range_t { + size_t count; /*!< Number of files */ + /*!< Dynamically-allocated array of info about individual files */ + struct files_t { + char name[FN_REFLEN];/*!< Name of a file */ + lsn_t start_lsn; /*!< Starting LSN of data in this + file */ + ulong seq_num; /*!< Sequence number of this file */ + } *files; +}; + +/* log0online.c */ + +/** File name stem for bitmap files. */ +static const char* bmp_file_name_stem = "ib_modified_log_"; + +/** The bitmap file block size in bytes. All writes will be multiples of this. + */ +enum { + MODIFIED_PAGE_BLOCK_SIZE = 4096 +}; + +/** Offsets in a file bitmap block */ +enum { + MODIFIED_PAGE_IS_LAST_BLOCK = 0,/* 1 if last block in the current + write, 0 otherwise. */ + MODIFIED_PAGE_START_LSN = 4, /* The starting tracked LSN of this and + other blocks in the same write */ + MODIFIED_PAGE_END_LSN = 12, /* The ending tracked LSN of this and + other blocks in the same write */ + MODIFIED_PAGE_SPACE_ID = 20, /* The space ID of tracked pages in + this block */ + MODIFIED_PAGE_1ST_PAGE_ID = 24, /* The page ID of the first tracked + page in this block */ + MODIFIED_PAGE_BLOCK_UNUSED_1 = 28,/* Unused in order to align the start + of bitmap at 8 byte boundary */ + MODIFIED_PAGE_BLOCK_BITMAP = 32,/* Start of the bitmap itself */ + MODIFIED_PAGE_BLOCK_UNUSED_2 = MODIFIED_PAGE_BLOCK_SIZE - 8, + /* Unused in order to align the end of + bitmap at 8 byte boundary */ + MODIFIED_PAGE_BLOCK_CHECKSUM = MODIFIED_PAGE_BLOCK_SIZE - 4 + /* The checksum of the current block */ +}; + +/** Length of the bitmap data in a block */ +enum { MODIFIED_PAGE_BLOCK_BITMAP_LEN + = MODIFIED_PAGE_BLOCK_UNUSED_2 - MODIFIED_PAGE_BLOCK_BITMAP }; + +/** Length of the bitmap data in a block in page ids */ +enum { MODIFIED_PAGE_BLOCK_ID_COUNT = MODIFIED_PAGE_BLOCK_BITMAP_LEN * 8 }; + +typedef ib_uint64_t bitmap_word_t; + +/****************************************************************//** +Calculate a bitmap block checksum. Algorithm borrowed from +log_block_calc_checksum. +@return checksum */ +UNIV_INLINE +ulint +log_online_calc_checksum( +/*=====================*/ + const byte* block); /*! p2 +*/ +static +int +log_online_compare_bmp_keys( +/*========================*/ + const void* p1, /*! k2_start_page ? 1 : 0; + } + return k1_space < k2_space ? -1 : 1; +} + +/****************************************************************//** +Calculate a bitmap block checksum. Algorithm borrowed from +log_block_calc_checksum. +@return checksum */ +UNIV_INLINE +ulint +log_online_calc_checksum( +/*=====================*/ + const byte* block) /*! 24) { + + sh = 0; + } + } + + return sum; +} + +/****************************************************************//** +Read one bitmap data page and check it for corruption. + +@return TRUE if page read OK, FALSE if I/O error */ +static +ibool +log_online_read_bitmap_page( +/*========================*/ + log_online_bitmap_file_t *bitmap_file, /*!size >= MODIFIED_PAGE_BLOCK_SIZE); + ut_a(bitmap_file->offset + <= bitmap_file->size - MODIFIED_PAGE_BLOCK_SIZE); + ut_a(bitmap_file->offset % MODIFIED_PAGE_BLOCK_SIZE == 0); + + success = os_file_read(bitmap_file->file, page, bitmap_file->offset, + MODIFIED_PAGE_BLOCK_SIZE); + + if (UNIV_UNLIKELY(!success)) { + + /* The following call prints an error message */ + os_file_get_last_error(TRUE); + msg("InnoDB: Warning: failed reading changed page bitmap " + "file \'%s\'\n", bitmap_file->name); + return FALSE; + } + + bitmap_file->offset += MODIFIED_PAGE_BLOCK_SIZE; + ut_ad(bitmap_file->offset <= bitmap_file->size); + + checksum = mach_read_from_4(page + MODIFIED_PAGE_BLOCK_CHECKSUM); + actual_checksum = log_online_calc_checksum(page); + *checksum_ok = (checksum == actual_checksum); + + return TRUE; +} + +/*********************************************************************//** +Check the name of a given file if it's a changed page bitmap file and +return file sequence and start LSN name components if it is. If is not, +the values of output parameters are undefined. + +@return TRUE if a given file is a changed page bitmap file. */ +static +ibool +log_online_is_bitmap_file( +/*======================*/ + const os_file_stat_t* file_info, /*!name) < OS_FILE_MAX_PATH); + + return ((file_info->type == OS_FILE_TYPE_FILE + || file_info->type == OS_FILE_TYPE_LINK) + && (sscanf(file_info->name, "%[a-z_]%lu_" LSN_PF ".xdb", stem, + bitmap_file_seq_num, bitmap_file_start_lsn) == 3) + && (!strcmp(stem, bmp_file_name_stem))); +} + +/*********************************************************************//** +List the bitmap files in srv_data_home and setup their range that contains the +specified LSN interval. This range, if non-empty, will start with a file that +has the greatest LSN equal to or less than the start LSN and will include all +the files up to the one with the greatest LSN less than the end LSN. Caller +must free bitmap_files->files when done if bitmap_files set to non-NULL and +this function returned TRUE. Field bitmap_files->count might be set to a +larger value than the actual count of the files, and space for the unused array +slots will be allocated but cleared to zeroes. + +@return TRUE if succeeded +*/ +static +ibool +log_online_setup_bitmap_file_range( +/*===============================*/ + log_online_bitmap_file_range_t *bitmap_files, /*!= range_start); + + bitmap_files->count = 0; + bitmap_files->files = NULL; + + /* 1st pass: size the info array */ + + bitmap_dir = os_file_opendir(srv_data_home, FALSE); + if (UNIV_UNLIKELY(!bitmap_dir)) { + + msg("InnoDB: Error: failed to open bitmap directory \'%s\'\n", + srv_data_home); + return FALSE; + } + + while (!os_file_readdir_next_file(srv_data_home, bitmap_dir, + &bitmap_dir_file_info)) { + + ulong file_seq_num; + lsn_t file_start_lsn; + + if (!log_online_is_bitmap_file(&bitmap_dir_file_info, + &file_seq_num, + &file_start_lsn) + || file_start_lsn >= range_end) { + + continue; + } + + if (file_seq_num > last_file_seq_num) { + + last_file_seq_num = file_seq_num; + } + + if (file_start_lsn >= range_start + || file_start_lsn == first_file_start_lsn + || first_file_start_lsn > range_start) { + + /* A file that falls into the range */ + + if (file_start_lsn < first_file_start_lsn) { + + first_file_start_lsn = file_start_lsn; + } + if (file_seq_num < first_file_seq_num) { + + first_file_seq_num = file_seq_num; + } + } else if (file_start_lsn > first_file_start_lsn) { + + /* A file that has LSN closer to the range start + but smaller than it, replacing another such file */ + first_file_start_lsn = file_start_lsn; + first_file_seq_num = file_seq_num; + } + } + + if (UNIV_UNLIKELY(os_file_closedir(bitmap_dir))) { + + os_file_get_last_error(TRUE); + msg("InnoDB: Error: cannot close \'%s\'\n",srv_data_home); + return FALSE; + } + + if (first_file_seq_num == ULONG_MAX && last_file_seq_num == 0) { + + bitmap_files->count = 0; + return TRUE; + } + + bitmap_files->count = last_file_seq_num - first_file_seq_num + 1; + + /* 2nd pass: get the file names in the file_seq_num order */ + + bitmap_dir = os_file_opendir(srv_data_home, FALSE); + if (UNIV_UNLIKELY(!bitmap_dir)) { + + msg("InnoDB: Error: failed to open bitmap directory \'%s\'\n", + srv_data_home); + return FALSE; + } + + bitmap_files->files = + static_cast + (ut_malloc(bitmap_files->count + * sizeof(bitmap_files->files[0]))); + memset(bitmap_files->files, 0, + bitmap_files->count * sizeof(bitmap_files->files[0])); + + while (!os_file_readdir_next_file(srv_data_home, bitmap_dir, + &bitmap_dir_file_info)) { + + ulong file_seq_num; + lsn_t file_start_lsn; + size_t array_pos; + + if (!log_online_is_bitmap_file(&bitmap_dir_file_info, + &file_seq_num, + &file_start_lsn) + || file_start_lsn >= range_end + || file_start_lsn < first_file_start_lsn) { + + continue; + } + + array_pos = file_seq_num - first_file_seq_num; + if (UNIV_UNLIKELY(array_pos >= bitmap_files->count)) { + + msg("InnoDB: Error: inconsistent bitmap file " + "directory\n"); + free(bitmap_files->files); + return FALSE; + } + + if (file_seq_num > bitmap_files->files[array_pos].seq_num) { + + bitmap_files->files[array_pos].seq_num = file_seq_num; + strncpy(bitmap_files->files[array_pos].name, + bitmap_dir_file_info.name, FN_REFLEN); + bitmap_files->files[array_pos].name[FN_REFLEN - 1] + = '\0'; + bitmap_files->files[array_pos].start_lsn + = file_start_lsn; + } + } + + if (UNIV_UNLIKELY(os_file_closedir(bitmap_dir))) { + + os_file_get_last_error(TRUE); + msg("InnoDB: Error: cannot close \'%s\'\n", srv_data_home); + free(bitmap_files->files); + return FALSE; + } + +#ifdef UNIV_DEBUG + ut_ad(bitmap_files->files[0].seq_num == first_file_seq_num); + + for (size_t i = 1; i < bitmap_files->count; i++) { + if (!bitmap_files->files[i].seq_num) { + + break; + } + ut_ad(bitmap_files->files[i].seq_num + > bitmap_files->files[i - 1].seq_num); + ut_ad(bitmap_files->files[i].start_lsn + >= bitmap_files->files[i - 1].start_lsn); + } +#endif + + return TRUE; +} + +/****************************************************************//** +Open a bitmap file for reading. + +@return TRUE if opened successfully */ +static +ibool +log_online_open_bitmap_file_read_only( +/*==================================*/ + const char* name, /*!name, FN_REFLEN, "%s%s", srv_data_home, name); + bitmap_file->file + = os_file_create_simple_no_error_handling(0, bitmap_file->name, + OS_FILE_OPEN, + OS_FILE_READ_ONLY, + &success); + if (UNIV_UNLIKELY(!success)) { + + /* Here and below assume that bitmap file names do not + contain apostrophes, thus no need for ut_print_filename(). */ + msg("InnoDB: Warning: error opening the changed page " + "bitmap \'%s\'\n", bitmap_file->name); + return FALSE; + } + + bitmap_file->size = os_file_get_size(bitmap_file->file); + bitmap_file->offset = 0; + +#ifdef UNIV_LINUX + posix_fadvise(bitmap_file->file, 0, 0, POSIX_FADV_SEQUENTIAL); + posix_fadvise(bitmap_file->file, 0, 0, POSIX_FADV_NOREUSE); +#endif + + return TRUE; +} + +/****************************************************************//** +Diagnose one or both of the following situations if we read close to +the end of bitmap file: +1) Warn if the remainder of the file is less than one page. +2) Error if we cannot read any more full pages but the last read page +did not have the last-in-run flag set. + +@return FALSE for the error */ +static +ibool +log_online_diagnose_bitmap_eof( +/*===========================*/ + const log_online_bitmap_file_t* bitmap_file, /*!< in: bitmap file */ + ibool last_page_in_run)/*!< in: "last page in + run" flag value in the + last read page */ +{ + /* Check if we are too close to EOF to read a full page */ + if ((bitmap_file->size < MODIFIED_PAGE_BLOCK_SIZE) + || (bitmap_file->offset + > bitmap_file->size - MODIFIED_PAGE_BLOCK_SIZE)) { + + if (UNIV_UNLIKELY(bitmap_file->offset != bitmap_file->size)) { + + /* If we are not at EOF and we have less than one page + to read, it's junk. This error is not fatal in + itself. */ + + msg("InnoDB: Warning: junk at the end of changed " + "page bitmap file \'%s\'.\n", bitmap_file->name); + } + + if (UNIV_UNLIKELY(!last_page_in_run)) { + + /* We are at EOF but the last read page did not finish + a run */ + /* It's a "Warning" here because it's not a fatal error + for the whole server */ + msg("InnoDB: Warning: changed page bitmap " + "file \'%s\' does not contain a complete run " + "at the end.\n", bitmap_file->name); + return FALSE; + } + } + return TRUE; +} + +/* End of copy-pasted definitions */ + +/** Iterator structure over changed page bitmap */ +struct xb_page_bitmap_range_struct { + const xb_page_bitmap *bitmap; /* Bitmap with data */ + ulint space_id; /* Space id for this + iterator */ + ulint bit_i; /* Bit index of the iterator + position in the current page */ + const ib_rbt_node_t *bitmap_node; /* Current bitmap tree node */ + const byte *bitmap_page; /* Current bitmap page */ + ulint current_page_id;/* Current page id */ +}; + +/****************************************************************//** +Print a diagnostic message on missing bitmap data for an LSN range. */ +static +void +xb_msg_missing_lsn_data( +/*====================*/ + lsn_t missing_interval_start, /*!size >= MODIFIED_PAGE_BLOCK_SIZE); + + *page_end_lsn = 0; + + while ((*page_end_lsn <= lsn) + && (bitmap_file->offset + <= bitmap_file->size - MODIFIED_PAGE_BLOCK_SIZE)) { + + next_to_last_page_ok = last_page_ok; + if (!log_online_read_bitmap_page(bitmap_file, page, + &last_page_ok)) { + + return FALSE; + } + + *page_end_lsn = mach_read_from_8(page + MODIFIED_PAGE_END_LSN); + } + + /* We check two pages here because the last read page already contains + the required LSN data. If the next to the last one page is corrupted, + then we have no way of telling if that page contained the required LSN + range data too */ + return last_page_ok && next_to_last_page_ok; +} + +/****************************************************************//** +Read the disk bitmap and build the changed page bitmap tree for the +LSN interval incremental_lsn to checkpoint_lsn_start. + +@return the built bitmap tree or NULL if unable to read the full interval for +any reason. */ +xb_page_bitmap* +xb_page_bitmap_init(void) +/*=====================*/ +{ + log_online_bitmap_file_t bitmap_file; + lsn_t bmp_start_lsn = incremental_lsn; + lsn_t bmp_end_lsn = checkpoint_lsn_start; + byte page[MODIFIED_PAGE_BLOCK_SIZE]; + lsn_t current_page_end_lsn; + xb_page_bitmap *result; + ibool last_page_in_run= FALSE; + log_online_bitmap_file_range_t bitmap_files; + size_t bmp_i; + ibool last_page_ok = TRUE; + + if (UNIV_UNLIKELY(bmp_start_lsn > bmp_end_lsn)) { + + msg("xtrabackup: incremental backup LSN " LSN_PF + " is larger than than the last checkpoint LSN " LSN_PF + "\n", bmp_start_lsn, bmp_end_lsn); + return NULL; + } + + if (!log_online_setup_bitmap_file_range(&bitmap_files, bmp_start_lsn, + bmp_end_lsn)) { + + return NULL; + } + + /* Only accept no bitmap files returned if start LSN == end LSN */ + if (bitmap_files.count == 0 && bmp_end_lsn != bmp_start_lsn) { + + return NULL; + } + + result = rbt_create(MODIFIED_PAGE_BLOCK_SIZE, + log_online_compare_bmp_keys); + + if (bmp_start_lsn == bmp_end_lsn) { + + /* Empty range - empty bitmap */ + return result; + } + + bmp_i = 0; + + if (UNIV_UNLIKELY(bitmap_files.files[bmp_i].start_lsn + > bmp_start_lsn)) { + + /* The 1st file does not have the starting LSN data */ + xb_msg_missing_lsn_data(bmp_start_lsn, + bitmap_files.files[bmp_i].start_lsn); + rbt_free(result); + free(bitmap_files.files); + return NULL; + } + + /* Skip any zero-sized files at the start */ + while ((bmp_i < bitmap_files.count - 1) + && (bitmap_files.files[bmp_i].start_lsn + == bitmap_files.files[bmp_i + 1].start_lsn)) { + + bmp_i++; + } + + /* Is the 1st bitmap file missing? */ + if (UNIV_UNLIKELY(bitmap_files.files[bmp_i].name[0] == '\0')) { + + /* TODO: this is not the exact missing range */ + xb_msg_missing_lsn_data(bmp_start_lsn, bmp_end_lsn); + rbt_free(result); + free(bitmap_files.files); + return NULL; + } + + /* Open the 1st bitmap file */ + if (UNIV_UNLIKELY(!log_online_open_bitmap_file_read_only( + bitmap_files.files[bmp_i].name, + &bitmap_file))) { + + rbt_free(result); + free(bitmap_files.files); + return NULL; + } + + /* If the 1st file is truncated, no data. Not merged with the case + below because zero-length file indicates not a corruption but missing + subsequent files instead. */ + if (UNIV_UNLIKELY(bitmap_file.size < MODIFIED_PAGE_BLOCK_SIZE)) { + + xb_msg_missing_lsn_data(bmp_start_lsn, bmp_end_lsn); + rbt_free(result); + free(bitmap_files.files); + os_file_close(bitmap_file.file); + return NULL; + } + + /* Find the start of the required LSN range in the file */ + if (UNIV_UNLIKELY(!xb_find_lsn_in_bitmap_file(&bitmap_file, page, + ¤t_page_end_lsn, + bmp_start_lsn))) { + + msg("xtrabackup: Warning: changed page bitmap file " + "\'%s\' corrupted\n", bitmap_file.name); + rbt_free(result); + free(bitmap_files.files); + os_file_close(bitmap_file.file); + return NULL; + } + + last_page_in_run + = mach_read_from_4(page + MODIFIED_PAGE_IS_LAST_BLOCK); + + if (UNIV_UNLIKELY(!log_online_diagnose_bitmap_eof(&bitmap_file, + last_page_in_run))) { + + rbt_free(result); + free(bitmap_files.files); + os_file_close(bitmap_file.file); + return NULL; + } + + if (UNIV_UNLIKELY(current_page_end_lsn < bmp_start_lsn)) { + + xb_msg_missing_lsn_data(current_page_end_lsn, bmp_start_lsn); + rbt_free(result); + free(bitmap_files.files); + os_file_close(bitmap_file.file); + return NULL; + } + + /* 1st bitmap page found, add it to the tree. */ + rbt_insert(result, page, page); + + /* Read next pages/files until all required data is read */ + while (last_page_ok + && (current_page_end_lsn < bmp_end_lsn + || (current_page_end_lsn == bmp_end_lsn + && !last_page_in_run))) { + + ib_rbt_bound_t tree_search_pos; + + /* If EOF, advance the file skipping over any empty files */ + while (bitmap_file.size < MODIFIED_PAGE_BLOCK_SIZE + || (bitmap_file.offset + > bitmap_file.size - MODIFIED_PAGE_BLOCK_SIZE)) { + + os_file_close(bitmap_file.file); + + if (UNIV_UNLIKELY( + !log_online_diagnose_bitmap_eof( + &bitmap_file, last_page_in_run))) { + + rbt_free(result); + free(bitmap_files.files); + return NULL; + } + + bmp_i++; + + if (UNIV_UNLIKELY(bmp_i == bitmap_files.count + || (bitmap_files.files[bmp_i].seq_num + == 0))) { + + xb_msg_missing_lsn_data(current_page_end_lsn, + bmp_end_lsn); + rbt_free(result); + free(bitmap_files.files); + return NULL; + } + + /* Is the next file missing? */ + if (UNIV_UNLIKELY(bitmap_files.files[bmp_i].name[0] + == '\0')) { + + /* TODO: this is not the exact missing range */ + xb_msg_missing_lsn_data(bitmap_files.files + [bmp_i - 1].start_lsn, + bmp_end_lsn); + rbt_free(result); + free(bitmap_files.files); + return NULL; + } + + if (UNIV_UNLIKELY( + !log_online_open_bitmap_file_read_only( + bitmap_files.files[bmp_i].name, + &bitmap_file))) { + + rbt_free(result); + free(bitmap_files.files); + return NULL; + } + } + + if (UNIV_UNLIKELY( + !log_online_read_bitmap_page(&bitmap_file, page, + &last_page_ok))) { + + rbt_free(result); + free(bitmap_files.files); + os_file_close(bitmap_file.file); + return NULL; + } + + if (UNIV_UNLIKELY(!last_page_ok)) { + + msg("xtrabackup: warning: changed page bitmap file " + "\'%s\' corrupted.\n", bitmap_file.name); + rbt_free(result); + free(bitmap_files.files); + os_file_close(bitmap_file.file); + return NULL; + } + + /* Merge the current page with an existing page or insert a new + page into the tree */ + + if (!rbt_search(result, &tree_search_pos, page)) { + + /* Merge the bitmap pages */ + byte *existing_page + = rbt_value(byte, tree_search_pos.last); + bitmap_word_t *bmp_word_1 = (bitmap_word_t *) + (existing_page + MODIFIED_PAGE_BLOCK_BITMAP); + bitmap_word_t *bmp_end = (bitmap_word_t *) + (existing_page + MODIFIED_PAGE_BLOCK_UNUSED_2); + bitmap_word_t *bmp_word_2 = (bitmap_word_t *) + (page + MODIFIED_PAGE_BLOCK_BITMAP); + while (bmp_word_1 < bmp_end) { + + *bmp_word_1++ |= *bmp_word_2++; + } + xb_a (bmp_word_1 == bmp_end); + } else { + + /* Add a new page */ + rbt_add_node(result, &tree_search_pos, page); + } + + current_page_end_lsn + = mach_read_from_8(page + MODIFIED_PAGE_END_LSN); + last_page_in_run + = mach_read_from_4(page + MODIFIED_PAGE_IS_LAST_BLOCK); + } + + xb_a (current_page_end_lsn >= bmp_end_lsn); + + free(bitmap_files.files); + os_file_close(bitmap_file.file); + + return result; +} + +/****************************************************************//** +Free the bitmap tree. */ +void +xb_page_bitmap_deinit( +/*==================*/ + xb_page_bitmap* bitmap) /*!bitmap_page has been +already found/bumped by rbt_search()/rbt_next(). + +@return FALSE if no more bitmap data for the range space ID */ +static +ibool +xb_page_bitmap_setup_next_page( +/*===========================*/ + xb_page_bitmap_range* bitmap_range) /*!bitmap_node == NULL) { + + bitmap_range->current_page_id = ULINT_UNDEFINED; + return FALSE; + } + + bitmap_range->bitmap_page = rbt_value(byte, bitmap_range->bitmap_node); + + new_space_id = mach_read_from_4(bitmap_range->bitmap_page + + MODIFIED_PAGE_SPACE_ID); + if (new_space_id != bitmap_range->space_id) { + + /* No more data for the current page id. */ + xb_a(new_space_id > bitmap_range->space_id); + bitmap_range->current_page_id = ULINT_UNDEFINED; + return FALSE; + } + + new_1st_page_id = mach_read_from_4(bitmap_range->bitmap_page + + MODIFIED_PAGE_1ST_PAGE_ID); + xb_a (new_1st_page_id >= bitmap_range->current_page_id + || bitmap_range->current_page_id == ULINT_UNDEFINED); + + bitmap_range->current_page_id = new_1st_page_id; + bitmap_range->bit_i = 0; + + return TRUE; +} + +/****************************************************************//** +Set up a new bitmap range iterator over a given space id changed +pages in a given bitmap. + +@return bitmap range iterator */ +xb_page_bitmap_range* +xb_page_bitmap_range_init( +/*======================*/ + xb_page_bitmap* bitmap, /*!< in: bitmap to iterate over */ + ulint space_id) /*!< in: space id */ +{ + byte search_page[MODIFIED_PAGE_BLOCK_SIZE]; + xb_page_bitmap_range *result + = static_cast + (ut_malloc(sizeof(*result))); + + memset(result, 0, sizeof(*result)); + result->bitmap = bitmap; + result->space_id = space_id; + result->current_page_id = ULINT_UNDEFINED; + + /* Search for the 1st page for the given space id */ + /* This also sets MODIFIED_PAGE_1ST_PAGE_ID to 0, which is what we + want. */ + memset(search_page, 0, MODIFIED_PAGE_BLOCK_SIZE); + mach_write_to_4(search_page + MODIFIED_PAGE_SPACE_ID, space_id); + + result->bitmap_node = rbt_lower_bound(result->bitmap, search_page); + + xb_page_bitmap_setup_next_page(result); + + return result; +} + +/****************************************************************//** +Get the value of the bitmap->range->bit_i bitmap bit + +@return the current bit value */ +static inline +ibool +is_bit_set( +/*=======*/ + const xb_page_bitmap_range* bitmap_range) /*!< in: bitmap + range */ +{ + return ((*(((bitmap_word_t *)(bitmap_range->bitmap_page + + MODIFIED_PAGE_BLOCK_BITMAP)) + + (bitmap_range->bit_i >> 6))) + & (1ULL << (bitmap_range->bit_i & 0x3F))) ? TRUE : FALSE; +} + +/****************************************************************//** +Get the next page id that has its bit set or cleared, i.e. equal to +bit_value. + +@return page id */ +ulint +xb_page_bitmap_range_get_next_bit( +/*==============================*/ + xb_page_bitmap_range* bitmap_range, /*!< in/out: bitmap range */ + ibool bit_value) /*!< in: bit value */ +{ + if (UNIV_UNLIKELY(bitmap_range->current_page_id + == ULINT_UNDEFINED)) { + + return ULINT_UNDEFINED; + } + + do { + while (bitmap_range->bit_i < MODIFIED_PAGE_BLOCK_ID_COUNT) { + + while (is_bit_set(bitmap_range) != bit_value + && (bitmap_range->bit_i + < MODIFIED_PAGE_BLOCK_ID_COUNT)) { + + bitmap_range->current_page_id++; + bitmap_range->bit_i++; + } + + if (bitmap_range->bit_i + < MODIFIED_PAGE_BLOCK_ID_COUNT) { + + ulint result = bitmap_range->current_page_id; + bitmap_range->current_page_id++; + bitmap_range->bit_i++; + return result; + } + } + + bitmap_range->bitmap_node + = rbt_next(bitmap_range->bitmap, + bitmap_range->bitmap_node); + + } while (xb_page_bitmap_setup_next_page(bitmap_range)); + + return ULINT_UNDEFINED; +} + +/****************************************************************//** +Free the bitmap range iterator. */ +void +xb_page_bitmap_range_deinit( +/*========================*/ + xb_page_bitmap_range* bitmap_range) /*! in/out: bitmap range */ +{ + ut_free(bitmap_range); +} diff --git a/extra/mariabackup/changed_page_bitmap.h b/extra/mariabackup/changed_page_bitmap.h new file mode 100644 index 00000000000..6f549f47400 --- /dev/null +++ b/extra/mariabackup/changed_page_bitmap.h @@ -0,0 +1,85 @@ +/****************************************************** +XtraBackup: hot backup tool for InnoDB +(c) 2009-2012 Percona Inc. +Originally Created 3/3/2009 Yasufumi Kinoshita +Written by Alexey Kopytov, Aleksandr Kuzminsky, Stewart Smith, Vadim Tkachenko, +Yasufumi Kinoshita, Ignacio Nin and Baron Schwartz. + +This program is free software; you can redistribute it and/or modify +it under the terms of the GNU General Public License as published by +the Free Software Foundation; version 2 of the License. + +This program is distributed in the hope that it will be useful, +but WITHOUT ANY WARRANTY; without even the implied warranty of +MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the +GNU General Public License for more details. + +You should have received a copy of the GNU General Public License +along with this program; if not, write to the Free Software +Foundation, Inc., 51 Franklin Street, Fifth Floor, Boston, MA 02110-1301, USA + +*******************************************************/ + +/* Changed page bitmap interface */ + +#ifndef XB_CHANGED_PAGE_BITMAP_H +#define XB_CHANGED_PAGE_BITMAP_H + +#include +#include + +/* The changed page bitmap structure */ +typedef ib_rbt_t xb_page_bitmap; + +struct xb_page_bitmap_range_struct; + +/* The bitmap range iterator over one space id */ +typedef struct xb_page_bitmap_range_struct xb_page_bitmap_range; + +/****************************************************************//** +Read the disk bitmap and build the changed page bitmap tree for the +LSN interval incremental_lsn to checkpoint_lsn_start. + +@return the built bitmap tree */ +xb_page_bitmap* +xb_page_bitmap_init(void); +/*=====================*/ + +/****************************************************************//** +Free the bitmap tree. */ +void +xb_page_bitmap_deinit( +/*==================*/ + xb_page_bitmap* bitmap); /*! +#include +#include +#include + +#define xb_a(expr) \ + do { \ + if (!(expr)) { \ + msg("Assertion \"%s\" failed at %s:%lu\n", \ + #expr, __FILE__, (ulong) __LINE__); \ + abort(); \ + } \ + } while (0); + +#ifdef XB_DEBUG +#define xb_ad(expr) xb_a(expr) +#else +#define xb_ad(expr) +#endif + +#define XB_DELTA_INFO_SUFFIX ".meta" + +static inline int msg(const char *fmt, ...) ATTRIBUTE_FORMAT(printf, 1, 2); +static inline int msg(const char *fmt, ...) +{ + int result; + va_list args; + + va_start(args, fmt); + result = vfprintf(stderr, fmt, args); + va_end(args); + + return result; +} + +static inline int msg_ts(const char *fmt, ...) ATTRIBUTE_FORMAT(printf, 1, 2); +static inline int msg_ts(const char *fmt, ...) +{ + int result; + time_t t = time(NULL); + char date[100]; + char *line; + va_list args; + + strftime(date, sizeof(date), "%y%m%d %H:%M:%S", localtime(&t)); + + va_start(args, fmt); + result = vasprintf(&line, fmt, args); + va_end(args); + + if (result != -1) { + result = fprintf(stderr, "%s %s", date, line); + free(line); + } + + return result; +} + +/* Use POSIX_FADV_NORMAL when available */ + +#ifdef POSIX_FADV_NORMAL +# define USE_POSIX_FADVISE +#else +# define POSIX_FADV_NORMAL +# define POSIX_FADV_SEQUENTIAL +# define POSIX_FADV_DONTNEED +# define posix_fadvise(a,b,c,d) do {} while(0) +#endif + +/*********************************************************************** +Computes bit shift for a given value. If the argument is not a power +of 2, returns 0.*/ +static inline ulong +get_bit_shift(ulong value) +{ + ulong shift; + + if (value == 0) + return 0; + + for (shift = 0; !(value & 1UL); shift++) { + value >>= 1; + } + return (value >> 1) ? 0 : shift; +} + +/**************************************************************************** +Read 'len' bytes from 'fd'. It is identical to my_read(..., MYF(MY_FULL_IO)), +i.e. tries to combine partial reads into a single block of size 'len', except +that it bails out on EOF or error, and returns the number of successfully read +bytes instead. */ +static inline size_t +xb_read_full(File fd, uchar *buf, size_t len) +{ + size_t tlen = 0; + size_t tbytes; + + while (tlen < len) { + tbytes = my_read(fd, buf, len - tlen, MYF(MY_WME)); + if (tbytes == 0 || tbytes == MY_FILE_ERROR) { + break; + } + + buf += tbytes; + tlen += tbytes; + } + + return tlen; +} + +#endif diff --git a/extra/mariabackup/compact.cc b/extra/mariabackup/compact.cc new file mode 100644 index 00000000000..5d08a6e02b2 --- /dev/null +++ b/extra/mariabackup/compact.cc @@ -0,0 +1,1059 @@ +/****************************************************** +XtraBackup: hot backup tool for InnoDB +(c) 2009-2014 Percona LLC and/or its affiliates. +Originally Created 3/3/2009 Yasufumi Kinoshita +Written by Alexey Kopytov, Aleksandr Kuzminsky, Stewart Smith, Vadim Tkachenko, +Yasufumi Kinoshita, Ignacio Nin and Baron Schwartz. + +This program is free software; you can redistribute it and/or modify +it under the terms of the GNU General Public License as published by +the Free Software Foundation; version 2 of the License. + +This program is distributed in the hope that it will be useful, +but WITHOUT ANY WARRANTY; without even the implied warranty of +MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the +GNU General Public License for more details. + +You should have received a copy of the GNU General Public License +along with this program; if not, write to the Free Software +Foundation, Inc., 51 Franklin Street, Fifth Floor, Boston, MA 02110-1301, USA + +*******************************************************/ + +/* Compact backups implementation */ + +#include +#include + +#include +#include +#include +#include +#include +#include +#include +#include +#include "common.h" +#include "write_filt.h" +#include "fil_cur.h" +#include "xtrabackup.h" +#include "ds_buffer.h" +#include "xb0xb.h" + +/* Number of the first primary key page in an .ibd file */ +#define XB_FIRST_CLUSTERED_INDEX_PAGE_NO 3 + +/* Suffix for page map files */ +#define XB_PAGE_MAP_SUFFIX ".pmap" +#define XB_TMPFILE_SUFFIX ".tmp" + +/* Page range */ +struct page_range_t { + ulint from; /*!< range start */ + ulint to; /*!< range end */ +}; + +/* Cursor in a page map file */ +struct page_map_cursor_t { + File fd; /*!< file descriptor */ + IO_CACHE cache; /*!< IO_CACHE associated with fd */ +}; + +/* Table descriptor for the index rebuild operation */ +struct index_rebuild_table_t { + char* name; /* table name */ + ulint space_id; /* space ID */ + UT_LIST_NODE_T(index_rebuild_table_t) list; /* list node */ +}; + +/* Thread descriptor for the index rebuild operation */ +struct index_rebuild_thread_t { + ulint num; /* thread number */ + pthread_t id; /* thread ID */ +}; + +/* Empty page use to replace skipped pages in the data files */ +static byte empty_page[UNIV_PAGE_SIZE_MAX]; +static const char compacted_page_magic[] = "COMPACTP"; +static const size_t compacted_page_magic_size = + sizeof(compacted_page_magic) - 1; +static const ulint compacted_page_magic_offset = FIL_PAGE_DATA; + +/* Mutex protecting table_list */ +static pthread_mutex_t table_list_mutex; +/* List of tablespaces to process by the index rebuild operation */ +static UT_LIST_BASE_NODE_T(index_rebuild_table_t) table_list; + + +/************************************************************************ +Compact page filter. */ +static my_bool wf_compact_init(xb_write_filt_ctxt_t *ctxt, char *dst_name, + xb_fil_cur_t *cursor); +static my_bool wf_compact_process(xb_write_filt_ctxt_t *ctxt, + ds_file_t *dstfile); +static my_bool wf_compact_finalize(xb_write_filt_ctxt_t *ctxt, + ds_file_t *dstfile); +xb_write_filt_t wf_compact = { + &wf_compact_init, + &wf_compact_process, + &wf_compact_finalize, + NULL +}; + +/************************************************************************ +Initialize the compact page filter. + +@return TRUE on success, FALSE on error. */ +static my_bool +wf_compact_init(xb_write_filt_ctxt_t *ctxt, + char *dst_name __attribute__((unused)), xb_fil_cur_t *cursor) +{ + xb_wf_compact_ctxt_t *cp = &(ctxt->u.wf_compact_ctxt); + char page_map_name[FN_REFLEN]; + MY_STAT mystat; + + ctxt->cursor = cursor; + cp->clustered_index_found = FALSE; + cp->inside_skipped_range = FALSE; + cp->free_limit = 0; + + /* Don't compact the system table space */ + cp->skip = cursor->is_system; + if (cp->skip) { + return(TRUE); + } + + snprintf(page_map_name, sizeof(page_map_name), "%s%s", dst_name, + XB_PAGE_MAP_SUFFIX); + + cp->ds_buffer = ds_create(xtrabackup_target_dir, DS_TYPE_BUFFER); + if (cp->ds_buffer == NULL) { + return(FALSE); + } + + ds_set_pipe(cp->ds_buffer, ds_meta); + + memset(&mystat, 0, sizeof(mystat)); + mystat.st_mtime = my_time(0); + cp->buffer = ds_open(cp->ds_buffer, page_map_name, &mystat); + if (cp->buffer == NULL) { + msg("xtrabackup: Error: cannot open output stream for %s\n", + page_map_name); + return(FALSE); + } + + return(TRUE); +} + +/************************************************************************ +Check if the specified page should be skipped. We currently skip all +non-clustered index pages for compact backups. + +@return TRUE if the page should be skipped. */ +static my_bool +check_if_skip_page(xb_wf_compact_ctxt_t *cp, xb_fil_cur_t *cursor, ulint offset) +{ + byte *page; + ulint page_no; + ulint page_type; + index_id_t index_id; + + + xb_ad(cursor->is_system == FALSE); + + page = cursor->buf + cursor->page_size * offset; + page_no = cursor->buf_page_no + offset; + page_type = fil_page_get_type(page); + + if (UNIV_UNLIKELY(page_no == 0)) { + + cp->free_limit = mach_read_from_4(page + FSP_HEADER_OFFSET + + FSP_FREE_LIMIT); + } else if (UNIV_UNLIKELY(page_no == XB_FIRST_CLUSTERED_INDEX_PAGE_NO)) { + + xb_ad(cp->clustered_index_found == FALSE); + + if (page_type != FIL_PAGE_INDEX) { + + /* Uninitialized clustered index root page, there's + nothing we can do to compact the space.*/ + + msg("[%02u] Uninitialized page type value (%lu) in the " + "clustered index root page of tablespace %s. " + "Will not be compacted.\n", + cursor->thread_n, + page_type, cursor->rel_path); + + cp->skip = TRUE; + + return(FALSE); + } + + cp->clustered_index = + mach_read_from_8(page + PAGE_HEADER + PAGE_INDEX_ID); + cp->clustered_index_found = TRUE; + } else if (UNIV_UNLIKELY(page_no >= cp->free_limit)) { + + /* Skip unused pages above free limit, if that value is set in + the FSP header.*/ + + return(cp->free_limit > 0); + } else if (cp->clustered_index_found && page_type == FIL_PAGE_INDEX) { + + index_id = mach_read_from_8(page + PAGE_HEADER + PAGE_INDEX_ID); + if (index_id != cp->clustered_index) { + + ulint fseg_hdr_space = + mach_read_from_4(page + PAGE_HEADER + + PAGE_BTR_SEG_TOP); + ulint fseg_hdr_page_no = + mach_read_from_4(page + PAGE_HEADER + + PAGE_BTR_SEG_TOP + 4); + ulint fseg_hdr_offset = + mach_read_from_2(page + PAGE_HEADER + + PAGE_BTR_SEG_TOP + 8); + + /* Don't skip root index pages, i.e. the ones where the + above fields are defined. We need root index pages to be + able to correctly drop the indexes later, as they + contain fseg inode pointers. */ + + return(fseg_hdr_space == 0 && + fseg_hdr_page_no == 0 && + fseg_hdr_offset == 0); + } + } + + return(FALSE); +} + +/************************************************************************ +Run the next batch of pages through the compact page filter. + +@return TRUE on success, FALSE on error. */ +static my_bool +wf_compact_process(xb_write_filt_ctxt_t *ctxt, ds_file_t *dstfile) +{ + xb_fil_cur_t *cursor = ctxt->cursor; + ulint page_size = cursor->page_size; + byte *page; + byte *buf_end; + byte *write_from; + xb_wf_compact_ctxt_t *cp = &(ctxt->u.wf_compact_ctxt); + ulint i; + ulint page_no; + byte tmp[4]; + + if (cp->skip) { + return(!ds_write(dstfile, cursor->buf, cursor->buf_read)); + } + + write_from = NULL; + buf_end = cursor->buf + cursor->buf_read; + for (i = 0, page = cursor->buf; page < buf_end; + i++, page += page_size) { + + page_no = cursor->buf_page_no + i; + + if (!check_if_skip_page(cp, cursor, i)) { + + if (write_from == NULL) { + write_from = page; + } + + if (cp->inside_skipped_range) { + cp->inside_skipped_range = FALSE; + + /* Write the last range endpoint to the + skipped pages map */ + + xb_ad(page_no > 0); + mach_write_to_4(tmp, page_no - 1); + if (ds_write(cp->buffer, tmp, sizeof(tmp))) { + return(FALSE); + } + } + continue; + } + + if (write_from != NULL) { + + /* The first skipped page in this block, write the + non-skipped ones to the data file */ + + if (ds_write(dstfile, write_from, page - write_from)) { + return(FALSE); + } + + write_from = NULL; + } + + if (!cp->inside_skipped_range) { + + /* The first skipped page in range, write the first + range endpoint to the skipped pages map */ + + cp->inside_skipped_range = TRUE; + + mach_write_to_4(tmp, page_no); + if (ds_write(cp->buffer, tmp, sizeof(tmp))) { + return(FALSE); + } + } + } + + /* Write the remaining pages in the buffer, if any */ + if (write_from != NULL && + ds_write(dstfile, write_from, buf_end - write_from)) { + return(FALSE); + } + + return(TRUE); +} + +/************************************************************************ +Close the compact filter's page map stream. + +@return TRUE on success, FALSE on error. */ +static my_bool +wf_compact_finalize(xb_write_filt_ctxt_t *ctxt, + ds_file_t *dstfile __attribute__((unused))) +{ + xb_fil_cur_t *cursor = ctxt->cursor; + xb_wf_compact_ctxt_t *cp = &(ctxt->u.wf_compact_ctxt); + my_bool rc = TRUE; + + /* Write the last endpoint of the current range, if the last pages of + the space have been skipped. */ + if (cp->inside_skipped_range) { + byte tmp[4]; + + mach_write_to_4(tmp, cursor->space_size - 1); + if (ds_write(cp->buffer, tmp, sizeof(tmp))) { + return(FALSE); + } + + cp->inside_skipped_range = FALSE; + } + + if (cp->buffer) { + if (ds_close(cp->buffer)) { + rc = FALSE; + } + } + if (cp->ds_buffer) { + ds_destroy(cp->ds_buffer); + } + + return(rc); +} + +/************************************************************************ +Open a page map file and return a cursor. + +@return page map cursor, or NULL if the file doesn't exist. */ +static page_map_cursor_t * +page_map_file_open(const char *path) +{ + MY_STAT statinfo; + page_map_cursor_t *pmap_cur; + int rc; + + if (my_stat(path, &statinfo, MYF(0)) == NULL) { + + return(NULL); + } + + /* The maximum possible page map file corresponds to a 64 TB tablespace + and the worst case when every other page was skipped. That is, 2^32/2 + page ranges = 16 GB. */ + xb_a(statinfo.st_size < (off_t) 16 * 1024 * 1024 * 1024); + + /* Must be a series of 8-byte tuples */ + xb_a(statinfo.st_size % 8 == 0); + + pmap_cur = (page_map_cursor_t *) my_malloc(sizeof(page_map_cursor_t), + MYF(MY_FAE)); + + pmap_cur->fd = my_open(path, O_RDONLY, MYF(MY_WME)); + xb_a(pmap_cur->fd != 0); + + rc = init_io_cache(&pmap_cur->cache, pmap_cur->fd, 0, READ_CACHE, + 0, 0, MYF(MY_WME)); + xb_a(rc == 0); + + return(pmap_cur); +} + +/************************************************************************ +Read the next range from a page map file and update the cursor. + +@return TRUE on success, FALSE on end-of-file. */ +static ibool +page_map_file_next(page_map_cursor_t *pmap_cur, page_range_t *range) +{ + byte buf[8]; + + xb_ad(pmap_cur != NULL); + + if (my_b_read(&pmap_cur->cache, buf, sizeof(buf))) { + return(FALSE); + } + + range->from = mach_read_from_4(buf); + range->to = mach_read_from_4(buf + 4); + + return(TRUE); +} + +/************************************************************************ +Close the page map cursor.*/ +static void +page_map_file_close(page_map_cursor_t *pmap_cur) +{ + int rc; + + xb_ad(pmap_cur != NULL); + + rc = end_io_cache(&pmap_cur->cache); + xb_a(rc == 0); + + posix_fadvise(pmap_cur->fd, 0, 0, POSIX_FADV_DONTNEED); + + rc = my_close(pmap_cur->fd, MY_WME); + xb_a(rc == 0); + + my_free(pmap_cur); +} + +/**************************************************************************** +Expand a single data file according to the skipped pages maps created by +--compact. + +@return TRUE on success, FALSE on failure. */ +static my_bool +xb_expand_file(fil_node_t *node) +{ + char pmapfile_path[FN_REFLEN]; + char tmpfile_path[FN_REFLEN]; + xb_fil_cur_t cursor; + xb_fil_cur_result_t res; + ds_ctxt_t *ds_local; + ds_ctxt_t *ds_buffer; + ds_file_t *tmpfile; + my_bool success = FALSE; + ulint i; + byte *page; + ulint page_expected_no; + page_map_cursor_t *pmap_cur; + ibool have_next_range; + page_range_t pmap_range; + + xb_ad(trx_sys_sys_space(node->space->id) == FALSE); + + snprintf(pmapfile_path, sizeof(pmapfile_path), "%s%s", + node->name, XB_PAGE_MAP_SUFFIX); + + /* Skip files that don't have a corresponding page map file */ + + if (!(pmap_cur = page_map_file_open(pmapfile_path))) { + + msg("Not expanding %s\n", node->name); + + return(FALSE); + } + + msg("Expanding %s\n", node->name); + + ds_local = ds_create(".", DS_TYPE_LOCAL); + ds_buffer = ds_create(".", DS_TYPE_BUFFER); + + xb_a(ds_local != NULL && ds_buffer != NULL); + + ds_buffer_set_size(ds_buffer, FSP_EXTENT_SIZE * UNIV_PAGE_SIZE_MAX); + + ds_set_pipe(ds_buffer, ds_local); + + res = xb_fil_cur_open(&cursor, &rf_pass_through, node, 1); + xb_a(res == XB_FIL_CUR_SUCCESS); + + snprintf(tmpfile_path, sizeof(tmpfile_path), "%s%s", + node->name, XB_TMPFILE_SUFFIX); + + tmpfile = ds_open(ds_buffer, tmpfile_path, &cursor.statinfo); + if (tmpfile == NULL) { + + msg("Could not open temporary file '%s'\n", tmpfile_path); + goto error; + } + + have_next_range = page_map_file_next(pmap_cur, &pmap_range); + + page_expected_no = 0; + + /* Initialize and mark the empty page which is used to replace + skipped pages. */ + memset(empty_page, 0, cursor.page_size); + memcpy(empty_page + compacted_page_magic_offset, + compacted_page_magic, compacted_page_magic_size); + mach_write_to_4(empty_page + FIL_PAGE_SPACE_OR_CHKSUM, + BUF_NO_CHECKSUM_MAGIC); + mach_write_to_4(empty_page + cursor.page_size - + FIL_PAGE_END_LSN_OLD_CHKSUM, + BUF_NO_CHECKSUM_MAGIC); + + + /* Main copy loop */ + + while ((res = xb_fil_cur_read(&cursor)) == XB_FIL_CUR_SUCCESS) { + + for (i = 0, page = cursor.buf; i < cursor.buf_npages; + i++, page += cursor.page_size) { + + ulint page_read_no; + + page_read_no = mach_read_from_4(page + FIL_PAGE_OFFSET); + xb_a(!page_read_no || page_expected_no <= page_read_no); + + if (have_next_range && + page_expected_no == pmap_range.from) { + + xb_a(pmap_range.from <= pmap_range.to); + + /* Write empty pages instead of skipped ones, if + necessary. */ + + while (page_expected_no <= pmap_range.to) { + + if (ds_write(tmpfile, empty_page, + cursor.page_size)) { + + goto write_error; + } + + page_expected_no++; + } + + have_next_range = + page_map_file_next(pmap_cur, + &pmap_range); + } + + /* Write the current page */ + + if (ds_write(tmpfile, page, cursor.page_size)) { + + goto write_error; + } + + page_expected_no++; + } + } + + if (res != XB_FIL_CUR_EOF) { + + goto error; + } + + /* Write empty pages instead of trailing skipped ones, if any */ + + if (have_next_range) { + + xb_a(page_expected_no == pmap_range.from); + xb_a(pmap_range.from <= pmap_range.to); + + while (page_expected_no <= pmap_range.to) { + + if (ds_write(tmpfile, empty_page, + cursor.page_size)) { + + goto write_error; + } + + page_expected_no++; + } + + xb_a(!page_map_file_next(pmap_cur, &pmap_range)); + } + + /* Replace the original .ibd file with the expanded file */ + if (my_rename(tmpfile_path, node->name, MYF(MY_WME))) { + + msg("Failed to rename '%s' to '%s'\n", + tmpfile_path, node->name); + goto error; + } + + my_delete(pmapfile_path, MYF(MY_WME)); + + if (!ds_close(tmpfile)) { + success = TRUE; + } + tmpfile = NULL; + + goto end; + +write_error: + msg("Write to '%s' failed\n", tmpfile_path); + +error: + if (tmpfile != NULL) { + + ds_close(tmpfile); + my_delete(tmpfile_path, MYF(MY_WME)); + } + +end: + ds_destroy(ds_buffer); + ds_destroy(ds_local); + + xb_fil_cur_close(&cursor); + + page_map_file_close(pmap_cur); + + return(success); +} + +/****************************************************************************** +Expand the data files according to the skipped pages maps created by --compact. +@return TRUE on success, FALSE on failure. */ +my_bool +xb_expand_datafiles(void) +/*=====================*/ +{ + ulint nfiles; + datafiles_iter_t *it = NULL; + fil_node_t *node; + fil_space_t *space; + + msg("Starting to expand compacted .ibd files.\n"); + + /* Initialize the tablespace cache */ + if (xb_data_files_init() != DB_SUCCESS) { + return(FALSE); + } + + nfiles = UT_LIST_GET_LEN(fil_system->space_list); + xb_a(nfiles > 0); + + it = datafiles_iter_new(fil_system); + if (it == NULL) { + msg("xtrabackup: error: datafiles_iter_new() failed.\n"); + goto error; + } + + while ((node = datafiles_iter_next(it)) != NULL) { + + space = node->space; + + /* System tablespace cannot be compacted */ + if (!fil_is_user_tablespace_id(space->id)) { + + continue; + } + + if (!xb_expand_file(node)) { + + goto error; + } + } + + datafiles_iter_free(it); + xb_data_files_close(); + + return(TRUE); + +error: + if (it != NULL) { + datafiles_iter_free(it); + } + + xb_data_files_close(); + + return(FALSE); +} + +/****************************************************************************** +Callback used in buf_page_io_complete() to detect compacted pages. +@return TRUE if the page is marked as compacted, FALSE otherwise. */ +ibool +buf_page_is_compacted( +/*==================*/ + const byte* page) /*!< in: a database page */ +{ + return !memcmp(page + compacted_page_magic_offset, + compacted_page_magic, compacted_page_magic_size); +} + +/***************************************************************************** +Builds an index definition corresponding to an index object. It is roughly +similar to innobase_create_index_def() / innobase_create_index_field_def() and +the opposite to dict_mem_index_create() / dict_mem_index_add_field(). */ +static +void +xb_build_index_def( +/*=======================*/ + mem_heap_t* heap, /*!< in: heap */ + const dict_index_t* index, /*!< in: index */ + index_def_t* index_def) /*!< out: index definition */ +{ + index_field_t* fields; + ulint n_fields; + ulint i; + + ut_a(index->n_fields); + ut_ad(index->magic_n == DICT_INDEX_MAGIC_N); + + /* Use n_user_defined_cols instead of n_fields, as the index will + contain a part of the primary key after n_user_defined_cols, and those + columns will be created automatically in + dict_index_build_internal_clust(). */ + n_fields = index->n_user_defined_cols; + + memset(index_def, 0, sizeof(*index_def)); + + index_def->name = mem_heap_strdup(heap, index->name); + index_def->ind_type = index->type; + + fields = static_cast + (mem_heap_alloc(heap, n_fields * sizeof(*fields))); + + for (i = 0; i < n_fields; i++) { + dict_field_t* field; + + field = dict_index_get_nth_field(index, i); + fields[i].col_no = dict_col_get_no(field->col); + fields[i].prefix_len = field->prefix_len; + } + + index_def->fields = fields; + index_def->n_fields = n_fields; +} + +/* A dummy autoc_inc sequence for row_merge_build_indexes(). */ +static ib_sequence_t null_seq(NULL, 0, 0); +/* A dummy table share and table for row_merge_build_indexes() error reporting. +Assumes that no errors are going to be reported. */ +static struct TABLE_SHARE dummy_table_share; +static struct TABLE dummy_table; + +/********************************************************************//** +Rebuild secondary indexes for a given table. */ +static +void +xb_rebuild_indexes_for_table( +/*=========================*/ + dict_table_t* table, /*!< in: table */ + trx_t* trx, /*!< in: transaction handle */ + ulint thread_n) /*!< in: thread number */ +{ + dict_index_t* index; + dict_index_t** indexes; + ulint n_indexes; + index_def_t* index_defs; + ulint i; + mem_heap_t* heap; + ulint error; + ulint* add_key_nums; + + ut_ad(!mutex_own(&(dict_sys->mutex))); + ut_ad(table); + + ut_a(UT_LIST_GET_LEN(table->indexes) > 0); + + n_indexes = UT_LIST_GET_LEN(table->indexes) - 1; + if (!n_indexes) { + /* Only the primary key, nothing to do. */ + return; + } + + heap = mem_heap_create(1024); + + indexes = (dict_index_t**) mem_heap_alloc(heap, + n_indexes * sizeof(*indexes)); + index_defs = (index_def_t*) mem_heap_alloc(heap, n_indexes * + sizeof(*index_defs)); + add_key_nums = static_cast + (mem_heap_alloc(heap, n_indexes * sizeof(*add_key_nums))); + + /* Skip the primary key. */ + index = dict_table_get_first_index(table); + ut_a(dict_index_is_clust(index)); + + row_mysql_lock_data_dictionary(trx); + + for (i = 0; (index = dict_table_get_next_index(index)); i++) { + + msg("[%02lu] Found index %s\n", thread_n, index->name); + + /* Pretend that it's the current trx that created this index. + Required to avoid 5.6+ debug assertions. */ + index->trx_id = trx->id; + + xb_build_index_def(heap, index, &index_defs[i]); + + /* In 5.6+, row_merge_drop_indexes() drops all the indexes on + the table that have the temp index prefix. It does not accept + an array of indexes to drop as in 5.5-. */ + row_merge_rename_index_to_drop(trx, table->id, index->id); + } + + ut_ad(i == n_indexes); + + row_merge_drop_indexes(trx, table, TRUE); + + index = dict_table_get_first_index(table); + ut_a(dict_index_is_clust(index)); + index = dict_table_get_next_index(index); + while (index) { + + /* In 5.6+, row_merge_drop_indexes() does not remove the + indexes from the dictionary cache nor from any foreign key + list. This may cause invalid dereferences as we try to access + the dropped indexes from other tables as FKs. */ + + dict_index_t* next_index = dict_table_get_next_index(index); + index->to_be_dropped = 1; + + /* Patch up any FK referencing this index with NULL */ + dict_foreign_replace_index(table, NULL, index); + + dict_index_remove_from_cache(table, index); + + index = next_index; + } + + msg("[%02lu] Rebuilding %lu index(es).\n", thread_n, n_indexes); + + error = row_merge_lock_table(trx, table, LOCK_X); + xb_a(error == DB_SUCCESS); + + for (i = 0; i < n_indexes; i++) { + indexes[i] = row_merge_create_index(trx, table, + &index_defs[i]); + add_key_nums[i] = index_defs[i].key_number; + } + + /* Commit trx to release latches on system tables */ + trx_commit_for_mysql(trx); + trx_start_for_ddl(trx, TRX_DICT_OP_INDEX); + + row_mysql_unlock_data_dictionary(trx); + + /* Reacquire table lock for row_merge_build_indexes() */ + error = row_merge_lock_table(trx, table, LOCK_X); + xb_a(error == DB_SUCCESS); + + error = row_merge_build_indexes(trx, table, table, FALSE, indexes, + add_key_nums, n_indexes, &dummy_table, + NULL, NULL, ULINT_UNDEFINED, null_seq); + ut_a(error == DB_SUCCESS); + + mem_heap_free(heap); + + trx_commit_for_mysql(trx); + + trx_start_for_ddl(trx, TRX_DICT_OP_INDEX); +} + +/************************************************************************** +Worker thread function for index rebuild. */ +static +void * +xb_rebuild_indexes_thread_func( +/*===========================*/ + void* arg) /* thread context */ +{ + dict_table_t* table; + index_rebuild_table_t* rebuild_table; + index_rebuild_thread_t* thread; + trx_t* trx; + + thread = (index_rebuild_thread_t *) arg; + + trx = trx_allocate_for_mysql(); + + /* Suppress foreign key checks, as we are going to drop and recreate all + secondary keys. */ + trx->check_foreigns = FALSE; + trx_start_for_ddl(trx, TRX_DICT_OP_INDEX); + + /* Loop until there are no more tables in tables list */ + for (;;) { + pthread_mutex_lock(&table_list_mutex); + + rebuild_table = UT_LIST_GET_FIRST(table_list); + + if (rebuild_table == NULL) { + + pthread_mutex_unlock(&table_list_mutex); + break; + } + + UT_LIST_REMOVE(list, table_list, rebuild_table); + + pthread_mutex_unlock(&table_list_mutex); + + ut_ad(rebuild_table->name); + ut_ad(fil_is_user_tablespace_id(rebuild_table->space_id)); + + row_mysql_lock_data_dictionary(trx); + + table = dict_table_get_low(rebuild_table->name); + + ut_d(table->n_ref_count++); + + row_mysql_unlock_data_dictionary(trx); + + ut_a(table != NULL); + ut_a(table->space == rebuild_table->space_id); + + /* Discard change buffer entries for this space */ + ibuf_delete_for_discarded_space(rebuild_table->space_id); + + msg("[%02lu] Checking if there are indexes to rebuild in table " + "%s (space id: %lu)\n", + thread->num, + rebuild_table->name, rebuild_table->space_id); + + xb_rebuild_indexes_for_table(table, trx, thread->num); + + ut_d(table->n_ref_count--); + + mem_free(rebuild_table->name); + mem_free(rebuild_table); + } + + trx_commit_for_mysql(trx); + + trx_free_for_mysql(trx); + + return(NULL); +} + +/****************************************************************************** +Rebuild all secondary indexes in all tables in separate spaces. Called from +innobase_start_or_create_for_mysql(). */ +void +xb_compact_rebuild_indexes(void) +/*=============================*/ +{ + dict_table_t* sys_tables; + dict_index_t* sys_index; + btr_pcur_t pcur; + const rec_t* rec; + mtr_t mtr; + const byte* field; + ulint len; + ulint space_id; + trx_t* trx; + index_rebuild_table_t* rebuild_table; + index_rebuild_thread_t* threads; + ulint i; + + /* Set up the dummy table for the index rebuild error reporting */ + dummy_table_share.fields = 0; + dummy_table.s = &dummy_table_share; + + /* Iterate all tables that are not in the system tablespace and add them + to the list of tables to be rebuilt later. */ + + trx = trx_allocate_for_mysql(); + trx_start_for_ddl(trx, TRX_DICT_OP_INDEX); + + row_mysql_lock_data_dictionary(trx); + + /* Enlarge the fatal lock wait timeout during index rebuild + operation. */ + os_increment_counter_by_amount(server_mutex, + srv_fatal_semaphore_wait_threshold, + 7200); + + mtr_start(&mtr); + + sys_tables = dict_table_get_low("SYS_TABLES"); + sys_index = UT_LIST_GET_FIRST(sys_tables->indexes); + ut_a(!dict_table_is_comp(sys_tables)); + + pthread_mutex_init(&table_list_mutex, NULL); + UT_LIST_INIT(table_list); + + btr_pcur_open_at_index_side(TRUE, sys_index, BTR_SEARCH_LEAF, &pcur, + TRUE, 0, &mtr); + for (;;) { + btr_pcur_move_to_next_user_rec(&pcur, &mtr); + + rec = btr_pcur_get_rec(&pcur); + + if (!btr_pcur_is_on_user_rec(&pcur)) { + /* end of index */ + + break; + } + + if (rec_get_deleted_flag(rec, 0)) { + continue; + } + + field = rec_get_nth_field_old(rec, 9, &len); + ut_a(len == 4); + + space_id = mach_read_from_4(field); + + /* Don't touch tables in the system tablespace */ + if (!fil_is_user_tablespace_id(space_id)) { + + continue; + } + + field = rec_get_nth_field_old(rec, 0, &len); + + rebuild_table = static_cast + (mem_alloc(sizeof(*rebuild_table))); + rebuild_table->name = mem_strdupl((char*) field, len); + rebuild_table->space_id = space_id; + + UT_LIST_ADD_LAST(list, table_list, rebuild_table); + } + + btr_pcur_close(&pcur); + mtr_commit(&mtr); + + row_mysql_unlock_data_dictionary(trx); + + trx_commit_for_mysql(trx); + + trx_free_for_mysql(trx); + + /* Start worker threads for the index rebuild operation */ + ut_ad(xtrabackup_rebuild_threads > 0); + + if (xtrabackup_rebuild_threads > 1) { + msg("Starting %lu threads to rebuild indexes.\n", + xtrabackup_rebuild_threads); + } + + threads = (index_rebuild_thread_t *) + mem_alloc(sizeof(*threads) * + xtrabackup_rebuild_threads); + + for (i = 0; i < xtrabackup_rebuild_threads; i++) { + + threads[i].num = i+1; + if (pthread_create(&threads[i].id, NULL, + xb_rebuild_indexes_thread_func, + &threads[i])) { + + msg("error: pthread_create() failed: errno = %d\n", + errno); + ut_a(0); + } + } + + /* Wait for worker threads to finish */ + for (i = 0; i < xtrabackup_rebuild_threads; i++) { + pthread_join(threads[i].id, NULL); + } + + mem_free(threads); +} diff --git a/extra/mariabackup/compact.h b/extra/mariabackup/compact.h new file mode 100644 index 00000000000..d0d9840f66d --- /dev/null +++ b/extra/mariabackup/compact.h @@ -0,0 +1,44 @@ +/****************************************************** +XtraBackup: hot backup tool for InnoDB +(c) 2009-2013 Percona LLC and/or its affiliates. +Originally Created 3/3/2009 Yasufumi Kinoshita +Written by Alexey Kopytov, Aleksandr Kuzminsky, Stewart Smith, Vadim Tkachenko, +Yasufumi Kinoshita, Ignacio Nin and Baron Schwartz. + +This program is free software; you can redistribute it and/or modify +it under the terms of the GNU General Public License as published by +the Free Software Foundation; version 2 of the License. + +This program is distributed in the hope that it will be useful, +but WITHOUT ANY WARRANTY; without even the implied warranty of +MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the +GNU General Public License for more details. + +You should have received a copy of the GNU General Public License +along with this program; if not, write to the Free Software +Foundation, Inc., 51 Franklin Street, Fifth Floor, Boston, MA 02110-1301, USA + +*******************************************************/ + +#ifndef XB_COMPACT_H +#define XB_COMPACT_H + +#include "write_filt.h" + +/* Compact page filter context */ +typedef struct { + my_bool skip; + ds_ctxt_t *ds_buffer; + ds_file_t *buffer; + index_id_t clustered_index; + my_bool clustered_index_found; + my_bool inside_skipped_range; + ulint free_limit; +} xb_wf_compact_ctxt_t; + +/****************************************************************************** +Expand the data files according to the skipped pages maps created by --compact. +@return TRUE on success, FALSE on failure. */ +my_bool xb_expand_datafiles(void); + +#endif diff --git a/extra/mariabackup/datasink.c b/extra/mariabackup/datasink.c new file mode 100644 index 00000000000..2f4233ddc98 --- /dev/null +++ b/extra/mariabackup/datasink.c @@ -0,0 +1,130 @@ +/****************************************************** +Copyright (c) 2011-2013 Percona LLC and/or its affiliates. + +Data sink interface. + +This program is free software; you can redistribute it and/or modify +it under the terms of the GNU General Public License as published by +the Free Software Foundation; version 2 of the License. + +This program is distributed in the hope that it will be useful, +but WITHOUT ANY WARRANTY; without even the implied warranty of +MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the +GNU General Public License for more details. + +You should have received a copy of the GNU General Public License +along with this program; if not, write to the Free Software +Foundation, Inc., 51 Franklin Street, Fifth Floor, Boston, MA 02110-1301, USA + +*******************************************************/ + +#include +#include "common.h" +#include "datasink.h" +#include "ds_compress.h" +#include "ds_archive.h" +#include "ds_xbstream.h" +#include "ds_local.h" +#include "ds_stdout.h" +#include "ds_tmpfile.h" +#include "ds_encrypt.h" +#include "ds_buffer.h" + +/************************************************************************ +Create a datasink of the specified type */ +ds_ctxt_t * +ds_create(const char *root, ds_type_t type) +{ + datasink_t *ds; + ds_ctxt_t *ctxt; + + switch (type) { + case DS_TYPE_STDOUT: + ds = &datasink_stdout; + break; + case DS_TYPE_LOCAL: + ds = &datasink_local; + break; + case DS_TYPE_ARCHIVE: + ds = &datasink_archive; + break; + case DS_TYPE_XBSTREAM: + ds = &datasink_xbstream; + break; + case DS_TYPE_COMPRESS: + ds = &datasink_compress; + break; + case DS_TYPE_ENCRYPT: + ds = &datasink_encrypt; + break; + case DS_TYPE_TMPFILE: + ds = &datasink_tmpfile; + break; + case DS_TYPE_BUFFER: + ds = &datasink_buffer; + break; + default: + msg("Unknown datasink type: %d\n", type); + xb_ad(0); + return NULL; + } + + ctxt = ds->init(root); + if (ctxt != NULL) { + ctxt->datasink = ds; + } else { + msg("Error: failed to initialize datasink.\n"); + exit(EXIT_FAILURE); + } + + return ctxt; +} + +/************************************************************************ +Open a datasink file */ +ds_file_t * +ds_open(ds_ctxt_t *ctxt, const char *path, MY_STAT *stat) +{ + ds_file_t *file; + + file = ctxt->datasink->open(ctxt, path, stat); + if (file != NULL) { + file->datasink = ctxt->datasink; + } + + return file; +} + +/************************************************************************ +Write to a datasink file. +@return 0 on success, 1 on error. */ +int +ds_write(ds_file_t *file, const void *buf, size_t len) +{ + return file->datasink->write(file, buf, len); +} + +/************************************************************************ +Close a datasink file. +@return 0 on success, 1, on error. */ +int +ds_close(ds_file_t *file) +{ + return file->datasink->close(file); +} + +/************************************************************************ +Destroy a datasink handle */ +void +ds_destroy(ds_ctxt_t *ctxt) +{ + ctxt->datasink->deinit(ctxt); +} + +/************************************************************************ +Set the destination pipe for a datasink (only makes sense for compress and +tmpfile). */ +void ds_set_pipe(ds_ctxt_t *ctxt, ds_ctxt_t *pipe_ctxt) +{ + ctxt->pipe_ctxt = pipe_ctxt; +} diff --git a/extra/mariabackup/datasink.h b/extra/mariabackup/datasink.h new file mode 100644 index 00000000000..36a3d564a57 --- /dev/null +++ b/extra/mariabackup/datasink.h @@ -0,0 +1,98 @@ +/****************************************************** +Copyright (c) 2011-2013 Percona LLC and/or its affiliates. + +Data sink interface. + +This program is free software; you can redistribute it and/or modify +it under the terms of the GNU General Public License as published by +the Free Software Foundation; version 2 of the License. + +This program is distributed in the hope that it will be useful, +but WITHOUT ANY WARRANTY; without even the implied warranty of +MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the +GNU General Public License for more details. + +You should have received a copy of the GNU General Public License +along with this program; if not, write to the Free Software +Foundation, Inc., 51 Franklin Street, Fifth Floor, Boston, MA 02110-1301, USA + +*******************************************************/ + +#ifndef XB_DATASINK_H +#define XB_DATASINK_H + +#include +#include + +#ifdef __cplusplus +extern "C" { +#endif + +struct datasink_struct; +typedef struct datasink_struct datasink_t; + +typedef struct ds_ctxt { + datasink_t *datasink; + char *root; + void *ptr; + struct ds_ctxt *pipe_ctxt; +} ds_ctxt_t; + +typedef struct { + void *ptr; + char *path; + datasink_t *datasink; +} ds_file_t; + +struct datasink_struct { + ds_ctxt_t *(*init)(const char *root); + ds_file_t *(*open)(ds_ctxt_t *ctxt, const char *path, MY_STAT *stat); + int (*write)(ds_file_t *file, const void *buf, size_t len); + int (*close)(ds_file_t *file); + void (*deinit)(ds_ctxt_t *ctxt); +}; + +/* Supported datasink types */ +typedef enum { + DS_TYPE_STDOUT, + DS_TYPE_LOCAL, + DS_TYPE_ARCHIVE, + DS_TYPE_XBSTREAM, + DS_TYPE_COMPRESS, + DS_TYPE_ENCRYPT, + DS_TYPE_TMPFILE, + DS_TYPE_BUFFER +} ds_type_t; + +/************************************************************************ +Create a datasink of the specified type */ +ds_ctxt_t *ds_create(const char *root, ds_type_t type); + +/************************************************************************ +Open a datasink file */ +ds_file_t *ds_open(ds_ctxt_t *ctxt, const char *path, MY_STAT *stat); + +/************************************************************************ +Write to a datasink file. +@return 0 on success, 1 on error. */ +int ds_write(ds_file_t *file, const void *buf, size_t len); + +/************************************************************************ +Close a datasink file. +@return 0 on success, 1, on error. */ +int ds_close(ds_file_t *file); + +/************************************************************************ +Destroy a datasink handle */ +void ds_destroy(ds_ctxt_t *ctxt); + +/************************************************************************ +Set the destination pipe for a datasink (only makes sense for compress and +tmpfile). */ +void ds_set_pipe(ds_ctxt_t *ctxt, ds_ctxt_t *pipe_ctxt); + +#ifdef __cplusplus +} /* extern "C" */ +#endif + +#endif /* XB_DATASINK_H */ diff --git a/extra/mariabackup/ds_archive.c b/extra/mariabackup/ds_archive.c new file mode 100644 index 00000000000..aa38b2f9530 --- /dev/null +++ b/extra/mariabackup/ds_archive.c @@ -0,0 +1,275 @@ +/****************************************************** +Copyright (c) 2013 Percona LLC and/or its affiliates. + +Streaming implementation for XtraBackup. + +This program is free software; you can redistribute it and/or modify +it under the terms of the GNU General Public License as published by +the Free Software Foundation; version 2 of the License. + +This program is distributed in the hope that it will be useful, +but WITHOUT ANY WARRANTY; without even the implied warranty of +MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the +GNU General Public License for more details. + +You should have received a copy of the GNU General Public License +along with this program; if not, write to the Free Software +Foundation, Inc., 51 Franklin Street, Fifth Floor, Boston, MA 02110-1301, USA + +*******************************************************/ + +#include +#include +#include +#include "common.h" +#include "datasink.h" + +typedef struct { + struct archive *archive; + ds_file_t *dest_file; + pthread_mutex_t mutex; +} ds_archive_ctxt_t; + +typedef struct { + struct archive_entry *entry; + ds_archive_ctxt_t *archive_ctxt; +} ds_archive_file_t; + + +/*********************************************************************** +General archive interface */ + +static ds_ctxt_t *archive_init(const char *root); +static ds_file_t *archive_open(ds_ctxt_t *ctxt, const char *path, + MY_STAT *mystat); +static int archive_write(ds_file_t *file, const void *buf, size_t len); +static int archive_close(ds_file_t *file); +static void archive_deinit(ds_ctxt_t *ctxt); + +datasink_t datasink_archive = { + &archive_init, + &archive_open, + &archive_write, + &archive_close, + &archive_deinit +}; + +static +int +my_archive_open_callback(struct archive *a __attribute__((unused)), + void *data __attribute__((unused))) +{ + return ARCHIVE_OK; +} + +static +ssize_t +my_archive_write_callback(struct archive *a __attribute__((unused)), + void *data, const void *buffer, size_t length) +{ + ds_archive_ctxt_t *archive_ctxt; + + archive_ctxt = (ds_archive_ctxt_t *) data; + + xb_ad(archive_ctxt != NULL); + xb_ad(archive_ctxt->dest_file != NULL); + + if (!ds_write(archive_ctxt->dest_file, buffer, length)) { + return length; + } + return -1; +} + +static +int +my_archive_close_callback(struct archive *a __attribute__((unused)), + void *data __attribute__((unused))) +{ + return ARCHIVE_OK; +} + +static +ds_ctxt_t * +archive_init(const char *root __attribute__((unused))) +{ + ds_ctxt_t *ctxt; + ds_archive_ctxt_t *archive_ctxt; + struct archive *a; + + ctxt = my_malloc(sizeof(ds_ctxt_t) + sizeof(ds_archive_ctxt_t), + MYF(MY_FAE)); + archive_ctxt = (ds_archive_ctxt_t *)(ctxt + 1); + + if (pthread_mutex_init(&archive_ctxt->mutex, NULL)) { + msg("archive_init: pthread_mutex_init() failed.\n"); + goto err; + } + + a = archive_write_new(); + if (a == NULL) { + msg("archive_write_new() failed.\n"); + goto err; + } + + archive_ctxt->archive = a; + archive_ctxt->dest_file = NULL; + + if (archive_write_set_compression_none(a) != ARCHIVE_OK || + archive_write_set_format_pax_restricted(a) != ARCHIVE_OK || + /* disable internal buffering so we don't have to flush the + output in xtrabackup */ + archive_write_set_bytes_per_block(a, 0) != ARCHIVE_OK) { + msg("failed to set libarchive archive options: %s\n", + archive_error_string(a)); + archive_write_finish(a); + goto err; + } + + if (archive_write_open(a, archive_ctxt, my_archive_open_callback, + my_archive_write_callback, + my_archive_close_callback) != ARCHIVE_OK) { + msg("cannot open output archive.\n"); + return NULL; + } + + ctxt->ptr = archive_ctxt; + + return ctxt; + +err: + my_free(ctxt); + return NULL; +} + +static +ds_file_t * +archive_open(ds_ctxt_t *ctxt, const char *path, MY_STAT *mystat) +{ + ds_archive_ctxt_t *archive_ctxt; + ds_ctxt_t *dest_ctxt; + ds_file_t *file; + ds_archive_file_t *archive_file; + + struct archive *a; + struct archive_entry *entry; + + xb_ad(ctxt->pipe_ctxt != NULL); + dest_ctxt = ctxt->pipe_ctxt; + + archive_ctxt = (ds_archive_ctxt_t *) ctxt->ptr; + + pthread_mutex_lock(&archive_ctxt->mutex); + if (archive_ctxt->dest_file == NULL) { + archive_ctxt->dest_file = ds_open(dest_ctxt, path, mystat); + if (archive_ctxt->dest_file == NULL) { + return NULL; + } + } + pthread_mutex_unlock(&archive_ctxt->mutex); + + file = (ds_file_t *) my_malloc(sizeof(ds_file_t) + + sizeof(ds_archive_file_t), + MYF(MY_FAE)); + + archive_file = (ds_archive_file_t *) (file + 1); + + a = archive_ctxt->archive; + + entry = archive_entry_new(); + if (entry == NULL) { + msg("archive_entry_new() failed.\n"); + goto err; + } + + archive_entry_set_size(entry, mystat->st_size); + archive_entry_set_mode(entry, 0660); + archive_entry_set_filetype(entry, AE_IFREG); + archive_entry_set_pathname(entry, path); + archive_entry_set_mtime(entry, mystat->st_mtime, 0); + + archive_file->entry = entry; + archive_file->archive_ctxt = archive_ctxt; + + if (archive_write_header(a, entry) != ARCHIVE_OK) { + msg("archive_write_header() failed.\n"); + archive_entry_free(entry); + goto err; + } + + file->ptr = archive_file; + file->path = archive_ctxt->dest_file->path; + + return file; + +err: + if (archive_ctxt->dest_file) { + ds_close(archive_ctxt->dest_file); + archive_ctxt->dest_file = NULL; + } + my_free(file); + + return NULL; +} + +static +int +archive_write(ds_file_t *file, const void *buf, size_t len) +{ + ds_archive_file_t *archive_file; + struct archive *a; + + archive_file = (ds_archive_file_t *) file->ptr; + + a = archive_file->archive_ctxt->archive; + + xb_ad(archive_file->archive_ctxt->dest_file != NULL); + if (archive_write_data(a, buf, len) < 0) { + msg("archive_write_data() failed: %s (errno = %d)\n", + archive_error_string(a), archive_errno(a)); + return 1; + } + + return 0; +} + +static +int +archive_close(ds_file_t *file) +{ + ds_archive_file_t *archive_file; + int rc = 0; + + archive_file = (ds_archive_file_t *)file->ptr; + + archive_entry_free(archive_file->entry); + + my_free(file); + + return rc; +} + +static +void +archive_deinit(ds_ctxt_t *ctxt) +{ + struct archive *a; + ds_archive_ctxt_t *archive_ctxt; + + archive_ctxt = (ds_archive_ctxt_t *) ctxt->ptr; + + a = archive_ctxt->archive; + + if (archive_write_close(a) != ARCHIVE_OK) { + msg("archive_write_close() failed.\n"); + } + archive_write_finish(a); + + if (archive_ctxt->dest_file) { + ds_close(archive_ctxt->dest_file); + archive_ctxt->dest_file = NULL; + } + + pthread_mutex_destroy(&archive_ctxt->mutex); + + my_free(ctxt); +} diff --git a/extra/mariabackup/ds_archive.h b/extra/mariabackup/ds_archive.h new file mode 100644 index 00000000000..3f4e4463c58 --- /dev/null +++ b/extra/mariabackup/ds_archive.h @@ -0,0 +1,28 @@ +/****************************************************** +Copyright (c) 2013 Percona LLC and/or its affiliates. + +Streaming interface for XtraBackup. + +This program is free software; you can redistribute it and/or modify +it under the terms of the GNU General Public License as published by +the Free Software Foundation; version 2 of the License. + +This program is distributed in the hope that it will be useful, +but WITHOUT ANY WARRANTY; without even the implied warranty of +MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the +GNU General Public License for more details. + +You should have received a copy of the GNU General Public License +along with this program; if not, write to the Free Software +Foundation, Inc., 51 Franklin Street, Fifth Floor, Boston, MA 02110-1301, USA + +*******************************************************/ + +#ifndef DS_ARCHIVE_H +#define DS_ARCHIVE_H + +#include "datasink.h" + +extern datasink_t datasink_archive; + +#endif diff --git a/extra/mariabackup/ds_buffer.c b/extra/mariabackup/ds_buffer.c new file mode 100644 index 00000000000..4bb314c0f50 --- /dev/null +++ b/extra/mariabackup/ds_buffer.c @@ -0,0 +1,189 @@ +/****************************************************** +Copyright (c) 2012-2013 Percona LLC and/or its affiliates. + +buffer datasink for XtraBackup. + +This program is free software; you can redistribute it and/or modify +it under the terms of the GNU General Public License as published by +the Free Software Foundation; version 2 of the License. + +This program is distributed in the hope that it will be useful, +but WITHOUT ANY WARRANTY; without even the implied warranty of +MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the +GNU General Public License for more details. + +You should have received a copy of the GNU General Public License +along with this program; if not, write to the Free Software +Foundation, Inc., 51 Franklin Street, Fifth Floor, Boston, MA 02110-1301, USA + +*******************************************************/ + +/* Does buffered output to a destination datasink set with ds_set_pipe(). +Writes to the destination datasink are guaranteed to not be smaller than a +specified buffer size (DS_DEFAULT_BUFFER_SIZE by default), with the only +exception for the last write for a file. */ + +#include +#include +#include "ds_buffer.h" +#include "common.h" +#include "datasink.h" + +#define DS_DEFAULT_BUFFER_SIZE (64 * 1024) + +typedef struct { + ds_file_t *dst_file; + char *buf; + size_t pos; + size_t size; +} ds_buffer_file_t; + +typedef struct { + size_t buffer_size; +} ds_buffer_ctxt_t; + +static ds_ctxt_t *buffer_init(const char *root); +static ds_file_t *buffer_open(ds_ctxt_t *ctxt, const char *path, + MY_STAT *mystat); +static int buffer_write(ds_file_t *file, const void *buf, size_t len); +static int buffer_close(ds_file_t *file); +static void buffer_deinit(ds_ctxt_t *ctxt); + +datasink_t datasink_buffer = { + &buffer_init, + &buffer_open, + &buffer_write, + &buffer_close, + &buffer_deinit +}; + +/* Change the default buffer size */ +void ds_buffer_set_size(ds_ctxt_t *ctxt, size_t size) +{ + ds_buffer_ctxt_t *buffer_ctxt = (ds_buffer_ctxt_t *) ctxt->ptr; + + buffer_ctxt->buffer_size = size; +} + +static ds_ctxt_t * +buffer_init(const char *root) +{ + ds_ctxt_t *ctxt; + ds_buffer_ctxt_t *buffer_ctxt; + + ctxt = my_malloc(sizeof(ds_ctxt_t) + sizeof(ds_buffer_ctxt_t), + MYF(MY_FAE)); + buffer_ctxt = (ds_buffer_ctxt_t *) (ctxt + 1); + buffer_ctxt->buffer_size = DS_DEFAULT_BUFFER_SIZE; + + ctxt->ptr = buffer_ctxt; + ctxt->root = my_strdup(root, MYF(MY_FAE)); + + return ctxt; +} + +static ds_file_t * +buffer_open(ds_ctxt_t *ctxt, const char *path, MY_STAT *mystat) +{ + ds_buffer_ctxt_t *buffer_ctxt; + ds_ctxt_t *pipe_ctxt; + ds_file_t *dst_file; + ds_file_t *file; + ds_buffer_file_t *buffer_file; + + pipe_ctxt = ctxt->pipe_ctxt; + xb_a(pipe_ctxt != NULL); + + dst_file = ds_open(pipe_ctxt, path, mystat); + if (dst_file == NULL) { + exit(EXIT_FAILURE); + } + + buffer_ctxt = (ds_buffer_ctxt_t *) ctxt->ptr; + + file = (ds_file_t *) my_malloc(sizeof(ds_file_t) + + sizeof(ds_buffer_file_t) + + buffer_ctxt->buffer_size, + MYF(MY_FAE)); + + buffer_file = (ds_buffer_file_t *) (file + 1); + buffer_file->dst_file = dst_file; + buffer_file->buf = (char *) (buffer_file + 1); + buffer_file->size = buffer_ctxt->buffer_size; + buffer_file->pos = 0; + + file->path = dst_file->path; + file->ptr = buffer_file; + + return file; +} + +static int +buffer_write(ds_file_t *file, const void *buf, size_t len) +{ + ds_buffer_file_t *buffer_file; + + buffer_file = (ds_buffer_file_t *) file->ptr; + + while (len > 0) { + if (buffer_file->pos + len > buffer_file->size) { + if (buffer_file->pos > 0) { + size_t bytes; + + bytes = buffer_file->size - buffer_file->pos; + memcpy(buffer_file->buf + buffer_file->pos, buf, + bytes); + + if (ds_write(buffer_file->dst_file, + buffer_file->buf, + buffer_file->size)) { + return 1; + } + + buffer_file->pos = 0; + + buf = (const char *) buf + bytes; + len -= bytes; + } else { + /* We don't have any buffered bytes, just write + the entire source buffer */ + if (ds_write(buffer_file->dst_file, buf, len)) { + return 1; + } + break; + } + } else { + memcpy(buffer_file->buf + buffer_file->pos, buf, len); + buffer_file->pos += len; + break; + } + } + + return 0; +} + +static int +buffer_close(ds_file_t *file) +{ + ds_buffer_file_t *buffer_file; + int ret; + + buffer_file = (ds_buffer_file_t *) file->ptr; + if (buffer_file->pos > 0) { + ds_write(buffer_file->dst_file, buffer_file->buf, + buffer_file->pos); + } + + ret = ds_close(buffer_file->dst_file); + + my_free(file); + + return ret; +} + +static void +buffer_deinit(ds_ctxt_t *ctxt) +{ + my_free(ctxt->root); + my_free(ctxt); +} diff --git a/extra/mariabackup/ds_buffer.h b/extra/mariabackup/ds_buffer.h new file mode 100644 index 00000000000..f8d2d63267d --- /dev/null +++ b/extra/mariabackup/ds_buffer.h @@ -0,0 +1,39 @@ +/****************************************************** +Copyright (c) 2012-2013 Percona LLC and/or its affiliates. + +buffer datasink for XtraBackup. + +This program is free software; you can redistribute it and/or modify +it under the terms of the GNU General Public License as published by +the Free Software Foundation; version 2 of the License. + +This program is distributed in the hope that it will be useful, +but WITHOUT ANY WARRANTY; without even the implied warranty of +MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the +GNU General Public License for more details. + +You should have received a copy of the GNU General Public License +along with this program; if not, write to the Free Software +Foundation, Inc., 51 Franklin Street, Fifth Floor, Boston, MA 02110-1301, USA + +*******************************************************/ + +#ifndef DS_BUFFER_H +#define DS_BUFFER_H + +#include "datasink.h" + +#ifdef __cplusplus +extern "C" { +#endif + +extern datasink_t datasink_buffer; + +/* Change the default buffer size */ +void ds_buffer_set_size(ds_ctxt_t *ctxt, size_t size); + +#ifdef __cplusplus +} +#endif + +#endif diff --git a/extra/mariabackup/ds_compress.c b/extra/mariabackup/ds_compress.c new file mode 100644 index 00000000000..c49fceaf7fa --- /dev/null +++ b/extra/mariabackup/ds_compress.c @@ -0,0 +1,462 @@ +/****************************************************** +Copyright (c) 2011-2013 Percona LLC and/or its affiliates. + +Compressing datasink implementation for XtraBackup. + +This program is free software; you can redistribute it and/or modify +it under the terms of the GNU General Public License as published by +the Free Software Foundation; version 2 of the License. + +This program is distributed in the hope that it will be useful, +but WITHOUT ANY WARRANTY; without even the implied warranty of +MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the +GNU General Public License for more details. + +You should have received a copy of the GNU General Public License +along with this program; if not, write to the Free Software +Foundation, Inc., 51 Franklin Street, Fifth Floor, Boston, MA 02110-1301, USA + +*******************************************************/ + +#include +#include +#include +#include +#include "common.h" +#include "datasink.h" + +#define COMPRESS_CHUNK_SIZE ((size_t) (xtrabackup_compress_chunk_size)) +#define MY_QLZ_COMPRESS_OVERHEAD 400 + +typedef struct { + pthread_t id; + uint num; + pthread_mutex_t ctrl_mutex; + pthread_cond_t ctrl_cond; + pthread_mutex_t data_mutex; + pthread_cond_t data_cond; + my_bool started; + my_bool data_avail; + my_bool cancelled; + const char *from; + size_t from_len; + char *to; + size_t to_len; + qlz_state_compress state; + ulong adler; +} comp_thread_ctxt_t; + +typedef struct { + comp_thread_ctxt_t *threads; + uint nthreads; +} ds_compress_ctxt_t; + +typedef struct { + ds_file_t *dest_file; + ds_compress_ctxt_t *comp_ctxt; + size_t bytes_processed; +} ds_compress_file_t; + +/* Compression options */ +extern char *xtrabackup_compress_alg; +extern uint xtrabackup_compress_threads; +extern ulonglong xtrabackup_compress_chunk_size; + +static ds_ctxt_t *compress_init(const char *root); +static ds_file_t *compress_open(ds_ctxt_t *ctxt, const char *path, + MY_STAT *mystat); +static int compress_write(ds_file_t *file, const void *buf, size_t len); +static int compress_close(ds_file_t *file); +static void compress_deinit(ds_ctxt_t *ctxt); + +datasink_t datasink_compress = { + &compress_init, + &compress_open, + &compress_write, + &compress_close, + &compress_deinit +}; + +static inline int write_uint32_le(ds_file_t *file, ulong n); +static inline int write_uint64_le(ds_file_t *file, ulonglong n); + +static comp_thread_ctxt_t *create_worker_threads(uint n); +static void destroy_worker_threads(comp_thread_ctxt_t *threads, uint n); +static void *compress_worker_thread_func(void *arg); + +static +ds_ctxt_t * +compress_init(const char *root) +{ + ds_ctxt_t *ctxt; + ds_compress_ctxt_t *compress_ctxt; + comp_thread_ctxt_t *threads; + + /* Create and initialize the worker threads */ + threads = create_worker_threads(xtrabackup_compress_threads); + if (threads == NULL) { + msg("compress: failed to create worker threads.\n"); + return NULL; + } + + ctxt = (ds_ctxt_t *) my_malloc(sizeof(ds_ctxt_t) + + sizeof(ds_compress_ctxt_t), + MYF(MY_FAE)); + + compress_ctxt = (ds_compress_ctxt_t *) (ctxt + 1); + compress_ctxt->threads = threads; + compress_ctxt->nthreads = xtrabackup_compress_threads; + + ctxt->ptr = compress_ctxt; + ctxt->root = my_strdup(root, MYF(MY_FAE)); + + return ctxt; +} + +static +ds_file_t * +compress_open(ds_ctxt_t *ctxt, const char *path, MY_STAT *mystat) +{ + ds_compress_ctxt_t *comp_ctxt; + ds_ctxt_t *dest_ctxt; + ds_file_t *dest_file; + char new_name[FN_REFLEN]; + size_t name_len; + ds_file_t *file; + ds_compress_file_t *comp_file; + + xb_ad(ctxt->pipe_ctxt != NULL); + dest_ctxt = ctxt->pipe_ctxt; + + comp_ctxt = (ds_compress_ctxt_t *) ctxt->ptr; + + /* Append the .qp extension to the filename */ + fn_format(new_name, path, "", ".qp", MYF(MY_APPEND_EXT)); + + dest_file = ds_open(dest_ctxt, new_name, mystat); + if (dest_file == NULL) { + return NULL; + } + + /* Write the qpress archive header */ + if (ds_write(dest_file, "qpress10", 8) || + write_uint64_le(dest_file, COMPRESS_CHUNK_SIZE)) { + goto err; + } + + /* We are going to create a one-file "flat" (i.e. with no + subdirectories) archive. So strip the directory part from the path and + remove the '.qp' suffix. */ + fn_format(new_name, path, "", "", MYF(MY_REPLACE_DIR)); + + /* Write the qpress file header */ + name_len = strlen(new_name); + if (ds_write(dest_file, "F", 1) || + write_uint32_le(dest_file, name_len) || + /* we want to write the terminating \0 as well */ + ds_write(dest_file, new_name, name_len + 1)) { + goto err; + } + + file = (ds_file_t *) my_malloc(sizeof(ds_file_t) + + sizeof(ds_compress_file_t), + MYF(MY_FAE)); + comp_file = (ds_compress_file_t *) (file + 1); + comp_file->dest_file = dest_file; + comp_file->comp_ctxt = comp_ctxt; + comp_file->bytes_processed = 0; + + file->ptr = comp_file; + file->path = dest_file->path; + + return file; + +err: + ds_close(dest_file); + return NULL; +} + +static +int +compress_write(ds_file_t *file, const void *buf, size_t len) +{ + ds_compress_file_t *comp_file; + ds_compress_ctxt_t *comp_ctxt; + comp_thread_ctxt_t *threads; + comp_thread_ctxt_t *thd; + uint nthreads; + uint i; + const char *ptr; + ds_file_t *dest_file; + + comp_file = (ds_compress_file_t *) file->ptr; + comp_ctxt = comp_file->comp_ctxt; + dest_file = comp_file->dest_file; + + threads = comp_ctxt->threads; + nthreads = comp_ctxt->nthreads; + + ptr = (const char *) buf; + while (len > 0) { + uint max_thread; + + /* Send data to worker threads for compression */ + for (i = 0; i < nthreads; i++) { + size_t chunk_len; + + thd = threads + i; + + pthread_mutex_lock(&thd->ctrl_mutex); + + chunk_len = (len > COMPRESS_CHUNK_SIZE) ? + COMPRESS_CHUNK_SIZE : len; + thd->from = ptr; + thd->from_len = chunk_len; + + pthread_mutex_lock(&thd->data_mutex); + thd->data_avail = TRUE; + pthread_cond_signal(&thd->data_cond); + pthread_mutex_unlock(&thd->data_mutex); + + len -= chunk_len; + if (len == 0) { + break; + } + ptr += chunk_len; + } + + max_thread = (i < nthreads) ? i : nthreads - 1; + + /* Reap and stream the compressed data */ + for (i = 0; i <= max_thread; i++) { + thd = threads + i; + + pthread_mutex_lock(&thd->data_mutex); + while (thd->data_avail == TRUE) { + pthread_cond_wait(&thd->data_cond, + &thd->data_mutex); + } + + xb_a(threads[i].to_len > 0); + + if (ds_write(dest_file, "NEWBNEWB", 8) || + write_uint64_le(dest_file, + comp_file->bytes_processed)) { + msg("compress: write to the destination stream " + "failed.\n"); + return 1; + } + + comp_file->bytes_processed += threads[i].from_len; + + if (write_uint32_le(dest_file, threads[i].adler) || + ds_write(dest_file, threads[i].to, + threads[i].to_len)) { + msg("compress: write to the destination stream " + "failed.\n"); + return 1; + } + + pthread_mutex_unlock(&threads[i].data_mutex); + pthread_mutex_unlock(&threads[i].ctrl_mutex); + } + } + + return 0; +} + +static +int +compress_close(ds_file_t *file) +{ + ds_compress_file_t *comp_file; + ds_file_t *dest_file; + int rc; + + comp_file = (ds_compress_file_t *) file->ptr; + dest_file = comp_file->dest_file; + + /* Write the qpress file trailer */ + ds_write(dest_file, "ENDSENDS", 8); + + /* Supposedly the number of written bytes should be written as a + "recovery information" in the file trailer, but in reality qpress + always writes 8 zeros here. Let's do the same */ + + write_uint64_le(dest_file, 0); + + rc = ds_close(dest_file); + + my_free(file); + + return rc; +} + +static +void +compress_deinit(ds_ctxt_t *ctxt) +{ + ds_compress_ctxt_t *comp_ctxt; + + xb_ad(ctxt->pipe_ctxt != NULL); + + comp_ctxt = (ds_compress_ctxt_t *) ctxt->ptr;; + + destroy_worker_threads(comp_ctxt->threads, comp_ctxt->nthreads); + + my_free(ctxt->root); + my_free(ctxt); +} + +static inline +int +write_uint32_le(ds_file_t *file, ulong n) +{ + char tmp[4]; + + int4store(tmp, n); + return ds_write(file, tmp, sizeof(tmp)); +} + +static inline +int +write_uint64_le(ds_file_t *file, ulonglong n) +{ + char tmp[8]; + + int8store(tmp, n); + return ds_write(file, tmp, sizeof(tmp)); +} + +static +comp_thread_ctxt_t * +create_worker_threads(uint n) +{ + comp_thread_ctxt_t *threads; + uint i; + + threads = (comp_thread_ctxt_t *) + my_malloc(sizeof(comp_thread_ctxt_t) * n, MYF(MY_FAE)); + + for (i = 0; i < n; i++) { + comp_thread_ctxt_t *thd = threads + i; + + thd->num = i + 1; + thd->started = FALSE; + thd->cancelled = FALSE; + thd->data_avail = FALSE; + + thd->to = (char *) my_malloc(COMPRESS_CHUNK_SIZE + + MY_QLZ_COMPRESS_OVERHEAD, + MYF(MY_FAE)); + + /* Initialize the control mutex and condition var */ + if (pthread_mutex_init(&thd->ctrl_mutex, NULL) || + pthread_cond_init(&thd->ctrl_cond, NULL)) { + goto err; + } + + /* Initialize and data mutex and condition var */ + if (pthread_mutex_init(&thd->data_mutex, NULL) || + pthread_cond_init(&thd->data_cond, NULL)) { + goto err; + } + + pthread_mutex_lock(&thd->ctrl_mutex); + + if (pthread_create(&thd->id, NULL, compress_worker_thread_func, + thd)) { + msg("compress: pthread_create() failed: " + "errno = %d\n", errno); + goto err; + } + } + + /* Wait for the threads to start */ + for (i = 0; i < n; i++) { + comp_thread_ctxt_t *thd = threads + i; + + while (thd->started == FALSE) + pthread_cond_wait(&thd->ctrl_cond, &thd->ctrl_mutex); + pthread_mutex_unlock(&thd->ctrl_mutex); + } + + return threads; + +err: + return NULL; +} + +static +void +destroy_worker_threads(comp_thread_ctxt_t *threads, uint n) +{ + uint i; + + for (i = 0; i < n; i++) { + comp_thread_ctxt_t *thd = threads + i; + + pthread_mutex_lock(&thd->data_mutex); + threads[i].cancelled = TRUE; + pthread_cond_signal(&thd->data_cond); + pthread_mutex_unlock(&thd->data_mutex); + + pthread_join(thd->id, NULL); + + pthread_cond_destroy(&thd->data_cond); + pthread_mutex_destroy(&thd->data_mutex); + pthread_cond_destroy(&thd->ctrl_cond); + pthread_mutex_destroy(&thd->ctrl_mutex); + + my_free(thd->to); + } + + my_free(threads); +} + +static +void * +compress_worker_thread_func(void *arg) +{ + comp_thread_ctxt_t *thd = (comp_thread_ctxt_t *) arg; + + pthread_mutex_lock(&thd->ctrl_mutex); + + pthread_mutex_lock(&thd->data_mutex); + + thd->started = TRUE; + pthread_cond_signal(&thd->ctrl_cond); + + pthread_mutex_unlock(&thd->ctrl_mutex); + + while (1) { + thd->data_avail = FALSE; + pthread_cond_signal(&thd->data_cond); + + while (!thd->data_avail && !thd->cancelled) { + pthread_cond_wait(&thd->data_cond, &thd->data_mutex); + } + + if (thd->cancelled) + break; + + thd->to_len = qlz_compress(thd->from, thd->to, thd->from_len, + &thd->state); + + /* qpress uses 0x00010000 as the initial value, but its own + Adler-32 implementation treats the value differently: + 1. higher order bits are the sum of all bytes in the sequence + 2. lower order bits are the sum of resulting values at every + step. + So it's the other way around as compared to zlib's adler32(). + That's why 0x00000001 is being passed here to be compatible + with qpress implementation. */ + + thd->adler = adler32(0x00000001, (uchar *) thd->to, + thd->to_len); + } + + pthread_mutex_unlock(&thd->data_mutex); + + return NULL; +} diff --git a/extra/mariabackup/ds_compress.h b/extra/mariabackup/ds_compress.h new file mode 100644 index 00000000000..8498c965e13 --- /dev/null +++ b/extra/mariabackup/ds_compress.h @@ -0,0 +1,28 @@ +/****************************************************** +Copyright (c) 2011-2013 Percona LLC and/or its affiliates. + +Compression interface for XtraBackup. + +This program is free software; you can redistribute it and/or modify +it under the terms of the GNU General Public License as published by +the Free Software Foundation; version 2 of the License. + +This program is distributed in the hope that it will be useful, +but WITHOUT ANY WARRANTY; without even the implied warranty of +MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the +GNU General Public License for more details. + +You should have received a copy of the GNU General Public License +along with this program; if not, write to the Free Software +Foundation, Inc., 51 Franklin Street, Fifth Floor, Boston, MA 02110-1301, USA + +*******************************************************/ + +#ifndef DS_COMPRESS_H +#define DS_COMPRESS_H + +#include "datasink.h" + +extern datasink_t datasink_compress; + +#endif diff --git a/extra/mariabackup/ds_encrypt.c b/extra/mariabackup/ds_encrypt.c new file mode 100644 index 00000000000..f8d62a03e13 --- /dev/null +++ b/extra/mariabackup/ds_encrypt.c @@ -0,0 +1,617 @@ +/****************************************************** +Copyright (c) 2013 Percona LLC and/or its affiliates. + +Encryption datasink implementation for XtraBackup. + +This program is free software; you can redistribute it and/or modify +it under the terms of the GNU General Public License as published by +the Free Software Foundation; version 2 of the License. + +This program is distributed in the hope that it will be useful, +but WITHOUT ANY WARRANTY; without even the implied warranty of +MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the +GNU General Public License for more details. + +You should have received a copy of the GNU General Public License +along with this program; if not, write to the Free Software +Foundation, Inc., 51 Franklin Street, Fifth Floor, Boston, MA 02110-1301, USA + +*******************************************************/ + + +#include +#include "common.h" +#include "datasink.h" + +#if GCC_VERSION >= 4002 +/* Workaround to avoid "gcry_ac_* is deprecated" warnings in gcrypt.h */ +# pragma GCC diagnostic ignored "-Wdeprecated-declarations" +#endif + +#include + +#if GCC_VERSION >= 4002 +# pragma GCC diagnostic warning "-Wdeprecated-declarations" +#endif + +#include "xbcrypt.h" + +#if !defined(GCRYPT_VERSION_NUMBER) || (GCRYPT_VERSION_NUMBER < 0x010600) +GCRY_THREAD_OPTION_PTHREAD_IMPL; +#endif + +#define XB_CRYPT_CHUNK_SIZE ((size_t) (xtrabackup_encrypt_chunk_size)) + +typedef struct { + pthread_t id; + uint num; + pthread_mutex_t ctrl_mutex; + pthread_cond_t ctrl_cond; + pthread_mutex_t data_mutex; + pthread_cond_t data_cond; + my_bool started; + my_bool data_avail; + my_bool cancelled; + const char *from; + size_t from_len; + char *to; + char *iv; + size_t to_len; + gcry_cipher_hd_t cipher_handle; +} crypt_thread_ctxt_t; + +typedef struct { + crypt_thread_ctxt_t *threads; + uint nthreads; +} ds_encrypt_ctxt_t; + +typedef struct { + xb_wcrypt_t *xbcrypt_file; + ds_encrypt_ctxt_t *crypt_ctxt; + size_t bytes_processed; + ds_file_t *dest_file; +} ds_encrypt_file_t; + +/* Encryption options */ +extern ulong xtrabackup_encrypt_algo; +extern char *xtrabackup_encrypt_key; +extern char *xtrabackup_encrypt_key_file; +extern uint xtrabackup_encrypt_threads; +extern ulonglong xtrabackup_encrypt_chunk_size; + +static ds_ctxt_t *encrypt_init(const char *root); +static ds_file_t *encrypt_open(ds_ctxt_t *ctxt, const char *path, + MY_STAT *mystat); +static int encrypt_write(ds_file_t *file, const void *buf, size_t len); +static int encrypt_close(ds_file_t *file); +static void encrypt_deinit(ds_ctxt_t *ctxt); + +datasink_t datasink_encrypt = { + &encrypt_init, + &encrypt_open, + &encrypt_write, + &encrypt_close, + &encrypt_deinit +}; + +static crypt_thread_ctxt_t *create_worker_threads(uint n); +static void destroy_worker_threads(crypt_thread_ctxt_t *threads, uint n); +static void *encrypt_worker_thread_func(void *arg); + +static uint encrypt_algos[] = { GCRY_CIPHER_NONE, GCRY_CIPHER_AES128, + GCRY_CIPHER_AES192, GCRY_CIPHER_AES256 }; +static uint encrypt_algo; +static const uint encrypt_mode = GCRY_CIPHER_MODE_CTR; +static uint encrypt_key_len = 0; +static size_t encrypt_iv_len = 0; + +static +ssize_t +my_xb_crypt_write_callback(void *userdata, const void *buf, size_t len) +{ + ds_encrypt_file_t *encrypt_file; + + encrypt_file = (ds_encrypt_file_t *) userdata; + + xb_ad(encrypt_file != NULL); + xb_ad(encrypt_file->dest_file != NULL); + + if (!ds_write(encrypt_file->dest_file, buf, len)) { + return len; + } + return -1; +} + +static +ds_ctxt_t * +encrypt_init(const char *root) +{ + ds_ctxt_t *ctxt; + ds_encrypt_ctxt_t *encrypt_ctxt; + crypt_thread_ctxt_t *threads; + gcry_error_t gcry_error; + + /* Acording to gcrypt docs (and my testing), setting up the threading + callbacks must be done first, so, lets give it a shot */ +#if !defined(GCRYPT_VERSION_NUMBER) || (GCRYPT_VERSION_NUMBER < 0x010600) + gcry_error = gcry_control(GCRYCTL_SET_THREAD_CBS, &gcry_threads_pthread); + if (gcry_error) { + msg("encrypt: unable to set libgcrypt thread cbs - " + "%s : %s\n", + gcry_strsource(gcry_error), + gcry_strerror(gcry_error)); + return NULL; + } +#endif + + /* Version check should be the very next call because it + makes sure that important subsystems are intialized. */ + if (!gcry_control(GCRYCTL_ANY_INITIALIZATION_P)) { + const char *gcrypt_version; + gcrypt_version = gcry_check_version(NULL); + /* No other library has already initialized libgcrypt. */ + if (!gcrypt_version) { + msg("encrypt: failed to initialize libgcrypt\n"); + return NULL; + } else { + msg("encrypt: using gcrypt %s\n", gcrypt_version); + } + } + + /* Disable the gcry secure memory, not dealing with this for now */ + gcry_error = gcry_control(GCRYCTL_DISABLE_SECMEM, 0); + if (gcry_error) { + msg("encrypt: unable to disable libgcrypt secmem - " + "%s : %s\n", + gcry_strsource(gcry_error), + gcry_strerror(gcry_error)); + return NULL; + } + + /* Finalize gcry initialization. */ + gcry_error = gcry_control(GCRYCTL_INITIALIZATION_FINISHED, 0); + if (gcry_error) { + msg("encrypt: unable to finish libgcrypt initialization - " + "%s : %s\n", + gcry_strsource(gcry_error), + gcry_strerror(gcry_error)); + return NULL; + } + + /* Determine the algorithm */ + encrypt_algo = encrypt_algos[xtrabackup_encrypt_algo]; + + /* Set up the iv length */ + encrypt_iv_len = gcry_cipher_get_algo_blklen(encrypt_algo); + xb_a(encrypt_iv_len > 0); + + /* Now set up the key */ + if (xtrabackup_encrypt_key == NULL && + xtrabackup_encrypt_key_file == NULL) { + msg("encrypt: no encryption key or key file specified.\n"); + return NULL; + } else if (xtrabackup_encrypt_key && xtrabackup_encrypt_key_file) { + msg("encrypt: both encryption key and key file specified.\n"); + return NULL; + } else if (xtrabackup_encrypt_key_file) { + if (!xb_crypt_read_key_file(xtrabackup_encrypt_key_file, + (void**)&xtrabackup_encrypt_key, + &encrypt_key_len)) { + msg("encrypt: unable to read encryption key file" + " \"%s\".\n", xtrabackup_encrypt_key_file); + return NULL; + } + } else if (xtrabackup_encrypt_key) { + encrypt_key_len = strlen(xtrabackup_encrypt_key); + } else { + msg("encrypt: no encryption key or key file specified.\n"); + return NULL; + } + + /* Create and initialize the worker threads */ + threads = create_worker_threads(xtrabackup_encrypt_threads); + if (threads == NULL) { + msg("encrypt: failed to create worker threads.\n"); + return NULL; + } + + ctxt = (ds_ctxt_t *) my_malloc(sizeof(ds_ctxt_t) + + sizeof(ds_encrypt_ctxt_t), + MYF(MY_FAE)); + + encrypt_ctxt = (ds_encrypt_ctxt_t *) (ctxt + 1); + encrypt_ctxt->threads = threads; + encrypt_ctxt->nthreads = xtrabackup_encrypt_threads; + + ctxt->ptr = encrypt_ctxt; + ctxt->root = my_strdup(root, MYF(MY_FAE)); + + return ctxt; +} + +static +ds_file_t * +encrypt_open(ds_ctxt_t *ctxt, const char *path, MY_STAT *mystat) +{ + ds_ctxt_t *dest_ctxt; + + ds_encrypt_ctxt_t *crypt_ctxt; + ds_encrypt_file_t *crypt_file; + + char new_name[FN_REFLEN]; + ds_file_t *file; + + xb_ad(ctxt->pipe_ctxt != NULL); + dest_ctxt = ctxt->pipe_ctxt; + + crypt_ctxt = (ds_encrypt_ctxt_t *) ctxt->ptr; + + + file = (ds_file_t *) my_malloc(sizeof(ds_file_t) + + sizeof(ds_encrypt_file_t), + MYF(MY_FAE|MY_ZEROFILL)); + + crypt_file = (ds_encrypt_file_t *) (file + 1); + + /* Append the .xbcrypt extension to the filename */ + fn_format(new_name, path, "", ".xbcrypt", MYF(MY_APPEND_EXT)); + crypt_file->dest_file = ds_open(dest_ctxt, new_name, mystat); + if (crypt_file->dest_file == NULL) { + msg("encrypt: ds_open(\"%s\") failed.\n", new_name); + goto err; + } + + crypt_file->crypt_ctxt = crypt_ctxt; + crypt_file->xbcrypt_file = xb_crypt_write_open(crypt_file, + my_xb_crypt_write_callback); + + if (crypt_file->xbcrypt_file == NULL) { + msg("encrypt: xb_crypt_write_open() failed.\n"); + goto err; + } + + + file->ptr = crypt_file; + file->path = crypt_file->dest_file->path; + + return file; + +err: + if (crypt_file->dest_file) { + ds_close(crypt_file->dest_file); + } + my_free(file); + return NULL; +} + +static +int +encrypt_write(ds_file_t *file, const void *buf, size_t len) +{ + ds_encrypt_file_t *crypt_file; + ds_encrypt_ctxt_t *crypt_ctxt; + crypt_thread_ctxt_t *threads; + crypt_thread_ctxt_t *thd; + uint nthreads; + uint i; + const char *ptr; + + crypt_file = (ds_encrypt_file_t *) file->ptr; + crypt_ctxt = crypt_file->crypt_ctxt; + + threads = crypt_ctxt->threads; + nthreads = crypt_ctxt->nthreads; + + ptr = (const char *) buf; + while (len > 0) { + uint max_thread; + + /* Send data to worker threads for encryption */ + for (i = 0; i < nthreads; i++) { + size_t chunk_len; + + thd = threads + i; + + pthread_mutex_lock(&thd->ctrl_mutex); + + chunk_len = (len > XB_CRYPT_CHUNK_SIZE) ? + XB_CRYPT_CHUNK_SIZE : len; + thd->from = ptr; + thd->from_len = chunk_len; + + pthread_mutex_lock(&thd->data_mutex); + thd->data_avail = TRUE; + pthread_cond_signal(&thd->data_cond); + pthread_mutex_unlock(&thd->data_mutex); + + len -= chunk_len; + if (len == 0) { + break; + } + ptr += chunk_len; + } + + max_thread = (i < nthreads) ? i : nthreads - 1; + + /* Reap and stream the encrypted data */ + for (i = 0; i <= max_thread; i++) { + thd = threads + i; + + pthread_mutex_lock(&thd->data_mutex); + while (thd->data_avail == TRUE) { + pthread_cond_wait(&thd->data_cond, + &thd->data_mutex); + } + + xb_a(threads[i].to_len > 0); + + if (xb_crypt_write_chunk(crypt_file->xbcrypt_file, + threads[i].to, + threads[i].from_len + + XB_CRYPT_HASH_LEN, + threads[i].to_len, + threads[i].iv, + encrypt_iv_len)) { + msg("encrypt: write to the destination file " + "failed.\n"); + return 1; + } + + crypt_file->bytes_processed += threads[i].from_len; + + pthread_mutex_unlock(&threads[i].data_mutex); + pthread_mutex_unlock(&threads[i].ctrl_mutex); + } + } + + return 0; +} + +static +int +encrypt_close(ds_file_t *file) +{ + ds_encrypt_file_t *crypt_file; + ds_file_t *dest_file; + int rc = 0; + + crypt_file = (ds_encrypt_file_t *) file->ptr; + dest_file = crypt_file->dest_file; + + rc = xb_crypt_write_close(crypt_file->xbcrypt_file); + + if (ds_close(dest_file)) { + rc = 1; + } + + my_free(file); + + return rc; +} + +static +void +encrypt_deinit(ds_ctxt_t *ctxt) +{ + ds_encrypt_ctxt_t *crypt_ctxt; + + xb_ad(ctxt->pipe_ctxt != NULL); + + crypt_ctxt = (ds_encrypt_ctxt_t *) ctxt->ptr; + + destroy_worker_threads(crypt_ctxt->threads, crypt_ctxt->nthreads); + + my_free(ctxt->root); + my_free(ctxt); + if (xtrabackup_encrypt_key) + my_free(xtrabackup_encrypt_key); + if (xtrabackup_encrypt_key_file) + my_free(xtrabackup_encrypt_key_file); +} + +static +crypt_thread_ctxt_t * +create_worker_threads(uint n) +{ + crypt_thread_ctxt_t *threads; + uint i; + + threads = (crypt_thread_ctxt_t *) + my_malloc(sizeof(crypt_thread_ctxt_t) * n, MYF(MY_FAE)); + + for (i = 0; i < n; i++) { + crypt_thread_ctxt_t *thd = threads + i; + + thd->num = i + 1; + thd->started = FALSE; + thd->cancelled = FALSE; + thd->data_avail = FALSE; + + thd->to = (char *) my_malloc(XB_CRYPT_CHUNK_SIZE + + XB_CRYPT_HASH_LEN, MYF(MY_FAE)); + + thd->iv = (char *) my_malloc(encrypt_iv_len, + MYF(MY_FAE)); + + /* Initialize the control mutex and condition var */ + if (pthread_mutex_init(&thd->ctrl_mutex, NULL) || + pthread_cond_init(&thd->ctrl_cond, NULL)) { + goto err; + } + + /* Initialize and data mutex and condition var */ + if (pthread_mutex_init(&thd->data_mutex, NULL) || + pthread_cond_init(&thd->data_cond, NULL)) { + goto err; + } + + if (encrypt_algo != GCRY_CIPHER_NONE) { + gcry_error_t gcry_error; + + gcry_error = gcry_cipher_open(&thd->cipher_handle, + encrypt_algo, + encrypt_mode, 0); + if (gcry_error) { + msg("encrypt: unable to open libgcrypt" + " cipher - %s : %s\n", + gcry_strsource(gcry_error), + gcry_strerror(gcry_error)); + gcry_cipher_close(thd->cipher_handle); + goto err; + } + + gcry_error = gcry_cipher_setkey(thd->cipher_handle, + xtrabackup_encrypt_key, + encrypt_key_len); + if (gcry_error) { + msg("encrypt: unable to set libgcrypt" + " cipher key - %s : %s\n", + gcry_strsource(gcry_error), + gcry_strerror(gcry_error)); + gcry_cipher_close(thd->cipher_handle); + goto err; + } + } + + pthread_mutex_lock(&thd->ctrl_mutex); + + if (pthread_create(&thd->id, NULL, encrypt_worker_thread_func, + thd)) { + msg("encrypt: pthread_create() failed: " + "errno = %d\n", errno); + goto err; + } + } + + /* Wait for the threads to start */ + for (i = 0; i < n; i++) { + crypt_thread_ctxt_t *thd = threads + i; + + while (thd->started == FALSE) + pthread_cond_wait(&thd->ctrl_cond, &thd->ctrl_mutex); + pthread_mutex_unlock(&thd->ctrl_mutex); + } + + return threads; + +err: + return NULL; +} + +static +void +destroy_worker_threads(crypt_thread_ctxt_t *threads, uint n) +{ + uint i; + + for (i = 0; i < n; i++) { + crypt_thread_ctxt_t *thd = threads + i; + + pthread_mutex_lock(&thd->data_mutex); + threads[i].cancelled = TRUE; + pthread_cond_signal(&thd->data_cond); + pthread_mutex_unlock(&thd->data_mutex); + + pthread_join(thd->id, NULL); + + pthread_cond_destroy(&thd->data_cond); + pthread_mutex_destroy(&thd->data_mutex); + pthread_cond_destroy(&thd->ctrl_cond); + pthread_mutex_destroy(&thd->ctrl_mutex); + + if (encrypt_algo != GCRY_CIPHER_NONE) + gcry_cipher_close(thd->cipher_handle); + + my_free(thd->to); + my_free(thd->iv); + } + + my_free(threads); +} + +static +void * +encrypt_worker_thread_func(void *arg) +{ + crypt_thread_ctxt_t *thd = (crypt_thread_ctxt_t *) arg; + + pthread_mutex_lock(&thd->ctrl_mutex); + + pthread_mutex_lock(&thd->data_mutex); + + thd->started = TRUE; + pthread_cond_signal(&thd->ctrl_cond); + + pthread_mutex_unlock(&thd->ctrl_mutex); + + while (1) { + thd->data_avail = FALSE; + pthread_cond_signal(&thd->data_cond); + + while (!thd->data_avail && !thd->cancelled) { + pthread_cond_wait(&thd->data_cond, &thd->data_mutex); + } + + if (thd->cancelled) + break; + + /* ensure that XB_CRYPT_HASH_LEN is the correct length + of XB_CRYPT_HASH hashing algorithm output */ + assert(gcry_md_get_algo_dlen(XB_CRYPT_HASH) == + XB_CRYPT_HASH_LEN); + + memcpy(thd->to, thd->from, thd->from_len); + gcry_md_hash_buffer(XB_CRYPT_HASH, thd->to + thd->from_len, + thd->from, thd->from_len); + thd->to_len = thd->from_len; + + if (encrypt_algo != GCRY_CIPHER_NONE) { + gcry_error_t gcry_error; + + gcry_error = gcry_cipher_reset(thd->cipher_handle); + if (gcry_error) { + msg("encrypt: unable to reset cipher - " + "%s : %s\n", + gcry_strsource(gcry_error), + gcry_strerror(gcry_error)); + thd->to_len = 0; + continue; + } + + xb_crypt_create_iv(thd->iv, encrypt_iv_len); + gcry_error = gcry_cipher_setctr(thd->cipher_handle, + thd->iv, + encrypt_iv_len); + if (gcry_error) { + msg("encrypt: unable to set cipher ctr - " + "%s : %s\n", + gcry_strsource(gcry_error), + gcry_strerror(gcry_error)); + thd->to_len = 0; + continue; + } + + gcry_error = gcry_cipher_encrypt(thd->cipher_handle, + thd->to, + thd->to_len + + XB_CRYPT_HASH_LEN, + thd->to, + thd->from_len + + XB_CRYPT_HASH_LEN); + if (gcry_error) { + msg("encrypt: unable to encrypt buffer - " + "%s : %s\n", gcry_strsource(gcry_error), + gcry_strerror(gcry_error)); + thd->to_len = 0; + } + } else { + memcpy(thd->to, thd->from, + thd->from_len + XB_CRYPT_HASH_LEN); + } + thd->to_len += XB_CRYPT_HASH_LEN; + } + + pthread_mutex_unlock(&thd->data_mutex); + + return NULL; +} diff --git a/extra/mariabackup/ds_encrypt.h b/extra/mariabackup/ds_encrypt.h new file mode 100644 index 00000000000..ed869747d79 --- /dev/null +++ b/extra/mariabackup/ds_encrypt.h @@ -0,0 +1,28 @@ +/****************************************************** +Copyright (c) 2013 Percona LLC and/or its affiliates. + +Encryption interface for XtraBackup. + +This program is free software; you can redistribute it and/or modify +it under the terms of the GNU General Public License as published by +the Free Software Foundation; version 2 of the License. + +This program is distributed in the hope that it will be useful, +but WITHOUT ANY WARRANTY; without even the implied warranty of +MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the +GNU General Public License for more details. + +You should have received a copy of the GNU General Public License +along with this program; if not, write to the Free Software +Foundation, Inc., 51 Franklin Street, Fifth Floor, Boston, MA 02110-1301, USA + +*******************************************************/ + +#ifndef DS_ENCRYPT_H +#define DS_ENCRYPT_H + +#include "datasink.h" + +extern datasink_t datasink_encrypt; + +#endif diff --git a/extra/mariabackup/ds_local.c b/extra/mariabackup/ds_local.c new file mode 100644 index 00000000000..dc13ed7595e --- /dev/null +++ b/extra/mariabackup/ds_local.c @@ -0,0 +1,151 @@ +/****************************************************** +Copyright (c) 2011-2013 Percona LLC and/or its affiliates. + +Local datasink implementation for XtraBackup. + +This program is free software; you can redistribute it and/or modify +it under the terms of the GNU General Public License as published by +the Free Software Foundation; version 2 of the License. + +This program is distributed in the hope that it will be useful, +but WITHOUT ANY WARRANTY; without even the implied warranty of +MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the +GNU General Public License for more details. + +You should have received a copy of the GNU General Public License +along with this program; if not, write to the Free Software +Foundation, Inc., 51 Franklin Street, Fifth Floor, Boston, MA 02110-1301, USA + +*******************************************************/ + +#include +#include +#include +#include "common.h" +#include "datasink.h" + +typedef struct { + File fd; +} ds_local_file_t; + +static ds_ctxt_t *local_init(const char *root); +static ds_file_t *local_open(ds_ctxt_t *ctxt, const char *path, + MY_STAT *mystat); +static int local_write(ds_file_t *file, const void *buf, size_t len); +static int local_close(ds_file_t *file); +static void local_deinit(ds_ctxt_t *ctxt); + +datasink_t datasink_local = { + &local_init, + &local_open, + &local_write, + &local_close, + &local_deinit +}; + +static +ds_ctxt_t * +local_init(const char *root) +{ + ds_ctxt_t *ctxt; + + if (my_mkdir(root, 0777, MYF(0)) < 0 + && my_errno != EEXIST && my_errno != EISDIR) + { + char errbuf[MYSYS_STRERROR_SIZE]; + my_error(EE_CANT_MKDIR, MYF(ME_BELL | ME_WAITTANG), + root, my_errno, my_strerror(errbuf, sizeof(errbuf), + my_errno)); + return NULL; + } + + ctxt = my_malloc(sizeof(ds_ctxt_t), MYF(MY_FAE)); + + ctxt->root = my_strdup(root, MYF(MY_FAE)); + + return ctxt; +} + +static +ds_file_t * +local_open(ds_ctxt_t *ctxt, const char *path, + MY_STAT *mystat __attribute__((unused))) +{ + char fullpath[FN_REFLEN]; + char dirpath[FN_REFLEN]; + size_t dirpath_len; + size_t path_len; + ds_local_file_t *local_file; + ds_file_t *file; + File fd; + + fn_format(fullpath, path, ctxt->root, "", MYF(MY_RELATIVE_PATH)); + + /* Create the directory if needed */ + dirname_part(dirpath, fullpath, &dirpath_len); + if (my_mkdir(dirpath, 0777, MYF(0)) < 0 && my_errno != EEXIST) { + char errbuf[MYSYS_STRERROR_SIZE]; + my_error(EE_CANT_MKDIR, MYF(ME_BELL | ME_WAITTANG), + dirpath, my_errno, my_strerror(errbuf, sizeof(errbuf), + my_errno)); + return NULL; + } + + fd = my_create(fullpath, 0, O_WRONLY | O_BINARY | O_EXCL | O_NOFOLLOW, + MYF(MY_WME)); + if (fd < 0) { + return NULL; + } + + path_len = strlen(fullpath) + 1; /* terminating '\0' */ + + file = (ds_file_t *) my_malloc(sizeof(ds_file_t) + + sizeof(ds_local_file_t) + + path_len, + MYF(MY_FAE)); + local_file = (ds_local_file_t *) (file + 1); + + local_file->fd = fd; + + file->path = (char *) local_file + sizeof(ds_local_file_t); + memcpy(file->path, fullpath, path_len); + + file->ptr = local_file; + + return file; +} + +static +int +local_write(ds_file_t *file, const void *buf, size_t len) +{ + File fd = ((ds_local_file_t *) file->ptr)->fd; + + if (!my_write(fd, buf, len, MYF(MY_WME | MY_NABP))) { + posix_fadvise(fd, 0, 0, POSIX_FADV_DONTNEED); + return 0; + } + + return 1; +} + +static +int +local_close(ds_file_t *file) +{ + File fd = ((ds_local_file_t *) file->ptr)->fd; + + my_free(file); + + my_sync(fd, MYF(MY_WME)); + + return my_close(fd, MYF(MY_WME)); +} + +static +void +local_deinit(ds_ctxt_t *ctxt) +{ + my_free(ctxt->root); + my_free(ctxt); +} diff --git a/extra/mariabackup/ds_local.h b/extra/mariabackup/ds_local.h new file mode 100644 index 00000000000..b0f0f04030c --- /dev/null +++ b/extra/mariabackup/ds_local.h @@ -0,0 +1,28 @@ +/****************************************************** +Copyright (c) 2011-2013 Percona LLC and/or its affiliates. + +Local datasink interface for XtraBackup. + +This program is free software; you can redistribute it and/or modify +it under the terms of the GNU General Public License as published by +the Free Software Foundation; version 2 of the License. + +This program is distributed in the hope that it will be useful, +but WITHOUT ANY WARRANTY; without even the implied warranty of +MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the +GNU General Public License for more details. + +You should have received a copy of the GNU General Public License +along with this program; if not, write to the Free Software +Foundation, Inc., 51 Franklin Street, Fifth Floor, Boston, MA 02110-1301, USA + +*******************************************************/ + +#ifndef DS_LOCAL_H +#define DS_LOCAL_H + +#include "datasink.h" + +extern datasink_t datasink_local; + +#endif diff --git a/extra/mariabackup/ds_stdout.c b/extra/mariabackup/ds_stdout.c new file mode 100644 index 00000000000..616bcbd831e --- /dev/null +++ b/extra/mariabackup/ds_stdout.c @@ -0,0 +1,121 @@ +/****************************************************** +Copyright (c) 2013 Percona LLC and/or its affiliates. + +Local datasink implementation for XtraBackup. + +This program is free software; you can redistribute it and/or modify +it under the terms of the GNU General Public License as published by +the Free Software Foundation; version 2 of the License. + +This program is distributed in the hope that it will be useful, +but WITHOUT ANY WARRANTY; without even the implied warranty of +MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the +GNU General Public License for more details. + +You should have received a copy of the GNU General Public License +along with this program; if not, write to the Free Software +Foundation, Inc., 51 Franklin Street, Fifth Floor, Boston, MA 02110-1301, USA + +*******************************************************/ + +#include +#include +#include "common.h" +#include "datasink.h" + +typedef struct { + File fd; +} ds_stdout_file_t; + +static ds_ctxt_t *stdout_init(const char *root); +static ds_file_t *stdout_open(ds_ctxt_t *ctxt, const char *path, + MY_STAT *mystat); +static int stdout_write(ds_file_t *file, const void *buf, size_t len); +static int stdout_close(ds_file_t *file); +static void stdout_deinit(ds_ctxt_t *ctxt); + +datasink_t datasink_stdout = { + &stdout_init, + &stdout_open, + &stdout_write, + &stdout_close, + &stdout_deinit +}; + +static +ds_ctxt_t * +stdout_init(const char *root) +{ + ds_ctxt_t *ctxt; + + ctxt = my_malloc(sizeof(ds_ctxt_t), MYF(MY_FAE)); + + ctxt->root = my_strdup(root, MYF(MY_FAE)); + + return ctxt; +} + +static +ds_file_t * +stdout_open(ds_ctxt_t *ctxt __attribute__((unused)), + const char *path __attribute__((unused)), + MY_STAT *mystat __attribute__((unused))) +{ + ds_stdout_file_t *stdout_file; + ds_file_t *file; + size_t pathlen; + const char *fullpath = ""; + + pathlen = strlen(fullpath) + 1; + + file = (ds_file_t *) my_malloc(sizeof(ds_file_t) + + sizeof(ds_stdout_file_t) + + pathlen, + MYF(MY_FAE)); + stdout_file = (ds_stdout_file_t *) (file + 1); + + +#ifdef __WIN__ + setmode(fileno(stdout), _O_BINARY); +#endif + + stdout_file->fd = fileno(stdout); + + file->path = (char *) stdout_file + sizeof(ds_stdout_file_t); + memcpy(file->path, fullpath, pathlen); + + file->ptr = stdout_file; + + return file; +} + +static +int +stdout_write(ds_file_t *file, const void *buf, size_t len) +{ + File fd = ((ds_stdout_file_t *) file->ptr)->fd; + + if (!my_write(fd, buf, len, MYF(MY_WME | MY_NABP))) { + posix_fadvise(fd, 0, 0, POSIX_FADV_DONTNEED); + return 0; + } + + return 1; +} + +static +int +stdout_close(ds_file_t *file) +{ + my_free(file); + + return 1; +} + +static +void +stdout_deinit(ds_ctxt_t *ctxt) +{ + my_free(ctxt->root); + my_free(ctxt); +} diff --git a/extra/mariabackup/ds_stdout.h b/extra/mariabackup/ds_stdout.h new file mode 100644 index 00000000000..58940264fef --- /dev/null +++ b/extra/mariabackup/ds_stdout.h @@ -0,0 +1,28 @@ +/****************************************************** +Copyright (c) 2013 Percona LLC and/or its affiliates. + +Local datasink interface for XtraBackup. + +This program is free software; you can redistribute it and/or modify +it under the terms of the GNU General Public License as published by +the Free Software Foundation; version 2 of the License. + +This program is distributed in the hope that it will be useful, +but WITHOUT ANY WARRANTY; without even the implied warranty of +MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the +GNU General Public License for more details. + +You should have received a copy of the GNU General Public License +along with this program; if not, write to the Free Software +Foundation, Inc., 51 Franklin Street, Fifth Floor, Boston, MA 02110-1301, USA + +*******************************************************/ + +#ifndef DS_STDOUT_H +#define DS_STDOUT_H + +#include "datasink.h" + +extern datasink_t datasink_stdout; + +#endif diff --git a/extra/mariabackup/ds_tmpfile.c b/extra/mariabackup/ds_tmpfile.c new file mode 100644 index 00000000000..915191dcdae --- /dev/null +++ b/extra/mariabackup/ds_tmpfile.c @@ -0,0 +1,248 @@ +/****************************************************** +Copyright (c) 2012 Percona LLC and/or its affiliates. + +tmpfile datasink for XtraBackup. + +This program is free software; you can redistribute it and/or modify +it under the terms of the GNU General Public License as published by +the Free Software Foundation; version 2 of the License. + +This program is distributed in the hope that it will be useful, +but WITHOUT ANY WARRANTY; without even the implied warranty of +MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the +GNU General Public License for more details. + +You should have received a copy of the GNU General Public License +along with this program; if not, write to the Free Software +Foundation, Inc., 51 Franklin Street, Fifth Floor, Boston, MA 02110-1301, USA + +*******************************************************/ + +/* Do all writes to temporary files first, then pipe them to the specified +datasink in a serialized way in deinit(). */ + +#include +#include "common.h" +#include "datasink.h" + +typedef struct { + pthread_mutex_t mutex; + LIST *file_list; +} ds_tmpfile_ctxt_t; + +typedef struct { + LIST list; + File fd; + char *orig_path; + MY_STAT mystat; + ds_file_t *file; +} ds_tmp_file_t; + +static ds_ctxt_t *tmpfile_init(const char *root); +static ds_file_t *tmpfile_open(ds_ctxt_t *ctxt, const char *path, + MY_STAT *mystat); +static int tmpfile_write(ds_file_t *file, const void *buf, size_t len); +static int tmpfile_close(ds_file_t *file); +static void tmpfile_deinit(ds_ctxt_t *ctxt); + +datasink_t datasink_tmpfile = { + &tmpfile_init, + &tmpfile_open, + &tmpfile_write, + &tmpfile_close, + &tmpfile_deinit +}; + +MY_TMPDIR mysql_tmpdir_list; + +static ds_ctxt_t * +tmpfile_init(const char *root) +{ + ds_ctxt_t *ctxt; + ds_tmpfile_ctxt_t *tmpfile_ctxt; + + ctxt = my_malloc(sizeof(ds_ctxt_t) + sizeof(ds_tmpfile_ctxt_t), + MYF(MY_FAE)); + tmpfile_ctxt = (ds_tmpfile_ctxt_t *) (ctxt + 1); + tmpfile_ctxt->file_list = NULL; + if (pthread_mutex_init(&tmpfile_ctxt->mutex, NULL)) { + + my_free(ctxt); + return NULL; + } + + ctxt->ptr = tmpfile_ctxt; + ctxt->root = my_strdup(root, MYF(MY_FAE)); + + return ctxt; +} + +static ds_file_t * +tmpfile_open(ds_ctxt_t *ctxt, const char *path, + MY_STAT *mystat) +{ + ds_tmpfile_ctxt_t *tmpfile_ctxt; + char tmp_path[FN_REFLEN]; + ds_tmp_file_t *tmp_file; + ds_file_t *file; + size_t path_len; + File fd; + + /* Create a temporary file in tmpdir. The file will be automatically + removed on close. Code copied from mysql_tmpfile(). */ + fd = create_temp_file(tmp_path, my_tmpdir(&mysql_tmpdir_list), + "xbtemp", +#ifdef __WIN__ + O_BINARY | O_TRUNC | O_SEQUENTIAL | + O_TEMPORARY | O_SHORT_LIVED | +#endif /* __WIN__ */ + O_CREAT | O_EXCL | O_RDWR, + MYF(MY_WME)); + +#ifndef __WIN__ + if (fd >= 0) { + /* On Windows, open files cannot be removed, but files can be + created with the O_TEMPORARY flag to the same effect + ("delete on close"). */ + unlink(tmp_path); + } +#endif /* !__WIN__ */ + + if (fd < 0) { + return NULL; + } + + path_len = strlen(path) + 1; /* terminating '\0' */ + + file = (ds_file_t *) my_malloc(sizeof(ds_file_t) + + sizeof(ds_tmp_file_t) + path_len, + MYF(MY_FAE)); + + tmp_file = (ds_tmp_file_t *) (file + 1); + tmp_file->file = file; + memcpy(&tmp_file->mystat, mystat, sizeof(MY_STAT)); + /* Save a copy of 'path', since it may not be accessible later */ + tmp_file->orig_path = (char *) tmp_file + sizeof(ds_tmp_file_t); + + tmp_file->fd = fd; + memcpy(tmp_file->orig_path, path, path_len); + + /* Store the real temporary file name in file->path */ + file->path = my_strdup(tmp_path, MYF(MY_FAE)); + file->ptr = tmp_file; + + /* Store the file object in the list to be piped later */ + tmpfile_ctxt = (ds_tmpfile_ctxt_t *) ctxt->ptr; + tmp_file->list.data = tmp_file; + + pthread_mutex_lock(&tmpfile_ctxt->mutex); + tmpfile_ctxt->file_list = list_add(tmpfile_ctxt->file_list, + &tmp_file->list); + pthread_mutex_unlock(&tmpfile_ctxt->mutex); + + return file; +} + +static int +tmpfile_write(ds_file_t *file, const void *buf, size_t len) +{ + File fd = ((ds_tmp_file_t *) file->ptr)->fd; + + if (!my_write(fd, buf, len, MYF(MY_WME | MY_NABP))) { + posix_fadvise(fd, 0, 0, POSIX_FADV_DONTNEED); + return 0; + } + + return 1; +} + +static int +tmpfile_close(ds_file_t *file) +{ + /* Do nothing -- we will close (and thus remove) the file after piping + it to the destination datasink in tmpfile_deinit(). */ + + my_free(file->path); + + return 0; +} + +static void +tmpfile_deinit(ds_ctxt_t *ctxt) +{ + LIST *list; + ds_tmpfile_ctxt_t *tmpfile_ctxt; + MY_STAT mystat; + ds_tmp_file_t *tmp_file; + ds_file_t *dst_file; + ds_ctxt_t *pipe_ctxt; + void *buf = NULL; + const size_t buf_size = 10 * 1024 * 1024; + size_t bytes; + size_t offset; + + pipe_ctxt = ctxt->pipe_ctxt; + xb_a(pipe_ctxt != NULL); + + buf = my_malloc(buf_size, MYF(MY_FAE)); + + tmpfile_ctxt = (ds_tmpfile_ctxt_t *) ctxt->ptr; + list = tmpfile_ctxt->file_list; + + /* Walk the files in the order they have been added */ + list = list_reverse(list); + while (list != NULL) { + tmp_file = list->data; + /* Stat the file to replace size and mtime on the original + * mystat struct */ + if (my_fstat(tmp_file->fd, &mystat, MYF(0))) { + msg("error: my_fstat() failed.\n"); + exit(EXIT_FAILURE); + } + tmp_file->mystat.st_size = mystat.st_size; + tmp_file->mystat.st_mtime = mystat.st_mtime; + + dst_file = ds_open(pipe_ctxt, tmp_file->orig_path, + &tmp_file->mystat); + if (dst_file == NULL) { + msg("error: could not stream a temporary file to " + "'%s'\n", tmp_file->orig_path); + exit(EXIT_FAILURE); + } + + /* copy to the destination datasink */ + posix_fadvise(tmp_file->fd, 0, 0, POSIX_FADV_SEQUENTIAL); + if (my_seek(tmp_file->fd, 0, SEEK_SET, MYF(0)) == + MY_FILEPOS_ERROR) { + msg("error: my_seek() failed for '%s', errno = %d.\n", + tmp_file->file->path, my_errno); + exit(EXIT_FAILURE); + } + offset = 0; + while ((bytes = my_read(tmp_file->fd, buf, buf_size, + MYF(MY_WME))) > 0) { + posix_fadvise(tmp_file->fd, offset, buf_size, POSIX_FADV_DONTNEED); + offset += buf_size; + if (ds_write(dst_file, buf, bytes)) { + msg("error: cannot write to stream for '%s'.\n", + tmp_file->orig_path); + exit(EXIT_FAILURE); + } + } + if (bytes == (size_t) -1) { + exit(EXIT_FAILURE); + } + + my_close(tmp_file->fd, MYF(MY_WME)); + ds_close(dst_file); + + list = list_rest(list); + my_free(tmp_file->file); + } + + pthread_mutex_destroy(&tmpfile_ctxt->mutex); + + my_free(buf); + my_free(ctxt->root); + my_free(ctxt); +} diff --git a/extra/mariabackup/ds_tmpfile.h b/extra/mariabackup/ds_tmpfile.h new file mode 100644 index 00000000000..c21f1a3f0b5 --- /dev/null +++ b/extra/mariabackup/ds_tmpfile.h @@ -0,0 +1,30 @@ +/****************************************************** +Copyright (c) 2012 Percona LLC and/or its affiliates. + +tmpfile datasink for XtraBackup. + +This program is free software; you can redistribute it and/or modify +it under the terms of the GNU General Public License as published by +the Free Software Foundation; version 2 of the License. + +This program is distributed in the hope that it will be useful, +but WITHOUT ANY WARRANTY; without even the implied warranty of +MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the +GNU General Public License for more details. + +You should have received a copy of the GNU General Public License +along with this program; if not, write to the Free Software +Foundation, Inc., 51 Franklin Street, Fifth Floor, Boston, MA 02110-1301, USA + +*******************************************************/ + +#ifndef DS_TMPFILE_H +#define DS_TMPFILE_H + +#include "datasink.h" + +extern datasink_t datasink_tmpfile; + +extern MY_TMPDIR mysql_tmpdir_list; + +#endif diff --git a/extra/mariabackup/ds_xbstream.c b/extra/mariabackup/ds_xbstream.c new file mode 100644 index 00000000000..42924a72d7f --- /dev/null +++ b/extra/mariabackup/ds_xbstream.c @@ -0,0 +1,223 @@ +/****************************************************** +Copyright (c) 2011-2013 Percona LLC and/or its affiliates. + +Streaming implementation for XtraBackup. + +This program is free software; you can redistribute it and/or modify +it under the terms of the GNU General Public License as published by +the Free Software Foundation; version 2 of the License. + +This program is distributed in the hope that it will be useful, +but WITHOUT ANY WARRANTY; without even the implied warranty of +MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the +GNU General Public License for more details. + +You should have received a copy of the GNU General Public License +along with this program; if not, write to the Free Software +Foundation, Inc., 51 Franklin Street, Fifth Floor, Boston, MA 02110-1301, USA + +*******************************************************/ + +#include +#include +#include "common.h" +#include "datasink.h" +#include "xbstream.h" + +typedef struct { + xb_wstream_t *xbstream; + ds_file_t *dest_file; + pthread_mutex_t mutex; +} ds_stream_ctxt_t; + +typedef struct { + xb_wstream_file_t *xbstream_file; + ds_stream_ctxt_t *stream_ctxt; +} ds_stream_file_t; + +/*********************************************************************** +General streaming interface */ + +static ds_ctxt_t *xbstream_init(const char *root); +static ds_file_t *xbstream_open(ds_ctxt_t *ctxt, const char *path, + MY_STAT *mystat); +static int xbstream_write(ds_file_t *file, const void *buf, size_t len); +static int xbstream_close(ds_file_t *file); +static void xbstream_deinit(ds_ctxt_t *ctxt); + +datasink_t datasink_xbstream = { + &xbstream_init, + &xbstream_open, + &xbstream_write, + &xbstream_close, + &xbstream_deinit +}; + +static +ssize_t +my_xbstream_write_callback(xb_wstream_file_t *f __attribute__((unused)), + void *userdata, const void *buf, size_t len) +{ + ds_stream_ctxt_t *stream_ctxt; + + stream_ctxt = (ds_stream_ctxt_t *) userdata; + + xb_ad(stream_ctxt != NULL); + xb_ad(stream_ctxt->dest_file != NULL); + + if (!ds_write(stream_ctxt->dest_file, buf, len)) { + return len; + } + return -1; +} + +static +ds_ctxt_t * +xbstream_init(const char *root __attribute__((unused))) +{ + ds_ctxt_t *ctxt; + ds_stream_ctxt_t *stream_ctxt; + xb_wstream_t *xbstream; + + ctxt = my_malloc(sizeof(ds_ctxt_t) + sizeof(ds_stream_ctxt_t), + MYF(MY_FAE)); + stream_ctxt = (ds_stream_ctxt_t *)(ctxt + 1); + + if (pthread_mutex_init(&stream_ctxt->mutex, NULL)) { + msg("xbstream_init: pthread_mutex_init() failed.\n"); + goto err; + } + + xbstream = xb_stream_write_new(); + if (xbstream == NULL) { + msg("xb_stream_write_new() failed.\n"); + goto err; + } + stream_ctxt->xbstream = xbstream; + stream_ctxt->dest_file = NULL; + + ctxt->ptr = stream_ctxt; + + return ctxt; + +err: + my_free(ctxt); + return NULL; +} + +static +ds_file_t * +xbstream_open(ds_ctxt_t *ctxt, const char *path, MY_STAT *mystat) +{ + ds_file_t *file; + ds_stream_file_t *stream_file; + ds_stream_ctxt_t *stream_ctxt; + ds_ctxt_t *dest_ctxt; + xb_wstream_t *xbstream; + xb_wstream_file_t *xbstream_file; + + + xb_ad(ctxt->pipe_ctxt != NULL); + dest_ctxt = ctxt->pipe_ctxt; + + stream_ctxt = (ds_stream_ctxt_t *) ctxt->ptr; + + pthread_mutex_lock(&stream_ctxt->mutex); + if (stream_ctxt->dest_file == NULL) { + stream_ctxt->dest_file = ds_open(dest_ctxt, path, mystat); + if (stream_ctxt->dest_file == NULL) { + return NULL; + } + } + pthread_mutex_unlock(&stream_ctxt->mutex); + + file = (ds_file_t *) my_malloc(sizeof(ds_file_t) + + sizeof(ds_stream_file_t), + MYF(MY_FAE)); + stream_file = (ds_stream_file_t *) (file + 1); + + xbstream = stream_ctxt->xbstream; + + xbstream_file = xb_stream_write_open(xbstream, path, mystat, + stream_ctxt, + my_xbstream_write_callback); + + if (xbstream_file == NULL) { + msg("xb_stream_write_open() failed.\n"); + goto err; + } + + stream_file->xbstream_file = xbstream_file; + stream_file->stream_ctxt = stream_ctxt; + file->ptr = stream_file; + file->path = stream_ctxt->dest_file->path; + + return file; + +err: + if (stream_ctxt->dest_file) { + ds_close(stream_ctxt->dest_file); + stream_ctxt->dest_file = NULL; + } + my_free(file); + + return NULL; +} + +static +int +xbstream_write(ds_file_t *file, const void *buf, size_t len) +{ + ds_stream_file_t *stream_file; + xb_wstream_file_t *xbstream_file; + + + stream_file = (ds_stream_file_t *) file->ptr; + + xbstream_file = stream_file->xbstream_file; + + if (xb_stream_write_data(xbstream_file, buf, len)) { + msg("xb_stream_write_data() failed.\n"); + return 1; + } + + return 0; +} + +static +int +xbstream_close(ds_file_t *file) +{ + ds_stream_file_t *stream_file; + int rc = 0; + + stream_file = (ds_stream_file_t *)file->ptr; + + rc = xb_stream_write_close(stream_file->xbstream_file); + + my_free(file); + + return rc; +} + +static +void +xbstream_deinit(ds_ctxt_t *ctxt) +{ + ds_stream_ctxt_t *stream_ctxt; + + stream_ctxt = (ds_stream_ctxt_t *) ctxt->ptr; + + if (xb_stream_write_done(stream_ctxt->xbstream)) { + msg("xb_stream_done() failed.\n"); + } + + if (stream_ctxt->dest_file) { + ds_close(stream_ctxt->dest_file); + stream_ctxt->dest_file = NULL; + } + + pthread_mutex_destroy(&stream_ctxt->mutex); + + my_free(ctxt); +} diff --git a/extra/mariabackup/ds_xbstream.h b/extra/mariabackup/ds_xbstream.h new file mode 100644 index 00000000000..30f34ac8318 --- /dev/null +++ b/extra/mariabackup/ds_xbstream.h @@ -0,0 +1,28 @@ +/****************************************************** +Copyright (c) 2011-2013 Percona LLC and/or its affiliates. + +Streaming interface for XtraBackup. + +This program is free software; you can redistribute it and/or modify +it under the terms of the GNU General Public License as published by +the Free Software Foundation; version 2 of the License. + +This program is distributed in the hope that it will be useful, +but WITHOUT ANY WARRANTY; without even the implied warranty of +MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the +GNU General Public License for more details. + +You should have received a copy of the GNU General Public License +along with this program; if not, write to the Free Software +Foundation, Inc., 51 Franklin Street, Fifth Floor, Boston, MA 02110-1301, USA + +*******************************************************/ + +#ifndef DS_XBSTREAM_H +#define DS_XBSTREAM_H + +#include "datasink.h" + +extern datasink_t datasink_xbstream; + +#endif diff --git a/extra/mariabackup/fil_cur.cc b/extra/mariabackup/fil_cur.cc new file mode 100644 index 00000000000..22ebdb90215 --- /dev/null +++ b/extra/mariabackup/fil_cur.cc @@ -0,0 +1,402 @@ +/****************************************************** +XtraBackup: hot backup tool for InnoDB +(c) 2009-2013 Percona LLC and/or its affiliates. +Originally Created 3/3/2009 Yasufumi Kinoshita +Written by Alexey Kopytov, Aleksandr Kuzminsky, Stewart Smith, Vadim Tkachenko, +Yasufumi Kinoshita, Ignacio Nin and Baron Schwartz. + +This program is free software; you can redistribute it and/or modify +it under the terms of the GNU General Public License as published by +the Free Software Foundation; version 2 of the License. + +This program is distributed in the hope that it will be useful, +but WITHOUT ANY WARRANTY; without even the implied warranty of +MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the +GNU General Public License for more details. + +You should have received a copy of the GNU General Public License +along with this program; if not, write to the Free Software +Foundation, Inc., 51 Franklin Street, Fifth Floor, Boston, MA 02110-1301, USA + +*******************************************************/ + +/* Source file cursor implementation */ + +#include + +#include +#include +#include +#include + +#include "fil_cur.h" +#include "common.h" +#include "read_filt.h" +#include "xtrabackup.h" + +/* Size of read buffer in pages (640 pages = 10M for 16K sized pages) */ +#define XB_FIL_CUR_PAGES 640 + +/*********************************************************************** +Extracts the relative path ("database/table.ibd") of a tablespace from a +specified possibly absolute path. + +For user tablespaces both "./database/table.ibd" and +"/remote/dir/database/table.ibd" result in "database/table.ibd". + +For system tablepsaces (i.e. When is_system is TRUE) both "/remote/dir/ibdata1" +and "./ibdata1" yield "ibdata1" in the output. */ +const char * +xb_get_relative_path( +/*=================*/ + const char* path, /*!< in: tablespace path (either + relative or absolute) */ + ibool is_system) /*!< in: TRUE for system tablespaces, + i.e. when only the filename must be + returned. */ +{ + const char *next; + const char *cur; + const char *prev; + + prev = NULL; + cur = path; + + while ((next = strchr(cur, SRV_PATH_SEPARATOR)) != NULL) { + + prev = cur; + cur = next + 1; + } + + if (is_system) { + + return(cur); + } else { + + return((prev == NULL) ? cur : prev); + } + +} + +/**********************************************************************//** +Closes a file. */ +static +void +xb_fil_node_close_file( +/*===================*/ + fil_node_t* node) /*!< in: file node */ +{ + ibool ret; + + mutex_enter(&fil_system->mutex); + + ut_ad(node); + ut_a(node->n_pending == 0); + ut_a(node->n_pending_flushes == 0); + ut_a(!node->being_extended); + + if (!node->open) { + + mutex_exit(&fil_system->mutex); + + return; + } + + ret = os_file_close(node->handle); + ut_a(ret); + + node->open = FALSE; + + ut_a(fil_system->n_open > 0); + fil_system->n_open--; + fil_n_file_opened--; + + if (node->space->purpose == FIL_TABLESPACE && + fil_is_user_tablespace_id(node->space->id)) { + + ut_a(UT_LIST_GET_LEN(fil_system->LRU) > 0); + + /* The node is in the LRU list, remove it */ + UT_LIST_REMOVE(LRU, fil_system->LRU, node); + } + + mutex_exit(&fil_system->mutex); +} + +/************************************************************************ +Open a source file cursor and initialize the associated read filter. + +@return XB_FIL_CUR_SUCCESS on success, XB_FIL_CUR_SKIP if the source file must +be skipped and XB_FIL_CUR_ERROR on error. */ +xb_fil_cur_result_t +xb_fil_cur_open( +/*============*/ + xb_fil_cur_t* cursor, /*!< out: source file cursor */ + xb_read_filt_t* read_filter, /*!< in/out: the read filter */ + fil_node_t* node, /*!< in: source tablespace node */ + uint thread_n) /*!< thread number for diagnostics */ +{ + ulint page_size; + ulint page_size_shift; + ulint zip_size; + ibool success; + + /* Initialize these first so xb_fil_cur_close() handles them correctly + in case of error */ + cursor->orig_buf = NULL; + cursor->node = NULL; + + cursor->space_id = node->space->id; + cursor->is_system = !fil_is_user_tablespace_id(node->space->id); + + strncpy(cursor->abs_path, node->name, sizeof(cursor->abs_path)); + + /* Get the relative path for the destination tablespace name, i.e. the + one that can be appended to the backup root directory. Non-system + tablespaces may have absolute paths for remote tablespaces in MySQL + 5.6+. We want to make "local" copies for the backup. */ + strncpy(cursor->rel_path, + xb_get_relative_path(cursor->abs_path, cursor->is_system), + sizeof(cursor->rel_path)); + + /* In the backup mode we should already have a tablespace handle created + by fil_load_single_table_tablespace() unless it is a system + tablespace. Otherwise we open the file here. */ + if (cursor->is_system || !srv_backup_mode || srv_close_files) { + node->handle = + os_file_create_simple_no_error_handling(0, node->name, + OS_FILE_OPEN, + OS_FILE_READ_ONLY, + &success); + if (!success) { + /* The following call prints an error message */ + os_file_get_last_error(TRUE); + + msg("[%02u] xtrabackup: error: cannot open " + "tablespace %s\n", + thread_n, cursor->abs_path); + + return(XB_FIL_CUR_ERROR); + } + mutex_enter(&fil_system->mutex); + + node->open = TRUE; + + fil_system->n_open++; + fil_n_file_opened++; + + if (node->space->purpose == FIL_TABLESPACE && + fil_is_user_tablespace_id(node->space->id)) { + + /* Put the node to the LRU list */ + UT_LIST_ADD_FIRST(LRU, fil_system->LRU, node); + } + + mutex_exit(&fil_system->mutex); + } + + ut_ad(node->open); + + cursor->node = node; + cursor->file = node->handle; + + if (my_fstat(cursor->file, &cursor->statinfo, MYF(MY_WME))) { + msg("[%02u] xtrabackup: error: cannot stat %s\n", + thread_n, cursor->abs_path); + + xb_fil_cur_close(cursor); + + return(XB_FIL_CUR_ERROR); + } + + if (srv_unix_file_flush_method == SRV_UNIX_O_DIRECT + || srv_unix_file_flush_method == SRV_UNIX_O_DIRECT_NO_FSYNC) { + + os_file_set_nocache(cursor->file, node->name, "OPEN"); + } + + posix_fadvise(cursor->file, 0, 0, POSIX_FADV_SEQUENTIAL); + + /* Determine the page size */ + zip_size = xb_get_zip_size(cursor->file); + if (zip_size == ULINT_UNDEFINED) { + xb_fil_cur_close(cursor); + return(XB_FIL_CUR_SKIP); + } else if (zip_size) { + page_size = zip_size; + page_size_shift = get_bit_shift(page_size); + msg("[%02u] %s is compressed with page size = " + "%lu bytes\n", thread_n, node->name, page_size); + if (page_size_shift < 10 || page_size_shift > 14) { + msg("[%02u] xtrabackup: Error: Invalid " + "page size: %lu.\n", thread_n, page_size); + ut_error; + } + } else { + page_size = UNIV_PAGE_SIZE; + page_size_shift = UNIV_PAGE_SIZE_SHIFT; + } + cursor->page_size = page_size; + cursor->page_size_shift = page_size_shift; + cursor->zip_size = zip_size; + + /* Allocate read buffer */ + cursor->buf_size = XB_FIL_CUR_PAGES * page_size; + cursor->orig_buf = static_cast + (ut_malloc(cursor->buf_size + UNIV_PAGE_SIZE)); + cursor->buf = static_cast + (ut_align(cursor->orig_buf, UNIV_PAGE_SIZE)); + + cursor->buf_read = 0; + cursor->buf_npages = 0; + cursor->buf_offset = 0; + cursor->buf_page_no = 0; + cursor->thread_n = thread_n; + + cursor->space_size = cursor->statinfo.st_size / page_size; + + cursor->read_filter = read_filter; + cursor->read_filter->init(&cursor->read_filter_ctxt, cursor, + node->space->id); + + return(XB_FIL_CUR_SUCCESS); +} + +/************************************************************************ +Reads and verifies the next block of pages from the source +file. Positions the cursor after the last read non-corrupted page. + +@return XB_FIL_CUR_SUCCESS if some have been read successfully, XB_FIL_CUR_EOF +if there are no more pages to read and XB_FIL_CUR_ERROR on error. */ +xb_fil_cur_result_t +xb_fil_cur_read( +/*============*/ + xb_fil_cur_t* cursor) /*!< in/out: source file cursor */ +{ + ibool success; + byte* page; + ulint i; + ulint npages; + ulint retry_count; + xb_fil_cur_result_t ret; + ib_int64_t offset; + ib_int64_t to_read; + + cursor->read_filter->get_next_batch(&cursor->read_filter_ctxt, + &offset, &to_read); + + if (to_read == 0LL) { + return(XB_FIL_CUR_EOF); + } + + if (to_read > (ib_int64_t) cursor->buf_size) { + to_read = (ib_int64_t) cursor->buf_size; + } + + xb_a(to_read > 0 && to_read <= 0xFFFFFFFFLL); + + if (to_read % cursor->page_size != 0 && + offset + to_read == cursor->statinfo.st_size) { + + if (to_read < (ib_int64_t) cursor->page_size) { + msg("[%02u] xtrabackup: Warning: junk at the end of " + "%s:\n", cursor->thread_n, cursor->abs_path); + msg("[%02u] xtrabackup: Warning: offset = %llu, " + "to_read = %llu\n", + cursor->thread_n, + (unsigned long long) offset, + (unsigned long long) to_read); + + return(XB_FIL_CUR_EOF); + } + + to_read = (ib_int64_t) (((ulint) to_read) & + ~(cursor->page_size - 1)); + } + + xb_a(to_read % cursor->page_size == 0); + + npages = (ulint) (to_read >> cursor->page_size_shift); + + retry_count = 10; + ret = XB_FIL_CUR_SUCCESS; + +read_retry: + xtrabackup_io_throttling(); + + cursor->buf_read = 0; + cursor->buf_npages = 0; + cursor->buf_offset = offset; + cursor->buf_page_no = (ulint) (offset >> cursor->page_size_shift); + + success = os_file_read(cursor->file, cursor->buf, offset, + to_read); + if (!success) { + return(XB_FIL_CUR_ERROR); + } + + /* check pages for corruption and re-read if necessary. i.e. in case of + partially written pages */ + for (page = cursor->buf, i = 0; i < npages; + page += cursor->page_size, i++) { + + if (buf_page_is_corrupted(TRUE, page, cursor->zip_size)) { + + ulint page_no = cursor->buf_page_no + i; + + if (cursor->is_system && + page_no >= FSP_EXTENT_SIZE && + page_no < FSP_EXTENT_SIZE * 3) { + /* skip doublewrite buffer pages */ + xb_a(cursor->page_size == UNIV_PAGE_SIZE); + msg("[%02u] xtrabackup: " + "Page %lu is a doublewrite buffer page, " + "skipping.\n", cursor->thread_n, page_no); + } else { + retry_count--; + if (retry_count == 0) { + msg("[%02u] xtrabackup: " + "Error: failed to read page after " + "10 retries. File %s seems to be " + "corrupted.\n", cursor->thread_n, + cursor->abs_path); + ret = XB_FIL_CUR_ERROR; + break; + } + msg("[%02u] xtrabackup: " + "Database page corruption detected at page " + "%lu, retrying...\n", cursor->thread_n, + page_no); + + os_thread_sleep(100000); + + goto read_retry; + } + } + cursor->buf_read += cursor->page_size; + cursor->buf_npages++; + } + + posix_fadvise(cursor->file, offset, to_read, POSIX_FADV_DONTNEED); + + return(ret); +} + +/************************************************************************ +Close the source file cursor opened with xb_fil_cur_open() and its +associated read filter. */ +void +xb_fil_cur_close( +/*=============*/ + xb_fil_cur_t *cursor) /*!< in/out: source file cursor */ +{ + cursor->read_filter->deinit(&cursor->read_filter_ctxt); + + if (cursor->orig_buf != NULL) { + ut_free(cursor->orig_buf); + } + if (cursor->node != NULL) { + xb_fil_node_close_file(cursor->node); + cursor->file = XB_FILE_UNDEFINED; + } +} diff --git a/extra/mariabackup/fil_cur.h b/extra/mariabackup/fil_cur.h new file mode 100644 index 00000000000..2057765dab5 --- /dev/null +++ b/extra/mariabackup/fil_cur.h @@ -0,0 +1,123 @@ +/****************************************************** +XtraBackup: hot backup tool for InnoDB +(c) 2009-2013 Percona LLC and/or its affiliates. +Originally Created 3/3/2009 Yasufumi Kinoshita +Written by Alexey Kopytov, Aleksandr Kuzminsky, Stewart Smith, Vadim Tkachenko, +Yasufumi Kinoshita, Ignacio Nin and Baron Schwartz. + +This program is free software; you can redistribute it and/or modify +it under the terms of the GNU General Public License as published by +the Free Software Foundation; version 2 of the License. + +This program is distributed in the hope that it will be useful, +but WITHOUT ANY WARRANTY; without even the implied warranty of +MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the +GNU General Public License for more details. + +You should have received a copy of the GNU General Public License +along with this program; if not, write to the Free Software +Foundation, Inc., 51 Franklin Street, Fifth Floor, Boston, MA 02110-1301, USA + +*******************************************************/ + +/* Source file cursor interface */ + +#ifndef FIL_CUR_H +#define FIL_CUR_H + +#include +#include "read_filt.h" + +struct xb_fil_cur_t { + os_file_t file; /*!< source file handle */ + fil_node_t* node; /*!< source tablespace node */ + char rel_path[FN_REFLEN]; + /*!< normalized file path */ + char abs_path[FN_REFLEN]; + /*!< absolute file path */ + MY_STAT statinfo; /*!< information about the file */ + ulint zip_size; /*!< compressed page size in bytes or 0 + for uncompressed pages */ + ulint page_size; /*!< = zip_size for compressed pages or + UNIV_PAGE_SIZE for uncompressed ones */ + ulint page_size_shift;/*!< bit shift corresponding to + page_size */ + my_bool is_system; /*!< TRUE for system tablespace, FALSE + otherwise */ + xb_read_filt_t* read_filter; /*!< read filter */ + xb_read_filt_ctxt_t read_filter_ctxt; + /*!< read filter context */ + byte* orig_buf; /*!< read buffer */ + byte* buf; /*!< aligned pointer for orig_buf */ + ulint buf_size; /*!< buffer size in bytes */ + ulint buf_read; /*!< number of read bytes in buffer + after the last cursor read */ + ulint buf_npages; /*!< number of pages in buffer after the + last cursor read */ + ib_int64_t buf_offset; /*!< file offset of the first page in + buffer */ + ulint buf_page_no; /*!< number of the first page in + buffer */ + uint thread_n; /*!< thread number for diagnostics */ + ulint space_id; /*!< ID of tablespace */ + ulint space_size; /*!< space size in pages */ +}; + +typedef enum { + XB_FIL_CUR_SUCCESS, + XB_FIL_CUR_SKIP, + XB_FIL_CUR_ERROR, + XB_FIL_CUR_EOF +} xb_fil_cur_result_t; + +/************************************************************************ +Open a source file cursor and initialize the associated read filter. + +@return XB_FIL_CUR_SUCCESS on success, XB_FIL_CUR_SKIP if the source file must +be skipped and XB_FIL_CUR_ERROR on error. */ +xb_fil_cur_result_t +xb_fil_cur_open( +/*============*/ + xb_fil_cur_t* cursor, /*!< out: source file cursor */ + xb_read_filt_t* read_filter, /*!< in/out: the read filter */ + fil_node_t* node, /*!< in: source tablespace node */ + uint thread_n); /*!< thread number for diagnostics */ + +/************************************************************************ +Reads and verifies the next block of pages from the source +file. Positions the cursor after the last read non-corrupted page. + +@return XB_FIL_CUR_SUCCESS if some have been read successfully, XB_FIL_CUR_EOF +if there are no more pages to read and XB_FIL_CUR_ERROR on error. */ +xb_fil_cur_result_t +xb_fil_cur_read( +/*============*/ + xb_fil_cur_t* cursor); /*!< in/out: source file cursor */ + +/************************************************************************ +Close the source file cursor opened with xb_fil_cur_open() and its +associated read filter. */ +void +xb_fil_cur_close( +/*=============*/ + xb_fil_cur_t *cursor); /*!< in/out: source file cursor */ + +/*********************************************************************** +Extracts the relative path ("database/table.ibd") of a tablespace from a +specified possibly absolute path. + +For user tablespaces both "./database/table.ibd" and +"/remote/dir/database/table.ibd" result in "database/table.ibd". + +For system tablepsaces (i.e. When is_system is TRUE) both "/remote/dir/ibdata1" +and "./ibdata1" yield "ibdata1" in the output. */ +const char * +xb_get_relative_path( +/*=================*/ + const char* path, /*!< in: tablespace path (either + relative or absolute) */ + ibool is_system); /*!< in: TRUE for system tablespaces, + i.e. when only the filename must be + returned. */ + +#endif diff --git a/extra/mariabackup/innobackupex.cc b/extra/mariabackup/innobackupex.cc new file mode 100644 index 00000000000..ec697bb56ff --- /dev/null +++ b/extra/mariabackup/innobackupex.cc @@ -0,0 +1,1153 @@ +/****************************************************** +hot backup tool for InnoDB +(c) 2009-2015 Percona LLC and/or its affiliates +Originally Created 3/3/2009 Yasufumi Kinoshita +Written by Alexey Kopytov, Aleksandr Kuzminsky, Stewart Smith, Vadim Tkachenko, +Yasufumi Kinoshita, Ignacio Nin and Baron Schwartz. + +This program is free software; you can redistribute it and/or modify +it under the terms of the GNU General Public License as published by +the Free Software Foundation; version 2 of the License. + +This program is distributed in the hope that it will be useful, +but WITHOUT ANY WARRANTY; without even the implied warranty of +MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the +GNU General Public License for more details. + +You should have received a copy of the GNU General Public License +along with this program; if not, write to the Free Software +Foundation, Inc., 51 Franklin Street, Fifth Floor, Boston, MA 02110-1301, USA + +******************************************************* + +This file incorporates work covered by the following copyright and +permission notice: + +Copyright (c) 2000, 2011, MySQL AB & Innobase Oy. All Rights Reserved. + +This program is free software; you can redistribute it and/or modify it under +the terms of the GNU General Public License as published by the Free Software +Foundation; version 2 of the License. + +This program is distributed in the hope that it will be useful, but WITHOUT +ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or FITNESS +FOR A PARTICULAR PURPOSE. See the GNU General Public License for more details. + +You should have received a copy of the GNU General Public License along with +this program; if not, write to the Free Software Foundation, Inc., 59 Temple +Place, Suite 330, Boston, MA 02111-1307 USA + +*******************************************************/ + +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include "common.h" +#include "innobackupex.h" +#include "xtrabackup.h" +#include "xtrabackup_version.h" +#include "xbstream.h" +#include "fil_cur.h" +#include "write_filt.h" +#include "backup_copy.h" + +using std::min; +using std::max; + +/* options */ +my_bool opt_ibx_version = FALSE; +my_bool opt_ibx_help = FALSE; +my_bool opt_ibx_apply_log = FALSE; +my_bool opt_ibx_redo_only = FALSE; +my_bool opt_ibx_incremental = FALSE; +my_bool opt_ibx_notimestamp = FALSE; + +my_bool opt_ibx_copy_back = FALSE; +my_bool opt_ibx_move_back = FALSE; +my_bool opt_ibx_galera_info = FALSE; +my_bool opt_ibx_slave_info = FALSE; +my_bool opt_ibx_no_lock = FALSE; +my_bool opt_ibx_safe_slave_backup = FALSE; +my_bool opt_ibx_rsync = FALSE; +my_bool opt_ibx_force_non_empty_dirs = FALSE; +my_bool opt_ibx_noversioncheck = FALSE; +my_bool opt_ibx_no_backup_locks = FALSE; +my_bool opt_ibx_decompress = FALSE; + +char *opt_ibx_incremental_history_name = NULL; +char *opt_ibx_incremental_history_uuid = NULL; + +char *opt_ibx_user = NULL; +char *opt_ibx_password = NULL; +char *opt_ibx_host = NULL; +char *opt_ibx_defaults_group = NULL; +char *opt_ibx_socket = NULL; +uint opt_ibx_port = 0; +char *opt_ibx_login_path = NULL; + + +ulong opt_ibx_lock_wait_query_type; +ulong opt_ibx_kill_long_query_type; + +ulong opt_ibx_decrypt_algo = 0; + +uint opt_ibx_kill_long_queries_timeout = 0; +uint opt_ibx_lock_wait_timeout = 0; +uint opt_ibx_lock_wait_threshold = 0; +uint opt_ibx_debug_sleep_before_unlock = 0; +uint opt_ibx_safe_slave_backup_timeout = 0; + +const char *opt_ibx_history = NULL; +bool opt_ibx_decrypt = false; + +char *opt_ibx_include = NULL; +char *opt_ibx_databases = NULL; +bool ibx_partial_backup = false; + +char *ibx_position_arg = NULL; +char *ibx_backup_directory = NULL; + +/* copy of proxied xtrabackup options */ +my_bool ibx_xb_close_files; +my_bool ibx_xtrabackup_compact; +const char *ibx_xtrabackup_compress_alg; +uint ibx_xtrabackup_compress_threads; +ulonglong ibx_xtrabackup_compress_chunk_size; +ulong ibx_xtrabackup_encrypt_algo; +char *ibx_xtrabackup_encrypt_key; +char *ibx_xtrabackup_encrypt_key_file; +uint ibx_xtrabackup_encrypt_threads; +ulonglong ibx_xtrabackup_encrypt_chunk_size; +my_bool ibx_xtrabackup_export; +char *ibx_xtrabackup_extra_lsndir; +char *ibx_xtrabackup_incremental_basedir; +char *ibx_xtrabackup_incremental_dir; +my_bool ibx_xtrabackup_incremental_force_scan; +ulint ibx_xtrabackup_log_copy_interval; +char *ibx_xtrabackup_incremental; +int ibx_xtrabackup_parallel; +my_bool ibx_xtrabackup_rebuild_indexes; +ulint ibx_xtrabackup_rebuild_threads; +char *ibx_xtrabackup_stream_str; +char *ibx_xtrabackup_tables_file; +long ibx_xtrabackup_throttle; +char *ibx_opt_mysql_tmpdir; +longlong ibx_xtrabackup_use_memory; + + +static inline int ibx_msg(const char *fmt, ...) ATTRIBUTE_FORMAT(printf, 1, 2); +static inline int ibx_msg(const char *fmt, ...) +{ + int result; + time_t t = time(NULL); + char date[100]; + char *line; + va_list args; + + strftime(date, sizeof(date), "%y%m%d %H:%M:%S", localtime(&t)); + + va_start(args, fmt); + + result = vasprintf(&line, fmt, args); + + va_end(args); + + if (result != -1) { + result = fprintf(stderr, "%s %s: %s", + date, INNOBACKUPEX_BIN_NAME, line); + free(line); + } + + return result; +} + +enum innobackupex_options +{ + OPT_APPLY_LOG = 256, + OPT_COPY_BACK, + OPT_MOVE_BACK, + OPT_REDO_ONLY, + OPT_GALERA_INFO, + OPT_SLAVE_INFO, + OPT_INCREMENTAL, + OPT_INCREMENTAL_HISTORY_NAME, + OPT_INCREMENTAL_HISTORY_UUID, + OPT_LOCK_WAIT_QUERY_TYPE, + OPT_KILL_LONG_QUERY_TYPE, + OPT_KILL_LONG_QUERIES_TIMEOUT, + OPT_LOCK_WAIT_TIMEOUT, + OPT_LOCK_WAIT_THRESHOLD, + OPT_DEBUG_SLEEP_BEFORE_UNLOCK, + OPT_NO_LOCK, + OPT_SAFE_SLAVE_BACKUP, + OPT_SAFE_SLAVE_BACKUP_TIMEOUT, + OPT_RSYNC, + OPT_HISTORY, + OPT_INCLUDE, + OPT_FORCE_NON_EMPTY_DIRS, + OPT_NO_TIMESTAMP, + OPT_NO_VERSION_CHECK, + OPT_NO_BACKUP_LOCKS, + OPT_DATABASES, + OPT_DECRYPT, + OPT_DECOMPRESS, + + /* options wich are passed directly to xtrabackup */ + OPT_CLOSE_FILES, + OPT_COMPACT, + OPT_COMPRESS, + OPT_COMPRESS_THREADS, + OPT_COMPRESS_CHUNK_SIZE, + OPT_ENCRYPT, + OPT_ENCRYPT_KEY, + OPT_ENCRYPT_KEY_FILE, + OPT_ENCRYPT_THREADS, + OPT_ENCRYPT_CHUNK_SIZE, + OPT_EXPORT, + OPT_EXTRA_LSNDIR, + OPT_INCREMENTAL_BASEDIR, + OPT_INCREMENTAL_DIR, + OPT_INCREMENTAL_FORCE_SCAN, + OPT_LOG_COPY_INTERVAL, + OPT_PARALLEL, + OPT_REBUILD_INDEXES, + OPT_REBUILD_THREADS, + OPT_STREAM, + OPT_TABLES_FILE, + OPT_THROTTLE, + OPT_USE_MEMORY +}; + +ibx_mode_t ibx_mode = IBX_MODE_BACKUP; + +static struct my_option ibx_long_options[] = +{ + {"version", 'v', "print xtrabackup version information", + (uchar *) &opt_ibx_version, (uchar *) &opt_ibx_version, 0, + GET_BOOL, NO_ARG, 0, 0, 0, 0, 0, 0}, + + {"help", '?', "This option displays a help screen and exits.", + (uchar *) &opt_ibx_help, (uchar *) &opt_ibx_help, 0, + GET_BOOL, NO_ARG, 0, 0, 0, 0, 0, 0}, + + {"apply-log", OPT_APPLY_LOG, "Prepare a backup in BACKUP-DIR by " + "applying the transaction log file named \"xtrabackup_logfile\" " + "located in the same directory. Also, create new transaction logs. " + "The InnoDB configuration is read from the file \"backup-my.cnf\".", + (uchar*) &opt_ibx_apply_log, (uchar*) &opt_ibx_apply_log, + 0, GET_BOOL, NO_ARG, 0, 0, 0, 0, 0, 0}, + + {"redo-only", OPT_REDO_ONLY, "This option should be used when " + "preparing the base full backup and when merging all incrementals " + "except the last one. This forces xtrabackup to skip the \"rollback\" " + "phase and do a \"redo\" only. This is necessary if the backup will " + "have incremental changes applied to it later. See the xtrabackup " + "documentation for details.", + (uchar *) &opt_ibx_redo_only, (uchar *) &opt_ibx_redo_only, 0, + GET_BOOL, NO_ARG, 0, 0, 0, 0, 0, 0}, + + {"copy-back", OPT_COPY_BACK, "Copy all the files in a previously made " + "backup from the backup directory to their original locations.", + (uchar *) &opt_ibx_copy_back, (uchar *) &opt_ibx_copy_back, 0, + GET_BOOL, NO_ARG, 0, 0, 0, 0, 0, 0}, + + {"move-back", OPT_MOVE_BACK, "Move all the files in a previously made " + "backup from the backup directory to the actual datadir location. " + "Use with caution, as it removes backup files.", + (uchar *) &opt_ibx_move_back, (uchar *) &opt_ibx_move_back, 0, + GET_BOOL, NO_ARG, 0, 0, 0, 0, 0, 0}, + + {"galera-info", OPT_GALERA_INFO, "This options creates the " + "xtrabackup_galera_info file which contains the local node state at " + "the time of the backup. Option should be used when performing the " + "backup of Percona-XtraDB-Cluster. Has no effect when backup locks " + "are used to create the backup.", + (uchar *) &opt_ibx_galera_info, (uchar *) &opt_ibx_galera_info, 0, + GET_BOOL, NO_ARG, 0, 0, 0, 0, 0, 0}, + + {"slave-info", OPT_SLAVE_INFO, "This option is useful when backing " + "up a replication slave server. It prints the binary log position " + "and name of the master server. It also writes this information to " + "the \"xtrabackup_slave_info\" file as a \"CHANGE MASTER\" command. " + "A new slave for this master can be set up by starting a slave server " + "on this backup and issuing a \"CHANGE MASTER\" command with the " + "binary log position saved in the \"xtrabackup_slave_info\" file.", + (uchar *) &opt_ibx_slave_info, (uchar *) &opt_ibx_slave_info, 0, + GET_BOOL, NO_ARG, 0, 0, 0, 0, 0, 0}, + + {"incremental", OPT_INCREMENTAL, "This option tells xtrabackup to " + "create an incremental backup, rather than a full one. It is passed " + "to the xtrabackup child process. When this option is specified, " + "either --incremental-lsn or --incremental-basedir can also be given. " + "If neither option is given, option --incremental-basedir is passed " + "to xtrabackup by default, set to the first timestamped backup " + "directory in the backup base directory.", + (uchar *) &opt_ibx_incremental, (uchar *) &opt_ibx_incremental, 0, + GET_BOOL, NO_ARG, 0, 0, 0, 0, 0, 0}, + + {"no-lock", OPT_NO_LOCK, "Use this option to disable table lock " + "with \"FLUSH TABLES WITH READ LOCK\". Use it only if ALL your " + "tables are InnoDB and you DO NOT CARE about the binary log " + "position of the backup. This option shouldn't be used if there " + "are any DDL statements being executed or if any updates are " + "happening on non-InnoDB tables (this includes the system MyISAM " + "tables in the mysql database), otherwise it could lead to an " + "inconsistent backup. If you are considering to use --no-lock " + "because your backups are failing to acquire the lock, this could " + "be because of incoming replication events preventing the lock " + "from succeeding. Please try using --safe-slave-backup to " + "momentarily stop the replication slave thread, this may help " + "the backup to succeed and you then don't need to resort to " + "using this option.", + (uchar *) &opt_ibx_no_lock, (uchar *) &opt_ibx_no_lock, 0, + GET_BOOL, NO_ARG, 0, 0, 0, 0, 0, 0}, + + {"safe-slave-backup", OPT_SAFE_SLAVE_BACKUP, "Stop slave SQL thread " + "and wait to start backup until Slave_open_temp_tables in " + "\"SHOW STATUS\" is zero. If there are no open temporary tables, " + "the backup will take place, otherwise the SQL thread will be " + "started and stopped until there are no open temporary tables. " + "The backup will fail if Slave_open_temp_tables does not become " + "zero after --safe-slave-backup-timeout seconds. The slave SQL " + "thread will be restarted when the backup finishes.", + (uchar *) &opt_ibx_safe_slave_backup, + (uchar *) &opt_ibx_safe_slave_backup, + 0, GET_BOOL, NO_ARG, 0, 0, 0, 0, 0, 0}, + + {"rsync", OPT_RSYNC, "Uses the rsync utility to optimize local file " + "transfers. When this option is specified, innobackupex uses rsync " + "to copy all non-InnoDB files instead of spawning a separate cp for " + "each file, which can be much faster for servers with a large number " + "of databases or tables. This option cannot be used together with " + "--stream.", + (uchar *) &opt_ibx_rsync, (uchar *) &opt_ibx_rsync, + 0, GET_BOOL, NO_ARG, 0, 0, 0, 0, 0, 0}, + + {"force-non-empty-directories", OPT_FORCE_NON_EMPTY_DIRS, "This " + "option, when specified, makes --copy-back or --move-back transfer " + "files to non-empty directories. Note that no existing files will be " + "overwritten. If --copy-back or --nove-back has to copy a file from " + "the backup directory which already exists in the destination " + "directory, it will still fail with an error.", + (uchar *) &opt_ibx_force_non_empty_dirs, + (uchar *) &opt_ibx_force_non_empty_dirs, + 0, GET_BOOL, NO_ARG, 0, 0, 0, 0, 0, 0}, + + {"no-timestamp", OPT_NO_TIMESTAMP, "This option prevents creation of a " + "time-stamped subdirectory of the BACKUP-ROOT-DIR given on the " + "command line. When it is specified, the backup is done in " + "BACKUP-ROOT-DIR instead.", + (uchar *) &opt_ibx_notimestamp, + (uchar *) &opt_ibx_notimestamp, + 0, GET_BOOL, NO_ARG, 0, 0, 0, 0, 0, 0}, + + {"no-version-check", OPT_NO_VERSION_CHECK, "This option disables the " + "version check which is enabled by the --version-check option.", + (uchar *) &opt_ibx_noversioncheck, + (uchar *) &opt_ibx_noversioncheck, + 0, GET_BOOL, NO_ARG, 0, 0, 0, 0, 0, 0}, + + {"no-backup-locks", OPT_NO_BACKUP_LOCKS, "This option controls if " + "backup locks should be used instead of FLUSH TABLES WITH READ LOCK " + "on the backup stage. The option has no effect when backup locks are " + "not supported by the server. This option is enabled by default, " + "disable with --no-backup-locks.", + (uchar *) &opt_ibx_no_backup_locks, + (uchar *) &opt_ibx_no_backup_locks, + 0, GET_BOOL, NO_ARG, 0, 0, 0, 0, 0, 0}, + + {"decompress", OPT_DECOMPRESS, "Decompresses all files with the .qp " + "extension in a backup previously made with the --compress option.", + (uchar *) &opt_ibx_decompress, + (uchar *) &opt_ibx_decompress, + 0, GET_BOOL, NO_ARG, 0, 0, 0, 0, 0, 0}, + + {"user", 'u', "This option specifies the MySQL username used " + "when connecting to the server, if that's not the current user. " + "The option accepts a string argument. See mysql --help for details.", + (uchar*) &opt_ibx_user, (uchar*) &opt_ibx_user, 0, GET_STR, + REQUIRED_ARG, 0, 0, 0, 0, 0, 0}, + + {"host", 'H', "This option specifies the host to use when " + "connecting to the database server with TCP/IP. The option accepts " + "a string argument. See mysql --help for details.", + (uchar*) &opt_ibx_host, (uchar*) &opt_ibx_host, 0, GET_STR, + REQUIRED_ARG, 0, 0, 0, 0, 0, 0}, + + {"port", 'P', "This option specifies the port to use when " + "connecting to the database server with TCP/IP. The option accepts " + "a string argument. See mysql --help for details.", + &opt_ibx_port, &opt_ibx_port, 0, GET_UINT, REQUIRED_ARG, + 0, 0, 0, 0, 0, 0}, + + {"password", 'p', "This option specifies the password to use " + "when connecting to the database. It accepts a string argument. " + "See mysql --help for details.", + 0, 0, 0, GET_STR, + REQUIRED_ARG, 0, 0, 0, 0, 0, 0}, + + {"socket", 'S', "This option specifies the socket to use when " + "connecting to the local database server with a UNIX domain socket. " + "The option accepts a string argument. See mysql --help for details.", + (uchar*) &opt_ibx_socket, (uchar*) &opt_ibx_socket, 0, GET_STR, + REQUIRED_ARG, 0, 0, 0, 0, 0, 0}, + + {"incremental-history-name", OPT_INCREMENTAL_HISTORY_NAME, + "This option specifies the name of the backup series stored in the " + "PERCONA_SCHEMA.xtrabackup_history history record to base an " + "incremental backup on. Xtrabackup will search the history table " + "looking for the most recent (highest innodb_to_lsn), successful " + "backup in the series and take the to_lsn value to use as the " + "starting lsn for the incremental backup. This will be mutually " + "exclusive with --incremental-history-uuid, --incremental-basedir " + "and --incremental-lsn. If no valid lsn can be found (no series by " + "that name, no successful backups by that name) xtrabackup will " + "return with an error. It is used with the --incremental option.", + (uchar*) &opt_ibx_incremental_history_name, + (uchar*) &opt_ibx_incremental_history_name, 0, GET_STR, + REQUIRED_ARG, 0, 0, 0, 0, 0, 0}, + + {"incremental-history-uuid", OPT_INCREMENTAL_HISTORY_UUID, + "This option specifies the UUID of the specific history record " + "stored in the PERCONA_SCHEMA.xtrabackup_history to base an " + "incremental backup on. --incremental-history-name, " + "--incremental-basedir and --incremental-lsn. If no valid lsn can be " + "found (no success record with that uuid) xtrabackup will return " + "with an error. It is used with the --incremental option.", + (uchar*) &opt_ibx_incremental_history_uuid, + (uchar*) &opt_ibx_incremental_history_uuid, 0, GET_STR, + REQUIRED_ARG, 0, 0, 0, 0, 0, 0}, + + {"decrypt", OPT_DECRYPT, "Decrypts all files with the .xbcrypt " + "extension in a backup previously made with --encrypt option.", + &opt_ibx_decrypt_algo, &opt_ibx_decrypt_algo, + &xtrabackup_encrypt_algo_typelib, GET_ENUM, REQUIRED_ARG, + 0, 0, 0, 0, 0, 0}, + + {"ftwrl-wait-query-type", OPT_LOCK_WAIT_QUERY_TYPE, + "This option specifies which types of queries are allowed to complete " + "before innobackupex will issue the global lock. Default is all.", + (uchar*) &opt_ibx_lock_wait_query_type, + (uchar*) &opt_ibx_lock_wait_query_type, &query_type_typelib, + GET_ENUM, REQUIRED_ARG, QUERY_TYPE_ALL, 0, 0, 0, 0, 0}, + + {"kill-long-query-type", OPT_KILL_LONG_QUERY_TYPE, + "This option specifies which types of queries should be killed to " + "unblock the global lock. Default is \"all\".", + (uchar*) &opt_ibx_kill_long_query_type, + (uchar*) &opt_ibx_kill_long_query_type, &query_type_typelib, + GET_ENUM, REQUIRED_ARG, QUERY_TYPE_SELECT, 0, 0, 0, 0, 0}, + + {"history", OPT_HISTORY, + "This option enables the tracking of backup history in the " + "PERCONA_SCHEMA.xtrabackup_history table. An optional history " + "series name may be specified that will be placed with the history " + "record for the current backup being taken.", + NULL, NULL, 0, GET_STR, OPT_ARG, 0, 0, 0, 0, 0, 0}, + + {"include", OPT_INCLUDE, + "This option is a regular expression to be matched against table " + "names in databasename.tablename format. It is passed directly to " + "xtrabackup's --tables option. See the xtrabackup documentation for " + "details.", + (uchar*) &opt_ibx_include, + (uchar*) &opt_ibx_include, 0, GET_STR, + REQUIRED_ARG, 0, 0, 0, 0, 0, 0}, + + {"databases", OPT_DATABASES, + "This option specifies the list of databases that innobackupex should " + "back up. The option accepts a string argument or path to file that " + "contains the list of databases to back up. The list is of the form " + "\"databasename1[.table_name1] databasename2[.table_name2] . . .\". " + "If this option is not specified, all databases containing MyISAM and " + "InnoDB tables will be backed up. Please make sure that --databases " + "contains all of the InnoDB databases and tables, so that all of the " + "innodb.frm files are also backed up. In case the list is very long, " + "this can be specified in a file, and the full path of the file can " + "be specified instead of the list. (See option --tables-file.)", + (uchar*) &opt_ibx_databases, + (uchar*) &opt_ibx_databases, 0, GET_STR, + REQUIRED_ARG, 0, 0, 0, 0, 0, 0}, + + {"kill-long-queries-timeout", OPT_KILL_LONG_QUERIES_TIMEOUT, + "This option specifies the number of seconds innobackupex waits " + "between starting FLUSH TABLES WITH READ LOCK and killing those " + "queries that block it. Default is 0 seconds, which means " + "innobackupex will not attempt to kill any queries.", + (uchar*) &opt_ibx_kill_long_queries_timeout, + (uchar*) &opt_ibx_kill_long_queries_timeout, 0, GET_UINT, + REQUIRED_ARG, 0, 0, 0, 0, 0, 0}, + + {"ftwrl-wait-timeout", OPT_LOCK_WAIT_TIMEOUT, + "This option specifies time in seconds that innobackupex should wait " + "for queries that would block FTWRL before running it. If there are " + "still such queries when the timeout expires, innobackupex terminates " + "with an error. Default is 0, in which case innobackupex does not " + "wait for queries to complete and starts FTWRL immediately.", + (uchar*) &opt_ibx_lock_wait_timeout, + (uchar*) &opt_ibx_lock_wait_timeout, 0, GET_UINT, + REQUIRED_ARG, 0, 0, 0, 0, 0, 0}, + + {"ftwrl-wait-threshold", OPT_LOCK_WAIT_THRESHOLD, + "This option specifies the query run time threshold which is used by " + "innobackupex to detect long-running queries with a non-zero value " + "of --ftwrl-wait-timeout. FTWRL is not started until such " + "long-running queries exist. This option has no effect if " + "--ftwrl-wait-timeout is 0. Default value is 60 seconds.", + (uchar*) &opt_ibx_lock_wait_threshold, + (uchar*) &opt_ibx_lock_wait_threshold, 0, GET_UINT, + REQUIRED_ARG, 60, 0, 0, 0, 0, 0}, + + {"debug-sleep-before-unlock", OPT_DEBUG_SLEEP_BEFORE_UNLOCK, + "This is a debug-only option used by the XtraBackup test suite.", + (uchar*) &opt_ibx_debug_sleep_before_unlock, + (uchar*) &opt_ibx_debug_sleep_before_unlock, 0, GET_UINT, + REQUIRED_ARG, 0, 0, 0, 0, 0, 0}, + + {"safe-slave-backup-timeout", OPT_SAFE_SLAVE_BACKUP_TIMEOUT, + "How many seconds --safe-slave-backup should wait for " + "Slave_open_temp_tables to become zero. (default 300)", + (uchar*) &opt_ibx_safe_slave_backup_timeout, + (uchar*) &opt_ibx_safe_slave_backup_timeout, 0, GET_UINT, + REQUIRED_ARG, 300, 0, 0, 0, 0, 0}, + + + /* Following command-line options are actually handled by xtrabackup. + We put them here with only purpose for them to showup in + innobackupex --help output */ + + {"close_files", OPT_CLOSE_FILES, "Do not keep files opened. This " + "option is passed directly to xtrabackup. Use at your own risk.", + (uchar*) &ibx_xb_close_files, (uchar*) &ibx_xb_close_files, 0, + GET_BOOL, NO_ARG, 0, 0, 0, 0, 0, 0}, + + {"compact", OPT_COMPACT, "Create a compact backup with all secondary " + "index pages omitted. This option is passed directly to xtrabackup. " + "See xtrabackup documentation for details.", + (uchar*) &ibx_xtrabackup_compact, (uchar*) &ibx_xtrabackup_compact, + 0, GET_BOOL, NO_ARG, 0, 0, 0, 0, 0, 0}, + + {"compress", OPT_COMPRESS, "This option instructs xtrabackup to " + "compress backup copies of InnoDB data files. It is passed directly " + "to the xtrabackup child process. Try 'xtrabackup --help' for more " + "details.", (uchar*) &ibx_xtrabackup_compress_alg, + (uchar*) &ibx_xtrabackup_compress_alg, 0, + GET_STR, OPT_ARG, 0, 0, 0, 0, 0, 0}, + + {"compress-threads", OPT_COMPRESS_THREADS, + "This option specifies the number of worker threads that will be used " + "for parallel compression. It is passed directly to the xtrabackup " + "child process. Try 'xtrabackup --help' for more details.", + (uchar*) &ibx_xtrabackup_compress_threads, + (uchar*) &ibx_xtrabackup_compress_threads, + 0, GET_UINT, REQUIRED_ARG, 1, 1, UINT_MAX, 0, 0, 0}, + + {"compress-chunk-size", OPT_COMPRESS_CHUNK_SIZE, "Size of working " + "buffer(s) for compression threads in bytes. The default value " + "is 64K.", (uchar*) &ibx_xtrabackup_compress_chunk_size, + (uchar*) &ibx_xtrabackup_compress_chunk_size, + 0, GET_ULL, REQUIRED_ARG, (1 << 16), 1024, ULONGLONG_MAX, 0, 0, 0}, + + {"encrypt", OPT_ENCRYPT, "This option instructs xtrabackup to encrypt " + "backup copies of InnoDB data files using the algorithm specified in " + "the ENCRYPTION-ALGORITHM. It is passed directly to the xtrabackup " + "child process. Try 'xtrabackup --help' for more details.", + &ibx_xtrabackup_encrypt_algo, &ibx_xtrabackup_encrypt_algo, + &xtrabackup_encrypt_algo_typelib, GET_ENUM, REQUIRED_ARG, + 0, 0, 0, 0, 0, 0}, + + {"encrypt-key", OPT_ENCRYPT_KEY, "This option instructs xtrabackup to " + "use the given ENCRYPTION-KEY when using the --encrypt or --decrypt " + "options. During backup it is passed directly to the xtrabackup child " + "process. Try 'xtrabackup --help' for more details.", + (uchar*) &ibx_xtrabackup_encrypt_key, + (uchar*) &ibx_xtrabackup_encrypt_key, 0, + GET_STR_ALLOC, REQUIRED_ARG, 0, 0, 0, 0, 0, 0}, + + {"encrypt-key-file", OPT_ENCRYPT_KEY_FILE, "This option instructs " + "xtrabackup to use the encryption key stored in the given " + "ENCRYPTION-KEY-FILE when using the --encrypt or --decrypt options.", + (uchar*) &ibx_xtrabackup_encrypt_key_file, + (uchar*) &ibx_xtrabackup_encrypt_key_file, 0, + GET_STR_ALLOC, REQUIRED_ARG, 0, 0, 0, 0, 0, 0}, + + {"encrypt-threads", OPT_ENCRYPT_THREADS, + "This option specifies the number of worker threads that will be used " + "for parallel encryption. It is passed directly to the xtrabackup " + "child process. Try 'xtrabackup --help' for more details.", + (uchar*) &ibx_xtrabackup_encrypt_threads, + (uchar*) &ibx_xtrabackup_encrypt_threads, + 0, GET_UINT, REQUIRED_ARG, 1, 1, UINT_MAX, 0, 0, 0}, + + {"encrypt-chunk-size", OPT_ENCRYPT_CHUNK_SIZE, + "This option specifies the size of the internal working buffer for " + "each encryption thread, measured in bytes. It is passed directly to " + "the xtrabackup child process. Try 'xtrabackup --help' for more " + "details.", + (uchar*) &ibx_xtrabackup_encrypt_chunk_size, + (uchar*) &ibx_xtrabackup_encrypt_chunk_size, + 0, GET_ULL, REQUIRED_ARG, (1 << 16), 1024, ULONGLONG_MAX, 0, 0, 0}, + + {"export", OPT_EXPORT, "This option is passed directly to xtrabackup's " + "--export option. It enables exporting individual tables for import " + "into another server. See the xtrabackup documentation for details.", + (uchar*) &ibx_xtrabackup_export, (uchar*) &ibx_xtrabackup_export, + 0, GET_BOOL, NO_ARG, 0, 0, 0, 0, 0, 0}, + + {"extra-lsndir", OPT_EXTRA_LSNDIR, "This option specifies the " + "directory in which to save an extra copy of the " + "\"xtrabackup_checkpoints\" file. The option accepts a string " + "argument. It is passed directly to xtrabackup's --extra-lsndir " + "option. See the xtrabackup documentation for details.", + (uchar*) &ibx_xtrabackup_extra_lsndir, + (uchar*) &ibx_xtrabackup_extra_lsndir, + 0, GET_STR, REQUIRED_ARG, 0, 0, 0, 0, 0, 0}, + + {"incremental-basedir", OPT_INCREMENTAL_BASEDIR, "This option " + "specifies the directory containing the full backup that is the base " + "dataset for the incremental backup. The option accepts a string " + "argument. It is used with the --incremental option.", + (uchar*) &ibx_xtrabackup_incremental_basedir, + (uchar*) &ibx_xtrabackup_incremental_basedir, + 0, GET_STR, REQUIRED_ARG, 0, 0, 0, 0, 0, 0}, + + {"incremental-dir", OPT_INCREMENTAL_DIR, "This option specifies the " + "directory where the incremental backup will be combined with the " + "full backup to make a new full backup. The option accepts a string " + "argument. It is used with the --incremental option.", + (uchar*) &ibx_xtrabackup_incremental_dir, + (uchar*) &ibx_xtrabackup_incremental_dir, + 0, GET_STR, REQUIRED_ARG, 0, 0, 0, 0, 0, 0}, + + {"incremental-force-scan", OPT_INCREMENTAL_FORCE_SCAN, + "This options tells xtrabackup to perform full scan of data files " + "for taking an incremental backup even if full changed page bitmap " + "data is available to enable the backup without the full scan.", + (uchar*)&ibx_xtrabackup_incremental_force_scan, + (uchar*)&ibx_xtrabackup_incremental_force_scan, 0, GET_BOOL, NO_ARG, + 0, 0, 0, 0, 0, 0}, + + {"log-copy-interval", OPT_LOG_COPY_INTERVAL, "This option specifies " + "time interval between checks done by log copying thread in " + "milliseconds.", (uchar*) &ibx_xtrabackup_log_copy_interval, + (uchar*) &ibx_xtrabackup_log_copy_interval, + 0, GET_LONG, REQUIRED_ARG, 1000, 0, LONG_MAX, 0, 1, 0}, + + {"incremental-lsn", OPT_INCREMENTAL, "This option specifies the log " + "sequence number (LSN) to use for the incremental backup. The option " + "accepts a string argument. It is used with the --incremental option. " + "It is used instead of specifying --incremental-basedir. For " + "databases created by MySQL and Percona Server 5.0-series versions, " + "specify the LSN as two 32-bit integers in high:low format. For " + "databases created in 5.1 and later, specify the LSN as a single " + "64-bit integer.", + (uchar*) &ibx_xtrabackup_incremental, + (uchar*) &ibx_xtrabackup_incremental, + 0, GET_STR, REQUIRED_ARG, 0, 0, 0, 0, 0, 0}, + + {"parallel", OPT_PARALLEL, "On backup, this option specifies the " + "number of threads the xtrabackup child process should use to back " + "up files concurrently. The option accepts an integer argument. It " + "is passed directly to xtrabackup's --parallel option. See the " + "xtrabackup documentation for details.", + (uchar*) &ibx_xtrabackup_parallel, (uchar*) &ibx_xtrabackup_parallel, + 0, GET_INT, REQUIRED_ARG, 1, 1, INT_MAX, 0, 0, 0}, + + {"rebuild-indexes", OPT_REBUILD_INDEXES, + "This option only has effect when used together with the --apply-log " + "option and is passed directly to xtrabackup. When used, makes " + "xtrabackup rebuild all secondary indexes after applying the log. " + "This option is normally used to prepare compact backups. See the " + "XtraBackup manual for more information.", + (uchar*) &ibx_xtrabackup_rebuild_indexes, + (uchar*) &ibx_xtrabackup_rebuild_indexes, + 0, GET_BOOL, NO_ARG, 0, 0, 0, 0, 0, 0}, + + {"rebuild-threads", OPT_REBUILD_THREADS, + "Use this number of threads to rebuild indexes in a compact backup. " + "Only has effect with --prepare and --rebuild-indexes.", + (uchar*) &ibx_xtrabackup_rebuild_threads, + (uchar*) &ibx_xtrabackup_rebuild_threads, + 0, GET_UINT, REQUIRED_ARG, 1, 1, UINT_MAX, 0, 0, 0}, + + {"stream", OPT_STREAM, "This option specifies the format in which to " + "do the streamed backup. The option accepts a string argument. The " + "backup will be done to STDOUT in the specified format. Currently, " + "the only supported formats are tar and xbstream. This option is " + "passed directly to xtrabackup's --stream option.", + (uchar*) &ibx_xtrabackup_stream_str, + (uchar*) &ibx_xtrabackup_stream_str, 0, GET_STR, + REQUIRED_ARG, 0, 0, 0, 0, 0, 0}, + + {"tables-file", OPT_TABLES_FILE, "This option specifies the file in " + "which there are a list of names of the form database. The option " + "accepts a string argument.table, one per line. The option is passed " + "directly to xtrabackup's --tables-file option.", + (uchar*) &ibx_xtrabackup_tables_file, + (uchar*) &ibx_xtrabackup_tables_file, + 0, GET_STR, REQUIRED_ARG, 0, 0, 0, 0, 0, 0}, + + {"throttle", OPT_THROTTLE, "This option specifies a number of I/O " + "operations (pairs of read+write) per second. It accepts an integer " + "argument. It is passed directly to xtrabackup's --throttle option.", + (uchar*) &ibx_xtrabackup_throttle, (uchar*) &ibx_xtrabackup_throttle, + 0, GET_LONG, REQUIRED_ARG, 0, 0, LONG_MAX, 0, 1, 0}, + + {"tmpdir", 't', "This option specifies the location where a temporary " + "files will be stored. If the option is not specified, the default is " + "to use the value of tmpdir read from the server configuration.", + (uchar*) &ibx_opt_mysql_tmpdir, + (uchar*) &ibx_opt_mysql_tmpdir, 0, GET_STR, REQUIRED_ARG, + 0, 0, 0, 0, 0, 0}, + + {"use-memory", OPT_USE_MEMORY, "This option accepts a string argument " + "that specifies the amount of memory in bytes for xtrabackup to use " + "for crash recovery while preparing a backup. Multiples are supported " + "providing the unit (e.g. 1MB, 1GB). It is used only with the option " + "--apply-log. It is passed directly to xtrabackup's --use-memory " + "option. See the xtrabackup documentation for details.", + (uchar*) &ibx_xtrabackup_use_memory, + (uchar*) &ibx_xtrabackup_use_memory, + 0, GET_LL, REQUIRED_ARG, 100*1024*1024L, 1024*1024L, LONGLONG_MAX, 0, + 1024*1024L, 0}, + + { 0, 0, 0, 0, 0, 0, GET_NO_ARG, NO_ARG, 0, 0, 0, 0, 0, 0} +}; + + +static void usage(void) +{ + puts("Open source backup tool for InnoDB and XtraDB\n\ +\n\ +Copyright (C) 2009-2015 Percona LLC and/or its affiliates.\n\ +Portions Copyright (C) 2000, 2011, MySQL AB & Innobase Oy. All Rights Reserved.\n\ +\n\ +This program is free software; you can redistribute it and/or\n\ +modify it under the terms of the GNU General Public License\n\ +as published by the Free Software Foundation version 2\n\ +of the License.\n\ +\n\ +This program is distributed in the hope that it will be useful,\n\ +but WITHOUT ANY WARRANTY; without even the implied warranty of\n\ +MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the\n\ +GNU General Public License for more details.\n\ +\n\ +You can download full text of the license on http://www.gnu.org/licenses/gpl-2.0.txt\n\n"); + + puts("innobackupex - Non-blocking backup tool for InnoDB, XtraDB and HailDB databases\n\ +\n\ +SYNOPOSIS\n\ +\n\ +innobackupex [--compress] [--compress-threads=NUMBER-OF-THREADS] [--compress-chunk-size=CHUNK-SIZE]\n\ + [--encrypt=ENCRYPTION-ALGORITHM] [--encrypt-threads=NUMBER-OF-THREADS] [--encrypt-chunk-size=CHUNK-SIZE]\n\ + [--encrypt-key=LITERAL-ENCRYPTION-KEY] | [--encryption-key-file=MY.KEY]\n\ + [--include=REGEXP] [--user=NAME]\n\ + [--password=WORD] [--port=PORT] [--socket=SOCKET]\n\ + [--no-timestamp] [--ibbackup=IBBACKUP-BINARY]\n\ + [--slave-info] [--galera-info] [--stream=tar|xbstream]\n\ + [--defaults-file=MY.CNF] [--defaults-group=GROUP-NAME]\n\ + [--databases=LIST] [--no-lock] \n\ + [--tmpdir=DIRECTORY] [--tables-file=FILE]\n\ + [--history=NAME]\n\ + [--incremental] [--incremental-basedir]\n\ + [--incremental-dir] [--incremental-force-scan] [--incremental-lsn]\n\ + [--incremental-history-name=NAME] [--incremental-history-uuid=UUID]\n\ + [--close-files] [--compact] \n\ + BACKUP-ROOT-DIR\n\ +\n\ +innobackupex --apply-log [--use-memory=B]\n\ + [--defaults-file=MY.CNF]\n\ + [--export] [--redo-only] [--ibbackup=IBBACKUP-BINARY]\n\ + BACKUP-DIR\n\ +\n\ +innobackupex --copy-back [--defaults-file=MY.CNF] [--defaults-group=GROUP-NAME] BACKUP-DIR\n\ +\n\ +innobackupex --move-back [--defaults-file=MY.CNF] [--defaults-group=GROUP-NAME] BACKUP-DIR\n\ +\n\ +innobackupex [--decompress] [--decrypt=ENCRYPTION-ALGORITHM]\n\ + [--encrypt-key=LITERAL-ENCRYPTION-KEY] | [--encryption-key-file=MY.KEY]\n\ + [--parallel=NUMBER-OF-FORKS] BACKUP-DIR\n\ +\n\ +DESCRIPTION\n\ +\n\ +The first command line above makes a hot backup of a MySQL database.\n\ +By default it creates a backup directory (named by the current date\n\ + and time) in the given backup root directory. With the --no-timestamp\n\ +option it does not create a time-stamped backup directory, but it puts\n\ +the backup in the given directory (which must not exist). This\n\ +command makes a complete backup of all MyISAM and InnoDB tables and\n\ +indexes in all databases or in all of the databases specified with the\n\ +--databases option. The created backup contains .frm, .MRG, .MYD,\n\ +.MYI, .MAD, .MAI, .TRG, .TRN, .ARM, .ARZ, .CSM, CSV, .opt, .par, and\n\ +InnoDB data and log files. The MY.CNF options file defines the\n\ +location of the database. This command connects to the MySQL server\n\ +using the mysql client program, and runs xtrabackup as a child\n\ +process.\n\ +\n\ +The --apply-log command prepares a backup for starting a MySQL\n\ +server on the backup. This command recovers InnoDB data files as specified\n\ +in BACKUP-DIR/backup-my.cnf using BACKUP-DIR/xtrabackup_logfile,\n\ +and creates new InnoDB log files as specified in BACKUP-DIR/backup-my.cnf.\n\ +The BACKUP-DIR should be the path to a backup directory created by\n\ +xtrabackup. This command runs xtrabackup as a child process, but it does not \n\ +connect to the database server.\n\ +\n\ +The --copy-back command copies data, index, and log files\n\ +from the backup directory back to their original locations.\n\ +The MY.CNF options file defines the original location of the database.\n\ +The BACKUP-DIR is the path to a backup directory created by xtrabackup.\n\ +\n\ +The --move-back command is similar to --copy-back with the only difference that\n\ +it moves files to their original locations rather than copies them. As this\n\ +option removes backup files, it must be used with caution. It may be useful in\n\ +cases when there is not enough free disk space to copy files.\n\ +\n\ +The --decompress --decrypt command will decrypt and/or decompress a backup made\n\ +with the --compress and/or --encrypt options. When decrypting, the encryption\n\ +algorithm and key used when the backup was taken MUST be provided via the\n\ +specified options. --decrypt and --decompress may be used together at the same\n\ +time to completely normalize a previously compressed and encrypted backup. The\n\ +--parallel option will allow multiple files to be decrypted and/or decompressed\n\ +simultaneously. In order to decompress, the qpress utility MUST be installed\n\ +and accessable within the path. This process will remove the original\n\ +compressed/encrypted files and leave the results in the same location.\n\ +\n\ +On success the exit code innobackupex is 0. A non-zero exit code \n\ +indicates an error.\n"); + printf("Usage: [%s [--defaults-file=#] --backup | %s [--defaults-file=#] --prepare] [OPTIONS]\n", my_progname, my_progname); + my_print_help(ibx_long_options); +} + + +static +my_bool +ibx_get_one_option(int optid, + const struct my_option *opt __attribute__((unused)), + char *argument) +{ + switch(optid) { + case '?': + usage(); + exit(0); + break; + case 'v': + msg("innobackupex version %s %s (%s) (revision id: %s)\n", + XTRABACKUP_VERSION, + SYSTEM_TYPE, MACHINE_TYPE, XTRABACKUP_REVISION); + exit(0); + break; + case OPT_HISTORY: + if (argument) { + opt_ibx_history = argument; + } else { + opt_ibx_history = ""; + } + break; + case OPT_DECRYPT: + if (argument == NULL) { + ibx_msg("Missing --decrypt argument, must specify a " + "valid encryption algorithm.\n"); + return(1); + } + opt_ibx_decrypt = true; + break; + case OPT_STREAM: + if (!strcasecmp(argument, "tar")) + xtrabackup_stream_fmt = XB_STREAM_FMT_TAR; + else if (!strcasecmp(argument, "xbstream")) + xtrabackup_stream_fmt = XB_STREAM_FMT_XBSTREAM; + else { + ibx_msg("Invalid --stream argument: %s\n", argument); + return 1; + } + xtrabackup_stream = TRUE; + break; + case OPT_COMPRESS: + if (argument == NULL) + xtrabackup_compress_alg = "quicklz"; + else if (strcasecmp(argument, "quicklz")) + { + ibx_msg("Invalid --compress argument: %s\n", argument); + return 1; + } + xtrabackup_compress = TRUE; + break; + case OPT_ENCRYPT: + if (argument == NULL) + { + msg("Missing --encrypt argument, must specify a " + "valid encryption algorithm.\n"); + return 1; + } + xtrabackup_encrypt = TRUE; + break; + case 'p': + if (argument) + { + char *start = argument; + my_free(opt_ibx_password); + opt_ibx_password= my_strdup(argument, MYF(MY_FAE)); + /* Destroy argument */ + while (*argument) + *argument++= 'x'; + if (*start) + start[1]=0 ; + } + break; + } + return(0); +} + +bool +make_backup_dir() +{ + time_t t = time(NULL); + char buf[100]; + + if (!opt_ibx_notimestamp && !ibx_xtrabackup_stream_str) { + strftime(buf, sizeof(buf), "%Y-%m-%d_%H-%M-%S", localtime(&t)); + ut_a(asprintf(&ibx_backup_directory, "%s/%s", + ibx_position_arg, buf) != -1); + } else { + ibx_backup_directory = strdup(ibx_position_arg); + } + + if (!directory_exists(ibx_backup_directory, true)) { + return(false); + } + + return(true); +} + +bool +ibx_handle_options(int *argc, char ***argv) +{ + int i, n_arguments; + + if (handle_options(argc, argv, ibx_long_options, ibx_get_one_option)) { + return(false); + } + + if (opt_ibx_apply_log) { + ibx_mode = IBX_MODE_APPLY_LOG; + } else if (opt_ibx_copy_back) { + ibx_mode = IBX_MODE_COPY_BACK; + } else if (opt_ibx_move_back) { + ibx_mode = IBX_MODE_MOVE_BACK; + } else if (opt_ibx_decrypt || opt_ibx_decompress) { + ibx_mode = IBX_MODE_DECRYPT_DECOMPRESS; + } else { + ibx_mode = IBX_MODE_BACKUP; + } + + /* find and save position argument */ + i = 0; + n_arguments = 0; + while (i < *argc) { + char *opt = (*argv)[i]; + + if (strncmp(opt, "--", 2) != 0 + && !(strlen(opt) == 2 && opt[0] == '-')) { + if (ibx_position_arg != NULL + && ibx_position_arg != opt) { + ibx_msg("Error: extra argument found %s\n", + opt); + } + ibx_position_arg = opt; + ++n_arguments; + } + ++i; + } + + *argc -= n_arguments; + if (n_arguments > 1) { + return(false); + } + + if (ibx_position_arg == NULL) { + ibx_msg("Missing argument\n"); + return(false); + } + + /* set argv[0] to be the program name */ + --(*argv); + ++(*argc); + + return(true); +} + +/*********************************************************************//** +Parse command-line options, connect to MySQL server, +detect server capabilities, etc. +@return true on success. */ +bool +ibx_init() +{ + const char *run; + + /*=====================*/ + xtrabackup_copy_back = opt_ibx_copy_back; + xtrabackup_move_back = opt_ibx_move_back; + opt_galera_info = opt_ibx_galera_info; + opt_slave_info = opt_ibx_slave_info; + opt_no_lock = opt_ibx_no_lock; + opt_safe_slave_backup = opt_ibx_safe_slave_backup; + opt_rsync = opt_ibx_rsync; + opt_force_non_empty_dirs = opt_ibx_force_non_empty_dirs; + opt_noversioncheck = opt_ibx_noversioncheck; + opt_no_backup_locks = opt_ibx_no_backup_locks; + opt_decompress = opt_ibx_decompress; + + opt_incremental_history_name = opt_ibx_incremental_history_name; + opt_incremental_history_uuid = opt_ibx_incremental_history_uuid; + + opt_user = opt_ibx_user; + opt_password = opt_ibx_password; + opt_host = opt_ibx_host; + opt_defaults_group = opt_ibx_defaults_group; + opt_socket = opt_ibx_socket; + opt_port = opt_ibx_port; + opt_login_path = opt_ibx_login_path; + + opt_lock_wait_query_type = opt_ibx_lock_wait_query_type; + opt_kill_long_query_type = opt_ibx_kill_long_query_type; + + opt_decrypt_algo = opt_ibx_decrypt_algo; + + opt_kill_long_queries_timeout = opt_ibx_kill_long_queries_timeout; + opt_lock_wait_timeout = opt_ibx_lock_wait_timeout; + opt_lock_wait_threshold = opt_ibx_lock_wait_threshold; + opt_debug_sleep_before_unlock = opt_ibx_debug_sleep_before_unlock; + opt_safe_slave_backup_timeout = opt_ibx_safe_slave_backup_timeout; + + opt_history = opt_ibx_history; + opt_decrypt = opt_ibx_decrypt; + + /* setup xtrabackup options */ + xb_close_files = ibx_xb_close_files; + xtrabackup_compact = ibx_xtrabackup_compact; + xtrabackup_compress_alg = ibx_xtrabackup_compress_alg; + xtrabackup_compress_threads = ibx_xtrabackup_compress_threads; + xtrabackup_compress_chunk_size = ibx_xtrabackup_compress_chunk_size; + xtrabackup_encrypt_algo = ibx_xtrabackup_encrypt_algo; + xtrabackup_encrypt_key = ibx_xtrabackup_encrypt_key; + xtrabackup_encrypt_key_file = ibx_xtrabackup_encrypt_key_file; + xtrabackup_encrypt_threads = ibx_xtrabackup_encrypt_threads; + xtrabackup_encrypt_chunk_size = ibx_xtrabackup_encrypt_chunk_size; + xtrabackup_export = ibx_xtrabackup_export; + xtrabackup_extra_lsndir = ibx_xtrabackup_extra_lsndir; + xtrabackup_incremental_basedir = ibx_xtrabackup_incremental_basedir; + xtrabackup_incremental_dir = ibx_xtrabackup_incremental_dir; + xtrabackup_incremental_force_scan = + ibx_xtrabackup_incremental_force_scan; + xtrabackup_log_copy_interval = ibx_xtrabackup_log_copy_interval; + xtrabackup_incremental = ibx_xtrabackup_incremental; + xtrabackup_parallel = ibx_xtrabackup_parallel; + xtrabackup_rebuild_indexes = ibx_xtrabackup_rebuild_indexes; + xtrabackup_rebuild_threads = ibx_xtrabackup_rebuild_threads; + xtrabackup_stream_str = ibx_xtrabackup_stream_str; + xtrabackup_tables_file = ibx_xtrabackup_tables_file; + xtrabackup_throttle = ibx_xtrabackup_throttle; + opt_mysql_tmpdir = ibx_opt_mysql_tmpdir; + xtrabackup_use_memory = ibx_xtrabackup_use_memory; + + if (!opt_ibx_incremental + && (xtrabackup_incremental + || xtrabackup_incremental_basedir + || opt_ibx_incremental_history_name + || opt_ibx_incremental_history_uuid)) { + ibx_msg("Error: --incremental-lsn, --incremental-basedir, " + "--incremental-history-name and " + "--incremental-history-uuid require the " + "--incremental option.\n"); + return(false); + } + + if (opt_ibx_databases != NULL) { + if (is_path_separator(*opt_ibx_databases)) { + xtrabackup_databases_file = opt_ibx_databases; + } else { + xtrabackup_databases = opt_ibx_databases; + } + } + + /* --tables and --tables-file options are xtrabackup only */ + ibx_partial_backup = (opt_ibx_include || opt_ibx_databases); + + if (ibx_mode == IBX_MODE_BACKUP) { + + if (!make_backup_dir()) { + return(false); + } + } + + /* --binlog-info is xtrabackup only, so force + --binlog-info=ON. i.e. behavior before the feature had been + implemented */ + opt_binlog_info = BINLOG_INFO_ON; + + switch (ibx_mode) { + case IBX_MODE_APPLY_LOG: + xtrabackup_prepare = TRUE; + if (opt_ibx_redo_only) { + xtrabackup_apply_log_only = TRUE; + } + xtrabackup_target_dir = ibx_position_arg; + run = "apply-log"; + break; + case IBX_MODE_BACKUP: + xtrabackup_backup = TRUE; + xtrabackup_target_dir = ibx_backup_directory; + if (opt_ibx_include != NULL) { + xtrabackup_tables = opt_ibx_include; + } + run = "backup"; + break; + case IBX_MODE_COPY_BACK: + xtrabackup_copy_back = TRUE; + xtrabackup_target_dir = ibx_position_arg; + run = "copy-back"; + break; + case IBX_MODE_MOVE_BACK: + xtrabackup_move_back = TRUE; + xtrabackup_target_dir = ibx_position_arg; + run = "move-back"; + break; + case IBX_MODE_DECRYPT_DECOMPRESS: + xtrabackup_decrypt_decompress = TRUE; + xtrabackup_target_dir = ibx_position_arg; + run = "decrypt and decompress"; + break; + default: + ut_error; + } + + ibx_msg("Starting the %s operation\n\n" + "IMPORTANT: Please check that the %s run completes " + "successfully.\n" + " At the end of a successful %s run innobackupex\n" + " prints \"completed OK!\".\n\n", run, run, run); + + + return(true); +} + +void +ibx_cleanup() +{ + free(ibx_backup_directory); +} diff --git a/extra/mariabackup/innobackupex.h b/extra/mariabackup/innobackupex.h new file mode 100644 index 00000000000..e2ad9bd2511 --- /dev/null +++ b/extra/mariabackup/innobackupex.h @@ -0,0 +1,45 @@ +/****************************************************** +Copyright (c) 2011-2014 Percona LLC and/or its affiliates. + +Declarations for innobackupex.cc + +This program is free software; you can redistribute it and/or modify +it under the terms of the GNU General Public License as published by +the Free Software Foundation; version 2 of the License. + +This program is distributed in the hope that it will be useful, +but WITHOUT ANY WARRANTY; without even the implied warranty of +MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the +GNU General Public License for more details. + +You should have received a copy of the GNU General Public License +along with this program; if not, write to the Free Software +Foundation, Inc., 51 Franklin Street, Fifth Floor, Boston, MA 02110-1301, USA + +*******************************************************/ + +#ifndef INNOBACKUPEX_H +#define INNOBACKUPEX_H + +#define INNOBACKUPEX_BIN_NAME "innobackupex" + +enum ibx_mode_t { + IBX_MODE_BACKUP, + IBX_MODE_APPLY_LOG, + IBX_MODE_COPY_BACK, + IBX_MODE_MOVE_BACK, + IBX_MODE_DECRYPT_DECOMPRESS +}; + +extern ibx_mode_t ibx_mode; + +bool +ibx_handle_options(int *argc, char ***argv); + +bool +ibx_init(); + +void +ibx_cleanup(); + +#endif diff --git a/extra/mariabackup/quicklz/quicklz.c b/extra/mariabackup/quicklz/quicklz.c new file mode 100644 index 00000000000..3742129023a --- /dev/null +++ b/extra/mariabackup/quicklz/quicklz.c @@ -0,0 +1,848 @@ +// Fast data compression library +// Copyright (C) 2006-2011 Lasse Mikkel Reinhold +// lar@quicklz.com +// +// QuickLZ can be used for free under the GPL 1, 2 or 3 license (where anything +// released into public must be open source) or under a commercial license if such +// has been acquired (see http://www.quicklz.com/order.html). The commercial license +// does not cover derived or ported versions created by third parties under GPL. + +// 1.5.0 final + +#include "quicklz.h" + +#if QLZ_VERSION_MAJOR != 1 || QLZ_VERSION_MINOR != 5 || QLZ_VERSION_REVISION != 0 + #error quicklz.c and quicklz.h have different versions +#endif + +#if (defined(__X86__) || defined(__i386__) || defined(i386) || defined(_M_IX86) || defined(__386__) || defined(__x86_64__) || defined(_M_X64)) + #define X86X64 +#endif + +#define MINOFFSET 2 +#define UNCONDITIONAL_MATCHLEN 6 +#define UNCOMPRESSED_END 4 +#define CWORD_LEN 4 + +#if QLZ_COMPRESSION_LEVEL == 1 && defined QLZ_PTR_64 && QLZ_STREAMING_BUFFER == 0 + #define OFFSET_BASE source + #define CAST (ui32)(size_t) +#else + #define OFFSET_BASE 0 + #define CAST +#endif + +int qlz_get_setting(int setting) +{ + switch (setting) + { + case 0: return QLZ_COMPRESSION_LEVEL; + case 1: return sizeof(qlz_state_compress); + case 2: return sizeof(qlz_state_decompress); + case 3: return QLZ_STREAMING_BUFFER; +#ifdef QLZ_MEMORY_SAFE + case 6: return 1; +#else + case 6: return 0; +#endif + case 7: return QLZ_VERSION_MAJOR; + case 8: return QLZ_VERSION_MINOR; + case 9: return QLZ_VERSION_REVISION; + } + return -1; +} + +#if QLZ_COMPRESSION_LEVEL == 1 +static int same(const unsigned char *src, size_t n) +{ + while(n > 0 && *(src + n) == *src) + n--; + return n == 0 ? 1 : 0; +} +#endif + +static void reset_table_compress(qlz_state_compress *state) +{ + int i; + for(i = 0; i < QLZ_HASH_VALUES; i++) + { +#if QLZ_COMPRESSION_LEVEL == 1 + state->hash[i].offset = 0; +#else + state->hash_counter[i] = 0; +#endif + } +} + +static void reset_table_decompress(qlz_state_decompress *state) +{ + int i; + (void)state; + (void)i; +#if QLZ_COMPRESSION_LEVEL == 2 + for(i = 0; i < QLZ_HASH_VALUES; i++) + { + state->hash_counter[i] = 0; + } +#endif +} + +static __inline ui32 hash_func(ui32 i) +{ +#if QLZ_COMPRESSION_LEVEL == 2 + return ((i >> 9) ^ (i >> 13) ^ i) & (QLZ_HASH_VALUES - 1); +#else + return ((i >> 12) ^ i) & (QLZ_HASH_VALUES - 1); +#endif +} + +static __inline ui32 fast_read(void const *src, ui32 bytes) +{ +#ifndef X86X64 + unsigned char *p = (unsigned char*)src; + switch (bytes) + { + case 4: + return(*p | *(p + 1) << 8 | *(p + 2) << 16 | *(p + 3) << 24); + case 3: + return(*p | *(p + 1) << 8 | *(p + 2) << 16); + case 2: + return(*p | *(p + 1) << 8); + case 1: + return(*p); + } + return 0; +#else + if (bytes >= 1 && bytes <= 4) + return *((ui32*)src); + else + return 0; +#endif +} + +static __inline ui32 hashat(const unsigned char *src) +{ + ui32 fetch, hash; + fetch = fast_read(src, 3); + hash = hash_func(fetch); + return hash; +} + +static __inline void fast_write(ui32 f, void *dst, size_t bytes) +{ +#ifndef X86X64 + unsigned char *p = (unsigned char*)dst; + + switch (bytes) + { + case 4: + *p = (unsigned char)f; + *(p + 1) = (unsigned char)(f >> 8); + *(p + 2) = (unsigned char)(f >> 16); + *(p + 3) = (unsigned char)(f >> 24); + return; + case 3: + *p = (unsigned char)f; + *(p + 1) = (unsigned char)(f >> 8); + *(p + 2) = (unsigned char)(f >> 16); + return; + case 2: + *p = (unsigned char)f; + *(p + 1) = (unsigned char)(f >> 8); + return; + case 1: + *p = (unsigned char)f; + return; + } +#else + switch (bytes) + { + case 4: + *((ui32*)dst) = f; + return; + case 3: + *((ui32*)dst) = f; + return; + case 2: + *((ui16 *)dst) = (ui16)f; + return; + case 1: + *((unsigned char*)dst) = (unsigned char)f; + return; + } +#endif +} + + +size_t qlz_size_decompressed(const char *source) +{ + ui32 n, r; + n = (((*source) & 2) == 2) ? 4 : 1; + r = fast_read(source + 1 + n, n); + r = r & (0xffffffff >> ((4 - n)*8)); + return r; +} + +size_t qlz_size_compressed(const char *source) +{ + ui32 n, r; + n = (((*source) & 2) == 2) ? 4 : 1; + r = fast_read(source + 1, n); + r = r & (0xffffffff >> ((4 - n)*8)); + return r; +} + +size_t qlz_size_header(const char *source) +{ + size_t n = 2*((((*source) & 2) == 2) ? 4 : 1) + 1; + return n; +} + + +static __inline void memcpy_up(unsigned char *dst, const unsigned char *src, ui32 n) +{ + // Caution if modifying memcpy_up! Overlap of dst and src must be special handled. +#ifndef X86X64 + unsigned char *end = dst + n; + while(dst < end) + { + *dst = *src; + dst++; + src++; + } +#else + ui32 f = 0; + do + { + *(ui32 *)(dst + f) = *(ui32 *)(src + f); + f += MINOFFSET + 1; + } + while (f < n); +#endif +} + +static __inline void update_hash(qlz_state_decompress *state, const unsigned char *s) +{ +#if QLZ_COMPRESSION_LEVEL == 1 + ui32 hash; + hash = hashat(s); + state->hash[hash].offset = s; + state->hash_counter[hash] = 1; +#elif QLZ_COMPRESSION_LEVEL == 2 + ui32 hash; + unsigned char c; + hash = hashat(s); + c = state->hash_counter[hash]; + state->hash[hash].offset[c & (QLZ_POINTERS - 1)] = s; + c++; + state->hash_counter[hash] = c; +#endif + (void)state; + (void)s; +} + +#if QLZ_COMPRESSION_LEVEL <= 2 +static void update_hash_upto(qlz_state_decompress *state, unsigned char **lh, const unsigned char *max) +{ + while(*lh < max) + { + (*lh)++; + update_hash(state, *lh); + } +} +#endif + +static size_t qlz_compress_core(const unsigned char *source, unsigned char *destination, size_t size, qlz_state_compress *state) +{ + const unsigned char *last_byte = source + size - 1; + const unsigned char *src = source; + unsigned char *cword_ptr = destination; + unsigned char *dst = destination + CWORD_LEN; + ui32 cword_val = 1U << 31; + const unsigned char *last_matchstart = last_byte - UNCONDITIONAL_MATCHLEN - UNCOMPRESSED_END; + ui32 fetch = 0; + unsigned int lits = 0; + + (void) lits; + + if(src <= last_matchstart) + fetch = fast_read(src, 3); + + while(src <= last_matchstart) + { + if ((cword_val & 1) == 1) + { + // store uncompressed if compression ratio is too low + if (src > source + (size >> 1) && dst - destination > src - source - ((src - source) >> 5)) + return 0; + + fast_write((cword_val >> 1) | (1U << 31), cword_ptr, CWORD_LEN); + + cword_ptr = dst; + dst += CWORD_LEN; + cword_val = 1U << 31; + fetch = fast_read(src, 3); + } +#if QLZ_COMPRESSION_LEVEL == 1 + { + const unsigned char *o; + ui32 hash, cached; + + hash = hash_func(fetch); + cached = fetch ^ state->hash[hash].cache; + state->hash[hash].cache = fetch; + + o = state->hash[hash].offset + OFFSET_BASE; + state->hash[hash].offset = CAST(src - OFFSET_BASE); + +#ifdef X86X64 + if ((cached & 0xffffff) == 0 && o != OFFSET_BASE && (src - o > MINOFFSET || (src == o + 1 && lits >= 3 && src > source + 3 && same(src - 3, 6)))) + { + if(cached != 0) + { +#else + if (cached == 0 && o != OFFSET_BASE && (src - o > MINOFFSET || (src == o + 1 && lits >= 3 && src > source + 3 && same(src - 3, 6)))) + { + if (*(o + 3) != *(src + 3)) + { +#endif + hash <<= 4; + cword_val = (cword_val >> 1) | (1U << 31); + fast_write((3 - 2) | hash, dst, 2); + src += 3; + dst += 2; + } + else + { + const unsigned char *old_src = src; + size_t matchlen; + hash <<= 4; + + cword_val = (cword_val >> 1) | (1U << 31); + src += 4; + + if(*(o + (src - old_src)) == *src) + { + src++; + if(*(o + (src - old_src)) == *src) + { + size_t q = last_byte - UNCOMPRESSED_END - (src - 5) + 1; + size_t remaining = q > 255 ? 255 : q; + src++; + while(*(o + (src - old_src)) == *src && (size_t)(src - old_src) < remaining) + src++; + } + } + + matchlen = src - old_src; + if (matchlen < 18) + { + fast_write((ui32)(matchlen - 2) | hash, dst, 2); + dst += 2; + } + else + { + fast_write((ui32)(matchlen << 16) | hash, dst, 3); + dst += 3; + } + } + fetch = fast_read(src, 3); + lits = 0; + } + else + { + lits++; + *dst = *src; + src++; + dst++; + cword_val = (cword_val >> 1); +#ifdef X86X64 + fetch = fast_read(src, 3); +#else + fetch = (fetch >> 8 & 0xffff) | (*(src + 2) << 16); +#endif + } + } +#elif QLZ_COMPRESSION_LEVEL >= 2 + { + const unsigned char *o, *offset2; + ui32 hash, matchlen, k, m, best_k = 0; + unsigned char c; + size_t remaining = (last_byte - UNCOMPRESSED_END - src + 1) > 255 ? 255 : (last_byte - UNCOMPRESSED_END - src + 1); + (void)best_k; + + + //hash = hashat(src); + fetch = fast_read(src, 3); + hash = hash_func(fetch); + + c = state->hash_counter[hash]; + + offset2 = state->hash[hash].offset[0]; + if(offset2 < src - MINOFFSET && c > 0 && ((fast_read(offset2, 3) ^ fetch) & 0xffffff) == 0) + { + matchlen = 3; + if(*(offset2 + matchlen) == *(src + matchlen)) + { + matchlen = 4; + while(*(offset2 + matchlen) == *(src + matchlen) && matchlen < remaining) + matchlen++; + } + } + else + matchlen = 0; + for(k = 1; k < QLZ_POINTERS && c > k; k++) + { + o = state->hash[hash].offset[k]; +#if QLZ_COMPRESSION_LEVEL == 3 + if(((fast_read(o, 3) ^ fetch) & 0xffffff) == 0 && o < src - MINOFFSET) +#elif QLZ_COMPRESSION_LEVEL == 2 + if(*(src + matchlen) == *(o + matchlen) && ((fast_read(o, 3) ^ fetch) & 0xffffff) == 0 && o < src - MINOFFSET) +#endif + { + m = 3; + while(*(o + m) == *(src + m) && m < remaining) + m++; +#if QLZ_COMPRESSION_LEVEL == 3 + if ((m > matchlen) || (m == matchlen && o > offset2)) +#elif QLZ_COMPRESSION_LEVEL == 2 + if (m > matchlen) +#endif + { + offset2 = o; + matchlen = m; + best_k = k; + } + } + } + o = offset2; + state->hash[hash].offset[c & (QLZ_POINTERS - 1)] = src; + c++; + state->hash_counter[hash] = c; + +#if QLZ_COMPRESSION_LEVEL == 3 + if(matchlen > 2 && src - o < 131071) + { + ui32 u; + size_t offset = src - o; + + for(u = 1; u < matchlen; u++) + { + hash = hashat(src + u); + c = state->hash_counter[hash]++; + state->hash[hash].offset[c & (QLZ_POINTERS - 1)] = src + u; + } + + cword_val = (cword_val >> 1) | (1U << 31); + src += matchlen; + + if(matchlen == 3 && offset <= 63) + { + *dst = (unsigned char)(offset << 2); + dst++; + } + else if (matchlen == 3 && offset <= 16383) + { + ui32 f = (ui32)((offset << 2) | 1); + fast_write(f, dst, 2); + dst += 2; + } + else if (matchlen <= 18 && offset <= 1023) + { + ui32 f = ((matchlen - 3) << 2) | ((ui32)offset << 6) | 2; + fast_write(f, dst, 2); + dst += 2; + } + + else if(matchlen <= 33) + { + ui32 f = ((matchlen - 2) << 2) | ((ui32)offset << 7) | 3; + fast_write(f, dst, 3); + dst += 3; + } + else + { + ui32 f = ((matchlen - 3) << 7) | ((ui32)offset << 15) | 3; + fast_write(f, dst, 4); + dst += 4; + } + } + else + { + *dst = *src; + src++; + dst++; + cword_val = (cword_val >> 1); + } +#elif QLZ_COMPRESSION_LEVEL == 2 + + if(matchlen > 2) + { + cword_val = (cword_val >> 1) | (1U << 31); + src += matchlen; + + if (matchlen < 10) + { + ui32 f = best_k | ((matchlen - 2) << 2) | (hash << 5); + fast_write(f, dst, 2); + dst += 2; + } + else + { + ui32 f = best_k | (matchlen << 16) | (hash << 5); + fast_write(f, dst, 3); + dst += 3; + } + } + else + { + *dst = *src; + src++; + dst++; + cword_val = (cword_val >> 1); + } +#endif + } +#endif + } + while (src <= last_byte) + { + if ((cword_val & 1) == 1) + { + fast_write((cword_val >> 1) | (1U << 31), cword_ptr, CWORD_LEN); + cword_ptr = dst; + dst += CWORD_LEN; + cword_val = 1U << 31; + } +#if QLZ_COMPRESSION_LEVEL < 3 + if (src <= last_byte - 3) + { +#if QLZ_COMPRESSION_LEVEL == 1 + ui32 hash, fetch; + fetch = fast_read(src, 3); + hash = hash_func(fetch); + state->hash[hash].offset = CAST(src - OFFSET_BASE); + state->hash[hash].cache = fetch; +#elif QLZ_COMPRESSION_LEVEL == 2 + ui32 hash; + unsigned char c; + hash = hashat(src); + c = state->hash_counter[hash]; + state->hash[hash].offset[c & (QLZ_POINTERS - 1)] = src; + c++; + state->hash_counter[hash] = c; +#endif + } +#endif + *dst = *src; + src++; + dst++; + cword_val = (cword_val >> 1); + } + + while((cword_val & 1) != 1) + cword_val = (cword_val >> 1); + + fast_write((cword_val >> 1) | (1U << 31), cword_ptr, CWORD_LEN); + + // min. size must be 9 bytes so that the qlz_size functions can take 9 bytes as argument + return dst - destination < 9 ? 9 : dst - destination; +} + +static size_t qlz_decompress_core(const unsigned char *source, unsigned char *destination, size_t size, qlz_state_decompress *state, const unsigned char *history) +{ + const unsigned char *src = source + qlz_size_header((const char *)source); + unsigned char *dst = destination; + const unsigned char *last_destination_byte = destination + size - 1; + ui32 cword_val = 1; + const unsigned char *last_matchstart = last_destination_byte - UNCONDITIONAL_MATCHLEN - UNCOMPRESSED_END; + unsigned char *last_hashed = destination - 1; + const unsigned char *last_source_byte = source + qlz_size_compressed((const char *)source) - 1; + static const ui32 bitlut[16] = {4, 0, 1, 0, 2, 0, 1, 0, 3, 0, 1, 0, 2, 0, 1, 0}; + + (void) last_source_byte; + (void) last_hashed; + (void) state; + (void) history; + + for(;;) + { + ui32 fetch; + + if (cword_val == 1) + { +#ifdef QLZ_MEMORY_SAFE + if(src + CWORD_LEN - 1 > last_source_byte) + return 0; +#endif + cword_val = fast_read(src, CWORD_LEN); + src += CWORD_LEN; + } + +#ifdef QLZ_MEMORY_SAFE + if(src + 4 - 1 > last_source_byte) + return 0; +#endif + + fetch = fast_read(src, 4); + + if ((cword_val & 1) == 1) + { + ui32 matchlen; + const unsigned char *offset2; + +#if QLZ_COMPRESSION_LEVEL == 1 + ui32 hash; + cword_val = cword_val >> 1; + hash = (fetch >> 4) & 0xfff; + offset2 = (const unsigned char *)(size_t)state->hash[hash].offset; + + if((fetch & 0xf) != 0) + { + matchlen = (fetch & 0xf) + 2; + src += 2; + } + else + { + matchlen = *(src + 2); + src += 3; + } + +#elif QLZ_COMPRESSION_LEVEL == 2 + ui32 hash; + unsigned char c; + cword_val = cword_val >> 1; + hash = (fetch >> 5) & 0x7ff; + c = (unsigned char)(fetch & 0x3); + offset2 = state->hash[hash].offset[c]; + + if((fetch & (28)) != 0) + { + matchlen = ((fetch >> 2) & 0x7) + 2; + src += 2; + } + else + { + matchlen = *(src + 2); + src += 3; + } + +#elif QLZ_COMPRESSION_LEVEL == 3 + ui32 offset; + cword_val = cword_val >> 1; + if ((fetch & 3) == 0) + { + offset = (fetch & 0xff) >> 2; + matchlen = 3; + src++; + } + else if ((fetch & 2) == 0) + { + offset = (fetch & 0xffff) >> 2; + matchlen = 3; + src += 2; + } + else if ((fetch & 1) == 0) + { + offset = (fetch & 0xffff) >> 6; + matchlen = ((fetch >> 2) & 15) + 3; + src += 2; + } + else if ((fetch & 127) != 3) + { + offset = (fetch >> 7) & 0x1ffff; + matchlen = ((fetch >> 2) & 0x1f) + 2; + src += 3; + } + else + { + offset = (fetch >> 15); + matchlen = ((fetch >> 7) & 255) + 3; + src += 4; + } + + offset2 = dst - offset; +#endif + +#ifdef QLZ_MEMORY_SAFE + if(offset2 < history || offset2 > dst - MINOFFSET - 1) + return 0; + + if(matchlen > (ui32)(last_destination_byte - dst - UNCOMPRESSED_END + 1)) + return 0; +#endif + + memcpy_up(dst, offset2, matchlen); + dst += matchlen; + +#if QLZ_COMPRESSION_LEVEL <= 2 + update_hash_upto(state, &last_hashed, dst - matchlen); + last_hashed = dst - 1; +#endif + } + else + { + if (dst < last_matchstart) + { + unsigned int n = bitlut[cword_val & 0xf]; +#ifdef X86X64 + *(ui32 *)dst = *(ui32 *)src; +#else + memcpy_up(dst, src, 4); +#endif + cword_val = cword_val >> n; + dst += n; + src += n; +#if QLZ_COMPRESSION_LEVEL <= 2 + update_hash_upto(state, &last_hashed, dst - 3); +#endif + } + else + { + while(dst <= last_destination_byte) + { + if (cword_val == 1) + { + src += CWORD_LEN; + cword_val = 1U << 31; + } +#ifdef QLZ_MEMORY_SAFE + if(src >= last_source_byte + 1) + return 0; +#endif + *dst = *src; + dst++; + src++; + cword_val = cword_val >> 1; + } + +#if QLZ_COMPRESSION_LEVEL <= 2 + update_hash_upto(state, &last_hashed, last_destination_byte - 3); // todo, use constant +#endif + return size; + } + + } + } +} + +size_t qlz_compress(const void *source, char *destination, size_t size, qlz_state_compress *state) +{ + size_t r; + ui32 compressed; + size_t base; + + if(size == 0 || size > 0xffffffff - 400) + return 0; + + if(size < 216) + base = 3; + else + base = 9; + +#if QLZ_STREAMING_BUFFER > 0 + if (state->stream_counter + size - 1 >= QLZ_STREAMING_BUFFER) +#endif + { + reset_table_compress(state); + r = base + qlz_compress_core((const unsigned char *)source, (unsigned char*)destination + base, size, state); +#if QLZ_STREAMING_BUFFER > 0 + reset_table_compress(state); +#endif + if(r == base) + { + memcpy(destination + base, source, size); + r = size + base; + compressed = 0; + } + else + { + compressed = 1; + } + state->stream_counter = 0; + } +#if QLZ_STREAMING_BUFFER > 0 + else + { + unsigned char *src = state->stream_buffer + state->stream_counter; + + memcpy(src, source, size); + r = base + qlz_compress_core(src, (unsigned char*)destination + base, size, state); + + if(r == base) + { + memcpy(destination + base, src, size); + r = size + base; + compressed = 0; + reset_table_compress(state); + } + else + { + compressed = 1; + } + state->stream_counter += size; + } +#endif + if(base == 3) + { + *destination = (unsigned char)(0 | compressed); + *(destination + 1) = (unsigned char)r; + *(destination + 2) = (unsigned char)size; + } + else + { + *destination = (unsigned char)(2 | compressed); + fast_write((ui32)r, destination + 1, 4); + fast_write((ui32)size, destination + 5, 4); + } + + *destination |= (QLZ_COMPRESSION_LEVEL << 2); + *destination |= (1 << 6); + *destination |= ((QLZ_STREAMING_BUFFER == 0 ? 0 : (QLZ_STREAMING_BUFFER == 100000 ? 1 : (QLZ_STREAMING_BUFFER == 1000000 ? 2 : 3))) << 4); + +// 76543210 +// 01SSLLHC + + return r; +} + +size_t qlz_decompress(const char *source, void *destination, qlz_state_decompress *state) +{ + size_t dsiz = qlz_size_decompressed(source); + +#if QLZ_STREAMING_BUFFER > 0 + if (state->stream_counter + qlz_size_decompressed(source) - 1 >= QLZ_STREAMING_BUFFER) +#endif + { + if((*source & 1) == 1) + { + reset_table_decompress(state); + dsiz = qlz_decompress_core((const unsigned char *)source, (unsigned char *)destination, dsiz, state, (const unsigned char *)destination); + } + else + { + memcpy(destination, source + qlz_size_header(source), dsiz); + } + state->stream_counter = 0; + reset_table_decompress(state); + } +#if QLZ_STREAMING_BUFFER > 0 + else + { + unsigned char *dst = state->stream_buffer + state->stream_counter; + if((*source & 1) == 1) + { + dsiz = qlz_decompress_core((const unsigned char *)source, dst, dsiz, state, (const unsigned char *)state->stream_buffer); + } + else + { + memcpy(dst, source + qlz_size_header(source), dsiz); + reset_table_decompress(state); + } + memcpy(destination, dst, dsiz); + state->stream_counter += dsiz; + } +#endif + return dsiz; +} + diff --git a/extra/mariabackup/quicklz/quicklz.h b/extra/mariabackup/quicklz/quicklz.h new file mode 100644 index 00000000000..6ffe00f3a91 --- /dev/null +++ b/extra/mariabackup/quicklz/quicklz.h @@ -0,0 +1,144 @@ +#ifndef QLZ_HEADER +#define QLZ_HEADER + +// Fast data compression library +// Copyright (C) 2006-2011 Lasse Mikkel Reinhold +// lar@quicklz.com +// +// QuickLZ can be used for free under the GPL 1, 2 or 3 license (where anything +// released into public must be open source) or under a commercial license if such +// has been acquired (see http://www.quicklz.com/order.html). The commercial license +// does not cover derived or ported versions created by third parties under GPL. + +// You can edit following user settings. Data must be decompressed with the same +// setting of QLZ_COMPRESSION_LEVEL and QLZ_STREAMING_BUFFER as it was compressed +// (see manual). If QLZ_STREAMING_BUFFER > 0, scratch buffers must be initially +// zeroed out (see manual). First #ifndef makes it possible to define settings from +// the outside like the compiler command line. + +// 1.5.0 final + +#ifndef QLZ_COMPRESSION_LEVEL + #define QLZ_COMPRESSION_LEVEL 1 + //#define QLZ_COMPRESSION_LEVEL 2 + //#define QLZ_COMPRESSION_LEVEL 3 + + #define QLZ_STREAMING_BUFFER 0 + //#define QLZ_STREAMING_BUFFER 100000 + //#define QLZ_STREAMING_BUFFER 1000000 + + //#define QLZ_MEMORY_SAFE +#endif + +#define QLZ_VERSION_MAJOR 1 +#define QLZ_VERSION_MINOR 5 +#define QLZ_VERSION_REVISION 0 + +// Using size_t, memset() and memcpy() +#include + +// Verify compression level +#if QLZ_COMPRESSION_LEVEL != 1 && QLZ_COMPRESSION_LEVEL != 2 && QLZ_COMPRESSION_LEVEL != 3 +#error QLZ_COMPRESSION_LEVEL must be 1, 2 or 3 +#endif + +typedef unsigned int ui32; +typedef unsigned short int ui16; + +// Decrease QLZ_POINTERS for level 3 to increase compression speed. Do not touch any other values! +#if QLZ_COMPRESSION_LEVEL == 1 +#define QLZ_POINTERS 1 +#define QLZ_HASH_VALUES 4096 +#elif QLZ_COMPRESSION_LEVEL == 2 +#define QLZ_POINTERS 4 +#define QLZ_HASH_VALUES 2048 +#elif QLZ_COMPRESSION_LEVEL == 3 +#define QLZ_POINTERS 16 +#define QLZ_HASH_VALUES 4096 +#endif + +// Detect if pointer size is 64-bit. It's not fatal if some 64-bit target is not detected because this is only for adding an optional 64-bit optimization. +#if defined _LP64 || defined __LP64__ || defined __64BIT__ || _ADDR64 || defined _WIN64 || defined __arch64__ || __WORDSIZE == 64 || (defined __sparc && defined __sparcv9) || defined __x86_64 || defined __amd64 || defined __x86_64__ || defined _M_X64 || defined _M_IA64 || defined __ia64 || defined __IA64__ + #define QLZ_PTR_64 +#endif + +// hash entry +typedef struct +{ +#if QLZ_COMPRESSION_LEVEL == 1 + ui32 cache; +#if defined QLZ_PTR_64 && QLZ_STREAMING_BUFFER == 0 + unsigned int offset; +#else + const unsigned char *offset; +#endif +#else + const unsigned char *offset[QLZ_POINTERS]; +#endif + +} qlz_hash_compress; + +typedef struct +{ +#if QLZ_COMPRESSION_LEVEL == 1 + const unsigned char *offset; +#else + const unsigned char *offset[QLZ_POINTERS]; +#endif +} qlz_hash_decompress; + + +// states +typedef struct +{ + #if QLZ_STREAMING_BUFFER > 0 + unsigned char stream_buffer[QLZ_STREAMING_BUFFER]; + #endif + size_t stream_counter; + qlz_hash_compress hash[QLZ_HASH_VALUES]; + unsigned char hash_counter[QLZ_HASH_VALUES]; +} qlz_state_compress; + + +#if QLZ_COMPRESSION_LEVEL == 1 || QLZ_COMPRESSION_LEVEL == 2 + typedef struct + { +#if QLZ_STREAMING_BUFFER > 0 + unsigned char stream_buffer[QLZ_STREAMING_BUFFER]; +#endif + qlz_hash_decompress hash[QLZ_HASH_VALUES]; + unsigned char hash_counter[QLZ_HASH_VALUES]; + size_t stream_counter; + } qlz_state_decompress; +#elif QLZ_COMPRESSION_LEVEL == 3 + typedef struct + { +#if QLZ_STREAMING_BUFFER > 0 + unsigned char stream_buffer[QLZ_STREAMING_BUFFER]; +#endif +#if QLZ_COMPRESSION_LEVEL <= 2 + qlz_hash_decompress hash[QLZ_HASH_VALUES]; +#endif + size_t stream_counter; + } qlz_state_decompress; +#endif + + +#if defined (__cplusplus) +extern "C" { +#endif + +// Public functions of QuickLZ +size_t qlz_size_decompressed(const char *source); +size_t qlz_size_compressed(const char *source); +size_t qlz_compress(const void *source, char *destination, size_t size, qlz_state_compress *state); +size_t qlz_decompress(const char *source, void *destination, qlz_state_decompress *state); +int qlz_get_setting(int setting); +size_t qlz_size_header(const char *source); + +#if defined (__cplusplus) +} +#endif + +#endif + diff --git a/extra/mariabackup/read_filt.cc b/extra/mariabackup/read_filt.cc new file mode 100644 index 00000000000..8ebc735e99e --- /dev/null +++ b/extra/mariabackup/read_filt.cc @@ -0,0 +1,206 @@ +/****************************************************** +XtraBackup: hot backup tool for InnoDB +(c) 2009-2012 Percona Inc. +Originally Created 3/3/2009 Yasufumi Kinoshita +Written by Alexey Kopytov, Aleksandr Kuzminsky, Stewart Smith, Vadim Tkachenko, +Yasufumi Kinoshita, Ignacio Nin and Baron Schwartz. + +This program is free software; you can redistribute it and/or modify +it under the terms of the GNU General Public License as published by +the Free Software Foundation; version 2 of the License. + +This program is distributed in the hope that it will be useful, +but WITHOUT ANY WARRANTY; without even the implied warranty of +MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the +GNU General Public License for more details. + +You should have received a copy of the GNU General Public License +along with this program; if not, write to the Free Software +Foundation, Inc., 51 Franklin Street, Fifth Floor, Boston, MA 02110-1301, USA + +*******************************************************/ + +/* Data file read filter implementation */ + +#include "read_filt.h" +#include "common.h" +#include "fil_cur.h" +#include "xtrabackup.h" + +/****************************************************************//** +Perform read filter context initialization that is common to all read +filters. */ +static +void +common_init( +/*========*/ + xb_read_filt_ctxt_t* ctxt, /*!offset = 0; + ctxt->data_file_size = cursor->statinfo.st_size; + ctxt->buffer_capacity = cursor->buf_size; + ctxt->page_size = cursor->page_size; +} + +/****************************************************************//** +Initialize the pass-through read filter. */ +static +void +rf_pass_through_init( +/*=================*/ + xb_read_filt_ctxt_t* ctxt, /*!offset; + *read_batch_len = ctxt->data_file_size - ctxt->offset; + + if (*read_batch_len > ctxt->buffer_capacity) { + *read_batch_len = ctxt->buffer_capacity; + } + + ctxt->offset += *read_batch_len; +} + +/****************************************************************//** +Deinitialize the pass-through read filter. */ +static +void +rf_pass_through_deinit( +/*===================*/ + xb_read_filt_ctxt_t* ctxt __attribute__((unused))) + /*!bitmap_range = xb_page_bitmap_range_init(changed_page_bitmap, + space_id); + ctxt->filter_batch_end = 0; +} + +/****************************************************************//** +Get the next batch of pages for the bitmap read filter. */ +static +void +rf_bitmap_get_next_batch( +/*=====================*/ + xb_read_filt_ctxt_t* ctxt, /*!offset / ctxt->page_size; + + xb_a (ctxt->offset % ctxt->page_size == 0); + + if (start_page_id == ctxt->filter_batch_end) { + + /* Used up all the previous bitmap range, get some more */ + ulint next_page_id; + + /* Find the next changed page using the bitmap */ + next_page_id = xb_page_bitmap_range_get_next_bit + (ctxt->bitmap_range, TRUE); + + if (next_page_id == ULINT_UNDEFINED) { + *read_batch_len = 0; + return; + } + + ctxt->offset = next_page_id * ctxt->page_size; + + /* Find the end of the current changed page block by searching + for the next cleared bitmap bit */ + ctxt->filter_batch_end + = xb_page_bitmap_range_get_next_bit(ctxt->bitmap_range, + FALSE); + xb_a(next_page_id < ctxt->filter_batch_end); + } + + *read_batch_start = ctxt->offset; + if (ctxt->filter_batch_end == ULINT_UNDEFINED) { + /* No more cleared bits in the bitmap, need to copy all the + remaining pages. */ + *read_batch_len = ctxt->data_file_size - ctxt->offset; + } else { + *read_batch_len = ctxt->filter_batch_end * ctxt->page_size + - ctxt->offset; + } + + /* If the page block is larger than the buffer capacity, limit it to + buffer capacity. The subsequent invocations will continue returning + the current block in buffer-sized pieces until ctxt->filter_batch_end + is reached, trigerring the next bitmap query. */ + if (*read_batch_len > ctxt->buffer_capacity) { + *read_batch_len = ctxt->buffer_capacity; + } + + ctxt->offset += *read_batch_len; + xb_a (ctxt->offset % ctxt->page_size == 0); + xb_a (*read_batch_start % ctxt->page_size == 0); + xb_a (*read_batch_len % ctxt->page_size == 0); +} + +/****************************************************************//** +Deinitialize the changed page bitmap-based read filter. */ +static +void +rf_bitmap_deinit( +/*=============*/ + xb_read_filt_ctxt_t* ctxt) /*!bitmap_range); +} + +/* The pass-through read filter */ +xb_read_filt_t rf_pass_through = { + &rf_pass_through_init, + &rf_pass_through_get_next_batch, + &rf_pass_through_deinit +}; + +/* The changed page bitmap-based read filter */ +xb_read_filt_t rf_bitmap = { + &rf_bitmap_init, + &rf_bitmap_get_next_batch, + &rf_bitmap_deinit +}; diff --git a/extra/mariabackup/read_filt.h b/extra/mariabackup/read_filt.h new file mode 100644 index 00000000000..73fef06a288 --- /dev/null +++ b/extra/mariabackup/read_filt.h @@ -0,0 +1,62 @@ +/****************************************************** +XtraBackup: hot backup tool for InnoDB +(c) 2009-2012 Percona Inc. +Originally Created 3/3/2009 Yasufumi Kinoshita +Written by Alexey Kopytov, Aleksandr Kuzminsky, Stewart Smith, Vadim Tkachenko, +Yasufumi Kinoshita, Ignacio Nin and Baron Schwartz. + +This program is free software; you can redistribute it and/or modify +it under the terms of the GNU General Public License as published by +the Free Software Foundation; version 2 of the License. + +This program is distributed in the hope that it will be useful, +but WITHOUT ANY WARRANTY; without even the implied warranty of +MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the +GNU General Public License for more details. + +You should have received a copy of the GNU General Public License +along with this program; if not, write to the Free Software +Foundation, Inc., 51 Franklin Street, Fifth Floor, Boston, MA 02110-1301, USA + +*******************************************************/ + +/* Data file read filter interface */ + +#ifndef XB_READ_FILT_H +#define XB_READ_FILT_H + +#include "changed_page_bitmap.h" + +struct xb_fil_cur_t; + +/* The read filter context */ +struct xb_read_filt_ctxt_t { + ib_int64_t offset; /*!< current file offset */ + ib_int64_t data_file_size; /*!< data file size */ + ib_int64_t buffer_capacity;/*!< read buffer capacity */ + ulint space_id; /*!< space id */ + /* The following fields used only in bitmap filter */ + /* Move these to union if any other filters are added in future */ + xb_page_bitmap_range *bitmap_range; /*!< changed page bitmap range + iterator for space_id */ + ulint page_size; /*!< page size */ + ulint filter_batch_end;/*!< the ending page id of the + current changed page block in + the bitmap */ +}; + +/* The read filter */ +struct xb_read_filt_t { + void (*init)(xb_read_filt_ctxt_t* ctxt, + const xb_fil_cur_t* cursor, + ulint space_id); + void (*get_next_batch)(xb_read_filt_ctxt_t* ctxt, + ib_int64_t* read_batch_start, + ib_int64_t* read_batch_len); + void (*deinit)(xb_read_filt_ctxt_t* ctxt); +}; + +extern xb_read_filt_t rf_pass_through; +extern xb_read_filt_t rf_bitmap; + +#endif diff --git a/extra/mariabackup/version_check.pl b/extra/mariabackup/version_check.pl new file mode 100644 index 00000000000..865e2eacb4a --- /dev/null +++ b/extra/mariabackup/version_check.pl @@ -0,0 +1,1373 @@ +use warnings FATAL => 'all'; +use strict; +use English qw(-no_match_vars); +use POSIX "strftime"; + +my @required_perl_version = (5, 0, 5); +my $required_perl_version_old_style = 5.005; + +# check existence of DBD::mysql module +eval { + require DBD::mysql; +}; +my $dbd_mysql_installed = $EVAL_ERROR ? 0 : 1; + +my $now; +my %mysql; +my $prefix = "version_check"; + + +# ########################################################################### +# HTTPMicro package +# This package is a copy without comments from the original. The original +# with comments and its test file can be found in the Bazaar repository at, +# lib/HTTPMicro.pm +# t/lib/HTTPMicro.t +# See https://launchpad.net/percona-toolkit for more information. +# ########################################################################### +{ + +package HTTPMicro; +BEGIN { + $HTTPMicro::VERSION = '0.001'; +} +use strict; +use warnings; + +use Carp (); + + +my @attributes; +BEGIN { + @attributes = qw(agent timeout); + no strict 'refs'; + for my $accessor ( @attributes ) { + *{$accessor} = sub { + @_ > 1 ? $_[0]->{$accessor} = $_[1] : $_[0]->{$accessor}; + }; + } +} + +sub new { + my($class, %args) = @_; + (my $agent = $class) =~ s{::}{-}g; + my $self = { + agent => $agent . "/" . ($class->VERSION || 0), + timeout => 60, + }; + for my $key ( @attributes ) { + $self->{$key} = $args{$key} if exists $args{$key} + } + return bless $self, $class; +} + +my %DefaultPort = ( + http => 80, + https => 443, +); + +sub request { + my ($self, $method, $url, $args) = @_; + @_ == 3 || (@_ == 4 && ref $args eq 'HASH') + or Carp::croak(q/Usage: $http->request(METHOD, URL, [HASHREF])/); + $args ||= {}; # we keep some state in this during _request + + my $response; + for ( 0 .. 1 ) { + $response = eval { $self->_request($method, $url, $args) }; + last unless $@ && $method eq 'GET' + && $@ =~ m{^(?:Socket closed|Unexpected end)}; + } + + if (my $e = "$@") { + $response = { + success => q{}, + status => 599, + reason => 'Internal Exception', + content => $e, + headers => { + 'content-type' => 'text/plain', + 'content-length' => length $e, + } + }; + } + return $response; +} + +sub _request { + my ($self, $method, $url, $args) = @_; + + my ($scheme, $host, $port, $path_query) = $self->_split_url($url); + + my $request = { + method => $method, + scheme => $scheme, + host_port => ($port == $DefaultPort{$scheme} ? $host : "$host:$port"), + uri => $path_query, + headers => {}, + }; + + my $handle = HTTPMicro::Handle->new(timeout => $self->{timeout}); + + $handle->connect($scheme, $host, $port); + + $self->_prepare_headers_and_cb($request, $args); + $handle->write_request_header(@{$request}{qw/method uri headers/}); + $handle->write_content_body($request) if $request->{content}; + + my $response; + do { $response = $handle->read_response_header } + until (substr($response->{status},0,1) ne '1'); + + if (!($method eq 'HEAD' || $response->{status} =~ /^[23]04/)) { + $response->{content} = ''; + $handle->read_content_body(sub { $_[1]->{content} .= $_[0] }, $response); + } + + $handle->close; + $response->{success} = substr($response->{status},0,1) eq '2'; + return $response; +} + +sub _prepare_headers_and_cb { + my ($self, $request, $args) = @_; + + for ($args->{headers}) { + next unless defined; + while (my ($k, $v) = each %$_) { + $request->{headers}{lc $k} = $v; + } + } + $request->{headers}{'host'} = $request->{host_port}; + $request->{headers}{'connection'} = "close"; + $request->{headers}{'user-agent'} ||= $self->{agent}; + + if (defined $args->{content}) { + $request->{headers}{'content-type'} ||= "application/octet-stream"; + utf8::downgrade($args->{content}, 1) + or Carp::croak(q/Wide character in request message body/); + $request->{headers}{'content-length'} = length $args->{content}; + $request->{content} = $args->{content}; + } + return; +} + +sub _split_url { + my $url = pop; + + my ($scheme, $authority, $path_query) = $url =~ m<\A([^:/?#]+)://([^/?#]*)([^#]*)> + or Carp::croak(qq/Cannot parse URL: '$url'/); + + $scheme = lc $scheme; + $path_query = "/$path_query" unless $path_query =~ m<\A/>; + + my $host = (length($authority)) ? lc $authority : 'localhost'; + $host =~ s/\A[^@]*@//; # userinfo + my $port = do { + $host =~ s/:([0-9]*)\z// && length $1 + ? $1 + : $DefaultPort{$scheme} + }; + + return ($scheme, $host, $port, $path_query); +} + +package + HTTPMicro::Handle; # hide from PAUSE/indexers +use strict; +use warnings; + +use Carp qw[croak]; +use Errno qw[EINTR EPIPE]; +use IO::Socket qw[SOCK_STREAM]; + +sub BUFSIZE () { 32768 } + +my $Printable = sub { + local $_ = shift; + s/\r/\\r/g; + s/\n/\\n/g; + s/\t/\\t/g; + s/([^\x20-\x7E])/sprintf('\\x%.2X', ord($1))/ge; + $_; +}; + +sub new { + my ($class, %args) = @_; + return bless { + rbuf => '', + timeout => 60, + max_line_size => 16384, + %args + }, $class; +} + +my $ssl_verify_args = { + check_cn => "when_only", + wildcards_in_alt => "anywhere", + wildcards_in_cn => "anywhere" +}; + +sub connect { + @_ == 4 || croak(q/Usage: $handle->connect(scheme, host, port)/); + my ($self, $scheme, $host, $port) = @_; + + if ( $scheme eq 'https' ) { + eval "require IO::Socket::SSL" + unless exists $INC{'IO/Socket/SSL.pm'}; + croak(qq/IO::Socket::SSL must be installed for https support\n/) + unless $INC{'IO/Socket/SSL.pm'}; + } + elsif ( $scheme ne 'http' ) { + croak(qq/Unsupported URL scheme '$scheme'\n/); + } + + $self->{fh} = 'IO::Socket::INET'->new( + PeerHost => $host, + PeerPort => $port, + Proto => 'tcp', + Type => SOCK_STREAM, + Timeout => $self->{timeout} + ) or croak(qq/Could not connect to '$host:$port': $@/); + + binmode($self->{fh}) + or croak(qq/Could not binmode() socket: '$!'/); + + if ( $scheme eq 'https') { + IO::Socket::SSL->start_SSL($self->{fh}); + ref($self->{fh}) eq 'IO::Socket::SSL' + or die(qq/SSL connection failed for $host\n/); + if ( $self->{fh}->can("verify_hostname") ) { + $self->{fh}->verify_hostname( $host, $ssl_verify_args ) + or die(qq/SSL certificate not valid for $host\n/); + } + else { + my $fh = $self->{fh}; + _verify_hostname_of_cert($host, _peer_certificate($fh), $ssl_verify_args) + or die(qq/SSL certificate not valid for $host\n/); + } + } + + $self->{host} = $host; + $self->{port} = $port; + + return $self; +} + +sub close { + @_ == 1 || croak(q/Usage: $handle->close()/); + my ($self) = @_; + CORE::close($self->{fh}) + or croak(qq/Could not close socket: '$!'/); +} + +sub write { + @_ == 2 || croak(q/Usage: $handle->write(buf)/); + my ($self, $buf) = @_; + + my $len = length $buf; + my $off = 0; + + local $SIG{PIPE} = 'IGNORE'; + + while () { + $self->can_write + or croak(q/Timed out while waiting for socket to become ready for writing/); + my $r = syswrite($self->{fh}, $buf, $len, $off); + if (defined $r) { + $len -= $r; + $off += $r; + last unless $len > 0; + } + elsif ($! == EPIPE) { + croak(qq/Socket closed by remote server: $!/); + } + elsif ($! != EINTR) { + croak(qq/Could not write to socket: '$!'/); + } + } + return $off; +} + +sub read { + @_ == 2 || @_ == 3 || croak(q/Usage: $handle->read(len)/); + my ($self, $len) = @_; + + my $buf = ''; + my $got = length $self->{rbuf}; + + if ($got) { + my $take = ($got < $len) ? $got : $len; + $buf = substr($self->{rbuf}, 0, $take, ''); + $len -= $take; + } + + while ($len > 0) { + $self->can_read + or croak(q/Timed out while waiting for socket to become ready for reading/); + my $r = sysread($self->{fh}, $buf, $len, length $buf); + if (defined $r) { + last unless $r; + $len -= $r; + } + elsif ($! != EINTR) { + croak(qq/Could not read from socket: '$!'/); + } + } + if ($len) { + croak(q/Unexpected end of stream/); + } + return $buf; +} + +sub readline { + @_ == 1 || croak(q/Usage: $handle->readline()/); + my ($self) = @_; + + while () { + if ($self->{rbuf} =~ s/\A ([^\x0D\x0A]* \x0D?\x0A)//x) { + return $1; + } + $self->can_read + or croak(q/Timed out while waiting for socket to become ready for reading/); + my $r = sysread($self->{fh}, $self->{rbuf}, BUFSIZE, length $self->{rbuf}); + if (defined $r) { + last unless $r; + } + elsif ($! != EINTR) { + croak(qq/Could not read from socket: '$!'/); + } + } + croak(q/Unexpected end of stream while looking for line/); +} + +sub read_header_lines { + @_ == 1 || @_ == 2 || croak(q/Usage: $handle->read_header_lines([headers])/); + my ($self, $headers) = @_; + $headers ||= {}; + my $lines = 0; + my $val; + + while () { + my $line = $self->readline; + + if ($line =~ /\A ([^\x00-\x1F\x7F:]+) : [\x09\x20]* ([^\x0D\x0A]*)/x) { + my ($field_name) = lc $1; + $val = \($headers->{$field_name} = $2); + } + elsif ($line =~ /\A [\x09\x20]+ ([^\x0D\x0A]*)/x) { + $val + or croak(q/Unexpected header continuation line/); + next unless length $1; + $$val .= ' ' if length $$val; + $$val .= $1; + } + elsif ($line =~ /\A \x0D?\x0A \z/x) { + last; + } + else { + croak(q/Malformed header line: / . $Printable->($line)); + } + } + return $headers; +} + +sub write_header_lines { + (@_ == 2 && ref $_[1] eq 'HASH') || croak(q/Usage: $handle->write_header_lines(headers)/); + my($self, $headers) = @_; + + my $buf = ''; + while (my ($k, $v) = each %$headers) { + my $field_name = lc $k; + $field_name =~ /\A [\x21\x23-\x27\x2A\x2B\x2D\x2E\x30-\x39\x41-\x5A\x5E-\x7A\x7C\x7E]+ \z/x + or croak(q/Invalid HTTP header field name: / . $Printable->($field_name)); + $field_name =~ s/\b(\w)/\u$1/g; + $buf .= "$field_name: $v\x0D\x0A"; + } + $buf .= "\x0D\x0A"; + return $self->write($buf); +} + +sub read_content_body { + @_ == 3 || @_ == 4 || croak(q/Usage: $handle->read_content_body(callback, response, [read_length])/); + my ($self, $cb, $response, $len) = @_; + $len ||= $response->{headers}{'content-length'}; + + croak("No content-length in the returned response, and this " + . "UA doesn't implement chunking") unless defined $len; + + while ($len > 0) { + my $read = ($len > BUFSIZE) ? BUFSIZE : $len; + $cb->($self->read($read), $response); + $len -= $read; + } + + return; +} + +sub write_content_body { + @_ == 2 || croak(q/Usage: $handle->write_content_body(request)/); + my ($self, $request) = @_; + my ($len, $content_length) = (0, $request->{headers}{'content-length'}); + + $len += $self->write($request->{content}); + + $len == $content_length + or croak(qq/Content-Length missmatch (got: $len expected: $content_length)/); + + return $len; +} + +sub read_response_header { + @_ == 1 || croak(q/Usage: $handle->read_response_header()/); + my ($self) = @_; + + my $line = $self->readline; + + $line =~ /\A (HTTP\/(0*\d+\.0*\d+)) [\x09\x20]+ ([0-9]{3}) [\x09\x20]+ ([^\x0D\x0A]*) \x0D?\x0A/x + or croak(q/Malformed Status-Line: / . $Printable->($line)); + + my ($protocol, $version, $status, $reason) = ($1, $2, $3, $4); + + return { + status => $status, + reason => $reason, + headers => $self->read_header_lines, + protocol => $protocol, + }; +} + +sub write_request_header { + @_ == 4 || croak(q/Usage: $handle->write_request_header(method, request_uri, headers)/); + my ($self, $method, $request_uri, $headers) = @_; + + return $self->write("$method $request_uri HTTP/1.1\x0D\x0A") + + $self->write_header_lines($headers); +} + +sub _do_timeout { + my ($self, $type, $timeout) = @_; + $timeout = $self->{timeout} + unless defined $timeout && $timeout >= 0; + + my $fd = fileno $self->{fh}; + defined $fd && $fd >= 0 + or croak(q/select(2): 'Bad file descriptor'/); + + my $initial = time; + my $pending = $timeout; + my $nfound; + + vec(my $fdset = '', $fd, 1) = 1; + + while () { + $nfound = ($type eq 'read') + ? select($fdset, undef, undef, $pending) + : select(undef, $fdset, undef, $pending) ; + if ($nfound == -1) { + $! == EINTR + or croak(qq/select(2): '$!'/); + redo if !$timeout || ($pending = $timeout - (time - $initial)) > 0; + $nfound = 0; + } + last; + } + $! = 0; + return $nfound; +} + +sub can_read { + @_ == 1 || @_ == 2 || croak(q/Usage: $handle->can_read([timeout])/); + my $self = shift; + return $self->_do_timeout('read', @_) +} + +sub can_write { + @_ == 1 || @_ == 2 || croak(q/Usage: $handle->can_write([timeout])/); + my $self = shift; + return $self->_do_timeout('write', @_) +} + +my $prog = <<'EOP'; +BEGIN { + if ( defined &IO::Socket::SSL::CAN_IPV6 ) { + *CAN_IPV6 = \*IO::Socket::SSL::CAN_IPV6; + } + else { + constant->import( CAN_IPV6 => '' ); + } + my %const = ( + NID_CommonName => 13, + GEN_DNS => 2, + GEN_IPADD => 7, + ); + while ( my ($name,$value) = each %const ) { + no strict 'refs'; + *{$name} = UNIVERSAL::can( 'Net::SSLeay', $name ) || sub { $value }; + } +} +{ + my %dispatcher = ( + issuer => sub { Net::SSLeay::X509_NAME_oneline( Net::SSLeay::X509_get_issuer_name( shift )) }, + subject => sub { Net::SSLeay::X509_NAME_oneline( Net::SSLeay::X509_get_subject_name( shift )) }, + ); + if ( $Net::SSLeay::VERSION >= 1.30 ) { + $dispatcher{commonName} = sub { + my $cn = Net::SSLeay::X509_NAME_get_text_by_NID( + Net::SSLeay::X509_get_subject_name( shift ), NID_CommonName); + $cn =~s{\0$}{}; # work around Bug in Net::SSLeay <1.33 + $cn; + } + } else { + $dispatcher{commonName} = sub { + croak "you need at least Net::SSLeay version 1.30 for getting commonName" + } + } + + if ( $Net::SSLeay::VERSION >= 1.33 ) { + $dispatcher{subjectAltNames} = sub { Net::SSLeay::X509_get_subjectAltNames( shift ) }; + } else { + $dispatcher{subjectAltNames} = sub { + return; + }; + } + + $dispatcher{authority} = $dispatcher{issuer}; + $dispatcher{owner} = $dispatcher{subject}; + $dispatcher{cn} = $dispatcher{commonName}; + + sub _peer_certificate { + my ($self, $field) = @_; + my $ssl = $self->_get_ssl_object or return; + + my $cert = ${*$self}{_SSL_certificate} + ||= Net::SSLeay::get_peer_certificate($ssl) + or return $self->error("Could not retrieve peer certificate"); + + if ($field) { + my $sub = $dispatcher{$field} or croak + "invalid argument for peer_certificate, valid are: ".join( " ",keys %dispatcher ). + "\nMaybe you need to upgrade your Net::SSLeay"; + return $sub->($cert); + } else { + return $cert + } + } + + + my %scheme = ( + ldap => { + wildcards_in_cn => 0, + wildcards_in_alt => 'leftmost', + check_cn => 'always', + }, + http => { + wildcards_in_cn => 'anywhere', + wildcards_in_alt => 'anywhere', + check_cn => 'when_only', + }, + smtp => { + wildcards_in_cn => 0, + wildcards_in_alt => 0, + check_cn => 'always' + }, + none => {}, # do not check + ); + + $scheme{www} = $scheme{http}; # alias + $scheme{xmpp} = $scheme{http}; # rfc 3920 + $scheme{pop3} = $scheme{ldap}; # rfc 2595 + $scheme{imap} = $scheme{ldap}; # rfc 2595 + $scheme{acap} = $scheme{ldap}; # rfc 2595 + $scheme{nntp} = $scheme{ldap}; # rfc 4642 + $scheme{ftp} = $scheme{http}; # rfc 4217 + + + sub _verify_hostname_of_cert { + my $identity = shift; + my $cert = shift; + my $scheme = shift || 'none'; + if ( ! ref($scheme) ) { + $scheme = $scheme{$scheme} or croak "scheme $scheme not defined"; + } + + return 1 if ! %$scheme; # 'none' + + my $commonName = $dispatcher{cn}->($cert); + my @altNames = $dispatcher{subjectAltNames}->($cert); + + if ( my $sub = $scheme->{callback} ) { + return $sub->($identity,$commonName,@altNames); + } + + + my $ipn; + if ( CAN_IPV6 and $identity =~m{:} ) { + $ipn = IO::Socket::SSL::inet_pton(IO::Socket::SSL::AF_INET6,$identity) + or croak "'$identity' is not IPv6, but neither IPv4 nor hostname"; + } elsif ( $identity =~m{^\d+\.\d+\.\d+\.\d+$} ) { + $ipn = IO::Socket::SSL::inet_aton( $identity ) or croak "'$identity' is not IPv4, but neither IPv6 nor hostname"; + } else { + if ( $identity =~m{[^a-zA-Z0-9_.\-]} ) { + $identity =~m{\0} and croak("name '$identity' has \\0 byte"); + $identity = IO::Socket::SSL::idn_to_ascii($identity) or + croak "Warning: Given name '$identity' could not be converted to IDNA!"; + } + } + + my $check_name = sub { + my ($name,$identity,$wtyp) = @_; + $wtyp ||= ''; + my $pattern; + if ( $wtyp eq 'anywhere' and $name =~m{^([a-zA-Z0-9_\-]*)\*(.+)} ) { + $pattern = qr{^\Q$1\E[a-zA-Z0-9_\-]*\Q$2\E$}i; + } elsif ( $wtyp eq 'leftmost' and $name =~m{^\*(\..+)$} ) { + $pattern = qr{^[a-zA-Z0-9_\-]*\Q$1\E$}i; + } else { + $pattern = qr{^\Q$name\E$}i; + } + return $identity =~ $pattern; + }; + + my $alt_dnsNames = 0; + while (@altNames) { + my ($type, $name) = splice (@altNames, 0, 2); + if ( $ipn and $type == GEN_IPADD ) { + return 1 if $ipn eq $name; + + } elsif ( ! $ipn and $type == GEN_DNS ) { + $name =~s/\s+$//; $name =~s/^\s+//; + $alt_dnsNames++; + $check_name->($name,$identity,$scheme->{wildcards_in_alt}) + and return 1; + } + } + + if ( ! $ipn and ( + $scheme->{check_cn} eq 'always' or + $scheme->{check_cn} eq 'when_only' and !$alt_dnsNames)) { + $check_name->($commonName,$identity,$scheme->{wildcards_in_cn}) + and return 1; + } + + return 0; # no match + } +} +EOP + +eval { require IO::Socket::SSL }; +if ( $INC{"IO/Socket/SSL.pm"} ) { + eval $prog; + die $@ if $@; +} + +1; +} +# ########################################################################### +# End HTTPMicro package +# ########################################################################### + +# ########################################################################### +# VersionCheck package +# This package is a copy without comments from the original. The original +# with comments and its test file can be found in the Bazaar repository at, +# lib/VersionCheck.pm +# t/lib/VersionCheck.t +# See https://launchpad.net/percona-toolkit for more information. +# ########################################################################### +{ +package VersionCheck; + + +use strict; +use warnings FATAL => 'all'; +use English qw(-no_match_vars); + +use constant PTDEBUG => $ENV{PTDEBUG} || 0; + +use Data::Dumper; +local $Data::Dumper::Indent = 1; +local $Data::Dumper::Sortkeys = 1; +local $Data::Dumper::Quotekeys = 0; + +use Digest::MD5 qw(md5_hex); +use Sys::Hostname qw(hostname); +use File::Basename qw(); +use File::Spec; +use FindBin qw(); + +eval { + require Percona::Toolkit; + require HTTPMicro; +}; + +{ + my $file = 'percona-version-check'; + my $home = $ENV{HOME} || $ENV{HOMEPATH} || $ENV{USERPROFILE} || '.'; + my @vc_dirs = ( + '/etc/percona', + '/etc/percona-toolkit', + '/tmp', + "$home", + ); + + if ($ENV{PTDEBUG_VERSION_CHECK_HOME}) { + @vc_dirs = ( $ENV{PTDEBUG_VERSION_CHECK_HOME} ); + } + + sub version_check_file { + foreach my $dir ( @vc_dirs ) { + if ( -d $dir && -w $dir ) { + PTDEBUG && _d('Version check file', $file, 'in', $dir); + return $dir . '/' . $file; + } + } + PTDEBUG && _d('Version check file', $file, 'in', $ENV{PWD}); + return $file; # in the CWD + } +} + +sub version_check_time_limit { + return 60 * 60 * 24; # one day +} + + +sub version_check { + my (%args) = @_; + + my $instances = $args{instances} || []; + my $instances_to_check; + + PTDEBUG && _d('FindBin::Bin:', $FindBin::Bin); + if ( !$args{force} ) { + if ( $FindBin::Bin + && (-d "$FindBin::Bin/../.bzr" || -d "$FindBin::Bin/../../.bzr") ) { + PTDEBUG && _d("$FindBin::Bin/../.bzr disables --version-check"); + return; + } + } + + eval { + foreach my $instance ( @$instances ) { + my ($name, $id) = get_instance_id($instance); + $instance->{name} = $name; + $instance->{id} = $id; + } + + push @$instances, { name => 'system', id => 0 }; + + $instances_to_check = get_instances_to_check( + instances => $instances, + vc_file => $args{vc_file}, # testing + now => $args{now}, # testing + ); + PTDEBUG && _d(scalar @$instances_to_check, 'instances to check'); + return unless @$instances_to_check; + + my $protocol = 'https'; + eval { require IO::Socket::SSL; }; + if ( $EVAL_ERROR ) { + PTDEBUG && _d($EVAL_ERROR); + PTDEBUG && _d("SSL not available, won't run version_check"); + return; + } + PTDEBUG && _d('Using', $protocol); + + my $advice = pingback( + instances => $instances_to_check, + protocol => $protocol, + url => $args{url} # testing + || $ENV{PERCONA_VERSION_CHECK_URL} # testing + || "$protocol://v.percona.com", + ); + if ( $advice ) { + PTDEBUG && _d('Advice:', Dumper($advice)); + if ( scalar @$advice > 1) { + print "\n# " . scalar @$advice . " software updates are " + . "available:\n"; + } + else { + print "\n# A software update is available:\n"; + } + print join("\n", map { "# * $_" } @$advice), "\n\n"; + } + }; + if ( $EVAL_ERROR ) { + PTDEBUG && _d('Version check failed:', $EVAL_ERROR); + } + + if ( @$instances_to_check ) { + eval { + update_check_times( + instances => $instances_to_check, + vc_file => $args{vc_file}, # testing + now => $args{now}, # testing + ); + }; + if ( $EVAL_ERROR ) { + PTDEBUG && _d('Error updating version check file:', $EVAL_ERROR); + } + } + + if ( $ENV{PTDEBUG_VERSION_CHECK} ) { + warn "Exiting because the PTDEBUG_VERSION_CHECK " + . "environment variable is defined.\n"; + exit 255; + } + + return; +} + +sub get_instances_to_check { + my (%args) = @_; + + my $instances = $args{instances}; + my $now = $args{now} || int(time); + my $vc_file = $args{vc_file} || version_check_file(); + + if ( !-f $vc_file ) { + PTDEBUG && _d('Version check file', $vc_file, 'does not exist;', + 'version checking all instances'); + return $instances; + } + + open my $fh, '<', $vc_file or die "Cannot open $vc_file: $OS_ERROR"; + chomp(my $file_contents = do { local $/ = undef; <$fh> }); + PTDEBUG && _d('Version check file', $vc_file, 'contents:', $file_contents); + close $fh; + my %last_check_time_for = $file_contents =~ /^([^,]+),(.+)$/mg; + + my $check_time_limit = version_check_time_limit(); + my @instances_to_check; + foreach my $instance ( @$instances ) { + my $last_check_time = $last_check_time_for{ $instance->{id} }; + PTDEBUG && _d('Intsance', $instance->{id}, 'last checked', + $last_check_time, 'now', $now, 'diff', $now - ($last_check_time || 0), + 'hours until next check', + sprintf '%.2f', + ($check_time_limit - ($now - ($last_check_time || 0))) / 3600); + if ( !defined $last_check_time + || ($now - $last_check_time) >= $check_time_limit ) { + PTDEBUG && _d('Time to check', Dumper($instance)); + push @instances_to_check, $instance; + } + } + + return \@instances_to_check; +} + +sub update_check_times { + my (%args) = @_; + + my $instances = $args{instances}; + my $now = $args{now} || int(time); + my $vc_file = $args{vc_file} || version_check_file(); + PTDEBUG && _d('Updating last check time:', $now); + + my %all_instances = map { + $_->{id} => { name => $_->{name}, ts => $now } + } @$instances; + + if ( -f $vc_file ) { + open my $fh, '<', $vc_file or die "Cannot read $vc_file: $OS_ERROR"; + my $contents = do { local $/ = undef; <$fh> }; + close $fh; + + foreach my $line ( split("\n", ($contents || '')) ) { + my ($id, $ts) = split(',', $line); + if ( !exists $all_instances{$id} ) { + $all_instances{$id} = { ts => $ts }; # original ts, not updated + } + } + } + + open my $fh, '>', $vc_file or die "Cannot write to $vc_file: $OS_ERROR"; + foreach my $id ( sort keys %all_instances ) { + PTDEBUG && _d('Updated:', $id, Dumper($all_instances{$id})); + print { $fh } $id . ',' . $all_instances{$id}->{ts} . "\n"; + } + close $fh; + + return; +} + +sub get_instance_id { + my ($instance) = @_; + + my $dbh = $instance->{dbh}; + my $dsn = $instance->{dsn}; + + my $sql = q{SELECT CONCAT(@@hostname, @@port)}; + PTDEBUG && _d($sql); + my ($name) = eval { $dbh->selectrow_array($sql) }; + if ( $EVAL_ERROR ) { + PTDEBUG && _d($EVAL_ERROR); + $sql = q{SELECT @@hostname}; + PTDEBUG && _d($sql); + ($name) = eval { $dbh->selectrow_array($sql) }; + if ( $EVAL_ERROR ) { + PTDEBUG && _d($EVAL_ERROR); + $name = ($dsn->{h} || 'localhost') . ($dsn->{P} || 3306); + } + else { + $sql = q{SHOW VARIABLES LIKE 'port'}; + PTDEBUG && _d($sql); + my (undef, $port) = eval { $dbh->selectrow_array($sql) }; + PTDEBUG && _d('port:', $port); + $name .= $port || ''; + } + } + my $id = md5_hex($name); + + PTDEBUG && _d('MySQL instance:', $id, $name, Dumper($dsn)); + + return $name, $id; +} + + +sub pingback { + my (%args) = @_; + my @required_args = qw(url instances); + foreach my $arg ( @required_args ) { + die "I need a $arg arugment" unless $args{$arg}; + } + my $url = $args{url}; + my $instances = $args{instances}; + + my $ua = $args{ua} || HTTPMicro->new( timeout => 3 ); + + my $response = $ua->request('GET', $url); + PTDEBUG && _d('Server response:', Dumper($response)); + die "No response from GET $url" + if !$response; + die("GET on $url returned HTTP status $response->{status}; expected 200\n", + ($response->{content} || '')) if $response->{status} != 200; + die("GET on $url did not return any programs to check") + if !$response->{content}; + + my $items = parse_server_response( + response => $response->{content} + ); + die "Failed to parse server requested programs: $response->{content}" + if !scalar keys %$items; + + my $versions = get_versions( + items => $items, + instances => $instances, + ); + die "Failed to get any program versions; should have at least gotten Perl" + if !scalar keys %$versions; + + my $client_content = encode_client_response( + items => $items, + versions => $versions, + general_id => md5_hex( hostname() ), + ); + + my $client_response = { + headers => { "X-Percona-Toolkit-Tool" => File::Basename::basename($0) }, + content => $client_content, + }; + PTDEBUG && _d('Client response:', Dumper($client_response)); + + $response = $ua->request('POST', $url, $client_response); + PTDEBUG && _d('Server suggestions:', Dumper($response)); + die "No response from POST $url $client_response" + if !$response; + die "POST $url returned HTTP status $response->{status}; expected 200" + if $response->{status} != 200; + + return unless $response->{content}; + + $items = parse_server_response( + response => $response->{content}, + split_vars => 0, + ); + die "Failed to parse server suggestions: $response->{content}" + if !scalar keys %$items; + my @suggestions = map { $_->{vars} } + sort { $a->{item} cmp $b->{item} } + values %$items; + + return \@suggestions; +} + +sub encode_client_response { + my (%args) = @_; + my @required_args = qw(items versions general_id); + foreach my $arg ( @required_args ) { + die "I need a $arg arugment" unless $args{$arg}; + } + my ($items, $versions, $general_id) = @args{@required_args}; + + my @lines; + foreach my $item ( sort keys %$items ) { + next unless exists $versions->{$item}; + if ( ref($versions->{$item}) eq 'HASH' ) { + my $mysql_versions = $versions->{$item}; + for my $id ( sort keys %$mysql_versions ) { + push @lines, join(';', $id, $item, $mysql_versions->{$id}); + } + } + else { + push @lines, join(';', $general_id, $item, $versions->{$item}); + } + } + + my $client_response = join("\n", @lines) . "\n"; + return $client_response; +} + +sub parse_server_response { + my (%args) = @_; + my @required_args = qw(response); + foreach my $arg ( @required_args ) { + die "I need a $arg arugment" unless $args{$arg}; + } + my ($response) = @args{@required_args}; + + my %items = map { + my ($item, $type, $vars) = split(";", $_); + if ( !defined $args{split_vars} || $args{split_vars} ) { + $vars = [ split(",", ($vars || '')) ]; + } + $item => { + item => $item, + type => $type, + vars => $vars, + }; + } split("\n", $response); + + PTDEBUG && _d('Items:', Dumper(\%items)); + + return \%items; +} + +my %sub_for_type = ( + os_version => \&get_os_version, + perl_version => \&get_perl_version, + perl_module_version => \&get_perl_module_version, + mysql_variable => \&get_mysql_variable, +); + +sub valid_item { + my ($item) = @_; + return unless $item; + if ( !exists $sub_for_type{ $item->{type} } ) { + PTDEBUG && _d('Invalid type:', $item->{type}); + return 0; + } + return 1; +} + +sub get_versions { + my (%args) = @_; + my @required_args = qw(items); + foreach my $arg ( @required_args ) { + die "I need a $arg arugment" unless $args{$arg}; + } + my ($items) = @args{@required_args}; + + my %versions; + foreach my $item ( values %$items ) { + next unless valid_item($item); + eval { + my $version = $sub_for_type{ $item->{type} }->( + item => $item, + instances => $args{instances}, + ); + if ( $version ) { + chomp $version unless ref($version); + $versions{$item->{item}} = $version; + } + }; + if ( $EVAL_ERROR ) { + PTDEBUG && _d('Error getting version for', Dumper($item), $EVAL_ERROR); + } + } + + return \%versions; +} + + +sub get_os_version { + if ( $OSNAME eq 'MSWin32' ) { + require Win32; + return Win32::GetOSDisplayName(); + } + + chomp(my $platform = `uname -s`); + PTDEBUG && _d('platform:', $platform); + return $OSNAME unless $platform; + + chomp(my $lsb_release + = `which lsb_release 2>/dev/null | awk '{print \$1}'` || ''); + PTDEBUG && _d('lsb_release:', $lsb_release); + + my $release = ""; + + if ( $platform eq 'Linux' ) { + if ( -f "/etc/fedora-release" ) { + $release = `cat /etc/fedora-release`; + } + elsif ( -f "/etc/redhat-release" ) { + $release = `cat /etc/redhat-release`; + } + elsif ( -f "/etc/system-release" ) { + $release = `cat /etc/system-release`; + } + elsif ( $lsb_release ) { + $release = `$lsb_release -ds`; + } + elsif ( -f "/etc/lsb-release" ) { + $release = `grep DISTRIB_DESCRIPTION /etc/lsb-release`; + $release =~ s/^\w+="([^"]+)".+/$1/; + } + elsif ( -f "/etc/debian_version" ) { + chomp(my $rel = `cat /etc/debian_version`); + $release = "Debian $rel"; + if ( -f "/etc/apt/sources.list" ) { + chomp(my $code_name = `awk '/^deb/ {print \$3}' /etc/apt/sources.list | awk -F/ '{print \$1}'| awk 'BEGIN {FS="|"} {print \$1}' | sort | uniq -c | sort -rn | head -n1 | awk '{print \$2}'`); + $release .= " ($code_name)" if $code_name; + } + } + elsif ( -f "/etc/os-release" ) { # openSUSE + chomp($release = `grep PRETTY_NAME /etc/os-release`); + $release =~ s/^PRETTY_NAME="(.+)"$/$1/; + } + elsif ( `ls /etc/*release 2>/dev/null` ) { + if ( `grep DISTRIB_DESCRIPTION /etc/*release 2>/dev/null` ) { + $release = `grep DISTRIB_DESCRIPTION /etc/*release | head -n1`; + } + else { + $release = `cat /etc/*release | head -n1`; + } + } + } + elsif ( $platform =~ m/(?:BSD|^Darwin)$/ ) { + my $rel = `uname -r`; + $release = "$platform $rel"; + } + elsif ( $platform eq "SunOS" ) { + my $rel = `head -n1 /etc/release` || `uname -r`; + $release = "$platform $rel"; + } + + if ( !$release ) { + PTDEBUG && _d('Failed to get the release, using platform'); + $release = $platform; + } + chomp($release); + + $release =~ s/^"|"$//g; + + PTDEBUG && _d('OS version =', $release); + return $release; +} + +sub get_perl_version { + my (%args) = @_; + my $item = $args{item}; + return unless $item; + + my $version = sprintf '%vd', $PERL_VERSION; + PTDEBUG && _d('Perl version', $version); + return $version; +} + +sub get_perl_module_version { + my (%args) = @_; + my $item = $args{item}; + return unless $item; + + my $var = '$' . $item->{item} . '::VERSION'; + my $version = eval "use $item->{item}; $var;"; + PTDEBUG && _d('Perl version for', $var, '=', $version); + return $version; +} + +sub get_mysql_variable { + return get_from_mysql( + show => 'VARIABLES', + @_, + ); +} + +sub get_from_mysql { + my (%args) = @_; + my $show = $args{show}; + my $item = $args{item}; + my $instances = $args{instances}; + return unless $show && $item; + + if ( !$instances || !@$instances ) { + PTDEBUG && _d('Cannot check', $item, + 'because there are no MySQL instances'); + return; + } + + if ($item->{item} eq 'MySQL' && $item->{type} eq 'mysql_variable') { + $item->{vars} = ['version_comment', 'version']; + } + + my @versions; + my %version_for; + foreach my $instance ( @$instances ) { + next unless $instance->{id}; # special system instance has id=0 + my $dbh = $instance->{dbh}; + local $dbh->{FetchHashKeyName} = 'NAME_lc'; + my $sql = qq/SHOW $show/; + PTDEBUG && _d($sql); + my $rows = $dbh->selectall_hashref($sql, 'variable_name'); + + my @versions; + foreach my $var ( @{$item->{vars}} ) { + $var = lc($var); + my $version = $rows->{$var}->{value}; + PTDEBUG && _d('MySQL version for', $item->{item}, '=', $version, + 'on', $instance->{name}); + push @versions, $version; + } + $version_for{ $instance->{id} } = join(' ', @versions); + } + + return \%version_for; +} + +sub _d { + my ($package, undef, $line) = caller 0; + @_ = map { (my $temp = $_) =~ s/\n/\n# /g; $temp; } + map { defined $_ ? $_ : 'undef' } + @_; + print STDERR "# $package:$line $PID ", join(' ', @_), "\n"; +} + +1; +} +# ########################################################################### +# End VersionCheck package +# ########################################################################### + +# +# parse_connection_options() subroutine parses connection-related command line +# options +# +sub parse_connection_options { + my $con = shift; + + $con->{dsn} = 'dbi:mysql:'; + + # this option has to be first + if ($ENV{option_defaults_file}) { + $con->{dsn} .= ";mysql_read_default_file=$ENV{option_defaults_file}"; + } + + if ($ENV{option_defaults_extra_file}) { + $con->{dsn} .= ";mysql_read_default_file=$ENV{option_defaults_extra_file}"; + } + + $con->{dsn} .= ";mysql_read_default_group=xtrabackup"; + + if ($ENV{option_mysql_password}) { + $con->{dsn_password} = "$ENV{option_mysql_password}"; + } + if ($ENV{option_mysql_user}) { + $con->{dsn_user} = "$ENV{option_mysql_user}"; + } + if ($ENV{option_mysql_host}) { + $con->{dsn} .= ";host=$ENV{option_mysql_host}"; + } + if ($ENV{option_mysql_port}) { + $con->{dsn} .= ";port=$ENV{option_mysql_port}"; + } + if ($ENV{option_mysql_socket}) { + $con->{dsn} .= ";mysql_socket=$ENV{option_mysql_socket}"; + } +} + +# +# mysql_connect subroutine connects to MySQL server +# +sub mysql_connect { + my %con; + my %args = ( + # Defaults + abort_on_error => 1, + @_ + ); + + $con{abort_on_error} = $args{abort_on_error}; + + parse_connection_options(\%con); + + $now = current_time(); + print STDERR "$now $prefix Connecting to MySQL server with DSN '$con{dsn}'" . + (defined($con{dsn_user}) ? " as '$con{dsn_user}' " : "") . + " (using password: "; + if (defined($con{dsn_password})) { + print STDERR "YES).\n"; + } else { + print STDERR "NO).\n"; + } + + eval { + $con{dbh}=DBI->connect($con{dsn}, $con{dsn_user}, + $con{dsn_password}, { RaiseError => 1 }); + }; + + if ($EVAL_ERROR) { + $con{connect_error}=$EVAL_ERROR; + } else { + $now = current_time(); + print STDERR "$now $prefix Connected to MySQL server\n"; + } + + if ($args{abort_on_error}) { + if (!$dbd_mysql_installed) { + die "Failed to connect to MySQL server as " . + "DBD::mysql module is not installed"; + } else { + if (!$con{dbh}) { + die "Failed to connect to MySQL server: " . + $con{connect_error}; + } + } + } + + if ($con{dbh}) { + $con{dbh}->do("SET SESSION wait_timeout=2147483"); + } + + return %con; +} + +# +# return current local time as string in form "070816 12:23:15" +# +sub current_time { + return strftime("%y%m%d %H:%M:%S", localtime()); +} + + +%mysql = mysql_connect(abort_on_error => 1); + +$now = current_time(); +print STDERR + "$now $prefix Executing a version check against the server...\n"; + +# Redirect STDOUT to STDERR, as VersionCheck prints alerts to STDOUT +select STDERR; + +VersionCheck::version_check( + force => 1, + instances => [ { + dbh => $mysql{dbh}, + dsn => $mysql{dsn} + } + ] + ); +# Restore STDOUT as the default filehandle +select STDOUT; + +$now = current_time(); +print STDERR "$now $prefix Done.\n"; diff --git a/extra/mariabackup/write_filt.cc b/extra/mariabackup/write_filt.cc new file mode 100644 index 00000000000..129302d7fa0 --- /dev/null +++ b/extra/mariabackup/write_filt.cc @@ -0,0 +1,219 @@ +/****************************************************** +XtraBackup: hot backup tool for InnoDB +(c) 2009-2013 Percona LLC and/or its affiliates. +Originally Created 3/3/2009 Yasufumi Kinoshita +Written by Alexey Kopytov, Aleksandr Kuzminsky, Stewart Smith, Vadim Tkachenko, +Yasufumi Kinoshita, Ignacio Nin and Baron Schwartz. + +This program is free software; you can redistribute it and/or modify +it under the terms of the GNU General Public License as published by +the Free Software Foundation; version 2 of the License. + +This program is distributed in the hope that it will be useful, +but WITHOUT ANY WARRANTY; without even the implied warranty of +MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the +GNU General Public License for more details. + +You should have received a copy of the GNU General Public License +along with this program; if not, write to the Free Software +Foundation, Inc., 51 Franklin Street, Fifth Floor, Boston, MA 02110-1301, USA + +*******************************************************/ + +/* Page write filters implementation */ + +#include +#include "common.h" +#include "write_filt.h" +#include "fil_cur.h" +#include "xtrabackup.h" + +/************************************************************************ +Write-through page write filter. */ +static my_bool wf_wt_init(xb_write_filt_ctxt_t *ctxt, char *dst_name, + xb_fil_cur_t *cursor); +static my_bool wf_wt_process(xb_write_filt_ctxt_t *ctxt, ds_file_t *dstfile); + +xb_write_filt_t wf_write_through = { + &wf_wt_init, + &wf_wt_process, + NULL, + NULL +}; + +/************************************************************************ +Incremental page write filter. */ +static my_bool wf_incremental_init(xb_write_filt_ctxt_t *ctxt, char *dst_name, + xb_fil_cur_t *cursor); +static my_bool wf_incremental_process(xb_write_filt_ctxt_t *ctxt, + ds_file_t *dstfile); +static my_bool wf_incremental_finalize(xb_write_filt_ctxt_t *ctxt, + ds_file_t *dstfile); +static void wf_incremental_deinit(xb_write_filt_ctxt_t *ctxt); + +xb_write_filt_t wf_incremental = { + &wf_incremental_init, + &wf_incremental_process, + &wf_incremental_finalize, + &wf_incremental_deinit +}; + +/************************************************************************ +Initialize incremental page write filter. + +@return TRUE on success, FALSE on error. */ +static my_bool +wf_incremental_init(xb_write_filt_ctxt_t *ctxt, char *dst_name, + xb_fil_cur_t *cursor) +{ + char meta_name[FN_REFLEN]; + xb_delta_info_t info; + ulint buf_size; + xb_wf_incremental_ctxt_t *cp = + &(ctxt->u.wf_incremental_ctxt); + + ctxt->cursor = cursor; + + /* allocate buffer for incremental backup (4096 pages) */ + buf_size = (UNIV_PAGE_SIZE_MAX / 4 + 1) * UNIV_PAGE_SIZE_MAX; + cp->delta_buf_base = static_cast(ut_malloc(buf_size)); + memset(cp->delta_buf_base, 0, buf_size); + cp->delta_buf = static_cast + (ut_align(cp->delta_buf_base, UNIV_PAGE_SIZE_MAX)); + + /* write delta meta info */ + snprintf(meta_name, sizeof(meta_name), "%s%s", dst_name, + XB_DELTA_INFO_SUFFIX); + info.page_size = cursor->page_size; + info.zip_size = cursor->zip_size; + info.space_id = cursor->space_id; + if (!xb_write_delta_metadata(meta_name, &info)) { + msg("[%02u] xtrabackup: Error: " + "failed to write meta info for %s\n", + cursor->thread_n, cursor->rel_path); + return(FALSE); + } + + /* change the target file name, since we are only going to write + delta pages */ + strcat(dst_name, ".delta"); + + mach_write_to_4(cp->delta_buf, 0x78747261UL); /*"xtra"*/ + cp->npages = 1; + + return(TRUE); +} + +/************************************************************************ +Run the next batch of pages through incremental page write filter. + +@return TRUE on success, FALSE on error. */ +static my_bool +wf_incremental_process(xb_write_filt_ctxt_t *ctxt, ds_file_t *dstfile) +{ + ulint i; + xb_fil_cur_t *cursor = ctxt->cursor; + ulint page_size = cursor->page_size; + byte *page; + xb_wf_incremental_ctxt_t *cp = &(ctxt->u.wf_incremental_ctxt); + + for (i = 0, page = cursor->buf; i < cursor->buf_npages; + i++, page += page_size) { + + if (incremental_lsn >= mach_read_from_8(page + FIL_PAGE_LSN)) { + + continue; + } + + /* updated page */ + if (cp->npages == page_size / 4) { + /* flush buffer */ + if (ds_write(dstfile, cp->delta_buf, + cp->npages * page_size)) { + return(FALSE); + } + + /* clear buffer */ + memset(cp->delta_buf, 0, page_size / 4 * page_size); + /*"xtra"*/ + mach_write_to_4(cp->delta_buf, 0x78747261UL); + cp->npages = 1; + } + + mach_write_to_4(cp->delta_buf + cp->npages * 4, + cursor->buf_page_no + i); + memcpy(cp->delta_buf + cp->npages * page_size, page, + page_size); + + cp->npages++; + } + + return(TRUE); +} + +/************************************************************************ +Flush the incremental page write filter's buffer. + +@return TRUE on success, FALSE on error. */ +static my_bool +wf_incremental_finalize(xb_write_filt_ctxt_t *ctxt, ds_file_t *dstfile) +{ + xb_fil_cur_t *cursor = ctxt->cursor; + ulint page_size = cursor->page_size; + xb_wf_incremental_ctxt_t *cp = &(ctxt->u.wf_incremental_ctxt); + + if (cp->npages != page_size / 4) { + mach_write_to_4(cp->delta_buf + cp->npages * 4, 0xFFFFFFFFUL); + } + + /* Mark the final block */ + mach_write_to_4(cp->delta_buf, 0x58545241UL); /*"XTRA"*/ + + /* flush buffer */ + if (ds_write(dstfile, cp->delta_buf, cp->npages * page_size)) { + return(FALSE); + } + + return(TRUE); +} + +/************************************************************************ +Free the incremental page write filter's buffer. */ +static void +wf_incremental_deinit(xb_write_filt_ctxt_t *ctxt) +{ + xb_wf_incremental_ctxt_t *cp = &(ctxt->u.wf_incremental_ctxt); + + if (cp->delta_buf_base != NULL) { + ut_free(cp->delta_buf_base); + } +} + +/************************************************************************ +Initialize the write-through page write filter. + +@return TRUE on success, FALSE on error. */ +static my_bool +wf_wt_init(xb_write_filt_ctxt_t *ctxt, char *dst_name __attribute__((unused)), + xb_fil_cur_t *cursor) +{ + ctxt->cursor = cursor; + + return(TRUE); +} + +/************************************************************************ +Write the next batch of pages to the destination datasink. + +@return TRUE on success, FALSE on error. */ +static my_bool +wf_wt_process(xb_write_filt_ctxt_t *ctxt, ds_file_t *dstfile) +{ + xb_fil_cur_t *cursor = ctxt->cursor; + + if (ds_write(dstfile, cursor->buf, cursor->buf_read)) { + return(FALSE); + } + + return(TRUE); +} diff --git a/extra/mariabackup/write_filt.h b/extra/mariabackup/write_filt.h new file mode 100644 index 00000000000..20213b6f523 --- /dev/null +++ b/extra/mariabackup/write_filt.h @@ -0,0 +1,61 @@ +/****************************************************** +XtraBackup: hot backup tool for InnoDB +(c) 2009-2013 Percona LLC and/or its affiliates. +Originally Created 3/3/2009 Yasufumi Kinoshita +Written by Alexey Kopytov, Aleksandr Kuzminsky, Stewart Smith, Vadim Tkachenko, +Yasufumi Kinoshita, Ignacio Nin and Baron Schwartz. + +This program is free software; you can redistribute it and/or modify +it under the terms of the GNU General Public License as published by +the Free Software Foundation; version 2 of the License. + +This program is distributed in the hope that it will be useful, +but WITHOUT ANY WARRANTY; without even the implied warranty of +MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the +GNU General Public License for more details. + +You should have received a copy of the GNU General Public License +along with this program; if not, write to the Free Software +Foundation, Inc., 51 Franklin Street, Fifth Floor, Boston, MA 02110-1301, USA + +*******************************************************/ + +/* Page write filter interface */ + +#ifndef XB_WRITE_FILT_H +#define XB_WRITE_FILT_H + +#include "fil_cur.h" +#include "datasink.h" +#include "compact.h" + +/* Incremental page filter context */ +typedef struct { + byte *delta_buf_base; + byte *delta_buf; + ulint npages; +} xb_wf_incremental_ctxt_t; + +/* Page filter context used as an opaque structure by callers */ +typedef struct { + xb_fil_cur_t *cursor; + union { + xb_wf_incremental_ctxt_t wf_incremental_ctxt; + xb_wf_compact_ctxt_t wf_compact_ctxt; + } u; +} xb_write_filt_ctxt_t; + + +typedef struct { + my_bool (*init)(xb_write_filt_ctxt_t *ctxt, char *dst_name, + xb_fil_cur_t *cursor); + my_bool (*process)(xb_write_filt_ctxt_t *ctxt, ds_file_t *dstfile); + my_bool (*finalize)(xb_write_filt_ctxt_t *, ds_file_t *dstfile); + void (*deinit)(xb_write_filt_ctxt_t *); +} xb_write_filt_t; + +extern xb_write_filt_t wf_write_through; +extern xb_write_filt_t wf_incremental; +extern xb_write_filt_t wf_compact; + +#endif /* XB_WRITE_FILT_H */ diff --git a/extra/mariabackup/wsrep.cc b/extra/mariabackup/wsrep.cc new file mode 100644 index 00000000000..420ada75f36 --- /dev/null +++ b/extra/mariabackup/wsrep.cc @@ -0,0 +1,219 @@ +/****************************************************** +Percona XtraBackup: hot backup tool for InnoDB +(c) 2009-2014 Percona LLC and/or its affiliates +Originally Created 3/3/2009 Yasufumi Kinoshita +Written by Alexey Kopytov, Aleksandr Kuzminsky, Stewart Smith, Vadim Tkachenko, +Yasufumi Kinoshita, Ignacio Nin and Baron Schwartz. + +This program is free software; you can redistribute it and/or modify +it under the terms of the GNU General Public License as published by +the Free Software Foundation; version 2 of the License. + +This program is distributed in the hope that it will be useful, +but WITHOUT ANY WARRANTY; without even the implied warranty of +MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the +GNU General Public License for more details. + +You should have received a copy of the GNU General Public License +along with this program; if not, write to the Free Software +Foundation, Inc., 51 Franklin Street, Fifth Floor, Boston, MA 02110-1301, USA + +******************************************************* + +This file incorporates work covered by the following copyright and +permission notice: + + Copyright 2010 Codership Oy + + This program is free software; you can redistribute it and/or modify + it under the terms of the GNU General Public License as published by + the Free Software Foundation; version 2 of the License. + + This program is distributed in the hope that it will be useful, + but WITHOUT ANY WARRANTY; without even the implied warranty of + MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the + GNU General Public License for more details. + + You should have received a copy of the GNU General Public License + along with this program; if not, write to the Free Software + Foundation, Inc., 51 Franklin Street, Fifth Floor, Boston, MA 02110-1301, USA + +*******************************************************/ + +#include +#include +#include +#include + +#include "common.h" + +#define WSREP_XID_PREFIX "WSREPXid" +#define WSREP_XID_PREFIX_LEN MYSQL_XID_PREFIX_LEN +#define WSREP_XID_UUID_OFFSET 8 +#define WSREP_XID_SEQNO_OFFSET (WSREP_XID_UUID_OFFSET + sizeof(wsrep_uuid_t)) +#define WSREP_XID_GTRID_LEN (WSREP_XID_SEQNO_OFFSET + sizeof(wsrep_seqno_t)) + +/*! undefined seqno */ +#define WSREP_SEQNO_UNDEFINED (-1) + +/*! Name of file where Galera info is stored on recovery */ +#define XB_GALERA_INFO_FILENAME "xtrabackup_galera_info" + +/* Galera UUID type - for all unique IDs */ +typedef struct wsrep_uuid { + uint8_t data[16]; +} wsrep_uuid_t; + +/* sequence number of a writeset, etc. */ +typedef int64_t wsrep_seqno_t; + +/* Undefined UUID */ +static const wsrep_uuid_t WSREP_UUID_UNDEFINED = {{0,}}; + +/***********************************************************************//** +Check if a given WSREP XID is valid. + +@return true if valid. +*/ +static +bool +wsrep_is_wsrep_xid( +/*===============*/ + const void* xid_ptr) +{ + const XID* xid = reinterpret_cast(xid_ptr); + + return((xid->formatID == 1 && + xid->gtrid_length == WSREP_XID_GTRID_LEN && + xid->bqual_length == 0 && + !memcmp(xid->data, WSREP_XID_PREFIX, WSREP_XID_PREFIX_LEN))); +} + +/***********************************************************************//** +Retrieve binary WSREP UUID from XID. + +@return binary WSREP UUID represenataion, if UUID is valid, or + WSREP_UUID_UNDEFINED otherwise. +*/ +static +const wsrep_uuid_t* +wsrep_xid_uuid( +/*===========*/ + const XID* xid) +{ + if (wsrep_is_wsrep_xid(xid)) { + return(reinterpret_cast + (xid->data + WSREP_XID_UUID_OFFSET)); + } else { + return(&WSREP_UUID_UNDEFINED); + } +} + +/***********************************************************************//** +Retrieve WSREP seqno from XID. + +@return WSREP seqno, if it is valid, or WSREP_SEQNO_UNDEFINED otherwise. +*/ +wsrep_seqno_t wsrep_xid_seqno( +/*==========================*/ + const XID* xid) +{ + if (wsrep_is_wsrep_xid(xid)) { + wsrep_seqno_t seqno; + memcpy(&seqno, xid->data + WSREP_XID_SEQNO_OFFSET, + sizeof(wsrep_seqno_t)); + + return(seqno); + } else { + return(WSREP_SEQNO_UNDEFINED); + } +} + +/***********************************************************************//** +Write UUID to string. + +@return length of UUID string representation or -EMSGSIZE if string is too +short. +*/ +static +int +wsrep_uuid_print( +/*=============*/ + const wsrep_uuid_t* uuid, + char* str, + size_t str_len) +{ + if (str_len > 36) { + const unsigned char* u = uuid->data; + return snprintf(str, str_len, + "%02x%02x%02x%02x-%02x%02x-%02x%02x-" + "%02x%02x-%02x%02x%02x%02x%02x%02x", + u[ 0], u[ 1], u[ 2], u[ 3], u[ 4], u[ 5], u[ 6], + u[ 7], u[ 8], u[ 9], u[10], u[11], u[12], u[13], + u[14], u[15]); + } + else { + return -EMSGSIZE; + } +} + +/*********************************************************************** +Store Galera checkpoint info in the 'xtrabackup_galera_info' file, if that +information is present in the trx system header. Otherwise, do nothing. */ +void +xb_write_galera_info(bool incremental_prepare) +/*==================*/ +{ + FILE* fp; + XID xid; + char uuid_str[40]; + wsrep_seqno_t seqno; + MY_STAT statinfo; + + /* Do not overwrite existing an existing file to be compatible with + servers with older server versions */ + if (!incremental_prepare && + my_stat(XB_GALERA_INFO_FILENAME, &statinfo, MYF(0)) != NULL) { + + return; + } + + memset(&xid, 0, sizeof(xid)); + xid.formatID = -1; + + if (!trx_sys_read_wsrep_checkpoint(&xid)) { + + return; + } + + if (wsrep_uuid_print(wsrep_xid_uuid(&xid), uuid_str, + sizeof(uuid_str)) < 0) { + return; + } + + fp = fopen(XB_GALERA_INFO_FILENAME, "w"); + if (fp == NULL) { + + msg("xtrabackup: error: " + "could not create " XB_GALERA_INFO_FILENAME + ", errno = %d\n", + errno); + exit(EXIT_FAILURE); + } + + seqno = wsrep_xid_seqno(&xid); + + msg("xtrabackup: Recovered WSREP position: %s:%lld\n", + uuid_str, (long long) seqno); + + if (fprintf(fp, "%s:%lld", uuid_str, (long long) seqno) < 0) { + + msg("xtrabackup: error: " + "could not write to " XB_GALERA_INFO_FILENAME + ", errno = %d\n", + errno); + exit(EXIT_FAILURE); + } + + fclose(fp); +} diff --git a/extra/mariabackup/wsrep.h b/extra/mariabackup/wsrep.h new file mode 100644 index 00000000000..7638d1f2b54 --- /dev/null +++ b/extra/mariabackup/wsrep.h @@ -0,0 +1,32 @@ +/****************************************************** +Percona XtraBackup: hot backup tool for InnoDB +(c) 2009-2014 Percona LLC and/or its affiliates +Originally Created 3/3/2009 Yasufumi Kinoshita +Written by Alexey Kopytov, Aleksandr Kuzminsky, Stewart Smith, Vadim Tkachenko, +Yasufumi Kinoshita, Ignacio Nin and Baron Schwartz. + +This program is free software; you can redistribute it and/or modify +it under the terms of the GNU General Public License as published by +the Free Software Foundation; version 2 of the License. + +This program is distributed in the hope that it will be useful, +but WITHOUT ANY WARRANTY; without even the implied warranty of +MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the +GNU General Public License for more details. + +You should have received a copy of the GNU General Public License +along with this program; if not, write to the Free Software +Foundation, Inc., 51 Franklin Street, Fifth Floor, Boston, MA 02110-1301, USA +*******************************************************/ + +#ifndef WSREP_H +#define WSREP_H + +/*********************************************************************** +Store Galera checkpoint info in the 'xtrabackup_galera_info' file, if that +information is present in the trx system header. Otherwise, do nothing. */ +void +xb_write_galera_info(bool incremental_prepare); +/*==================*/ + +#endif diff --git a/extra/mariabackup/xb_regex.h b/extra/mariabackup/xb_regex.h new file mode 100644 index 00000000000..94c5e2a5fa7 --- /dev/null +++ b/extra/mariabackup/xb_regex.h @@ -0,0 +1,71 @@ +/****************************************************** +Copyright (c) 2011-2013 Percona LLC and/or its affiliates. + +This program is free software; you can redistribute it and/or modify +it under the terms of the GNU General Public License as published by +the Free Software Foundation; version 2 of the License. + +This program is distributed in the hope that it will be useful, +but WITHOUT ANY WARRANTY; without even the implied warranty of +MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the +GNU General Public License for more details. + +You should have received a copy of the GNU General Public License +along with this program; if not, write to the Free Software +Foundation, Inc., 51 Franklin Street, Fifth Floor, Boston, MA 02110-1301, USA + +*******************************************************/ + +/* This file is required to abstract away regex(3) calls so that +my_regex is used on Windows and native calls are used on POSIX platforms. */ + +#ifndef XB_REGEX_H +#define XB_REGEX_H + +#ifdef _WIN32 + +#include + +typedef my_regex_t xb_regex_t; +typedef my_regmatch_t xb_regmatch_t; + +#define xb_regex_init() my_regex_init(&my_charset_latin1) + +#define xb_regexec(preg,string,nmatch,pmatch,eflags) \ + my_regexec(preg, string, nmatch, pmatch, eflags) + +#define xb_regerror(errcode,preg,errbuf,errbuf_size) \ + my_regerror(errcode, preg, errbuf, errbuf_size) + +#define xb_regcomp(preg,regex,cflags) \ + my_regcomp(preg, regex, cflags, &my_charset_latin1) + +#define xb_regfree(preg) my_regfree(preg) + +#define xb_regex_end() my_regex_end() + +#else /* ! _WIN32 */ + +#include + +typedef regex_t xb_regex_t; +typedef regmatch_t xb_regmatch_t; + +#define xb_regex_init() do { } while(0) + +#define xb_regexec(preg,string,nmatch,pmatch,eflags) \ + regexec(preg, string, nmatch, pmatch, eflags) + +#define xb_regerror(errcode,preg,errbuf,errbuf_size) \ + regerror(errcode, preg, errbuf, errbuf_size) + +#define xb_regcomp(preg,regex,cflags) \ + regcomp(preg, regex, cflags) + +#define xb_regfree(preg) regfree(preg) + +#define xb_regex_end() do { } while (0) + +#endif /* _WIN32 */ + +#endif /* XB_REGEX_H */ diff --git a/extra/mariabackup/xbcloud.cc b/extra/mariabackup/xbcloud.cc new file mode 100644 index 00000000000..56661b03dd0 --- /dev/null +++ b/extra/mariabackup/xbcloud.cc @@ -0,0 +1,2721 @@ +/****************************************************** +Copyright (c) 2014 Percona LLC and/or its affiliates. + +The xbstream utility: serialize/deserialize files in the XBSTREAM format. + +This program is free software; you can redistribute it and/or modify +it under the terms of the GNU General Public License as published by +the Free Software Foundation; version 2 of the License. + +This program is distributed in the hope that it will be useful, +but WITHOUT ANY WARRANTY; without even the implied warranty of +MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the +GNU General Public License for more details. + +You should have received a copy of the GNU General Public License +along with this program; if not, write to the Free Software +Foundation, Inc., 51 Franklin Street, Fifth Floor, Boston, MA 02110-1301, USA + +*******************************************************/ + +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include "xbstream.h" + +using std::min; +using std::max; +using std::map; +using std::string; + +#define XBCLOUD_VERSION "1.0" + +#define SWIFT_MAX_URL_SIZE 8192 +#define SWIFT_MAX_HDR_SIZE 8192 + +#define SWIFT_CHUNK_SIZE 11 * 1024 * 1024 + +#if ((LIBCURL_VERSION_MAJOR >= 7) && (LIBCURL_VERSION_MINOR >= 16)) +#define OLD_CURL_MULTI 0 +#else +#define OLD_CURL_MULTI 1 +#endif + +/*****************************************************************************/ + +typedef struct swift_auth_info_struct swift_auth_info; +typedef struct connection_info_struct connection_info; +typedef struct socket_info_struct socket_info; +typedef struct global_io_info_struct global_io_info; +typedef struct slo_chunk_struct slo_chunk; +typedef struct container_list_struct container_list; +typedef struct object_info_struct object_info; + +struct swift_auth_info_struct { + char url[SWIFT_MAX_URL_SIZE]; + char token[SWIFT_MAX_HDR_SIZE]; +}; + +struct global_io_info_struct { + struct ev_loop *loop; + struct ev_io input_event; + struct ev_timer timer_event; + CURLM *multi; + int still_running; + int eof; + curl_socket_t input_fd; + connection_info **connections; + long chunk_no; + connection_info *current_connection; + const char *url; + const char *container; + const char *token; + const char *backup_name; +}; + +struct socket_info_struct { + curl_socket_t sockfd; + CURL *easy; + int action; + long timeout; + struct ev_io ev; + int evset; + global_io_info *global; +}; + +struct connection_info_struct { + CURL *easy; + global_io_info *global; + char *buffer; + size_t buffer_size; + size_t filled_size; + size_t upload_size; + bool chunk_uploaded; + bool chunk_acked; + char error[CURL_ERROR_SIZE]; + struct curl_slist *slist; + char *name; + size_t name_len; + char hash[33]; + size_t chunk_no; + bool magic_verified; + size_t chunk_path_len; + xb_chunk_type_t chunk_type; + size_t payload_size; + size_t chunk_size; + int retry_count; + bool upload_started; + ulong global_idx; +}; + +struct slo_chunk_struct { + char name[SWIFT_MAX_URL_SIZE]; + char md5[33]; + int idx; + size_t size; +}; + +struct object_info_struct { + char hash[33]; + char name[SWIFT_MAX_URL_SIZE]; + size_t bytes; +}; + +struct container_list_struct { + size_t content_length; + size_t content_bufsize; + char *content_json; + size_t object_count; + size_t idx; + object_info *objects; + bool final; +}; + +enum {SWIFT, S3}; +const char *storage_names[] = +{ "SWIFT", "S3", NullS}; + +static my_bool opt_verbose = 0; +static ulong opt_storage = SWIFT; +static const char *opt_swift_user = NULL; +static const char *opt_swift_user_id = NULL; +static const char *opt_swift_password = NULL; +static const char *opt_swift_tenant = NULL; +static const char *opt_swift_tenant_id = NULL; +static const char *opt_swift_project = NULL; +static const char *opt_swift_project_id = NULL; +static const char *opt_swift_domain = NULL; +static const char *opt_swift_domain_id = NULL; +static const char *opt_swift_region = NULL; +static const char *opt_swift_container = NULL; +static const char *opt_swift_storage_url = NULL; +static const char *opt_swift_auth_url = NULL; +static const char *opt_swift_key = NULL; +static const char *opt_swift_auth_version = NULL; +static const char *opt_name = NULL; +static const char *opt_cacert = NULL; +static ulong opt_parallel = 1; +static my_bool opt_insecure = 0; +static enum {MODE_GET, MODE_PUT, MODE_DELETE} opt_mode; + +static char **file_list = NULL; +static int file_list_size = 0; + +TYPELIB storage_typelib = +{array_elements(storage_names)-1, "", storage_names, NULL}; + +enum { + OPT_STORAGE = 256, + OPT_SWIFT_CONTAINER, + OPT_SWIFT_AUTH_URL, + OPT_SWIFT_KEY, + OPT_SWIFT_USER, + OPT_SWIFT_USER_ID, + OPT_SWIFT_PASSWORD, + OPT_SWIFT_TENANT, + OPT_SWIFT_TENANT_ID, + OPT_SWIFT_PROJECT, + OPT_SWIFT_PROJECT_ID, + OPT_SWIFT_DOMAIN, + OPT_SWIFT_DOMAIN_ID, + OPT_SWIFT_REGION, + OPT_SWIFT_STORAGE_URL, + OPT_SWIFT_AUTH_VERSION, + OPT_PARALLEL, + OPT_CACERT, + OPT_INSECURE, + OPT_VERBOSE +}; + + +static struct my_option my_long_options[] = +{ + {"help", '?', "Display this help and exit.", + 0, 0, 0, GET_NO_ARG, NO_ARG, 0, 0, 0, 0, 0, 0}, + + {"storage", OPT_STORAGE, "Specify storage type S3/SWIFT.", + &opt_storage, &opt_storage, &storage_typelib, + GET_ENUM, REQUIRED_ARG, 0, 0, 0, 0, 0, 0}, + + {"swift-auth-version", OPT_SWIFT_AUTH_VERSION, + "Swift authentication verison to use.", + &opt_swift_auth_version, &opt_swift_auth_version, 0, + GET_STR_ALLOC, REQUIRED_ARG, + 0, 0, 0, 0, 0, 0}, + + {"swift-container", OPT_SWIFT_CONTAINER, + "Swift container to store backups into.", + &opt_swift_container, &opt_swift_container, 0, + GET_STR_ALLOC, REQUIRED_ARG, + 0, 0, 0, 0, 0, 0}, + + {"swift-user", OPT_SWIFT_USER, + "Swift user name.", + &opt_swift_user, &opt_swift_user, 0, GET_STR_ALLOC, REQUIRED_ARG, + 0, 0, 0, 0, 0, 0}, + + {"swift-user-id", OPT_SWIFT_USER_ID, + "Swift user ID.", + &opt_swift_user_id, &opt_swift_user_id, 0, GET_STR_ALLOC, REQUIRED_ARG, + 0, 0, 0, 0, 0, 0}, + + {"swift-auth-url", OPT_SWIFT_AUTH_URL, + "Base URL of SWIFT authentication service.", + &opt_swift_auth_url, &opt_swift_auth_url, 0, + GET_STR_ALLOC, REQUIRED_ARG, + 0, 0, 0, 0, 0, 0}, + + {"swift-storage-url", OPT_SWIFT_STORAGE_URL, + "URL of object-store endpoint. Usually received from authentication " + "service. Specify to override this value.", + &opt_swift_storage_url, &opt_swift_storage_url, 0, + GET_STR_ALLOC, REQUIRED_ARG, + 0, 0, 0, 0, 0, 0}, + + {"swift-key", OPT_SWIFT_KEY, + "Swift key.", + &opt_swift_key, &opt_swift_key, 0, GET_STR_ALLOC, REQUIRED_ARG, + 0, 0, 0, 0, 0, 0}, + + {"swift-tenant", OPT_SWIFT_TENANT, + "The tenant name. Both the --swift-tenant and --swift-tenant-id " + "options are optional, but should not be specified together.", + &opt_swift_tenant, &opt_swift_tenant, 0, GET_STR_ALLOC, REQUIRED_ARG, + 0, 0, 0, 0, 0, 0}, + + {"swift-tenant-id", OPT_SWIFT_TENANT_ID, + "The tenant ID. Both the --swift-tenant and --swift-tenant-id " + "options are optional, but should not be specified together.", + &opt_swift_tenant_id, &opt_swift_tenant_id, 0, + GET_STR_ALLOC, REQUIRED_ARG, + 0, 0, 0, 0, 0, 0}, + + {"swift-project", OPT_SWIFT_PROJECT, + "The project name.", + &opt_swift_project, &opt_swift_project, 0, GET_STR_ALLOC, REQUIRED_ARG, + 0, 0, 0, 0, 0, 0}, + + {"swift-project-id", OPT_SWIFT_PROJECT_ID, + "The project ID.", + &opt_swift_project_id, &opt_swift_project_id, 0, + GET_STR_ALLOC, REQUIRED_ARG, + 0, 0, 0, 0, 0, 0}, + + {"swift-domain", OPT_SWIFT_DOMAIN, + "The domain name.", + &opt_swift_domain, &opt_swift_domain, 0, GET_STR_ALLOC, REQUIRED_ARG, + 0, 0, 0, 0, 0, 0}, + + {"swift-domain-id", OPT_SWIFT_DOMAIN_ID, + "The domain ID.", + &opt_swift_domain_id, &opt_swift_domain_id, 0, + GET_STR_ALLOC, REQUIRED_ARG, + 0, 0, 0, 0, 0, 0}, + + {"swift-password", OPT_SWIFT_PASSWORD, + "The password of the user.", + &opt_swift_password, &opt_swift_password, 0, + GET_STR_ALLOC, REQUIRED_ARG, + 0, 0, 0, 0, 0, 0}, + + {"swift-region", OPT_SWIFT_REGION, + "The region object-store endpoint.", + &opt_swift_region, &opt_swift_region, 0, + GET_STR_ALLOC, REQUIRED_ARG, + 0, 0, 0, 0, 0, 0}, + + {"parallel", OPT_PARALLEL, + "Number of parallel chunk uploads.", + &opt_parallel, &opt_parallel, 0, GET_ULONG, REQUIRED_ARG, + 1, 0, 0, 0, 0, 0}, + + {"cacert", OPT_CACERT, + "CA certificate file.", + &opt_cacert, &opt_cacert, 0, GET_STR_ALLOC, REQUIRED_ARG, + 0, 0, 0, 0, 0, 0}, + + {"insecure", OPT_INSECURE, + "Do not verify server SSL certificate.", + &opt_insecure, &opt_insecure, 0, GET_BOOL, NO_ARG, + 0, 0, 0, 0, 0, 0}, + + {"verbose", OPT_VERBOSE, + "Turn ON cURL tracing.", + &opt_verbose, &opt_verbose, 0, GET_BOOL, NO_ARG, + 0, 0, 0, 0, 0, 0}, + + {0, 0, 0, 0, 0, 0, GET_NO_ARG, NO_ARG, 0, 0, 0, 0, 0, 0} +}; + +/* The values of these arguments should be masked + on the command line */ +static const char * const masked_args[] = { + "--swift-password", + "--swift-key", + "--swift-auth-url", + "--swift-storage-url", + "--swift-container", + "--swift-user", + "--swift-tenant", + "--swift-user-id", + "--swift-tenant-id", + 0 +}; + +static map file_chunk_count; + +static +void +print_version() +{ + printf("%s Ver %s for %s (%s)\n", my_progname, XBCLOUD_VERSION, + SYSTEM_TYPE, MACHINE_TYPE); +} + +static +void +usage() +{ + print_version(); + puts("Copyright (C) 2015 Percona LLC and/or its affiliates."); + puts("This software comes with ABSOLUTELY NO WARRANTY. " + "This is free software,\nand you are welcome to modify and " + "redistribute it under the GPL license.\n"); + + puts("Manage backups on Cloud services.\n"); + + puts("Usage: "); + printf(" %s -c put [OPTIONS...] upload backup from STDIN into " + "the cloud service with given name.\n", my_progname); + printf(" %s -c get [OPTIONS...] [FILES...] stream specified " + "backup or individual files from cloud service into STDOUT.\n", + my_progname); + + puts("\nOptions:"); + my_print_help(my_long_options); +} + +static +my_bool +get_one_option(int optid, const struct my_option *opt __attribute__((unused)), + char *argument __attribute__((unused))) +{ + switch (optid) { + case '?': + usage(); + exit(0); + } + + return(FALSE); +} + +static const char *load_default_groups[]= + { "xbcloud", 0 }; + +/*********************************************************************//** +mask sensitive values on the command line */ +static +void +mask_args(int argc, char **argv) +{ + int i; + for (i = 0; i < argc-1; i++) { + int j = 0; + if (argv[i]) while (masked_args[j]) { + char *p; + if ((p = strstr(argv[i], masked_args[j]))) { + p += strlen(masked_args[j]); + while (*p && *p != '=') { + p++; + } + if (*p == '=') { + p++; + while (*p) { + *p++ = 'x'; + } + } + } + j++; + } + } +} + +static +int parse_args(int argc, char **argv) +{ + const char *command; + + if (argc < 2) { + fprintf(stderr, "Command isn't specified. " + "Supported commands are put and get\n"); + usage(); + exit(EXIT_FAILURE); + } + + command = argv[1]; + argc--; argv++; + + if (strcasecmp(command, "put") == 0) { + opt_mode = MODE_PUT; + } else if (strcasecmp(command, "get") == 0) { + opt_mode = MODE_GET; + } else if (strcasecmp(command, "delete") == 0) { + opt_mode = MODE_DELETE; + } else { + fprintf(stderr, "Unknown command %s. " + "Supported commands are put and get\n", command); + usage(); + exit(EXIT_FAILURE); + } + + if (load_defaults("my", load_default_groups, &argc, &argv)) { + exit(EXIT_FAILURE); + } + + if (handle_options(&argc, &argv, my_long_options, get_one_option)) { + exit(EXIT_FAILURE); + } + + /* make sure name is specified */ + if (argc < 1) { + fprintf(stderr, "Backup name is required argument\n"); + exit(EXIT_FAILURE); + } + opt_name = argv[0]; + argc--; argv++; + + /* validate arguments */ + if (opt_storage == SWIFT) { + if (opt_swift_user == NULL) { + fprintf(stderr, "Swift user is not specified\n"); + exit(EXIT_FAILURE); + } + if (opt_swift_container == NULL) { + fprintf(stderr, + "Swift container is not specified\n"); + exit(EXIT_FAILURE); + } + if (opt_swift_auth_url == NULL) { + fprintf(stderr, "Swift auth URL is not specified\n"); + exit(EXIT_FAILURE); + } + } else { + fprintf(stderr, "Swift is only supported storage API\n"); + } + + if (argc > 0) { + file_list = argv; + file_list_size = argc; + } + + return(0); +} + +static char *hex_md5(const unsigned char *hash, char *out) +{ + enum { hash_len = 16 }; + char *p; + int i; + + for (i = 0, p = out; i < hash_len; i++, p+=2) { + sprintf(p, "%02x", hash[i]); + } + + return out; +} + +/* If header starts with prefix it's value will be copied into output buffer */ +static +int get_http_header(const char *prefix, const char *buffer, + char *out, size_t out_size) +{ + const char *beg, *end; + size_t len, prefix_len; + + prefix_len = strlen(prefix); + + if (strncasecmp(buffer, prefix, prefix_len) == 0) { + beg = buffer + prefix_len; + end = strchr(beg, '\r'); + + len = min(end - beg, out_size - 1); + + strncpy(out, beg, len); + + out[len] = 0; + + return 1; + } + + return 0; +} + +static +size_t swift_auth_header_read_cb(char *ptr, size_t size, size_t nmemb, + void *data) +{ + swift_auth_info *info = (swift_auth_info*)(data); + + get_http_header("X-Storage-Url: ", ptr, + info->url, array_elements(info->url)); + get_http_header("X-Auth-Token: ", ptr, + info->token, array_elements(info->token)); + + return nmemb * size; +} + +/*********************************************************************//** +Authenticate against Swift TempAuth. Fills swift_auth_info struct. +Uses creadentials privided as global variables. +@returns true if access is granted and token received. */ +static +bool +swift_temp_auth(const char *auth_url, swift_auth_info *info) +{ + CURL *curl; + CURLcode res; + long http_code; + char *hdr_buf = NULL; + struct curl_slist *slist = NULL; + + if (opt_swift_user == NULL) { + fprintf(stderr, "Swift user must be specified for TempAuth.\n"); + return(false); + } + + if (opt_swift_key == NULL) { + fprintf(stderr, "Swift key must be specified for TempAuth.\n"); + return(false); + } + + curl = curl_easy_init(); + + if (curl != NULL) { + + hdr_buf = (char *)(calloc(14 + max(strlen(opt_swift_user), + strlen(opt_swift_key)), 1)); + + if (!hdr_buf) { + res = CURLE_FAILED_INIT; + goto cleanup; + } + + sprintf(hdr_buf, "X-Auth-User: %s", opt_swift_user); + slist = curl_slist_append(slist, hdr_buf); + + sprintf(hdr_buf, "X-Auth-Key: %s", opt_swift_key); + slist = curl_slist_append(slist, hdr_buf); + + curl_easy_setopt(curl, CURLOPT_VERBOSE, opt_verbose); + curl_easy_setopt(curl, CURLOPT_URL, auth_url); + curl_easy_setopt(curl, CURLOPT_FOLLOWLOCATION, 1L); + curl_easy_setopt(curl, CURLOPT_HEADERFUNCTION, + swift_auth_header_read_cb); + curl_easy_setopt(curl, CURLOPT_HEADERDATA, info); + curl_easy_setopt(curl, CURLOPT_HTTPHEADER, slist); + if (opt_cacert != NULL) + curl_easy_setopt(curl, CURLOPT_CAINFO, opt_cacert); + if (opt_insecure) + curl_easy_setopt(curl, CURLOPT_SSL_VERIFYPEER, FALSE); + + res = curl_easy_perform(curl); + + if (res != CURLE_OK) { + fprintf(stderr, "error: authentication failed: " + "curl_easy_perform(): %s\n", + curl_easy_strerror(res)); + goto cleanup; + } + curl_easy_getinfo(curl, CURLINFO_RESPONSE_CODE, &http_code); + if (http_code != 200 && + http_code != 204) { + fprintf(stderr, "error: authentication failed " + "with response code: %ld\n", http_code); + res = CURLE_LOGIN_DENIED; + goto cleanup; + } + } else { + res = CURLE_FAILED_INIT; + fprintf(stderr, "error: curl_easy_init() failed\n"); + goto cleanup; + } + +cleanup: + if (hdr_buf) { + free(hdr_buf); + } + if (slist) { + curl_slist_free_all(slist); + } + if (curl) { + curl_easy_cleanup(curl); + } + + if (res == CURLE_OK) { + /* check that we received token and storage URL */ + if (*info->url == 0) { + fprintf(stderr, "error: malformed response: " + "X-Storage-Url is missing\n"); + return(false); + } + if (*info->token == 0) { + fprintf(stderr, "error: malformed response: " + "X-Auth-Token is missing\n"); + return(false); + } + return(true); + } + + return(false); +} + +static +size_t +write_null_cb(char *buffer, size_t size, size_t nmemb, void *stream) +{ + return fwrite(buffer, size, nmemb, stderr); +} + + +static +size_t +read_null_cb(char *ptr, size_t size, size_t nmemb, void *data) +{ + return 0; +} + + +static +int +swift_create_container(swift_auth_info *info, const char *name) +{ + char url[SWIFT_MAX_URL_SIZE]; + char auth_token[SWIFT_MAX_HDR_SIZE]; + CURLcode res; + long http_code; + CURL *curl; + struct curl_slist *slist = NULL; + + snprintf(url, array_elements(url), "%s/%s", info->url, name); + snprintf(auth_token, array_elements(auth_token), "X-Auth-Token: %s", + info->token); + + curl = curl_easy_init(); + + if (curl != NULL) { + slist = curl_slist_append(slist, auth_token); + slist = curl_slist_append(slist, "Content-Length: 0"); + + curl_easy_setopt(curl, CURLOPT_VERBOSE, opt_verbose); + curl_easy_setopt(curl, CURLOPT_URL, url); + curl_easy_setopt(curl, CURLOPT_FOLLOWLOCATION, 1L); + curl_easy_setopt(curl, CURLOPT_HTTPHEADER, slist); + curl_easy_setopt(curl, CURLOPT_WRITEFUNCTION, write_null_cb); + curl_easy_setopt(curl, CURLOPT_READFUNCTION, read_null_cb); + curl_easy_setopt(curl, CURLOPT_INFILESIZE, 0L); + curl_easy_setopt(curl, CURLOPT_PUT, 1L); + if (opt_cacert != NULL) + curl_easy_setopt(curl, CURLOPT_CAINFO, opt_cacert); + if (opt_insecure) + curl_easy_setopt(curl, CURLOPT_SSL_VERIFYPEER, FALSE); + + res = curl_easy_perform(curl); + + if (res != CURLE_OK) { + fprintf(stderr, + "error: curl_easy_perform() failed: %s\n", + curl_easy_strerror(res)); + goto cleanup; + } + curl_easy_getinfo(curl, CURLINFO_RESPONSE_CODE, &http_code); + if (http_code != 201 && /* created */ + http_code != 202 /* accepted (already exists) */) { + fprintf(stderr, "error: request failed " + "with response code: %ld\n", http_code); + res = CURLE_LOGIN_DENIED; + goto cleanup; + } + } else { + res = CURLE_FAILED_INIT; + fprintf(stderr, "error: curl_easy_init() failed\n"); + goto cleanup; + } + +cleanup: + if (slist) { + curl_slist_free_all(slist); + } + if (curl) { + curl_easy_cleanup(curl); + } + + return res; +} + + +/*********************************************************************//** +Delete object with given url. +@returns true if object deleted successfully. */ +static +bool +swift_delete_object(swift_auth_info *info, const char *url) +{ + char auth_token[SWIFT_MAX_HDR_SIZE]; + CURLcode res; + long http_code; + CURL *curl; + struct curl_slist *slist = NULL; + bool ret = false; + + snprintf(auth_token, array_elements(auth_token), "X-Auth-Token: %s", + info->token); + + curl = curl_easy_init(); + + if (curl != NULL) { + slist = curl_slist_append(slist, auth_token); + + curl_easy_setopt(curl, CURLOPT_VERBOSE, opt_verbose); + curl_easy_setopt(curl, CURLOPT_URL, url); + curl_easy_setopt(curl, CURLOPT_FOLLOWLOCATION, 1L); + curl_easy_setopt(curl, CURLOPT_HTTPHEADER, slist); + curl_easy_setopt(curl, CURLOPT_CUSTOMREQUEST, "DELETE"); + if (opt_cacert != NULL) + curl_easy_setopt(curl, CURLOPT_CAINFO, opt_cacert); + if (opt_insecure) + curl_easy_setopt(curl, CURLOPT_SSL_VERIFYPEER, FALSE); + + res = curl_easy_perform(curl); + + if (res != CURLE_OK) { + fprintf(stderr, + "error: curl_easy_perform() failed: %s\n", + curl_easy_strerror(res)); + goto cleanup; + } + curl_easy_getinfo(curl, CURLINFO_RESPONSE_CODE, &http_code); + if (http_code != 200 && /* OK */ + http_code != 204 /* no content */) { + fprintf(stderr, "error: request failed " + "with response code: %ld\n", http_code); + goto cleanup; + } + ret = true; + } else { + fprintf(stderr, "error: curl_easy_init() failed\n"); + goto cleanup; + } + +cleanup: + if (slist) { + curl_slist_free_all(slist); + } + if (curl) { + curl_easy_cleanup(curl); + } + + return ret; +} + +static int conn_upload_init(connection_info *conn); +static void conn_buffer_updated(connection_info *conn); +static connection_info *conn_new(global_io_info *global, ulong global_idx); +static void conn_cleanup(connection_info *conn); +static void conn_upload_retry(connection_info *conn); + +/* Check for completed transfers, and remove their easy handles */ +static void check_multi_info(global_io_info *g) +{ + char *eff_url; + CURLMsg *msg; + int msgs_left; + connection_info *conn; + CURL *easy; + + while ((msg = curl_multi_info_read(g->multi, &msgs_left))) { + if (msg->msg == CURLMSG_DONE) { + easy = msg->easy_handle; + curl_easy_getinfo(easy, CURLINFO_PRIVATE, &conn); + curl_easy_getinfo(easy, CURLINFO_EFFECTIVE_URL, + &eff_url); + curl_multi_remove_handle(g->multi, easy); + curl_easy_cleanup(easy); + conn->easy = NULL; + if (conn->chunk_acked) { + conn->chunk_uploaded = true; + fprintf(stderr, "%s is done\n", conn->hash); + } else { + fprintf(stderr, "error: chunk %zu '%s' %s " + "is not uploaded, but socket closed " + "(%zu bytes of %zu left to upload)\n", + conn->chunk_no, + conn->name, + conn->hash, + conn->chunk_size - conn->upload_size, + conn->chunk_size); + conn_upload_retry(conn); + } + } + } +} + +/* Die if we get a bad CURLMcode somewhere */ +static void mcode_or_die(const char *where, CURLMcode code) +{ + if (code != CURLM_OK) + { + const char *s; + switch (code) + { + case CURLM_BAD_HANDLE: + s = "CURLM_BAD_HANDLE"; + break; + case CURLM_BAD_EASY_HANDLE: + s = "CURLM_BAD_EASY_HANDLE"; + break; + case CURLM_OUT_OF_MEMORY: + s = "CURLM_OUT_OF_MEMORY"; + break; + case CURLM_INTERNAL_ERROR: + s = "CURLM_INTERNAL_ERROR"; + break; + case CURLM_UNKNOWN_OPTION: + s = "CURLM_UNKNOWN_OPTION"; + break; + case CURLM_LAST: + s = "CURLM_LAST"; + break; + default: + s = "CURLM_unknown"; + break; + case CURLM_BAD_SOCKET: + s = "CURLM_BAD_SOCKET"; + fprintf(stderr, "error: %s returns (%d) %s\n", + where, code, s); + /* ignore this error */ + return; + } + fprintf(stderr, "error: %s returns (%d) %s\n", + where, code, s); + assert(0); + } +} + +/* Called by libev when we get action on a multi socket */ +static void event_cb(EV_P_ struct ev_io *w, int revents) +{ + global_io_info *global = (global_io_info*)(w->data); + CURLMcode rc; + +#if !(OLD_CURL_MULTI) + int action = (revents & EV_READ ? CURL_POLL_IN : 0) | + (revents & EV_WRITE ? CURL_POLL_OUT : 0); + + do { + rc = curl_multi_socket_action(global->multi, w->fd, action, + &global->still_running); + } while (rc == CURLM_CALL_MULTI_PERFORM); +#else + do { + rc = curl_multi_socket(global->multi, w->fd, + &global->still_running); + } while (rc == CURLM_CALL_MULTI_PERFORM); +#endif + mcode_or_die("error: event_cb: curl_multi_socket_action", rc); + check_multi_info(global); + if (global->still_running <= 0) { + ev_timer_stop(global->loop, &global->timer_event); + } +} + +static void remsock(curl_socket_t s, socket_info *fdp, global_io_info *global) +{ + if (fdp) { + if (fdp->evset) { + ev_io_stop(global->loop, &fdp->ev); + } + free(fdp); + } +} + +static void setsock(socket_info *fdp, curl_socket_t s, CURL *easy, int action, + global_io_info *global) +{ + int kind = (action & CURL_POLL_IN ? (int)(EV_READ) : 0) | + (action & CURL_POLL_OUT ? (int)(EV_WRITE) : 0); + + fdp->sockfd = s; + fdp->action = action; + fdp->easy = easy; + if (fdp->evset) + ev_io_stop(global->loop, &fdp->ev); + ev_io_init(&fdp->ev, event_cb, fdp->sockfd, kind); + fdp->ev.data = global; + fdp->evset = 1; + ev_io_start(global->loop, &fdp->ev); +} + +static void addsock(curl_socket_t s, CURL *easy, int action, + global_io_info *global) +{ + socket_info *fdp = (socket_info *)(calloc(sizeof(socket_info), 1)); + + fdp->global = global; + setsock(fdp, s, easy, action, global); + curl_multi_assign(global->multi, s, fdp); +} + +static int sock_cb(CURL *easy, curl_socket_t s, int what, void *cbp, + void *sockp) +{ + global_io_info *global = (global_io_info*)(cbp); + socket_info *fdp = (socket_info*)(sockp); + + if (what == CURL_POLL_REMOVE) { + remsock(s, fdp, global); + } else { + if (!fdp) { + addsock(s, easy, what, global); + } else { + setsock(fdp, s, easy, what, global); + } + } + return 0; +} + +/* Called by libev when our timeout expires */ +static void timer_cb(EV_P_ struct ev_timer *w, int revents) +{ + global_io_info *io_global = (global_io_info*)(w->data); + CURLMcode rc; + +#if !(OLD_CURL_MULTI) + do { + rc = curl_multi_socket_action(io_global->multi, + CURL_SOCKET_TIMEOUT, 0, + &io_global->still_running); + } while (rc == CURLM_CALL_MULTI_PERFORM); +#else + do { + rc = curl_multi_socket_all(io_global->multi, + &io_global->still_running); + } while (rc == CURLM_CALL_MULTI_PERFORM); +#endif + mcode_or_die("timer_cb: curl_multi_socket_action", rc); + check_multi_info(io_global); +} + +static connection_info *get_current_connection(global_io_info *global) +{ + connection_info *conn = global->current_connection; + ulong i; + + if (conn && conn->filled_size < conn->chunk_size) + return conn; + + for (i = 0; i < opt_parallel; i++) { + conn = global->connections[i]; + if (conn->chunk_uploaded || conn->filled_size == 0) { + global->current_connection = conn; + conn_upload_init(conn); + return conn; + } + } + + return NULL; +} + +/* This gets called whenever data is received from the input */ +static void input_cb(EV_P_ struct ev_io *w, int revents) +{ + global_io_info *io_global = (global_io_info *)(w->data); + connection_info *conn = get_current_connection(io_global); + + if (conn == NULL) + return; + + if (conn->filled_size < conn->chunk_size) { + if (revents & EV_READ) { + ssize_t nbytes = read(io_global->input_fd, + conn->buffer + conn->filled_size, + conn->chunk_size - + conn->filled_size); + if (nbytes > 0) { + conn->filled_size += nbytes; + conn_buffer_updated(conn); + } else if (nbytes < 0) { + if (errno != EAGAIN && errno != EINTR) { + char error[200]; + my_strerror(error, sizeof(error), + errno); + fprintf(stderr, "error: failed to read " + "input stream (%s)\n", error); + /* failed to read input */ + exit(1); + } + } else { + io_global->eof = 1; + ev_io_stop(io_global->loop, w); + } + } + } + + assert(conn->filled_size <= conn->chunk_size); +} + +static int swift_upload_read_cb(char *ptr, size_t size, size_t nmemb, + void *data) +{ + size_t realsize; + + connection_info *conn = (connection_info*)(data); + + if (conn->filled_size == conn->upload_size && + conn->upload_size < conn->chunk_size && !conn->global->eof) { + ssize_t nbytes; + assert(conn->global->current_connection == conn); + do { + nbytes = read(conn->global->input_fd, + conn->buffer + conn->filled_size, + conn->chunk_size - conn->filled_size); + } while (nbytes == -1 && errno == EAGAIN); + if (nbytes > 0) { + conn->filled_size += nbytes; + conn_buffer_updated(conn); + } else { + conn->global->eof = 1; + } + } + + realsize = min(size * nmemb, conn->filled_size - conn->upload_size); + + memcpy(ptr, conn->buffer + conn->upload_size, realsize); + conn->upload_size += realsize; + + assert(conn->filled_size <= conn->chunk_size); + assert(conn->upload_size <= conn->filled_size); + + return realsize; +} + +static +size_t upload_header_read_cb(char *ptr, size_t size, size_t nmemb, + void *data) +{ + connection_info *conn = (connection_info *)(data); + char etag[33]; + + if (get_http_header("Etag: ", ptr, etag, array_elements(etag))) { + if (strcmp(conn->hash, etag) != 0) { + fprintf(stderr, "error: ETag mismatch\n"); + exit(EXIT_FAILURE); + } + fprintf(stderr, "acked chunk %s\n", etag); + conn->chunk_acked = true; + } + + return nmemb * size; +} + +static int conn_upload_init(connection_info *conn) +{ + conn->filled_size = 0; + conn->upload_size = 0; + conn->chunk_uploaded = false; + conn->chunk_acked = false; + conn->chunk_size = CHUNK_HEADER_CONSTANT_LEN; + conn->magic_verified = false; + conn->chunk_path_len = 0; + conn->chunk_type = XB_CHUNK_TYPE_UNKNOWN; + conn->payload_size = 0; + conn->upload_started = false; + conn->retry_count = 0; + if (conn->name != NULL) { + conn->name[0] = 0; + } + + if (conn->easy != NULL) { + conn->easy = 0; + } + + if (conn->slist != NULL) { + curl_slist_free_all(conn->slist); + conn->slist = NULL; + } + + return 0; +} + +static void conn_upload_prepare(connection_info *conn) +{ + gcry_md_hd_t md5; + + gcry_md_open(&md5, GCRY_MD_MD5, 0); + gcry_md_write(md5, conn->buffer, conn->chunk_size); + hex_md5(gcry_md_read(md5, GCRY_MD_MD5), conn->hash); + gcry_md_close(md5); +} + +static int conn_upload_start(connection_info *conn) +{ + char token_header[SWIFT_MAX_HDR_SIZE]; + char object_url[SWIFT_MAX_URL_SIZE]; + char content_len[200], etag[200]; + global_io_info *global; + CURLMcode rc; + + global = conn->global; + + fprintf(stderr, "uploading chunk %s/%s/%s.%020zu " + "(md5: %s, size: %zu)\n", + global->container, global->backup_name, conn->name, + conn->chunk_no, conn->hash, conn->chunk_size); + + snprintf(object_url, array_elements(object_url), "%s/%s/%s/%s.%020zu", + global->url, global->container, global->backup_name, + conn->name, conn->chunk_no); + + snprintf(content_len, sizeof(content_len), "Content-Length: %lu", + (ulong)(conn->chunk_size)); + + snprintf(etag, sizeof(etag), "ETag: %s", conn->hash); + + snprintf(token_header, array_elements(token_header), + "X-Auth-Token: %s", global->token); + + conn->slist = curl_slist_append(conn->slist, token_header); + conn->slist = curl_slist_append(conn->slist, + "Connection: keep-alive"); + conn->slist = curl_slist_append(conn->slist, + "Content-Type: " + "application/octet-stream"); + conn->slist = curl_slist_append(conn->slist, content_len); + conn->slist = curl_slist_append(conn->slist, etag); + + conn->easy = curl_easy_init(); + if (!conn->easy) { + fprintf(stderr, "error: curl_easy_init() failed\n"); + return 1; + } + curl_easy_setopt(conn->easy, CURLOPT_URL, object_url); + curl_easy_setopt(conn->easy, CURLOPT_READFUNCTION, + swift_upload_read_cb); + curl_easy_setopt(conn->easy, CURLOPT_READDATA, conn); + curl_easy_setopt(conn->easy, CURLOPT_VERBOSE, opt_verbose); + curl_easy_setopt(conn->easy, CURLOPT_ERRORBUFFER, conn->error); + curl_easy_setopt(conn->easy, CURLOPT_PRIVATE, conn); + curl_easy_setopt(conn->easy, CURLOPT_NOPROGRESS, 1L); + curl_easy_setopt(conn->easy, CURLOPT_LOW_SPEED_TIME, 5L); + curl_easy_setopt(conn->easy, CURLOPT_LOW_SPEED_LIMIT, 1024L); + curl_easy_setopt(conn->easy, CURLOPT_PUT, 1L); + curl_easy_setopt(conn->easy, CURLOPT_HTTPHEADER, conn->slist); + curl_easy_setopt(conn->easy, CURLOPT_HEADERFUNCTION, + upload_header_read_cb); + curl_easy_setopt(conn->easy, CURLOPT_HEADERDATA, conn); + curl_easy_setopt(conn->easy, CURLOPT_INFILESIZE, + (long) conn->chunk_size); + if (opt_cacert != NULL) + curl_easy_setopt(conn->easy, CURLOPT_CAINFO, opt_cacert); + if (opt_insecure) + curl_easy_setopt(conn->easy, CURLOPT_SSL_VERIFYPEER, FALSE); + + rc = curl_multi_add_handle(conn->global->multi, conn->easy); + mcode_or_die("conn_upload_init: curl_multi_add_handle", rc); + +#if (OLD_CURL_MULTI) + do { + rc = curl_multi_socket_all(global->multi, + &global->still_running); + } while(rc == CURLM_CALL_MULTI_PERFORM); +#endif + + conn->upload_started = true; + + return 0; +} + +static void conn_cleanup(connection_info *conn) +{ + if (conn) { + free(conn->name); + free(conn->buffer); + if (conn->slist) { + curl_slist_free_all(conn->slist); + conn->slist = NULL; + } + if (conn->easy) { + curl_easy_cleanup(conn->easy); + conn->easy = NULL; + } + } + free(conn); +} + +static void conn_upload_retry(connection_info *conn) +{ + /* already closed by cURL */ + conn->easy = NULL; + + if (conn->slist != NULL) { + curl_slist_free_all(conn->slist); + conn->slist = NULL; + } + + if (conn->retry_count++ > 3) { + fprintf(stderr, "error: retry count limit reached\n"); + exit(EXIT_FAILURE); + } + + fprintf(stderr, "warning: retrying to upload chunk %zu of '%s'\n", + conn->chunk_no, conn->name); + + conn->upload_size = 0; + + conn_upload_start(conn); +} + +static connection_info *conn_new(global_io_info *global, ulong global_idx) +{ + connection_info *conn; + + conn = (connection_info *)(calloc(1, sizeof(connection_info))); + if (conn == NULL) { + goto error; + } + + conn->global = global; + conn->global_idx = global_idx; + conn->buffer_size = SWIFT_CHUNK_SIZE; + if ((conn->buffer = (char *)(calloc(conn->buffer_size, 1))) == + NULL) { + goto error; + } + + return conn; + +error: + if (conn != NULL) { + conn_cleanup(conn); + } + + fprintf(stderr, "error: out of memory\n"); + exit(EXIT_FAILURE); + + return NULL; +} + +/*********************************************************************//** +Handle input buffer updates. Parse chunk header and set appropriate +buffer size. */ +static +void +conn_buffer_updated(connection_info *conn) +{ + bool ready_for_upload = false; + + /* chunk header */ + if (!conn->magic_verified && + conn->filled_size >= CHUNK_HEADER_CONSTANT_LEN) { + if (strncmp(XB_STREAM_CHUNK_MAGIC, conn->buffer, + sizeof(XB_STREAM_CHUNK_MAGIC) - 1) != 0) { + + fprintf(stderr, "Error: magic expected\n"); + exit(EXIT_FAILURE); + } + conn->magic_verified = true; + conn->chunk_path_len = uint4korr(conn->buffer + + PATH_LENGTH_OFFSET); + conn->chunk_type = (xb_chunk_type_t) + (conn->buffer[CHUNK_TYPE_OFFSET]); + conn->chunk_size = CHUNK_HEADER_CONSTANT_LEN + + conn->chunk_path_len; + if (conn->chunk_type != XB_CHUNK_TYPE_EOF) { + conn->chunk_size += 16; + } + } + + /* ordinary chunk */ + if (conn->magic_verified && + conn->payload_size == 0 && + conn->chunk_type != XB_CHUNK_TYPE_EOF && + conn->filled_size >= CHUNK_HEADER_CONSTANT_LEN + + conn->chunk_path_len + 16) { + + conn->payload_size = uint8korr(conn->buffer + + CHUNK_HEADER_CONSTANT_LEN + + conn->chunk_path_len); + + conn->chunk_size = conn->payload_size + 4 + 16 + + conn->chunk_path_len + + CHUNK_HEADER_CONSTANT_LEN; + + if (conn->name == NULL) { + conn->name = (char*)(malloc(conn->chunk_path_len + 1)); + } else if (conn->name_len < conn->chunk_path_len + 1) { + conn->name = (char*)(realloc(conn->name, + conn->chunk_path_len + 1)); + } + conn->name_len = conn->chunk_path_len + 1; + + memcpy(conn->name, conn->buffer + CHUNK_HEADER_CONSTANT_LEN, + conn->chunk_path_len); + conn->name[conn->chunk_path_len] = 0; + + if (conn->buffer_size < conn->chunk_size) { + conn->buffer = + (char *)(realloc(conn->buffer, conn->chunk_size)); + conn->buffer_size = conn->chunk_size; + } + } + + /* EOF chunk has no payload */ + if (conn->magic_verified && + conn->chunk_type == XB_CHUNK_TYPE_EOF && + conn->filled_size >= CHUNK_HEADER_CONSTANT_LEN + + conn->chunk_path_len) { + + if (conn->name == NULL) { + conn->name = (char*)(malloc(conn->chunk_path_len + 1)); + } else if (conn->name_len < conn->chunk_path_len + 1) { + conn->name = (char*)(realloc(conn->name, + conn->chunk_path_len + 1)); + } + conn->name_len = conn->chunk_path_len + 1; + + memcpy(conn->name, conn->buffer + CHUNK_HEADER_CONSTANT_LEN, + conn->chunk_path_len); + conn->name[conn->chunk_path_len] = 0; + } + + if (conn->filled_size > 0 && conn->filled_size == conn->chunk_size) { + ready_for_upload = true; + } + + /* start upload once recieved the size of the chunk */ + if (!conn->upload_started && ready_for_upload) { + conn->chunk_no = file_chunk_count[conn->name]++; + conn_upload_prepare(conn); + conn_upload_start(conn); + } +} + +static int init_input(global_io_info *io_global) +{ + ev_io_init(&io_global->input_event, input_cb, STDIN_FILENO, EV_READ); + io_global->input_event.data = io_global; + ev_io_start(io_global->loop, &io_global->input_event); + + return 0; +} + +/* Update the event timer after curl_multi library calls */ +static int multi_timer_cb(CURLM *multi, long timeout_ms, global_io_info *global) +{ + ev_timer_stop(global->loop, &global->timer_event); + if (timeout_ms > 0) { + double t = timeout_ms / 1000.0; + ev_timer_init(&global->timer_event, timer_cb, t, 0.); + ev_timer_start(global->loop, &global->timer_event); + } else { + timer_cb(global->loop, &global->timer_event, 0); + } + return 0; +} + +static +int swift_upload_parts(swift_auth_info *auth, const char *container, + const char *name) +{ + global_io_info io_global; + ulong i; +#if (OLD_CURL_MULTI) + long timeout; +#endif + CURLMcode rc; + int n_dirty_buffers; + + memset(&io_global, 0, sizeof(io_global)); + + io_global.loop = ev_default_loop(0); + init_input(&io_global); + io_global.multi = curl_multi_init(); + ev_timer_init(&io_global.timer_event, timer_cb, 0., 0.); + io_global.timer_event.data = &io_global; + io_global.connections = (connection_info **) + (calloc(opt_parallel, sizeof(connection_info))); + io_global.url = auth->url; + io_global.container = container; + io_global.backup_name = name; + io_global.token = auth->token; + for (i = 0; i < opt_parallel; i++) { + io_global.connections[i] = conn_new(&io_global, i); + } + + /* setup the generic multi interface options we want */ + curl_multi_setopt(io_global.multi, CURLMOPT_SOCKETFUNCTION, sock_cb); + curl_multi_setopt(io_global.multi, CURLMOPT_SOCKETDATA, &io_global); +#if !(OLD_CURL_MULTI) + curl_multi_setopt(io_global.multi, CURLMOPT_TIMERFUNCTION, multi_timer_cb); + curl_multi_setopt(io_global.multi, CURLMOPT_TIMERDATA, &io_global); + do { + rc = curl_multi_socket_action(io_global.multi, + CURL_SOCKET_TIMEOUT, 0, + &io_global.still_running); + } while (rc == CURLM_CALL_MULTI_PERFORM); +#else + curl_multi_timeout(io_global.multi, &timeout); + if (timeout >= 0) { + multi_timer_cb(io_global.multi, timeout, &io_global); + } + do { + rc = curl_multi_socket_all(io_global.multi, &io_global.still_running); + } while(rc == CURLM_CALL_MULTI_PERFORM); +#endif + + ev_loop(io_global.loop, 0); + check_multi_info(&io_global); + curl_multi_cleanup(io_global.multi); + + n_dirty_buffers = 0; + for (i = 0; i < opt_parallel; i++) { + connection_info *conn = io_global.connections[i]; + if (conn && conn->upload_size != conn->filled_size) { + fprintf(stderr, "error: upload failed: %lu bytes left " + "in the buffer %s (uploaded = %d)\n", + (ulong)(conn->filled_size - conn->upload_size), + conn->name, conn->chunk_uploaded); + ++n_dirty_buffers; + } + } + + for (i = 0; i < opt_parallel; i++) { + if (io_global.connections[i] != NULL) { + conn_cleanup(io_global.connections[i]); + } + } + free(io_global.connections); + + if (n_dirty_buffers > 0) { + return(EXIT_FAILURE); + } + + return 0; +} + +struct download_buffer_info { + off_t offset; + size_t size; + size_t result_len; + char *buf; + curl_read_callback custom_header_callback; + void *custom_header_callback_data; +}; + +/*********************************************************************//** +Callback to parse header of GET request on swift contaier. */ +static +size_t fetch_buffer_header_cb(char *ptr, size_t size, size_t nmemb, + void *data) +{ + download_buffer_info *buffer_info = (download_buffer_info*)(data); + size_t buf_size; + char content_length_str[100]; + char *endptr; + + if (get_http_header("Content-Length: ", ptr, + content_length_str, sizeof(content_length_str))) { + + buf_size = strtoull(content_length_str, &endptr, 10); + + if (buffer_info->buf == NULL) { + buffer_info->buf = (char*)(malloc(buf_size)); + buffer_info->size = buf_size; + } + + if (buf_size > buffer_info->size) { + buffer_info->buf = (char*) + (realloc(buffer_info->buf, buf_size)); + buffer_info->size = buf_size; + } + + buffer_info->result_len = buf_size; + } + + if (buffer_info->custom_header_callback) { + buffer_info->custom_header_callback(ptr, size, nmemb, + buffer_info->custom_header_callback_data); + } + + return nmemb * size; +} + +/*********************************************************************//** +Write contents into string buffer */ +static +size_t +fetch_buffer_cb(char *buffer, size_t size, size_t nmemb, void *out_buffer) +{ + download_buffer_info *buffer_info = (download_buffer_info*)(out_buffer); + + assert(buffer_info->size >= buffer_info->offset + size * nmemb); + + memcpy(buffer_info->buf + buffer_info->offset, buffer, size * nmemb); + buffer_info->offset += size * nmemb; + + return size * nmemb; +} + + +/*********************************************************************//** +Downloads contents of URL into buffer. Caller is responsible for +deallocating the buffer. +@return pointer to a buffer or NULL */ +static +char * +swift_fetch_into_buffer(swift_auth_info *auth, const char *url, + char **buf, size_t *buf_size, size_t *result_len, + curl_read_callback header_callback, + void *header_callback_data) +{ + char auth_token[SWIFT_MAX_HDR_SIZE]; + download_buffer_info buffer_info; + struct curl_slist *slist = NULL; + long http_code; + CURL *curl; + CURLcode res; + + memset(&buffer_info, 0, sizeof(buffer_info)); + buffer_info.buf = *buf; + buffer_info.size = *buf_size; + buffer_info.custom_header_callback = header_callback; + buffer_info.custom_header_callback_data = header_callback_data; + + snprintf(auth_token, array_elements(auth_token), "X-Auth-Token: %s", + auth->token); + + curl = curl_easy_init(); + + if (curl != NULL) { + slist = curl_slist_append(slist, auth_token); + + curl_easy_setopt(curl, CURLOPT_VERBOSE, opt_verbose); + curl_easy_setopt(curl, CURLOPT_URL, url); + curl_easy_setopt(curl, CURLOPT_FOLLOWLOCATION, 1L); + curl_easy_setopt(curl, CURLOPT_HTTPHEADER, slist); + curl_easy_setopt(curl, CURLOPT_WRITEFUNCTION, fetch_buffer_cb); + curl_easy_setopt(curl, CURLOPT_WRITEDATA, &buffer_info); + curl_easy_setopt(curl, CURLOPT_HEADERFUNCTION, + fetch_buffer_header_cb); + curl_easy_setopt(curl, CURLOPT_HEADERDATA, + &buffer_info); + if (opt_cacert != NULL) + curl_easy_setopt(curl, CURLOPT_CAINFO, opt_cacert); + if (opt_insecure) + curl_easy_setopt(curl, CURLOPT_SSL_VERIFYPEER, FALSE); + + res = curl_easy_perform(curl); + + if (res != CURLE_OK) { + fprintf(stderr, + "error: curl_easy_perform() failed: %s\n", + curl_easy_strerror(res)); + goto cleanup; + } + curl_easy_getinfo(curl, CURLINFO_RESPONSE_CODE, &http_code); + if (http_code < 200 || http_code >= 300) { + fprintf(stderr, "error: request failed " + "with response code: %ld\n", http_code); + res = CURLE_LOGIN_DENIED; + goto cleanup; + } + } else { + res = CURLE_FAILED_INIT; + fprintf(stderr, "error: curl_easy_init() failed\n"); + goto cleanup; + } + +cleanup: + if (slist) { + curl_slist_free_all(slist); + } + if (curl) { + curl_easy_cleanup(curl); + } + + if (res == CURLE_OK) { + *buf = buffer_info.buf; + *buf_size = buffer_info.size; + *result_len = buffer_info.result_len; + return(buffer_info.buf); + } + + free(buffer_info.buf); + *buf = NULL; + *buf_size = 0; + *result_len = 0; + + return(NULL); +} + +static +container_list * +container_list_new() +{ + container_list *list = + (container_list *)(calloc(1, sizeof(container_list))); + + list->object_count = 1000; + list->objects = (object_info*) + (calloc(list->object_count, sizeof(object_info))); + + if (list->objects == NULL) { + fprintf(stderr, "error: out of memory\n"); + free(list); + return(NULL); + } + + return(list); +} + +static +void +container_list_free(container_list *list) +{ + free(list->content_json); + free(list->objects); + free(list); +} + +static +void +container_list_add_object(container_list *list, const char *name, + const char *hash, size_t bytes) +{ + const size_t object_count_step = 1000; + + if (list->idx >= list->object_count) { + list->objects = (object_info*) + realloc(list->objects, + (list->object_count + object_count_step) * + sizeof(object_info)); + memset(list->objects + list->object_count, 0, + object_count_step * sizeof(object_info)); + list->object_count += object_count_step; + } + assert(list->idx <= list->object_count); + strcpy(list->objects[list->idx].name, name); + strcpy(list->objects[list->idx].hash, hash); + list->objects[list->idx].bytes = bytes; + ++list->idx; +} + + +/*********************************************************************//** +Tokenize json string. Return array of tokens. Caller is responsoble for +deallocating the array. */ +jsmntok_t * +json_tokenise(char *json, size_t len, int initial_tokens) +{ + jsmn_parser parser; + jsmn_init(&parser); + + unsigned int n = initial_tokens; + jsmntok_t *tokens = (jsmntok_t *)(malloc(sizeof(jsmntok_t) * n)); + + int ret = jsmn_parse(&parser, json, len, tokens, n); + + while (ret == JSMN_ERROR_NOMEM) + { + n = n * 2 + 1; + tokens = (jsmntok_t*)(realloc(tokens, sizeof(jsmntok_t) * n)); + ret = jsmn_parse(&parser, json, len, tokens, n); + } + + if (ret == JSMN_ERROR_INVAL) { + fprintf(stderr, "error: invalid JSON string\n"); + + } + if (ret == JSMN_ERROR_PART) { + fprintf(stderr, "error: truncated JSON string\n"); + } + + return tokens; +} + +/*********************************************************************//** +Return true if token representation equal to given string. */ +static +bool +json_token_eq(const char *buf, jsmntok_t *t, const char *s) +{ + size_t len = strlen(s); + + assert(t->end > t->start); + + return((size_t)(t->end - t->start) == len && + (strncmp(buf + t->start, s, len) == 0)); +} + +/*********************************************************************//** +Copy given token as string. */ +static +bool +json_token_str(const char *buf, jsmntok_t *t, char *out, int out_size) +{ + size_t len = min(t->end - t->start, out_size - 1); + + memcpy(out, buf + t->start, len); + out[len] = 0; + + return(true); +} + +/*********************************************************************//** +Parse SWIFT container list response and fill output array with values +sorted by object name. */ +static +bool +swift_parse_container_list(container_list *list) +{ + enum {MAX_DEPTH=20}; + enum label_t {NONE, OBJECT}; + + char name[SWIFT_MAX_URL_SIZE]; + char hash[33]; + char bytes[30]; + char *response = list->content_json; + + struct stack_t { + jsmntok_t *t; + int n_items; + label_t label; + }; + + stack_t stack[MAX_DEPTH]; + jsmntok_t *tokens; + int level; + size_t count = 0; + + tokens = json_tokenise(list->content_json, list->content_length, 200); + + stack[0].t = &tokens[0]; + stack[0].label = NONE; + stack[0].n_items = 1; + level = 0; + + for (size_t i = 0, j = 1; j > 0; i++, j--) { + jsmntok_t *t = &tokens[i]; + + assert(t->start != -1 && t->end != -1); + assert(level >= 0); + + --stack[level].n_items; + + switch (t->type) { + case JSMN_ARRAY: + case JSMN_OBJECT: + if (level < MAX_DEPTH - 1) { + level++; + } + stack[level].t = t; + stack[level].label = NONE; + if (t->type == JSMN_ARRAY) { + stack[level].n_items = t->size; + j += t->size; + } else { + stack[level].n_items = t->size * 2; + j += t->size * 2; + } + break; + case JSMN_PRIMITIVE: + case JSMN_STRING: + if (stack[level].t->type == JSMN_OBJECT && + stack[level].n_items % 2 == 1) { + /* key */ + if (json_token_eq(response, t, "name")) { + json_token_str(response, &tokens[i + 1], + name, sizeof(name)); + } + if (json_token_eq(response, t, "hash")) { + json_token_str(response, &tokens[i + 1], + hash, sizeof(hash)); + } + if (json_token_eq(response, t, "bytes")) { + json_token_str(response, &tokens[i + 1], + bytes, sizeof(bytes)); + } + } + break; + } + + while (stack[level].n_items == 0 && level > 0) { + if (stack[level].t->type == JSMN_OBJECT + && level == 2) { + char *endptr; + container_list_add_object(list, name, hash, + strtoull(bytes, &endptr, 10)); + ++count; + } + --level; + } + } + + if (count == 0) { + list->final = true; + } + + free(tokens); + + return(true); +} + +/*********************************************************************//** +List swift container with given name. Return list of objects sorted by +object name. */ +static +container_list * +swift_list(swift_auth_info *auth, const char *container, const char *path) +{ + container_list *list; + char url[SWIFT_MAX_URL_SIZE]; + + list = container_list_new(); + + while (!list->final) { + + /* download the list in json format */ + snprintf(url, array_elements(url), + "%s/%s?format=json&limit=1000%s%s%s%s", + auth->url, container, path ? "&prefix=" : "", + path ? path : "", list->idx > 0 ? "&marker=" : "", + list->idx > 0 ? + list->objects[list->idx - 1].name : ""); + + list->content_json = swift_fetch_into_buffer(auth, url, + &list->content_json, &list->content_bufsize, + &list->content_length, NULL, NULL); + + if (list->content_json == NULL) { + container_list_free(list); + return(NULL); + } + + /* parse downloaded list */ + if (!swift_parse_container_list(list)) { + fprintf(stderr, "error: unable to parse " + "container list\n"); + container_list_free(list); + return(NULL); + } + } + + return(list); +} + + +/*********************************************************************//** +Return true if chunk is a part of backup with given name. */ +static +bool +chunk_belongs_to(const char *chunk_name, const char *backup_name) +{ + size_t backup_name_len = strlen(backup_name); + + return((strlen(chunk_name) > backup_name_len) + && (chunk_name[backup_name_len] == '/') + && strncmp(chunk_name, backup_name, backup_name_len) == 0); +} + +/*********************************************************************//** +Return true if chunk is in given list. */ +static +bool +chunk_in_list(const char *chunk_name, char **list, int list_size) +{ + size_t chunk_name_len; + + if (list_size == 0) { + return(true); + } + + chunk_name_len = strlen(chunk_name); + if (chunk_name_len < 20) { + return(false); + } + + for (int i = 0; i < list_size; i++) { + size_t item_len = strlen(list[i]); + + if ((strncmp(chunk_name - item_len + chunk_name_len - 21, + list[i], item_len) == 0) + && (chunk_name[chunk_name_len - 21] == '.') + && (chunk_name[chunk_name_len - item_len - 22] == '/')) { + return(true); + } + } + + return(false); +} + +static +int swift_download(swift_auth_info *auth, const char *container, + const char *name) +{ + container_list *list; + char *buf = NULL; + size_t buf_size = 0; + size_t result_len = 0; + + if ((list = swift_list(auth, container, name)) == NULL) { + return(CURLE_FAILED_INIT); + } + + for (size_t i = 0; i < list->idx; i++) { + const char *chunk_name = list->objects[i].name; + + if (chunk_belongs_to(chunk_name, name) + && chunk_in_list(chunk_name, file_list, file_list_size)) { + char url[SWIFT_MAX_URL_SIZE]; + + snprintf(url, sizeof(url), "%s/%s/%s", + auth->url, container, chunk_name); + + if ((buf = swift_fetch_into_buffer( + auth, url, &buf, &buf_size, &result_len, + NULL, NULL)) == NULL) { + fprintf(stderr, "error: failed to download " + "chunk %s\n", chunk_name); + container_list_free(list); + return(CURLE_FAILED_INIT); + } + + fwrite(buf, 1, result_len, stdout); + } + } + + free(buf); + + container_list_free(list); + + return(CURLE_OK); +} + + +/*********************************************************************//** +Delete backup with given name from given container. +@return true if backup deleted successfully */ +static +bool swift_delete(swift_auth_info *auth, const char *container, + const char *name) +{ + container_list *list; + + if ((list = swift_list(auth, container, name)) == NULL) { + return(CURLE_FAILED_INIT); + } + + for (size_t i = 0; i < list->object_count; i++) { + const char *chunk_name = list->objects[i].name; + + if (chunk_belongs_to(chunk_name, name)) { + char url[SWIFT_MAX_URL_SIZE]; + + snprintf(url, sizeof(url), "%s/%s/%s", + auth->url, container, chunk_name); + + fprintf(stderr, "delete %s\n", chunk_name); + if (!swift_delete_object(auth, url)) { + fprintf(stderr, "error: failed to delete " + "chunk %s\n", chunk_name); + container_list_free(list); + return(CURLE_FAILED_INIT); + } + } + } + + container_list_free(list); + + return(CURLE_OK); +} + +/*********************************************************************//** +Check if backup with given name exists. +@return true if backup exists */ +static +bool swift_backup_exists(swift_auth_info *auth, const char *container, + const char *backup_name) +{ + container_list *list; + + if ((list = swift_list(auth, container, backup_name)) == NULL) { + fprintf(stderr, "error: unable to list container %s\n", + container); + exit(EXIT_FAILURE); + } + + for (size_t i = 0; i < list->object_count; i++) { + if (chunk_belongs_to(list->objects[i].name, backup_name)) { + container_list_free(list); + return(true); + } + } + + container_list_free(list); + + return(false); +} + +/*********************************************************************//** +Fills auth_info with response from keystone response. +@return true is response parsed successfully */ +static +bool +swift_parse_keystone_response_v2(char *response, size_t response_length, + swift_auth_info *auth_info) +{ + enum {MAX_DEPTH=20}; + enum label_t {NONE, ACCESS, CATALOG, ENDPOINTS, TOKEN}; + + char filtered_url[SWIFT_MAX_URL_SIZE]; + char public_url[SWIFT_MAX_URL_SIZE]; + char region[SWIFT_MAX_URL_SIZE]; + char id[SWIFT_MAX_URL_SIZE]; + char token_id[SWIFT_MAX_URL_SIZE]; + char type[SWIFT_MAX_URL_SIZE]; + + struct stack_t { + jsmntok_t *t; + int n_items; + label_t label; + }; + + stack_t stack[MAX_DEPTH]; + jsmntok_t *tokens; + int level; + + tokens = json_tokenise(response, response_length, 200); + + stack[0].t = &tokens[0]; + stack[0].label = NONE; + stack[0].n_items = 1; + level = 0; + + for (size_t i = 0, j = 1; j > 0; i++, j--) { + jsmntok_t *t = &tokens[i]; + + assert(t->start != -1 && t->end != -1); + assert(level >= 0); + + --stack[level].n_items; + + switch (t->type) { + case JSMN_ARRAY: + case JSMN_OBJECT: + if (level < MAX_DEPTH - 1) { + level++; + } + stack[level].t = t; + stack[level].label = NONE; + if (t->type == JSMN_ARRAY) { + stack[level].n_items = t->size; + j += t->size; + } else { + stack[level].n_items = t->size * 2; + j += t->size * 2; + } + break; + case JSMN_PRIMITIVE: + case JSMN_STRING: + if (stack[level].t->type == JSMN_OBJECT && + stack[level].n_items % 2 == 1) { + /* key */ + if (json_token_eq(response, t, "access")) { + stack[level].label = ACCESS; + } + if (json_token_eq(response, t, + "serviceCatalog")) { + stack[level].label = CATALOG; + } + if (json_token_eq(response, t, "endpoints")) { + stack[level].label = ENDPOINTS; + } + if (json_token_eq(response, t, "token")) { + stack[level].label = TOKEN; + } + if (json_token_eq(response, t, "id")) { + json_token_str(response, &tokens[i + 1], + id, sizeof(id)); + } + if (json_token_eq(response, t, "id") + && stack[level - 1].label == TOKEN) { + json_token_str(response, &tokens[i + 1], + token_id, sizeof(token_id)); + } + if (json_token_eq(response, t, "region")) { + json_token_str(response, &tokens[i + 1], + region, sizeof(region)); + } + if (json_token_eq(response, t, "publicURL")) { + json_token_str(response, &tokens[i + 1], + public_url, sizeof(public_url)); + } + if (json_token_eq(response, t, "type")) { + json_token_str(response, &tokens[i + 1], + type, sizeof(type)); + } + } + break; + } + + while (stack[level].n_items == 0 && level > 0) { + if (stack[level].t->type == JSMN_OBJECT + && level == 6 + && stack[level - 1].t->type == JSMN_ARRAY + && stack[level - 2].label == ENDPOINTS) { + if (opt_swift_region == NULL + || strcmp(opt_swift_region, region) == 0) { + strncpy(filtered_url, public_url, + sizeof(filtered_url)); + } + } + if (stack[level].t->type == JSMN_OBJECT && + level == 4 && + stack[level - 1].t->type == JSMN_ARRAY && + stack[level - 2].label == CATALOG) { + if (strcmp(type, "object-store") == 0) { + strncpy(auth_info->url, filtered_url, + sizeof(auth_info->url)); + } + } + --level; + } + } + + free(tokens); + + strncpy(auth_info->token, token_id, sizeof(auth_info->token)); + + assert(level == 0); + + if (*auth_info->token == 0) { + fprintf(stderr, "error: can not receive token from response\n"); + return(false); + } + + if (*auth_info->url == 0) { + fprintf(stderr, "error: can not get URL from response\n"); + return(false); + } + + return(true); +} + +/*********************************************************************//** +Authenticate against Swift TempAuth. Fills swift_auth_info struct. +Uses creadentials privided as global variables. +@returns true if access is granted and token received. */ +static +bool +swift_keystone_auth_v2(const char *auth_url, swift_auth_info *info) +{ + char tenant_arg[SWIFT_MAX_URL_SIZE]; + char payload[SWIFT_MAX_URL_SIZE]; + struct curl_slist *slist = NULL; + download_buffer_info buf_info; + long http_code; + CURLcode res; + CURL *curl; + bool auth_res = false; + + memset(&buf_info, 0, sizeof(buf_info)); + + if (opt_swift_user == NULL) { + fprintf(stderr, "error: both --swift-user is required " + "for keystone authentication.\n"); + return(false); + } + + if (opt_swift_password == NULL) { + fprintf(stderr, "error: both --swift-password is required " + "for keystone authentication.\n"); + return(false); + } + + if (opt_swift_tenant != NULL && opt_swift_tenant_id != NULL) { + fprintf(stderr, "error: both --swift-tenant and " + "--swift-tenant-id specified for keystone " + "authentication.\n"); + return(false); + } + + if (opt_swift_tenant != NULL) { + snprintf(tenant_arg, sizeof(tenant_arg), ",\"%s\":\"%s\"", + "tenantName", opt_swift_tenant); + } else if (opt_swift_tenant_id != NULL) { + snprintf(tenant_arg, sizeof(tenant_arg), ",\"%s\":\"%s\"", + "tenantId", opt_swift_tenant_id); + } else { + *tenant_arg = 0; + } + + snprintf(payload, sizeof(payload), "{\"auth\": " + "{\"passwordCredentials\": {\"username\":\"%s\"," + "\"password\":\"%s\"}%s}}", + opt_swift_user, opt_swift_password, tenant_arg); + + curl = curl_easy_init(); + + if (curl != NULL) { + + slist = curl_slist_append(slist, + "Content-Type: application/json"); + slist = curl_slist_append(slist, + "Accept: application/json"); + + curl_easy_setopt(curl, CURLOPT_VERBOSE, opt_verbose); + curl_easy_setopt(curl, CURLOPT_POST, 1L); + curl_easy_setopt(curl, CURLOPT_URL, auth_url); + curl_easy_setopt(curl, CURLOPT_FOLLOWLOCATION, 1L); + curl_easy_setopt(curl, CURLOPT_POSTFIELDS, payload); + curl_easy_setopt(curl, CURLOPT_WRITEFUNCTION, fetch_buffer_cb); + curl_easy_setopt(curl, CURLOPT_WRITEDATA, &buf_info); + curl_easy_setopt(curl, CURLOPT_HEADERFUNCTION, + fetch_buffer_header_cb); + curl_easy_setopt(curl, CURLOPT_HEADERDATA, + &buf_info); + curl_easy_setopt(curl, CURLOPT_HTTPHEADER, slist); + + if (opt_cacert != NULL) + curl_easy_setopt(curl, CURLOPT_CAINFO, opt_cacert); + if (opt_insecure) + curl_easy_setopt(curl, CURLOPT_SSL_VERIFYPEER, FALSE); + + res = curl_easy_perform(curl); + + if (res != CURLE_OK) { + fprintf(stderr, + "error: curl_easy_perform() failed: %s\n", + curl_easy_strerror(res)); + goto cleanup; + } + curl_easy_getinfo(curl, CURLINFO_RESPONSE_CODE, &http_code); + if (http_code < 200 || http_code >= 300) { + fprintf(stderr, "error: request failed " + "with response code: %ld\n", http_code); + res = CURLE_LOGIN_DENIED; + goto cleanup; + } + } else { + res = CURLE_FAILED_INIT; + fprintf(stderr, "error: curl_easy_init() failed\n"); + goto cleanup; + } + + if (!swift_parse_keystone_response_v2(buf_info.buf, + buf_info.size, info)) { + goto cleanup; + } + + auth_res = true; + +cleanup: + if (slist) { + curl_slist_free_all(slist); + } + if (curl) { + curl_easy_cleanup(curl); + } + + free(buf_info.buf); + + return(auth_res); +} + + +/*********************************************************************//** +Fills auth_info with response from keystone response. +@return true is response parsed successfully */ +static +bool +swift_parse_keystone_response_v3(char *response, size_t response_length, + swift_auth_info *auth_info) +{ + enum {MAX_DEPTH=20}; + enum label_t {NONE, TOKEN, CATALOG, ENDPOINTS}; + + char url[SWIFT_MAX_URL_SIZE]; + char filtered_url[SWIFT_MAX_URL_SIZE]; + char region[SWIFT_MAX_URL_SIZE]; + char interface[SWIFT_MAX_URL_SIZE]; + char type[SWIFT_MAX_URL_SIZE]; + + struct stack_t { + jsmntok_t *t; + int n_items; + label_t label; + }; + + stack_t stack[MAX_DEPTH]; + jsmntok_t *tokens; + int level; + + tokens = json_tokenise(response, response_length, 200); + + stack[0].t = &tokens[0]; + stack[0].label = NONE; + stack[0].n_items = 1; + level = 0; + + for (size_t i = 0, j = 1; j > 0; i++, j--) { + jsmntok_t *t = &tokens[i]; + + assert(t->start != -1 && t->end != -1); + assert(level >= 0); + + --stack[level].n_items; + + switch (t->type) { + case JSMN_ARRAY: + case JSMN_OBJECT: + if (level < MAX_DEPTH - 1) { + level++; + } + stack[level].t = t; + stack[level].label = NONE; + if (t->type == JSMN_ARRAY) { + stack[level].n_items = t->size; + j += t->size; + } else { + stack[level].n_items = t->size * 2; + j += t->size * 2; + } + break; + case JSMN_PRIMITIVE: + case JSMN_STRING: + if (stack[level].t->type == JSMN_OBJECT && + stack[level].n_items % 2 == 1) { + /* key */ + if (json_token_eq(response, t, "token")) { + stack[level].label = TOKEN; + fprintf(stderr, "token\n"); + } + if (json_token_eq(response, t, + "catalog")) { + stack[level].label = CATALOG; + fprintf(stderr, "catalog\n"); + } + if (json_token_eq(response, t, "endpoints")) { + stack[level].label = ENDPOINTS; + } + if (json_token_eq(response, t, "region")) { + json_token_str(response, &tokens[i + 1], + region, sizeof(region)); + } + if (json_token_eq(response, t, "url")) { + json_token_str(response, &tokens[i + 1], + url, sizeof(url)); + } + if (json_token_eq(response, t, "interface")) { + json_token_str(response, &tokens[i + 1], + interface, sizeof(interface)); + } + if (json_token_eq(response, t, "type")) { + json_token_str(response, &tokens[i + 1], + type, sizeof(type)); + } + } + break; + } + + while (stack[level].n_items == 0 && level > 0) { + if (stack[level].t->type == JSMN_OBJECT + && level == 6 + && stack[level - 1].t->type == JSMN_ARRAY + && stack[level - 2].label == ENDPOINTS) { + if ((opt_swift_region == NULL + || strcmp(opt_swift_region, region) == 0) + && strcmp(interface, "public") == 0) { + strncpy(filtered_url, url, + sizeof(filtered_url)); + } + } + if (stack[level].t->type == JSMN_OBJECT && + level == 4 && + stack[level - 1].t->type == JSMN_ARRAY && + stack[level - 2].label == CATALOG) { + if (strcmp(type, "object-store") == 0) { + strncpy(auth_info->url, filtered_url, + sizeof(auth_info->url)); + } + } + --level; + } + } + + free(tokens); + + assert(level == 0); + + if (*auth_info->url == 0) { + fprintf(stderr, "error: can not get URL from response\n"); + return(false); + } + + return(true); +} + +/*********************************************************************//** +Captures X-Subject-Token header. */ +static +size_t keystone_v3_header_cb(char *ptr, size_t size, size_t nmemb, void *data) +{ + swift_auth_info *info = (swift_auth_info*)(data); + + get_http_header("X-Subject-Token: ", ptr, + info->token, array_elements(info->token)); + + return nmemb * size; +} + +/*********************************************************************//** +Authenticate against Swift TempAuth. Fills swift_auth_info struct. +Uses creadentials privided as global variables. +@returns true if access is granted and token received. */ +static +bool +swift_keystone_auth_v3(const char *auth_url, swift_auth_info *info) +{ + char scope[SWIFT_MAX_URL_SIZE]; + char domain[SWIFT_MAX_URL_SIZE]; + char payload[SWIFT_MAX_URL_SIZE]; + struct curl_slist *slist = NULL; + download_buffer_info buf_info; + long http_code; + CURLcode res; + CURL *curl; + bool auth_res = false; + + memset(&buf_info, 0, sizeof(buf_info)); + buf_info.custom_header_callback = keystone_v3_header_cb; + buf_info.custom_header_callback_data = info; + + if (opt_swift_user == NULL) { + fprintf(stderr, "error: both --swift-user is required " + "for keystone authentication.\n"); + return(false); + } + + if (opt_swift_password == NULL) { + fprintf(stderr, "error: both --swift-password is required " + "for keystone authentication.\n"); + return(false); + } + + if (opt_swift_project_id != NULL && opt_swift_project != NULL) { + fprintf(stderr, "error: both --swift-project and " + "--swift-project-id specified for keystone " + "authentication.\n"); + return(false); + } + + if (opt_swift_domain_id != NULL && opt_swift_domain != NULL) { + fprintf(stderr, "error: both --swift-domain and " + "--swift-domain-id specified for keystone " + "authentication.\n"); + return(false); + } + + if (opt_swift_project_id != NULL && opt_swift_domain != NULL) { + fprintf(stderr, "error: both --swift-project-id and " + "--swift-domain specified for keystone " + "authentication.\n"); + return(false); + } + + if (opt_swift_project_id != NULL && opt_swift_domain_id != NULL) { + fprintf(stderr, "error: both --swift-project-id and " + "--swift-domain-id specified for keystone " + "authentication.\n"); + return(false); + } + + scope[0] = 0; domain[0] = 0; + + if (opt_swift_domain != NULL) { + snprintf(domain, sizeof(domain), + ",{\"domain\":{\"name\":\"%s\"}}", + opt_swift_domain); + } else if (opt_swift_domain_id != NULL) { + snprintf(domain, sizeof(domain), + ",{\"domain\":{\"id\":\"%s\"}}", + opt_swift_domain_id); + } + + if (opt_swift_project_id != NULL) { + snprintf(scope, sizeof(scope), + ",\"scope\":{\"project\":{\"id\":\"%s\"}}", + opt_swift_project_id); + } else if (opt_swift_project != NULL) { + snprintf(scope, sizeof(scope), + ",\"scope\":{\"project\":{\"name\":\"%s\"%s}}", + opt_swift_project_id, domain); + } + + snprintf(payload, sizeof(payload), "{\"auth\":{\"identity\":" + "{\"methods\":[\"password\"],\"password\":{\"user\":" + "{\"name\":\"%s\",\"password\":\"%s\"%s}}}%s}}", + opt_swift_user, opt_swift_password, + *scope ? "" : ",\"domain\":{\"id\":\"default\"}", + scope); + + curl = curl_easy_init(); + + if (curl != NULL) { + + slist = curl_slist_append(slist, + "Content-Type: application/json"); + slist = curl_slist_append(slist, + "Accept: application/json"); + + curl_easy_setopt(curl, CURLOPT_VERBOSE, opt_verbose); + curl_easy_setopt(curl, CURLOPT_POST, 1L); + curl_easy_setopt(curl, CURLOPT_URL, auth_url); + curl_easy_setopt(curl, CURLOPT_FOLLOWLOCATION, 1L); + curl_easy_setopt(curl, CURLOPT_POSTFIELDS, payload); + curl_easy_setopt(curl, CURLOPT_WRITEFUNCTION, fetch_buffer_cb); + curl_easy_setopt(curl, CURLOPT_WRITEDATA, &buf_info); + curl_easy_setopt(curl, CURLOPT_HEADERFUNCTION, + fetch_buffer_header_cb); + curl_easy_setopt(curl, CURLOPT_HEADERDATA, + &buf_info); + curl_easy_setopt(curl, CURLOPT_HTTPHEADER, slist); + + if (opt_cacert != NULL) + curl_easy_setopt(curl, CURLOPT_CAINFO, opt_cacert); + if (opt_insecure) + curl_easy_setopt(curl, CURLOPT_SSL_VERIFYPEER, FALSE); + + res = curl_easy_perform(curl); + + if (res != CURLE_OK) { + fprintf(stderr, + "error: curl_easy_perform() failed: %s\n", + curl_easy_strerror(res)); + goto cleanup; + } + curl_easy_getinfo(curl, CURLINFO_RESPONSE_CODE, &http_code); + if (http_code < 200 || http_code >= 300) { + fprintf(stderr, "error: request failed " + "with response code: %ld\n", http_code); + res = CURLE_LOGIN_DENIED; + goto cleanup; + } + } else { + res = CURLE_FAILED_INIT; + fprintf(stderr, "error: curl_easy_init() failed\n"); + goto cleanup; + } + + if (!swift_parse_keystone_response_v3(buf_info.buf, + buf_info.size, info)) { + goto cleanup; + } + + auth_res = true; + +cleanup: + if (slist) { + curl_slist_free_all(slist); + } + if (curl) { + curl_easy_cleanup(curl); + } + + free(buf_info.buf); + + return(auth_res); +} + +int main(int argc, char **argv) +{ + swift_auth_info info; + char auth_url[SWIFT_MAX_URL_SIZE]; + + MY_INIT(argv[0]); + + /* handle_options in parse_args is destructive so + * we make a copy of our argument pointers so we can + * mask the sensitive values afterwards */ + char **mask_argv = (char **)malloc(sizeof(char *) * (argc - 1)); + memcpy(mask_argv, argv + 1, sizeof(char *) * (argc - 1)); + + if (parse_args(argc, argv)) { + return(EXIT_FAILURE); + } + + mask_args(argc, mask_argv); /* mask args on cmdline */ + + curl_global_init(CURL_GLOBAL_ALL); + + if (opt_swift_auth_version == NULL || *opt_swift_auth_version == '1') { + /* TempAuth */ + snprintf(auth_url, SWIFT_MAX_URL_SIZE, "%sauth/v%s/", + opt_swift_auth_url, opt_swift_auth_version ? + opt_swift_auth_version : "1.0"); + + if (!swift_temp_auth(auth_url, &info)) { + fprintf(stderr, "error: failed to authenticate\n"); + return(EXIT_FAILURE); + } + + } else if (*opt_swift_auth_version == '2') { + /* Keystone v2 */ + snprintf(auth_url, SWIFT_MAX_URL_SIZE, "%sv%s/tokens", + opt_swift_auth_url, opt_swift_auth_version); + + if (!swift_keystone_auth_v2(auth_url, &info)) { + fprintf(stderr, "error: failed to authenticate\n"); + return(EXIT_FAILURE); + } + + } else if (*opt_swift_auth_version == '3') { + /* Keystone v3 */ + snprintf(auth_url, SWIFT_MAX_URL_SIZE, "%sv%s/auth/tokens", + opt_swift_auth_url, opt_swift_auth_version); + + if (!swift_keystone_auth_v3(auth_url, &info)) { + fprintf(stderr, "error: failed to authenticate\n"); + exit(EXIT_FAILURE); + } + + } + + if (opt_swift_storage_url != NULL) { + snprintf(info.url, sizeof(info.url), "%s", + opt_swift_storage_url); + } + + fprintf(stderr, "Object store URL: %s\n", info.url); + + if (opt_mode == MODE_PUT) { + + if (swift_create_container(&info, opt_swift_container) != 0) { + fprintf(stderr, "error: failed to create " + "container %s\n", + opt_swift_container); + return(EXIT_FAILURE); + } + + if (swift_backup_exists(&info, opt_swift_container, opt_name)) { + fprintf(stderr, "error: backup named '%s' " + "already exists!\n", + opt_name); + return(EXIT_FAILURE); + } + + if (swift_upload_parts(&info, opt_swift_container, + opt_name) != 0) { + fprintf(stderr, "error: upload failed\n"); + return(EXIT_FAILURE); + } + + } else if (opt_mode == MODE_GET) { + + if (swift_download(&info, opt_swift_container, opt_name) + != CURLE_OK) { + fprintf(stderr, "error: download failed\n"); + return(EXIT_FAILURE); + } + + } else if (opt_mode == MODE_DELETE) { + + if (swift_delete(&info, opt_swift_container, opt_name) + != CURLE_OK) { + fprintf(stderr, "error: delete failed\n"); + return(EXIT_FAILURE); + } + + } else { + fprintf(stderr, "Unknown command supplied.\n"); + exit(EXIT_FAILURE); + } + + curl_global_cleanup(); + + return(EXIT_SUCCESS); +} diff --git a/extra/mariabackup/xbcrypt.c b/extra/mariabackup/xbcrypt.c new file mode 100644 index 00000000000..255da875de4 --- /dev/null +++ b/extra/mariabackup/xbcrypt.c @@ -0,0 +1,694 @@ +/****************************************************** +Copyright (c) 2013 Percona LLC and/or its affiliates. + +The xbcrypt utility: decrypt files in the XBCRYPT format. + +This program is free software; you can redistribute it and/or modify +it under the terms of the GNU General Public License as published by +the Free Software Foundation; version 2 of the License. + +This program is distributed in the hope that it will be useful, +but WITHOUT ANY WARRANTY; without even the implied warranty of +MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the +GNU General Public License for more details. + +You should have received a copy of the GNU General Public License +along with this program; if not, write to the Free Software +Foundation, Inc., 51 Franklin Street, Fifth Floor, Boston, MA 02110-1301, USA + +*******************************************************/ + +#include +#include +#include "common.h" +#include "xbcrypt.h" +#include + +#if !defined(GCRYPT_VERSION_NUMBER) || (GCRYPT_VERSION_NUMBER < 0x010600) +GCRY_THREAD_OPTION_PTHREAD_IMPL; +#endif + +#define XBCRYPT_VERSION "1.1" + +typedef enum { + RUN_MODE_NONE, + RUN_MODE_ENCRYPT, + RUN_MODE_DECRYPT +} run_mode_t; + +const char *xbcrypt_encrypt_algo_names[] = +{ "NONE", "AES128", "AES192", "AES256", NullS}; +TYPELIB xbcrypt_encrypt_algo_typelib= +{array_elements(xbcrypt_encrypt_algo_names)-1,"", + xbcrypt_encrypt_algo_names, NULL}; + +static run_mode_t opt_run_mode = RUN_MODE_ENCRYPT; +static char *opt_input_file = NULL; +static char *opt_output_file = NULL; +static ulong opt_encrypt_algo; +static char *opt_encrypt_key_file = NULL; +static void *opt_encrypt_key = NULL; +static ulonglong opt_encrypt_chunk_size = 0; +static my_bool opt_verbose = FALSE; + +static uint encrypt_algos[] = { GCRY_CIPHER_NONE, + GCRY_CIPHER_AES128, + GCRY_CIPHER_AES192, + GCRY_CIPHER_AES256 }; +static int encrypt_algo = 0; +static int encrypt_mode = GCRY_CIPHER_MODE_CTR; +static uint encrypt_key_len = 0; +static size_t encrypt_iv_len = 0; + +static struct my_option my_long_options[] = +{ + {"help", '?', "Display this help and exit.", + 0, 0, 0, GET_NO_ARG, NO_ARG, 0, 0, 0, 0, 0, 0}, + + {"decrypt", 'd', "Decrypt data input to output.", + 0, 0, 0, + GET_NO_ARG, NO_ARG, 0, 0, 0, 0, 0, 0}, + + {"input", 'i', "Optional input file. If not specified, input" + " will be read from standard input.", + &opt_input_file, &opt_input_file, 0, + GET_STR_ALLOC, REQUIRED_ARG, 0, 0, 0, 0, 0, 0}, + + {"output", 'o', "Optional output file. If not specified, output" + " will be written to standard output.", + &opt_output_file, &opt_output_file, 0, + GET_STR_ALLOC, REQUIRED_ARG, 0, 0, 0, 0, 0, 0}, + + {"encrypt-algo", 'a', "Encryption algorithm.", + &opt_encrypt_algo, &opt_encrypt_algo, &xbcrypt_encrypt_algo_typelib, + GET_ENUM, REQUIRED_ARG, 0, 0, 0, 0, 0, 0}, + + {"encrypt-key", 'k', "Encryption key.", + &opt_encrypt_key, &opt_encrypt_key, 0, + GET_STR_ALLOC, REQUIRED_ARG, 0, 0, 0, 0, 0, 0}, + + {"encrypt-key-file", 'f', "File which contains encryption key.", + &opt_encrypt_key_file, &opt_encrypt_key_file, 0, + GET_STR_ALLOC, REQUIRED_ARG, 0, 0, 0, 0, 0, 0}, + + {"encrypt-chunk-size", 's', "Size of working buffer for encryption in" + " bytes. The default value is 64K.", + &opt_encrypt_chunk_size, &opt_encrypt_chunk_size, 0, + GET_ULL, REQUIRED_ARG, (1 << 16), 1024, ULONGLONG_MAX, 0, 0, 0}, + + {"verbose", 'v', "Display verbose status output.", + &opt_verbose, &opt_verbose, + 0, GET_BOOL, NO_ARG, 0, 0, 0, 0, 0, 0}, + {0, 0, 0, 0, 0, 0, GET_NO_ARG, NO_ARG, 0, 0, 0, 0, 0, 0} +}; + +static +int +get_options(int *argc, char ***argv); + +static +my_bool +get_one_option(int optid, const struct my_option *opt __attribute__((unused)), + char *argument __attribute__((unused))); + +static +void +print_version(void); + +static +void +usage(void); + +static +int +mode_decrypt(File filein, File fileout); + +static +int +mode_encrypt(File filein, File fileout); + +int +main(int argc, char **argv) +{ +#if !defined(GCRYPT_VERSION_NUMBER) || (GCRYPT_VERSION_NUMBER < 0x010600) + gcry_error_t gcry_error; +#endif + File filein = 0; + File fileout = 0; + + MY_INIT(argv[0]); + + if (get_options(&argc, &argv)) { + goto err; + } + + /* Acording to gcrypt docs (and my testing), setting up the threading + callbacks must be done first, so, lets give it a shot */ +#if !defined(GCRYPT_VERSION_NUMBER) || (GCRYPT_VERSION_NUMBER < 0x010600) + gcry_error = gcry_control(GCRYCTL_SET_THREAD_CBS, &gcry_threads_pthread); + if (gcry_error) { + msg("%s: unable to set libgcrypt thread cbs - " + "%s : %s\n", my_progname, + gcry_strsource(gcry_error), + gcry_strerror(gcry_error)); + return 1; + } +#endif + + /* Version check should be the very first call because it + makes sure that important subsystems are intialized. */ + if (!gcry_control(GCRYCTL_ANY_INITIALIZATION_P)) { + const char *gcrypt_version; + gcrypt_version = gcry_check_version(NULL); + /* No other library has already initialized libgcrypt. */ + if (!gcrypt_version) { + msg("%s: failed to initialize libgcrypt\n", + my_progname); + return 1; + } else if (opt_verbose) { + msg("%s: using gcrypt %s\n", my_progname, + gcrypt_version); + } + } + gcry_control(GCRYCTL_DISABLE_SECMEM, 0); + gcry_control(GCRYCTL_INITIALIZATION_FINISHED, 0); + + /* Determine the algorithm */ + encrypt_algo = encrypt_algos[opt_encrypt_algo]; + + /* Set up the iv length */ + encrypt_iv_len = gcry_cipher_get_algo_blklen(encrypt_algo); + + /* Now set up the key */ + if (opt_encrypt_key == NULL && opt_encrypt_key_file == NULL) { + msg("%s: no encryption key or key file specified.\n", + my_progname); + return 1; + } else if (opt_encrypt_key && opt_encrypt_key_file) { + msg("%s: both encryption key and key file specified.\n", + my_progname); + return 1; + } else if (opt_encrypt_key_file) { + if (!xb_crypt_read_key_file(opt_encrypt_key_file, + &opt_encrypt_key, + &encrypt_key_len)) { + msg("%s: unable to read encryption key file \"%s\".\n", + opt_encrypt_key_file, my_progname); + return 1; + } + } else { + encrypt_key_len = strlen(opt_encrypt_key); + } + + if (opt_input_file) { + MY_STAT mystat; + + if (opt_verbose) + msg("%s: input file \"%s\".\n", my_progname, + opt_input_file); + + if (my_stat(opt_input_file, &mystat, MYF(MY_WME)) == NULL) { + goto err; + } + if (!MY_S_ISREG(mystat.st_mode)) { + msg("%s: \"%s\" is not a regular file, exiting.\n", + my_progname, opt_input_file); + goto err; + } + if ((filein = my_open(opt_input_file, O_RDONLY, MYF(MY_WME))) + < 0) { + msg("%s: failed to open \"%s\".\n", my_progname, + opt_input_file); + goto err; + } + } else { + if (opt_verbose) + msg("%s: input from standard input.\n", my_progname); + filein = fileno(stdin); + } + + if (opt_output_file) { + if (opt_verbose) + msg("%s: output file \"%s\".\n", my_progname, + opt_output_file); + + if ((fileout = my_create(opt_output_file, 0, + O_WRONLY|O_BINARY|O_EXCL|O_NOFOLLOW, + MYF(MY_WME))) < 0) { + msg("%s: failed to create output file \"%s\".\n", + my_progname, opt_output_file); + goto err; + } + } else { + if (opt_verbose) + msg("%s: output to standard output.\n", my_progname); + fileout = fileno(stdout); + } + + if (opt_run_mode == RUN_MODE_DECRYPT + && mode_decrypt(filein, fileout)) { + goto err; + } else if (opt_run_mode == RUN_MODE_ENCRYPT + && mode_encrypt(filein, fileout)) { + goto err; + } + + if (opt_input_file && filein) { + my_close(filein, MYF(MY_WME)); + } + if (opt_output_file && fileout) { + my_close(fileout, MYF(MY_WME)); + } + + my_cleanup_options(my_long_options); + + my_end(0); + + return EXIT_SUCCESS; +err: + if (opt_input_file && filein) { + my_close(filein, MYF(MY_WME)); + } + if (opt_output_file && fileout) { + my_close(fileout, MYF(MY_WME)); + } + + my_cleanup_options(my_long_options); + + my_end(0); + + exit(EXIT_FAILURE); + +} + + +static +size_t +my_xb_crypt_read_callback(void *userdata, void *buf, size_t len) +{ + File* file = (File *) userdata; + return xb_read_full(*file, buf, len); +} + +static +int +mode_decrypt(File filein, File fileout) +{ + xb_rcrypt_t *xbcrypt_file = NULL; + void *chunkbuf = NULL; + size_t chunksize; + size_t originalsize; + void *ivbuf = NULL; + size_t ivsize; + void *decryptbuf = NULL; + size_t decryptbufsize = 0; + ulonglong ttlchunksread = 0; + ulonglong ttlbytesread = 0; + xb_rcrypt_result_t result; + gcry_cipher_hd_t cipher_handle; + gcry_error_t gcry_error; + my_bool hash_appended; + + if (encrypt_algo != GCRY_CIPHER_NONE) { + gcry_error = gcry_cipher_open(&cipher_handle, + encrypt_algo, + encrypt_mode, 0); + if (gcry_error) { + msg("%s:decrypt: unable to open libgcrypt" + " cipher - %s : %s\n", my_progname, + gcry_strsource(gcry_error), + gcry_strerror(gcry_error)); + return 1; + } + + gcry_error = gcry_cipher_setkey(cipher_handle, + opt_encrypt_key, + encrypt_key_len); + if (gcry_error) { + msg("%s:decrypt: unable to set libgcrypt cipher" + "key - %s : %s\n", my_progname, + gcry_strsource(gcry_error), + gcry_strerror(gcry_error)); + goto err; + } + } + + /* Initialize the xb_crypt format reader */ + xbcrypt_file = xb_crypt_read_open(&filein, my_xb_crypt_read_callback); + if (xbcrypt_file == NULL) { + msg("%s:decrypt: xb_crypt_read_open() failed.\n", my_progname); + goto err; + } + + /* Walk the encrypted chunks, decrypting them and writing out */ + while ((result = xb_crypt_read_chunk(xbcrypt_file, &chunkbuf, + &originalsize, &chunksize, + &ivbuf, &ivsize, &hash_appended)) + == XB_CRYPT_READ_CHUNK) { + + if (encrypt_algo != GCRY_CIPHER_NONE) { + gcry_error = gcry_cipher_reset(cipher_handle); + if (gcry_error) { + msg("%s:decrypt: unable to reset libgcrypt" + " cipher - %s : %s\n", my_progname, + gcry_strsource(gcry_error), + gcry_strerror(gcry_error)); + goto err; + } + + if (ivsize) { + gcry_error = gcry_cipher_setctr(cipher_handle, + ivbuf, + ivsize); + } + if (gcry_error) { + msg("%s:decrypt: unable to set cipher iv - " + "%s : %s\n", my_progname, + gcry_strsource(gcry_error), + gcry_strerror(gcry_error)); + continue; + } + + if (decryptbufsize < originalsize) { + decryptbuf = my_realloc(decryptbuf, + originalsize, + MYF(MY_WME | MY_ALLOW_ZERO_PTR)); + decryptbufsize = originalsize; + } + + /* Try to decrypt it */ + gcry_error = gcry_cipher_decrypt(cipher_handle, + decryptbuf, + originalsize, + chunkbuf, + chunksize); + if (gcry_error) { + msg("%s:decrypt: unable to decrypt chunk - " + "%s : %s\n", my_progname, + gcry_strsource(gcry_error), + gcry_strerror(gcry_error)); + gcry_cipher_close(cipher_handle); + goto err; + } + + } else { + decryptbuf = chunkbuf; + } + + if (hash_appended) { + uchar hash[XB_CRYPT_HASH_LEN]; + + originalsize -= XB_CRYPT_HASH_LEN; + + /* ensure that XB_CRYPT_HASH_LEN is the correct length + of XB_CRYPT_HASH hashing algorithm output */ + assert(gcry_md_get_algo_dlen(XB_CRYPT_HASH) == + XB_CRYPT_HASH_LEN); + gcry_md_hash_buffer(XB_CRYPT_HASH, hash, decryptbuf, + originalsize); + if (memcmp(hash, (char *) decryptbuf + originalsize, + XB_CRYPT_HASH_LEN) != 0) { + msg("%s:%s invalid plaintext hash. " + "Wrong encrytion key specified?\n", + my_progname, __FUNCTION__); + result = XB_CRYPT_READ_ERROR; + goto err; + } + } + + /* Write it out */ + if (my_write(fileout, (const uchar *) decryptbuf, originalsize, + MYF(MY_WME | MY_NABP))) { + msg("%s:decrypt: unable to write output chunk.\n", + my_progname); + goto err; + } + ttlchunksread++; + ttlbytesread += chunksize; + if (opt_verbose) + msg("%s:decrypt: %llu chunks read, %llu bytes read\n.", + my_progname, ttlchunksread, ttlbytesread); + } + + xb_crypt_read_close(xbcrypt_file); + + if (encrypt_algo != GCRY_CIPHER_NONE) + gcry_cipher_close(cipher_handle); + + if (decryptbuf && decryptbufsize) + my_free(decryptbuf); + + if (opt_verbose) + msg("\n%s:decrypt: done\n", my_progname); + + return 0; +err: + if (xbcrypt_file) + xb_crypt_read_close(xbcrypt_file); + + if (encrypt_algo != GCRY_CIPHER_NONE) + gcry_cipher_close(cipher_handle); + + if (decryptbuf && decryptbufsize) + my_free(decryptbuf); + + return 1; +} + +static +ssize_t +my_xb_crypt_write_callback(void *userdata, const void *buf, size_t len) +{ + File* file = (File *) userdata; + + ssize_t ret = my_write(*file, buf, len, MYF(MY_WME)); + posix_fadvise(*file, 0, 0, POSIX_FADV_DONTNEED); + return ret; +} + +static +int +mode_encrypt(File filein, File fileout) +{ + size_t bytesread; + size_t chunkbuflen; + uchar *chunkbuf = NULL; + void *ivbuf = NULL; + size_t encryptbuflen = 0; + size_t encryptedlen = 0; + void *encryptbuf = NULL; + ulonglong ttlchunkswritten = 0; + ulonglong ttlbyteswritten = 0; + xb_wcrypt_t *xbcrypt_file = NULL; + gcry_cipher_hd_t cipher_handle; + gcry_error_t gcry_error; + + if (encrypt_algo != GCRY_CIPHER_NONE) { + gcry_error = gcry_cipher_open(&cipher_handle, + encrypt_algo, + encrypt_mode, 0); + if (gcry_error) { + msg("%s:encrypt: unable to open libgcrypt cipher - " + "%s : %s\n", my_progname, + gcry_strsource(gcry_error), + gcry_strerror(gcry_error)); + return 1; + } + + gcry_error = gcry_cipher_setkey(cipher_handle, + opt_encrypt_key, + encrypt_key_len); + if (gcry_error) { + msg("%s:encrypt: unable to set libgcrypt cipher key - " + "%s : %s\n", my_progname, + gcry_strsource(gcry_error), + gcry_strerror(gcry_error)); + goto err; + } + } + + posix_fadvise(filein, 0, 0, POSIX_FADV_SEQUENTIAL); + + xbcrypt_file = xb_crypt_write_open(&fileout, + my_xb_crypt_write_callback); + if (xbcrypt_file == NULL) { + msg("%s:encrypt: xb_crypt_write_open() failed.\n", + my_progname); + goto err; + } + + ivbuf = my_malloc(encrypt_iv_len, MYF(MY_FAE)); + + /* now read in data in chunk size, encrypt and write out */ + chunkbuflen = opt_encrypt_chunk_size + XB_CRYPT_HASH_LEN; + chunkbuf = (uchar *) my_malloc(chunkbuflen, MYF(MY_FAE)); + while ((bytesread = my_read(filein, chunkbuf, opt_encrypt_chunk_size, + MYF(MY_WME))) > 0) { + + size_t origbuflen = bytesread + XB_CRYPT_HASH_LEN; + + /* ensure that XB_CRYPT_HASH_LEN is the correct length + of XB_CRYPT_HASH hashing algorithm output */ + assert(XB_CRYPT_HASH_LEN == + gcry_md_get_algo_dlen(XB_CRYPT_HASH)); + gcry_md_hash_buffer(XB_CRYPT_HASH, chunkbuf + bytesread, + chunkbuf, bytesread); + + if (encrypt_algo != GCRY_CIPHER_NONE) { + gcry_error = gcry_cipher_reset(cipher_handle); + + if (gcry_error) { + msg("%s:encrypt: unable to reset cipher - " + "%s : %s\n", my_progname, + gcry_strsource(gcry_error), + gcry_strerror(gcry_error)); + goto err; + } + + xb_crypt_create_iv(ivbuf, encrypt_iv_len); + gcry_error = gcry_cipher_setctr(cipher_handle, + ivbuf, + encrypt_iv_len); + + if (gcry_error) { + msg("%s:encrypt: unable to set cipher iv - " + "%s : %s\n", my_progname, + gcry_strsource(gcry_error), + gcry_strerror(gcry_error)); + continue; + } + + if (encryptbuflen < origbuflen) { + encryptbuf = my_realloc(encryptbuf, origbuflen, + MYF(MY_WME | MY_ALLOW_ZERO_PTR)); + encryptbuflen = origbuflen; + } + + gcry_error = gcry_cipher_encrypt(cipher_handle, + encryptbuf, + encryptbuflen, + chunkbuf, + origbuflen); + + encryptedlen = origbuflen; + + if (gcry_error) { + msg("%s:encrypt: unable to encrypt chunk - " + "%s : %s\n", my_progname, + gcry_strsource(gcry_error), + gcry_strerror(gcry_error)); + gcry_cipher_close(cipher_handle); + goto err; + } + } else { + encryptedlen = origbuflen; + encryptbuf = chunkbuf; + } + + if (xb_crypt_write_chunk(xbcrypt_file, encryptbuf, + bytesread + XB_CRYPT_HASH_LEN, + encryptedlen, ivbuf, encrypt_iv_len)) { + msg("%s:encrypt: abcrypt_write_chunk() failed.\n", + my_progname); + goto err; + } + + ttlchunkswritten++; + ttlbyteswritten += encryptedlen; + + if (opt_verbose) + msg("%s:encrypt: %llu chunks written, %llu bytes " + "written\n.", my_progname, ttlchunkswritten, + ttlbyteswritten); + } + + my_free(ivbuf); + my_free(chunkbuf); + + if (encryptbuf && encryptbuflen) + my_free(encryptbuf); + + xb_crypt_write_close(xbcrypt_file); + + if (encrypt_algo != GCRY_CIPHER_NONE) + gcry_cipher_close(cipher_handle); + + if (opt_verbose) + msg("\n%s:encrypt: done\n", my_progname); + + return 0; +err: + if (chunkbuf) + my_free(chunkbuf); + + if (encryptbuf && encryptbuflen) + my_free(encryptbuf); + + if (xbcrypt_file) + xb_crypt_write_close(xbcrypt_file); + + if (encrypt_algo != GCRY_CIPHER_NONE) + gcry_cipher_close(cipher_handle); + + return 1; +} + +static +int +get_options(int *argc, char ***argv) +{ + int ho_error; + + if ((ho_error= handle_options(argc, argv, my_long_options, + get_one_option))) { + exit(EXIT_FAILURE); + } + + return 0; +} + +static +my_bool +get_one_option(int optid, const struct my_option *opt __attribute__((unused)), + char *argument __attribute__((unused))) +{ + switch (optid) { + case 'd': + opt_run_mode = RUN_MODE_DECRYPT; + break; + case '?': + usage(); + exit(0); + } + + return FALSE; +} + +static +void +print_version(void) +{ + printf("%s Ver %s for %s (%s)\n", my_progname, XBCRYPT_VERSION, + SYSTEM_TYPE, MACHINE_TYPE); +} + +static +void +usage(void) +{ + print_version(); + puts("Copyright (C) 2011 Percona Inc."); + puts("This software comes with ABSOLUTELY NO WARRANTY. " + "This is free software,\nand you are welcome to modify and " + "redistribute it under the GPL license.\n"); + + puts("Encrypt or decrypt files in the XBCRYPT format.\n"); + + puts("Usage: "); + printf(" %s [OPTIONS...]" + " # read data from specified input, encrypting or decrypting " + " and writing the result to the specified output.\n", + my_progname); + puts("\nOptions:"); + my_print_help(my_long_options); +} diff --git a/extra/mariabackup/xbcrypt.h b/extra/mariabackup/xbcrypt.h new file mode 100644 index 00000000000..cdabf56a21a --- /dev/null +++ b/extra/mariabackup/xbcrypt.h @@ -0,0 +1,84 @@ +/****************************************************** +Copyright (c) 2011 Percona LLC and/or its affiliates. + +Encryption interface for XtraBackup. + +This program is free software; you can redistribute it and/or modify +it under the terms of the GNU General Public License as published by +the Free Software Foundation; version 2 of the License. + +This program is distributed in the hope that it will be useful, +but WITHOUT ANY WARRANTY; without even the implied warranty of +MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the +GNU General Public License for more details. + +You should have received a copy of the GNU General Public License +along with this program; if not, write to the Free Software +Foundation, Inc., 51 Franklin Street, Fifth Floor, Boston, MA 02110-1301, USA + +*******************************************************/ + +#ifndef XBCRYPT_H +#define XBCRYPT_H + +#include +#include "common.h" + +#define XB_CRYPT_CHUNK_MAGIC1 "XBCRYP01" +#define XB_CRYPT_CHUNK_MAGIC2 "XBCRYP02" +#define XB_CRYPT_CHUNK_MAGIC3 "XBCRYP03" /* must be same size as ^^ */ +#define XB_CRYPT_CHUNK_MAGIC_CURRENT XB_CRYPT_CHUNK_MAGIC3 +#define XB_CRYPT_CHUNK_MAGIC_SIZE (sizeof(XB_CRYPT_CHUNK_MAGIC1)-1) + +#define XB_CRYPT_HASH GCRY_MD_SHA256 +#define XB_CRYPT_HASH_LEN 32 + +/****************************************************************************** +Write interface */ +typedef struct xb_wcrypt_struct xb_wcrypt_t; + +/* Callback on write for i/o, must return # of bytes written or -1 on error */ +typedef ssize_t xb_crypt_write_callback(void *userdata, + const void *buf, size_t len); + +xb_wcrypt_t *xb_crypt_write_open(void *userdata, + xb_crypt_write_callback *onwrite); + +/* Takes buffer, original length, encrypted length iv and iv length, formats + output buffer and calls write callback. + Returns 0 on success, 1 on error */ +int xb_crypt_write_chunk(xb_wcrypt_t *crypt, const void *buf, size_t olen, + size_t elen, const void *iv, size_t ivlen); + +/* Returns 0 on success, 1 on error */ +int xb_crypt_write_close(xb_wcrypt_t *crypt); + +/****************************************************************************** +Read interface */ +typedef struct xb_rcrypt_struct xb_rcrypt_t; + +/* Callback on read for i/o, must return # of bytes read or -1 on error */ +typedef size_t xb_crypt_read_callback(void *userdata, void *buf, size_t len); + +xb_rcrypt_t *xb_crypt_read_open(void *userdata, + xb_crypt_read_callback *onread); + +typedef enum { + XB_CRYPT_READ_CHUNK, + XB_CRYPT_READ_EOF, + XB_CRYPT_READ_ERROR +} xb_rcrypt_result_t; + +xb_rcrypt_result_t xb_crypt_read_chunk(xb_rcrypt_t *crypt, void **buf, + size_t *olen, size_t *elen, void **iv, + size_t *ivlen, my_bool *hash_appended); + +int xb_crypt_read_close(xb_rcrypt_t *crypt); + +/****************************************************************************** +Utility interface */ +my_bool xb_crypt_read_key_file(const char *filename, + void** key, uint *keylength); + +void xb_crypt_create_iv(void* ivbuf, size_t ivlen); +#endif diff --git a/extra/mariabackup/xbcrypt_common.c b/extra/mariabackup/xbcrypt_common.c new file mode 100644 index 00000000000..fe34fcb3bb0 --- /dev/null +++ b/extra/mariabackup/xbcrypt_common.c @@ -0,0 +1,60 @@ +/****************************************************** +Copyright (c) 2013 Percona LLC and/or its affiliates. + +Encryption configuration file interface for XtraBackup. + +This program is free software; you can redistribute it and/or modify +it under the terms of the GNU General Public License as published by +the Free Software Foundation; version 2 of the License. + +This program is distributed in the hope that it will be useful, +but WITHOUT ANY WARRANTY; without even the implied warranty of +MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the +GNU General Public License for more details. + +You should have received a copy of the GNU General Public License +along with this program; if not, write to the Free Software +Foundation, Inc., 51 Franklin Street, Fifth Floor, Boston, MA 02110-1301, USA + +*******************************************************/ + +#include +#include "common.h" +#include "xbcrypt.h" + +#if GCC_VERSION >= 4002 +/* Workaround to avoid "gcry_ac_* is deprecated" warnings in gcrypt.h */ +# pragma GCC diagnostic ignored "-Wdeprecated-declarations" +#endif + +#include + +#if GCC_VERSION >= 4002 +# pragma GCC diagnostic warning "-Wdeprecated-declarations" +#endif + +my_bool +xb_crypt_read_key_file(const char *filename, void** key, uint *keylength) +{ + FILE *fp; + + if (!(fp = my_fopen(filename, O_RDONLY, MYF(0)))) { + msg("%s:%s: unable to open config file \"%s\", errno(%d)\n", + my_progname, __FUNCTION__, filename, my_errno); + return FALSE; + } + + fseek(fp, 0 , SEEK_END); + *keylength = ftell(fp); + rewind(fp); + *key = my_malloc(*keylength, MYF(MY_FAE)); + *keylength = fread(*key, 1, *keylength, fp); + my_fclose(fp, MYF(0)); + return TRUE; +} + +void +xb_crypt_create_iv(void* ivbuf, size_t ivlen) +{ + gcry_create_nonce(ivbuf, ivlen); +} diff --git a/extra/mariabackup/xbcrypt_read.c b/extra/mariabackup/xbcrypt_read.c new file mode 100644 index 00000000000..6522333f023 --- /dev/null +++ b/extra/mariabackup/xbcrypt_read.c @@ -0,0 +1,251 @@ +/****************************************************** +Copyright (c) 2013 Percona LLC and/or its affiliates. + +The xbcrypt format reader implementation. + +This program is free software; you can redistribute it and/or modify +it under the terms of the GNU General Public License as published by +the Free Software Foundation; version 2 of the License. + +This program is distributed in the hope that it will be useful, +but WITHOUT ANY WARRANTY; without even the implied warranty of +MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the +GNU General Public License for more details. + +You should have received a copy of the GNU General Public License +along with this program; if not, write to the Free Software +Foundation, Inc., 51 Franklin Street, Fifth Floor, Boston, MA 02110-1301, USA + +*******************************************************/ + +#include "xbcrypt.h" + +struct xb_rcrypt_struct { + void *userdata; + xb_crypt_read_callback *read; + void *buffer; + size_t bufsize; + void *ivbuffer; + size_t ivbufsize; + ulonglong offset; +}; + +xb_rcrypt_t * +xb_crypt_read_open(void *userdata, xb_crypt_read_callback *onread) +{ + xb_rcrypt_t *crypt; + + xb_ad(onread); + + crypt = (xb_rcrypt_t *) my_malloc(sizeof(xb_rcrypt_t), MYF(MY_FAE)); + + crypt->userdata = userdata; + crypt->read = onread; + crypt->buffer = NULL; + crypt->bufsize = 0; + crypt->offset = 0; + crypt->ivbuffer = NULL; + crypt->ivbufsize = 0; + return crypt; +} + +xb_rcrypt_result_t +xb_crypt_read_chunk(xb_rcrypt_t *crypt, void **buf, size_t *olen, size_t *elen, + void **iv, size_t *ivlen, my_bool *hash_appended) + +{ + uchar tmpbuf[XB_CRYPT_CHUNK_MAGIC_SIZE + 8 + 8 + 8 + 4]; + uchar *ptr; + ulonglong tmp; + ulong checksum, checksum_exp, version; + size_t bytesread; + xb_rcrypt_result_t result = XB_CRYPT_READ_CHUNK; + + if ((bytesread = crypt->read(crypt->userdata, tmpbuf, sizeof(tmpbuf))) + != sizeof(tmpbuf)) { + if (bytesread == 0) { + result = XB_CRYPT_READ_EOF; + goto err; + } else { + msg("%s:%s: unable to read chunk header data at " + "offset 0x%llx.\n", + my_progname, __FUNCTION__, crypt->offset); + result = XB_CRYPT_READ_ERROR; + goto err; + } + } + + ptr = tmpbuf; + + if (memcmp(ptr, XB_CRYPT_CHUNK_MAGIC3, + XB_CRYPT_CHUNK_MAGIC_SIZE) == 0) { + version = 3; + } else if (memcmp(ptr, XB_CRYPT_CHUNK_MAGIC2, + XB_CRYPT_CHUNK_MAGIC_SIZE) == 0) { + version = 2; + } else if (memcmp(ptr, XB_CRYPT_CHUNK_MAGIC1, + XB_CRYPT_CHUNK_MAGIC_SIZE) == 0) { + version = 1; + } else { + msg("%s:%s: wrong chunk magic at offset 0x%llx.\n", + my_progname, __FUNCTION__, crypt->offset); + result = XB_CRYPT_READ_ERROR; + goto err; + } + + ptr += XB_CRYPT_CHUNK_MAGIC_SIZE; + crypt->offset += XB_CRYPT_CHUNK_MAGIC_SIZE; + + tmp = uint8korr(ptr); /* reserved */ + ptr += 8; + crypt->offset += 8; + + tmp = uint8korr(ptr); /* original size */ + ptr += 8; + if (tmp > INT_MAX) { + msg("%s:%s: invalid original size at offset 0x%llx.\n", + my_progname, __FUNCTION__, crypt->offset); + result = XB_CRYPT_READ_ERROR; + goto err; + } + crypt->offset += 8; + *olen = (size_t)tmp; + + tmp = uint8korr(ptr); /* encrypted size */ + ptr += 8; + if (tmp > INT_MAX) { + msg("%s:%s: invalid encrypted size at offset 0x%llx.\n", + my_progname, __FUNCTION__, crypt->offset); + result = XB_CRYPT_READ_ERROR; + goto err; + } + crypt->offset += 8; + *elen = (size_t)tmp; + + checksum_exp = uint4korr(ptr); /* checksum */ + ptr += 4; + crypt->offset += 4; + + /* iv size */ + if (version == 1) { + *ivlen = 0; + *iv = 0; + } else { + if ((bytesread = crypt->read(crypt->userdata, tmpbuf, 8)) + != 8) { + if (bytesread == 0) { + result = XB_CRYPT_READ_EOF; + goto err; + } else { + msg("%s:%s: unable to read chunk iv size at " + "offset 0x%llx.\n", + my_progname, __FUNCTION__, crypt->offset); + result = XB_CRYPT_READ_ERROR; + goto err; + } + } + + tmp = uint8korr(tmpbuf); + if (tmp > INT_MAX) { + msg("%s:%s: invalid iv size at offset 0x%llx.\n", + my_progname, __FUNCTION__, crypt->offset); + result = XB_CRYPT_READ_ERROR; + goto err; + } + crypt->offset += 8; + *ivlen = (size_t)tmp; + } + + if (*ivlen > crypt->ivbufsize) { + crypt->ivbuffer = my_realloc(crypt->ivbuffer, *ivlen, + MYF(MY_WME | MY_ALLOW_ZERO_PTR)); + if (crypt->ivbuffer == NULL) { + msg("%s:%s: failed to increase iv buffer to " + "%llu bytes.\n", my_progname, __FUNCTION__, + (ulonglong)*ivlen); + result = XB_CRYPT_READ_ERROR; + goto err; + } + crypt->ivbufsize = *ivlen; + } + + if (*ivlen > 0) { + if (crypt->read(crypt->userdata, crypt->ivbuffer, *ivlen) + != *ivlen) { + msg("%s:%s: failed to read %lld bytes for chunk iv " + "at offset 0x%llx.\n", my_progname, __FUNCTION__, + (ulonglong)*ivlen, crypt->offset); + result = XB_CRYPT_READ_ERROR; + goto err; + } + *iv = crypt->ivbuffer; + } + + /* for version euqals 2 we need to read in the iv data but do not init + CTR with it */ + if (version == 2) { + *ivlen = 0; + *iv = 0; + } + + if (*olen > crypt->bufsize) { + crypt->buffer = my_realloc(crypt->buffer, *olen, + MYF(MY_WME | MY_ALLOW_ZERO_PTR)); + if (crypt->buffer == NULL) { + msg("%s:%s: failed to increase buffer to " + "%llu bytes.\n", my_progname, __FUNCTION__, + (ulonglong)*olen); + result = XB_CRYPT_READ_ERROR; + goto err; + } + crypt->bufsize = *olen; + } + + if (*elen > 0) { + if (crypt->read(crypt->userdata, crypt->buffer, *elen) + != *elen) { + msg("%s:%s: failed to read %lld bytes for chunk payload " + "at offset 0x%llx.\n", my_progname, __FUNCTION__, + (ulonglong)*elen, crypt->offset); + result = XB_CRYPT_READ_ERROR; + goto err; + } + } + + checksum = crc32(0, crypt->buffer, *elen); + if (checksum != checksum_exp) { + msg("%s:%s invalid checksum at offset 0x%llx, " + "expected 0x%lx, actual 0x%lx.\n", my_progname, __FUNCTION__, + crypt->offset, checksum_exp, checksum); + result = XB_CRYPT_READ_ERROR; + goto err; + } + + crypt->offset += *elen; + *buf = crypt->buffer; + + *hash_appended = version > 2; + + goto exit; + +err: + *buf = NULL; + *olen = 0; + *elen = 0; + *ivlen = 0; + *iv = 0; +exit: + return result; +} + +int xb_crypt_read_close(xb_rcrypt_t *crypt) +{ + if (crypt->buffer) + my_free(crypt->buffer); + if (crypt->ivbuffer) + my_free(crypt->ivbuffer); + my_free(crypt); + + return 0; +} + diff --git a/extra/mariabackup/xbcrypt_write.c b/extra/mariabackup/xbcrypt_write.c new file mode 100644 index 00000000000..5cbeb67f227 --- /dev/null +++ b/extra/mariabackup/xbcrypt_write.c @@ -0,0 +1,104 @@ +/****************************************************** +Copyright (c) 2013 Percona LLC and/or its affiliates. + +The xbcrypt format writer implementation. + +This program is free software; you can redistribute it and/or modify +it under the terms of the GNU General Public License as published by +the Free Software Foundation; version 2 of the License. + +This program is distributed in the hope that it will be useful, +but WITHOUT ANY WARRANTY; without even the implied warranty of +MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the +GNU General Public License for more details. + +You should have received a copy of the GNU General Public License +along with this program; if not, write to the Free Software +Foundation, Inc., 51 Franklin Street, Fifth Floor, Boston, MA 02110-1301, USA + +*******************************************************/ + +#include "xbcrypt.h" + +struct xb_wcrypt_struct { + void *userdata; + xb_crypt_write_callback *write; +}; + +xb_wcrypt_t * +xb_crypt_write_open(void *userdata, xb_crypt_write_callback *onwrite) +{ + xb_wcrypt_t *crypt; + + xb_ad(onwrite); + + crypt = (xb_wcrypt_t *) my_malloc(sizeof(xb_wcrypt_t), MYF(MY_FAE)); + + crypt->userdata = userdata; + crypt->write = onwrite; + + return crypt; +} + +int xb_crypt_write_chunk(xb_wcrypt_t *crypt, const void *buf, size_t olen, + size_t elen, const void *iv, size_t ivlen) +{ + uchar tmpbuf[XB_CRYPT_CHUNK_MAGIC_SIZE + 8 + 8 + 8 + 4 + 8]; + uchar *ptr; + ulong checksum; + + xb_ad(olen <= INT_MAX); + if (olen > INT_MAX) + return 0; + + xb_ad(elen <= INT_MAX); + if (elen > INT_MAX) + return 0; + + xb_ad(ivlen <= INT_MAX); + if (ivlen > INT_MAX) + return 0; + + ptr = tmpbuf; + + memcpy(ptr, XB_CRYPT_CHUNK_MAGIC_CURRENT, XB_CRYPT_CHUNK_MAGIC_SIZE); + ptr += XB_CRYPT_CHUNK_MAGIC_SIZE; + + int8store(ptr, (ulonglong)0); /* reserved */ + ptr += 8; + + int8store(ptr, (ulonglong)olen); /* original size */ + ptr += 8; + + int8store(ptr, (ulonglong)elen); /* encrypted (actual) size */ + ptr += 8; + + checksum = crc32(0, buf, elen); + int4store(ptr, checksum); /* checksum */ + ptr += 4; + + int8store(ptr, (ulonglong)ivlen); /* iv size */ + ptr += 8; + + xb_ad(ptr <= tmpbuf + sizeof(tmpbuf)); + + if (crypt->write(crypt->userdata, tmpbuf, ptr-tmpbuf) == -1) + return 1; + + if (crypt->write(crypt->userdata, iv, ivlen) == -1) + return 1; + + if (crypt->write(crypt->userdata, buf, elen) == -1) + return 1; + + return 0; +} + +int xb_crypt_write_close(xb_wcrypt_t *crypt) +{ + my_free(crypt); + + return 0; +} + + diff --git a/extra/mariabackup/xbstream.c b/extra/mariabackup/xbstream.c new file mode 100644 index 00000000000..ba3412a359b --- /dev/null +++ b/extra/mariabackup/xbstream.c @@ -0,0 +1,456 @@ +/****************************************************** +Copyright (c) 2011-2013 Percona LLC and/or its affiliates. + +The xbstream utility: serialize/deserialize files in the XBSTREAM format. + +This program is free software; you can redistribute it and/or modify +it under the terms of the GNU General Public License as published by +the Free Software Foundation; version 2 of the License. + +This program is distributed in the hope that it will be useful, +but WITHOUT ANY WARRANTY; without even the implied warranty of +MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the +GNU General Public License for more details. + +You should have received a copy of the GNU General Public License +along with this program; if not, write to the Free Software +Foundation, Inc., 51 Franklin Street, Fifth Floor, Boston, MA 02110-1301, USA + +*******************************************************/ + +#include +#include +#include +#include +#include "common.h" +#include "xbstream.h" +#include "ds_local.h" +#include "ds_stdout.h" + +#define XBSTREAM_VERSION "1.0" +#define XBSTREAM_BUFFER_SIZE (10 * 1024 * 1024UL) + +#define START_FILE_HASH_SIZE 16 + +typedef enum { + RUN_MODE_NONE, + RUN_MODE_CREATE, + RUN_MODE_EXTRACT +} run_mode_t; + +/* Need the following definitions to avoid linking with ds_*.o and their link +dependencies */ +datasink_t datasink_archive; +datasink_t datasink_xbstream; +datasink_t datasink_compress; +datasink_t datasink_tmpfile; +datasink_t datasink_encrypt; +datasink_t datasink_buffer; + +static run_mode_t opt_mode; +static char * opt_directory = NULL; +static my_bool opt_verbose = 0; + +static struct my_option my_long_options[] = +{ + {"help", '?', "Display this help and exit.", + 0, 0, 0, GET_NO_ARG, NO_ARG, 0, 0, 0, 0, 0, 0}, + {"create", 'c', "Stream the specified files to the standard output.", + 0, 0, 0, GET_NO_ARG, NO_ARG, 0, 0, 0, 0, 0, 0}, + {"extract", 'x', "Extract to disk files from the stream on the " + "standard input.", + 0, 0, 0, GET_NO_ARG, NO_ARG, 0, 0, 0, 0, 0, 0}, + {"directory", 'C', "Change the current directory to the specified one " + "before streaming or extracting.", &opt_directory, &opt_directory, 0, + GET_STR_ALLOC, REQUIRED_ARG, 0, 0, 0, 0, 0, 0}, + {"verbose", 'v', "Print verbose output.", &opt_verbose, &opt_verbose, + 0, GET_BOOL, NO_ARG, 0, 0, 0, 0, 0, 0}, + + {0, 0, 0, 0, 0, 0, GET_NO_ARG, NO_ARG, 0, 0, 0, 0, 0, 0} +}; + +typedef struct { + char *path; + uint pathlen; + my_off_t offset; + ds_ctxt_t *ds_ctxt; + ds_file_t *file; +} file_entry_t; + +static int get_options(int *argc, char ***argv); +static int mode_create(int argc, char **argv); +static int mode_extract(int argc, char **argv); +static my_bool get_one_option(int optid, const struct my_option *opt, + char *argument); + +int +main(int argc, char **argv) +{ + MY_INIT(argv[0]); + + if (get_options(&argc, &argv)) { + goto err; + } + + if (opt_mode == RUN_MODE_NONE) { + msg("%s: either -c or -x must be specified.\n", my_progname); + goto err; + } + + /* Change the current directory if -C is specified */ + if (opt_directory && my_setwd(opt_directory, MYF(MY_WME))) { + goto err; + } + + if (opt_mode == RUN_MODE_CREATE && mode_create(argc, argv)) { + goto err; + } else if (opt_mode == RUN_MODE_EXTRACT && mode_extract(argc, argv)) { + goto err; + } + + my_cleanup_options(my_long_options); + + my_end(0); + + return EXIT_SUCCESS; +err: + my_cleanup_options(my_long_options); + + my_end(0); + + exit(EXIT_FAILURE); +} + +static +int +get_options(int *argc, char ***argv) +{ + int ho_error; + + if ((ho_error= handle_options(argc, argv, my_long_options, + get_one_option))) { + exit(EXIT_FAILURE); + } + + return 0; +} + +static +void +print_version(void) +{ + printf("%s Ver %s for %s (%s)\n", my_progname, XBSTREAM_VERSION, + SYSTEM_TYPE, MACHINE_TYPE); +} + +static +void +usage(void) +{ + print_version(); + puts("Copyright (C) 2011-2013 Percona LLC and/or its affiliates."); + puts("This software comes with ABSOLUTELY NO WARRANTY. " + "This is free software,\nand you are welcome to modify and " + "redistribute it under the GPL license.\n"); + + puts("Serialize/deserialize files in the XBSTREAM format.\n"); + + puts("Usage: "); + printf(" %s -c [OPTIONS...] FILES... # stream specified files to " + "standard output.\n", my_progname); + printf(" %s -x [OPTIONS...] # extract files from the stream" + "on the standard input.\n", my_progname); + + puts("\nOptions:"); + my_print_help(my_long_options); +} + +static +int +set_run_mode(run_mode_t mode) +{ + if (opt_mode != RUN_MODE_NONE) { + msg("%s: can't set specify both -c and -x.\n", my_progname); + return 1; + } + + opt_mode = mode; + + return 0; +} + +static +my_bool +get_one_option(int optid, const struct my_option *opt __attribute__((unused)), + char *argument __attribute__((unused))) +{ + switch (optid) { + case 'c': + if (set_run_mode(RUN_MODE_CREATE)) { + return TRUE; + } + break; + case 'x': + if (set_run_mode(RUN_MODE_EXTRACT)) { + return TRUE; + } + break; + case '?': + usage(); + exit(0); + } + + return FALSE; +} + +static +int +stream_one_file(File file, xb_wstream_file_t *xbfile) +{ + uchar *buf; + size_t bytes; + size_t offset; + + posix_fadvise(file, 0, 0, POSIX_FADV_SEQUENTIAL); + offset = my_tell(file, MYF(MY_WME)); + + buf = (uchar*)(my_malloc(XBSTREAM_BUFFER_SIZE, MYF(MY_FAE))); + + while ((bytes = my_read(file, buf, XBSTREAM_BUFFER_SIZE, + MYF(MY_WME))) > 0) { + if (xb_stream_write_data(xbfile, buf, bytes)) { + msg("%s: xb_stream_write_data() failed.\n", + my_progname); + my_free(buf); + return 1; + } + posix_fadvise(file, offset, XBSTREAM_BUFFER_SIZE, + POSIX_FADV_DONTNEED); + offset += XBSTREAM_BUFFER_SIZE; + + } + + my_free(buf); + + if (bytes == (size_t) -1) { + return 1; + } + + return 0; +} + +static +int +mode_create(int argc, char **argv) +{ + int i; + MY_STAT mystat; + xb_wstream_t *stream; + + if (argc < 1) { + msg("%s: no files are specified.\n", my_progname); + return 1; + } + + stream = xb_stream_write_new(); + if (stream == NULL) { + msg("%s: xb_stream_write_new() failed.\n", my_progname); + return 1; + } + + for (i = 0; i < argc; i++) { + char *filepath = argv[i]; + File src_file; + xb_wstream_file_t *file; + + if (my_stat(filepath, &mystat, MYF(MY_WME)) == NULL) { + goto err; + } + if (!MY_S_ISREG(mystat.st_mode)) { + msg("%s: %s is not a regular file, exiting.\n", + my_progname, filepath); + goto err; + } + + if ((src_file = my_open(filepath, O_RDONLY, MYF(MY_WME))) < 0) { + msg("%s: failed to open %s.\n", my_progname, filepath); + goto err; + } + + file = xb_stream_write_open(stream, filepath, &mystat, NULL, NULL); + if (file == NULL) { + goto err; + } + + if (opt_verbose) { + msg("%s\n", filepath); + } + + if (stream_one_file(src_file, file) || + xb_stream_write_close(file) || + my_close(src_file, MYF(MY_WME))) { + goto err; + } + } + + xb_stream_write_done(stream); + + return 0; +err: + xb_stream_write_done(stream); + + return 1; +} + +static +file_entry_t * +file_entry_new(ds_ctxt_t *ds_ctxt, const char *path, uint pathlen) +{ + file_entry_t *entry; + ds_file_t *file; + + entry = (file_entry_t *) my_malloc(sizeof(file_entry_t), + MYF(MY_WME | MY_ZEROFILL)); + if (entry == NULL) { + return NULL; + } + + entry->path = my_strndup(path, pathlen, MYF(MY_WME)); + if (entry->path == NULL) { + goto err; + } + entry->pathlen = pathlen; + + file = ds_open(ds_ctxt, path, NULL); + if (file == NULL) { + msg("%s: failed to create file.\n", my_progname); + goto err; + } + + if (opt_verbose) { + msg("%s\n", entry->path); + } + + entry->file = file; + entry->ds_ctxt = ds_ctxt; + + return entry; + +err: + if (entry->path != NULL) { + my_free(entry->path); + } + my_free(entry); + + return NULL; +} + +static +uchar * +get_file_entry_key(file_entry_t *entry, size_t *length, + my_bool not_used __attribute__((unused))) +{ + *length = entry->pathlen; + return (uchar *) entry->path; +} + +static +void +file_entry_free(file_entry_t *entry) +{ + ds_close(entry->file); + my_free(entry->path); + my_free(entry); +} + +static +int +mode_extract(int argc __attribute__((unused)), + char **argv __attribute__((unused))) +{ + xb_rstream_t *stream; + xb_rstream_result_t res; + xb_rstream_chunk_t chunk; + HASH filehash; + file_entry_t *entry; + ds_ctxt_t *ds_ctxt; + + stream = xb_stream_read_new(); + if (stream == NULL) { + msg("%s: xb_stream_read_new() failed.\n", my_progname); + return 1; + } + + /* If --directory is specified, it is already set as CWD by now. */ + ds_ctxt = ds_create(".", DS_TYPE_LOCAL); + + if (my_hash_init(&filehash, &my_charset_bin, START_FILE_HASH_SIZE, + 0, 0, (my_hash_get_key) get_file_entry_key, + (my_hash_free_key) file_entry_free, MYF(0))) { + msg("%s: failed to initialize file hash.\n", my_progname); + goto err; + } + + while ((res = xb_stream_read_chunk(stream, &chunk)) == + XB_STREAM_READ_CHUNK) { + /* If unknown type and ignorable flag is set, skip this chunk */ + if (chunk.type == XB_CHUNK_TYPE_UNKNOWN && \ + !(chunk.flags & XB_STREAM_FLAG_IGNORABLE)) { + continue; + } + + /* See if we already have this file open */ + entry = (file_entry_t *) my_hash_search(&filehash, + (uchar *) chunk.path, + chunk.pathlen); + + if (entry == NULL) { + entry = file_entry_new(ds_ctxt, chunk.path, + chunk.pathlen); + if (entry == NULL) { + goto err; + } + if (my_hash_insert(&filehash, (uchar *) entry)) { + msg("%s: my_hash_insert() failed.\n", + my_progname); + goto err; + } + } + + if (chunk.type == XB_CHUNK_TYPE_EOF) { + my_hash_delete(&filehash, (uchar *) entry); + + continue; + } + + if (entry->offset != chunk.offset) { + msg("%s: out-of-order chunk: real offset = 0x%llx, " + "expected offset = 0x%llx\n", my_progname, + chunk.offset, entry->offset); + goto err; + } + + if (ds_write(entry->file, chunk.data, chunk.length)) { + msg("%s: my_write() failed.\n", my_progname); + goto err; + } + + entry->offset += chunk.length; + }; + + if (res == XB_STREAM_READ_ERROR) { + goto err; + } + + my_hash_free(&filehash); + ds_destroy(ds_ctxt); + xb_stream_read_done(stream); + + return 0; +err: + my_hash_free(&filehash); + ds_destroy(ds_ctxt); + xb_stream_read_done(stream); + + return 1; +} diff --git a/extra/mariabackup/xbstream.h b/extra/mariabackup/xbstream.h new file mode 100644 index 00000000000..e9f1468e58d --- /dev/null +++ b/extra/mariabackup/xbstream.h @@ -0,0 +1,103 @@ +/****************************************************** +Copyright (c) 2011-2013 Percona LLC and/or its affiliates. + +The xbstream format interface. + +This program is free software; you can redistribute it and/or modify +it under the terms of the GNU General Public License as published by +the Free Software Foundation; version 2 of the License. + +This program is distributed in the hope that it will be useful, +but WITHOUT ANY WARRANTY; without even the implied warranty of +MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the +GNU General Public License for more details. + +You should have received a copy of the GNU General Public License +along with this program; if not, write to the Free Software +Foundation, Inc., 51 Franklin Street, Fifth Floor, Boston, MA 02110-1301, USA + +*******************************************************/ + +#ifndef XBSTREAM_H +#define XBSTREAM_H + +#include + +/* Magic value in a chunk header */ +#define XB_STREAM_CHUNK_MAGIC "XBSTCK01" + +/* Chunk flags */ +/* Chunk can be ignored if unknown version/format */ +#define XB_STREAM_FLAG_IGNORABLE 0x01 + +/* Magic + flags + type + path len */ +#define CHUNK_HEADER_CONSTANT_LEN ((sizeof(XB_STREAM_CHUNK_MAGIC) - 1) + \ + 1 + 1 + 4) +#define CHUNK_TYPE_OFFSET (sizeof(XB_STREAM_CHUNK_MAGIC) - 1 + 1) +#define PATH_LENGTH_OFFSET (sizeof(XB_STREAM_CHUNK_MAGIC) - 1 + 1 + 1) + +typedef struct xb_wstream_struct xb_wstream_t; + +typedef struct xb_wstream_file_struct xb_wstream_file_t; + +typedef enum { + XB_STREAM_FMT_NONE, + XB_STREAM_FMT_TAR, + XB_STREAM_FMT_XBSTREAM +} xb_stream_fmt_t; + +/************************************************************************ +Write interface. */ + +typedef ssize_t xb_stream_write_callback(xb_wstream_file_t *file, + void *userdata, + const void *buf, size_t len); + +xb_wstream_t *xb_stream_write_new(void); + +xb_wstream_file_t *xb_stream_write_open(xb_wstream_t *stream, const char *path, + MY_STAT *mystat, void *userdata, + xb_stream_write_callback *onwrite); + +int xb_stream_write_data(xb_wstream_file_t *file, const void *buf, size_t len); + +int xb_stream_write_close(xb_wstream_file_t *file); + +int xb_stream_write_done(xb_wstream_t *stream); + +/************************************************************************ +Read interface. */ + +typedef enum { + XB_STREAM_READ_CHUNK, + XB_STREAM_READ_EOF, + XB_STREAM_READ_ERROR +} xb_rstream_result_t; + +typedef enum { + XB_CHUNK_TYPE_UNKNOWN = '\0', + XB_CHUNK_TYPE_PAYLOAD = 'P', + XB_CHUNK_TYPE_EOF = 'E' +} xb_chunk_type_t; + +typedef struct xb_rstream_struct xb_rstream_t; + +typedef struct { + uchar flags; + xb_chunk_type_t type; + uint pathlen; + char path[FN_REFLEN]; + size_t length; + my_off_t offset; + void *data; + ulong checksum; +} xb_rstream_chunk_t; + +xb_rstream_t *xb_stream_read_new(void); + +xb_rstream_result_t xb_stream_read_chunk(xb_rstream_t *stream, + xb_rstream_chunk_t *chunk); + +int xb_stream_read_done(xb_rstream_t *stream); + +#endif diff --git a/extra/mariabackup/xbstream_read.c b/extra/mariabackup/xbstream_read.c new file mode 100644 index 00000000000..0ffcabd9270 --- /dev/null +++ b/extra/mariabackup/xbstream_read.c @@ -0,0 +1,227 @@ +/****************************************************** +Copyright (c) 2011-2013 Percona LLC and/or its affiliates. + +The xbstream format reader implementation. + +This program is free software; you can redistribute it and/or modify +it under the terms of the GNU General Public License as published by +the Free Software Foundation; version 2 of the License. + +This program is distributed in the hope that it will be useful, +but WITHOUT ANY WARRANTY; without even the implied warranty of +MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the +GNU General Public License for more details. + +You should have received a copy of the GNU General Public License +along with this program; if not, write to the Free Software +Foundation, Inc., 51 Franklin Street, Fifth Floor, Boston, MA 02110-1301, USA + +*******************************************************/ + +#include +#include +#include +#include "common.h" +#include "xbstream.h" + +/* Allocate 1 MB for the payload buffer initially */ +#define INIT_BUFFER_LEN (1024 * 1024) + +#ifndef MY_OFF_T_MAX +#define MY_OFF_T_MAX (~(my_off_t)0UL) +#endif + +struct xb_rstream_struct { + my_off_t offset; + File fd; + void *buffer; + size_t buflen; +}; + +xb_rstream_t * +xb_stream_read_new(void) +{ + xb_rstream_t *stream; + + stream = (xb_rstream_t *) my_malloc(sizeof(xb_rstream_t), MYF(MY_FAE)); + + stream->buffer = my_malloc(INIT_BUFFER_LEN, MYF(MY_FAE)); + stream->buflen = INIT_BUFFER_LEN; + + stream->fd = fileno(stdin); + stream->offset = 0; + +#ifdef __WIN__ + setmode(stream->fd, _O_BINARY); +#endif + + return stream; +} + +static inline +xb_chunk_type_t +validate_chunk_type(uchar code) +{ + switch ((xb_chunk_type_t) code) { + case XB_CHUNK_TYPE_PAYLOAD: + case XB_CHUNK_TYPE_EOF: + return (xb_chunk_type_t) code; + default: + return XB_CHUNK_TYPE_UNKNOWN; + } +} + +#define F_READ(buf,len) \ + do { \ + if (xb_read_full(fd, buf, len) < len) { \ + msg("xb_stream_read_chunk(): my_read() failed.\n"); \ + goto err; \ + } \ + } while (0) + +xb_rstream_result_t +xb_stream_read_chunk(xb_rstream_t *stream, xb_rstream_chunk_t *chunk) +{ + uchar tmpbuf[16]; + uchar *ptr = tmpbuf; + uint pathlen; + size_t tbytes; + ulonglong ullval; + ulong checksum_exp; + ulong checksum; + File fd = stream->fd; + + xb_ad(sizeof(tmpbuf) >= CHUNK_HEADER_CONSTANT_LEN); + + /* This is the only place where we expect EOF, so read with + xb_read_full() rather than F_READ() */ + tbytes = xb_read_full(fd, ptr, CHUNK_HEADER_CONSTANT_LEN); + if (tbytes == 0) { + return XB_STREAM_READ_EOF; + } else if (tbytes < CHUNK_HEADER_CONSTANT_LEN) { + msg("xb_stream_read_chunk(): unexpected end of stream at " + "offset 0x%llx.\n", stream->offset); + goto err; + } + + ptr = tmpbuf; + + /* Chunk magic value */ + if (memcmp(tmpbuf, XB_STREAM_CHUNK_MAGIC, 8)) { + msg("xb_stream_read_chunk(): wrong chunk magic at offset " + "0x%llx.\n", (ulonglong) stream->offset); + goto err; + } + ptr += 8; + stream->offset += 8; + + /* Chunk flags */ + chunk->flags = *ptr++; + stream->offset++; + + /* Chunk type, ignore unknown ones if ignorable flag is set */ + chunk->type = validate_chunk_type(*ptr); + if (chunk->type == XB_CHUNK_TYPE_UNKNOWN && + !(chunk->flags & XB_STREAM_FLAG_IGNORABLE)) { + msg("xb_stream_read_chunk(): unknown chunk type 0x%lu at " + "offset 0x%llx.\n", (ulong) *ptr, + (ulonglong) stream->offset); + goto err; + } + ptr++; + stream->offset++; + + /* Path length */ + pathlen = uint4korr(ptr); + if (pathlen >= FN_REFLEN) { + msg("xb_stream_read_chunk(): path length (%lu) is too large at " + "offset 0x%llx.\n", (ulong) pathlen, stream->offset); + goto err; + } + chunk->pathlen = pathlen; + stream->offset +=4; + + xb_ad((ptr + 4 - tmpbuf) == CHUNK_HEADER_CONSTANT_LEN); + + /* Path */ + if (chunk->pathlen > 0) { + F_READ((uchar *) chunk->path, pathlen); + stream->offset += pathlen; + } + chunk->path[pathlen] = '\0'; + + if (chunk->type == XB_CHUNK_TYPE_EOF) { + return XB_STREAM_READ_CHUNK; + } + + /* Payload length */ + F_READ(tmpbuf, 16); + ullval = uint8korr(tmpbuf); + if (ullval > (ulonglong) SIZE_T_MAX) { + msg("xb_stream_read_chunk(): chunk length is too large at " + "offset 0x%llx: 0x%llx.\n", (ulonglong) stream->offset, + ullval); + goto err; + } + chunk->length = (size_t) ullval; + stream->offset += 8; + + /* Payload offset */ + ullval = uint8korr(tmpbuf + 8); + if (ullval > (ulonglong) MY_OFF_T_MAX) { + msg("xb_stream_read_chunk(): chunk offset is too large at " + "offset 0x%llx: 0x%llx.\n", (ulonglong) stream->offset, + ullval); + goto err; + } + chunk->offset = (my_off_t) ullval; + stream->offset += 8; + + /* Reallocate the buffer if needed */ + if (chunk->length > stream->buflen) { + stream->buffer = my_realloc(stream->buffer, chunk->length, + MYF(MY_WME)); + if (stream->buffer == NULL) { + msg("xb_stream_read_chunk(): failed to increase buffer " + "to %lu bytes.\n", (ulong) chunk->length); + goto err; + } + stream->buflen = chunk->length; + } + + /* Checksum */ + F_READ(tmpbuf, 4); + checksum_exp = uint4korr(tmpbuf); + + /* Payload */ + if (chunk->length > 0) { + F_READ(stream->buffer, chunk->length); + stream->offset += chunk->length; + } + + checksum = crc32(0, stream->buffer, chunk->length); + if (checksum != checksum_exp) { + msg("xb_stream_read_chunk(): invalid checksum at offset " + "0x%llx: expected 0x%lx, read 0x%lx.\n", + (ulonglong) stream->offset, checksum_exp, checksum); + goto err; + } + stream->offset += 4; + + chunk->data = stream->buffer; + chunk->checksum = checksum; + + return XB_STREAM_READ_CHUNK; + +err: + return XB_STREAM_READ_ERROR; +} + +int +xb_stream_read_done(xb_rstream_t *stream) +{ + my_free(stream->buffer); + my_free(stream); + + return 0; +} diff --git a/extra/mariabackup/xbstream_write.c b/extra/mariabackup/xbstream_write.c new file mode 100644 index 00000000000..7b042eea49c --- /dev/null +++ b/extra/mariabackup/xbstream_write.c @@ -0,0 +1,280 @@ +/****************************************************** +Copyright (c) 2011-2013 Percona LLC and/or its affiliates. + +The xbstream format writer implementation. + +This program is free software; you can redistribute it and/or modify +it under the terms of the GNU General Public License as published by +the Free Software Foundation; version 2 of the License. + +This program is distributed in the hope that it will be useful, +but WITHOUT ANY WARRANTY; without even the implied warranty of +MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the +GNU General Public License for more details. + +You should have received a copy of the GNU General Public License +along with this program; if not, write to the Free Software +Foundation, Inc., 51 Franklin Street, Fifth Floor, Boston, MA 02110-1301, USA + +*******************************************************/ + +#include +#include +#include +#include "common.h" +#include "xbstream.h" + +/* Group writes smaller than this into a single chunk */ +#define XB_STREAM_MIN_CHUNK_SIZE (10 * 1024 * 1024) + +struct xb_wstream_struct { + pthread_mutex_t mutex; +}; + +struct xb_wstream_file_struct { + xb_wstream_t *stream; + char *path; + ulong path_len; + char chunk[XB_STREAM_MIN_CHUNK_SIZE]; + char *chunk_ptr; + size_t chunk_free; + my_off_t offset; + void *userdata; + xb_stream_write_callback *write; +}; + +static int xb_stream_flush(xb_wstream_file_t *file); +static int xb_stream_write_chunk(xb_wstream_file_t *file, + const void *buf, size_t len); +static int xb_stream_write_eof(xb_wstream_file_t *file); + +static +ssize_t +xb_stream_default_write_callback(xb_wstream_file_t *file __attribute__((unused)), + void *userdata __attribute__((unused)), + const void *buf, size_t len) +{ + if (my_write(fileno(stdout), buf, len, MYF(MY_WME | MY_NABP))) + return -1; + return len; +} + +xb_wstream_t * +xb_stream_write_new(void) +{ + xb_wstream_t *stream; + + stream = (xb_wstream_t *) my_malloc(sizeof(xb_wstream_t), MYF(MY_FAE)); + pthread_mutex_init(&stream->mutex, NULL); + + return stream;; +} + +xb_wstream_file_t * +xb_stream_write_open(xb_wstream_t *stream, const char *path, + MY_STAT *mystat __attribute__((unused)), + void *userdata, + xb_stream_write_callback *onwrite) +{ + xb_wstream_file_t *file; + ulong path_len; + + path_len = strlen(path); + + if (path_len > FN_REFLEN) { + msg("xb_stream_write_open(): file path is too long.\n"); + return NULL; + } + + file = (xb_wstream_file_t *) my_malloc(sizeof(xb_wstream_file_t) + + path_len + 1, MYF(MY_FAE)); + + file->path = (char *) (file + 1); + memcpy(file->path, path, path_len + 1); + file->path_len = path_len; + + file->stream = stream; + file->offset = 0; + file->chunk_ptr = file->chunk; + file->chunk_free = XB_STREAM_MIN_CHUNK_SIZE; + if (onwrite) { +#ifdef __WIN__ + setmode(fileno(stdout), _O_BINARY); +#endif + file->userdata = userdata; + file->write = onwrite; + } else { + file->userdata = NULL; + file->write = xb_stream_default_write_callback; + } + + return file; +} + +int +xb_stream_write_data(xb_wstream_file_t *file, const void *buf, size_t len) +{ + if (len < file->chunk_free) { + memcpy(file->chunk_ptr, buf, len); + file->chunk_ptr += len; + file->chunk_free -= len; + + return 0; + } + + if (xb_stream_flush(file)) + return 1; + + return xb_stream_write_chunk(file, buf, len); +} + +int +xb_stream_write_close(xb_wstream_file_t *file) +{ + if (xb_stream_flush(file) || + xb_stream_write_eof(file)) { + my_free(file); + return 1; + } + + my_free(file); + + return 0; +} + +int +xb_stream_write_done(xb_wstream_t *stream) +{ + pthread_mutex_destroy(&stream->mutex); + + my_free(stream); + + return 0; +} + +static +int +xb_stream_flush(xb_wstream_file_t *file) +{ + if (file->chunk_ptr == file->chunk) { + return 0; + } + + if (xb_stream_write_chunk(file, file->chunk, + file->chunk_ptr - file->chunk)) { + return 1; + } + + file->chunk_ptr = file->chunk; + file->chunk_free = XB_STREAM_MIN_CHUNK_SIZE; + + return 0; +} + +static +int +xb_stream_write_chunk(xb_wstream_file_t *file, const void *buf, size_t len) +{ + /* Chunk magic + flags + chunk type + path_len + path + len + offset + + checksum */ + uchar tmpbuf[sizeof(XB_STREAM_CHUNK_MAGIC) - 1 + 1 + 1 + 4 + + FN_REFLEN + 8 + 8 + 4]; + uchar *ptr; + xb_wstream_t *stream = file->stream; + ulong checksum; + + /* Write xbstream header */ + ptr = tmpbuf; + + /* Chunk magic */ + memcpy(ptr, XB_STREAM_CHUNK_MAGIC, sizeof(XB_STREAM_CHUNK_MAGIC) - 1); + ptr += sizeof(XB_STREAM_CHUNK_MAGIC) - 1; + + *ptr++ = 0; /* Chunk flags */ + + *ptr++ = (uchar) XB_CHUNK_TYPE_PAYLOAD; /* Chunk type */ + + int4store(ptr, file->path_len); /* Path length */ + ptr += 4; + + memcpy(ptr, file->path, file->path_len); /* Path */ + ptr += file->path_len; + + int8store(ptr, len); /* Payload length */ + ptr += 8; + + pthread_mutex_lock(&stream->mutex); + + int8store(ptr, file->offset); /* Payload offset */ + ptr += 8; + + checksum = crc32(0, buf, len); /* checksum */ + int4store(ptr, checksum); + ptr += 4; + + xb_ad(ptr <= tmpbuf + sizeof(tmpbuf)); + + if (file->write(file, file->userdata, tmpbuf, ptr-tmpbuf) == -1) + goto err; + + + if (file->write(file, file->userdata, buf, len) == -1) /* Payload */ + goto err; + + file->offset+= len; + + pthread_mutex_unlock(&stream->mutex); + + return 0; + +err: + + pthread_mutex_unlock(&stream->mutex); + + return 1; +} + +static +int +xb_stream_write_eof(xb_wstream_file_t *file) +{ + /* Chunk magic + flags + chunk type + path_len + path */ + uchar tmpbuf[sizeof(XB_STREAM_CHUNK_MAGIC) - 1 + 1 + 1 + 4 + + FN_REFLEN]; + uchar *ptr; + xb_wstream_t *stream = file->stream; + + pthread_mutex_lock(&stream->mutex); + + /* Write xbstream header */ + ptr = tmpbuf; + + /* Chunk magic */ + memcpy(ptr, XB_STREAM_CHUNK_MAGIC, sizeof(XB_STREAM_CHUNK_MAGIC) - 1); + ptr += sizeof(XB_STREAM_CHUNK_MAGIC) - 1; + + *ptr++ = 0; /* Chunk flags */ + + *ptr++ = (uchar) XB_CHUNK_TYPE_EOF; /* Chunk type */ + + int4store(ptr, file->path_len); /* Path length */ + ptr += 4; + + memcpy(ptr, file->path, file->path_len); /* Path */ + ptr += file->path_len; + + xb_ad(ptr <= tmpbuf + sizeof(tmpbuf)); + + if (file->write(file, file->userdata, tmpbuf, + (ulonglong) (ptr - tmpbuf)) == -1) + goto err; + + pthread_mutex_unlock(&stream->mutex); + + return 0; +err: + + pthread_mutex_unlock(&stream->mutex); + + return 1; +} diff --git a/extra/mariabackup/xtrabackup.cc b/extra/mariabackup/xtrabackup.cc new file mode 100644 index 00000000000..d24c915bb48 --- /dev/null +++ b/extra/mariabackup/xtrabackup.cc @@ -0,0 +1,7220 @@ +/****************************************************** +XtraBackup: hot backup tool for InnoDB +(c) 2009-2015 Percona LLC and/or its affiliates +Originally Created 3/3/2009 Yasufumi Kinoshita +Written by Alexey Kopytov, Aleksandr Kuzminsky, Stewart Smith, Vadim Tkachenko, +Yasufumi Kinoshita, Ignacio Nin and Baron Schwartz. + +This program is free software; you can redistribute it and/or modify +it under the terms of the GNU General Public License as published by +the Free Software Foundation; version 2 of the License. + +This program is distributed in the hope that it will be useful, +but WITHOUT ANY WARRANTY; without even the implied warranty of +MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the +GNU General Public License for more details. + +You should have received a copy of the GNU General Public License +along with this program; if not, write to the Free Software +Foundation, Inc., 51 Franklin Street, Fifth Floor, Boston, MA 02110-1301, USA + +******************************************************* + +This file incorporates work covered by the following copyright and +permission notice: + +Copyright (c) 2000, 2011, MySQL AB & Innobase Oy. All Rights Reserved. + +This program is free software; you can redistribute it and/or modify it under +the terms of the GNU General Public License as published by the Free Software +Foundation; version 2 of the License. + +This program is distributed in the hope that it will be useful, but WITHOUT +ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or FITNESS +FOR A PARTICULAR PURPOSE. See the GNU General Public License for more details. + +You should have received a copy of the GNU General Public License along with +this program; if not, write to the Free Software Foundation, Inc., 59 Temple +Place, Suite 330, Boston, MA 02111-1307 USA + +*******************************************************/ + +//#define XTRABACKUP_TARGET_IS_PLUGIN + +#include +#include +#include +#include +#include +#include + +#include +#include + +#ifdef __linux__ +# include +#endif + +#include + +#include +#include +#include +#include +#include +#include +#include +#include +#include + +#include +#include +#include + +#define G_PTR uchar* + +#include "common.h" +#include "xtrabackup_version.h" +#include "datasink.h" + +#include "xb_regex.h" +#include "fil_cur.h" +#include "write_filt.h" +#include "xtrabackup.h" +#include "ds_buffer.h" +#include "ds_tmpfile.h" +#include "xbstream.h" +#include "changed_page_bitmap.h" +#include "read_filt.h" +#include "wsrep.h" +#include "innobackupex.h" +#include "backup_mysql.h" +#include "backup_copy.h" +#include "backup_mysql.h" + +/* TODO: replace with appropriate macros used in InnoDB 5.6 */ +#define PAGE_ZIP_MIN_SIZE_SHIFT 10 +#define DICT_TF_ZSSIZE_SHIFT 1 +#define DICT_TF_FORMAT_ZIP 1 +#define DICT_TF_FORMAT_SHIFT 5 + +int sys_var_init(); + +my_bool innodb_inited= 0; + +/* === xtrabackup specific options === */ +char xtrabackup_real_target_dir[FN_REFLEN] = "./xtrabackup_backupfiles/"; +char *xtrabackup_target_dir= xtrabackup_real_target_dir; +my_bool xtrabackup_version = FALSE; +my_bool xtrabackup_backup = FALSE; +my_bool xtrabackup_stats = FALSE; +my_bool xtrabackup_prepare = FALSE; +my_bool xtrabackup_copy_back = FALSE; +my_bool xtrabackup_move_back = FALSE; +my_bool xtrabackup_decrypt_decompress = FALSE; +my_bool xtrabackup_print_param = FALSE; + +my_bool xtrabackup_export = FALSE; +my_bool xtrabackup_apply_log_only = FALSE; + +longlong xtrabackup_use_memory = 100*1024*1024L; +my_bool xtrabackup_create_ib_logfile = FALSE; + +long xtrabackup_throttle = 0; /* 0:unlimited */ +lint io_ticket; +os_event_t wait_throttle = NULL; +os_event_t log_copying_stop = NULL; + +char *xtrabackup_incremental = NULL; +lsn_t incremental_lsn; +lsn_t incremental_to_lsn; +lsn_t incremental_last_lsn; +xb_page_bitmap *changed_page_bitmap = NULL; + +char *xtrabackup_incremental_basedir = NULL; /* for --backup */ +char *xtrabackup_extra_lsndir = NULL; /* for --backup with --extra-lsndir */ +char *xtrabackup_incremental_dir = NULL; /* for --prepare */ + +char xtrabackup_real_incremental_basedir[FN_REFLEN]; +char xtrabackup_real_extra_lsndir[FN_REFLEN]; +char xtrabackup_real_incremental_dir[FN_REFLEN]; + +lsn_t xtrabackup_archived_to_lsn = 0; /* for --archived-to-lsn */ + +char *xtrabackup_tables = NULL; + +/* List of regular expressions for filtering */ +typedef struct xb_regex_list_node_struct xb_regex_list_node_t; +struct xb_regex_list_node_struct { + UT_LIST_NODE_T(xb_regex_list_node_t) regex_list; + xb_regex_t regex; +}; +static UT_LIST_BASE_NODE_T(xb_regex_list_node_t) regex_list; + +static xb_regmatch_t tables_regmatch[1]; + +char *xtrabackup_tables_file = NULL; +static hash_table_t* tables_hash = NULL; + +char *xtrabackup_databases = NULL; +char *xtrabackup_databases_file = NULL; +static hash_table_t* databases_hash = NULL; + +static hash_table_t* inc_dir_tables_hash; + +struct xb_filter_entry_struct{ + char* name; + ibool has_tables; + hash_node_t name_hash; +}; +typedef struct xb_filter_entry_struct xb_filter_entry_t; + +static ulint thread_nr[SRV_MAX_N_IO_THREADS + 6]; +static os_thread_id_t thread_ids[SRV_MAX_N_IO_THREADS + 6]; + +lsn_t checkpoint_lsn_start; +lsn_t checkpoint_no_start; +lsn_t log_copy_scanned_lsn; +ibool log_copying = TRUE; +ibool log_copying_running = FALSE; +ibool io_watching_thread_running = FALSE; + +ibool xtrabackup_logfile_is_renamed = FALSE; + +int xtrabackup_parallel; + +char *xtrabackup_stream_str = NULL; +xb_stream_fmt_t xtrabackup_stream_fmt = XB_STREAM_FMT_NONE; +ibool xtrabackup_stream = FALSE; + +const char *xtrabackup_compress_alg = NULL; +ibool xtrabackup_compress = FALSE; +uint xtrabackup_compress_threads; +ulonglong xtrabackup_compress_chunk_size = 0; + +const char *xtrabackup_encrypt_algo_names[] = +{ "NONE", "AES128", "AES192", "AES256", NullS}; +TYPELIB xtrabackup_encrypt_algo_typelib= +{array_elements(xtrabackup_encrypt_algo_names)-1,"", + xtrabackup_encrypt_algo_names, NULL}; + +ibool xtrabackup_encrypt = FALSE; +ulong xtrabackup_encrypt_algo; +char *xtrabackup_encrypt_key = NULL; +char *xtrabackup_encrypt_key_file = NULL; +uint xtrabackup_encrypt_threads; +ulonglong xtrabackup_encrypt_chunk_size = 0; + +ulint xtrabackup_rebuild_threads = 1; + +/* sleep interval beetween log copy iterations in log copying thread +in milliseconds (default is 1 second) */ +ulint xtrabackup_log_copy_interval = 1000; + +/* Ignored option (--log) for MySQL option compatibility */ +char* log_ignored_opt = NULL; + +/* === metadata of backup === */ +#define XTRABACKUP_METADATA_FILENAME "xtrabackup_checkpoints" +char metadata_type[30] = ""; /*[full-backuped|log-applied| + full-prepared|incremental]*/ +lsn_t metadata_from_lsn = 0; +lsn_t metadata_to_lsn = 0; +lsn_t metadata_last_lsn = 0; + +#define XB_LOG_FILENAME "xtrabackup_logfile" + +ds_file_t *dst_log_file = NULL; + +static char mysql_data_home_buff[2]; + +const char *defaults_group = "mysqld"; + +/* === static parameters in ha_innodb.cc */ + +#define HA_INNOBASE_ROWS_IN_TABLE 10000 /* to get optimization right */ +#define HA_INNOBASE_RANGE_COUNT 100 + +ulong innobase_large_page_size = 0; + +/* The default values for the following, type long or longlong, start-up +parameters are declared in mysqld.cc: */ + +long innobase_additional_mem_pool_size = 1*1024*1024L; +long innobase_buffer_pool_awe_mem_mb = 0; +long innobase_file_io_threads = 4; +long innobase_read_io_threads = 4; +long innobase_write_io_threads = 4; +long innobase_force_recovery = 0; +long innobase_log_buffer_size = 1024*1024L; +long innobase_log_files_in_group = 2; +long innobase_open_files = 300L; + +longlong innobase_page_size = (1LL << 14); /* 16KB */ +static ulong innobase_log_block_size = 512; +my_bool innobase_fast_checksum = FALSE; +char* innobase_doublewrite_file = NULL; +char* innobase_buffer_pool_filename = NULL; + +longlong innobase_buffer_pool_size = 8*1024*1024L; +longlong innobase_log_file_size = 48*1024*1024L; + +/* The default values for the following char* start-up parameters +are determined in innobase_init below: */ + +char* innobase_ignored_opt = NULL; +char* innobase_data_home_dir = NULL; +char* innobase_data_file_path = NULL; +char* innobase_log_arch_dir = NULL;/* unused */ +/* The following has a misleading name: starting from 4.0.5, this also +affects Windows: */ +char* innobase_unix_file_flush_method = NULL; + +/* Below we have boolean-valued start-up parameters, and their default +values */ + +ulong innobase_fast_shutdown = 1; +my_bool innobase_log_archive = FALSE;/* unused */ +my_bool innobase_use_doublewrite = TRUE; +my_bool innobase_use_checksums = TRUE; +my_bool innobase_use_large_pages = FALSE; +my_bool innobase_file_per_table = FALSE; +my_bool innobase_locks_unsafe_for_binlog = FALSE; +my_bool innobase_rollback_on_timeout = FALSE; +my_bool innobase_create_status_file = FALSE; +my_bool innobase_adaptive_hash_index = TRUE; + +static char *internal_innobase_data_file_path = NULL; + +/* The following counter is used to convey information to InnoDB +about server activity: in selects it is not sensible to call +srv_active_wake_master_thread after each fetch or search, we only do +it every INNOBASE_WAKE_INTERVAL'th step. */ + +#define INNOBASE_WAKE_INTERVAL 32 +ulong innobase_active_counter = 0; + +ibool srv_compact_backup = FALSE; +ibool srv_rebuild_indexes = FALSE; + +static char *xtrabackup_debug_sync = NULL; + +my_bool xtrabackup_compact = FALSE; +my_bool xtrabackup_rebuild_indexes = FALSE; + +my_bool xtrabackup_incremental_force_scan = FALSE; + +/* The flushed lsn which is read from data files */ +lsn_t min_flushed_lsn= 0; +lsn_t max_flushed_lsn= 0; + +/* The size of archived log file */ +size_t xtrabackup_arch_file_size = 0ULL; +/* The minimal LSN of found archived log files */ +lsn_t xtrabackup_arch_first_file_lsn = 0ULL; +/* The maximum LSN of found archived log files */ +lsn_t xtrabackup_arch_last_file_lsn = 0ULL; + +ulong xb_open_files_limit= 0; +my_bool xb_close_files= FALSE; + +/* Datasinks */ +ds_ctxt_t *ds_data = NULL; +ds_ctxt_t *ds_meta = NULL; +ds_ctxt_t *ds_redo = NULL; + +static bool innobackupex_mode = false; + +static long innobase_log_files_in_group_save; +static char *srv_log_group_home_dir_save; +static longlong innobase_log_file_size_save; + +/* String buffer used by --print-param to accumulate server options as they are +parsed from the defaults file */ +static std::ostringstream print_param_str; + +/* Set of specified parameters */ +std::set param_set; + +static ulonglong global_max_value; + +extern "C" sig_handler handle_fatal_signal(int sig); + +my_bool opt_galera_info = FALSE; +my_bool opt_slave_info = FALSE; +my_bool opt_no_lock = FALSE; +my_bool opt_safe_slave_backup = FALSE; +my_bool opt_rsync = FALSE; +my_bool opt_force_non_empty_dirs = FALSE; +my_bool opt_noversioncheck = FALSE; +my_bool opt_no_backup_locks = FALSE; +my_bool opt_decompress = FALSE; +my_bool opt_remove_original = FALSE; + +static const char *binlog_info_values[] = {"off", "lockless", "on", "auto", + NullS}; +static TYPELIB binlog_info_typelib = {array_elements(binlog_info_values)-1, "", + binlog_info_values, NULL}; +ulong opt_binlog_info; + +char *opt_incremental_history_name = NULL; +char *opt_incremental_history_uuid = NULL; + +char *opt_user = NULL; +char *opt_password = NULL; +char *opt_host = NULL; +char *opt_defaults_group = NULL; +char *opt_socket = NULL; +uint opt_port = 0; +char *opt_login_path = NULL; +char *opt_log_bin = NULL; + +const char *query_type_names[] = { "ALL", "UPDATE", "SELECT", NullS}; + +TYPELIB query_type_typelib= {array_elements(query_type_names) - 1, "", + query_type_names, NULL}; + +ulong opt_lock_wait_query_type; +ulong opt_kill_long_query_type; + +ulong opt_decrypt_algo = 0; + +uint opt_kill_long_queries_timeout = 0; +uint opt_lock_wait_timeout = 0; +uint opt_lock_wait_threshold = 0; +uint opt_debug_sleep_before_unlock = 0; +uint opt_safe_slave_backup_timeout = 0; + +const char *opt_history = NULL; +my_bool opt_decrypt = FALSE; + +#if defined(HAVE_OPENSSL) +my_bool opt_ssl_verify_server_cert = FALSE; +#if !defined(HAVE_YASSL) +char *opt_server_public_key = NULL; +#endif +#endif + +/* Whether xtrabackup_binlog_info should be created on recovery */ +static bool recover_binlog_info; + +/* Simple datasink creation tracking...add datasinks in the reverse order you +want them destroyed. */ +#define XTRABACKUP_MAX_DATASINKS 10 +static ds_ctxt_t *datasinks[XTRABACKUP_MAX_DATASINKS]; +static uint actual_datasinks = 0; +static inline +void +xtrabackup_add_datasink(ds_ctxt_t *ds) +{ + xb_ad(actual_datasinks < XTRABACKUP_MAX_DATASINKS); + datasinks[actual_datasinks] = ds; actual_datasinks++; +} + +/* ======== Datafiles iterator ======== */ +datafiles_iter_t * +datafiles_iter_new(fil_system_t *f_system) +{ + datafiles_iter_t *it; + + it = static_cast + (ut_malloc(sizeof(datafiles_iter_t))); + it->mutex = os_mutex_create(); + + it->system = f_system; + it->space = NULL; + it->node = NULL; + it->started = FALSE; + + return it; +} + +fil_node_t * +datafiles_iter_next(datafiles_iter_t *it) +{ + fil_node_t *new_node; + + os_mutex_enter(it->mutex); + + if (it->node == NULL) { + if (it->started) + goto end; + it->started = TRUE; + } else { + it->node = UT_LIST_GET_NEXT(chain, it->node); + if (it->node != NULL) + goto end; + } + + it->space = (it->space == NULL) ? + UT_LIST_GET_FIRST(it->system->space_list) : + UT_LIST_GET_NEXT(space_list, it->space); + + while (it->space != NULL && + (it->space->purpose != FIL_TABLESPACE || + UT_LIST_GET_LEN(it->space->chain) == 0)) + it->space = UT_LIST_GET_NEXT(space_list, it->space); + if (it->space == NULL) + goto end; + + it->node = UT_LIST_GET_FIRST(it->space->chain); + +end: + new_node = it->node; + os_mutex_exit(it->mutex); + + return new_node; +} + +void +datafiles_iter_free(datafiles_iter_t *it) +{ + os_mutex_free(it->mutex); + ut_free(it); +} + +/* ======== Date copying thread context ======== */ + +typedef struct { + datafiles_iter_t *it; + uint num; + uint *count; + os_ib_mutex_t count_mutex; + os_thread_id_t id; +} data_thread_ctxt_t; + +/* ======== for option and variables ======== */ + +enum options_xtrabackup +{ + OPT_XTRA_TARGET_DIR = 1000, /* make sure it is larger + than OPT_MAX_CLIENT_OPTION */ + OPT_XTRA_BACKUP, + OPT_XTRA_STATS, + OPT_XTRA_PREPARE, + OPT_XTRA_EXPORT, + OPT_XTRA_APPLY_LOG_ONLY, + OPT_XTRA_PRINT_PARAM, + OPT_XTRA_USE_MEMORY, + OPT_XTRA_THROTTLE, + OPT_XTRA_LOG_COPY_INTERVAL, + OPT_XTRA_INCREMENTAL, + OPT_XTRA_INCREMENTAL_BASEDIR, + OPT_XTRA_EXTRA_LSNDIR, + OPT_XTRA_INCREMENTAL_DIR, + OPT_XTRA_ARCHIVED_TO_LSN, + OPT_XTRA_TABLES, + OPT_XTRA_TABLES_FILE, + OPT_XTRA_DATABASES, + OPT_XTRA_DATABASES_FILE, + OPT_XTRA_CREATE_IB_LOGFILE, + OPT_XTRA_PARALLEL, + OPT_XTRA_STREAM, + OPT_XTRA_COMPRESS, + OPT_XTRA_COMPRESS_THREADS, + OPT_XTRA_COMPRESS_CHUNK_SIZE, + OPT_XTRA_ENCRYPT, + OPT_XTRA_ENCRYPT_KEY, + OPT_XTRA_ENCRYPT_KEY_FILE, + OPT_XTRA_ENCRYPT_THREADS, + OPT_XTRA_ENCRYPT_CHUNK_SIZE, + OPT_LOG, + OPT_INNODB, + OPT_INNODB_CHECKSUMS, + OPT_INNODB_DATA_FILE_PATH, + OPT_INNODB_DATA_HOME_DIR, + OPT_INNODB_ADAPTIVE_HASH_INDEX, + OPT_INNODB_DOUBLEWRITE, + OPT_INNODB_FAST_SHUTDOWN, + OPT_INNODB_FILE_PER_TABLE, + OPT_INNODB_FLUSH_LOG_AT_TRX_COMMIT, + OPT_INNODB_FLUSH_METHOD, + OPT_INNODB_LOCKS_UNSAFE_FOR_BINLOG, + OPT_INNODB_LOG_ARCH_DIR, + OPT_INNODB_LOG_ARCHIVE, + OPT_INNODB_LOG_GROUP_HOME_DIR, + OPT_INNODB_MAX_DIRTY_PAGES_PCT, + OPT_INNODB_MAX_PURGE_LAG, + OPT_INNODB_ROLLBACK_ON_TIMEOUT, + OPT_INNODB_STATUS_FILE, + OPT_INNODB_ADDITIONAL_MEM_POOL_SIZE, + OPT_INNODB_AUTOEXTEND_INCREMENT, + OPT_INNODB_BUFFER_POOL_SIZE, + OPT_INNODB_COMMIT_CONCURRENCY, + OPT_INNODB_CONCURRENCY_TICKETS, + OPT_INNODB_FILE_IO_THREADS, + OPT_INNODB_IO_CAPACITY, + OPT_INNODB_READ_IO_THREADS, + OPT_INNODB_WRITE_IO_THREADS, + OPT_INNODB_USE_NATIVE_AIO, + OPT_INNODB_PAGE_SIZE, + OPT_INNODB_LOG_BLOCK_SIZE, + OPT_INNODB_FAST_CHECKSUM, + OPT_INNODB_EXTRA_UNDOSLOTS, + OPT_INNODB_DOUBLEWRITE_FILE, + OPT_INNODB_BUFFER_POOL_FILENAME, + OPT_INNODB_FORCE_RECOVERY, + OPT_INNODB_LOCK_WAIT_TIMEOUT, + OPT_INNODB_LOG_BUFFER_SIZE, + OPT_INNODB_LOG_FILE_SIZE, + OPT_INNODB_LOG_FILES_IN_GROUP, + OPT_INNODB_MIRRORED_LOG_GROUPS, + OPT_INNODB_OPEN_FILES, + OPT_INNODB_SYNC_SPIN_LOOPS, + OPT_INNODB_THREAD_CONCURRENCY, + OPT_INNODB_THREAD_SLEEP_DELAY, + OPT_XTRA_DEBUG_SYNC, + OPT_XTRA_COMPACT, + OPT_XTRA_REBUILD_INDEXES, + OPT_XTRA_REBUILD_THREADS, + OPT_INNODB_CHECKSUM_ALGORITHM, + OPT_INNODB_UNDO_DIRECTORY, + OPT_INNODB_UNDO_TABLESPACES, + OPT_INNODB_LOG_CHECKSUM_ALGORITHM, + OPT_XTRA_INCREMENTAL_FORCE_SCAN, + OPT_DEFAULTS_GROUP, + OPT_OPEN_FILES_LIMIT, + OPT_CLOSE_FILES, + OPT_CORE_FILE, + + OPT_COPY_BACK, + OPT_MOVE_BACK, + OPT_GALERA_INFO, + OPT_SLAVE_INFO, + OPT_NO_LOCK, + OPT_SAFE_SLAVE_BACKUP, + OPT_RSYNC, + OPT_FORCE_NON_EMPTY_DIRS, + OPT_NO_VERSION_CHECK, + OPT_NO_BACKUP_LOCKS, + OPT_DECOMPRESS, + OPT_INCREMENTAL_HISTORY_NAME, + OPT_INCREMENTAL_HISTORY_UUID, + OPT_DECRYPT, + OPT_REMOVE_ORIGINAL, + OPT_LOCK_WAIT_QUERY_TYPE, + OPT_KILL_LONG_QUERY_TYPE, + OPT_HISTORY, + OPT_KILL_LONG_QUERIES_TIMEOUT, + OPT_LOCK_WAIT_TIMEOUT, + OPT_LOCK_WAIT_THRESHOLD, + OPT_DEBUG_SLEEP_BEFORE_UNLOCK, + OPT_SAFE_SLAVE_BACKUP_TIMEOUT, + OPT_BINLOG_INFO, + OPT_XB_SECURE_AUTH, + + OPT_SSL_SSL, + OPT_SSL_VERIFY_SERVER_CERT, + OPT_SERVER_PUBLIC_KEY, + +}; + +struct my_option xb_client_options[] = +{ + {"version", 'v', "print xtrabackup version information", + (G_PTR *) &xtrabackup_version, (G_PTR *) &xtrabackup_version, 0, GET_BOOL, + NO_ARG, 0, 0, 0, 0, 0, 0}, + {"target-dir", OPT_XTRA_TARGET_DIR, "destination directory", (G_PTR*) &xtrabackup_target_dir, + (G_PTR*) &xtrabackup_target_dir, 0, GET_STR, REQUIRED_ARG, 0, 0, 0, 0, 0, 0}, + {"backup", OPT_XTRA_BACKUP, "take backup to target-dir", + (G_PTR*) &xtrabackup_backup, (G_PTR*) &xtrabackup_backup, + 0, GET_BOOL, NO_ARG, 0, 0, 0, 0, 0, 0}, + {"stats", OPT_XTRA_STATS, "calc statistic of datadir (offline mysqld is recommended)", + (G_PTR*) &xtrabackup_stats, (G_PTR*) &xtrabackup_stats, + 0, GET_BOOL, NO_ARG, 0, 0, 0, 0, 0, 0}, + {"prepare", OPT_XTRA_PREPARE, "prepare a backup for starting mysql server on the backup.", + (G_PTR*) &xtrabackup_prepare, (G_PTR*) &xtrabackup_prepare, + 0, GET_BOOL, NO_ARG, 0, 0, 0, 0, 0, 0}, + {"export", OPT_XTRA_EXPORT, "create files to import to another database when prepare.", + (G_PTR*) &xtrabackup_export, (G_PTR*) &xtrabackup_export, + 0, GET_BOOL, NO_ARG, 0, 0, 0, 0, 0, 0}, + {"apply-log-only", OPT_XTRA_APPLY_LOG_ONLY, + "stop recovery process not to progress LSN after applying log when prepare.", + (G_PTR*) &xtrabackup_apply_log_only, (G_PTR*) &xtrabackup_apply_log_only, + 0, GET_BOOL, NO_ARG, 0, 0, 0, 0, 0, 0}, + {"print-param", OPT_XTRA_PRINT_PARAM, "print parameter of mysqld needed for copyback.", + (G_PTR*) &xtrabackup_print_param, (G_PTR*) &xtrabackup_print_param, + 0, GET_BOOL, NO_ARG, 0, 0, 0, 0, 0, 0}, + {"use-memory", OPT_XTRA_USE_MEMORY, "The value is used instead of buffer_pool_size", + (G_PTR*) &xtrabackup_use_memory, (G_PTR*) &xtrabackup_use_memory, + 0, GET_LL, REQUIRED_ARG, 100*1024*1024L, 1024*1024L, LONGLONG_MAX, 0, + 1024*1024L, 0}, + {"throttle", OPT_XTRA_THROTTLE, "limit count of IO operations (pairs of read&write) per second to IOS values (for '--backup')", + (G_PTR*) &xtrabackup_throttle, (G_PTR*) &xtrabackup_throttle, + 0, GET_LONG, REQUIRED_ARG, 0, 0, LONG_MAX, 0, 1, 0}, + {"log", OPT_LOG, "Ignored option for MySQL option compatibility", + (G_PTR*) &log_ignored_opt, (G_PTR*) &log_ignored_opt, 0, + GET_STR, OPT_ARG, 0, 0, 0, 0, 0, 0}, + {"log-copy-interval", OPT_XTRA_LOG_COPY_INTERVAL, "time interval between checks done by log copying thread in milliseconds (default is 1 second).", + (G_PTR*) &xtrabackup_log_copy_interval, (G_PTR*) &xtrabackup_log_copy_interval, + 0, GET_LONG, REQUIRED_ARG, 1000, 0, LONG_MAX, 0, 1, 0}, + {"extra-lsndir", OPT_XTRA_EXTRA_LSNDIR, "(for --backup): save an extra copy of the xtrabackup_checkpoints file in this directory.", + (G_PTR*) &xtrabackup_extra_lsndir, (G_PTR*) &xtrabackup_extra_lsndir, + 0, GET_STR, REQUIRED_ARG, 0, 0, 0, 0, 0, 0}, + {"incremental-lsn", OPT_XTRA_INCREMENTAL, "(for --backup): copy only .ibd pages newer than specified LSN 'high:low'. ##ATTENTION##: If a wrong LSN value is specified, it is impossible to diagnose this, causing the backup to be unusable. Be careful!", + (G_PTR*) &xtrabackup_incremental, (G_PTR*) &xtrabackup_incremental, + 0, GET_STR, REQUIRED_ARG, 0, 0, 0, 0, 0, 0}, + {"incremental-basedir", OPT_XTRA_INCREMENTAL_BASEDIR, "(for --backup): copy only .ibd pages newer than backup at specified directory.", + (G_PTR*) &xtrabackup_incremental_basedir, (G_PTR*) &xtrabackup_incremental_basedir, + 0, GET_STR, REQUIRED_ARG, 0, 0, 0, 0, 0, 0}, + {"incremental-dir", OPT_XTRA_INCREMENTAL_DIR, "(for --prepare): apply .delta files and logfile in the specified directory.", + (G_PTR*) &xtrabackup_incremental_dir, (G_PTR*) &xtrabackup_incremental_dir, + 0, GET_STR, REQUIRED_ARG, 0, 0, 0, 0, 0, 0}, + {"to-archived-lsn", OPT_XTRA_ARCHIVED_TO_LSN, + "Don't apply archived logs with bigger log sequence number.", + (G_PTR*) &xtrabackup_archived_to_lsn, (G_PTR*) &xtrabackup_archived_to_lsn, 0, + GET_LL, REQUIRED_ARG, 0, 0, LONGLONG_MAX, 0, 0, 0}, + {"tables", OPT_XTRA_TABLES, "filtering by regexp for table names.", + (G_PTR*) &xtrabackup_tables, (G_PTR*) &xtrabackup_tables, + 0, GET_STR, REQUIRED_ARG, 0, 0, 0, 0, 0, 0}, + {"tables_file", OPT_XTRA_TABLES_FILE, "filtering by list of the exact database.table name in the file.", + (G_PTR*) &xtrabackup_tables_file, (G_PTR*) &xtrabackup_tables_file, + 0, GET_STR, REQUIRED_ARG, 0, 0, 0, 0, 0, 0}, + {"databases", OPT_XTRA_DATABASES, "filtering by list of databases.", + (G_PTR*) &xtrabackup_databases, (G_PTR*) &xtrabackup_databases, + 0, GET_STR, REQUIRED_ARG, 0, 0, 0, 0, 0, 0}, + {"databases_file", OPT_XTRA_TABLES_FILE, + "filtering by list of databases in the file.", + (G_PTR*) &xtrabackup_databases_file, (G_PTR*) &xtrabackup_databases_file, + 0, GET_STR, REQUIRED_ARG, 0, 0, 0, 0, 0, 0}, + {"create-ib-logfile", OPT_XTRA_CREATE_IB_LOGFILE, "** not work for now** creates ib_logfile* also after '--prepare'. ### If you want create ib_logfile*, only re-execute this command in same options. ###", + (G_PTR*) &xtrabackup_create_ib_logfile, (G_PTR*) &xtrabackup_create_ib_logfile, + 0, GET_BOOL, NO_ARG, 0, 0, 0, 0, 0, 0}, + + {"stream", OPT_XTRA_STREAM, "Stream all backup files to the standard output " + "in the specified format. Currently the only supported format is 'tar'.", + (G_PTR*) &xtrabackup_stream_str, (G_PTR*) &xtrabackup_stream_str, 0, GET_STR, + REQUIRED_ARG, 0, 0, 0, 0, 0, 0}, + + {"compress", OPT_XTRA_COMPRESS, "Compress individual backup files using the " + "specified compression algorithm. Currently the only supported algorithm " + "is 'quicklz'. It is also the default algorithm, i.e. the one used when " + "--compress is used without an argument.", + (G_PTR*) &xtrabackup_compress_alg, (G_PTR*) &xtrabackup_compress_alg, 0, + GET_STR, OPT_ARG, 0, 0, 0, 0, 0, 0}, + + {"compress-threads", OPT_XTRA_COMPRESS_THREADS, + "Number of threads for parallel data compression. The default value is 1.", + (G_PTR*) &xtrabackup_compress_threads, (G_PTR*) &xtrabackup_compress_threads, + 0, GET_UINT, REQUIRED_ARG, 1, 1, UINT_MAX, 0, 0, 0}, + + {"compress-chunk-size", OPT_XTRA_COMPRESS_CHUNK_SIZE, + "Size of working buffer(s) for compression threads in bytes. The default value is 64K.", + (G_PTR*) &xtrabackup_compress_chunk_size, (G_PTR*) &xtrabackup_compress_chunk_size, + 0, GET_ULL, REQUIRED_ARG, (1 << 16), 1024, ULONGLONG_MAX, 0, 0, 0}, + + {"encrypt", OPT_XTRA_ENCRYPT, "Encrypt individual backup files using the " + "specified encryption algorithm.", + &xtrabackup_encrypt_algo, &xtrabackup_encrypt_algo, + &xtrabackup_encrypt_algo_typelib, GET_ENUM, REQUIRED_ARG, 0, 0, 0, 0, 0, 0}, + + {"encrypt-key", OPT_XTRA_ENCRYPT_KEY, "Encryption key to use.", + (G_PTR*) &xtrabackup_encrypt_key, (G_PTR*) &xtrabackup_encrypt_key, 0, + GET_STR_ALLOC, REQUIRED_ARG, 0, 0, 0, 0, 0, 0}, + + {"encrypt-key-file", OPT_XTRA_ENCRYPT_KEY_FILE, "File which contains encryption key to use.", + (G_PTR*) &xtrabackup_encrypt_key_file, (G_PTR*) &xtrabackup_encrypt_key_file, 0, + GET_STR_ALLOC, REQUIRED_ARG, 0, 0, 0, 0, 0, 0}, + + {"encrypt-threads", OPT_XTRA_ENCRYPT_THREADS, + "Number of threads for parallel data encryption. The default value is 1.", + (G_PTR*) &xtrabackup_encrypt_threads, (G_PTR*) &xtrabackup_encrypt_threads, + 0, GET_UINT, REQUIRED_ARG, 1, 1, UINT_MAX, 0, 0, 0}, + + {"encrypt-chunk-size", OPT_XTRA_ENCRYPT_CHUNK_SIZE, + "Size of working buffer(S) for encryption threads in bytes. The default value is 64K.", + (G_PTR*) &xtrabackup_encrypt_chunk_size, (G_PTR*) &xtrabackup_encrypt_chunk_size, + 0, GET_ULL, REQUIRED_ARG, (1 << 16), 1024, ULONGLONG_MAX, 0, 0, 0}, + + {"compact", OPT_XTRA_COMPACT, + "Create a compact backup by skipping secondary index pages.", + (G_PTR*) &xtrabackup_compact, (G_PTR*) &xtrabackup_compact, + 0, GET_BOOL, NO_ARG, 0, 0, 0, 0, 0, 0}, + + {"rebuild_indexes", OPT_XTRA_REBUILD_INDEXES, + "Rebuild secondary indexes in InnoDB tables after applying the log. " + "Only has effect with --prepare.", + (G_PTR*) &xtrabackup_rebuild_indexes, (G_PTR*) &xtrabackup_rebuild_indexes, + 0, GET_BOOL, NO_ARG, 0, 0, 0, 0, 0, 0}, + + {"rebuild_threads", OPT_XTRA_REBUILD_THREADS, + "Use this number of threads to rebuild indexes in a compact backup. " + "Only has effect with --prepare and --rebuild-indexes.", + (G_PTR*) &xtrabackup_rebuild_threads, (G_PTR*) &xtrabackup_rebuild_threads, + 0, GET_UINT, REQUIRED_ARG, 1, 1, UINT_MAX, 0, 0, 0}, + + {"incremental-force-scan", OPT_XTRA_INCREMENTAL_FORCE_SCAN, + "Perform a full-scan incremental backup even in the presence of changed " + "page bitmap data", + (G_PTR*)&xtrabackup_incremental_force_scan, + (G_PTR*)&xtrabackup_incremental_force_scan, 0, GET_BOOL, NO_ARG, + 0, 0, 0, 0, 0, 0}, + + + {"close_files", OPT_CLOSE_FILES, "do not keep files opened. Use at your own " + "risk.", (G_PTR*) &xb_close_files, (G_PTR*) &xb_close_files, 0, GET_BOOL, + NO_ARG, 0, 0, 0, 0, 0, 0}, + + {"core-file", OPT_CORE_FILE, "Write core on fatal signals", 0, 0, 0, + GET_NO_ARG, NO_ARG, 0, 0, 0, 0, 0, 0}, + + + {"copy-back", OPT_COPY_BACK, "Copy all the files in a previously made " + "backup from the backup directory to their original locations.", + (uchar *) &xtrabackup_copy_back, (uchar *) &xtrabackup_copy_back, 0, + GET_BOOL, NO_ARG, 0, 0, 0, 0, 0, 0}, + + {"move-back", OPT_MOVE_BACK, "Move all the files in a previously made " + "backup from the backup directory to the actual datadir location. " + "Use with caution, as it removes backup files.", + (uchar *) &xtrabackup_move_back, (uchar *) &xtrabackup_move_back, 0, + GET_BOOL, NO_ARG, 0, 0, 0, 0, 0, 0}, + + {"galera-info", OPT_GALERA_INFO, "This options creates the " + "xtrabackup_galera_info file which contains the local node state at " + "the time of the backup. Option should be used when performing the " + "backup of Percona-XtraDB-Cluster. Has no effect when backup locks " + "are used to create the backup.", + (uchar *) &opt_galera_info, (uchar *) &opt_galera_info, 0, + GET_BOOL, NO_ARG, 0, 0, 0, 0, 0, 0}, + + {"slave-info", OPT_SLAVE_INFO, "This option is useful when backing " + "up a replication slave server. It prints the binary log position " + "and name of the master server. It also writes this information to " + "the \"xtrabackup_slave_info\" file as a \"CHANGE MASTER\" command. " + "A new slave for this master can be set up by starting a slave server " + "on this backup and issuing a \"CHANGE MASTER\" command with the " + "binary log position saved in the \"xtrabackup_slave_info\" file.", + (uchar *) &opt_slave_info, (uchar *) &opt_slave_info, 0, + GET_BOOL, NO_ARG, 0, 0, 0, 0, 0, 0}, + + {"no-lock", OPT_NO_LOCK, "Use this option to disable table lock " + "with \"FLUSH TABLES WITH READ LOCK\". Use it only if ALL your " + "tables are InnoDB and you DO NOT CARE about the binary log " + "position of the backup. This option shouldn't be used if there " + "are any DDL statements being executed or if any updates are " + "happening on non-InnoDB tables (this includes the system MyISAM " + "tables in the mysql database), otherwise it could lead to an " + "inconsistent backup. If you are considering to use --no-lock " + "because your backups are failing to acquire the lock, this could " + "be because of incoming replication events preventing the lock " + "from succeeding. Please try using --safe-slave-backup to " + "momentarily stop the replication slave thread, this may help " + "the backup to succeed and you then don't need to resort to " + "using this option.", + (uchar *) &opt_no_lock, (uchar *) &opt_no_lock, 0, + GET_BOOL, NO_ARG, 0, 0, 0, 0, 0, 0}, + + {"safe-slave-backup", OPT_SAFE_SLAVE_BACKUP, "Stop slave SQL thread " + "and wait to start backup until Slave_open_temp_tables in " + "\"SHOW STATUS\" is zero. If there are no open temporary tables, " + "the backup will take place, otherwise the SQL thread will be " + "started and stopped until there are no open temporary tables. " + "The backup will fail if Slave_open_temp_tables does not become " + "zero after --safe-slave-backup-timeout seconds. The slave SQL " + "thread will be restarted when the backup finishes.", + (uchar *) &opt_safe_slave_backup, + (uchar *) &opt_safe_slave_backup, + 0, GET_BOOL, NO_ARG, 0, 0, 0, 0, 0, 0}, + + {"rsync", OPT_RSYNC, "Uses the rsync utility to optimize local file " + "transfers. When this option is specified, innobackupex uses rsync " + "to copy all non-InnoDB files instead of spawning a separate cp for " + "each file, which can be much faster for servers with a large number " + "of databases or tables. This option cannot be used together with " + "--stream.", + (uchar *) &opt_rsync, (uchar *) &opt_rsync, + 0, GET_BOOL, NO_ARG, 0, 0, 0, 0, 0, 0}, + + {"force-non-empty-directories", OPT_FORCE_NON_EMPTY_DIRS, "This " + "option, when specified, makes --copy-back or --move-back transfer " + "files to non-empty directories. Note that no existing files will be " + "overwritten. If --copy-back or --nove-back has to copy a file from " + "the backup directory which already exists in the destination " + "directory, it will still fail with an error.", + (uchar *) &opt_force_non_empty_dirs, + (uchar *) &opt_force_non_empty_dirs, + 0, GET_BOOL, NO_ARG, 0, 0, 0, 0, 0, 0}, + + {"no-version-check", OPT_NO_VERSION_CHECK, "This option disables the " + "version check which is enabled by the --version-check option.", + (uchar *) &opt_noversioncheck, + (uchar *) &opt_noversioncheck, + 0, GET_BOOL, NO_ARG, 0, 0, 0, 0, 0, 0}, + + {"no-backup-locks", OPT_NO_BACKUP_LOCKS, "This option controls if " + "backup locks should be used instead of FLUSH TABLES WITH READ LOCK " + "on the backup stage. The option has no effect when backup locks are " + "not supported by the server. This option is enabled by default, " + "disable with --no-backup-locks.", + (uchar *) &opt_no_backup_locks, + (uchar *) &opt_no_backup_locks, + 0, GET_BOOL, NO_ARG, 0, 0, 0, 0, 0, 0}, + + {"decompress", OPT_DECOMPRESS, "Decompresses all files with the .qp " + "extension in a backup previously made with the --compress option.", + (uchar *) &opt_decompress, + (uchar *) &opt_decompress, + 0, GET_BOOL, NO_ARG, 0, 0, 0, 0, 0, 0}, + + {"user", 'u', "This option specifies the MySQL username used " + "when connecting to the server, if that's not the current user. " + "The option accepts a string argument. See mysql --help for details.", + (uchar*) &opt_user, (uchar*) &opt_user, 0, GET_STR, + REQUIRED_ARG, 0, 0, 0, 0, 0, 0}, + + {"host", 'H', "This option specifies the host to use when " + "connecting to the database server with TCP/IP. The option accepts " + "a string argument. See mysql --help for details.", + (uchar*) &opt_host, (uchar*) &opt_host, 0, GET_STR, + REQUIRED_ARG, 0, 0, 0, 0, 0, 0}, + + {"port", 'P', "This option specifies the port to use when " + "connecting to the database server with TCP/IP. The option accepts " + "a string argument. See mysql --help for details.", + &opt_port, &opt_port, 0, GET_UINT, REQUIRED_ARG, + 0, 0, 0, 0, 0, 0}, + + {"password", 'p', "This option specifies the password to use " + "when connecting to the database. It accepts a string argument. " + "See mysql --help for details.", + 0, 0, 0, GET_STR, + REQUIRED_ARG, 0, 0, 0, 0, 0, 0}, + + {"socket", 'S', "This option specifies the socket to use when " + "connecting to the local database server with a UNIX domain socket. " + "The option accepts a string argument. See mysql --help for details.", + (uchar*) &opt_socket, (uchar*) &opt_socket, 0, GET_STR, + REQUIRED_ARG, 0, 0, 0, 0, 0, 0}, + + {"incremental-history-name", OPT_INCREMENTAL_HISTORY_NAME, + "This option specifies the name of the backup series stored in the " + "PERCONA_SCHEMA.xtrabackup_history history record to base an " + "incremental backup on. Xtrabackup will search the history table " + "looking for the most recent (highest innodb_to_lsn), successful " + "backup in the series and take the to_lsn value to use as the " + "starting lsn for the incremental backup. This will be mutually " + "exclusive with --incremental-history-uuid, --incremental-basedir " + "and --incremental-lsn. If no valid lsn can be found (no series by " + "that name, no successful backups by that name) xtrabackup will " + "return with an error. It is used with the --incremental option.", + (uchar*) &opt_incremental_history_name, + (uchar*) &opt_incremental_history_name, 0, GET_STR, + REQUIRED_ARG, 0, 0, 0, 0, 0, 0}, + + {"incremental-history-uuid", OPT_INCREMENTAL_HISTORY_UUID, + "This option specifies the UUID of the specific history record " + "stored in the PERCONA_SCHEMA.xtrabackup_history to base an " + "incremental backup on. --incremental-history-name, " + "--incremental-basedir and --incremental-lsn. If no valid lsn can be " + "found (no success record with that uuid) xtrabackup will return " + "with an error. It is used with the --incremental option.", + (uchar*) &opt_incremental_history_uuid, + (uchar*) &opt_incremental_history_uuid, 0, GET_STR, + REQUIRED_ARG, 0, 0, 0, 0, 0, 0}, + + {"decrypt", OPT_DECRYPT, "Decrypts all files with the .xbcrypt " + "extension in a backup previously made with --encrypt option.", + &opt_decrypt_algo, &opt_decrypt_algo, + &xtrabackup_encrypt_algo_typelib, GET_ENUM, REQUIRED_ARG, + 0, 0, 0, 0, 0, 0}, + + {"remove-original", OPT_REMOVE_ORIGINAL, "Remove .qp and .xbcrypt files " + "after decryption and decompression.", + (uchar *) &opt_remove_original, + (uchar *) &opt_remove_original, + 0, GET_BOOL, NO_ARG, 0, 0, 0, 0, 0, 0}, + + {"ftwrl-wait-query-type", OPT_LOCK_WAIT_QUERY_TYPE, + "This option specifies which types of queries are allowed to complete " + "before innobackupex will issue the global lock. Default is all.", + (uchar*) &opt_lock_wait_query_type, + (uchar*) &opt_lock_wait_query_type, &query_type_typelib, + GET_ENUM, REQUIRED_ARG, QUERY_TYPE_ALL, 0, 0, 0, 0, 0}, + + {"kill-long-query-type", OPT_KILL_LONG_QUERY_TYPE, + "This option specifies which types of queries should be killed to " + "unblock the global lock. Default is \"all\".", + (uchar*) &opt_kill_long_query_type, + (uchar*) &opt_kill_long_query_type, &query_type_typelib, + GET_ENUM, REQUIRED_ARG, QUERY_TYPE_SELECT, 0, 0, 0, 0, 0}, + + {"history", OPT_HISTORY, + "This option enables the tracking of backup history in the " + "PERCONA_SCHEMA.xtrabackup_history table. An optional history " + "series name may be specified that will be placed with the history " + "record for the current backup being taken.", + NULL, NULL, 0, GET_STR, OPT_ARG, 0, 0, 0, 0, 0, 0}, + + {"kill-long-queries-timeout", OPT_KILL_LONG_QUERIES_TIMEOUT, + "This option specifies the number of seconds innobackupex waits " + "between starting FLUSH TABLES WITH READ LOCK and killing those " + "queries that block it. Default is 0 seconds, which means " + "innobackupex will not attempt to kill any queries.", + (uchar*) &opt_kill_long_queries_timeout, + (uchar*) &opt_kill_long_queries_timeout, 0, GET_UINT, + REQUIRED_ARG, 0, 0, 0, 0, 0, 0}, + + {"ftwrl-wait-timeout", OPT_LOCK_WAIT_TIMEOUT, + "This option specifies time in seconds that innobackupex should wait " + "for queries that would block FTWRL before running it. If there are " + "still such queries when the timeout expires, innobackupex terminates " + "with an error. Default is 0, in which case innobackupex does not " + "wait for queries to complete and starts FTWRL immediately.", + (uchar*) &opt_lock_wait_timeout, + (uchar*) &opt_lock_wait_timeout, 0, GET_UINT, + REQUIRED_ARG, 0, 0, 0, 0, 0, 0}, + + {"ftwrl-wait-threshold", OPT_LOCK_WAIT_THRESHOLD, + "This option specifies the query run time threshold which is used by " + "innobackupex to detect long-running queries with a non-zero value " + "of --ftwrl-wait-timeout. FTWRL is not started until such " + "long-running queries exist. This option has no effect if " + "--ftwrl-wait-timeout is 0. Default value is 60 seconds.", + (uchar*) &opt_lock_wait_threshold, + (uchar*) &opt_lock_wait_threshold, 0, GET_UINT, + REQUIRED_ARG, 60, 0, 0, 0, 0, 0}, + + {"debug-sleep-before-unlock", OPT_DEBUG_SLEEP_BEFORE_UNLOCK, + "This is a debug-only option used by the XtraBackup test suite.", + (uchar*) &opt_debug_sleep_before_unlock, + (uchar*) &opt_debug_sleep_before_unlock, 0, GET_UINT, + REQUIRED_ARG, 0, 0, 0, 0, 0, 0}, + + {"safe-slave-backup-timeout", OPT_SAFE_SLAVE_BACKUP_TIMEOUT, + "How many seconds --safe-slave-backup should wait for " + "Slave_open_temp_tables to become zero. (default 300)", + (uchar*) &opt_safe_slave_backup_timeout, + (uchar*) &opt_safe_slave_backup_timeout, 0, GET_UINT, + REQUIRED_ARG, 300, 0, 0, 0, 0, 0}, + + {"binlog-info", OPT_BINLOG_INFO, + "This option controls how XtraBackup should retrieve server's binary log " + "coordinates corresponding to the backup. Possible values are OFF, ON, " + "LOCKLESS and AUTO. See the XtraBackup manual for more information", + &opt_binlog_info, &opt_binlog_info, + &binlog_info_typelib, GET_ENUM, OPT_ARG, BINLOG_INFO_AUTO, 0, 0, 0, 0, 0}, + + {"secure-auth", OPT_XB_SECURE_AUTH, "Refuse client connecting to server if it" + " uses old (pre-4.1.1) protocol.", &opt_secure_auth, + &opt_secure_auth, 0, GET_BOOL, NO_ARG, 1, 0, 0, 0, 0, 0}, + +#include "sslopt-longopts.h" + +#if !defined(HAVE_YASSL) + {"server-public-key-path", OPT_SERVER_PUBLIC_KEY, + "File path to the server public RSA key in PEM format.", + &opt_server_public_key, &opt_server_public_key, 0, + GET_STR, REQUIRED_ARG, 0, 0, 0, 0, 0, 0}, +#endif + + { 0, 0, 0, 0, 0, 0, GET_NO_ARG, NO_ARG, 0, 0, 0, 0, 0, 0} +}; + +uint xb_client_options_count = array_elements(xb_client_options); + +struct my_option xb_server_options[] = +{ + {"datadir", 'h', "Path to the database root.", (G_PTR*) &mysql_data_home, + (G_PTR*) &mysql_data_home, 0, GET_STR, REQUIRED_ARG, 0, 0, 0, 0, 0, 0}, + {"tmpdir", 't', + "Path for temporary files. Several paths may be specified, separated by a " +#if defined(__WIN__) || defined(OS2) || defined(__NETWARE__) + "semicolon (;)" +#else + "colon (:)" +#endif + ", in this case they are used in a round-robin fashion.", + (G_PTR*) &opt_mysql_tmpdir, + (G_PTR*) &opt_mysql_tmpdir, 0, GET_STR, REQUIRED_ARG, 0, 0, 0, 0, 0, 0}, + {"parallel", OPT_XTRA_PARALLEL, + "Number of threads to use for parallel datafiles transfer. Does not have " + "any effect in the stream mode. The default value is 1.", + (G_PTR*) &xtrabackup_parallel, (G_PTR*) &xtrabackup_parallel, 0, GET_INT, + REQUIRED_ARG, 1, 1, INT_MAX, 0, 0, 0}, + + {"log", OPT_LOG, "Ignored option for MySQL option compatibility", + (G_PTR*) &log_ignored_opt, (G_PTR*) &log_ignored_opt, 0, + GET_STR, OPT_ARG, 0, 0, 0, 0, 0, 0}, + + {"log_bin", OPT_LOG, "Base name for the log sequence", + &opt_log_bin, &opt_log_bin, 0, GET_STR_ALLOC, OPT_ARG, 0, 0, 0, 0, 0, 0}, + + {"innodb", OPT_INNODB, "Ignored option for MySQL option compatibility", + (G_PTR*) &innobase_ignored_opt, (G_PTR*) &innobase_ignored_opt, 0, + GET_STR, OPT_ARG, 0, 0, 0, 0, 0, 0}, + + {"innodb_adaptive_hash_index", OPT_INNODB_ADAPTIVE_HASH_INDEX, + "Enable InnoDB adaptive hash index (enabled by default). " + "Disable with --skip-innodb-adaptive-hash-index.", + (G_PTR*) &innobase_adaptive_hash_index, + (G_PTR*) &innobase_adaptive_hash_index, + 0, GET_BOOL, NO_ARG, 1, 0, 0, 0, 0, 0}, + {"innodb_additional_mem_pool_size", OPT_INNODB_ADDITIONAL_MEM_POOL_SIZE, + "Size of a memory pool InnoDB uses to store data dictionary information and other internal data structures.", + (G_PTR*) &innobase_additional_mem_pool_size, + (G_PTR*) &innobase_additional_mem_pool_size, 0, GET_LONG, REQUIRED_ARG, + 1*1024*1024L, 512*1024L, LONG_MAX, 0, 1024, 0}, + {"innodb_autoextend_increment", OPT_INNODB_AUTOEXTEND_INCREMENT, + "Data file autoextend increment in megabytes", + (G_PTR*) &srv_auto_extend_increment, + (G_PTR*) &srv_auto_extend_increment, + 0, GET_ULONG, REQUIRED_ARG, 8L, 1L, 1000L, 0, 1L, 0}, + {"innodb_buffer_pool_size", OPT_INNODB_BUFFER_POOL_SIZE, + "The size of the memory buffer InnoDB uses to cache data and indexes of its tables.", + (G_PTR*) &innobase_buffer_pool_size, (G_PTR*) &innobase_buffer_pool_size, 0, + GET_LL, REQUIRED_ARG, 8*1024*1024L, 1024*1024L, LONGLONG_MAX, 0, + 1024*1024L, 0}, + {"innodb_checksums", OPT_INNODB_CHECKSUMS, "Enable InnoDB checksums validation (enabled by default). \ +Disable with --skip-innodb-checksums.", (G_PTR*) &innobase_use_checksums, + (G_PTR*) &innobase_use_checksums, 0, GET_BOOL, NO_ARG, 1, 0, 0, 0, 0, 0}, + {"innodb_data_file_path", OPT_INNODB_DATA_FILE_PATH, + "Path to individual files and their sizes.", &innobase_data_file_path, + &innobase_data_file_path, 0, GET_STR_ALLOC, REQUIRED_ARG, 0, 0, 0, 0, 0, 0}, + {"innodb_data_home_dir", OPT_INNODB_DATA_HOME_DIR, + "The common part for InnoDB table spaces.", &innobase_data_home_dir, + &innobase_data_home_dir, 0, GET_STR_ALLOC, REQUIRED_ARG, 0, 0, 0, 0, 0, 0}, + {"innodb_doublewrite", OPT_INNODB_DOUBLEWRITE, "Enable InnoDB doublewrite buffer (enabled by default). \ +Disable with --skip-innodb-doublewrite.", (G_PTR*) &innobase_use_doublewrite, + (G_PTR*) &innobase_use_doublewrite, 0, GET_BOOL, NO_ARG, 1, 0, 0, 0, 0, 0}, + {"innodb_io_capacity", OPT_INNODB_IO_CAPACITY, + "Number of IOPs the server can do. Tunes the background IO rate", + (G_PTR*) &srv_io_capacity, (G_PTR*) &srv_io_capacity, + 0, GET_ULONG, OPT_ARG, 200, 100, ~0UL, 0, 0, 0}, + {"innodb_file_io_threads", OPT_INNODB_FILE_IO_THREADS, + "Number of file I/O threads in InnoDB.", (G_PTR*) &innobase_file_io_threads, + (G_PTR*) &innobase_file_io_threads, 0, GET_LONG, REQUIRED_ARG, 4, 4, 64, 0, + 1, 0}, + {"innodb_read_io_threads", OPT_INNODB_READ_IO_THREADS, + "Number of background read I/O threads in InnoDB.", (G_PTR*) &innobase_read_io_threads, + (G_PTR*) &innobase_read_io_threads, 0, GET_LONG, REQUIRED_ARG, 4, 1, 64, 0, + 1, 0}, + {"innodb_write_io_threads", OPT_INNODB_WRITE_IO_THREADS, + "Number of background write I/O threads in InnoDB.", (G_PTR*) &innobase_write_io_threads, + (G_PTR*) &innobase_write_io_threads, 0, GET_LONG, REQUIRED_ARG, 4, 1, 64, 0, + 1, 0}, + {"innodb_file_per_table", OPT_INNODB_FILE_PER_TABLE, + "Stores each InnoDB table to an .ibd file in the database dir.", + (G_PTR*) &innobase_file_per_table, + (G_PTR*) &innobase_file_per_table, 0, GET_BOOL, NO_ARG, + FALSE, 0, 0, 0, 0, 0}, + {"innodb_flush_log_at_trx_commit", OPT_INNODB_FLUSH_LOG_AT_TRX_COMMIT, + "Set to 0 (write and flush once per second), 1 (write and flush at each commit) or 2 (write at commit, flush once per second).", + (G_PTR*) &srv_flush_log_at_trx_commit, + (G_PTR*) &srv_flush_log_at_trx_commit, + 0, GET_ULONG, OPT_ARG, 1, 0, 2, 0, 0, 0}, + {"innodb_flush_method", OPT_INNODB_FLUSH_METHOD, + "With which method to flush data.", (G_PTR*) &innobase_unix_file_flush_method, + (G_PTR*) &innobase_unix_file_flush_method, 0, GET_STR, REQUIRED_ARG, 0, 0, 0, + 0, 0, 0}, + +/* ####### Should we use this option? ####### */ + {"innodb_force_recovery", OPT_INNODB_FORCE_RECOVERY, + "Helps to save your data in case the disk image of the database becomes corrupt.", + (G_PTR*) &innobase_force_recovery, (G_PTR*) &innobase_force_recovery, 0, + GET_LONG, REQUIRED_ARG, 0, 0, 6, 0, 1, 0}, + + {"innodb_log_arch_dir", OPT_INNODB_LOG_ARCH_DIR, + "Where full logs should be archived.", (G_PTR*) &innobase_log_arch_dir, + (G_PTR*) &innobase_log_arch_dir, 0, GET_STR, REQUIRED_ARG, 0, 0, 0, 0, 0, 0}, + {"innodb_log_buffer_size", OPT_INNODB_LOG_BUFFER_SIZE, + "The size of the buffer which InnoDB uses to write log to the log files on disk.", + (G_PTR*) &innobase_log_buffer_size, (G_PTR*) &innobase_log_buffer_size, 0, + GET_LONG, REQUIRED_ARG, 1024*1024L, 256*1024L, LONG_MAX, 0, 1024, 0}, + {"innodb_log_file_size", OPT_INNODB_LOG_FILE_SIZE, + "Size of each log file in a log group.", + (G_PTR*) &innobase_log_file_size, (G_PTR*) &innobase_log_file_size, 0, + GET_LL, REQUIRED_ARG, 48*1024*1024L, 1*1024*1024L, LONGLONG_MAX, 0, + 1024*1024L, 0}, + {"innodb_log_files_in_group", OPT_INNODB_LOG_FILES_IN_GROUP, + "Number of log files in the log group. InnoDB writes to the files in a " + "circular fashion. Value 3 is recommended here.", + &innobase_log_files_in_group, &innobase_log_files_in_group, + 0, GET_LONG, REQUIRED_ARG, 2, 2, 100, 0, 1, 0}, + {"innodb_log_group_home_dir", OPT_INNODB_LOG_GROUP_HOME_DIR, + "Path to InnoDB log files.", &srv_log_group_home_dir, + &srv_log_group_home_dir, 0, GET_STR_ALLOC, REQUIRED_ARG, 0, 0, 0, 0, 0, 0}, + {"innodb_max_dirty_pages_pct", OPT_INNODB_MAX_DIRTY_PAGES_PCT, + "Percentage of dirty pages allowed in bufferpool.", (G_PTR*) &srv_max_buf_pool_modified_pct, + (G_PTR*) &srv_max_buf_pool_modified_pct, 0, GET_ULONG, REQUIRED_ARG, 90, 0, 100, 0, 0, 0}, + {"innodb_open_files", OPT_INNODB_OPEN_FILES, + "How many files at the maximum InnoDB keeps open at the same time.", + (G_PTR*) &innobase_open_files, (G_PTR*) &innobase_open_files, 0, + GET_LONG, REQUIRED_ARG, 300L, 10L, LONG_MAX, 0, 1L, 0}, + {"innodb_use_native_aio", OPT_INNODB_USE_NATIVE_AIO, + "Use native AIO if supported on this platform.", + (G_PTR*) &srv_use_native_aio, + (G_PTR*) &srv_use_native_aio, 0, GET_BOOL, NO_ARG, + FALSE, 0, 0, 0, 0, 0}, + {"innodb_page_size", OPT_INNODB_PAGE_SIZE, + "The universal page size of the database.", + (G_PTR*) &innobase_page_size, (G_PTR*) &innobase_page_size, 0, + /* Use GET_LL to support numeric suffixes in 5.6 */ + GET_LL, REQUIRED_ARG, + (1LL << 14), (1LL << 12), (1LL << UNIV_PAGE_SIZE_SHIFT_MAX), 0, 1L, 0}, + {"innodb_log_block_size", OPT_INNODB_LOG_BLOCK_SIZE, + "The log block size of the transaction log file. " + "Changing for created log file is not supported. Use on your own risk!", + (G_PTR*) &innobase_log_block_size, (G_PTR*) &innobase_log_block_size, 0, + GET_ULONG, REQUIRED_ARG, 512, 512, 1 << UNIV_PAGE_SIZE_SHIFT_MAX, 0, 1L, 0}, + {"innodb_fast_checksum", OPT_INNODB_FAST_CHECKSUM, + "Change the algorithm of checksum for the whole of datapage to 4-bytes word based.", + (G_PTR*) &innobase_fast_checksum, + (G_PTR*) &innobase_fast_checksum, 0, GET_BOOL, NO_ARG, 0, 0, 0, 0, 0, 0}, + {"innodb_doublewrite_file", OPT_INNODB_DOUBLEWRITE_FILE, + "Path to special datafile for doublewrite buffer. (default is "": not used)", + (G_PTR*) &innobase_doublewrite_file, (G_PTR*) &innobase_doublewrite_file, + 0, GET_STR, REQUIRED_ARG, 0, 0, 0, 0, 0, 0}, + {"innodb_buffer_pool_filename", OPT_INNODB_BUFFER_POOL_FILENAME, + "Filename to/from which to dump/load the InnoDB buffer pool", + (G_PTR*) &innobase_buffer_pool_filename, + (G_PTR*) &innobase_buffer_pool_filename, + 0, GET_STR, REQUIRED_ARG, 0, 0, 0, 0, 0, 0}, + +#ifndef __WIN__ + {"debug-sync", OPT_XTRA_DEBUG_SYNC, + "Debug sync point. This is only used by the xtrabackup test suite", + (G_PTR*) &xtrabackup_debug_sync, + (G_PTR*) &xtrabackup_debug_sync, + 0, GET_STR, REQUIRED_ARG, 0, 0, 0, 0, 0, 0}, +#endif + + {"innodb_checksum_algorithm", OPT_INNODB_CHECKSUM_ALGORITHM, + "The algorithm InnoDB uses for page checksumming. [CRC32, STRICT_CRC32, " + "INNODB, STRICT_INNODB, NONE, STRICT_NONE]", &srv_checksum_algorithm, + &srv_checksum_algorithm, &innodb_checksum_algorithm_typelib, GET_ENUM, + REQUIRED_ARG, SRV_CHECKSUM_ALGORITHM_INNODB, 0, 0, 0, 0, 0}, + {"innodb_log_checksum_algorithm", OPT_INNODB_LOG_CHECKSUM_ALGORITHM, + "The algorithm InnoDB uses for log checksumming. [CRC32, STRICT_CRC32, " + "INNODB, STRICT_INNODB, NONE, STRICT_NONE]", &srv_log_checksum_algorithm, + &srv_log_checksum_algorithm, &innodb_checksum_algorithm_typelib, GET_ENUM, + REQUIRED_ARG, SRV_CHECKSUM_ALGORITHM_INNODB, 0, 0, 0, 0, 0}, + {"innodb_undo_directory", OPT_INNODB_UNDO_DIRECTORY, + "Directory where undo tablespace files live, this path can be absolute.", + &srv_undo_dir, &srv_undo_dir, 0, GET_STR_ALLOC, REQUIRED_ARG, 0, 0, 0, 0, 0, + 0}, + + {"innodb_undo_tablespaces", OPT_INNODB_UNDO_TABLESPACES, + "Number of undo tablespaces to use.", + (G_PTR*)&srv_undo_tablespaces, (G_PTR*)&srv_undo_tablespaces, + 0, GET_ULONG, REQUIRED_ARG, 0, 0, 126, 0, 1, 0}, + + {"defaults_group", OPT_DEFAULTS_GROUP, "defaults group in config file (default \"mysqld\").", + (G_PTR*) &defaults_group, (G_PTR*) &defaults_group, + 0, GET_STR, REQUIRED_ARG, 0, 0, 0, 0, 0, 0}, + + {"open_files_limit", OPT_OPEN_FILES_LIMIT, "the maximum number of file " + "descriptors to reserve with setrlimit().", + (G_PTR*) &xb_open_files_limit, (G_PTR*) &xb_open_files_limit, 0, GET_ULONG, + REQUIRED_ARG, 0, 0, UINT_MAX, 0, 1, 0}, + + { 0, 0, 0, 0, 0, 0, GET_NO_ARG, NO_ARG, 0, 0, 0, 0, 0, 0} +}; + +uint xb_server_options_count = array_elements(xb_server_options); + +#ifndef __WIN__ +static int debug_sync_resumed; + +static void sigcont_handler(int sig); + +static void sigcont_handler(int sig __attribute__((unused))) +{ + debug_sync_resumed= 1; +} +#endif + +static inline +void +debug_sync_point(const char *name) +{ +#ifndef __WIN__ + FILE *fp; + pid_t pid; + char pid_path[FN_REFLEN]; + + if (xtrabackup_debug_sync == NULL) { + return; + } + + if (strcmp(xtrabackup_debug_sync, name)) { + return; + } + + pid = getpid(); + + snprintf(pid_path, sizeof(pid_path), "%s/xtrabackup_debug_sync", + xtrabackup_target_dir); + fp = fopen(pid_path, "w"); + if (fp == NULL) { + msg("xtrabackup: Error: cannot open %s\n", pid_path); + exit(EXIT_FAILURE); + } + fprintf(fp, "%u\n", (uint) pid); + fclose(fp); + + msg("xtrabackup: DEBUG: Suspending at debug sync point '%s'. " + "Resume with 'kill -SIGCONT %u'.\n", name, (uint) pid); + + debug_sync_resumed= 0; + kill(pid, SIGSTOP); + while (!debug_sync_resumed) { + sleep(1); + } + + /* On resume */ + msg("xtrabackup: DEBUG: removing the pid file.\n"); + my_delete(pid_path, MYF(MY_WME)); +#endif +} + +static const char *xb_client_default_groups[]= + { "xtrabackup", "client", 0, 0, 0 }; + +static const char *xb_server_default_groups[]= + { "xtrabackup", "mysqld", 0, 0, 0 }; + +static void print_version(void) +{ + msg("%s version %s based on MySQL server %s %s (%s) (revision id: %s)\n", + my_progname, XTRABACKUP_VERSION, MYSQL_SERVER_VERSION, SYSTEM_TYPE, + MACHINE_TYPE, XTRABACKUP_REVISION); +} + +static void usage(void) +{ + puts("Open source backup tool for InnoDB and XtraDB\n\ +\n\ +Copyright (C) 2009-2015 Percona LLC and/or its affiliates.\n\ +Portions Copyright (C) 2000, 2011, MySQL AB & Innobase Oy. All Rights Reserved.\n\ +\n\ +This program is free software; you can redistribute it and/or\n\ +modify it under the terms of the GNU General Public License\n\ +as published by the Free Software Foundation version 2\n\ +of the License.\n\ +\n\ +This program is distributed in the hope that it will be useful,\n\ +but WITHOUT ANY WARRANTY; without even the implied warranty of\n\ +MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the\n\ +GNU General Public License for more details.\n\ +\n\ +You can download full text of the license on http://www.gnu.org/licenses/gpl-2.0.txt\n"); + + printf("Usage: [%s [--defaults-file=#] --backup | %s [--defaults-file=#] --prepare] [OPTIONS]\n",my_progname,my_progname); + print_defaults("my", xb_server_default_groups); + my_print_help(xb_client_options); + my_print_help(xb_server_options); + my_print_variables(xb_server_options); + my_print_variables(xb_client_options); +} + +#define ADD_PRINT_PARAM_OPT(value) \ + { \ + print_param_str << opt->name << "=" << value << "\n"; \ + param_set.insert(opt->name); \ + } + +/************************************************************************ +Check if parameter is set in defaults file or via command line argument +@return true if parameter is set. */ +bool +check_if_param_set(const char *param) +{ + return param_set.find(param) != param_set.end(); +} + +my_bool +xb_get_one_option(int optid, + const struct my_option *opt __attribute__((unused)), + char *argument) +{ + switch(optid) { + case 'h': + strmake(mysql_real_data_home,argument, FN_REFLEN - 1); + mysql_data_home= mysql_real_data_home; + + ADD_PRINT_PARAM_OPT(mysql_real_data_home); + break; + + case 't': + + ADD_PRINT_PARAM_OPT(opt_mysql_tmpdir); + break; + + case OPT_INNODB_DATA_HOME_DIR: + + ADD_PRINT_PARAM_OPT(innobase_data_home_dir); + break; + + case OPT_INNODB_DATA_FILE_PATH: + + ADD_PRINT_PARAM_OPT(innobase_data_file_path); + break; + + case OPT_INNODB_LOG_GROUP_HOME_DIR: + + ADD_PRINT_PARAM_OPT(srv_log_group_home_dir); + break; + + case OPT_INNODB_LOG_FILES_IN_GROUP: + + ADD_PRINT_PARAM_OPT(innobase_log_files_in_group); + break; + + case OPT_INNODB_LOG_FILE_SIZE: + + ADD_PRINT_PARAM_OPT(innobase_log_file_size); + break; + + case OPT_INNODB_FLUSH_METHOD: + + ADD_PRINT_PARAM_OPT(innobase_unix_file_flush_method); + break; + + case OPT_INNODB_PAGE_SIZE: + + ADD_PRINT_PARAM_OPT(innobase_page_size); + break; + + case OPT_INNODB_FAST_CHECKSUM: + + ADD_PRINT_PARAM_OPT(!!innobase_fast_checksum); + break; + + case OPT_INNODB_LOG_BLOCK_SIZE: + + ADD_PRINT_PARAM_OPT(innobase_log_block_size); + break; + + case OPT_INNODB_DOUBLEWRITE_FILE: + + ADD_PRINT_PARAM_OPT(innobase_doublewrite_file); + break; + + case OPT_INNODB_UNDO_DIRECTORY: + + ADD_PRINT_PARAM_OPT(srv_undo_dir); + break; + + case OPT_INNODB_UNDO_TABLESPACES: + + ADD_PRINT_PARAM_OPT(srv_undo_tablespaces); + break; + + case OPT_INNODB_CHECKSUM_ALGORITHM: + + ut_a(srv_checksum_algorithm <= SRV_CHECKSUM_ALGORITHM_STRICT_NONE); + + ADD_PRINT_PARAM_OPT(innodb_checksum_algorithm_names[srv_checksum_algorithm]); + break; + + case OPT_INNODB_LOG_CHECKSUM_ALGORITHM: + + ut_a(srv_log_checksum_algorithm <= SRV_CHECKSUM_ALGORITHM_STRICT_NONE); + + ADD_PRINT_PARAM_OPT(innodb_checksum_algorithm_names[srv_log_checksum_algorithm]); + break; + + case OPT_INNODB_BUFFER_POOL_FILENAME: + + ADD_PRINT_PARAM_OPT(innobase_buffer_pool_filename); + break; + + case OPT_XTRA_TARGET_DIR: + strmake(xtrabackup_real_target_dir,argument, sizeof(xtrabackup_real_target_dir)-1); + xtrabackup_target_dir= xtrabackup_real_target_dir; + break; + case OPT_XTRA_STREAM: + if (!strcasecmp(argument, "tar")) + xtrabackup_stream_fmt = XB_STREAM_FMT_TAR; + else if (!strcasecmp(argument, "xbstream")) + xtrabackup_stream_fmt = XB_STREAM_FMT_XBSTREAM; + else + { + msg("Invalid --stream argument: %s\n", argument); + return 1; + } + xtrabackup_stream = TRUE; + break; + case OPT_XTRA_COMPRESS: + if (argument == NULL) + xtrabackup_compress_alg = "quicklz"; + else if (strcasecmp(argument, "quicklz")) + { + msg("Invalid --compress argument: %s\n", argument); + return 1; + } + xtrabackup_compress = TRUE; + break; + case OPT_XTRA_ENCRYPT: + if (argument == NULL) + { + msg("Missing --encrypt argument, must specify a valid encryption " + " algorithm.\n"); + return 1; + } + xtrabackup_encrypt = TRUE; + break; + case OPT_DECRYPT: + if (argument == NULL) { + msg("Missing --decrypt argument, must specify a " + "valid encryption algorithm.\n"); + return(1); + } + opt_decrypt = TRUE; + xtrabackup_decrypt_decompress = true; + break; + case OPT_DECOMPRESS: + opt_decompress = TRUE; + xtrabackup_decrypt_decompress = true; + break; + case (int) OPT_CORE_FILE: + test_flags |= TEST_CORE_ON_SIGNAL; + break; + case OPT_HISTORY: + if (argument) { + opt_history = argument; + } else { + opt_history = ""; + } + break; + case 'p': + if (argument) + { + char *start= argument; + my_free(opt_password); + opt_password= my_strdup(argument, MYF(MY_FAE)); + while (*argument) *argument++= 'x'; // Destroy argument + if (*start) + start[1]=0 ; + } + break; + + +#include "sslopt-case.h" + + case '?': + usage(); + exit(EXIT_SUCCESS); + break; + case 'v': + print_version(); + exit(EXIT_SUCCESS); + break; + default: + break; + } + return 0; +} + +/*********************************************************************** +Initializes log_block_size */ +static +ibool +xb_init_log_block_size(void) +{ + srv_log_block_size = 0; + if (innobase_log_block_size != 512) { + uint n_shift = get_bit_shift(innobase_log_block_size);; + + if (n_shift > 0) { + srv_log_block_size = (1 << n_shift); + msg("InnoDB: The log block size is set to %lu.\n", + srv_log_block_size); + } + } else { + srv_log_block_size = 512; + } + if (!srv_log_block_size) { + msg("InnoDB: Error: %lu is not valid value for " + "innodb_log_block_size.\n", innobase_log_block_size); + return FALSE; + } + + return TRUE; +} + +static my_bool +innodb_init_param(void) +{ + /* innobase_init */ + static char current_dir[3]; /* Set if using current lib */ + my_bool ret; + char *default_path; + + /* === some variables from mysqld === */ + memset((G_PTR) &mysql_tmpdir_list, 0, sizeof(mysql_tmpdir_list)); + + if (init_tmpdir(&mysql_tmpdir_list, opt_mysql_tmpdir)) + exit(EXIT_FAILURE); + + /* dummy for initialize all_charsets[] */ + get_charset_name(0); + + srv_page_size = 0; + srv_page_size_shift = 0; + + if (innobase_page_size != (1LL << 14)) { + int n_shift = get_bit_shift((ulint) innobase_page_size); + + if (n_shift >= 12 && n_shift <= UNIV_PAGE_SIZE_SHIFT_MAX) { + srv_page_size_shift = n_shift; + srv_page_size = 1 << n_shift; + msg("InnoDB: The universal page size of the " + "database is set to %lu.\n", srv_page_size); + } else { + msg("InnoDB: Error: invalid value of " + "innobase_page_size: %lld", innobase_page_size); + exit(EXIT_FAILURE); + } + } else { + srv_page_size_shift = 14; + srv_page_size = (1 << srv_page_size_shift); + } + + if (!xb_init_log_block_size()) { + goto error; + } + + srv_fast_checksum = (ibool) innobase_fast_checksum; + + /* Check that values don't overflow on 32-bit systems. */ + if (sizeof(ulint) == 4) { + if (xtrabackup_use_memory > UINT_MAX32) { + msg("xtrabackup: use-memory can't be over 4GB" + " on 32-bit systems\n"); + } + + if (innobase_buffer_pool_size > UINT_MAX32) { + msg("xtrabackup: innobase_buffer_pool_size can't be " + "over 4GB on 32-bit systems\n"); + + goto error; + } + + if (innobase_log_file_size > UINT_MAX32) { + msg("xtrabackup: innobase_log_file_size can't be " + "over 4GB on 32-bit systemsi\n"); + + goto error; + } + } + + os_innodb_umask = (ulint)0664; + + /* First calculate the default path for innodb_data_home_dir etc., + in case the user has not given any value. + + Note that when using the embedded server, the datadirectory is not + necessarily the current directory of this program. */ + + /* It's better to use current lib, to keep paths short */ + current_dir[0] = FN_CURLIB; + current_dir[1] = FN_LIBCHAR; + current_dir[2] = 0; + default_path = current_dir; + + ut_a(default_path); + + /* Set InnoDB initialization parameters according to the values + read from MySQL .cnf file */ + + if (xtrabackup_backup || xtrabackup_stats) { + msg("xtrabackup: using the following InnoDB configuration:\n"); + } else { + msg("xtrabackup: using the following InnoDB configuration " + "for recovery:\n"); + } + + /*--------------- Data files -------------------------*/ + + /* The default dir for data files is the datadir of MySQL */ + + srv_data_home = ((xtrabackup_backup || xtrabackup_stats) && innobase_data_home_dir + ? innobase_data_home_dir : default_path); + msg("xtrabackup: innodb_data_home_dir = %s\n", srv_data_home); + + /* Set default InnoDB data file size to 10 MB and let it be + auto-extending. Thus users can use InnoDB in >= 4.0 without having + to specify any startup options. */ + + if (!innobase_data_file_path) { + innobase_data_file_path = (char*) "ibdata1:10M:autoextend"; + } + msg("xtrabackup: innodb_data_file_path = %s\n", + innobase_data_file_path); + + /* Since InnoDB edits the argument in the next call, we make another + copy of it: */ + + internal_innobase_data_file_path = strdup(innobase_data_file_path); + + ret = (my_bool) srv_parse_data_file_paths_and_sizes( + internal_innobase_data_file_path); + if (ret == FALSE) { + msg("xtrabackup: syntax error in innodb_data_file_path\n"); +mem_free_and_error: + free(internal_innobase_data_file_path); + internal_innobase_data_file_path = NULL; + goto error; + } + + if (xtrabackup_prepare) { + /* "--prepare" needs filenames only */ + ulint i; + + for (i=0; i < srv_n_data_files; i++) { + char *p; + + p = srv_data_file_names[i]; + while ((p = strchr(p, SRV_PATH_SEPARATOR)) != NULL) + { + p++; + srv_data_file_names[i] = p; + } + } + } + + /* -------------- Log files ---------------------------*/ + + /* The default dir for log files is the datadir of MySQL */ + + if (!((xtrabackup_backup || xtrabackup_stats) && + srv_log_group_home_dir)) { + srv_log_group_home_dir = default_path; + } + if (xtrabackup_prepare && xtrabackup_incremental_dir) { + srv_log_group_home_dir = xtrabackup_incremental_dir; + } + msg("xtrabackup: innodb_log_group_home_dir = %s\n", + srv_log_group_home_dir); + + srv_normalize_path_for_win(srv_log_group_home_dir); + + if (strchr(srv_log_group_home_dir, ';')) { + + msg("syntax error in innodb_log_group_home_dir, "); + + goto mem_free_and_error; + } + + srv_adaptive_flushing = FALSE; + srv_use_sys_malloc = TRUE; + srv_file_format = 1; /* Barracuda */ + srv_max_file_format_at_startup = UNIV_FORMAT_MIN; /* on */ + /* --------------------------------------------------*/ + + srv_file_flush_method_str = innobase_unix_file_flush_method; + + srv_n_log_files = (ulint) innobase_log_files_in_group; + srv_log_file_size = (ulint) innobase_log_file_size; + msg("xtrabackup: innodb_log_files_in_group = %ld\n", + srv_n_log_files); + msg("xtrabackup: innodb_log_file_size = %lld\n", + (long long int) srv_log_file_size); + + srv_log_archive_on = (ulint) innobase_log_archive; + srv_log_buffer_size = (ulint) innobase_log_buffer_size; + + /* We set srv_pool_size here in units of 1 kB. InnoDB internally + changes the value so that it becomes the number of database pages. */ + + //srv_buf_pool_size = (ulint) innobase_buffer_pool_size; + srv_buf_pool_size = (ulint) xtrabackup_use_memory; + + srv_mem_pool_size = (ulint) innobase_additional_mem_pool_size; + + srv_n_file_io_threads = (ulint) innobase_file_io_threads; + srv_n_read_io_threads = (ulint) innobase_read_io_threads; + srv_n_write_io_threads = (ulint) innobase_write_io_threads; + + srv_force_recovery = (ulint) innobase_force_recovery; + + srv_use_doublewrite_buf = (ibool) innobase_use_doublewrite; + + if (!innobase_use_checksums) { + + srv_checksum_algorithm = SRV_CHECKSUM_ALGORITHM_NONE; + } + + btr_search_enabled = (char) innobase_adaptive_hash_index; + + os_use_large_pages = (ibool) innobase_use_large_pages; + os_large_page_size = (ulint) innobase_large_page_size; + + row_rollback_on_timeout = (ibool) innobase_rollback_on_timeout; + + srv_file_per_table = (my_bool) innobase_file_per_table; + + srv_locks_unsafe_for_binlog = (ibool) innobase_locks_unsafe_for_binlog; + + srv_max_n_open_files = (ulint) innobase_open_files; + srv_innodb_status = (ibool) innobase_create_status_file; + + srv_print_verbose_log = 1; + + /* Store the default charset-collation number of this MySQL + installation */ + + /* We cannot treat characterset here for now!! */ + data_mysql_default_charset_coll = (ulint)default_charset_info->number; + + ut_a(DATA_MYSQL_LATIN1_SWEDISH_CHARSET_COLL == + my_charset_latin1.number); + ut_a(DATA_MYSQL_BINARY_CHARSET_COLL == my_charset_bin.number); + + /* Store the latin1_swedish_ci character ordering table to InnoDB. For + non-latin1_swedish_ci charsets we use the MySQL comparison functions, + and consequently we do not need to know the ordering internally in + InnoDB. */ + + ut_a(0 == strcmp(my_charset_latin1.name, "latin1_swedish_ci")); + srv_latin1_ordering = my_charset_latin1.sort_order; + + //innobase_commit_concurrency_init_default(); + + /* Since we in this module access directly the fields of a trx + struct, and due to different headers and flags it might happen that + mutex_t has a different size in this module and in InnoDB + modules, we check at run time that the size is the same in + these compilation modules. */ + + /* On 5.5+ srv_use_native_aio is TRUE by default. It is later reset + if it is not supported by the platform in + innobase_start_or_create_for_mysql(). As we don't call it in xtrabackup, + we have to duplicate checks from that function here. */ + +#ifdef __WIN__ + switch (os_get_os_version()) { + case OS_WIN95: + case OS_WIN31: + case OS_WINNT: + /* On Win 95, 98, ME, Win32 subsystem for Windows 3.1, + and NT use simulated aio. In NT Windows provides async i/o, + but when run in conjunction with InnoDB Hot Backup, it seemed + to corrupt the data files. */ + + srv_use_native_aio = FALSE; + break; + + case OS_WIN2000: + case OS_WINXP: + /* On 2000 and XP, async IO is available. */ + srv_use_native_aio = TRUE; + break; + + default: + /* Vista and later have both async IO and condition variables */ + srv_use_native_aio = TRUE; + srv_use_native_conditions = TRUE; + break; + } + +#elif defined(LINUX_NATIVE_AIO) + + if (srv_use_native_aio) { + ut_print_timestamp(stderr); + msg(" InnoDB: Using Linux native AIO\n"); + } +#else + /* Currently native AIO is supported only on windows and linux + and that also when the support is compiled in. In all other + cases, we ignore the setting of innodb_use_native_aio. */ + srv_use_native_aio = FALSE; + +#endif + + /* Assign the default value to srv_undo_dir if it's not specified, as + my_getopt does not support default values for string options. We also + ignore the option and override innodb_undo_directory on --prepare, + because separate undo tablespaces are copied to the root backup + directory. */ + + if (!srv_undo_dir || !xtrabackup_backup) { + my_free(srv_undo_dir); + srv_undo_dir = my_strdup(".", MYF(MY_FAE)); + } + + innodb_log_checksum_func_update(srv_log_checksum_algorithm); + + return(FALSE); + +error: + msg("xtrabackup: innodb_init_param(): Error occured.\n"); + return(TRUE); +} + +static my_bool +innodb_init(void) +{ + int err; + + err = innobase_start_or_create_for_mysql(); + + if (err != DB_SUCCESS) { + free(internal_innobase_data_file_path); + internal_innobase_data_file_path = NULL; + goto error; + } + + /* They may not be needed for now */ +// (void) hash_init(&innobase_open_tables,system_charset_info, 32, 0, 0, +// (hash_get_key) innobase_get_key, 0, 0); +// pthread_mutex_init(&innobase_share_mutex, MY_MUTEX_INIT_FAST); +// pthread_mutex_init(&prepare_commit_mutex, MY_MUTEX_INIT_FAST); +// pthread_mutex_init(&commit_threads_m, MY_MUTEX_INIT_FAST); +// pthread_mutex_init(&commit_cond_m, MY_MUTEX_INIT_FAST); +// pthread_cond_init(&commit_cond, NULL); + + innodb_inited= 1; + + return(FALSE); + +error: + msg("xtrabackup: innodb_init(): Error occured.\n"); + return(TRUE); +} + +static my_bool +innodb_end(void) +{ + srv_fast_shutdown = (ulint) innobase_fast_shutdown; + innodb_inited = 0; + + msg("xtrabackup: starting shutdown with innodb_fast_shutdown = %lu\n", + srv_fast_shutdown); + + if (innobase_shutdown_for_mysql() != DB_SUCCESS) { + goto error; + } + free(internal_innobase_data_file_path); + internal_innobase_data_file_path = NULL; + + /* They may not be needed for now */ +// hash_free(&innobase_open_tables); +// pthread_mutex_destroy(&innobase_share_mutex); +// pthread_mutex_destroy(&prepare_commit_mutex); +// pthread_mutex_destroy(&commit_threads_m); +// pthread_mutex_destroy(&commit_cond_m); +// pthread_cond_destroy(&commit_cond); + + return(FALSE); + +error: + msg("xtrabackup: innodb_end(): Error occured.\n"); + return(TRUE); +} + +/* ================= common ================= */ + +/*********************************************************************** +Read backup meta info. +@return TRUE on success, FALSE on failure. */ +static +my_bool +xtrabackup_read_metadata(char *filename) +{ + FILE *fp; + my_bool r = TRUE; + int t; + + fp = fopen(filename,"r"); + if(!fp) { + msg("xtrabackup: Error: cannot open %s\n", filename); + return(FALSE); + } + + if (fscanf(fp, "backup_type = %29s\n", metadata_type) + != 1) { + r = FALSE; + goto end; + } + /* Use UINT64PF instead of LSN_PF here, as we have to maintain the file + format. */ + if (fscanf(fp, "from_lsn = " UINT64PF "\n", &metadata_from_lsn) + != 1) { + r = FALSE; + goto end; + } + if (fscanf(fp, "to_lsn = " UINT64PF "\n", &metadata_to_lsn) + != 1) { + r = FALSE; + goto end; + } + if (fscanf(fp, "last_lsn = " UINT64PF "\n", &metadata_last_lsn) + != 1) { + metadata_last_lsn = 0; + } + /* Optional fields */ + + if (fscanf(fp, "compact = %d\n", &t) == 1) { + xtrabackup_compact = (t == 1); + } else { + xtrabackup_compact = 0; + } + + if (fscanf(fp, "recover_binlog_info = %d\n", &t) == 1) { + recover_binlog_info = (t == 1); + } +end: + fclose(fp); + + return(r); +} + +/*********************************************************************** +Print backup meta info to a specified buffer. */ +static +void +xtrabackup_print_metadata(char *buf, size_t buf_len) +{ + /* Use UINT64PF instead of LSN_PF here, as we have to maintain the file + format. */ + snprintf(buf, buf_len, + "backup_type = %s\n" + "from_lsn = " UINT64PF "\n" + "to_lsn = " UINT64PF "\n" + "last_lsn = " UINT64PF "\n" + "compact = %d\n" + "recover_binlog_info = %d\n", + metadata_type, + metadata_from_lsn, + metadata_to_lsn, + metadata_last_lsn, + MY_TEST(xtrabackup_compact == TRUE), + MY_TEST(opt_binlog_info == BINLOG_INFO_LOCKLESS)); +} + +/*********************************************************************** +Stream backup meta info to a specified datasink. +@return TRUE on success, FALSE on failure. */ +static +my_bool +xtrabackup_stream_metadata(ds_ctxt_t *ds_ctxt) +{ + char buf[1024]; + size_t len; + ds_file_t *stream; + MY_STAT mystat; + my_bool rc = TRUE; + + xtrabackup_print_metadata(buf, sizeof(buf)); + + len = strlen(buf); + + mystat.st_size = len; + mystat.st_mtime = my_time(0); + + stream = ds_open(ds_ctxt, XTRABACKUP_METADATA_FILENAME, &mystat); + if (stream == NULL) { + msg("xtrabackup: Error: cannot open output stream " + "for %s\n", XTRABACKUP_METADATA_FILENAME); + return(FALSE); + } + + if (ds_write(stream, buf, len)) { + rc = FALSE; + } + + if (ds_close(stream)) { + rc = FALSE; + } + + return(rc); +} + +/*********************************************************************** +Write backup meta info to a specified file. +@return TRUE on success, FALSE on failure. */ +static +my_bool +xtrabackup_write_metadata(const char *filepath) +{ + char buf[1024]; + size_t len; + FILE *fp; + + xtrabackup_print_metadata(buf, sizeof(buf)); + + len = strlen(buf); + + fp = fopen(filepath, "w"); + if(!fp) { + msg("xtrabackup: Error: cannot open %s\n", filepath); + return(FALSE); + } + if (fwrite(buf, len, 1, fp) < 1) { + fclose(fp); + return(FALSE); + } + + fclose(fp); + + return(TRUE); +} + +/*********************************************************************** +Read meta info for an incremental delta. +@return TRUE on success, FALSE on failure. */ +static my_bool +xb_read_delta_metadata(const char *filepath, xb_delta_info_t *info) +{ + FILE* fp; + char key[51]; + char value[51]; + my_bool r = TRUE; + + /* set defaults */ + info->page_size = ULINT_UNDEFINED; + info->zip_size = ULINT_UNDEFINED; + info->space_id = ULINT_UNDEFINED; + + fp = fopen(filepath, "r"); + if (!fp) { + /* Meta files for incremental deltas are optional */ + return(TRUE); + } + + while (!feof(fp)) { + if (fscanf(fp, "%50s = %50s\n", key, value) == 2) { + if (strcmp(key, "page_size") == 0) { + info->page_size = strtoul(value, NULL, 10); + } else if (strcmp(key, "zip_size") == 0) { + info->zip_size = strtoul(value, NULL, 10); + } else if (strcmp(key, "space_id") == 0) { + info->space_id = strtoul(value, NULL, 10); + } + } + } + + fclose(fp); + + if (info->page_size == ULINT_UNDEFINED) { + msg("xtrabackup: page_size is required in %s\n", filepath); + r = FALSE; + } + if (info->space_id == ULINT_UNDEFINED) { + msg("xtrabackup: Warning: This backup was taken with XtraBackup 2.0.1 " + "or earlier, some DDL operations between full and incremental " + "backups may be handled incorrectly\n"); + } + + return(r); +} + +/*********************************************************************** +Write meta info for an incremental delta. +@return TRUE on success, FALSE on failure. */ +my_bool +xb_write_delta_metadata(const char *filename, const xb_delta_info_t *info) +{ + ds_file_t *f; + char buf[64]; + my_bool ret; + size_t len; + MY_STAT mystat; + + snprintf(buf, sizeof(buf), + "page_size = %lu\n" + "zip_size = %lu\n" + "space_id = %lu\n", + info->page_size, info->zip_size, info->space_id); + len = strlen(buf); + + mystat.st_size = len; + mystat.st_mtime = my_time(0); + + f = ds_open(ds_meta, filename, &mystat); + if (f == NULL) { + msg("xtrabackup: Error: cannot open output stream for %s\n", + filename); + return(FALSE); + } + + ret = (ds_write(f, buf, len) == 0); + + if (ds_close(f)) { + ret = FALSE; + } + + return(ret); +} + +/* ================= backup ================= */ +void +xtrabackup_io_throttling(void) +{ + if (xtrabackup_throttle && (io_ticket--) < 0) { + os_event_reset(wait_throttle); + os_event_wait(wait_throttle); + } +} + +/************************************************************************ +Checks if a given table name matches any of specifications in the --tables or +--tables-file options. + +@return TRUE on match. */ +static my_bool +check_if_table_matches_filters(const char *name) +{ + int regres; + xb_filter_entry_t* table; + xb_regex_list_node_t* node; + + if (UT_LIST_GET_LEN(regex_list)) { + /* Check against regular expressions list */ + for (node = UT_LIST_GET_FIRST(regex_list); node; + node = UT_LIST_GET_NEXT(regex_list, node)) { + regres = xb_regexec(&node->regex, name, 1, + tables_regmatch, 0); + if (regres != REG_NOMATCH) { + + return(TRUE); + } + } + } + + if (tables_hash) { + HASH_SEARCH(name_hash, tables_hash, ut_fold_string(name), + xb_filter_entry_t*, + table, (void) 0, + !strcmp(table->name, name)); + if (table) { + + return(TRUE); + } + } + + return(FALSE); +} + +/************************************************************************ +Checks if a table specified as a name in the form "database/name" (InnoDB 5.6) +or "./database/name.ibd" (InnoDB 5.5-) should be skipped from backup based on +the --tables or --tables-file options. + +@return TRUE if the table should be skipped. */ +my_bool +check_if_skip_table( +/******************/ + const char* name) /*!< in: path to the table */ +{ + char buf[FN_REFLEN]; + const char *dbname, *tbname; + const char *ptr; + char *eptr; + + if (UT_LIST_GET_LEN(regex_list) == 0 && + tables_hash == NULL && + databases_hash == NULL) { + return(FALSE); + } + + dbname = NULL; + tbname = name; + while ((ptr = strchr(tbname, SRV_PATH_SEPARATOR)) != NULL) { + dbname = tbname; + tbname = ptr + 1; + } + + if (dbname == NULL) { + return(FALSE); + } + + strncpy(buf, dbname, FN_REFLEN); + buf[tbname - 1 - dbname] = 0; + + if (databases_hash) { + /* There are some filters for databases, check them */ + xb_filter_entry_t* database; + + HASH_SEARCH(name_hash, databases_hash, ut_fold_string(buf), + xb_filter_entry_t*, + database, (void) 0, + !strcmp(database->name, buf)); + /* Table's database isn't found, skip the table */ + if (!database) { + return(TRUE); + } + /* There aren't tables specified for the database, + it should be backed up entirely */ + if (!database->has_tables) { + return(FALSE); + } + } + + buf[FN_REFLEN - 1] = '\0'; + buf[tbname - 1 - dbname] = '.'; + + /* Check if there's a suffix in the table name. If so, truncate it. We + rely on the fact that a dot cannot be a part of a table name (it is + encoded by the server with the @NNNN syntax). */ + if ((eptr = strchr(&buf[tbname - dbname], '.')) != NULL) { + + *eptr = '\0'; + } + + /* For partitioned tables first try to match against the regexp + without truncating the #P#... suffix so we can backup individual + partitions with regexps like '^test[.]t#P#p5' */ + if (check_if_table_matches_filters(buf)) { + + return(FALSE); + } + if ((eptr = strstr(buf, "#P#")) != NULL) { + + *eptr = 0; + + if (check_if_table_matches_filters(buf)) { + + return(FALSE); + } + } + + return(TRUE); +} + +/*********************************************************************** +Reads the space flags from a given data file and returns the compressed +page size, or 0 if the space is not compressed. */ +ulint +xb_get_zip_size(os_file_t file) +{ + byte *buf; + byte *page; + ulint zip_size = ULINT_UNDEFINED; + ibool success; + ulint space; + + buf = static_cast(ut_malloc(2 * UNIV_PAGE_SIZE_MAX)); + page = static_cast(ut_align(buf, UNIV_PAGE_SIZE_MAX)); + + success = os_file_read(file, page, 0, UNIV_PAGE_SIZE_MAX); + if (!success) { + goto end; + } + + space = mach_read_from_4(page + FIL_PAGE_ARCH_LOG_NO_OR_SPACE_ID); + zip_size = (space == 0 ) ? 0 : + dict_tf_get_zip_size(fsp_header_get_flags(page)); +end: + ut_free(buf); + + return(zip_size); +} + +const char* +xb_get_copy_action(const char *dflt) +{ + const char *action; + + if (xtrabackup_stream) { + if (xtrabackup_compress) { + if (xtrabackup_encrypt) { + action = "Compressing, encrypting and streaming"; + } else { + action = "Compressing and streaming"; + } + } else if (xtrabackup_encrypt) { + action = "Encrypting and streaming"; + } else { + action = "Streaming"; + } + } else { + if (xtrabackup_compress) { + if (xtrabackup_encrypt) { + action = "Compressing and encrypting"; + } else { + action = "Compressing"; + } + } else if (xtrabackup_encrypt) { + action = "Encrypting"; + } else { + action = dflt; + } + } + + return(action); +} + +/* TODO: We may tune the behavior (e.g. by fil_aio)*/ + +static +my_bool +xtrabackup_copy_datafile(fil_node_t* node, uint thread_n) +{ + char dst_name[FN_REFLEN]; + ds_file_t *dstfile = NULL; + xb_fil_cur_t cursor; + xb_fil_cur_result_t res; + xb_write_filt_t *write_filter = NULL; + xb_write_filt_ctxt_t write_filt_ctxt; + const char *action; + xb_read_filt_t *read_filter; + ibool is_system; + my_bool rc = FALSE; + + /* Get the name and the path for the tablespace. node->name always + contains the path (which may be absolute for remote tablespaces in + 5.6+). space->name contains the tablespace name in the form + "./database/table.ibd" (in 5.5-) or "database/table" (in 5.6+). For a + multi-node shared tablespace, space->name contains the name of the first + node, but that's irrelevant, since we only need node_name to match them + against filters, and the shared tablespace is always copied regardless + of the filters value. */ + + const char* const node_name = node->space->name; + const char* const node_path = node->name; + + is_system = !fil_is_user_tablespace_id(node->space->id); + + if (!is_system && check_if_skip_table(node_name)) { + msg("[%02u] Skipping %s.\n", thread_n, node_name); + return(FALSE); + } + + if (!changed_page_bitmap) { + read_filter = &rf_pass_through; + } + else { + read_filter = &rf_bitmap; + } + res = xb_fil_cur_open(&cursor, read_filter, node, thread_n); + if (res == XB_FIL_CUR_SKIP) { + goto skip; + } else if (res == XB_FIL_CUR_ERROR) { + goto error; + } + + strncpy(dst_name, cursor.rel_path, sizeof(dst_name)); + + /* Setup the page write filter */ + if (xtrabackup_incremental) { + write_filter = &wf_incremental; + } else if (xtrabackup_compact) { + write_filter = &wf_compact; + } else { + write_filter = &wf_write_through; + } + + memset(&write_filt_ctxt, 0, sizeof(xb_write_filt_ctxt_t)); + ut_a(write_filter->process != NULL); + + if (write_filter->init != NULL && + !write_filter->init(&write_filt_ctxt, dst_name, &cursor)) { + msg("[%02u] xtrabackup: error: " + "failed to initialize page write filter.\n", thread_n); + goto error; + } + + dstfile = ds_open(ds_data, dst_name, &cursor.statinfo); + if (dstfile == NULL) { + msg("[%02u] xtrabackup: error: " + "cannot open the destination stream for %s\n", + thread_n, dst_name); + goto error; + } + + action = xb_get_copy_action(); + + if (xtrabackup_stream) { + msg_ts("[%02u] %s %s\n", thread_n, action, node_path); + } else { + msg_ts("[%02u] %s %s to %s\n", thread_n, action, + node_path, dstfile->path); + } + + /* The main copy loop */ + while ((res = xb_fil_cur_read(&cursor)) == XB_FIL_CUR_SUCCESS) { + if (!write_filter->process(&write_filt_ctxt, dstfile)) { + goto error; + } + } + + if (res == XB_FIL_CUR_ERROR) { + goto error; + } + + if (write_filter->finalize + && !write_filter->finalize(&write_filt_ctxt, dstfile)) { + goto error; + } + + /* close */ + msg_ts("[%02u] ...done\n", thread_n); + xb_fil_cur_close(&cursor); + if (ds_close(dstfile)) { + rc = TRUE; + } + if (write_filter && write_filter->deinit) { + write_filter->deinit(&write_filt_ctxt); + } + return(rc); + +error: + xb_fil_cur_close(&cursor); + if (dstfile != NULL) { + ds_close(dstfile); + } + if (write_filter && write_filter->deinit) { + write_filter->deinit(&write_filt_ctxt);; + } + msg("[%02u] xtrabackup: Error: " + "xtrabackup_copy_datafile() failed.\n", thread_n); + return(TRUE); /*ERROR*/ + +skip: + + if (dstfile != NULL) { + ds_close(dstfile); + } + if (write_filter && write_filter->deinit) { + write_filter->deinit(&write_filt_ctxt); + } + msg("[%02u] xtrabackup: Warning: We assume the " + "table was dropped during xtrabackup execution " + "and ignore the file.\n", thread_n); + msg("[%02u] xtrabackup: Warning: skipping tablespace %s.\n", + thread_n, node_name); + return(FALSE); +} + +static +void +xtrabackup_choose_lsn_offset(lsn_t start_lsn) +{ + ulint no, alt_no, expected_no; + ulint blocks_in_group; + lsn_t tmp_offset, end_lsn; + int lsn_chosen = 0; + log_group_t *group; + + start_lsn = ut_uint64_align_down(start_lsn, OS_FILE_LOG_BLOCK_SIZE); + end_lsn = start_lsn + RECV_SCAN_SIZE; + + group = UT_LIST_GET_FIRST(log_sys->log_groups); + + if (mysql_server_version < 50500 || mysql_server_version > 50600) { + /* only make sense for Percona Server 5.5 */ + return; + } + + if (server_flavor == FLAVOR_PERCONA_SERVER) { + /* it is Percona Server 5.5 */ + group->alt_offset_chosen = true; + group->lsn_offset = group->lsn_offset_alt; + return; + } + + if (group->lsn_offset_alt == group->lsn_offset || + group->lsn_offset_alt == (lsn_t) -1) { + /* we have only one option */ + return; + } + + no = alt_no = (ulint) -1; + lsn_chosen = 0; + + blocks_in_group = log_block_convert_lsn_to_no( + log_group_get_capacity(group)) - 1; + + /* read log block number from usual offset */ + if (group->lsn_offset < group->file_size * group->n_files && + (log_group_calc_lsn_offset(start_lsn, group) % + UNIV_PAGE_SIZE) % OS_MIN_LOG_BLOCK_SIZE == 0) { + log_group_read_log_seg(LOG_RECOVER, log_sys->buf, + group, start_lsn, end_lsn); + no = log_block_get_hdr_no(log_sys->buf); + } + + /* read log block number from Percona Server 5.5 offset */ + tmp_offset = group->lsn_offset; + group->lsn_offset = group->lsn_offset_alt; + + if (group->lsn_offset < group->file_size * group->n_files && + (log_group_calc_lsn_offset(start_lsn, group) % + UNIV_PAGE_SIZE) % OS_MIN_LOG_BLOCK_SIZE == 0) { + log_group_read_log_seg(LOG_RECOVER, log_sys->buf, + group, start_lsn, end_lsn); + alt_no = log_block_get_hdr_no(log_sys->buf); + } + + expected_no = log_block_convert_lsn_to_no(start_lsn); + + ut_a(!(no == expected_no && alt_no == expected_no)); + + group->lsn_offset = tmp_offset; + + if ((no <= expected_no && + ((expected_no - no) % blocks_in_group) == 0) || + ((expected_no | 0x40000000UL) - no) % blocks_in_group == 0) { + /* default offset looks ok */ + ++lsn_chosen; + } + + if ((alt_no <= expected_no && + ((expected_no - alt_no) % blocks_in_group) == 0) || + ((expected_no | 0x40000000UL) - alt_no) % blocks_in_group == 0) { + /* PS 5.5 style offset looks ok */ + ++lsn_chosen; + group->alt_offset_chosen = true; + group->lsn_offset = group->lsn_offset_alt; + } + + /* We are in trouble, because we can not make a + decision to choose one over the other. Die just + like a Buridan's ass */ + ut_a(lsn_chosen == 1); +} + +/*******************************************************//** +Scans log from a buffer and writes new log data to the outpud datasinc. +@return true if success */ +static +bool +xtrabackup_scan_log_recs( +/*===============*/ + log_group_t* group, /*!< in: log group */ + bool is_last, /*!< in: whether it is last segment + to copy */ + lsn_t start_lsn, /*!< in: buffer start lsn */ + lsn_t* contiguous_lsn, /*!< in/out: it is known that all log + groups contain contiguous log data up + to this lsn */ + lsn_t* group_scanned_lsn,/*!< out: scanning succeeded up to + this lsn */ + bool* finished) /*!< out: false if is not able to scan + any more in this log group */ +{ + lsn_t scanned_lsn; + ulint data_len; + ulint write_size; + const byte* log_block; + + ulint scanned_checkpoint_no = 0; + + *finished = false; + scanned_lsn = start_lsn; + log_block = log_sys->buf; + + while (log_block < log_sys->buf + RECV_SCAN_SIZE && !*finished) { + ulint no = log_block_get_hdr_no(log_block); + ulint scanned_no = log_block_convert_lsn_to_no(scanned_lsn); + ibool checksum_is_ok = + log_block_checksum_is_ok_or_old_format(log_block); + + if (no != scanned_no && checksum_is_ok) { + ulint blocks_in_group; + + blocks_in_group = log_block_convert_lsn_to_no( + log_group_get_capacity(group)) - 1; + + if ((no < scanned_no && + ((scanned_no - no) % blocks_in_group) == 0) || + no == 0 || + /* Log block numbers wrap around at 0x3FFFFFFF */ + ((scanned_no | 0x40000000UL) - no) % + blocks_in_group == 0) { + + /* old log block, do nothing */ + *finished = true; + break; + } + + msg("xtrabackup: error:" + " log block numbers mismatch:\n" + "xtrabackup: error: expected log block no. %lu," + " but got no. %lu from the log file.\n", + (ulong) scanned_no, (ulong) no); + + if ((no - scanned_no) % blocks_in_group == 0) { + msg("xtrabackup: error:" + " it looks like InnoDB log has wrapped" + " around before xtrabackup could" + " process all records due to either" + " log copying being too slow, or " + " log files being too small.\n"); + } + + return(false); + } else if (!checksum_is_ok) { + /* Garbage or an incompletely written log block */ + + msg("xtrabackup: warning: Log block checksum mismatch" + " (block no %lu at lsn " LSN_PF "): \n" + "expected %lu, calculated checksum %lu\n", + (ulong) no, + scanned_lsn, + (ulong) log_block_get_checksum(log_block), + (ulong) log_block_calc_checksum(log_block)); + msg("xtrabackup: warning: this is possible when the " + "log block has not been fully written by the " + "server, will retry later.\n"); + *finished = true; + break; + } + + if (log_block_get_flush_bit(log_block)) { + /* This block was a start of a log flush operation: + we know that the previous flush operation must have + been completed for all log groups before this block + can have been flushed to any of the groups. Therefore, + we know that log data is contiguous up to scanned_lsn + in all non-corrupt log groups. */ + + if (scanned_lsn > *contiguous_lsn) { + + *contiguous_lsn = scanned_lsn; + } + } + + data_len = log_block_get_data_len(log_block); + + if ( + (scanned_checkpoint_no > 0) + && (log_block_get_checkpoint_no(log_block) + < scanned_checkpoint_no) + && (scanned_checkpoint_no + - log_block_get_checkpoint_no(log_block) + > 0x80000000UL)) { + + /* Garbage from a log buffer flush which was made + before the most recent database recovery */ + + *finished = true; + break; + } + + scanned_lsn = scanned_lsn + data_len; + scanned_checkpoint_no = log_block_get_checkpoint_no(log_block); + + if (data_len < OS_FILE_LOG_BLOCK_SIZE) { + /* Log data for this group ends here */ + + *finished = true; + } else { + log_block += OS_FILE_LOG_BLOCK_SIZE; + } + } + + *group_scanned_lsn = scanned_lsn; + + /* ===== write log to 'xtrabackup_logfile' ====== */ + if (!*finished) { + write_size = RECV_SCAN_SIZE; + } else { + write_size = ut_uint64_align_up(scanned_lsn, + OS_FILE_LOG_BLOCK_SIZE) - start_lsn; + if (!is_last && scanned_lsn % OS_FILE_LOG_BLOCK_SIZE) { + write_size -= OS_FILE_LOG_BLOCK_SIZE; + } + } + + if (ds_write(dst_log_file, log_sys->buf, write_size)) { + msg("xtrabackup: Error: " + "write to logfile failed\n"); + return(false); + } + + return(true); +} + +static my_bool +xtrabackup_copy_logfile(lsn_t from_lsn, my_bool is_last) +{ + /* definition from recv_recovery_from_checkpoint_start() */ + log_group_t* group; + lsn_t group_scanned_lsn; + lsn_t contiguous_lsn; + + ut_a(dst_log_file != NULL); + + /* read from checkpoint_lsn_start to current */ + contiguous_lsn = ut_uint64_align_down(from_lsn, OS_FILE_LOG_BLOCK_SIZE); + + /* TODO: We must check the contiguous_lsn still exists in log file.. */ + + group = UT_LIST_GET_FIRST(log_sys->log_groups); + + while (group) { + bool finished; + lsn_t start_lsn; + lsn_t end_lsn; + + /* reference recv_group_scan_log_recs() */ + finished = false; + + start_lsn = contiguous_lsn; + + while (!finished) { + + end_lsn = start_lsn + RECV_SCAN_SIZE; + + xtrabackup_io_throttling(); + + mutex_enter(&log_sys->mutex); + + log_group_read_log_seg(LOG_RECOVER, log_sys->buf, + group, start_lsn, end_lsn); + + if (!xtrabackup_scan_log_recs(group, is_last, + start_lsn, &contiguous_lsn, &group_scanned_lsn, + &finished)) { + goto error; + } + + mutex_exit(&log_sys->mutex); + + start_lsn = end_lsn; + + } + + group->scanned_lsn = group_scanned_lsn; + + msg_ts(">> log scanned up to (" LSN_PF ")\n", + group->scanned_lsn); + + group = UT_LIST_GET_NEXT(log_groups, group); + + /* update global variable*/ + log_copy_scanned_lsn = group_scanned_lsn; + + /* innodb_mirrored_log_groups must be 1, no other groups */ + ut_a(group == NULL); + + debug_sync_point("xtrabackup_copy_logfile_pause"); + + } + + + return(FALSE); + +error: + mutex_exit(&log_sys->mutex); + ds_close(dst_log_file); + msg("xtrabackup: Error: xtrabackup_copy_logfile() failed.\n"); + return(TRUE); +} + +static +#ifndef __WIN__ +void* +#else +ulint +#endif +log_copying_thread( + void* arg __attribute__((unused))) +{ + /* + Initialize mysys thread-specific memory so we can + use mysys functions in this thread. + */ + my_thread_init(); + + ut_a(dst_log_file != NULL); + + log_copying_running = TRUE; + + while(log_copying) { + os_event_reset(log_copying_stop); + os_event_wait_time_low(log_copying_stop, + xtrabackup_log_copy_interval * 1000ULL, + 0); + if (log_copying) { + if(xtrabackup_copy_logfile(log_copy_scanned_lsn, + FALSE)) { + + exit(EXIT_FAILURE); + } + } + } + + /* last copying */ + if(xtrabackup_copy_logfile(log_copy_scanned_lsn, TRUE)) { + + exit(EXIT_FAILURE); + } + + log_copying_running = FALSE; + my_thread_end(); + os_thread_exit(NULL); + + return(0); +} + +/* io throttle watching (rough) */ +static +#ifndef __WIN__ +void* +#else +ulint +#endif +io_watching_thread( + void* arg) +{ + (void)arg; + /* currently, for --backup only */ + ut_a(xtrabackup_backup); + + io_watching_thread_running = TRUE; + + while (log_copying) { + os_thread_sleep(1000000); /*1 sec*/ + io_ticket = xtrabackup_throttle; + os_event_set(wait_throttle); + } + + /* stop io throttle */ + xtrabackup_throttle = 0; + os_event_set(wait_throttle); + + io_watching_thread_running = FALSE; + + os_thread_exit(NULL); + + return(0); +} + +/************************************************************************ +I/o-handler thread function. */ +static + +#ifndef __WIN__ +void* +#else +ulint +#endif +io_handler_thread( +/*==============*/ + void* arg) +{ + ulint segment; + + + segment = *((ulint*)arg); + + while (srv_shutdown_state != SRV_SHUTDOWN_EXIT_THREADS) { + fil_aio_wait(segment); + } + + /* We count the number of threads in os_thread_exit(). A created + thread should always use that to exit and not use return() to exit. + The thread actually never comes here because it is exited in an + os_event_wait(). */ + + os_thread_exit(NULL); + +#ifndef __WIN__ + return(NULL); /* Not reached */ +#else + return(0); +#endif +} + +/************************************************************************** +Datafiles copying thread.*/ +static +os_thread_ret_t +data_copy_thread_func( +/*==================*/ + void *arg) /* thread context */ +{ + data_thread_ctxt_t *ctxt = (data_thread_ctxt_t *) arg; + uint num = ctxt->num; + fil_node_t* node; + + /* + Initialize mysys thread-specific memory so we can + use mysys functions in this thread. + */ + my_thread_init(); + + debug_sync_point("data_copy_thread_func"); + + while ((node = datafiles_iter_next(ctxt->it)) != NULL) { + + /* copy the datafile */ + if(xtrabackup_copy_datafile(node, num)) { + msg("[%02u] xtrabackup: Error: " + "failed to copy datafile.\n", num); + exit(EXIT_FAILURE); + } + } + + os_mutex_enter(ctxt->count_mutex); + (*ctxt->count)--; + os_mutex_exit(ctxt->count_mutex); + + my_thread_end(); + os_thread_exit(NULL); + OS_THREAD_DUMMY_RETURN; +} + +/************************************************************************ +Initialize the appropriate datasink(s). Both local backups and streaming in the +'xbstream' format allow parallel writes so we can write directly. + +Otherwise (i.e. when streaming in the 'tar' format) we need 2 separate datasinks +for the data stream (and don't allow parallel data copying) and for metainfo +files (including xtrabackup_logfile). The second datasink writes to temporary +files first, and then streams them in a serialized way when closed. */ +static void +xtrabackup_init_datasinks(void) +{ + if (xtrabackup_parallel > 1 && xtrabackup_stream && + xtrabackup_stream_fmt == XB_STREAM_FMT_TAR) { + msg("xtrabackup: warning: the --parallel option does not have " + "any effect when streaming in the 'tar' format. " + "You can use the 'xbstream' format instead.\n"); + xtrabackup_parallel = 1; + } + + /* Start building out the pipelines from the terminus back */ + if (xtrabackup_stream) { + /* All streaming goes to stdout */ + ds_data = ds_meta = ds_redo = ds_create(xtrabackup_target_dir, + DS_TYPE_STDOUT); + } else { + /* Local filesystem */ + ds_data = ds_meta = ds_redo = ds_create(xtrabackup_target_dir, + DS_TYPE_LOCAL); + } + + /* Track it for destruction */ + xtrabackup_add_datasink(ds_data); + + /* Stream formatting */ + if (xtrabackup_stream) { + ds_ctxt_t *ds; + if (xtrabackup_stream_fmt == XB_STREAM_FMT_TAR) { + ds = ds_create(xtrabackup_target_dir, DS_TYPE_ARCHIVE); + } else if (xtrabackup_stream_fmt == XB_STREAM_FMT_XBSTREAM) { + ds = ds_create(xtrabackup_target_dir, DS_TYPE_XBSTREAM); + } else { + /* bad juju... */ + ds = NULL; + } + + xtrabackup_add_datasink(ds); + + ds_set_pipe(ds, ds_data); + ds_data = ds; + + if (xtrabackup_stream_fmt != XB_STREAM_FMT_XBSTREAM) { + + /* 'tar' does not allow parallel streams */ + ds_redo = ds_meta = ds_create(xtrabackup_target_dir, + DS_TYPE_TMPFILE); + xtrabackup_add_datasink(ds_meta); + ds_set_pipe(ds_meta, ds); + } else { + ds_redo = ds_meta = ds_data; + } + } + + /* Encryption */ + if (xtrabackup_encrypt) { + ds_ctxt_t *ds; + + ds = ds_create(xtrabackup_target_dir, DS_TYPE_ENCRYPT); + xtrabackup_add_datasink(ds); + + ds_set_pipe(ds, ds_data); + if (ds_data != ds_meta) { + ds_data = ds; + ds = ds_create(xtrabackup_target_dir, DS_TYPE_ENCRYPT); + xtrabackup_add_datasink(ds); + + ds_set_pipe(ds, ds_meta); + ds_redo = ds_meta = ds; + } else { + ds_redo = ds_data = ds_meta = ds; + } + } + + /* Compression for ds_data and ds_redo */ + if (xtrabackup_compress) { + ds_ctxt_t *ds; + + /* Use a 1 MB buffer for compressed output stream */ + ds = ds_create(xtrabackup_target_dir, DS_TYPE_BUFFER); + ds_buffer_set_size(ds, 1024 * 1024); + xtrabackup_add_datasink(ds); + ds_set_pipe(ds, ds_data); + if (ds_data != ds_redo) { + ds_data = ds; + ds = ds_create(xtrabackup_target_dir, DS_TYPE_BUFFER); + ds_buffer_set_size(ds, 1024 * 1024); + xtrabackup_add_datasink(ds); + ds_set_pipe(ds, ds_redo); + ds_redo = ds; + } else { + ds_redo = ds_data = ds; + } + + ds = ds_create(xtrabackup_target_dir, DS_TYPE_COMPRESS); + xtrabackup_add_datasink(ds); + ds_set_pipe(ds, ds_data); + if (ds_data != ds_redo) { + ds_data = ds; + ds = ds_create(xtrabackup_target_dir, DS_TYPE_COMPRESS); + xtrabackup_add_datasink(ds); + ds_set_pipe(ds, ds_redo); + ds_redo = ds; + } else { + ds_redo = ds_data = ds; + } + } +} + +/************************************************************************ +Destroy datasinks. + +Destruction is done in the specific order to not violate their order in the +pipeline so that each datasink is able to flush data down the pipeline. */ +static void xtrabackup_destroy_datasinks(void) +{ + for (uint i = actual_datasinks; i > 0; i--) { + ds_destroy(datasinks[i-1]); + datasinks[i-1] = NULL; + } + ds_data = NULL; + ds_meta = NULL; + ds_redo = NULL; +} + +#define SRV_N_PENDING_IOS_PER_THREAD OS_AIO_N_PENDING_IOS_PER_THREAD +#define SRV_MAX_N_PENDING_SYNC_IOS 100 + +/************************************************************************ +@return TRUE if table should be opened. */ +static +ibool +xb_check_if_open_tablespace( + const char* db, + const char* table) +{ + char buf[FN_REFLEN]; + + snprintf(buf, sizeof(buf), "%s/%s", db, table); + + return !check_if_skip_table(buf); +} + +/************************************************************************ +Initializes the I/O and tablespace cache subsystems. */ +static +void +xb_fil_io_init(void) +/*================*/ +{ + srv_n_file_io_threads = srv_n_read_io_threads; + + os_aio_init(8 * SRV_N_PENDING_IOS_PER_THREAD, + srv_n_read_io_threads, + srv_n_write_io_threads, + SRV_MAX_N_PENDING_SYNC_IOS); + + fil_init(srv_file_per_table ? 50000 : 5000, LONG_MAX); + + fsp_init(); +} + +/**************************************************************************** +Populates the tablespace memory cache by scanning for and opening data files. +@returns DB_SUCCESS or error code.*/ +static +ulint +xb_load_tablespaces(void) +/*=====================*/ +{ + ulint i; + ibool create_new_db; + ulint err; + ulint sum_of_new_sizes; + + for (i = 0; i < srv_n_file_io_threads; i++) { + thread_nr[i] = i; + + os_thread_create(io_handler_thread, thread_nr + i, + thread_ids + i); + } + + os_thread_sleep(200000); /*0.2 sec*/ + + err = open_or_create_data_files(&create_new_db, + &min_flushed_lsn, &max_flushed_lsn, + &sum_of_new_sizes); + if (err != DB_SUCCESS) { + msg("xtrabackup: Could not open or create data files.\n" + "xtrabackup: If you tried to add new data files, and it " + "failed here,\n" + "xtrabackup: you should now edit innodb_data_file_path in " + "my.cnf back\n" + "xtrabackup: to what it was, and remove the new ibdata " + "files InnoDB created\n" + "xtrabackup: in this failed attempt. InnoDB only wrote " + "those files full of\n" + "xtrabackup: zeros, but did not yet use them in any way. " + "But be careful: do not\n" + "xtrabackup: remove old data files which contain your " + "precious data!\n"); + return(err); + } + + /* create_new_db must not be TRUE.. */ + if (create_new_db) { + msg("xtrabackup: could not find data files at the " + "specified datadir\n"); + return(DB_ERROR); + } + + /* Add separate undo tablespaces to fil_system */ + + err = srv_undo_tablespaces_init(FALSE, + TRUE, + srv_undo_tablespaces, + &srv_undo_tablespaces_open); + if (err != DB_SUCCESS) { + return(err); + } + + /* It is important to call fil_load_single_table_tablespace() after + srv_undo_tablespaces_init(), because fil_is_user_tablespace_id() * + relies on srv_undo_tablespaces_open to be properly initialized */ + + msg("xtrabackup: Generating a list of tablespaces\n"); + + err = fil_load_single_table_tablespaces(xb_check_if_open_tablespace); + if (err != DB_SUCCESS) { + return(err); + } + + debug_sync_point("xtrabackup_load_tablespaces_pause"); + + return(DB_SUCCESS); +} + +/************************************************************************ +Initialize the tablespace memory cache and populate it by scanning for and +opening data files. +@returns DB_SUCCESS or error code.*/ +ulint +xb_data_files_init(void) +/*====================*/ +{ + xb_fil_io_init(); + + return(xb_load_tablespaces()); +} + +/************************************************************************ +Destroy the tablespace memory cache. */ +void +xb_data_files_close(void) +/*====================*/ +{ + ulint i; + + /* Shutdown the aio threads. This has been copied from + innobase_shutdown_for_mysql(). */ + + srv_shutdown_state = SRV_SHUTDOWN_EXIT_THREADS; + + for (i = 0; i < 1000; i++) { + os_aio_wake_all_threads_at_shutdown(); + + os_mutex_enter(os_sync_mutex); + + if (os_thread_count == 0) { + + os_mutex_exit(os_sync_mutex); + + os_thread_sleep(10000); + + break; + } + + os_mutex_exit(os_sync_mutex); + + os_thread_sleep(10000); + } + + if (i == 1000) { + msg("xtrabackup: Warning: %lu threads created by InnoDB" + " had not exited at shutdown!\n", + (ulong) os_thread_count); + } + + os_aio_free(); + + fil_close_all_files(); + + /* Free the double write data structures. */ + if (buf_dblwr) { + buf_dblwr_free(); + } + + /* Reset srv_file_io_threads to its default value to avoid confusing + warning on --prepare in innobase_start_or_create_for_mysql()*/ + srv_n_file_io_threads = 4; + + srv_shutdown_state = SRV_SHUTDOWN_NONE; +} + +/*********************************************************************** +Allocate and initialize the entry for databases and tables filtering +hash tables. If memory allocation is not successful, terminate program. +@return pointer to the created entry. */ +static +xb_filter_entry_t * +xb_new_filter_entry( +/*================*/ + const char* name) /*!< in: name of table/database */ +{ + xb_filter_entry_t *entry; + ulint namelen = strlen(name); + + ut_a(namelen <= NAME_LEN * 2 + 1); + + entry = static_cast + (ut_malloc(sizeof(xb_filter_entry_t) + namelen + 1)); + memset(entry, '\0', sizeof(xb_filter_entry_t) + namelen + 1); + entry->name = ((char*)entry) + sizeof(xb_filter_entry_t); + strcpy(entry->name, name); + entry->has_tables = FALSE; + + return entry; +} + +/*********************************************************************** +Add entry to hash table. If hash table is NULL, allocate and initialize +new hash table */ +static +xb_filter_entry_t* +xb_add_filter( +/*========================*/ + const char* name, /*!< in: name of table/database */ + hash_table_t** hash) /*!< in/out: hash to insert into */ +{ + xb_filter_entry_t* entry; + + entry = xb_new_filter_entry(name); + + if (UNIV_UNLIKELY(*hash == NULL)) { + *hash = hash_create(1000); + } + HASH_INSERT(xb_filter_entry_t, + name_hash, *hash, + ut_fold_string(entry->name), + entry); + + return entry; +} + +/*********************************************************************** +Validate name of table or database. If name is invalid, program will +be finished with error code */ +static +void +xb_validate_name( +/*=============*/ + const char* name, /*!< in: name */ + size_t len) /*!< in: length of name */ +{ + const char* p; + + /* perform only basic validation. validate length and + path symbols */ + if (len > NAME_LEN) { + msg("xtrabackup: name `%s` is too long.\n", name); + exit(EXIT_FAILURE); + } + p = strpbrk(name, "/\\~"); + if (p && p - name < NAME_LEN) { + msg("xtrabackup: name `%s` is not valid.\n", name); + exit(EXIT_FAILURE); + } +} + +/*********************************************************************** +Register new filter entry which can be either database +or table name. */ +static +void +xb_register_filter_entry( +/*=====================*/ + const char* name) /*!< in: name */ +{ + const char* p; + size_t namelen; + xb_filter_entry_t* db_entry = NULL; + + namelen = strlen(name); + if ((p = strchr(name, '.')) != NULL) { + char dbname[NAME_LEN + 1]; + + xb_validate_name(name, p - name); + xb_validate_name(p + 1, namelen - (p - name)); + + strncpy(dbname, name, p - name); + dbname[p - name] = 0; + + if (databases_hash) { + HASH_SEARCH(name_hash, databases_hash, + ut_fold_string(dbname), + xb_filter_entry_t*, + db_entry, (void) 0, + !strcmp(db_entry->name, dbname)); + } + if (!db_entry) { + db_entry = xb_add_filter(dbname, &databases_hash); + } + db_entry->has_tables = TRUE; + xb_add_filter(name, &tables_hash); + } else { + xb_validate_name(name, namelen); + + xb_add_filter(name, &databases_hash); + } +} + +/*********************************************************************** +Register new table for the filter. */ +static +void +xb_register_table( +/*==============*/ + const char* name) /*!< in: name of table */ +{ + if (strchr(name, '.') == NULL) { + msg("xtrabackup: `%s` is not fully qualified name.\n", name); + exit(EXIT_FAILURE); + } + + xb_register_filter_entry(name); +} + +/*********************************************************************** +Register new regex for the filter. */ +static +void +xb_register_regex( +/*==============*/ + const char* regex) /*!< in: regex */ +{ + xb_regex_list_node_t* node; + char errbuf[100]; + int ret; + + node = static_cast + (ut_malloc(sizeof(xb_regex_list_node_t))); + + ret = xb_regcomp(&node->regex, regex, REG_EXTENDED); + if (ret != 0) { + xb_regerror(ret, &node->regex, errbuf, sizeof(errbuf)); + msg("xtrabackup: error: tables regcomp(%s): %s\n", + regex, errbuf); + exit(EXIT_FAILURE); + } + + UT_LIST_ADD_LAST(regex_list, regex_list, node); +} + +typedef void (*insert_entry_func_t)(const char*); + +/*********************************************************************** +Scan string and load filter entries from it. */ +static +void +xb_load_list_string( +/*================*/ + char* list, /*!< in: string representing a list */ + const char* delimiters, /*!< in: delimiters of entries */ + insert_entry_func_t ins) /*!< in: callback to add entry */ +{ + char* p; + char* saveptr; + + p = strtok_r(list, delimiters, &saveptr); + while (p) { + + ins(p); + + p = strtok_r(NULL, delimiters, &saveptr); + } +} + +/*********************************************************************** +Scan file and load filter entries from it. */ +static +void +xb_load_list_file( +/*==============*/ + const char* filename, /*!< in: name of file */ + insert_entry_func_t ins) /*!< in: callback to add entry */ +{ + char name_buf[NAME_LEN*2+2]; + FILE* fp; + + /* read and store the filenames */ + fp = fopen(filename, "r"); + if (!fp) { + msg("xtrabackup: cannot open %s\n", + filename); + exit(EXIT_FAILURE); + } + while (fgets(name_buf, sizeof(name_buf), fp) != NULL) { + char* p = strchr(name_buf, '\n'); + if (p) { + *p = '\0'; + } else { + msg("xtrabackup: `%s...` name is too long", name_buf); + exit(EXIT_FAILURE); + } + + ins(name_buf); + } + + fclose(fp); +} + + +static +void +xb_filters_init() +{ + UT_LIST_INIT(regex_list); + + if (xtrabackup_databases) { + xb_load_list_string(xtrabackup_databases, " \t", + xb_register_filter_entry); + } + + if (xtrabackup_databases_file) { + xb_load_list_file(xtrabackup_databases_file, + xb_register_filter_entry); + } + + if (xtrabackup_tables) { + xb_load_list_string(xtrabackup_tables, ",", + xb_register_regex); + } + + if (xtrabackup_tables_file) { + xb_load_list_file(xtrabackup_tables_file, xb_register_table); + } +} + +static +void +xb_filter_hash_free(hash_table_t* hash) +{ + ulint i; + + /* free the hash elements */ + for (i = 0; i < hash_get_n_cells(hash); i++) { + xb_filter_entry_t* table; + + table = static_cast + (HASH_GET_FIRST(hash, i)); + + while (table) { + xb_filter_entry_t* prev_table = table; + + table = static_cast + (HASH_GET_NEXT(name_hash, prev_table)); + + HASH_DELETE(xb_filter_entry_t, name_hash, hash, + ut_fold_string(prev_table->name), prev_table); + ut_free(prev_table); + } + } + + /* free hash */ + hash_table_free(hash); +} + +/************************************************************************ +Destroy table filters for partial backup. */ +static +void +xb_filters_free() +{ + while (UT_LIST_GET_LEN(regex_list) > 0) { + xb_regex_list_node_t* node = UT_LIST_GET_FIRST(regex_list); + UT_LIST_REMOVE(regex_list, regex_list, node); + xb_regfree(&node->regex); + ut_free(node); + } + + if (tables_hash) { + xb_filter_hash_free(tables_hash); + } + + if (databases_hash) { + xb_filter_hash_free(databases_hash); + } +} + +/*********************************************************************//** +Creates or opens the log files and closes them. +@return DB_SUCCESS or error code */ +static +ulint +open_or_create_log_file( +/*====================*/ + ibool create_new_db, /*!< in: TRUE if we should create a + new database */ + ibool* log_file_created, /*!< out: TRUE if new log file + created */ + ibool log_file_has_been_opened,/*!< in: TRUE if a log file has been + opened before: then it is an error + to try to create another log file */ + ulint k, /*!< in: log group number */ + ulint i) /*!< in: log file number in group */ +{ + ibool ret; + os_offset_t size; + char name[10000]; + ulint dirnamelen; + + UT_NOT_USED(create_new_db); + UT_NOT_USED(log_file_has_been_opened); + UT_NOT_USED(k); + ut_ad(k == 0); + + *log_file_created = FALSE; + + srv_normalize_path_for_win(srv_log_group_home_dir); + + dirnamelen = strlen(srv_log_group_home_dir); + ut_a(dirnamelen < (sizeof name) - 10 - sizeof "ib_logfile"); + memcpy(name, srv_log_group_home_dir, dirnamelen); + + /* Add a path separator if needed. */ + if (dirnamelen && name[dirnamelen - 1] != SRV_PATH_SEPARATOR) { + name[dirnamelen++] = SRV_PATH_SEPARATOR; + } + + sprintf(name + dirnamelen, "%s%lu", "ib_logfile", (ulong) i); + + files[i] = os_file_create(innodb_file_log_key, name, + OS_FILE_OPEN, OS_FILE_NORMAL, + OS_LOG_FILE, &ret); + if (ret == FALSE) { + fprintf(stderr, "InnoDB: Error in opening %s\n", name); + + return(DB_ERROR); + } + + size = os_file_get_size(files[i]); + + if (size != srv_log_file_size * UNIV_PAGE_SIZE) { + + fprintf(stderr, + "InnoDB: Error: log file %s is" + " of different size " UINT64PF " bytes\n" + "InnoDB: than specified in the .cnf" + " file " UINT64PF " bytes!\n", + name, size, srv_log_file_size * UNIV_PAGE_SIZE); + + return(DB_ERROR); + } + + ret = os_file_close(files[i]); + ut_a(ret); + + if (i == 0) { + /* Create in memory the file space object + which is for this log group */ + + fil_space_create(name, + 2 * k + SRV_LOG_SPACE_FIRST_ID, 0, FIL_LOG); + } + + ut_a(fil_validate()); + + ut_a(fil_node_create(name, srv_log_file_size, + 2 * k + SRV_LOG_SPACE_FIRST_ID, FALSE)); + if (i == 0) { + log_group_init(k, srv_n_log_files, + srv_log_file_size * UNIV_PAGE_SIZE, + 2 * k + SRV_LOG_SPACE_FIRST_ID, + SRV_LOG_SPACE_FIRST_ID + 1); /* dummy arch + space id */ + } + + return(DB_SUCCESS); +} + +/*********************************************************************//** +Normalizes init parameter values to use units we use inside InnoDB. +@return DB_SUCCESS or error code */ +static +void +xb_normalize_init_values(void) +/*==========================*/ +{ + ulint i; + + for (i = 0; i < srv_n_data_files; i++) { + srv_data_file_sizes[i] = srv_data_file_sizes[i] + * ((1024 * 1024) / UNIV_PAGE_SIZE); + } + + srv_last_file_size_max = srv_last_file_size_max + * ((1024 * 1024) / UNIV_PAGE_SIZE); + + srv_log_file_size = srv_log_file_size / UNIV_PAGE_SIZE; + + srv_log_buffer_size = srv_log_buffer_size / UNIV_PAGE_SIZE; + + srv_lock_table_size = 5 * (srv_buf_pool_size / UNIV_PAGE_SIZE); +} + +/*********************************************************************** +Set the open files limit. Based on set_max_open_files(). + +@return the resulting open files limit. May be less or more than the requested +value. */ +static uint +xb_set_max_open_files( +/*==================*/ + uint max_file_limit) /*!= max_file_limit) { + + max_file_limit = rlimit.rlim_cur; + goto end; + } + + rlimit.rlim_cur = rlimit.rlim_max = max_file_limit; + + if (setrlimit(RLIMIT_NOFILE, &rlimit)) { + + max_file_limit = old_cur; /* Use original value */ + } else { + + rlimit.rlim_cur = 0; /* Safety if next call fails */ + + (void) getrlimit(RLIMIT_NOFILE, &rlimit); + + if (rlimit.rlim_cur) { + + /* If call didn't fail */ + max_file_limit = (uint) rlimit.rlim_cur; + } + } + +end: + return(max_file_limit); +#else + return(0); +#endif +} + +void +xtrabackup_backup_func(void) +{ + MY_STAT stat_info; + lsn_t latest_cp; + uint i; + uint count; + os_ib_mutex_t count_mutex; + data_thread_ctxt_t *data_threads; + +#ifdef USE_POSIX_FADVISE + msg("xtrabackup: uses posix_fadvise().\n"); +#endif + + /* cd to datadir */ + + if (my_setwd(mysql_real_data_home,MYF(MY_WME))) + { + msg("xtrabackup: cannot my_setwd %s\n", mysql_real_data_home); + exit(EXIT_FAILURE); + } + msg("xtrabackup: cd to %s\n", mysql_real_data_home); + + msg("xtrabackup: open files limit requested %u, set to %u\n", + (uint) xb_open_files_limit, + xb_set_max_open_files(xb_open_files_limit)); + + mysql_data_home= mysql_data_home_buff; + mysql_data_home[0]=FN_CURLIB; // all paths are relative from here + mysql_data_home[1]=0; + + srv_read_only_mode = TRUE; + + srv_backup_mode = TRUE; + srv_close_files = xb_close_files; + + if (srv_close_files) + msg("xtrabackup: warning: close-files specified. Use it " + "at your own risk. If there are DDL operations like table DROP TABLE " + "or RENAME TABLE during the backup, inconsistent backup will be " + "produced.\n"); + + /* initialize components */ + if(innodb_init_param()) + exit(EXIT_FAILURE); + + xb_normalize_init_values(); + +#ifndef __WIN__ + if (srv_file_flush_method_str == NULL) { + /* These are the default options */ + srv_unix_file_flush_method = SRV_UNIX_FSYNC; + } else if (0 == ut_strcmp(srv_file_flush_method_str, "fsync")) { + srv_unix_file_flush_method = SRV_UNIX_FSYNC; + } else if (0 == ut_strcmp(srv_file_flush_method_str, "O_DSYNC")) { + srv_unix_file_flush_method = SRV_UNIX_O_DSYNC; + + } else if (0 == ut_strcmp(srv_file_flush_method_str, "O_DIRECT")) { + srv_unix_file_flush_method = SRV_UNIX_O_DIRECT; + msg("xtrabackup: using O_DIRECT\n"); + } else if (0 == ut_strcmp(srv_file_flush_method_str, "littlesync")) { + srv_unix_file_flush_method = SRV_UNIX_LITTLESYNC; + + } else if (0 == ut_strcmp(srv_file_flush_method_str, "nosync")) { + srv_unix_file_flush_method = SRV_UNIX_NOSYNC; + } else if (0 == ut_strcmp(srv_file_flush_method_str, "ALL_O_DIRECT")) { + srv_unix_file_flush_method = SRV_UNIX_ALL_O_DIRECT; + msg("xtrabackup: using ALL_O_DIRECT\n"); + } else if (0 == ut_strcmp(srv_file_flush_method_str, + "O_DIRECT_NO_FSYNC")) { + srv_unix_file_flush_method = SRV_UNIX_O_DIRECT_NO_FSYNC; + msg("xtrabackup: using O_DIRECT_NO_FSYNC\n"); + } else { + msg("xtrabackup: Unrecognized value %s for " + "innodb_flush_method\n", srv_file_flush_method_str); + exit(EXIT_FAILURE); + } +#else /* __WIN__ */ + /* We can only use synchronous unbuffered IO on Windows for now */ + if (srv_file_flush_method_str != NULL) { + msg("xtrabackupp: Warning: " + "ignoring innodb_flush_method = %s on Windows.\n"); + } + + srv_win_file_flush_method = SRV_WIN_IO_UNBUFFERED; + srv_use_native_aio = FALSE; +#endif + + if (srv_buf_pool_size >= 1000 * 1024 * 1024) { + /* Here we still have srv_pool_size counted + in kilobytes (in 4.0 this was in bytes) + srv_boot() converts the value to + pages; if buffer pool is less than 1000 MB, + assume fewer threads. */ + srv_max_n_threads = 50000; + + } else if (srv_buf_pool_size >= 8 * 1024 * 1024) { + + srv_max_n_threads = 10000; + } else { + srv_max_n_threads = 1000; /* saves several MB of memory, + especially in 64-bit + computers */ + } + + os_sync_mutex = NULL; + srv_general_init(); + ut_crc32_init(); + + xb_filters_init(); + + { + ibool log_file_created; + ibool log_created = FALSE; + ibool log_opened = FALSE; + ulint err; + ulint i; + + xb_fil_io_init(); + + log_init(); + + lock_sys_create(srv_lock_table_size); + + for (i = 0; i < srv_n_log_files; i++) { + err = open_or_create_log_file(FALSE, &log_file_created, + log_opened, 0, i); + if (err != DB_SUCCESS) { + + //return((int) err); + exit(EXIT_FAILURE); + } + + if (log_file_created) { + log_created = TRUE; + } else { + log_opened = TRUE; + } + if ((log_opened && log_created)) { + msg( + "xtrabackup: Error: all log files must be created at the same time.\n" + "xtrabackup: All log files must be created also in database creation.\n" + "xtrabackup: If you want bigger or smaller log files, shut down the\n" + "xtrabackup: database and make sure there were no errors in shutdown.\n" + "xtrabackup: Then delete the existing log files. Edit the .cnf file\n" + "xtrabackup: and start the database again.\n"); + + //return(DB_ERROR); + exit(EXIT_FAILURE); + } + } + + /* log_file_created must not be TRUE, if online */ + if (log_file_created) { + msg("xtrabackup: Something wrong with source files...\n"); + exit(EXIT_FAILURE); + } + + } + + /* create extra LSN dir if it does not exist. */ + if (xtrabackup_extra_lsndir + &&!my_stat(xtrabackup_extra_lsndir,&stat_info,MYF(0)) + && (my_mkdir(xtrabackup_extra_lsndir,0777,MYF(0)) < 0)) { + msg("xtrabackup: Error: cannot mkdir %d: %s\n", + my_errno, xtrabackup_extra_lsndir); + exit(EXIT_FAILURE); + } + + /* create target dir if not exist */ + if (!my_stat(xtrabackup_target_dir,&stat_info,MYF(0)) + && (my_mkdir(xtrabackup_target_dir,0777,MYF(0)) < 0)){ + msg("xtrabackup: Error: cannot mkdir %d: %s\n", + my_errno, xtrabackup_target_dir); + exit(EXIT_FAILURE); + } + + { + fil_system_t* f_system = fil_system; + + /* definition from recv_recovery_from_checkpoint_start() */ + log_group_t* max_cp_group; + ulint max_cp_field; + byte* buf; + byte* log_hdr_buf_; + byte* log_hdr_buf; + ulint err; + + /* start back ground thread to copy newer log */ + os_thread_id_t log_copying_thread_id; + datafiles_iter_t *it; + + log_hdr_buf_ = static_cast + (ut_malloc(LOG_FILE_HDR_SIZE + UNIV_PAGE_SIZE_MAX)); + log_hdr_buf = static_cast + (ut_align(log_hdr_buf_, UNIV_PAGE_SIZE_MAX)); + + /* get current checkpoint_lsn */ + /* Look for the latest checkpoint from any of the log groups */ + + mutex_enter(&log_sys->mutex); + + err = recv_find_max_checkpoint(&max_cp_group, &max_cp_field); + + if (err != DB_SUCCESS) { + + ut_free(log_hdr_buf_); + exit(EXIT_FAILURE); + } + + log_group_read_checkpoint_info(max_cp_group, max_cp_field); + buf = log_sys->checkpoint_buf; + + checkpoint_lsn_start = mach_read_from_8(buf + LOG_CHECKPOINT_LSN); + checkpoint_no_start = mach_read_from_8(buf + LOG_CHECKPOINT_NO); + + mutex_exit(&log_sys->mutex); + +reread_log_header: + fil_io(OS_FILE_READ | OS_FILE_LOG, TRUE, max_cp_group->space_id, + 0, + 0, 0, LOG_FILE_HDR_SIZE, + log_hdr_buf, max_cp_group); + + /* check consistency of log file header to copy */ + mutex_enter(&log_sys->mutex); + + err = recv_find_max_checkpoint(&max_cp_group, &max_cp_field); + + if (err != DB_SUCCESS) { + + ut_free(log_hdr_buf_); + exit(EXIT_FAILURE); + } + + log_group_read_checkpoint_info(max_cp_group, max_cp_field); + buf = log_sys->checkpoint_buf; + + if(checkpoint_no_start != mach_read_from_8(buf + LOG_CHECKPOINT_NO)) { + + checkpoint_lsn_start = mach_read_from_8(buf + LOG_CHECKPOINT_LSN); + checkpoint_no_start = mach_read_from_8(buf + LOG_CHECKPOINT_NO); + mutex_exit(&log_sys->mutex); + goto reread_log_header; + } + + mutex_exit(&log_sys->mutex); + + xtrabackup_init_datasinks(); + + if (!select_history()) { + exit(EXIT_FAILURE); + } + + /* open the log file */ + memset(&stat_info, 0, sizeof(MY_STAT)); + dst_log_file = ds_open(ds_redo, XB_LOG_FILENAME, &stat_info); + if (dst_log_file == NULL) { + msg("xtrabackup: error: failed to open the target stream for " + "'%s'.\n", XB_LOG_FILENAME); + ut_free(log_hdr_buf_); + exit(EXIT_FAILURE); + } + + /* label it */ + strcpy((char*) log_hdr_buf + LOG_FILE_WAS_CREATED_BY_HOT_BACKUP, + "xtrabkup "); + ut_sprintf_timestamp( + (char*) log_hdr_buf + (LOG_FILE_WAS_CREATED_BY_HOT_BACKUP + + (sizeof "xtrabkup ") - 1)); + + if (ds_write(dst_log_file, log_hdr_buf, LOG_FILE_HDR_SIZE)) { + msg("xtrabackup: error: write to logfile failed\n"); + ut_free(log_hdr_buf_); + exit(EXIT_FAILURE); + } + + ut_free(log_hdr_buf_); + + /* start flag */ + log_copying = TRUE; + + /* start io throttle */ + if(xtrabackup_throttle) { + os_thread_id_t io_watching_thread_id; + + io_ticket = xtrabackup_throttle; + wait_throttle = os_event_create(); + + os_thread_create(io_watching_thread, NULL, + &io_watching_thread_id); + } + + mutex_enter(&log_sys->mutex); + xtrabackup_choose_lsn_offset(checkpoint_lsn_start); + mutex_exit(&log_sys->mutex); + + /* copy log file by current position */ + if(xtrabackup_copy_logfile(checkpoint_lsn_start, FALSE)) + exit(EXIT_FAILURE); + + + log_copying_stop = os_event_create(); + os_thread_create(log_copying_thread, NULL, &log_copying_thread_id); + + /* Populate fil_system with tablespaces to copy */ + err = xb_load_tablespaces(); + if (err != DB_SUCCESS) { + msg("xtrabackup: error: xb_load_tablespaces() failed with" + "error code %lu\n", err); + exit(EXIT_FAILURE); + } + + /* FLUSH CHANGED_PAGE_BITMAPS call */ + if (!flush_changed_page_bitmaps()) { + exit(EXIT_FAILURE); + } + debug_sync_point("xtrabackup_suspend_at_start"); + + if (xtrabackup_incremental) { + if (!xtrabackup_incremental_force_scan) { + changed_page_bitmap = xb_page_bitmap_init(); + } + if (!changed_page_bitmap) { + msg("xtrabackup: using the full scan for incremental " + "backup\n"); + } else if (incremental_lsn != checkpoint_lsn_start) { + /* Do not print that bitmaps are used when dummy bitmap + is build for an empty LSN range. */ + msg("xtrabackup: using the changed page bitmap\n"); + } + } + + ut_a(xtrabackup_parallel > 0); + + if (xtrabackup_parallel > 1) { + msg("xtrabackup: Starting %u threads for parallel data " + "files transfer\n", xtrabackup_parallel); + } + + it = datafiles_iter_new(f_system); + if (it == NULL) { + msg("xtrabackup: Error: datafiles_iter_new() failed.\n"); + exit(EXIT_FAILURE); + } + + /* Create data copying threads */ + data_threads = (data_thread_ctxt_t *) + ut_malloc(sizeof(data_thread_ctxt_t) * xtrabackup_parallel); + count = xtrabackup_parallel; + count_mutex = os_mutex_create(); + + for (i = 0; i < (uint) xtrabackup_parallel; i++) { + data_threads[i].it = it; + data_threads[i].num = i+1; + data_threads[i].count = &count; + data_threads[i].count_mutex = count_mutex; + os_thread_create(data_copy_thread_func, data_threads + i, + &data_threads[i].id); + } + + /* Wait for threads to exit */ + while (1) { + os_thread_sleep(1000000); + os_mutex_enter(count_mutex); + if (count == 0) { + os_mutex_exit(count_mutex); + break; + } + os_mutex_exit(count_mutex); + } + + os_mutex_free(count_mutex); + ut_free(data_threads); + datafiles_iter_free(it); + + if (changed_page_bitmap) { + xb_page_bitmap_deinit(changed_page_bitmap); + } + } + + if (!backup_start()) { + exit(EXIT_FAILURE); + } + + /* read the latest checkpoint lsn */ + latest_cp = 0; + { + log_group_t* max_cp_group; + ulint max_cp_field; + ulint err; + + mutex_enter(&log_sys->mutex); + + err = recv_find_max_checkpoint(&max_cp_group, &max_cp_field); + + if (err != DB_SUCCESS) { + msg("xtrabackup: Error: recv_find_max_checkpoint() failed.\n"); + mutex_exit(&log_sys->mutex); + goto skip_last_cp; + } + + log_group_read_checkpoint_info(max_cp_group, max_cp_field); + + xtrabackup_choose_lsn_offset(checkpoint_lsn_start); + + latest_cp = mach_read_from_8(log_sys->checkpoint_buf + + LOG_CHECKPOINT_LSN); + + mutex_exit(&log_sys->mutex); + + msg("xtrabackup: The latest check point (for incremental): " + "'" LSN_PF "'\n", latest_cp); + } +skip_last_cp: + /* stop log_copying_thread */ + log_copying = FALSE; + os_event_set(log_copying_stop); + msg("xtrabackup: Stopping log copying thread.\n"); + while (log_copying_running) { + msg("."); + os_thread_sleep(200000); /*0.2 sec*/ + } + msg("\n"); + + os_event_free(log_copying_stop); + if (ds_close(dst_log_file)) { + exit(EXIT_FAILURE); + } + + if(!xtrabackup_incremental) { + strcpy(metadata_type, "full-backuped"); + metadata_from_lsn = 0; + } else { + strcpy(metadata_type, "incremental"); + metadata_from_lsn = incremental_lsn; + } + metadata_to_lsn = latest_cp; + metadata_last_lsn = log_copy_scanned_lsn; + + if (!xtrabackup_stream_metadata(ds_meta)) { + msg("xtrabackup: Error: failed to stream metadata.\n"); + exit(EXIT_FAILURE); + } + if (xtrabackup_extra_lsndir) { + char filename[FN_REFLEN]; + + sprintf(filename, "%s/%s", xtrabackup_extra_lsndir, + XTRABACKUP_METADATA_FILENAME); + if (!xtrabackup_write_metadata(filename)) { + msg("xtrabackup: Error: failed to write metadata " + "to '%s'.\n", filename); + exit(EXIT_FAILURE); + } + + } + + if (!backup_finish()) { + exit(EXIT_FAILURE); + } + + xtrabackup_destroy_datasinks(); + + if (wait_throttle) { + /* wait for io_watching_thread completion */ + while (io_watching_thread_running) { + os_thread_sleep(1000000); + } + os_event_free(wait_throttle); + wait_throttle = NULL; + } + + msg("xtrabackup: Transaction log of lsn (" LSN_PF ") to (" LSN_PF + ") was copied.\n", checkpoint_lsn_start, log_copy_scanned_lsn); + xb_filters_free(); + + xb_data_files_close(); + + /* Make sure that the latest checkpoint made it to xtrabackup_logfile */ + if (latest_cp > log_copy_scanned_lsn) { + msg("xtrabackup: error: last checkpoint LSN (" LSN_PF + ") is larger than last copied LSN (" LSN_PF ").\n", + latest_cp, log_copy_scanned_lsn); + exit(EXIT_FAILURE); + } +} + +/* ================= stats ================= */ +static my_bool +xtrabackup_stats_level( + dict_index_t* index, + ulint level) +{ + ulint space; + page_t* page; + + rec_t* node_ptr; + + ulint right_page_no; + + page_cur_t cursor; + + mtr_t mtr; + mem_heap_t* heap = mem_heap_create(256); + + ulint* offsets = NULL; + + ulonglong n_pages, n_pages_extern; + ulonglong sum_data, sum_data_extern; + ulonglong n_recs; + ulint page_size; + buf_block_t* block; + ulint zip_size; + + n_pages = sum_data = n_recs = 0; + n_pages_extern = sum_data_extern = 0; + + + if (level == 0) + fprintf(stdout, " leaf pages: "); + else + fprintf(stdout, " level %lu pages: ", level); + + mtr_start(&mtr); + + mtr_x_lock(&(index->lock), &mtr); + block = btr_root_block_get(index, RW_X_LATCH, &mtr); + page = buf_block_get_frame(block); + + space = page_get_space_id(page); + zip_size = fil_space_get_zip_size(space); + + while (level != btr_page_get_level(page, &mtr)) { + + ut_a(space == buf_block_get_space(block)); + ut_a(space == page_get_space_id(page)); + ut_a(!page_is_leaf(page)); + + page_cur_set_before_first(block, &cursor); + page_cur_move_to_next(&cursor); + + node_ptr = page_cur_get_rec(&cursor); + offsets = rec_get_offsets(node_ptr, index, offsets, + ULINT_UNDEFINED, &heap); + block = btr_node_ptr_get_child(node_ptr, index, offsets, &mtr); + page = buf_block_get_frame(block); + } + +loop: + mem_heap_empty(heap); + offsets = NULL; + mtr_x_lock(&(index->lock), &mtr); + + right_page_no = btr_page_get_next(page, &mtr); + + + /*=================================*/ + //fprintf(stdout, "%lu ", (ulint) buf_frame_get_page_no(page)); + + n_pages++; + sum_data += page_get_data_size(page); + n_recs += page_get_n_recs(page); + + + if (level == 0) { + page_cur_t cur; + ulint n_fields; + ulint i; + mem_heap_t* local_heap = NULL; + ulint offsets_[REC_OFFS_NORMAL_SIZE]; + ulint* local_offsets = offsets_; + + *offsets_ = (sizeof offsets_) / sizeof *offsets_; + + page_cur_set_before_first(block, &cur); + page_cur_move_to_next(&cur); + + for (;;) { + if (page_cur_is_after_last(&cur)) { + break; + } + + local_offsets = rec_get_offsets(cur.rec, index, local_offsets, + ULINT_UNDEFINED, &local_heap); + n_fields = rec_offs_n_fields(local_offsets); + + for (i = 0; i < n_fields; i++) { + if (rec_offs_nth_extern(local_offsets, i)) { + page_t* local_page; + ulint space_id; + ulint page_no; + ulint offset; + byte* blob_header; + ulint part_len; + mtr_t local_mtr; + ulint local_len; + byte* data; + buf_block_t* local_block; + + data = rec_get_nth_field(cur.rec, local_offsets, i, &local_len); + + ut_a(local_len >= BTR_EXTERN_FIELD_REF_SIZE); + local_len -= BTR_EXTERN_FIELD_REF_SIZE; + + space_id = mach_read_from_4(data + local_len + BTR_EXTERN_SPACE_ID); + page_no = mach_read_from_4(data + local_len + BTR_EXTERN_PAGE_NO); + offset = mach_read_from_4(data + local_len + BTR_EXTERN_OFFSET); + + if (offset != FIL_PAGE_DATA) + msg("\nWarning: several record may share same external page.\n"); + + for (;;) { + mtr_start(&local_mtr); + + local_block = btr_block_get(space_id, zip_size, page_no, RW_S_LATCH, index, &local_mtr); + local_page = buf_block_get_frame(local_block); + blob_header = local_page + offset; +#define BTR_BLOB_HDR_PART_LEN 0 +#define BTR_BLOB_HDR_NEXT_PAGE_NO 4 + //part_len = btr_blob_get_part_len(blob_header); + part_len = mach_read_from_4(blob_header + BTR_BLOB_HDR_PART_LEN); + + //page_no = btr_blob_get_next_page_no(blob_header); + page_no = mach_read_from_4(blob_header + BTR_BLOB_HDR_NEXT_PAGE_NO); + + offset = FIL_PAGE_DATA; + + + + + /*=================================*/ + //fprintf(stdout, "[%lu] ", (ulint) buf_frame_get_page_no(page)); + + n_pages_extern++; + sum_data_extern += part_len; + + + mtr_commit(&local_mtr); + + if (page_no == FIL_NULL) + break; + } + } + } + + page_cur_move_to_next(&cur); + } + } + + + + + mtr_commit(&mtr); + if (right_page_no != FIL_NULL) { + mtr_start(&mtr); + block = btr_block_get(space, zip_size, right_page_no, + RW_X_LATCH, index, &mtr); + page = buf_block_get_frame(block); + goto loop; + } + mem_heap_free(heap); + + if (zip_size) { + page_size = zip_size; + } else { + page_size = UNIV_PAGE_SIZE; + } + + if (level == 0) + fprintf(stdout, "recs=%llu, ", n_recs); + + fprintf(stdout, "pages=%llu, data=%llu bytes, data/pages=%lld%%", + n_pages, sum_data, + ((sum_data * 100)/ page_size)/n_pages); + + + if (level == 0 && n_pages_extern) { + putc('\n', stdout); + /* also scan blob pages*/ + fprintf(stdout, " external pages: "); + + fprintf(stdout, "pages=%llu, data=%llu bytes, data/pages=%lld%%", + n_pages_extern, sum_data_extern, + ((sum_data_extern * 100)/ page_size)/n_pages_extern); + } + + putc('\n', stdout); + + if (level > 0) { + xtrabackup_stats_level(index, level - 1); + } + + return(TRUE); +} + +static void +xtrabackup_stats_func(void) +{ + ulint n; + + /* cd to datadir */ + + if (my_setwd(mysql_real_data_home,MYF(MY_WME))) + { + msg("xtrabackup: cannot my_setwd %s\n", mysql_real_data_home); + exit(EXIT_FAILURE); + } + msg("xtrabackup: cd to %s\n", mysql_real_data_home); + + mysql_data_home= mysql_data_home_buff; + mysql_data_home[0]=FN_CURLIB; // all paths are relative from here + mysql_data_home[1]=0; + + /* set read only */ + srv_read_only_mode = TRUE; + + /* initialize components */ + if(innodb_init_param()) + exit(EXIT_FAILURE); + + /* Check if the log files have been created, otherwise innodb_init() + will crash when called with srv_read_only == TRUE */ + for (n = 0; n < srv_n_log_files; n++) { + char logname[FN_REFLEN]; + ibool exists; + os_file_type_t type; + + snprintf(logname, sizeof(logname), "%s%c%s%lu", + srv_log_group_home_dir, SRV_PATH_SEPARATOR, + "ib_logfile", (ulong) n); + srv_normalize_path_for_win(logname); + + if (!os_file_status(logname, &exists, &type) || !exists || + type != OS_FILE_TYPE_FILE) { + msg("xtrabackup: Error: " + "Cannot find log file %s.\n", logname); + msg("xtrabackup: Error: " + "to use the statistics feature, you need a " + "clean copy of the database including " + "correctly sized log files, so you need to " + "execute with --prepare twice to use this " + "functionality on a backup.\n"); + exit(EXIT_FAILURE); + } + } + + msg("xtrabackup: Starting 'read-only' InnoDB instance to gather " + "index statistics.\n" + "xtrabackup: Using %lld bytes for buffer pool (set by " + "--use-memory parameter)\n", xtrabackup_use_memory); + + if(innodb_init()) + exit(EXIT_FAILURE); + + xb_filters_init(); + + fprintf(stdout, "\n\n\n"); + + /* gather stats */ + + { + dict_table_t* sys_tables; + dict_index_t* sys_index; + dict_table_t* table; + btr_pcur_t pcur; + rec_t* rec; + byte* field; + ulint len; + mtr_t mtr; + + /* Enlarge the fatal semaphore wait timeout during the InnoDB table + monitor printout */ + + os_increment_counter_by_amount(server_mutex, + srv_fatal_semaphore_wait_threshold, + 72000); + + mutex_enter(&(dict_sys->mutex)); + + mtr_start(&mtr); + + sys_tables = dict_table_get_low("SYS_TABLES"); + sys_index = UT_LIST_GET_FIRST(sys_tables->indexes); + + btr_pcur_open_at_index_side(TRUE, sys_index, BTR_SEARCH_LEAF, &pcur, + TRUE, 0, &mtr); +loop: + btr_pcur_move_to_next_user_rec(&pcur, &mtr); + + rec = btr_pcur_get_rec(&pcur); + + if (!btr_pcur_is_on_user_rec(&pcur)) + { + /* end of index */ + + btr_pcur_close(&pcur); + mtr_commit(&mtr); + + mutex_exit(&(dict_sys->mutex)); + + /* Restore the fatal semaphore wait timeout */ + os_increment_counter_by_amount(server_mutex, + srv_fatal_semaphore_wait_threshold, + -72000); + + goto end; + } + + field = rec_get_nth_field_old(rec, 0, &len); + + if (!rec_get_deleted_flag(rec, 0)) { + + /* We found one */ + + char* table_name = mem_strdupl((char*) field, len); + + btr_pcur_store_position(&pcur, &mtr); + + mtr_commit(&mtr); + + table = dict_table_get_low(table_name); + mem_free(table_name); + + if (table && check_if_skip_table(table->name)) + goto skip; + + + if (table == NULL) { + fputs("InnoDB: Failed to load table ", stderr); + ut_print_namel(stderr, NULL, TRUE, (char*) field, len); + putc('\n', stderr); + } else { + dict_index_t* index; + + /* The table definition was corrupt if there + is no index */ + + if (dict_table_get_first_index(table)) { + dict_stats_update_transient(table); + } + + //dict_table_print_low(table); + + index = UT_LIST_GET_FIRST(table->indexes); + while (index != NULL) { +{ + ib_int64_t n_vals; + + if (index->n_user_defined_cols > 0) { + n_vals = index->stat_n_diff_key_vals[ + index->n_user_defined_cols]; + } else { + n_vals = index->stat_n_diff_key_vals[1]; + } + + fprintf(stdout, + " table: %s, index: %s, space id: %lu, root page: %lu" + ", zip size: %lu" + "\n estimated statistics in dictionary:\n" + " key vals: %lu, leaf pages: %lu, size pages: %lu\n" + " real statistics:\n", + table->name, index->name, + (ulong) index->space, + (ulong) index->page, + (ulong) fil_space_get_zip_size(index->space), + (ulong) n_vals, + (ulong) index->stat_n_leaf_pages, + (ulong) index->stat_index_size); + + { + mtr_t local_mtr; + page_t* root; + ulint page_level; + + mtr_start(&local_mtr); + + mtr_x_lock(&(index->lock), &local_mtr); + root = btr_root_get(index, &local_mtr); + page_level = btr_page_get_level(root, &local_mtr); + + xtrabackup_stats_level(index, page_level); + + mtr_commit(&local_mtr); + } + + putc('\n', stdout); +} + index = UT_LIST_GET_NEXT(indexes, index); + } + } + +skip: + mtr_start(&mtr); + + btr_pcur_restore_position(BTR_SEARCH_LEAF, &pcur, &mtr); + } + + goto loop; + } + +end: + putc('\n', stdout); + + fflush(stdout); + + xb_filters_free(); + + /* shutdown InnoDB */ + if(innodb_end()) + exit(EXIT_FAILURE); +} + +/* ================= prepare ================= */ + +static my_bool +xtrabackup_init_temp_log(void) +{ + os_file_t src_file = XB_FILE_UNDEFINED; + char src_path[FN_REFLEN]; + char dst_path[FN_REFLEN]; + ibool success; + + ulint field; + byte log_buf[UNIV_PAGE_SIZE_MAX * 128]; /* 2 MB */ + + ib_int64_t file_size; + + lsn_t max_no; + lsn_t max_lsn; + lsn_t checkpoint_no; + + ulint fold; + + bool checkpoint_found; + + max_no = 0; + + if (!xb_init_log_block_size()) { + goto error; + } + + if(!xtrabackup_incremental_dir) { + sprintf(dst_path, "%s/ib_logfile0", xtrabackup_target_dir); + sprintf(src_path, "%s/%s", xtrabackup_target_dir, + XB_LOG_FILENAME); + } else { + sprintf(dst_path, "%s/ib_logfile0", xtrabackup_incremental_dir); + sprintf(src_path, "%s/%s", xtrabackup_incremental_dir, + XB_LOG_FILENAME); + } + + srv_normalize_path_for_win(dst_path); + srv_normalize_path_for_win(src_path); +retry: + src_file = os_file_create_simple_no_error_handling(0, src_path, + OS_FILE_OPEN, + OS_FILE_READ_WRITE, + &success); + if (!success) { + /* The following call prints an error message */ + os_file_get_last_error(TRUE); + + msg("xtrabackup: Warning: cannot open %s. will try to find.\n", + src_path); + + /* check if ib_logfile0 may be xtrabackup_logfile */ + src_file = os_file_create_simple_no_error_handling(0, dst_path, + OS_FILE_OPEN, + OS_FILE_READ_WRITE, + &success); + if (!success) { + os_file_get_last_error(TRUE); + msg(" xtrabackup: Fatal error: cannot find %s.\n", + src_path); + + goto error; + } + + success = os_file_read(src_file, log_buf, 0, + LOG_FILE_HDR_SIZE); + if (!success) { + goto error; + } + + if ( ut_memcmp(log_buf + LOG_FILE_WAS_CREATED_BY_HOT_BACKUP, + (byte*)"xtrabkup", (sizeof "xtrabkup") - 1) == 0) { + msg(" xtrabackup: 'ib_logfile0' seems to be " + "'xtrabackup_logfile'. will retry.\n"); + + os_file_close(src_file); + src_file = XB_FILE_UNDEFINED; + + /* rename and try again */ + success = os_file_rename(0, dst_path, src_path); + if (!success) { + goto error; + } + + goto retry; + } + + msg(" xtrabackup: Fatal error: cannot find %s.\n", + src_path); + + os_file_close(src_file); + src_file = XB_FILE_UNDEFINED; + + goto error; + } + + file_size = os_file_get_size(src_file); + + + /* TODO: We should skip the following modifies, if it is not the first time. */ + + /* read log file header */ + success = os_file_read(src_file, log_buf, 0, LOG_FILE_HDR_SIZE); + if (!success) { + goto error; + } + + if ( ut_memcmp(log_buf + LOG_FILE_WAS_CREATED_BY_HOT_BACKUP, + (byte*)"xtrabkup", (sizeof "xtrabkup") - 1) != 0 ) { + msg("xtrabackup: notice: xtrabackup_logfile was already used " + "to '--prepare'.\n"); + goto skip_modify; + } else { + /* clear it later */ + //memset(log_buf + LOG_FILE_WAS_CREATED_BY_HOT_BACKUP, + // ' ', 4); + } + + checkpoint_found = false; + + /* read last checkpoint lsn */ + for (field = LOG_CHECKPOINT_1; field <= LOG_CHECKPOINT_2; + field += LOG_CHECKPOINT_2 - LOG_CHECKPOINT_1) { + if (!recv_check_cp_is_consistent(const_cast + (log_buf + field))) + goto not_consistent; + + checkpoint_no = mach_read_from_8(log_buf + field + + LOG_CHECKPOINT_NO); + + if (checkpoint_no >= max_no) { + + max_no = checkpoint_no; + max_lsn = mach_read_from_8(log_buf + field + + LOG_CHECKPOINT_LSN); + checkpoint_found = true; + } +not_consistent: + ; + } + + if (!checkpoint_found) { + msg("xtrabackup: No valid checkpoint found.\n"); + goto error; + } + + + /* It seems to be needed to overwrite the both checkpoint area. */ + mach_write_to_8(log_buf + LOG_CHECKPOINT_1 + LOG_CHECKPOINT_LSN, + max_lsn); + mach_write_to_4(log_buf + LOG_CHECKPOINT_1 + + LOG_CHECKPOINT_OFFSET_LOW32, + LOG_FILE_HDR_SIZE + + (max_lsn - + ut_uint64_align_down(max_lsn, + OS_FILE_LOG_BLOCK_SIZE))); + mach_write_to_4(log_buf + LOG_CHECKPOINT_1 + + LOG_CHECKPOINT_OFFSET_HIGH32, 0); + fold = ut_fold_binary(log_buf + LOG_CHECKPOINT_1, LOG_CHECKPOINT_CHECKSUM_1); + mach_write_to_4(log_buf + LOG_CHECKPOINT_1 + LOG_CHECKPOINT_CHECKSUM_1, fold); + + fold = ut_fold_binary(log_buf + LOG_CHECKPOINT_1 + LOG_CHECKPOINT_LSN, + LOG_CHECKPOINT_CHECKSUM_2 - LOG_CHECKPOINT_LSN); + mach_write_to_4(log_buf + LOG_CHECKPOINT_1 + LOG_CHECKPOINT_CHECKSUM_2, fold); + + mach_write_to_8(log_buf + LOG_CHECKPOINT_2 + LOG_CHECKPOINT_LSN, + max_lsn); + mach_write_to_4(log_buf + LOG_CHECKPOINT_2 + + LOG_CHECKPOINT_OFFSET_LOW32, + LOG_FILE_HDR_SIZE + + (max_lsn - + ut_uint64_align_down(max_lsn, + OS_FILE_LOG_BLOCK_SIZE))); + mach_write_to_4(log_buf + LOG_CHECKPOINT_2 + + LOG_CHECKPOINT_OFFSET_HIGH32, 0); + fold = ut_fold_binary(log_buf + LOG_CHECKPOINT_2, LOG_CHECKPOINT_CHECKSUM_1); + mach_write_to_4(log_buf + LOG_CHECKPOINT_2 + LOG_CHECKPOINT_CHECKSUM_1, fold); + + fold = ut_fold_binary(log_buf + LOG_CHECKPOINT_2 + LOG_CHECKPOINT_LSN, + LOG_CHECKPOINT_CHECKSUM_2 - LOG_CHECKPOINT_LSN); + mach_write_to_4(log_buf + LOG_CHECKPOINT_2 + LOG_CHECKPOINT_CHECKSUM_2, fold); + + + success = os_file_write(src_path, src_file, log_buf, 0, + LOG_FILE_HDR_SIZE); + if (!success) { + goto error; + } + + /* expand file size (9/8) and align to UNIV_PAGE_SIZE_MAX */ + + if (file_size % UNIV_PAGE_SIZE_MAX) { + memset(log_buf, 0, UNIV_PAGE_SIZE_MAX); + success = os_file_write(src_path, src_file, log_buf, + file_size, + UNIV_PAGE_SIZE_MAX + - (ulint) (file_size + % UNIV_PAGE_SIZE_MAX)); + if (!success) { + goto error; + } + + file_size = os_file_get_size(src_file); + } + + /* TODO: We should judge whether the file is already expanded or not... */ + { + ulint expand; + + memset(log_buf, 0, UNIV_PAGE_SIZE_MAX * 128); + expand = (ulint) (file_size / UNIV_PAGE_SIZE_MAX / 8); + + for (; expand > 128; expand -= 128) { + success = os_file_write(src_path, src_file, log_buf, + file_size, + UNIV_PAGE_SIZE_MAX * 128); + if (!success) { + goto error; + } + file_size += UNIV_PAGE_SIZE_MAX * 128; + } + + if (expand) { + success = os_file_write(src_path, src_file, log_buf, + file_size, + expand * UNIV_PAGE_SIZE_MAX); + if (!success) { + goto error; + } + file_size += UNIV_PAGE_SIZE_MAX * expand; + } + } + + /* make larger than 2MB */ + if (file_size < 2*1024*1024L) { + memset(log_buf, 0, UNIV_PAGE_SIZE_MAX); + while (file_size < 2*1024*1024L) { + success = os_file_write(src_path, src_file, log_buf, + file_size, + UNIV_PAGE_SIZE_MAX); + if (!success) { + goto error; + } + file_size += UNIV_PAGE_SIZE_MAX; + } + file_size = os_file_get_size(src_file); + } + + msg("xtrabackup: xtrabackup_logfile detected: size=" INT64PF ", " + "start_lsn=(" LSN_PF ")\n", file_size, max_lsn); + + os_file_close(src_file); + src_file = XB_FILE_UNDEFINED; + + /* fake InnoDB */ + innobase_log_files_in_group_save = innobase_log_files_in_group; + srv_log_group_home_dir_save = srv_log_group_home_dir; + innobase_log_file_size_save = innobase_log_file_size; + + srv_log_group_home_dir = NULL; + innobase_log_file_size = file_size; + innobase_log_files_in_group = 1; + + srv_thread_concurrency = 0; + + /* rename 'xtrabackup_logfile' to 'ib_logfile0' */ + success = os_file_rename(0, src_path, dst_path); + if (!success) { + goto error; + } + xtrabackup_logfile_is_renamed = TRUE; + + return(FALSE); + +skip_modify: + os_file_close(src_file); + src_file = XB_FILE_UNDEFINED; + return(FALSE); + +error: + if (src_file != XB_FILE_UNDEFINED) + os_file_close(src_file); + msg("xtrabackup: Error: xtrabackup_init_temp_log() failed.\n"); + return(TRUE); /*ERROR*/ +} + +/*********************************************************************** +Generates path to the meta file path from a given path to an incremental .delta +by replacing trailing ".delta" with ".meta", or returns error if 'delta_path' +does not end with the ".delta" character sequence. +@return TRUE on success, FALSE on error. */ +static +ibool +get_meta_path( + const char *delta_path, /* in: path to a .delta file */ + char *meta_path) /* out: path to the corresponding .meta + file */ +{ + size_t len = strlen(delta_path); + + if (len <= 6 || strcmp(delta_path + len - 6, ".delta")) { + return FALSE; + } + memcpy(meta_path, delta_path, len - 6); + strcpy(meta_path + len - 6, XB_DELTA_INFO_SUFFIX); + + return TRUE; +} + +/****************************************************************//** +Create a new tablespace on disk and return the handle to its opened +file. Code adopted from fil_create_new_single_table_tablespace with +the main difference that only disk file is created without updating +the InnoDB in-memory dictionary data structures. + +@return TRUE on success, FALSE on error. */ +static +ibool +xb_space_create_file( +/*==================*/ + const char* path, /*!(ut_malloc(3 * UNIV_PAGE_SIZE)); + /* Align the memory for file i/o if we might have O_DIRECT set */ + page = static_cast(ut_align(buf, UNIV_PAGE_SIZE)); + + memset(page, '\0', UNIV_PAGE_SIZE); + + fsp_header_init_fields(page, space_id, flags); + mach_write_to_4(page + FIL_PAGE_ARCH_LOG_NO_OR_SPACE_ID, space_id); + + if (!fsp_flags_is_compressed(flags)) { + buf_flush_init_for_writing(page, NULL, 0); + + ret = os_file_write(path, *file, page, 0, UNIV_PAGE_SIZE); + } + else { + page_zip_des_t page_zip; + ulint zip_size; + + zip_size = fsp_flags_get_zip_size(flags); + page_zip_set_size(&page_zip, zip_size); + page_zip.data = page + UNIV_PAGE_SIZE; + fprintf(stderr, "zip_size = %lu\n", zip_size); + +#ifdef UNIV_DEBUG + page_zip.m_start = +#endif /* UNIV_DEBUG */ + page_zip.m_end = page_zip.m_nonempty = + page_zip.n_blobs = 0; + + buf_flush_init_for_writing(page, &page_zip, 0); + + ret = os_file_write(path, *file, page_zip.data, 0, + zip_size); + } + + ut_free(buf); + + if (!ret) { + msg("xtrabackup: could not write the first page to %s\n", + path); + os_file_close(*file); + os_file_delete(0, path); + return ret; + } + + return TRUE; +} + +/*********************************************************************** +Searches for matching tablespace file for given .delta file and space_id +in given directory. When matching tablespace found, renames it to match the +name of .delta file. If there was a tablespace with matching name and +mismatching ID, renames it to xtrabackup_tmp_#ID.ibd. If there was no +matching file, creates a new tablespace. +@return file handle of matched or created file */ +static +os_file_t +xb_delta_open_matching_space( + const char* dbname, /* in: path to destination database dir */ + const char* name, /* in: name of delta file (without .delta) */ + ulint space_id, /* in: space id of delta file */ + ulint zip_size, /* in: zip_size of tablespace */ + char* real_name, /* out: full path of destination file */ + size_t real_name_len, /* out: buffer size for real_name */ + ibool* success) /* out: indicates error. TRUE = success */ +{ + char dest_dir[FN_REFLEN]; + char dest_space_name[FN_REFLEN]; + ibool ok; + fil_space_t* fil_space; + os_file_t file = 0; + ulint tablespace_flags; + xb_filter_entry_t* table; + + ut_a(dbname != NULL || + !fil_is_user_tablespace_id(space_id) || + space_id == ULINT_UNDEFINED); + + *success = FALSE; + + if (dbname) { + snprintf(dest_dir, FN_REFLEN, "%s/%s", + xtrabackup_target_dir, dbname); + srv_normalize_path_for_win(dest_dir); + + snprintf(dest_space_name, FN_REFLEN, "%s/%s", dbname, name); + } else { + snprintf(dest_dir, FN_REFLEN, "%s", xtrabackup_target_dir); + srv_normalize_path_for_win(dest_dir); + + snprintf(dest_space_name, FN_REFLEN, "%s", name); + } + + snprintf(real_name, real_name_len, + "%s/%s", + xtrabackup_target_dir, dest_space_name); + srv_normalize_path_for_win(real_name); + /* Truncate ".ibd" */ + dest_space_name[strlen(dest_space_name) - 4] = '\0'; + + /* Create the database directory if it doesn't exist yet */ + if (!os_file_create_directory(dest_dir, FALSE)) { + msg("xtrabackup: error: cannot create dir %s\n", dest_dir); + return file; + } + + if (!fil_is_user_tablespace_id(space_id)) { + goto found; + } + + /* remember space name for further reference */ + table = static_cast + (ut_malloc(sizeof(xb_filter_entry_t) + + strlen(dest_space_name) + 1)); + + table->name = ((char*)table) + sizeof(xb_filter_entry_t); + strcpy(table->name, dest_space_name); + HASH_INSERT(xb_filter_entry_t, name_hash, inc_dir_tables_hash, + ut_fold_string(table->name), table); + + mutex_enter(&fil_system->mutex); + fil_space = fil_space_get_by_name(dest_space_name); + mutex_exit(&fil_system->mutex); + + if (fil_space != NULL) { + if (fil_space->id == space_id || space_id == ULINT_UNDEFINED) { + /* we found matching space */ + goto found; + } else { + + char tmpname[FN_REFLEN]; + + snprintf(tmpname, FN_REFLEN, "%s/xtrabackup_tmp_#%lu", + dbname, fil_space->id); + + msg("xtrabackup: Renaming %s to %s.ibd\n", + fil_space->name, tmpname); + + if (!fil_rename_tablespace(NULL, fil_space->id, + tmpname, NULL)) + { + msg("xtrabackup: Cannot rename %s to %s\n", + fil_space->name, tmpname); + goto exit; + } + } + } + + if (space_id == ULINT_UNDEFINED) + { + msg("xtrabackup: Error: Cannot handle DDL operation on tablespace " + "%s\n", dest_space_name); + exit(EXIT_FAILURE); + } + mutex_enter(&fil_system->mutex); + fil_space = fil_space_get_by_id(space_id); + mutex_exit(&fil_system->mutex); + if (fil_space != NULL) { + char tmpname[FN_REFLEN]; + + strncpy(tmpname, dest_space_name, FN_REFLEN); + + msg("xtrabackup: Renaming %s to %s\n", + fil_space->name, dest_space_name); + + if (!fil_rename_tablespace(NULL, fil_space->id, tmpname, + NULL)) + { + msg("xtrabackup: Cannot rename %s to %s\n", + fil_space->name, dest_space_name); + goto exit; + } + + goto found; + } + + /* No matching space found. create the new one. */ + + if (!fil_space_create(dest_space_name, space_id, 0, + FIL_TABLESPACE)) { + msg("xtrabackup: Cannot create tablespace %s\n", + dest_space_name); + goto exit; + } + + /* Calculate correct tablespace flags for compressed tablespaces. */ + if (!zip_size || zip_size == ULINT_UNDEFINED) { + tablespace_flags = 0; + } + else { + tablespace_flags + = (get_bit_shift(zip_size >> PAGE_ZIP_MIN_SIZE_SHIFT + << 1) + << DICT_TF_ZSSIZE_SHIFT) + | DICT_TF_COMPACT + | (DICT_TF_FORMAT_ZIP << DICT_TF_FORMAT_SHIFT); + ut_a(dict_tf_get_zip_size(tablespace_flags) + == zip_size); + } + *success = xb_space_create_file(real_name, space_id, tablespace_flags, + &file); + goto exit; + +found: + /* open the file and return it's handle */ + + file = os_file_create_simple_no_error_handling(0, real_name, + OS_FILE_OPEN, + OS_FILE_READ_WRITE, + &ok); + + if (ok) { + *success = TRUE; + } else { + msg("xtrabackup: Cannot open file %s\n", real_name); + } + +exit: + + return file; +} + +/************************************************************************ +Applies a given .delta file to the corresponding data file. +@return TRUE on success */ +static +ibool +xtrabackup_apply_delta( + const char* dirname, /* in: dir name of incremental */ + const char* dbname, /* in: database name (ibdata: NULL) */ + const char* filename, /* in: file name (not a path), + including the .delta extension */ + void* /*data*/) +{ + os_file_t src_file = XB_FILE_UNDEFINED; + os_file_t dst_file = XB_FILE_UNDEFINED; + char src_path[FN_REFLEN]; + char dst_path[FN_REFLEN]; + char meta_path[FN_REFLEN]; + char space_name[FN_REFLEN]; + ibool success; + + ibool last_buffer = FALSE; + ulint page_in_buffer; + ulint incremental_buffers = 0; + + xb_delta_info_t info; + ulint page_size; + ulint page_size_shift; + byte* incremental_buffer_base = NULL; + byte* incremental_buffer; + + size_t offset; + + ut_a(xtrabackup_incremental); + + if (dbname) { + snprintf(src_path, sizeof(src_path), "%s/%s/%s", + dirname, dbname, filename); + snprintf(dst_path, sizeof(dst_path), "%s/%s/%s", + xtrabackup_real_target_dir, dbname, filename); + } else { + snprintf(src_path, sizeof(src_path), "%s/%s", + dirname, filename); + snprintf(dst_path, sizeof(dst_path), "%s/%s", + xtrabackup_real_target_dir, filename); + } + dst_path[strlen(dst_path) - 6] = '\0'; + + strncpy(space_name, filename, FN_REFLEN); + space_name[strlen(space_name) - 6] = 0; + + if (!get_meta_path(src_path, meta_path)) { + goto error; + } + + srv_normalize_path_for_win(dst_path); + srv_normalize_path_for_win(src_path); + srv_normalize_path_for_win(meta_path); + + if (!xb_read_delta_metadata(meta_path, &info)) { + goto error; + } + + page_size = info.page_size; + page_size_shift = get_bit_shift(page_size); + msg("xtrabackup: page size for %s is %lu bytes\n", + src_path, page_size); + if (page_size_shift < 10 || + page_size_shift > UNIV_PAGE_SIZE_SHIFT_MAX) { + msg("xtrabackup: error: invalid value of page_size " + "(%lu bytes) read from %s\n", page_size, meta_path); + goto error; + } + + src_file = os_file_create_simple_no_error_handling(0, src_path, + OS_FILE_OPEN, + OS_FILE_READ_WRITE, + &success); + if (!success) { + os_file_get_last_error(TRUE); + msg("xtrabackup: error: cannot open %s\n", src_path); + goto error; + } + + posix_fadvise(src_file, 0, 0, POSIX_FADV_SEQUENTIAL); + + os_file_set_nocache(src_file, src_path, "OPEN"); + + dst_file = xb_delta_open_matching_space( + dbname, space_name, info.space_id, info.zip_size, + dst_path, sizeof(dst_path), &success); + if (!success) { + msg("xtrabackup: error: cannot open %s\n", dst_path); + goto error; + } + + posix_fadvise(dst_file, 0, 0, POSIX_FADV_DONTNEED); + + os_file_set_nocache(dst_file, dst_path, "OPEN"); + + /* allocate buffer for incremental backup (4096 pages) */ + incremental_buffer_base = static_cast + (ut_malloc((UNIV_PAGE_SIZE_MAX / 4 + 1) * + UNIV_PAGE_SIZE_MAX)); + incremental_buffer = static_cast + (ut_align(incremental_buffer_base, + UNIV_PAGE_SIZE_MAX)); + + msg("Applying %s to %s...\n", src_path, dst_path); + + while (!last_buffer) { + ulint cluster_header; + + /* read to buffer */ + /* first block of block cluster */ + offset = ((incremental_buffers * (page_size / 4)) + << page_size_shift); + success = os_file_read(src_file, incremental_buffer, + offset, page_size); + if (!success) { + goto error; + } + + cluster_header = mach_read_from_4(incremental_buffer); + switch(cluster_header) { + case 0x78747261UL: /*"xtra"*/ + break; + case 0x58545241UL: /*"XTRA"*/ + last_buffer = TRUE; + break; + default: + msg("xtrabackup: error: %s seems not " + ".delta file.\n", src_path); + goto error; + } + + for (page_in_buffer = 1; page_in_buffer < page_size / 4; + page_in_buffer++) { + if (mach_read_from_4(incremental_buffer + page_in_buffer * 4) + == 0xFFFFFFFFUL) + break; + } + + ut_a(last_buffer || page_in_buffer == page_size / 4); + + /* read whole of the cluster */ + success = os_file_read(src_file, incremental_buffer, + offset, page_in_buffer * page_size); + if (!success) { + goto error; + } + + posix_fadvise(src_file, offset, page_in_buffer * page_size, + POSIX_FADV_DONTNEED); + + for (page_in_buffer = 1; page_in_buffer < page_size / 4; + page_in_buffer++) { + ulint offset_on_page; + + offset_on_page = mach_read_from_4(incremental_buffer + page_in_buffer * 4); + + if (offset_on_page == 0xFFFFFFFFUL) + break; + + success = os_file_write(dst_path, dst_file, + incremental_buffer + + page_in_buffer * page_size, + (offset_on_page << + page_size_shift), + page_size); + if (!success) { + goto error; + } + } + + incremental_buffers++; + } + + if (incremental_buffer_base) + ut_free(incremental_buffer_base); + if (src_file != XB_FILE_UNDEFINED) + os_file_close(src_file); + if (dst_file != XB_FILE_UNDEFINED) + os_file_close(dst_file); + return TRUE; + +error: + if (incremental_buffer_base) + ut_free(incremental_buffer_base); + if (src_file != XB_FILE_UNDEFINED) + os_file_close(src_file); + if (dst_file != XB_FILE_UNDEFINED) + os_file_close(dst_file); + msg("xtrabackup: Error: xtrabackup_apply_delta(): " + "failed to apply %s to %s.\n", src_path, dst_path); + return FALSE; +} + +/************************************************************************ +Callback to handle datadir entry. Function of this type will be called +for each entry which matches the mask by xb_process_datadir. +@return should return TRUE on success */ +typedef ibool (*handle_datadir_entry_func_t)( +/*=========================================*/ + const char* data_home_dir, /*!name, name)); + + if (!table) { + snprintf(name, FN_REFLEN, "%s/%s/%s", data_home_dir, + db_name, file_name); + return os_file_delete(0, name); + } + + return(TRUE); +} + +/************************************************************************ +Function enumerates files in datadir (provided by path) which are matched +by provided suffix. For each entry callback is called. +@return FALSE if callback for some entry returned FALSE */ +static +ibool +xb_process_datadir( + const char* path, /*! suffix_len + && 0 == strcmp(fileinfo.name + + strlen(fileinfo.name) - suffix_len, + suffix)) { + if (!func( + path, NULL, + fileinfo.name, data)) + { + return(FALSE); + } + } +next_file_item_1: + ret = fil_file_readdir_next_file(&err, + path, dbdir, + &fileinfo); + } + + os_file_closedir(dbdir); + } else { + msg("xtrabackup: Cannot open dir %s\n", + path); + } + + /* single table tablespaces */ + dir = os_file_opendir(path, FALSE); + + if (dir == NULL) { + msg("xtrabackup: Cannot open dir %s\n", + path); + } + + ret = fil_file_readdir_next_file(&err, path, dir, + &dbinfo); + while (ret == 0) { + if (dbinfo.type == OS_FILE_TYPE_FILE + || dbinfo.type == OS_FILE_TYPE_UNKNOWN) { + + goto next_datadir_item; + } + + sprintf(dbpath, "%s/%s", path, + dbinfo.name); + srv_normalize_path_for_win(dbpath); + + dbdir = os_file_opendir(dbpath, FALSE); + + if (dbdir != NULL) { + + ret = fil_file_readdir_next_file(&err, dbpath, dbdir, + &fileinfo); + while (ret == 0) { + + if (fileinfo.type == OS_FILE_TYPE_DIR) { + + goto next_file_item_2; + } + + if (strlen(fileinfo.name) > suffix_len + && 0 == strcmp(fileinfo.name + + strlen(fileinfo.name) - + suffix_len, + suffix)) { + /* The name ends in suffix; process + the file */ + if (!func( + path, + dbinfo.name, + fileinfo.name, data)) + { + return(FALSE); + } + } +next_file_item_2: + ret = fil_file_readdir_next_file(&err, + dbpath, dbdir, + &fileinfo); + } + + os_file_closedir(dbdir); + } +next_datadir_item: + ret = fil_file_readdir_next_file(&err, + path, + dir, &dbinfo); + } + + os_file_closedir(dir); + + return(TRUE); +} + +/************************************************************************ +Applies all .delta files from incremental_dir to the full backup. +@return TRUE on success. */ +static +ibool +xtrabackup_apply_deltas() +{ + return xb_process_datadir(xtrabackup_incremental_dir, ".delta", + xtrabackup_apply_delta, NULL); +} + +static my_bool +xtrabackup_close_temp_log(my_bool clear_flag) +{ + os_file_t src_file = XB_FILE_UNDEFINED; + char src_path[FN_REFLEN]; + char dst_path[FN_REFLEN]; + ibool success; + byte log_buf[UNIV_PAGE_SIZE_MAX]; + + if (!xtrabackup_logfile_is_renamed) + return(FALSE); + + /* rename 'ib_logfile0' to 'xtrabackup_logfile' */ + if(!xtrabackup_incremental_dir) { + sprintf(dst_path, "%s/ib_logfile0", xtrabackup_target_dir); + sprintf(src_path, "%s/%s", xtrabackup_target_dir, + XB_LOG_FILENAME); + } else { + sprintf(dst_path, "%s/ib_logfile0", xtrabackup_incremental_dir); + sprintf(src_path, "%s/%s", xtrabackup_incremental_dir, + XB_LOG_FILENAME); + } + + srv_normalize_path_for_win(dst_path); + srv_normalize_path_for_win(src_path); + + success = os_file_rename(0, dst_path, src_path); + if (!success) { + goto error; + } + xtrabackup_logfile_is_renamed = FALSE; + + if (!clear_flag) + return(FALSE); + + /* clear LOG_FILE_WAS_CREATED_BY_HOT_BACKUP field */ + src_file = os_file_create_simple_no_error_handling(0, src_path, + OS_FILE_OPEN, + OS_FILE_READ_WRITE, + &success); + if (!success) { + goto error; + } + + success = os_file_read(src_file, log_buf, 0, LOG_FILE_HDR_SIZE); + if (!success) { + goto error; + } + + memset(log_buf + LOG_FILE_WAS_CREATED_BY_HOT_BACKUP, ' ', 4); + + success = os_file_write(src_path, src_file, log_buf, 0, + LOG_FILE_HDR_SIZE); + if (!success) { + goto error; + } + + os_file_close(src_file); + src_file = XB_FILE_UNDEFINED; + + innobase_log_files_in_group = innobase_log_files_in_group_save; + srv_log_group_home_dir = srv_log_group_home_dir_save; + innobase_log_file_size = innobase_log_file_size_save; + + return(FALSE); +error: + if (src_file != XB_FILE_UNDEFINED) + os_file_close(src_file); + msg("xtrabackup: Error: xtrabackup_close_temp_log() failed.\n"); + return(TRUE); /*ERROR*/ +} + + +/*********************************************************************//** +Write the meta data (index user fields) config file. +@return true in case of success otherwise false. */ +static +bool +xb_export_cfg_write_index_fields( +/*===========================*/ + const dict_index_t* index, /*!< in: write the meta data for + this index */ + FILE* file) /*!< in: file to write to */ +{ + byte row[sizeof(ib_uint32_t) * 2]; + + for (ulint i = 0; i < index->n_fields; ++i) { + byte* ptr = row; + const dict_field_t* field = &index->fields[i]; + + mach_write_to_4(ptr, field->prefix_len); + ptr += sizeof(ib_uint32_t); + + mach_write_to_4(ptr, field->fixed_len); + + if (fwrite(row, 1, sizeof(row), file) != sizeof(row)) { + + msg("xtrabackup: Error: writing index fields."); + + return(false); + } + + /* Include the NUL byte in the length. */ + ib_uint32_t len = strlen(field->name) + 1; + ut_a(len > 1); + + mach_write_to_4(row, len); + + if (fwrite(row, 1, sizeof(len), file) != sizeof(len) + || fwrite(field->name, 1, len, file) != len) { + + msg("xtrabackup: Error: writing index column."); + + return(false); + } + } + + return(true); +} + +/*********************************************************************//** +Write the meta data config file index information. +@return true in case of success otherwise false. */ +static __attribute__((nonnull, warn_unused_result)) +bool +xb_export_cfg_write_indexes( +/*======================*/ + const dict_table_t* table, /*!< in: write the meta data for + this table */ + FILE* file) /*!< in: file to write to */ +{ + { + byte row[sizeof(ib_uint32_t)]; + + /* Write the number of indexes in the table. */ + mach_write_to_4(row, UT_LIST_GET_LEN(table->indexes)); + + if (fwrite(row, 1, sizeof(row), file) != sizeof(row)) { + msg("xtrabackup: Error: writing index count."); + + return(false); + } + } + + bool ret = true; + + /* Write the index meta data. */ + for (const dict_index_t* index = UT_LIST_GET_FIRST(table->indexes); + index != 0 && ret; + index = UT_LIST_GET_NEXT(indexes, index)) { + + byte* ptr; + byte row[sizeof(ib_uint64_t) + + sizeof(ib_uint32_t) * 8]; + + ptr = row; + + ut_ad(sizeof(ib_uint64_t) == 8); + mach_write_to_8(ptr, index->id); + ptr += sizeof(ib_uint64_t); + + mach_write_to_4(ptr, index->space); + ptr += sizeof(ib_uint32_t); + + mach_write_to_4(ptr, index->page); + ptr += sizeof(ib_uint32_t); + + mach_write_to_4(ptr, index->type); + ptr += sizeof(ib_uint32_t); + + mach_write_to_4(ptr, index->trx_id_offset); + ptr += sizeof(ib_uint32_t); + + mach_write_to_4(ptr, index->n_user_defined_cols); + ptr += sizeof(ib_uint32_t); + + mach_write_to_4(ptr, index->n_uniq); + ptr += sizeof(ib_uint32_t); + + mach_write_to_4(ptr, index->n_nullable); + ptr += sizeof(ib_uint32_t); + + mach_write_to_4(ptr, index->n_fields); + + if (fwrite(row, 1, sizeof(row), file) != sizeof(row)) { + + msg("xtrabackup: Error: writing index meta-data."); + + return(false); + } + + /* Write the length of the index name. + NUL byte is included in the length. */ + ib_uint32_t len = strlen(index->name) + 1; + ut_a(len > 1); + + mach_write_to_4(row, len); + + if (fwrite(row, 1, sizeof(len), file) != sizeof(len) + || fwrite(index->name, 1, len, file) != len) { + + msg("xtrabackup: Error: writing index name."); + + return(false); + } + + ret = xb_export_cfg_write_index_fields(index, file); + } + + return(ret); +} + +/*********************************************************************//** +Write the meta data (table columns) config file. Serialise the contents of +dict_col_t structure, along with the column name. All fields are serialized +as ib_uint32_t. +@return true in case of success otherwise false. */ +static __attribute__((nonnull, warn_unused_result)) +bool +xb_export_cfg_write_table( +/*====================*/ + const dict_table_t* table, /*!< in: write the meta data for + this table */ + FILE* file) /*!< in: file to write to */ +{ + dict_col_t* col; + byte row[sizeof(ib_uint32_t) * 7]; + + col = table->cols; + + for (ulint i = 0; i < table->n_cols; ++i, ++col) { + byte* ptr = row; + + mach_write_to_4(ptr, col->prtype); + ptr += sizeof(ib_uint32_t); + + mach_write_to_4(ptr, col->mtype); + ptr += sizeof(ib_uint32_t); + + mach_write_to_4(ptr, col->len); + ptr += sizeof(ib_uint32_t); + + mach_write_to_4(ptr, col->mbminmaxlen); + ptr += sizeof(ib_uint32_t); + + mach_write_to_4(ptr, col->ind); + ptr += sizeof(ib_uint32_t); + + mach_write_to_4(ptr, col->ord_part); + ptr += sizeof(ib_uint32_t); + + mach_write_to_4(ptr, col->max_prefix); + + if (fwrite(row, 1, sizeof(row), file) != sizeof(row)) { + msg("xtrabackup: Error: writing table column data."); + + return(false); + } + + /* Write out the column name as [len, byte array]. The len + includes the NUL byte. */ + ib_uint32_t len; + const char* col_name; + + col_name = dict_table_get_col_name(table, dict_col_get_no(col)); + + /* Include the NUL byte in the length. */ + len = strlen(col_name) + 1; + ut_a(len > 1); + + mach_write_to_4(row, len); + + if (fwrite(row, 1, sizeof(len), file) != sizeof(len) + || fwrite(col_name, 1, len, file) != len) { + + msg("xtrabackup: Error: writing column name."); + + return(false); + } + } + + return(true); +} + +/*********************************************************************//** +Write the meta data config file header. +@return true in case of success otherwise false. */ +static __attribute__((nonnull, warn_unused_result)) +bool +xb_export_cfg_write_header( +/*=====================*/ + const dict_table_t* table, /*!< in: write the meta data for + this table */ + FILE* file) /*!< in: file to write to */ +{ + byte value[sizeof(ib_uint32_t)]; + + /* Write the meta-data version number. */ + mach_write_to_4(value, IB_EXPORT_CFG_VERSION_V1); + + if (fwrite(&value, 1, sizeof(value), file) != sizeof(value)) { + msg("xtrabackup: Error: writing meta-data version number."); + + return(false); + } + + /* Write the server hostname. */ + ib_uint32_t len; + const char* hostname = "Hostname unknown"; + + /* The server hostname includes the NUL byte. */ + len = strlen(hostname) + 1; + mach_write_to_4(value, len); + + if (fwrite(&value, 1, sizeof(value), file) != sizeof(value) + || fwrite(hostname, 1, len, file) != len) { + + msg("xtrabackup: Error: writing hostname."); + + return(false); + } + + /* The table name includes the NUL byte. */ + ut_a(table->name != 0); + len = strlen(table->name) + 1; + + /* Write the table name. */ + mach_write_to_4(value, len); + + if (fwrite(&value, 1, sizeof(value), file) != sizeof(value) + || fwrite(table->name, 1, len, file) != len) { + + msg("xtrabackup: Error: writing table name."); + + return(false); + } + + byte row[sizeof(ib_uint32_t) * 3]; + + /* Write the next autoinc value. */ + mach_write_to_8(row, table->autoinc); + + if (fwrite(row, 1, sizeof(ib_uint64_t), file) != sizeof(ib_uint64_t)) { + msg("xtrabackup: Error: writing table autoinc value."); + + return(false); + } + + byte* ptr = row; + + /* Write the system page size. */ + mach_write_to_4(ptr, UNIV_PAGE_SIZE); + ptr += sizeof(ib_uint32_t); + + /* Write the table->flags. */ + mach_write_to_4(ptr, table->flags); + ptr += sizeof(ib_uint32_t); + + /* Write the number of columns in the table. */ + mach_write_to_4(ptr, table->n_cols); + + if (fwrite(row, 1, sizeof(row), file) != sizeof(row)) { + msg("xtrabackup: Error: writing table meta-data."); + + return(false); + } + + return(true); +} + +/*********************************************************************//** +Write MySQL 5.6-style meta data config file. +@return true in case of success otherwise false. */ +static +bool +xb_export_cfg_write( + const fil_node_t* node, + const dict_table_t* table) /*!< in: write the meta data for + this table */ +{ + char file_path[FN_REFLEN]; + FILE* file; + bool success; + + strcpy(file_path, node->name); + strcpy(file_path + strlen(file_path) - 4, ".cfg"); + + file = fopen(file_path, "w+b"); + + if (file == NULL) { + msg("xtrabackup: Error: cannot close %s\n", node->name); + + success = false; + } else { + + success = xb_export_cfg_write_header(table, file); + + if (success) { + success = xb_export_cfg_write_table(table, file); + } + + if (success) { + success = xb_export_cfg_write_indexes(table, file); + } + + if (fclose(file) != 0) { + msg("xtrabackup: Error: cannot close %s\n", node->name); + success = false; + } + + } + + return(success); + +} + +/********************************************************************//** +Searches archived log files in archived log directory. The min and max +LSN's of found files as well as archived log file size are stored in +xtrabackup_arch_first_file_lsn, xtrabackup_arch_last_file_lsn and +xtrabackup_arch_file_size respectively. +@return true on success +*/ +static +bool +xtrabackup_arch_search_files( +/*=========================*/ + ib_uint64_t start_lsn) /*!< in: filter out log files + witch does not contain data + with lsn < start_lsn */ +{ + os_file_dir_t dir; + os_file_stat_t fileinfo; + ut_ad(innobase_log_arch_dir); + + dir = os_file_opendir(innobase_log_arch_dir, FALSE); + if (!dir) { + msg("xtrabackup: error: cannot open archived log directory %s\n", + innobase_log_arch_dir); + return false; + } + + while(!os_file_readdir_next_file(innobase_log_arch_dir, + dir, + &fileinfo) ) { + lsn_t log_file_lsn; + char* log_str_end_lsn_ptr; + + if (strncmp(fileinfo.name, + IB_ARCHIVED_LOGS_PREFIX, + sizeof(IB_ARCHIVED_LOGS_PREFIX) - 1)) { + continue; + } + + log_file_lsn = strtoll(fileinfo.name + + sizeof(IB_ARCHIVED_LOGS_PREFIX) - 1, + &log_str_end_lsn_ptr, 10); + + if (*log_str_end_lsn_ptr) { + continue; + } + + if (log_file_lsn + (fileinfo.size - LOG_FILE_HDR_SIZE) < start_lsn) { + continue; + } + + if (!xtrabackup_arch_first_file_lsn || + log_file_lsn < xtrabackup_arch_first_file_lsn) { + xtrabackup_arch_first_file_lsn = log_file_lsn; + } + if (log_file_lsn > xtrabackup_arch_last_file_lsn) { + xtrabackup_arch_last_file_lsn = log_file_lsn; + } + + //TODO: find the more suitable way to extract archived log file + //size + if (fileinfo.size > (ib_int64_t)xtrabackup_arch_file_size) { + xtrabackup_arch_file_size = fileinfo.size; + } + } + + return xtrabackup_arch_first_file_lsn != 0; +} + +static +void +innodb_free_param() +{ + srv_free_paths_and_sizes(); + free(internal_innobase_data_file_path); + internal_innobase_data_file_path = NULL; + free_tmpdir(&mysql_tmpdir_list); +} + + +/************************************************************************** +Store the current binary log coordinates in a specified file. +@return 'false' on error. */ +static bool +store_binlog_info( +/*==============*/ + const char *filename) /*!< in: output file name */ +{ + FILE *fp; + + if (trx_sys_mysql_bin_log_name[0] == '\0') { + return(true); + } + + fp = fopen(filename, "w"); + + if (!fp) { + msg("xtrabackup: failed to open '%s'\n", filename); + return(false); + } + + fprintf(fp, "%s\t" UINT64PF "\n", + trx_sys_mysql_bin_log_name, trx_sys_mysql_bin_log_pos); + fclose(fp); + + return(true); +} + +static void +xtrabackup_prepare_func(void) +{ + ulint err; + datafiles_iter_t *it; + fil_node_t *node; + fil_space_t *space; + char metadata_path[FN_REFLEN]; + + /* cd to target-dir */ + + if (my_setwd(xtrabackup_real_target_dir,MYF(MY_WME))) + { + msg("xtrabackup: cannot my_setwd %s\n", + xtrabackup_real_target_dir); + exit(EXIT_FAILURE); + } + msg("xtrabackup: cd to %s\n", xtrabackup_real_target_dir); + + xtrabackup_target_dir= mysql_data_home_buff; + xtrabackup_target_dir[0]=FN_CURLIB; // all paths are relative from here + xtrabackup_target_dir[1]=0; + + /* + read metadata of target, we don't need metadata reading in the case + archived logs applying + */ + sprintf(metadata_path, "%s/%s", xtrabackup_target_dir, + XTRABACKUP_METADATA_FILENAME); + + if (!xtrabackup_read_metadata(metadata_path)) { + msg("xtrabackup: Error: failed to read metadata from '%s'\n", + metadata_path); + exit(EXIT_FAILURE); + } + + if (!innobase_log_arch_dir) + { + if (!strcmp(metadata_type, "full-backuped")) { + msg("xtrabackup: This target seems to be not prepared " + "yet.\n"); + } else if (!strcmp(metadata_type, "log-applied")) { + msg("xtrabackup: This target seems to be already " + "prepared with --apply-log-only.\n"); + goto skip_check; + } else if (!strcmp(metadata_type, "full-prepared")) { + msg("xtrabackup: This target seems to be already " + "prepared.\n"); + } else { + msg("xtrabackup: This target seems not to have correct " + "metadata...\n"); + exit(EXIT_FAILURE); + } + + if (xtrabackup_incremental) { + msg("xtrabackup: error: applying incremental backup " + "needs target prepared with --apply-log-only.\n"); + exit(EXIT_FAILURE); + } +skip_check: + if (xtrabackup_incremental + && metadata_to_lsn != incremental_lsn) { + msg("xtrabackup: error: This incremental backup seems " + "not to be proper for the target.\n" + "xtrabackup: Check 'to_lsn' of the target and " + "'from_lsn' of the incremental.\n"); + exit(EXIT_FAILURE); + } + } + + /* Create logfiles for recovery from 'xtrabackup_logfile', before start InnoDB */ + srv_max_n_threads = 1000; + os_sync_mutex = NULL; + ut_mem_init(); + /* temporally dummy value to avoid crash */ + srv_page_size_shift = 14; + srv_page_size = (1 << srv_page_size_shift); + os_sync_init(); + sync_init(); + os_io_init_simple(); + mem_init(srv_mem_pool_size); + ut_crc32_init(); + + xb_filters_init(); + + if(!innobase_log_arch_dir && xtrabackup_init_temp_log()) + goto error_cleanup; + + if(innodb_init_param()) { + goto error_cleanup; + } + + /* Expand compacted datafiles */ + + if (xtrabackup_compact) { + srv_compact_backup = TRUE; + + if (!xb_expand_datafiles()) { + goto error_cleanup; + } + + /* Reset the 'compact' flag in xtrabackup_checkpoints so we + don't expand on subsequent invocations. */ + xtrabackup_compact = FALSE; + if (!xtrabackup_write_metadata(metadata_path)) { + msg("xtrabackup: error: xtrabackup_write_metadata() " + "failed\n"); + goto error_cleanup; + } + } + + xb_normalize_init_values(); + + if (xtrabackup_incremental || innobase_log_arch_dir) { + err = xb_data_files_init(); + if (err != DB_SUCCESS) { + msg("xtrabackup: error: xb_data_files_init() failed " + "with error code %lu\n", err); + goto error_cleanup; + } + } + if (xtrabackup_incremental) { + inc_dir_tables_hash = hash_create(1000); + + if(!xtrabackup_apply_deltas()) { + xb_data_files_close(); + xb_filter_hash_free(inc_dir_tables_hash); + goto error_cleanup; + } + } + if (xtrabackup_incremental || innobase_log_arch_dir) { + xb_data_files_close(); + } + if (xtrabackup_incremental) { + /* Cleanup datadir from tablespaces deleted between full and + incremental backups */ + + xb_process_datadir("./", ".ibd", rm_if_not_found, NULL); + + xb_filter_hash_free(inc_dir_tables_hash); + } + sync_close(); + sync_initialized = FALSE; + if (fil_system) { + fil_close(); + } + os_sync_free(); + mem_close(); + os_sync_mutex = NULL; + ut_free_all_mem(); + + innodb_free_param(); + + /* Reset the configuration as it might have been changed by + xb_data_files_init(). */ + if(innodb_init_param()) { + goto error_cleanup; + } + + srv_apply_log_only = (ibool) xtrabackup_apply_log_only; + srv_rebuild_indexes = (ibool) xtrabackup_rebuild_indexes; + + /* increase IO threads */ + if(srv_n_file_io_threads < 10) { + srv_n_read_io_threads = 4; + srv_n_write_io_threads = 4; + } + + if (innobase_log_arch_dir) { + srv_arch_dir = innobase_log_arch_dir; + srv_archive_recovery = TRUE; + if (xtrabackup_archived_to_lsn) { + if (xtrabackup_archived_to_lsn < metadata_last_lsn) { + msg("xtrabackup: warning: logs applying lsn " + "limit " UINT64PF " is " + "less than metadata last-lsn " UINT64PF + " and will be set to metadata last-lsn value\n", + xtrabackup_archived_to_lsn, + metadata_last_lsn); + xtrabackup_archived_to_lsn = metadata_last_lsn; + } + if (xtrabackup_archived_to_lsn < min_flushed_lsn) { + msg("xtrabackup: error: logs applying " + "lsn limit " UINT64PF " is less than " + "min_flushed_lsn " UINT64PF + ", there is nothing to do\n", + xtrabackup_archived_to_lsn, + min_flushed_lsn); + goto error_cleanup; + } + } + srv_archive_recovery_limit_lsn= xtrabackup_archived_to_lsn; + /* + Unfinished transactions are not rolled back during log applying + as they can be finished at the firther files applyings. + */ + srv_apply_log_only = xtrabackup_apply_log_only = TRUE; + + if (!xtrabackup_arch_search_files(min_flushed_lsn)) { + goto error_cleanup; + } + + /* + Check if last log file last lsn is big enough to overlap + last scanned lsn read from metadata. + */ + if (xtrabackup_arch_last_file_lsn + + xtrabackup_arch_file_size - + LOG_FILE_HDR_SIZE < metadata_last_lsn) { + msg("xtrabackup: error: there are no enough archived logs " + "to apply\n"); + goto error_cleanup; + } + } + + msg("xtrabackup: Starting InnoDB instance for recovery.\n" + "xtrabackup: Using %lld bytes for buffer pool " + "(set by --use-memory parameter)\n", xtrabackup_use_memory); + + if(innodb_init()) + goto error_cleanup; + + it = datafiles_iter_new(fil_system); + if (it == NULL) { + msg("xtrabackup: Error: datafiles_iter_new() failed.\n"); + exit(EXIT_FAILURE); + } + + while ((node = datafiles_iter_next(it)) != NULL) { + byte *header; + ulint size; + ulint actual_size; + mtr_t mtr; + buf_block_t *block; + ulint flags; + + space = node->space; + + /* Align space sizes along with fsp header. We want to process + each space once, so skip all nodes except the first one in a + multi-node space. */ + if (UT_LIST_GET_PREV(chain, node) != NULL) { + continue; + } + + mtr_start(&mtr); + + mtr_s_lock(fil_space_get_latch(space->id, &flags), &mtr); + + block = buf_page_get(space->id, + dict_tf_get_zip_size(flags), + 0, RW_S_LATCH, &mtr); + header = FSP_HEADER_OFFSET + buf_block_get_frame(block); + + size = mtr_read_ulint(header + FSP_SIZE, MLOG_4BYTES, + &mtr); + + mtr_commit(&mtr); + + fil_extend_space_to_desired_size(&actual_size, space->id, size); + } + + datafiles_iter_free(it); + + if (xtrabackup_export) { + msg("xtrabackup: export option is specified.\n"); + os_file_t info_file = XB_FILE_UNDEFINED; + char info_file_path[FN_REFLEN]; + ibool success; + char table_name[FN_REFLEN]; + + byte* page; + byte* buf = NULL; + + buf = static_cast(ut_malloc(UNIV_PAGE_SIZE * 2)); + page = static_cast(ut_align(buf, UNIV_PAGE_SIZE)); + + /* flush insert buffer at shutdwon */ + innobase_fast_shutdown = 0; + + it = datafiles_iter_new(fil_system); + if (it == NULL) { + msg("xtrabackup: Error: datafiles_iter_new() " + "failed.\n"); + exit(EXIT_FAILURE); + } + while ((node = datafiles_iter_next(it)) != NULL) { + int len; + char *next, *prev, *p; + dict_table_t* table; + dict_index_t* index; + ulint n_index; + + space = node->space; + + /* treat file_per_table only */ + if (!fil_is_user_tablespace_id(space->id)) { + continue; + } + + /* node exist == file exist, here */ + strcpy(info_file_path, node->name); + strcpy(info_file_path + + strlen(info_file_path) - + 4, ".exp"); + + len = strlen(info_file_path); + + p = info_file_path; + prev = NULL; + while ((next = strchr(p, SRV_PATH_SEPARATOR)) != NULL) + { + prev = p; + p = next + 1; + } + info_file_path[len - 4] = 0; + strncpy(table_name, prev, FN_REFLEN); + + info_file_path[len - 4] = '.'; + + mutex_enter(&(dict_sys->mutex)); + + table = dict_table_get_low(table_name); + if (!table) { + msg("xtrabackup: error: " + "cannot find dictionary " + "record of table %s\n", + table_name); + goto next_node; + } + index = dict_table_get_first_index(table); + n_index = UT_LIST_GET_LEN(table->indexes); + if (n_index > 31) { + msg("xtrabackup: error: " + "sorry, cannot export over " + "31 indexes for now.\n"); + goto next_node; + } + + /* Write MySQL 5.6 .cfg file */ + if (!xb_export_cfg_write(node, table)) { + goto next_node; + } + + /* init exp file */ + memset(page, 0, UNIV_PAGE_SIZE); + mach_write_to_4(page , 0x78706f72UL); + mach_write_to_4(page + 4, 0x74696e66UL);/*"xportinf"*/ + mach_write_to_4(page + 8, n_index); + strncpy((char *) page + 12, + table_name, 500); + + msg("xtrabackup: export metadata of " + "table '%s' to file `%s` " + "(%lu indexes)\n", + table_name, info_file_path, + n_index); + + n_index = 1; + while (index) { + mach_write_to_8(page + n_index * 512, index->id); + mach_write_to_4(page + n_index * 512 + 8, + index->page); + strncpy((char *) page + n_index * 512 + + 12, index->name, 500); + + msg("xtrabackup: name=%s, " + "id.low=%lu, page=%lu\n", + index->name, + (ulint)(index->id & + 0xFFFFFFFFUL), + (ulint) index->page); + index = dict_table_get_next_index(index); + n_index++; + } + + srv_normalize_path_for_win(info_file_path); + info_file = os_file_create( + 0, + info_file_path, + OS_FILE_OVERWRITE, + OS_FILE_NORMAL, OS_DATA_FILE, + &success); + if (!success) { + os_file_get_last_error(TRUE); + goto next_node; + } + success = os_file_write(info_file_path, + info_file, page, + 0, UNIV_PAGE_SIZE); + if (!success) { + os_file_get_last_error(TRUE); + goto next_node; + } + success = os_file_flush(info_file); + if (!success) { + os_file_get_last_error(TRUE); + goto next_node; + } +next_node: + if (info_file != XB_FILE_UNDEFINED) { + os_file_close(info_file); + info_file = XB_FILE_UNDEFINED; + } + mutex_exit(&(dict_sys->mutex)); + } + + ut_free(buf); + } + + /* print the binary log position */ + trx_sys_print_mysql_binlog_offset(); + msg("\n"); + + /* output to xtrabackup_binlog_pos_innodb and (if + backup_safe_binlog_info was available on the server) to + xtrabackup_binlog_info. In the latter case xtrabackup_binlog_pos_innodb + becomes redundant and is created only for compatibility. */ + if (!store_binlog_info("xtrabackup_binlog_pos_innodb") || + (recover_binlog_info && + !store_binlog_info(XTRABACKUP_BINLOG_INFO))) { + + exit(EXIT_FAILURE); + } + + if (innobase_log_arch_dir) + srv_start_lsn = log_sys->lsn = recv_sys->recovered_lsn; + + /* Check whether the log is applied enough or not. */ + if ((xtrabackup_incremental + && srv_start_lsn < incremental_to_lsn) + ||(!xtrabackup_incremental + && srv_start_lsn < metadata_to_lsn)) { + msg("xtrabackup: error: " + "The transaction log file is corrupted.\n" + "xtrabackup: error: " + "The log was not applied to the intended LSN!\n"); + msg("xtrabackup: Log applied to lsn " LSN_PF "\n", + srv_start_lsn); + if (xtrabackup_incremental) { + msg("xtrabackup: The intended lsn is " LSN_PF "\n", + incremental_to_lsn); + } else { + msg("xtrabackup: The intended lsn is " LSN_PF "\n", + metadata_to_lsn); + } + exit(EXIT_FAILURE); + } + + xb_write_galera_info(xtrabackup_incremental); + + if(innodb_end()) + goto error_cleanup; + + innodb_free_param(); + + sync_initialized = FALSE; + os_sync_mutex = NULL; + + /* re-init necessary components */ + ut_mem_init(); + os_sync_init(); + sync_init(); + os_io_init_simple(); + + if(xtrabackup_close_temp_log(TRUE)) + exit(EXIT_FAILURE); + + /* output to metadata file */ + { + char filename[FN_REFLEN]; + + strcpy(metadata_type, srv_apply_log_only ? + "log-applied" : "full-prepared"); + + if(xtrabackup_incremental + && metadata_to_lsn < incremental_to_lsn) + { + metadata_to_lsn = incremental_to_lsn; + metadata_last_lsn = incremental_last_lsn; + } + + sprintf(filename, "%s/%s", xtrabackup_target_dir, XTRABACKUP_METADATA_FILENAME); + if (!xtrabackup_write_metadata(filename)) { + + msg("xtrabackup: Error: failed to write metadata " + "to '%s'\n", filename); + exit(EXIT_FAILURE); + } + + if(xtrabackup_extra_lsndir) { + sprintf(filename, "%s/%s", xtrabackup_extra_lsndir, XTRABACKUP_METADATA_FILENAME); + if (!xtrabackup_write_metadata(filename)) { + msg("xtrabackup: Error: failed to write " + "metadata to '%s'\n", filename); + exit(EXIT_FAILURE); + } + } + } + + if (!apply_log_finish()) { + exit(EXIT_FAILURE); + } + + sync_close(); + sync_initialized = FALSE; + if (fil_system) { + fil_close(); + } + os_sync_free(); + // mem_close(); + os_sync_mutex = NULL; + ut_free_all_mem(); + + /* start InnoDB once again to create log files */ + + if (!xtrabackup_apply_log_only) { + + if(innodb_init_param()) { + goto error; + } + + srv_apply_log_only = FALSE; + srv_rebuild_indexes = FALSE; + + /* increase IO threads */ + if(srv_n_file_io_threads < 10) { + srv_n_read_io_threads = 4; + srv_n_write_io_threads = 4; + } + + srv_shutdown_state = SRV_SHUTDOWN_NONE; + + if(innodb_init()) + goto error; + + if(innodb_end()) + goto error; + + innodb_free_param(); + + } + + xb_filters_free(); + + return; + +error_cleanup: + xtrabackup_close_temp_log(FALSE); + xb_filters_free(); + +error: + exit(EXIT_FAILURE); +} + +/************************************************************************** +Signals-related setup. */ +static +void +setup_signals() +/*===========*/ +{ + struct sigaction sa; + + /* Print a stacktrace on some signals */ + sa.sa_flags = SA_RESETHAND | SA_NODEFER; + sigemptyset(&sa.sa_mask); + sigprocmask(SIG_SETMASK,&sa.sa_mask,NULL); +#ifdef HAVE_STACKTRACE + my_init_stacktrace(); +#endif + sa.sa_handler = handle_fatal_signal; + sigaction(SIGSEGV, &sa, NULL); + sigaction(SIGABRT, &sa, NULL); + sigaction(SIGBUS, &sa, NULL); + sigaction(SIGILL, &sa, NULL); + sigaction(SIGFPE, &sa, NULL); + +#ifdef __linux__ + /* Ensure xtrabackup process is killed when the parent one + (innobackupex) is terminated with an unhandled signal */ + + if (prctl(PR_SET_PDEATHSIG, SIGKILL)) { + msg("prctl() failed with errno = %d\n", errno); + exit(EXIT_FAILURE); + } +#endif +} + +/************************************************************************** +Append group name to xb_load_default_groups list. */ +static +void +append_defaults_group(const char *group, const char *default_groups[], + size_t default_groups_size) +{ + uint i; + bool appended = false; + for (i = 0; i < default_groups_size - 1; i++) { + if (default_groups[i] == NULL) { + default_groups[i] = group; + appended = true; + break; + } + } + ut_a(appended); +} + +bool +xb_init() +{ + const char *mixed_options[4] = {NULL, NULL, NULL, NULL}; + int n_mixed_options; + + /* sanity checks */ + + if (opt_slave_info + && opt_no_lock + && !opt_safe_slave_backup) { + msg("Error: --slave-info is used with --no-lock but " + "without --safe-slave-backup. The binlog position " + "cannot be consistent with the backup data.\n"); + return(false); + } + + if (opt_rsync && xtrabackup_stream_fmt) { + msg("Error: --rsync doesn't work with --stream\n"); + return(false); + } + + n_mixed_options = 0; + + if (opt_decompress) { + mixed_options[n_mixed_options++] = "--decompress"; + } else if (opt_decrypt) { + mixed_options[n_mixed_options++] = "--decrypt"; + } + + if (xtrabackup_copy_back) { + mixed_options[n_mixed_options++] = "--copy-back"; + } + + if (xtrabackup_move_back) { + mixed_options[n_mixed_options++] = "--move-back"; + } + + if (xtrabackup_prepare) { + mixed_options[n_mixed_options++] = "--apply-log"; + } + + if (n_mixed_options > 1) { + msg("Error: %s and %s are mutually exclusive\n", + mixed_options[0], mixed_options[1]); + return(false); + } + + if (xtrabackup_backup) { + + if (!opt_noversioncheck) { + version_check(); + } + + if ((mysql_connection = xb_mysql_connect()) == NULL) { + return(false); + } + + if (!get_mysql_vars(mysql_connection)) { + return(false); + } + + history_start_time = time(NULL); + + } + + return(true); +} + +void +handle_options(int argc, char **argv, char ***argv_client, char ***argv_server) +{ + int i; + int ho_error; + + char* target_dir = NULL; + bool prepare = false; + + char conf_file[FN_REFLEN]; + int argc_client = argc; + int argc_server = argc; + + *argv_client = argv; + *argv_server = argv; + + /* scan options for group and config file to load defaults from */ + for (i = 1; i < argc; i++) { + + char *optend = strcend(argv[i], '='); + + if (strncmp(argv[i], "--defaults-group", + optend - argv[i]) == 0) { + defaults_group = optend + 1; + append_defaults_group(defaults_group, + xb_server_default_groups, + array_elements(xb_server_default_groups)); + } + + if (strncmp(argv[i], "--login-path", + optend - argv[i]) == 0) { + append_defaults_group(optend + 1, + xb_client_default_groups, + array_elements(xb_client_default_groups)); + } + + if (!strncmp(argv[i], "--prepare", + optend - argv[i])) { + prepare = true; + } + + if (!strncmp(argv[i], "--apply-log", + optend - argv[i])) { + prepare = true; + } + + if (!strncmp(argv[i], "--target-dir", + optend - argv[i]) && *optend) { + target_dir = optend + 1; + } + + if (!*optend && argv[i][0] != '-') { + target_dir = argv[i]; + } + } + + snprintf(conf_file, sizeof(conf_file), "my"); + + if (prepare && target_dir) { + snprintf(conf_file, sizeof(conf_file), + "%s/backup-my.cnf", target_dir); + } + if (load_defaults(conf_file, xb_server_default_groups, + &argc_server, argv_server)) { + exit(EXIT_FAILURE); + } + + print_param_str << + "# This MySQL options file was generated by XtraBackup.\n" + "[" << defaults_group << "]\n"; + + /* We want xtrabackup to ignore unknown options, because it only + recognizes a small subset of server variables */ + my_getopt_skip_unknown = TRUE; + + /* Reset u_max_value for all options, as we don't want the + --maximum-... modifier to set the actual option values */ + for (my_option *optp= xb_server_options; optp->name; optp++) { + optp->u_max_value = (G_PTR *) &global_max_value; + } + + /* Throw a descriptive error if --defaults-file or --defaults-extra-file + is not the first command line argument */ + for (int i = 2 ; i < argc ; i++) { + char *optend = strcend((argv)[i], '='); + + if (optend - argv[i] == 15 && + !strncmp(argv[i], "--defaults-file", optend - argv[i])) { + + msg("xtrabackup: Error: --defaults-file " + "must be specified first on the command " + "line\n"); + exit(EXIT_FAILURE); + } + if (optend - argv[i] == 21 && + !strncmp(argv[i], "--defaults-extra-file", + optend - argv[i])) { + + msg("xtrabackup: Error: --defaults-extra-file " + "must be specified first on the command " + "line\n"); + exit(EXIT_FAILURE); + } + } + + if (argc_server > 0 + && (ho_error=handle_options(&argc_server, argv_server, + xb_server_options, xb_get_one_option))) + exit(ho_error); + + if (load_defaults(conf_file, xb_client_default_groups, + &argc_client, argv_client)) { + exit(EXIT_FAILURE); + } + + if (strcmp(base_name(my_progname), INNOBACKUPEX_BIN_NAME) == 0 && + argc_client > 0) { + /* emulate innobackupex script */ + innobackupex_mode = true; + if (!ibx_handle_options(&argc_client, argv_client)) { + exit(EXIT_FAILURE); + } + } + + if (argc_client > 0 + && (ho_error=handle_options(&argc_client, argv_client, + xb_client_options, xb_get_one_option))) + exit(ho_error); + + /* Reject command line arguments that don't look like options, i.e. are + not of the form '-X' (single-character options) or '--option' (long + options) */ + for (int i = 0 ; i < argc_client ; i++) { + const char * const opt = (*argv_client)[i]; + + if (strncmp(opt, "--", 2) && + !(strlen(opt) == 2 && opt[0] == '-')) { + bool server_option = true; + + for (int j = 0; j < argc_server; j++) { + if (opt == (*argv_server)[j]) { + server_option = false; + break; + } + } + + if (!server_option) { + msg("xtrabackup: Error:" + " unknown argument: '%s'\n", opt); + exit(EXIT_FAILURE); + } + } + } +} + +/* ================= main =================== */ + +int main(int argc, char **argv) +{ + char **client_defaults, **server_defaults; + char cwd[FN_REFLEN]; + my_bool is_symdir; + + setup_signals(); + + MY_INIT(argv[0]); + + pthread_key_create(&THR_THD, NULL); + my_pthread_setspecific_ptr(THR_THD, NULL); + + xb_regex_init(); + + capture_tool_command(argc, argv); + + if (mysql_server_init(-1, NULL, NULL)) + { + exit(EXIT_FAILURE); + } + + system_charset_info= &my_charset_utf8_general_ci; + key_map_full.set_all(); + + handle_options(argc, argv, &client_defaults, &server_defaults); + + if (innobackupex_mode) { + if (!ibx_init()) { + exit(EXIT_FAILURE); + } + } + + if ((!xtrabackup_print_param) && (!xtrabackup_prepare) && (strcmp(mysql_data_home, "./") == 0)) { + if (!xtrabackup_print_param) + usage(); + msg("\nxtrabackup: Error: Please set parameter 'datadir'\n"); + exit(EXIT_FAILURE); + } + + /* Expand target-dir, incremental-basedir, etc. */ + + my_getwd(cwd, sizeof(cwd), MYF(0)); + + my_load_path(xtrabackup_real_target_dir, + xtrabackup_target_dir, cwd); + unpack_dirname(xtrabackup_real_target_dir, + xtrabackup_real_target_dir, &is_symdir); + xtrabackup_target_dir= xtrabackup_real_target_dir; + + if (xtrabackup_incremental_basedir) { + my_load_path(xtrabackup_real_incremental_basedir, + xtrabackup_incremental_basedir, cwd); + unpack_dirname(xtrabackup_real_incremental_basedir, + xtrabackup_real_incremental_basedir, &is_symdir); + xtrabackup_incremental_basedir = + xtrabackup_real_incremental_basedir; + } + + if (xtrabackup_incremental_dir) { + my_load_path(xtrabackup_real_incremental_dir, + xtrabackup_incremental_dir, cwd); + unpack_dirname(xtrabackup_real_incremental_dir, + xtrabackup_real_incremental_dir, &is_symdir); + xtrabackup_incremental_dir = xtrabackup_real_incremental_dir; + } + + if (xtrabackup_extra_lsndir) { + my_load_path(xtrabackup_real_extra_lsndir, + xtrabackup_extra_lsndir, cwd); + unpack_dirname(xtrabackup_real_extra_lsndir, + xtrabackup_real_extra_lsndir, &is_symdir); + xtrabackup_extra_lsndir = xtrabackup_real_extra_lsndir; + } + + /* get default temporary directory */ + if (!opt_mysql_tmpdir || !opt_mysql_tmpdir[0]) { + opt_mysql_tmpdir = getenv("TMPDIR"); +#if defined(__WIN__) + if (!opt_mysql_tmpdir) { + opt_mysql_tmpdir = getenv("TEMP"); + } + if (!opt_mysql_tmpdir) { + opt_mysql_tmpdir = getenv("TMP"); + } +#endif + if (!opt_mysql_tmpdir || !opt_mysql_tmpdir[0]) { + opt_mysql_tmpdir = const_cast(DEFAULT_TMPDIR); + } + } + + /* temporary setting of enough size */ + srv_page_size_shift = UNIV_PAGE_SIZE_SHIFT_MAX; + srv_page_size = UNIV_PAGE_SIZE_MAX; + if (xtrabackup_backup && xtrabackup_incremental) { + /* direct specification is only for --backup */ + /* and the lsn is prior to the other option */ + + char* endchar; + int error = 0; + incremental_lsn = strtoll(xtrabackup_incremental, &endchar, 10); + if (*endchar != '\0') + error = 1; + + if (error) { + msg("xtrabackup: value '%s' may be wrong format for " + "incremental option.\n", xtrabackup_incremental); + exit(EXIT_FAILURE); + } + } else if (xtrabackup_backup && xtrabackup_incremental_basedir) { + char filename[FN_REFLEN]; + + sprintf(filename, "%s/%s", xtrabackup_incremental_basedir, XTRABACKUP_METADATA_FILENAME); + + if (!xtrabackup_read_metadata(filename)) { + msg("xtrabackup: error: failed to read metadata from " + "%s\n", filename); + exit(EXIT_FAILURE); + } + + incremental_lsn = metadata_to_lsn; + xtrabackup_incremental = xtrabackup_incremental_basedir; //dummy + } else if (xtrabackup_prepare && xtrabackup_incremental_dir) { + char filename[FN_REFLEN]; + + sprintf(filename, "%s/%s", xtrabackup_incremental_dir, XTRABACKUP_METADATA_FILENAME); + + if (!xtrabackup_read_metadata(filename)) { + msg("xtrabackup: error: failed to read metadata from " + "%s\n", filename); + exit(EXIT_FAILURE); + } + + incremental_lsn = metadata_from_lsn; + incremental_to_lsn = metadata_to_lsn; + incremental_last_lsn = metadata_last_lsn; + xtrabackup_incremental = xtrabackup_incremental_dir; //dummy + + } else if (opt_incremental_history_name) { + xtrabackup_incremental = opt_incremental_history_name; + } else if (opt_incremental_history_uuid) { + xtrabackup_incremental = opt_incremental_history_uuid; + } else { + xtrabackup_incremental = NULL; + } + + if (!xb_init()) { + exit(EXIT_FAILURE); + } + + /* --print-param */ + if (xtrabackup_print_param) { + + printf("%s", print_param_str.str().c_str()); + + exit(EXIT_SUCCESS); + } + + print_version(); + if (xtrabackup_incremental) { + msg("incremental backup from " LSN_PF " is enabled.\n", + incremental_lsn); + } + + if (xtrabackup_export && innobase_file_per_table == FALSE) { + msg("xtrabackup: auto-enabling --innodb-file-per-table due to " + "the --export option\n"); + innobase_file_per_table = TRUE; + } + + if (xtrabackup_incremental && xtrabackup_stream && + xtrabackup_stream_fmt == XB_STREAM_FMT_TAR) { + msg("xtrabackup: error: " + "streaming incremental backups are incompatible with the \n" + "'tar' streaming format. Use --stream=xbstream instead.\n"); + exit(EXIT_FAILURE); + } + + if ((xtrabackup_compress || xtrabackup_encrypt) && xtrabackup_stream && + xtrabackup_stream_fmt == XB_STREAM_FMT_TAR) { + msg("xtrabackup: error: " + "compressed and encrypted backups are incompatible with the \n" + "'tar' streaming format. Use --stream=xbstream instead.\n"); + exit(EXIT_FAILURE); + } + + if (!xtrabackup_prepare && + (innobase_log_arch_dir || xtrabackup_archived_to_lsn)) { + + /* Default my.cnf can contain innobase_log_arch_dir option set + for server, reset it to allow backup. */ + innobase_log_arch_dir= NULL; + xtrabackup_archived_to_lsn= 0; + msg("xtrabackup: warning: " + "as --innodb-log-arch-dir and --to-archived-lsn can be used " + "only with --prepare they will be reset\n"); + } + + /* cannot execute both for now */ + { + int num = 0; + + if (xtrabackup_backup) num++; + if (xtrabackup_stats) num++; + if (xtrabackup_prepare) num++; + if (xtrabackup_copy_back) num++; + if (xtrabackup_move_back) num++; + if (xtrabackup_decrypt_decompress) num++; + if (num != 1) { /* !XOR (for now) */ + usage(); + exit(EXIT_FAILURE); + } + } + +#ifndef __WIN__ + if (xtrabackup_debug_sync) { + signal(SIGCONT, sigcont_handler); + } +#endif + + /* --backup */ + if (xtrabackup_backup) + xtrabackup_backup_func(); + + /* --stats */ + if (xtrabackup_stats) + xtrabackup_stats_func(); + + /* --prepare */ + if (xtrabackup_prepare) + xtrabackup_prepare_func(); + + if (xtrabackup_copy_back || xtrabackup_move_back) { + if (!check_if_param_set("datadir")) { + msg("Error: datadir must be specified.\n"); + exit(EXIT_FAILURE); + } + if (!copy_back()) + exit(EXIT_FAILURE); + } + + if (xtrabackup_decrypt_decompress && !decrypt_decompress()) { + exit(EXIT_FAILURE); + } + + backup_cleanup(); + + if (innobackupex_mode) { + ibx_cleanup(); + } + + xb_regex_end(); + + free_defaults(client_defaults); + free_defaults(server_defaults); + + if (THR_THD) + (void) pthread_key_delete(THR_THD); + + msg_ts("completed OK!\n"); + + exit(EXIT_SUCCESS); +} diff --git a/extra/mariabackup/xtrabackup.h b/extra/mariabackup/xtrabackup.h new file mode 100644 index 00000000000..b31028175c9 --- /dev/null +++ b/extra/mariabackup/xtrabackup.h @@ -0,0 +1,232 @@ +/****************************************************** +Copyright (c) 2011-2015 Percona LLC and/or its affiliates. + +Declarations for xtrabackup.cc + +This program is free software; you can redistribute it and/or modify +it under the terms of the GNU General Public License as published by +the Free Software Foundation; version 2 of the License. + +This program is distributed in the hope that it will be useful, +but WITHOUT ANY WARRANTY; without even the implied warranty of +MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the +GNU General Public License for more details. + +You should have received a copy of the GNU General Public License +along with this program; if not, write to the Free Software +Foundation, Inc., 51 Franklin Street, Fifth Floor, Boston, MA 02110-1301, USA + +*******************************************************/ + +#ifndef XB_XTRABACKUP_H +#define XB_XTRABACKUP_H + +#include +#include "datasink.h" +#include "xbstream.h" +#include "changed_page_bitmap.h" + +#ifdef __WIN__ +#define XB_FILE_UNDEFINED NULL +#else +#define XB_FILE_UNDEFINED (-1) +#endif + +typedef struct { + ulint page_size; + ulint zip_size; + ulint space_id; +} xb_delta_info_t; + +/* ======== Datafiles iterator ======== */ +typedef struct { + fil_system_t *system; + fil_space_t *space; + fil_node_t *node; + ibool started; + os_ib_mutex_t mutex; +} datafiles_iter_t; + +/* value of the --incremental option */ +extern lsn_t incremental_lsn; + +extern char *xtrabackup_target_dir; +extern char *xtrabackup_incremental_dir; +extern char *xtrabackup_incremental_basedir; +extern char *innobase_data_home_dir; +extern char *innobase_buffer_pool_filename; +extern ds_ctxt_t *ds_meta; +extern ds_ctxt_t *ds_data; + +/* The last checkpoint LSN at the backup startup time */ +extern lsn_t checkpoint_lsn_start; + +extern xb_page_bitmap *changed_page_bitmap; + +extern ulint xtrabackup_rebuild_threads; + +extern char *xtrabackup_incremental; +extern my_bool xtrabackup_incremental_force_scan; + +extern lsn_t metadata_from_lsn; +extern lsn_t metadata_to_lsn; +extern lsn_t metadata_last_lsn; + +extern xb_stream_fmt_t xtrabackup_stream_fmt; +extern ibool xtrabackup_stream; + +extern char *xtrabackup_tables; +extern char *xtrabackup_tables_file; +extern char *xtrabackup_databases; +extern char *xtrabackup_databases_file; + +extern my_bool xtrabackup_compact; +extern ibool xtrabackup_compress; +extern ibool xtrabackup_encrypt; + +extern my_bool xtrabackup_backup; +extern my_bool xtrabackup_prepare; +extern my_bool xtrabackup_apply_log_only; +extern my_bool xtrabackup_copy_back; +extern my_bool xtrabackup_move_back; +extern my_bool xtrabackup_decrypt_decompress; + +extern char *innobase_data_file_path; +extern char *innobase_doublewrite_file; +extern char *xtrabackup_encrypt_key; +extern char *xtrabackup_encrypt_key_file; +extern longlong innobase_log_file_size; +extern long innobase_log_files_in_group; +extern longlong innobase_page_size; + +extern const char *xtrabackup_encrypt_algo_names[]; +extern TYPELIB xtrabackup_encrypt_algo_typelib; + +extern int xtrabackup_parallel; + +extern my_bool xb_close_files; +extern const char *xtrabackup_compress_alg; +extern uint xtrabackup_compress_threads; +extern ulonglong xtrabackup_compress_chunk_size; +extern ulong xtrabackup_encrypt_algo; +extern uint xtrabackup_encrypt_threads; +extern ulonglong xtrabackup_encrypt_chunk_size; +extern my_bool xtrabackup_export; +extern char *xtrabackup_incremental_basedir; +extern char *xtrabackup_extra_lsndir; +extern char *xtrabackup_incremental_dir; +extern ulint xtrabackup_log_copy_interval; +extern my_bool xtrabackup_rebuild_indexes; +extern char *xtrabackup_stream_str; +extern long xtrabackup_throttle; +extern longlong xtrabackup_use_memory; + +extern my_bool opt_galera_info; +extern my_bool opt_slave_info; +extern my_bool opt_no_lock; +extern my_bool opt_safe_slave_backup; +extern my_bool opt_rsync; +extern my_bool opt_force_non_empty_dirs; +extern my_bool opt_noversioncheck; +extern my_bool opt_no_backup_locks; +extern my_bool opt_decompress; +extern my_bool opt_remove_original; + +extern char *opt_incremental_history_name; +extern char *opt_incremental_history_uuid; + +extern char *opt_user; +extern char *opt_password; +extern char *opt_host; +extern char *opt_defaults_group; +extern char *opt_socket; +extern uint opt_port; +extern char *opt_login_path; +extern char *opt_log_bin; + +extern const char *query_type_names[]; + +enum query_type_t {QUERY_TYPE_ALL, QUERY_TYPE_UPDATE, + QUERY_TYPE_SELECT}; + +extern TYPELIB query_type_typelib; + +extern ulong opt_lock_wait_query_type; +extern ulong opt_kill_long_query_type; + +extern ulong opt_decrypt_algo; + +extern uint opt_kill_long_queries_timeout; +extern uint opt_lock_wait_timeout; +extern uint opt_lock_wait_threshold; +extern uint opt_debug_sleep_before_unlock; +extern uint opt_safe_slave_backup_timeout; + +extern const char *opt_history; +extern my_bool opt_decrypt; + +#if defined(HAVE_OPENSSL) +extern my_bool opt_use_ssl; +extern my_bool opt_ssl_verify_server_cert; +#if !defined(HAVE_YASSL) +extern char *opt_server_public_key; +#endif +#endif + +enum binlog_info_enum { BINLOG_INFO_OFF, BINLOG_INFO_LOCKLESS, BINLOG_INFO_ON, + BINLOG_INFO_AUTO}; + +extern ulong opt_binlog_info; + +void xtrabackup_io_throttling(void); +my_bool xb_write_delta_metadata(const char *filename, + const xb_delta_info_t *info); + +datafiles_iter_t *datafiles_iter_new(fil_system_t *f_system); +fil_node_t *datafiles_iter_next(datafiles_iter_t *it); +void datafiles_iter_free(datafiles_iter_t *it); + +/************************************************************************ +Initialize the tablespace memory cache and populate it by scanning for and +opening data files */ +ulint xb_data_files_init(void); + +/************************************************************************ +Destroy the tablespace memory cache. */ +void xb_data_files_close(void); + +/*********************************************************************** +Reads the space flags from a given data file and returns the compressed +page size, or 0 if the space is not compressed. */ +ulint xb_get_zip_size(os_file_t file); + +/************************************************************************ +Checks if a table specified as a name in the form "database/name" (InnoDB 5.6) +or "./database/name.ibd" (InnoDB 5.5-) should be skipped from backup based on +the --tables or --tables-file options. + +@return TRUE if the table should be skipped. */ +my_bool +check_if_skip_table( +/******************/ + const char* name); /*!< in: path to the table */ + +/************************************************************************ +Check if parameter is set in defaults file or via command line argument +@return true if parameter is set. */ +bool +check_if_param_set(const char *param); + + +void +xtrabackup_backup_func(void); + +my_bool +xb_get_one_option(int optid, + const struct my_option *opt __attribute__((unused)), + char *argument); + +const char* +xb_get_copy_action(const char *dflt = "Copying"); + +#endif /* XB_XTRABACKUP_H */ diff --git a/extra/mariabackup/xtrabackup_version.h.in b/extra/mariabackup/xtrabackup_version.h.in new file mode 100644 index 00000000000..dc4c7992f8f --- /dev/null +++ b/extra/mariabackup/xtrabackup_version.h.in @@ -0,0 +1,27 @@ +/****************************************************** +Copyright (c) 2013 Percona LLC and/or its affiliates. + +Version numbers definitions. + +This program is free software; you can redistribute it and/or modify +it under the terms of the GNU General Public License as published by +the Free Software Foundation; version 2 of the License. + +This program is distributed in the hope that it will be useful, +but WITHOUT ANY WARRANTY; without even the implied warranty of +MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the +GNU General Public License for more details. + +You should have received a copy of the GNU General Public License +along with this program; if not, write to the Free Software +Foundation, Inc., 51 Franklin Street, Fifth Floor, Boston, MA 02110-1301, USA + +*******************************************************/ + +#ifndef XB_VERSION_H +#define XB_VERSION_H + +#define XTRABACKUP_VERSION "@XB_VERSION@" +#define XTRABACKUP_REVISION "@XB_REVISION@" + +#endif /* XB_VERSION_H */ From ce4c56db0c5806ccbea8ad78585f95f6c335b0ea Mon Sep 17 00:00:00 2001 From: Vladislav Vaintroub Date: Tue, 18 Apr 2017 19:05:57 +0000 Subject: [PATCH 10/28] MDEV-9566 Port Percona Xtrabackup to MariaDB as mariabackup - Modify backup code to work with MariaDB's 10.1 xtradb - Remove crypt encryption, version_check.pl, "compact backup" - Add encryption plugin --- extra/mariabackup/CMakeLists.txt | 231 ++-- extra/mariabackup/backup_copy.cc | 141 ++- extra/mariabackup/backup_copy.h | 2 - extra/mariabackup/backup_mysql.cc | 256 ++-- extra/mariabackup/changed_page_bitmap.cc | 2 +- extra/mariabackup/common.h | 48 +- extra/mariabackup/compact.cc | 1059 ---------------- extra/mariabackup/compact.h | 44 - extra/mariabackup/datasink.c | 9 +- extra/mariabackup/datasink.h | 1 + extra/mariabackup/ds_compress.c | 4 +- extra/mariabackup/ds_encrypt.c | 3 +- extra/mariabackup/ds_encrypt.h | 4 +- extra/mariabackup/ds_local.c | 8 +- extra/mariabackup/ds_stdout.c | 2 +- extra/mariabackup/ds_tmpfile.c | 3 +- extra/mariabackup/encryption_plugin.cc | 157 +++ extra/mariabackup/encryption_plugin.h | 7 + extra/mariabackup/fil_cur.cc | 25 +- extra/mariabackup/fil_cur.h | 6 +- extra/mariabackup/innobackupex.cc | 27 +- extra/mariabackup/read_filt.cc | 6 +- extra/mariabackup/read_filt.h | 8 +- extra/mariabackup/version_check.pl | 1373 --------------------- extra/mariabackup/write_filt.cc | 2 +- extra/mariabackup/write_filt.h | 3 - extra/mariabackup/wsrep.cc | 7 +- extra/mariabackup/xb0xb.h | 78 ++ extra/mariabackup/xb_regex.h | 43 +- extra/mariabackup/xbcrypt_common.c | 2 + extra/mariabackup/xbcrypt_write.c | 2 +- extra/mariabackup/xbstream.c | 8 +- extra/mariabackup/xbstream_read.c | 8 +- extra/mariabackup/xbstream_write.c | 20 +- extra/mariabackup/xtrabackup.cc | 443 ++++--- extra/mariabackup/xtrabackup.h | 30 +- extra/mariabackup/xtrabackup_version.h.in | 27 - storage/xtradb/CMakeLists.txt | 3 + 38 files changed, 944 insertions(+), 3158 deletions(-) delete mode 100644 extra/mariabackup/compact.cc delete mode 100644 extra/mariabackup/compact.h create mode 100644 extra/mariabackup/encryption_plugin.cc create mode 100644 extra/mariabackup/encryption_plugin.h delete mode 100644 extra/mariabackup/version_check.pl create mode 100644 extra/mariabackup/xb0xb.h delete mode 100644 extra/mariabackup/xtrabackup_version.h.in diff --git a/extra/mariabackup/CMakeLists.txt b/extra/mariabackup/CMakeLists.txt index a84ac304486..f3369b49cee 100644 --- a/extra/mariabackup/CMakeLists.txt +++ b/extra/mariabackup/CMakeLists.txt @@ -13,65 +13,138 @@ # along with this program; if not, write to the Free Software # Foundation, Inc., 51 Franklin Street, Fifth Floor, Boston, MA 02110-1301, USA -INCLUDE(gcrypt) -INCLUDE(curl) -INCLUDE(libev) -ADD_SUBDIRECTORY(libarchive) -ADD_SUBDIRECTORY(jsmn) +OPTION(WITH_MARIABACKUP "Include mariabackup" ON) +IF(NOT WITH_MARIABACKUP) + RETURN() +ENDIF() -FIND_GCRYPT() -FIND_CURL() -FIND_EV() -# xxd is needed to embed version_check script -FIND_PROGRAM(XXD_PATH xxd) +IF(NOT WIN32) + CHECK_SYMBOL_EXISTS(regcomp regex.h HAVE_SYSTEM_REGEX) + IF(HAVE_SYSTEM_REGEX) + ADD_DEFINITIONS(-DHAVE_SYSTEM_REGEX) + ENDIF() +ENDIF() -IF(NOT XXD_PATH) - MESSAGE(FATAL_ERROR "xxd not found. Try to install vim-common.") -ENDIF(NOT XXD_PATH) +IF(WITH_LIBARCHIVE STREQUAL "STATIC") + SET(CMAKE_FIND_LIBRARY_SUFFIXES .a .lib) +ENDIF() + +FIND_PACKAGE(LibArchive) + +IF(NOT DEFINED WITH_LIBARCHIVE) + IF(LibArchive_FOUND) + SET(WITH_LIBARCHIVE_DEFAULT ON) + ELSE() + SET(WITH_LIBARCHIVE_DEFAULT OFF) + ENDIF() + SET(WITH_LIBARCHIVE ${WITH_LIBARCHIVE_DEFAULT} CACHE STRING "Use libarchive for streaming features (ON, OFF or STATIC)" ) +ENDIF() + +IF(NOT WITH_LIBARCHIVE MATCHES "^(ON|OFF|STATIC)$") + MESSAGE(FATAL_ERROR "Invalid value for WITH_LIBARCHIVE: '${WITH_LIBARCHIVE}'. Use one of ON, OFF or STATIC") +ENDIF() + +IF(UNIX) + SET(PIC_FLAG -fPIC) +ENDIF() + +IF((NOT WITH_LIBARCHIVE STREQUAL "OFF") AND (NOT LibArchive_FOUND)) + IF(CMAKE_VERSION VERSION_LESS "2.8.12") + MESSAGE("libarchive can't be built, old cmake") + ELSE() + # Build a local version + INCLUDE(ExternalProject) + SET(libarchive_PREFIX ${CMAKE_CURRENT_BINARY_DIR}/libarchive) + SET(libarchive_CMAKE_ARGS + -DCMAKE_INSTALL_PREFIX:PATH= + -DCMAKE_BUILD_TYPE=${CMAKE_BUILD_TYPE} + -DENABLE_ICONV=OFF + -DENABLE_TAR=ON + -DENABLE_OPENSSL=OFF + -DENABLE_TEST=OFF + "-DCMAKE_C_FLAGS_DEBUG=${CMAKE_C_FLAGS_DEBUG} ${PIC_FLAG}" + "-DCMAKE_C_FLAGS_RELWITHDEBINFO=${CMAKE_C_FLAGS_RELWITHDEBINFO} ${PIC_FLAG}" + "-DCMAKE_C_FLAGS_RELEASE=${CMAKE_C_FLAGS_RELEASE} ${PIC_FLAG}" + "-DCMAKE_C_FLAGS_MINSIZEREL=${CMAKE_C_FLAGS_MINSIZEREL} ${PIC_FLAG}" + ) + IF(WIN32) + SET(libarchive_CMAKE_ARGS ${libarchive_CMAKE_ARGS} -DWINDOWS_VERSION=WIN7 -DCMAKE_DEBUG_POSTFIX=d) + ENDIF() + + SET(LIBARCHIVE_DIR ${CMAKE_CURRENT_BINARY_DIR}/libarchive) + ExternalProject_Add(libarchive + PREFIX ${libarchive_PREFIX} + DOWNLOAD_DIR ${LIBARCHIVE_DIR} + URL http://www.libarchive.org/downloads/libarchive-3.2.2.tar.gz + INSTALL_DIR ${LIBARCHIVE_DIR} + CMAKE_ARGS ${libarchive_CMAKE_ARGS} + ) + ADD_LIBRARY(archive_static STATIC IMPORTED) + ADD_DEPENDENCIES(archive_static libarchive) + IF(WIN32) + SET(LIBARCHIVE_RELEASE_LIB ${LIBARCHIVE_DIR}/lib/${CMAKE_STATIC_LIBRARY_PREFIX}archive_static${CMAKE_STATIC_LIBRARY_SUFFIX}) + SET(LIBARCHIVE_DEBUG_LIB ${LIBARCHIVE_DIR}/lib/${CMAKE_STATIC_LIBRARY_PREFIX}archive_staticd${CMAKE_STATIC_LIBRARY_SUFFIX}) + SET_PROPERTY(TARGET archive_static PROPERTY IMPORTED_LOCATION_RELWITHDEBINFO ${LIBARCHIVE_RELEASE_LIB}) + SET_PROPERTY(TARGET archive_static PROPERTY IMPORTED_LOCATION_RELEASE ${LIBARCHIVE_RELEASE_LIB}) + SET_PROPERTY(TARGET archive_static PROPERTY IMPORTED_LOCATION_DEBUG ${LIBARCHIVE_DEBUG_LIB}) + SET_PROPERTY(TARGET archive_static PROPERTY IMPORTED_LOCATION_MINSIZEREL ${LIBARCHIVE_RELEASE_LIB}) + ELSE() + SET_PROPERTY(TARGET archive_static PROPERTY IMPORTED_LOCATION ${LIBARCHIVE_DIR}/lib/${CMAKE_STATIC_LIBRARY_PREFIX}archive${CMAKE_STATIC_LIBRARY_SUFFIX}) + ENDIF() + + SET(LibArchive_FOUND ON ) + SET(LibArchive_INCLUDE_DIRS ${LIBARCHIVE_DIR}/include ) + SET(LibArchive_LIBRARIES archive_static) + IF(WIN32) + SET(LIBARCHIVE_STATIC 1) + ENDIF() + ENDIF() +ENDIF() + + +IF(WITH_LIBARCHIVE AND LibArchive_FOUND) + ADD_DEFINITIONS(-DHAVE_LIBARCHIVE) + IF(LIBARCHIVE_STATIC) + ADD_DEFINITIONS(-DLIBARCHIVE_STATIC) + ENDIF() + INCLUDE_DIRECTORIES(${LibArchive_INCLUDE_DIRS}) + LINK_LIBRARIES(${LibArchive_LIBRARIES}) + SET(DS_ARCHIVE_SOURCE ds_archive.c) +ENDIF() INCLUDE_DIRECTORIES( ${CMAKE_SOURCE_DIR}/include - ${CMAKE_SOURCE_DIR}/storage/innobase/include + ${CMAKE_SOURCE_DIR}/storage/xtradb/include ${CMAKE_SOURCE_DIR}/sql - ${CMAKE_SOURCE_DIR}/storage/innobase/xtrabackup/src/libarchive/libarchive - ${CMAKE_SOURCE_DIR}/storage/innobase/xtrabackup/src/quicklz - ${CMAKE_SOURCE_DIR}/storage/innobase/xtrabackup/src/jsmn - ${GCRYPT_INCLUDE_DIR} - ${CURL_INCLUDE_DIRS} - ${LIBEV_INCLUDE_DIRS} - ${CMAKE_CURRENT_BINARY_DIR} + ${CMAKE_CURRENT_SOURCE_DIR}/quicklz + ${CMAKE_CURRENT_SOURCE_DIR} ) -ADD_DEFINITIONS(${SSL_DEFINES}) +IF(NOT HAVE_SYSTEM_REGEX) + INCLUDE_DIRECTORIES(${CMAKE_SOURCE_DIR}/pcre) +ENDIF() +ADD_DEFINITIONS(-UMYSQL_SERVER) ######################################################################## # xtrabackup binary ######################################################################## -CONFIGURE_FILE(${CMAKE_CURRENT_SOURCE_DIR}/xtrabackup_version.h.in - ${CMAKE_CURRENT_BINARY_DIR}/xtrabackup_version.h ) -ADD_CUSTOM_COMMAND(OUTPUT ${CMAKE_CURRENT_BINARY_DIR}/version_check_pl.h - COMMAND ${XXD_PATH} --include version_check.pl - ${CMAKE_CURRENT_BINARY_DIR}/version_check_pl.h - WORKING_DIRECTORY ${CMAKE_CURRENT_SOURCE_DIR}) +IF(WIN32) + SET(NT_SERVICE_SOURCE ${PROJECT_SOURCE_DIR}/sql/nt_servc.cc) +ELSE() + SET(NT_SERVICE_SOURCE) +ENDIF() -ADD_CUSTOM_TARGET(GenVersionCheck - DEPENDS ${CMAKE_CURRENT_BINARY_DIR}/version_check_pl.h) +ADD_DEFINITIONS(-DPCRE_STATIC=1) -SET_SOURCE_FILES_PROPERTIES( - xtrabackup.cc - backup_mysql.cc - PROPERTIES COMPILE_FLAGS -DMYSQL_CLIENT) - -MYSQL_ADD_EXECUTABLE(xtrabackup +MYSQL_ADD_EXECUTABLE(mariabackup xtrabackup.cc innobackupex.cc changed_page_bitmap.cc - compact.cc datasink.c - ds_archive.c + ${DS_ARCHIVE_SOURCE} ds_buffer.c ds_compress.c ds_encrypt.c @@ -89,31 +162,29 @@ MYSQL_ADD_EXECUTABLE(xtrabackup xbstream_write.c backup_mysql.cc backup_copy.cc - ../../../../sql-common/client_authentication.cc + encryption_plugin.cc + ${PROJECT_SOURCE_DIR}/libmysql/libmysql.c + ${PROJECT_SOURCE_DIR}/sql/net_serv.cc + ${NT_SERVICE_SOURCE} + COMPONENT backup ) -SET_TARGET_PROPERTIES(xtrabackup PROPERTIES ENABLE_EXPORTS TRUE) -TARGET_LINK_LIBRARIES(xtrabackup - mysqlserver - ${GCRYPT_LIBS} - archive_static - ) +# Export all symbols on Unix, for better crash callstacks +SET_TARGET_PROPERTIES(mariabackup PROPERTIES ENABLE_EXPORTS TRUE) -ADD_DEPENDENCIES(xtrabackup GenVersionCheck) -######################################################################## -# innobackupex symlink -######################################################################## -ADD_CUSTOM_COMMAND(TARGET xtrabackup - COMMAND ${CMAKE_COMMAND} ARGS -E create_symlink - xtrabackup innobackupex) -INSTALL(FILES ${CMAKE_CURRENT_BINARY_DIR}/innobackupex DESTINATION bin) +TARGET_LINK_LIBRARIES(mariabackup sql) + +IF(NOT HAVE_SYSTEM_REGEX) + TARGET_LINK_LIBRARIES(mariabackup pcreposix) +ENDIF() + ######################################################################## # xbstream binary ######################################################################## -MYSQL_ADD_EXECUTABLE(xbstream +MYSQL_ADD_EXECUTABLE(mbstream ds_buffer.c ds_local.c ds_stdout.c @@ -121,53 +192,15 @@ MYSQL_ADD_EXECUTABLE(xbstream xbstream.c xbstream_read.c xbstream_write.c + + COMPONENT backup ) -SET_TARGET_PROPERTIES(xbstream - PROPERTIES LINKER_LANGUAGE CXX - ) -TARGET_LINK_LIBRARIES(xbstream +TARGET_LINK_LIBRARIES(mbstream mysys - mysys_ssl - ) +) -######################################################################## -# xbcrypt binary -######################################################################## -MYSQL_ADD_EXECUTABLE(xbcrypt - xbcrypt.c - xbcrypt_common.c - xbcrypt_read.c - xbcrypt_write.c - ) - -SET_TARGET_PROPERTIES(xbcrypt - PROPERTIES LINKER_LANGUAGE CXX - ) - -TARGET_LINK_LIBRARIES(xbcrypt - ${GCRYPT_LIBS} - mysys - mysys_ssl - ) - -######################################################################## -# xbcloud binary -######################################################################## -MYSQL_ADD_EXECUTABLE(xbcloud - xbcloud.cc - ) - -SET_TARGET_PROPERTIES(xbcloud - PROPERTIES LINKER_LANGUAGE CXX - ) - -TARGET_LINK_LIBRARIES(xbcloud - ${GCRYPT_LIBS} - ${LIBEV_LIBRARIES} - ${CURL_LIBRARIES} - mysys - mysys_ssl - jsmn - ) +IF(MSVC) + SET_TARGET_PROPERTIES(mbstream PROPERTIES LINK_FLAGS setargv.obj) +ENDIF() diff --git a/extra/mariabackup/backup_copy.cc b/extra/mariabackup/backup_copy.cc index a051c64ee3b..478a6a66b97 100644 --- a/extra/mariabackup/backup_copy.cc +++ b/extra/mariabackup/backup_copy.cc @@ -1,6 +1,7 @@ /****************************************************** hot backup tool for InnoDB (c) 2009-2015 Percona LLC and/or its affiliates +(c) 2017 MariaDB Originally Created 3/3/2009 Yasufumi Kinoshita Written by Alexey Kopytov, Aleksandr Kuzminsky, Stewart Smith, Vadim Tkachenko, Yasufumi Kinoshita, Ignacio Nin and Baron Schwartz. @@ -48,13 +49,14 @@ Place, Suite 330, Boston, MA 02111-1307 USA #include #include #include -#include #include #include "fil_cur.h" #include "xtrabackup.h" #include "common.h" #include "backup_copy.h" #include "backup_mysql.h" +#include +#include "xb0xb.h" /* list of files to sync for --rsync mode */ @@ -110,6 +112,7 @@ struct datadir_thread_ctxt_t { bool ret; }; +static bool backup_files_from_datadir(const char *dir_path); /************************************************************************ Retirn true if character if file separator */ @@ -225,7 +228,7 @@ datadir_iter_next_database(datadir_iter_t *it) it->dbdir = NULL; } - while (fil_file_readdir_next_file(&it->err, it->datadir_path, + while (os_file_readdir_next_file(it->datadir_path, it->dir, &it->dbinfo) == 0) { ulint len; @@ -340,7 +343,7 @@ datadir_iter_next_file(datadir_iter_t *it) return(false); } - while (fil_file_readdir_next_file(&it->err, it->dbpath, it->dbdir, + while (os_file_readdir_next_file(it->dbpath, it->dbdir, &it->fileinfo) == 0) { if (it->fileinfo.type == OS_FILE_TYPE_DIR) { @@ -449,9 +452,9 @@ struct datafile_cur_t { uint thread_n; byte* orig_buf; byte* buf; - ib_int64_t buf_size; - ib_int64_t buf_read; - ib_int64_t buf_offset; + size_t buf_size; + size_t buf_read; + size_t buf_offset; }; static @@ -486,7 +489,7 @@ datafile_open(const char *file, datafile_cur_t *cursor, uint thread_n) cursor->abs_path, OS_FILE_OPEN, OS_FILE_READ_ONLY, - &success); + &success, 0); if (!success) { /* The following call prints an error message */ os_file_get_last_error(TRUE); @@ -498,7 +501,7 @@ datafile_open(const char *file, datafile_cur_t *cursor, uint thread_n) return(false); } - if (my_fstat(cursor->file, &cursor->statinfo, MYF(MY_WME))) { + if (!my_stat(cursor->abs_path, &cursor->statinfo, 0)) { msg("[%02u] error: cannot stat %s\n", thread_n, cursor->abs_path); @@ -510,7 +513,7 @@ datafile_open(const char *file, datafile_cur_t *cursor, uint thread_n) posix_fadvise(cursor->file, 0, 0, POSIX_FADV_SEQUENTIAL); cursor->buf_size = 10 * 1024 * 1024; - cursor->buf = static_cast(ut_malloc(cursor->buf_size)); + cursor->buf = static_cast(ut_malloc((ulint)cursor->buf_size)); return(true); } @@ -525,7 +528,7 @@ datafile_read(datafile_cur_t *cursor) xtrabackup_io_throttling(); - to_read = min(cursor->statinfo.st_size - cursor->buf_offset, + to_read = (ulint)MY_MIN(cursor->statinfo.st_size - cursor->buf_offset, cursor->buf_size); if (to_read == 0) { @@ -603,6 +606,11 @@ ends_with(const char *str, const char *suffix) && strcmp(str + str_len - suffix_len, suffix) == 0); } +static bool starts_with(const char *str, const char *prefix) +{ + return strncmp(str, prefix, strlen(prefix)) == 0; +} + /************************************************************************ Create directories recursively. @return 0 if directories created successfully. */ @@ -644,6 +652,7 @@ Return true if first and second arguments are the same path. */ bool equal_paths(const char *first, const char *second) { +#ifdef HAVE_REALPATH char real_first[PATH_MAX]; char real_second[PATH_MAX]; @@ -655,6 +664,9 @@ equal_paths(const char *first, const char *second) } return (strcmp(real_first, real_second) == 0); +#else + return strcmp(first, second) == 0; +#endif } /************************************************************************ @@ -675,10 +687,8 @@ directory_exists(const char *dir, bool create) } if (mkdirp(dir, 0777, MYF(0)) < 0) { - - msg("Can not create directory %s: %s\n", dir, - my_strerror(errbuf, sizeof(errbuf), my_errno)); - + my_strerror(errbuf, sizeof(errbuf), my_errno); + msg("Can not create directory %s: %s\n", dir, errbuf); return(false); } @@ -688,9 +698,9 @@ directory_exists(const char *dir, bool create) os_dir = os_file_opendir(dir, FALSE); if (os_dir == NULL) { - + my_strerror(errbuf, sizeof(errbuf), my_errno); msg("Can not open directory %s: %s\n", dir, - my_strerror(errbuf, sizeof(errbuf), my_errno)); + errbuf); return(false); } @@ -1054,16 +1064,17 @@ move_file(ds_ctxt_t *datasink, dst_file_path, thread_n); msg_ts("[%02u] Removing %s\n", thread_n, src_file_path); if (unlink(src_file_path) != 0) { + my_strerror(errbuf, sizeof(errbuf), errno); msg("Error: unlink %s failed: %s\n", src_file_path, - my_strerror(errbuf, - sizeof(errbuf), errno)); + errbuf); } return(ret); } + my_strerror(errbuf, sizeof(errbuf), my_errno); msg("Can not move file %s to %s: %s\n", src_file_path, dst_file_path_abs, - my_strerror(errbuf, sizeof(errbuf), my_errno)); + errbuf); return(false); } @@ -1360,6 +1371,10 @@ backup_start() return(false); } + if (!backup_files_from_datadir(fil_path_to_mysql_datadir)) { + return false; + } + // There is no need to stop slave thread before coping non-Innodb data when // --no-lock option is used because --no-lock option requires that no DDL or // DML to non-transaction tables can occur. @@ -1578,7 +1593,11 @@ ibx_cleanup_full_backup() while (datadir_iter_next(it, &node)) { if (node.is_empty_dir) { +#ifdef _WIN32 + DeleteFile(node.filepath); +#else rmdir(node.filepath); +#endif } if (xtrabackup_incremental && !node.is_empty_dir @@ -1605,6 +1624,9 @@ apply_log_finish() return(true); } +extern void +os_io_init_simple(void); + bool copy_back() { @@ -1661,7 +1683,7 @@ copy_back() } srv_max_n_threads = 1000; - os_sync_mutex = NULL; + //os_sync_mutex = NULL; ut_mem_init(); /* temporally dummy value to avoid crash */ srv_page_size_shift = 14; @@ -1764,10 +1786,9 @@ copy_back() if (mkdirp(path, 0777, MYF(0)) < 0) { char errbuf[MYSYS_STRERROR_SIZE]; - + my_strerror(errbuf, sizeof(errbuf), my_errno); msg("Can not create directory %s: %s\n", - path, my_strerror(errbuf, - sizeof(errbuf), my_errno)); + path, errbuf); ret = false; goto cleanup; @@ -1855,13 +1876,12 @@ cleanup: ds_data = NULL; + //os_sync_free(); + mem_close(); + //os_sync_mutex = NULL; + ut_free_all_mem(); sync_close(); sync_initialized = FALSE; - os_sync_free(); - mem_close(); - os_sync_mutex = NULL; - ut_free_all_mem(); - return(ret); } @@ -1872,7 +1892,7 @@ decrypt_decompress_file(const char *filepath, uint thread_n) char *dest_filepath = strdup(filepath); bool needs_action = false; - cmd << "cat " << filepath; + cmd << IF_WIN("type ","cat ") << filepath; if (ends_with(filepath, ".xbcrypt") && opt_decrypt) { cmd << " | xbcrypt --decrypt --encrypt-algo=" @@ -1926,7 +1946,7 @@ decrypt_decompress_file(const char *filepath, uint thread_n) } static -os_thread_ret_t +os_thread_ret_t STDCALL decrypt_decompress_thread_func(void *arg) { bool ret = true; @@ -1974,7 +1994,7 @@ decrypt_decompress() datadir_iter_t *it = NULL; srv_max_n_threads = 1000; - os_sync_mutex = NULL; + //os_sync_mutex = NULL; ut_mem_init(); os_sync_init(); sync_init(); @@ -2008,40 +2028,43 @@ decrypt_decompress() sync_close(); sync_initialized = FALSE; - os_sync_free(); - os_sync_mutex = NULL; + //os_sync_free(); + //os_sync_mutex = NULL; ut_free_all_mem(); return(ret); } -void -version_check() +/* + Copy some files from top level datadir. + Do not copy the Innodb files (ibdata1, redo log files), + as this is done in a separate step. +*/ +static bool backup_files_from_datadir(const char *dir_path) { - if (opt_password != NULL) { - setenv("option_mysql_password", opt_password, 1); - } - if (opt_user != NULL) { - setenv("option_mysql_user", opt_user, 1); - } - if (opt_host != NULL) { - setenv("option_mysql_host", opt_host, 1); - } - if (opt_socket != NULL) { - setenv("option_mysql_socket", opt_socket, 1); - } - if (opt_port != 0) { - char port[20]; - snprintf(port, sizeof(port), "%u", opt_port); - setenv("option_mysql_port", port, 1); - } + os_file_dir_t dir = os_file_opendir(dir_path, TRUE); + os_file_stat_t info; + bool ret = true; + while (os_file_readdir_next_file(dir_path, dir, &info) == 0) { - FILE *pipe = popen("perl", "w"); - if (pipe == NULL) { - return; + if (info.type != OS_FILE_TYPE_FILE) + continue; + + const char *pname = strrchr(info.name, IF_WIN('\\', '/')); + if (!pname) + pname = info.name; + + /* Copy aria log files, and aws keys for encryption plugins.*/ + const char *prefixes[] = { "aria_log", "aws-kms-key" }; + for (size_t i = 0; i < array_elements(prefixes); i++) { + if (starts_with(pname, prefixes[i])) { + ret = copy_file(ds_data, info.name, info.name, 1); + if (!ret) { + break; + } + } + } } - - fputs((const char *)version_check_pl, pipe); - - pclose(pipe); + os_file_closedir(dir); + return ret; } diff --git a/extra/mariabackup/backup_copy.h b/extra/mariabackup/backup_copy.h index c8fc5fc8ba9..4b829982764 100644 --- a/extra/mariabackup/backup_copy.h +++ b/extra/mariabackup/backup_copy.h @@ -41,8 +41,6 @@ bool copy_back(); bool decrypt_decompress(); -void -version_check(); bool is_path_separator(char); bool diff --git a/extra/mariabackup/backup_mysql.cc b/extra/mariabackup/backup_mysql.cc index 1ae8d10053e..ed5446b4806 100644 --- a/extra/mariabackup/backup_mysql.cc +++ b/extra/mariabackup/backup_mysql.cc @@ -38,6 +38,7 @@ this program; if not, write to the Free Software Foundation, Inc., 59 Temple Place, Suite 330, Boston, MA 02111-1307 USA *******************************************************/ +#define MYSQL_CLIENT #include #include @@ -47,10 +48,12 @@ Place, Suite 330, Boston, MA 02111-1307 USA #include #include "common.h" #include "xtrabackup.h" -#include "xtrabackup_version.h" +#include "mysql_version.h" #include "backup_copy.h" #include "backup_mysql.h" #include "mysqld.h" +#include "encryption_plugin.h" +#include char *tool_name; @@ -88,15 +91,7 @@ time_t history_lock_time; MYSQL *mysql_connection; -extern "C" { -MYSQL * STDCALL -cli_mysql_real_connect(MYSQL *mysql,const char *host, const char *user, - const char *passwd, const char *db, - uint port, const char *unix_socket,ulong client_flag); -} - -#define mysql_real_connect cli_mysql_real_connect - +my_bool opt_ssl_verify_server_cert; MYSQL * xb_mysql_connect() @@ -136,11 +131,6 @@ xb_mysql_connect() } mysql_options(connection,MYSQL_OPT_SSL_VERIFY_SERVER_CERT, (char*)&opt_ssl_verify_server_cert); -#if !defined(HAVE_YASSL) - if (opt_server_public_key && *opt_server_public_key) - mysql_options(connection, MYSQL_SERVER_PUBLIC_KEY, - opt_server_public_key); -#endif #endif if (!mysql_real_connect(connection, @@ -310,7 +300,7 @@ check_server_version(unsigned long version_number, version_supported = version_supported || (version_number > 50500 && version_number < 50700); version_supported = version_supported - || ((version_number > 100000 && version_number < 100300) + || ((version_number > 100000) && server_flavor == FLAVOR_MARIADB); if (mysql51 && innodb_version == NULL) { @@ -596,7 +586,7 @@ select_incremental_lsn_from_history(lsn_t *incremental_lsn) if (opt_incremental_history_name) { mysql_real_escape_string(mysql_connection, buf, opt_incremental_history_name, - strlen(opt_incremental_history_name)); + (unsigned long)strlen(opt_incremental_history_name)); ut_snprintf(query, sizeof(query), "SELECT innodb_to_lsn " "FROM PERCONA_SCHEMA.xtrabackup_history " @@ -609,7 +599,7 @@ select_incremental_lsn_from_history(lsn_t *incremental_lsn) if (opt_incremental_history_uuid) { mysql_real_escape_string(mysql_connection, buf, opt_incremental_history_uuid, - strlen(opt_incremental_history_uuid)); + (unsigned long)strlen(opt_incremental_history_uuid)); ut_snprintf(query, sizeof(query), "SELECT innodb_to_lsn " "FROM PERCONA_SCHEMA.xtrabackup_history " @@ -756,7 +746,7 @@ have_queries_to_wait_for(MYSQL *connection, uint threshold) static void -kill_long_queries(MYSQL *connection, uint timeout) +kill_long_queries(MYSQL *connection, time_t timeout) { MYSQL_RES *result; MYSQL_ROW row; @@ -768,15 +758,15 @@ kill_long_queries(MYSQL *connection, uint timeout) all_queries = (opt_kill_long_query_type == QUERY_TYPE_ALL); while ((row = mysql_fetch_row(result)) != NULL) { const char *info = row[7]; - int duration = atoi(row[5]); + long long duration = atoll(row[5]); char *id = row[0]; if (info != NULL && - duration >= (int)timeout && + (time_t)duration >= timeout && ((all_queries && is_query(info)) || is_select_query(info))) { msg_ts("Killing query %s (duration %d sec): %s\n", - id, duration, info); + id, (int)duration, info); ut_snprintf(kill_stmt, sizeof(kill_stmt), "KILL %s", id); xb_mysql_query(connection, kill_stmt, false, false); @@ -1378,7 +1368,18 @@ cleanup: return(result); } - +static string escape_and_quote(MYSQL *mysql,const char *str) +{ + if (!str) + return "NULL"; + size_t len = strlen(str); + char* escaped = (char *)alloca(2 * len + 3); + escaped[0] = '\''; + size_t new_len = mysql_real_escape_string(mysql, escaped+1, str, len); + escaped[new_len + 1] = '\''; + escaped[new_len + 2] = 0; + return string(escaped); +} /*********************************************************************//** Writes xtrabackup_info file and if backup_history is enable creates @@ -1388,25 +1389,16 @@ backup. */ bool write_xtrabackup_info(MYSQL *connection) { - MYSQL_STMT *stmt; - MYSQL_BIND bind[19]; + char *uuid = NULL; char *server_version = NULL; char buf_start_time[100]; char buf_end_time[100]; - int idx; tm tm; my_bool null = TRUE; - + ostringstream oss; const char *xb_stream_name[] = {"file", "tar", "xbstream"}; - const char *ins_query = "insert into PERCONA_SCHEMA.xtrabackup_history(" - "uuid, name, tool_name, tool_command, tool_version, " - "ibbackup_version, server_version, start_time, end_time, " - "lock_time, binlog_pos, innodb_from_lsn, innodb_to_lsn, " - "partial, incremental, format, compact, compressed, " - "encrypted) " - "values(?,?,?,?,?,?,?,from_unixtime(?),from_unixtime(?)," - "?,?,?,?,?,?,?,?,?,?)"; + ut_ad(xtrabackup_stream_fmt < 3); @@ -1419,6 +1411,11 @@ write_xtrabackup_info(MYSQL *connection) localtime_r(&history_end_time, &tm); strftime(buf_end_time, sizeof(buf_end_time), "%Y-%m-%d %H:%M:%S", &tm); + bool is_partial = (xtrabackup_tables + || xtrabackup_tables_file + || xtrabackup_databases + || xtrabackup_databases_file); + backup_file_printf(XTRABACKUP_INFO, "uuid = %s\n" "name = %s\n" @@ -1443,23 +1440,20 @@ write_xtrabackup_info(MYSQL *connection) opt_history ? opt_history : "", /* name */ tool_name, /* tool_name */ tool_args, /* tool_command */ - XTRABACKUP_VERSION, /* tool_version */ - XTRABACKUP_VERSION, /* ibbackup_version */ + MYSQL_SERVER_VERSION, /* tool_version */ + MYSQL_SERVER_VERSION, /* ibbackup_version */ server_version, /* server_version */ buf_start_time, /* start_time */ buf_end_time, /* end_time */ - history_lock_time, /* lock_time */ + (int)history_lock_time, /* lock_time */ mysql_binlog_position ? mysql_binlog_position : "", /* binlog_pos */ incremental_lsn, /* innodb_from_lsn */ metadata_to_lsn, /* innodb_to_lsn */ - (xtrabackup_tables /* partial */ - || xtrabackup_tables_file - || xtrabackup_databases - || xtrabackup_databases_file) ? "Y" : "N", + is_partial? "Y" : "N", xtrabackup_incremental ? "Y" : "N", /* incremental */ xb_stream_name[xtrabackup_stream_fmt], /* format */ - xtrabackup_compact ? "Y" : "N", /* compact */ + "N", /* compact */ xtrabackup_compress ? "compressed" : "N", /* compressed */ xtrabackup_encrypt ? "Y" : "N"); /* encrypted */ @@ -1492,142 +1486,36 @@ write_xtrabackup_info(MYSQL *connection) "encrypted ENUM('Y', 'N') DEFAULT NULL" ") CHARACTER SET utf8 ENGINE=innodb", false); - stmt = mysql_stmt_init(connection); - mysql_stmt_prepare(stmt, ins_query, strlen(ins_query)); +#define ESCAPE_BOOL(expr) ((expr)?"'Y'":"'N'") - memset(bind, 0, sizeof(bind)); - idx = 0; + oss << "insert into PERCONA_SCHEMA.xtrabackup_history(" + << "uuid, name, tool_name, tool_command, tool_version," + << "ibbackup_version, server_version, start_time, end_time," + << "lock_time, binlog_pos, innodb_from_lsn, innodb_to_lsn," + << "partial, incremental, format, compact, compressed, " + << "encrypted) values(" + << escape_and_quote(connection, uuid) << "," + << escape_and_quote(connection, opt_history) << "," + << escape_and_quote(connection, tool_name) << "," + << escape_and_quote(connection, tool_args) << "," + << escape_and_quote(connection, MYSQL_SERVER_VERSION) << "," + << escape_and_quote(connection, MYSQL_SERVER_VERSION) << "," + << escape_and_quote(connection, server_version) << "," + << "from_unixtime(" << history_start_time << ")," + << "from_unixtime(" << history_end_time << ")," + << history_lock_time << "," + << escape_and_quote(connection, mysql_binlog_position) << "," + << incremental_lsn << "," + << metadata_to_lsn << "," + << ESCAPE_BOOL(is_partial) << "," + << ESCAPE_BOOL(xtrabackup_incremental)<< "," + << escape_and_quote(connection,xb_stream_name[xtrabackup_stream_fmt]) <<"," + << ESCAPE_BOOL(false) << "," + << ESCAPE_BOOL(xtrabackup_compress) << "," + << ESCAPE_BOOL(xtrabackup_encrypt) <<")"; - /* uuid */ - bind[idx].buffer_type = MYSQL_TYPE_STRING; - bind[idx].buffer = uuid; - bind[idx].buffer_length = strlen(uuid); - ++idx; - - /* name */ - bind[idx].buffer_type = MYSQL_TYPE_STRING; - bind[idx].buffer = (char*)(opt_history); - bind[idx].buffer_length = strlen(opt_history); - if (!(opt_history && *opt_history)) { - bind[idx].is_null = &null; - } - ++idx; - - /* tool_name */ - bind[idx].buffer_type = MYSQL_TYPE_STRING; - bind[idx].buffer = tool_name; - bind[idx].buffer_length = strlen(tool_name); - ++idx; - - /* tool_command */ - bind[idx].buffer_type = MYSQL_TYPE_STRING; - bind[idx].buffer = tool_args; - bind[idx].buffer_length = strlen(tool_args); - ++idx; - - /* tool_version */ - bind[idx].buffer_type = MYSQL_TYPE_STRING; - bind[idx].buffer = (char*)(XTRABACKUP_VERSION); - bind[idx].buffer_length = strlen(XTRABACKUP_VERSION); - ++idx; - - /* ibbackup_version */ - bind[idx].buffer_type = MYSQL_TYPE_STRING; - bind[idx].buffer = (char*)(XTRABACKUP_VERSION); - bind[idx].buffer_length = strlen(XTRABACKUP_VERSION); - ++idx; - - /* server_version */ - bind[idx].buffer_type = MYSQL_TYPE_STRING; - bind[idx].buffer = server_version; - bind[idx].buffer_length = strlen(server_version); - ++idx; - - /* start_time */ - bind[idx].buffer_type = MYSQL_TYPE_LONG; - bind[idx].buffer = &history_start_time; - ++idx; - - /* end_time */ - bind[idx].buffer_type = MYSQL_TYPE_LONG; - bind[idx].buffer = &history_end_time; - ++idx; - - /* lock_time */ - bind[idx].buffer_type = MYSQL_TYPE_LONG; - bind[idx].buffer = &history_lock_time; - ++idx; - - /* binlog_pos */ - bind[idx].buffer_type = MYSQL_TYPE_STRING; - bind[idx].buffer = mysql_binlog_position; - if (mysql_binlog_position != NULL) { - bind[idx].buffer_length = strlen(mysql_binlog_position); - } else { - bind[idx].is_null = &null; - } - ++idx; - - /* innodb_from_lsn */ - bind[idx].buffer_type = MYSQL_TYPE_LONGLONG; - bind[idx].buffer = (char*)(&incremental_lsn); - ++idx; - - /* innodb_to_lsn */ - bind[idx].buffer_type = MYSQL_TYPE_LONGLONG; - bind[idx].buffer = (char*)(&metadata_to_lsn); - ++idx; - - /* partial (Y | N) */ - bind[idx].buffer_type = MYSQL_TYPE_STRING; - bind[idx].buffer = (char*)((xtrabackup_tables - || xtrabackup_tables_file - || xtrabackup_databases - || xtrabackup_databases_file) ? "Y" : "N"); - bind[idx].buffer_length = 1; - ++idx; - - /* incremental (Y | N) */ - bind[idx].buffer_type = MYSQL_TYPE_STRING; - bind[idx].buffer = (char*)( - (xtrabackup_incremental - || xtrabackup_incremental_basedir - || opt_incremental_history_name - || opt_incremental_history_uuid) ? "Y" : "N"); - bind[idx].buffer_length = 1; - ++idx; - - /* format (file | tar | xbstream) */ - bind[idx].buffer_type = MYSQL_TYPE_STRING; - bind[idx].buffer = (char*)(xb_stream_name[xtrabackup_stream_fmt]); - bind[idx].buffer_length = strlen(xb_stream_name[xtrabackup_stream_fmt]); - ++idx; - - /* compact (Y | N) */ - bind[idx].buffer_type = MYSQL_TYPE_STRING; - bind[idx].buffer = (char*)(xtrabackup_compact ? "Y" : "N"); - bind[idx].buffer_length = 1; - ++idx; - - /* compressed (Y | N) */ - bind[idx].buffer_type = MYSQL_TYPE_STRING; - bind[idx].buffer = (char*)(xtrabackup_compress ? "Y" : "N"); - bind[idx].buffer_length = 1; - ++idx; - - /* encrypted (Y | N) */ - bind[idx].buffer_type = MYSQL_TYPE_STRING; - bind[idx].buffer = (char*)(xtrabackup_encrypt ? "Y" : "N"); - bind[idx].buffer_length = 1; - ++idx; - - ut_ad(idx == 19); - - mysql_stmt_bind_param(stmt, bind); - - mysql_stmt_execute(stmt); - mysql_stmt_close(stmt); + xb_mysql_query(mysql_connection, oss.str().c_str(), false); cleanup: @@ -1637,10 +1525,11 @@ cleanup: return(true); } -bool -write_backup_config_file() +extern const char *innodb_checksum_algorithm_names[]; + +bool write_backup_config_file() { - return backup_file_printf("backup-my.cnf", + int rc= backup_file_printf("backup-my.cnf", "# This MySQL options file was generated by innobackupex.\n\n" "# The MySQL server\n" "[mysqld]\n" @@ -1649,19 +1538,18 @@ write_backup_config_file() "innodb_data_file_path=%s\n" "innodb_log_files_in_group=%lu\n" "innodb_log_file_size=%lld\n" - "innodb_fast_checksum=%s\n" "innodb_page_size=%lu\n" "innodb_log_block_size=%lu\n" "innodb_undo_directory=%s\n" "innodb_undo_tablespaces=%lu\n" "%s%s\n" - "%s%s\n", + "%s%s\n" + "%s\n", innodb_checksum_algorithm_names[srv_checksum_algorithm], innodb_checksum_algorithm_names[srv_log_checksum_algorithm], innobase_data_file_path, srv_n_log_files, innobase_log_file_size, - srv_fast_checksum ? "true" : "false", srv_page_size, srv_log_block_size, srv_undo_dir, @@ -1671,7 +1559,9 @@ write_backup_config_file() innobase_buffer_pool_filename ? "innodb_buffer_pool_filename=" : "", innobase_buffer_pool_filename ? - innobase_buffer_pool_filename : ""); + innobase_buffer_pool_filename : "", + encryption_plugin_get_config()); + return rc; } diff --git a/extra/mariabackup/changed_page_bitmap.cc b/extra/mariabackup/changed_page_bitmap.cc index e385474c7aa..435b7fb6172 100644 --- a/extra/mariabackup/changed_page_bitmap.cc +++ b/extra/mariabackup/changed_page_bitmap.cc @@ -447,7 +447,7 @@ log_online_open_bitmap_file_read_only( = os_file_create_simple_no_error_handling(0, bitmap_file->name, OS_FILE_OPEN, OS_FILE_READ_ONLY, - &success); + &success,0); if (UNIV_UNLIKELY(!success)) { /* Here and below assume that bitmap file names do not diff --git a/extra/mariabackup/common.h b/extra/mariabackup/common.h index abbc4b41a85..7b1dfd7a0db 100644 --- a/extra/mariabackup/common.h +++ b/extra/mariabackup/common.h @@ -26,6 +26,46 @@ Foundation, Inc., 51 Franklin Street, Fifth Floor, Boston, MA 02110-1301, USA #include #include + +# define fil_is_user_tablespace_id(i) ((i) > srv_undo_tablespaces_open) + +#ifdef _MSC_VER +#define stat _stati64 +#define PATH_MAX MAX_PATH +#endif + +#ifndef HAVE_VASPRINTF +static inline int vasprintf(char **strp, const char *fmt, va_list args) +{ + int len; +#ifdef _MSC_VER + len = _vscprintf(fmt, args); +#else + len = vsnprintf(NULL, 0, fmt, args); +#endif + if (len < 0) + { + return -1; + } + *strp = (char *)malloc(len + 1); + if (!*strp) + { + return -1; + } + vsprintf(*strp, fmt, args); + return len; +} + +static inline int asprintf(char **strp, const char *fmt,...) +{ + va_list args; + va_start(args, fmt); + int len = vasprintf(strp, fmt, args); + va_end(args); + return len; +} +#endif + #define xb_a(expr) \ do { \ if (!(expr)) { \ @@ -93,15 +133,15 @@ static inline int msg_ts(const char *fmt, ...) /*********************************************************************** Computes bit shift for a given value. If the argument is not a power of 2, returns 0.*/ -static inline ulong -get_bit_shift(ulong value) +static inline size_t +get_bit_shift(size_t value) { - ulong shift; + size_t shift; if (value == 0) return 0; - for (shift = 0; !(value & 1UL); shift++) { + for (shift = 0; !(value & 1); shift++) { value >>= 1; } return (value >> 1) ? 0 : shift; diff --git a/extra/mariabackup/compact.cc b/extra/mariabackup/compact.cc deleted file mode 100644 index 5d08a6e02b2..00000000000 --- a/extra/mariabackup/compact.cc +++ /dev/null @@ -1,1059 +0,0 @@ -/****************************************************** -XtraBackup: hot backup tool for InnoDB -(c) 2009-2014 Percona LLC and/or its affiliates. -Originally Created 3/3/2009 Yasufumi Kinoshita -Written by Alexey Kopytov, Aleksandr Kuzminsky, Stewart Smith, Vadim Tkachenko, -Yasufumi Kinoshita, Ignacio Nin and Baron Schwartz. - -This program is free software; you can redistribute it and/or modify -it under the terms of the GNU General Public License as published by -the Free Software Foundation; version 2 of the License. - -This program is distributed in the hope that it will be useful, -but WITHOUT ANY WARRANTY; without even the implied warranty of -MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the -GNU General Public License for more details. - -You should have received a copy of the GNU General Public License -along with this program; if not, write to the Free Software -Foundation, Inc., 51 Franklin Street, Fifth Floor, Boston, MA 02110-1301, USA - -*******************************************************/ - -/* Compact backups implementation */ - -#include -#include - -#include -#include -#include -#include -#include -#include -#include -#include -#include "common.h" -#include "write_filt.h" -#include "fil_cur.h" -#include "xtrabackup.h" -#include "ds_buffer.h" -#include "xb0xb.h" - -/* Number of the first primary key page in an .ibd file */ -#define XB_FIRST_CLUSTERED_INDEX_PAGE_NO 3 - -/* Suffix for page map files */ -#define XB_PAGE_MAP_SUFFIX ".pmap" -#define XB_TMPFILE_SUFFIX ".tmp" - -/* Page range */ -struct page_range_t { - ulint from; /*!< range start */ - ulint to; /*!< range end */ -}; - -/* Cursor in a page map file */ -struct page_map_cursor_t { - File fd; /*!< file descriptor */ - IO_CACHE cache; /*!< IO_CACHE associated with fd */ -}; - -/* Table descriptor for the index rebuild operation */ -struct index_rebuild_table_t { - char* name; /* table name */ - ulint space_id; /* space ID */ - UT_LIST_NODE_T(index_rebuild_table_t) list; /* list node */ -}; - -/* Thread descriptor for the index rebuild operation */ -struct index_rebuild_thread_t { - ulint num; /* thread number */ - pthread_t id; /* thread ID */ -}; - -/* Empty page use to replace skipped pages in the data files */ -static byte empty_page[UNIV_PAGE_SIZE_MAX]; -static const char compacted_page_magic[] = "COMPACTP"; -static const size_t compacted_page_magic_size = - sizeof(compacted_page_magic) - 1; -static const ulint compacted_page_magic_offset = FIL_PAGE_DATA; - -/* Mutex protecting table_list */ -static pthread_mutex_t table_list_mutex; -/* List of tablespaces to process by the index rebuild operation */ -static UT_LIST_BASE_NODE_T(index_rebuild_table_t) table_list; - - -/************************************************************************ -Compact page filter. */ -static my_bool wf_compact_init(xb_write_filt_ctxt_t *ctxt, char *dst_name, - xb_fil_cur_t *cursor); -static my_bool wf_compact_process(xb_write_filt_ctxt_t *ctxt, - ds_file_t *dstfile); -static my_bool wf_compact_finalize(xb_write_filt_ctxt_t *ctxt, - ds_file_t *dstfile); -xb_write_filt_t wf_compact = { - &wf_compact_init, - &wf_compact_process, - &wf_compact_finalize, - NULL -}; - -/************************************************************************ -Initialize the compact page filter. - -@return TRUE on success, FALSE on error. */ -static my_bool -wf_compact_init(xb_write_filt_ctxt_t *ctxt, - char *dst_name __attribute__((unused)), xb_fil_cur_t *cursor) -{ - xb_wf_compact_ctxt_t *cp = &(ctxt->u.wf_compact_ctxt); - char page_map_name[FN_REFLEN]; - MY_STAT mystat; - - ctxt->cursor = cursor; - cp->clustered_index_found = FALSE; - cp->inside_skipped_range = FALSE; - cp->free_limit = 0; - - /* Don't compact the system table space */ - cp->skip = cursor->is_system; - if (cp->skip) { - return(TRUE); - } - - snprintf(page_map_name, sizeof(page_map_name), "%s%s", dst_name, - XB_PAGE_MAP_SUFFIX); - - cp->ds_buffer = ds_create(xtrabackup_target_dir, DS_TYPE_BUFFER); - if (cp->ds_buffer == NULL) { - return(FALSE); - } - - ds_set_pipe(cp->ds_buffer, ds_meta); - - memset(&mystat, 0, sizeof(mystat)); - mystat.st_mtime = my_time(0); - cp->buffer = ds_open(cp->ds_buffer, page_map_name, &mystat); - if (cp->buffer == NULL) { - msg("xtrabackup: Error: cannot open output stream for %s\n", - page_map_name); - return(FALSE); - } - - return(TRUE); -} - -/************************************************************************ -Check if the specified page should be skipped. We currently skip all -non-clustered index pages for compact backups. - -@return TRUE if the page should be skipped. */ -static my_bool -check_if_skip_page(xb_wf_compact_ctxt_t *cp, xb_fil_cur_t *cursor, ulint offset) -{ - byte *page; - ulint page_no; - ulint page_type; - index_id_t index_id; - - - xb_ad(cursor->is_system == FALSE); - - page = cursor->buf + cursor->page_size * offset; - page_no = cursor->buf_page_no + offset; - page_type = fil_page_get_type(page); - - if (UNIV_UNLIKELY(page_no == 0)) { - - cp->free_limit = mach_read_from_4(page + FSP_HEADER_OFFSET + - FSP_FREE_LIMIT); - } else if (UNIV_UNLIKELY(page_no == XB_FIRST_CLUSTERED_INDEX_PAGE_NO)) { - - xb_ad(cp->clustered_index_found == FALSE); - - if (page_type != FIL_PAGE_INDEX) { - - /* Uninitialized clustered index root page, there's - nothing we can do to compact the space.*/ - - msg("[%02u] Uninitialized page type value (%lu) in the " - "clustered index root page of tablespace %s. " - "Will not be compacted.\n", - cursor->thread_n, - page_type, cursor->rel_path); - - cp->skip = TRUE; - - return(FALSE); - } - - cp->clustered_index = - mach_read_from_8(page + PAGE_HEADER + PAGE_INDEX_ID); - cp->clustered_index_found = TRUE; - } else if (UNIV_UNLIKELY(page_no >= cp->free_limit)) { - - /* Skip unused pages above free limit, if that value is set in - the FSP header.*/ - - return(cp->free_limit > 0); - } else if (cp->clustered_index_found && page_type == FIL_PAGE_INDEX) { - - index_id = mach_read_from_8(page + PAGE_HEADER + PAGE_INDEX_ID); - if (index_id != cp->clustered_index) { - - ulint fseg_hdr_space = - mach_read_from_4(page + PAGE_HEADER + - PAGE_BTR_SEG_TOP); - ulint fseg_hdr_page_no = - mach_read_from_4(page + PAGE_HEADER + - PAGE_BTR_SEG_TOP + 4); - ulint fseg_hdr_offset = - mach_read_from_2(page + PAGE_HEADER + - PAGE_BTR_SEG_TOP + 8); - - /* Don't skip root index pages, i.e. the ones where the - above fields are defined. We need root index pages to be - able to correctly drop the indexes later, as they - contain fseg inode pointers. */ - - return(fseg_hdr_space == 0 && - fseg_hdr_page_no == 0 && - fseg_hdr_offset == 0); - } - } - - return(FALSE); -} - -/************************************************************************ -Run the next batch of pages through the compact page filter. - -@return TRUE on success, FALSE on error. */ -static my_bool -wf_compact_process(xb_write_filt_ctxt_t *ctxt, ds_file_t *dstfile) -{ - xb_fil_cur_t *cursor = ctxt->cursor; - ulint page_size = cursor->page_size; - byte *page; - byte *buf_end; - byte *write_from; - xb_wf_compact_ctxt_t *cp = &(ctxt->u.wf_compact_ctxt); - ulint i; - ulint page_no; - byte tmp[4]; - - if (cp->skip) { - return(!ds_write(dstfile, cursor->buf, cursor->buf_read)); - } - - write_from = NULL; - buf_end = cursor->buf + cursor->buf_read; - for (i = 0, page = cursor->buf; page < buf_end; - i++, page += page_size) { - - page_no = cursor->buf_page_no + i; - - if (!check_if_skip_page(cp, cursor, i)) { - - if (write_from == NULL) { - write_from = page; - } - - if (cp->inside_skipped_range) { - cp->inside_skipped_range = FALSE; - - /* Write the last range endpoint to the - skipped pages map */ - - xb_ad(page_no > 0); - mach_write_to_4(tmp, page_no - 1); - if (ds_write(cp->buffer, tmp, sizeof(tmp))) { - return(FALSE); - } - } - continue; - } - - if (write_from != NULL) { - - /* The first skipped page in this block, write the - non-skipped ones to the data file */ - - if (ds_write(dstfile, write_from, page - write_from)) { - return(FALSE); - } - - write_from = NULL; - } - - if (!cp->inside_skipped_range) { - - /* The first skipped page in range, write the first - range endpoint to the skipped pages map */ - - cp->inside_skipped_range = TRUE; - - mach_write_to_4(tmp, page_no); - if (ds_write(cp->buffer, tmp, sizeof(tmp))) { - return(FALSE); - } - } - } - - /* Write the remaining pages in the buffer, if any */ - if (write_from != NULL && - ds_write(dstfile, write_from, buf_end - write_from)) { - return(FALSE); - } - - return(TRUE); -} - -/************************************************************************ -Close the compact filter's page map stream. - -@return TRUE on success, FALSE on error. */ -static my_bool -wf_compact_finalize(xb_write_filt_ctxt_t *ctxt, - ds_file_t *dstfile __attribute__((unused))) -{ - xb_fil_cur_t *cursor = ctxt->cursor; - xb_wf_compact_ctxt_t *cp = &(ctxt->u.wf_compact_ctxt); - my_bool rc = TRUE; - - /* Write the last endpoint of the current range, if the last pages of - the space have been skipped. */ - if (cp->inside_skipped_range) { - byte tmp[4]; - - mach_write_to_4(tmp, cursor->space_size - 1); - if (ds_write(cp->buffer, tmp, sizeof(tmp))) { - return(FALSE); - } - - cp->inside_skipped_range = FALSE; - } - - if (cp->buffer) { - if (ds_close(cp->buffer)) { - rc = FALSE; - } - } - if (cp->ds_buffer) { - ds_destroy(cp->ds_buffer); - } - - return(rc); -} - -/************************************************************************ -Open a page map file and return a cursor. - -@return page map cursor, or NULL if the file doesn't exist. */ -static page_map_cursor_t * -page_map_file_open(const char *path) -{ - MY_STAT statinfo; - page_map_cursor_t *pmap_cur; - int rc; - - if (my_stat(path, &statinfo, MYF(0)) == NULL) { - - return(NULL); - } - - /* The maximum possible page map file corresponds to a 64 TB tablespace - and the worst case when every other page was skipped. That is, 2^32/2 - page ranges = 16 GB. */ - xb_a(statinfo.st_size < (off_t) 16 * 1024 * 1024 * 1024); - - /* Must be a series of 8-byte tuples */ - xb_a(statinfo.st_size % 8 == 0); - - pmap_cur = (page_map_cursor_t *) my_malloc(sizeof(page_map_cursor_t), - MYF(MY_FAE)); - - pmap_cur->fd = my_open(path, O_RDONLY, MYF(MY_WME)); - xb_a(pmap_cur->fd != 0); - - rc = init_io_cache(&pmap_cur->cache, pmap_cur->fd, 0, READ_CACHE, - 0, 0, MYF(MY_WME)); - xb_a(rc == 0); - - return(pmap_cur); -} - -/************************************************************************ -Read the next range from a page map file and update the cursor. - -@return TRUE on success, FALSE on end-of-file. */ -static ibool -page_map_file_next(page_map_cursor_t *pmap_cur, page_range_t *range) -{ - byte buf[8]; - - xb_ad(pmap_cur != NULL); - - if (my_b_read(&pmap_cur->cache, buf, sizeof(buf))) { - return(FALSE); - } - - range->from = mach_read_from_4(buf); - range->to = mach_read_from_4(buf + 4); - - return(TRUE); -} - -/************************************************************************ -Close the page map cursor.*/ -static void -page_map_file_close(page_map_cursor_t *pmap_cur) -{ - int rc; - - xb_ad(pmap_cur != NULL); - - rc = end_io_cache(&pmap_cur->cache); - xb_a(rc == 0); - - posix_fadvise(pmap_cur->fd, 0, 0, POSIX_FADV_DONTNEED); - - rc = my_close(pmap_cur->fd, MY_WME); - xb_a(rc == 0); - - my_free(pmap_cur); -} - -/**************************************************************************** -Expand a single data file according to the skipped pages maps created by ---compact. - -@return TRUE on success, FALSE on failure. */ -static my_bool -xb_expand_file(fil_node_t *node) -{ - char pmapfile_path[FN_REFLEN]; - char tmpfile_path[FN_REFLEN]; - xb_fil_cur_t cursor; - xb_fil_cur_result_t res; - ds_ctxt_t *ds_local; - ds_ctxt_t *ds_buffer; - ds_file_t *tmpfile; - my_bool success = FALSE; - ulint i; - byte *page; - ulint page_expected_no; - page_map_cursor_t *pmap_cur; - ibool have_next_range; - page_range_t pmap_range; - - xb_ad(trx_sys_sys_space(node->space->id) == FALSE); - - snprintf(pmapfile_path, sizeof(pmapfile_path), "%s%s", - node->name, XB_PAGE_MAP_SUFFIX); - - /* Skip files that don't have a corresponding page map file */ - - if (!(pmap_cur = page_map_file_open(pmapfile_path))) { - - msg("Not expanding %s\n", node->name); - - return(FALSE); - } - - msg("Expanding %s\n", node->name); - - ds_local = ds_create(".", DS_TYPE_LOCAL); - ds_buffer = ds_create(".", DS_TYPE_BUFFER); - - xb_a(ds_local != NULL && ds_buffer != NULL); - - ds_buffer_set_size(ds_buffer, FSP_EXTENT_SIZE * UNIV_PAGE_SIZE_MAX); - - ds_set_pipe(ds_buffer, ds_local); - - res = xb_fil_cur_open(&cursor, &rf_pass_through, node, 1); - xb_a(res == XB_FIL_CUR_SUCCESS); - - snprintf(tmpfile_path, sizeof(tmpfile_path), "%s%s", - node->name, XB_TMPFILE_SUFFIX); - - tmpfile = ds_open(ds_buffer, tmpfile_path, &cursor.statinfo); - if (tmpfile == NULL) { - - msg("Could not open temporary file '%s'\n", tmpfile_path); - goto error; - } - - have_next_range = page_map_file_next(pmap_cur, &pmap_range); - - page_expected_no = 0; - - /* Initialize and mark the empty page which is used to replace - skipped pages. */ - memset(empty_page, 0, cursor.page_size); - memcpy(empty_page + compacted_page_magic_offset, - compacted_page_magic, compacted_page_magic_size); - mach_write_to_4(empty_page + FIL_PAGE_SPACE_OR_CHKSUM, - BUF_NO_CHECKSUM_MAGIC); - mach_write_to_4(empty_page + cursor.page_size - - FIL_PAGE_END_LSN_OLD_CHKSUM, - BUF_NO_CHECKSUM_MAGIC); - - - /* Main copy loop */ - - while ((res = xb_fil_cur_read(&cursor)) == XB_FIL_CUR_SUCCESS) { - - for (i = 0, page = cursor.buf; i < cursor.buf_npages; - i++, page += cursor.page_size) { - - ulint page_read_no; - - page_read_no = mach_read_from_4(page + FIL_PAGE_OFFSET); - xb_a(!page_read_no || page_expected_no <= page_read_no); - - if (have_next_range && - page_expected_no == pmap_range.from) { - - xb_a(pmap_range.from <= pmap_range.to); - - /* Write empty pages instead of skipped ones, if - necessary. */ - - while (page_expected_no <= pmap_range.to) { - - if (ds_write(tmpfile, empty_page, - cursor.page_size)) { - - goto write_error; - } - - page_expected_no++; - } - - have_next_range = - page_map_file_next(pmap_cur, - &pmap_range); - } - - /* Write the current page */ - - if (ds_write(tmpfile, page, cursor.page_size)) { - - goto write_error; - } - - page_expected_no++; - } - } - - if (res != XB_FIL_CUR_EOF) { - - goto error; - } - - /* Write empty pages instead of trailing skipped ones, if any */ - - if (have_next_range) { - - xb_a(page_expected_no == pmap_range.from); - xb_a(pmap_range.from <= pmap_range.to); - - while (page_expected_no <= pmap_range.to) { - - if (ds_write(tmpfile, empty_page, - cursor.page_size)) { - - goto write_error; - } - - page_expected_no++; - } - - xb_a(!page_map_file_next(pmap_cur, &pmap_range)); - } - - /* Replace the original .ibd file with the expanded file */ - if (my_rename(tmpfile_path, node->name, MYF(MY_WME))) { - - msg("Failed to rename '%s' to '%s'\n", - tmpfile_path, node->name); - goto error; - } - - my_delete(pmapfile_path, MYF(MY_WME)); - - if (!ds_close(tmpfile)) { - success = TRUE; - } - tmpfile = NULL; - - goto end; - -write_error: - msg("Write to '%s' failed\n", tmpfile_path); - -error: - if (tmpfile != NULL) { - - ds_close(tmpfile); - my_delete(tmpfile_path, MYF(MY_WME)); - } - -end: - ds_destroy(ds_buffer); - ds_destroy(ds_local); - - xb_fil_cur_close(&cursor); - - page_map_file_close(pmap_cur); - - return(success); -} - -/****************************************************************************** -Expand the data files according to the skipped pages maps created by --compact. -@return TRUE on success, FALSE on failure. */ -my_bool -xb_expand_datafiles(void) -/*=====================*/ -{ - ulint nfiles; - datafiles_iter_t *it = NULL; - fil_node_t *node; - fil_space_t *space; - - msg("Starting to expand compacted .ibd files.\n"); - - /* Initialize the tablespace cache */ - if (xb_data_files_init() != DB_SUCCESS) { - return(FALSE); - } - - nfiles = UT_LIST_GET_LEN(fil_system->space_list); - xb_a(nfiles > 0); - - it = datafiles_iter_new(fil_system); - if (it == NULL) { - msg("xtrabackup: error: datafiles_iter_new() failed.\n"); - goto error; - } - - while ((node = datafiles_iter_next(it)) != NULL) { - - space = node->space; - - /* System tablespace cannot be compacted */ - if (!fil_is_user_tablespace_id(space->id)) { - - continue; - } - - if (!xb_expand_file(node)) { - - goto error; - } - } - - datafiles_iter_free(it); - xb_data_files_close(); - - return(TRUE); - -error: - if (it != NULL) { - datafiles_iter_free(it); - } - - xb_data_files_close(); - - return(FALSE); -} - -/****************************************************************************** -Callback used in buf_page_io_complete() to detect compacted pages. -@return TRUE if the page is marked as compacted, FALSE otherwise. */ -ibool -buf_page_is_compacted( -/*==================*/ - const byte* page) /*!< in: a database page */ -{ - return !memcmp(page + compacted_page_magic_offset, - compacted_page_magic, compacted_page_magic_size); -} - -/***************************************************************************** -Builds an index definition corresponding to an index object. It is roughly -similar to innobase_create_index_def() / innobase_create_index_field_def() and -the opposite to dict_mem_index_create() / dict_mem_index_add_field(). */ -static -void -xb_build_index_def( -/*=======================*/ - mem_heap_t* heap, /*!< in: heap */ - const dict_index_t* index, /*!< in: index */ - index_def_t* index_def) /*!< out: index definition */ -{ - index_field_t* fields; - ulint n_fields; - ulint i; - - ut_a(index->n_fields); - ut_ad(index->magic_n == DICT_INDEX_MAGIC_N); - - /* Use n_user_defined_cols instead of n_fields, as the index will - contain a part of the primary key after n_user_defined_cols, and those - columns will be created automatically in - dict_index_build_internal_clust(). */ - n_fields = index->n_user_defined_cols; - - memset(index_def, 0, sizeof(*index_def)); - - index_def->name = mem_heap_strdup(heap, index->name); - index_def->ind_type = index->type; - - fields = static_cast - (mem_heap_alloc(heap, n_fields * sizeof(*fields))); - - for (i = 0; i < n_fields; i++) { - dict_field_t* field; - - field = dict_index_get_nth_field(index, i); - fields[i].col_no = dict_col_get_no(field->col); - fields[i].prefix_len = field->prefix_len; - } - - index_def->fields = fields; - index_def->n_fields = n_fields; -} - -/* A dummy autoc_inc sequence for row_merge_build_indexes(). */ -static ib_sequence_t null_seq(NULL, 0, 0); -/* A dummy table share and table for row_merge_build_indexes() error reporting. -Assumes that no errors are going to be reported. */ -static struct TABLE_SHARE dummy_table_share; -static struct TABLE dummy_table; - -/********************************************************************//** -Rebuild secondary indexes for a given table. */ -static -void -xb_rebuild_indexes_for_table( -/*=========================*/ - dict_table_t* table, /*!< in: table */ - trx_t* trx, /*!< in: transaction handle */ - ulint thread_n) /*!< in: thread number */ -{ - dict_index_t* index; - dict_index_t** indexes; - ulint n_indexes; - index_def_t* index_defs; - ulint i; - mem_heap_t* heap; - ulint error; - ulint* add_key_nums; - - ut_ad(!mutex_own(&(dict_sys->mutex))); - ut_ad(table); - - ut_a(UT_LIST_GET_LEN(table->indexes) > 0); - - n_indexes = UT_LIST_GET_LEN(table->indexes) - 1; - if (!n_indexes) { - /* Only the primary key, nothing to do. */ - return; - } - - heap = mem_heap_create(1024); - - indexes = (dict_index_t**) mem_heap_alloc(heap, - n_indexes * sizeof(*indexes)); - index_defs = (index_def_t*) mem_heap_alloc(heap, n_indexes * - sizeof(*index_defs)); - add_key_nums = static_cast - (mem_heap_alloc(heap, n_indexes * sizeof(*add_key_nums))); - - /* Skip the primary key. */ - index = dict_table_get_first_index(table); - ut_a(dict_index_is_clust(index)); - - row_mysql_lock_data_dictionary(trx); - - for (i = 0; (index = dict_table_get_next_index(index)); i++) { - - msg("[%02lu] Found index %s\n", thread_n, index->name); - - /* Pretend that it's the current trx that created this index. - Required to avoid 5.6+ debug assertions. */ - index->trx_id = trx->id; - - xb_build_index_def(heap, index, &index_defs[i]); - - /* In 5.6+, row_merge_drop_indexes() drops all the indexes on - the table that have the temp index prefix. It does not accept - an array of indexes to drop as in 5.5-. */ - row_merge_rename_index_to_drop(trx, table->id, index->id); - } - - ut_ad(i == n_indexes); - - row_merge_drop_indexes(trx, table, TRUE); - - index = dict_table_get_first_index(table); - ut_a(dict_index_is_clust(index)); - index = dict_table_get_next_index(index); - while (index) { - - /* In 5.6+, row_merge_drop_indexes() does not remove the - indexes from the dictionary cache nor from any foreign key - list. This may cause invalid dereferences as we try to access - the dropped indexes from other tables as FKs. */ - - dict_index_t* next_index = dict_table_get_next_index(index); - index->to_be_dropped = 1; - - /* Patch up any FK referencing this index with NULL */ - dict_foreign_replace_index(table, NULL, index); - - dict_index_remove_from_cache(table, index); - - index = next_index; - } - - msg("[%02lu] Rebuilding %lu index(es).\n", thread_n, n_indexes); - - error = row_merge_lock_table(trx, table, LOCK_X); - xb_a(error == DB_SUCCESS); - - for (i = 0; i < n_indexes; i++) { - indexes[i] = row_merge_create_index(trx, table, - &index_defs[i]); - add_key_nums[i] = index_defs[i].key_number; - } - - /* Commit trx to release latches on system tables */ - trx_commit_for_mysql(trx); - trx_start_for_ddl(trx, TRX_DICT_OP_INDEX); - - row_mysql_unlock_data_dictionary(trx); - - /* Reacquire table lock for row_merge_build_indexes() */ - error = row_merge_lock_table(trx, table, LOCK_X); - xb_a(error == DB_SUCCESS); - - error = row_merge_build_indexes(trx, table, table, FALSE, indexes, - add_key_nums, n_indexes, &dummy_table, - NULL, NULL, ULINT_UNDEFINED, null_seq); - ut_a(error == DB_SUCCESS); - - mem_heap_free(heap); - - trx_commit_for_mysql(trx); - - trx_start_for_ddl(trx, TRX_DICT_OP_INDEX); -} - -/************************************************************************** -Worker thread function for index rebuild. */ -static -void * -xb_rebuild_indexes_thread_func( -/*===========================*/ - void* arg) /* thread context */ -{ - dict_table_t* table; - index_rebuild_table_t* rebuild_table; - index_rebuild_thread_t* thread; - trx_t* trx; - - thread = (index_rebuild_thread_t *) arg; - - trx = trx_allocate_for_mysql(); - - /* Suppress foreign key checks, as we are going to drop and recreate all - secondary keys. */ - trx->check_foreigns = FALSE; - trx_start_for_ddl(trx, TRX_DICT_OP_INDEX); - - /* Loop until there are no more tables in tables list */ - for (;;) { - pthread_mutex_lock(&table_list_mutex); - - rebuild_table = UT_LIST_GET_FIRST(table_list); - - if (rebuild_table == NULL) { - - pthread_mutex_unlock(&table_list_mutex); - break; - } - - UT_LIST_REMOVE(list, table_list, rebuild_table); - - pthread_mutex_unlock(&table_list_mutex); - - ut_ad(rebuild_table->name); - ut_ad(fil_is_user_tablespace_id(rebuild_table->space_id)); - - row_mysql_lock_data_dictionary(trx); - - table = dict_table_get_low(rebuild_table->name); - - ut_d(table->n_ref_count++); - - row_mysql_unlock_data_dictionary(trx); - - ut_a(table != NULL); - ut_a(table->space == rebuild_table->space_id); - - /* Discard change buffer entries for this space */ - ibuf_delete_for_discarded_space(rebuild_table->space_id); - - msg("[%02lu] Checking if there are indexes to rebuild in table " - "%s (space id: %lu)\n", - thread->num, - rebuild_table->name, rebuild_table->space_id); - - xb_rebuild_indexes_for_table(table, trx, thread->num); - - ut_d(table->n_ref_count--); - - mem_free(rebuild_table->name); - mem_free(rebuild_table); - } - - trx_commit_for_mysql(trx); - - trx_free_for_mysql(trx); - - return(NULL); -} - -/****************************************************************************** -Rebuild all secondary indexes in all tables in separate spaces. Called from -innobase_start_or_create_for_mysql(). */ -void -xb_compact_rebuild_indexes(void) -/*=============================*/ -{ - dict_table_t* sys_tables; - dict_index_t* sys_index; - btr_pcur_t pcur; - const rec_t* rec; - mtr_t mtr; - const byte* field; - ulint len; - ulint space_id; - trx_t* trx; - index_rebuild_table_t* rebuild_table; - index_rebuild_thread_t* threads; - ulint i; - - /* Set up the dummy table for the index rebuild error reporting */ - dummy_table_share.fields = 0; - dummy_table.s = &dummy_table_share; - - /* Iterate all tables that are not in the system tablespace and add them - to the list of tables to be rebuilt later. */ - - trx = trx_allocate_for_mysql(); - trx_start_for_ddl(trx, TRX_DICT_OP_INDEX); - - row_mysql_lock_data_dictionary(trx); - - /* Enlarge the fatal lock wait timeout during index rebuild - operation. */ - os_increment_counter_by_amount(server_mutex, - srv_fatal_semaphore_wait_threshold, - 7200); - - mtr_start(&mtr); - - sys_tables = dict_table_get_low("SYS_TABLES"); - sys_index = UT_LIST_GET_FIRST(sys_tables->indexes); - ut_a(!dict_table_is_comp(sys_tables)); - - pthread_mutex_init(&table_list_mutex, NULL); - UT_LIST_INIT(table_list); - - btr_pcur_open_at_index_side(TRUE, sys_index, BTR_SEARCH_LEAF, &pcur, - TRUE, 0, &mtr); - for (;;) { - btr_pcur_move_to_next_user_rec(&pcur, &mtr); - - rec = btr_pcur_get_rec(&pcur); - - if (!btr_pcur_is_on_user_rec(&pcur)) { - /* end of index */ - - break; - } - - if (rec_get_deleted_flag(rec, 0)) { - continue; - } - - field = rec_get_nth_field_old(rec, 9, &len); - ut_a(len == 4); - - space_id = mach_read_from_4(field); - - /* Don't touch tables in the system tablespace */ - if (!fil_is_user_tablespace_id(space_id)) { - - continue; - } - - field = rec_get_nth_field_old(rec, 0, &len); - - rebuild_table = static_cast - (mem_alloc(sizeof(*rebuild_table))); - rebuild_table->name = mem_strdupl((char*) field, len); - rebuild_table->space_id = space_id; - - UT_LIST_ADD_LAST(list, table_list, rebuild_table); - } - - btr_pcur_close(&pcur); - mtr_commit(&mtr); - - row_mysql_unlock_data_dictionary(trx); - - trx_commit_for_mysql(trx); - - trx_free_for_mysql(trx); - - /* Start worker threads for the index rebuild operation */ - ut_ad(xtrabackup_rebuild_threads > 0); - - if (xtrabackup_rebuild_threads > 1) { - msg("Starting %lu threads to rebuild indexes.\n", - xtrabackup_rebuild_threads); - } - - threads = (index_rebuild_thread_t *) - mem_alloc(sizeof(*threads) * - xtrabackup_rebuild_threads); - - for (i = 0; i < xtrabackup_rebuild_threads; i++) { - - threads[i].num = i+1; - if (pthread_create(&threads[i].id, NULL, - xb_rebuild_indexes_thread_func, - &threads[i])) { - - msg("error: pthread_create() failed: errno = %d\n", - errno); - ut_a(0); - } - } - - /* Wait for worker threads to finish */ - for (i = 0; i < xtrabackup_rebuild_threads; i++) { - pthread_join(threads[i].id, NULL); - } - - mem_free(threads); -} diff --git a/extra/mariabackup/compact.h b/extra/mariabackup/compact.h deleted file mode 100644 index d0d9840f66d..00000000000 --- a/extra/mariabackup/compact.h +++ /dev/null @@ -1,44 +0,0 @@ -/****************************************************** -XtraBackup: hot backup tool for InnoDB -(c) 2009-2013 Percona LLC and/or its affiliates. -Originally Created 3/3/2009 Yasufumi Kinoshita -Written by Alexey Kopytov, Aleksandr Kuzminsky, Stewart Smith, Vadim Tkachenko, -Yasufumi Kinoshita, Ignacio Nin and Baron Schwartz. - -This program is free software; you can redistribute it and/or modify -it under the terms of the GNU General Public License as published by -the Free Software Foundation; version 2 of the License. - -This program is distributed in the hope that it will be useful, -but WITHOUT ANY WARRANTY; without even the implied warranty of -MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the -GNU General Public License for more details. - -You should have received a copy of the GNU General Public License -along with this program; if not, write to the Free Software -Foundation, Inc., 51 Franklin Street, Fifth Floor, Boston, MA 02110-1301, USA - -*******************************************************/ - -#ifndef XB_COMPACT_H -#define XB_COMPACT_H - -#include "write_filt.h" - -/* Compact page filter context */ -typedef struct { - my_bool skip; - ds_ctxt_t *ds_buffer; - ds_file_t *buffer; - index_id_t clustered_index; - my_bool clustered_index_found; - my_bool inside_skipped_range; - ulint free_limit; -} xb_wf_compact_ctxt_t; - -/****************************************************************************** -Expand the data files according to the skipped pages maps created by --compact. -@return TRUE on success, FALSE on failure. */ -my_bool xb_expand_datafiles(void); - -#endif diff --git a/extra/mariabackup/datasink.c b/extra/mariabackup/datasink.c index 2f4233ddc98..199eb77ad55 100644 --- a/extra/mariabackup/datasink.c +++ b/extra/mariabackup/datasink.c @@ -46,7 +46,12 @@ ds_create(const char *root, ds_type_t type) ds = &datasink_local; break; case DS_TYPE_ARCHIVE: +#ifdef HAVE_LIBARCHIVE ds = &datasink_archive; +#else + msg("Error : mariabackup was built without libarchive support"); + exit(EXIT_FAILURE); +#endif break; case DS_TYPE_XBSTREAM: ds = &datasink_xbstream; @@ -55,8 +60,10 @@ ds_create(const char *root, ds_type_t type) ds = &datasink_compress; break; case DS_TYPE_ENCRYPT: - ds = &datasink_encrypt; + msg("Error : mariabackup does not support encrypted backups."); + exit(EXIT_FAILURE); break; + case DS_TYPE_TMPFILE: ds = &datasink_tmpfile; break; diff --git a/extra/mariabackup/datasink.h b/extra/mariabackup/datasink.h index 36a3d564a57..e378a806441 100644 --- a/extra/mariabackup/datasink.h +++ b/extra/mariabackup/datasink.h @@ -28,6 +28,7 @@ Foundation, Inc., 51 Franklin Street, Fifth Floor, Boston, MA 02110-1301, USA extern "C" { #endif +extern char *xtrabackup_tmpdir; struct datasink_struct; typedef struct datasink_struct datasink_t; diff --git a/extra/mariabackup/ds_compress.c b/extra/mariabackup/ds_compress.c index c49fceaf7fa..15801c8abd4 100644 --- a/extra/mariabackup/ds_compress.c +++ b/extra/mariabackup/ds_compress.c @@ -152,7 +152,7 @@ compress_open(ds_ctxt_t *ctxt, const char *path, MY_STAT *mystat) /* Write the qpress file header */ name_len = strlen(new_name); if (ds_write(dest_file, "F", 1) || - write_uint32_le(dest_file, name_len) || + write_uint32_le(dest_file, (uint)name_len) || /* we want to write the terminating \0 as well */ ds_write(dest_file, new_name, name_len + 1)) { goto err; @@ -453,7 +453,7 @@ compress_worker_thread_func(void *arg) with qpress implementation. */ thd->adler = adler32(0x00000001, (uchar *) thd->to, - thd->to_len); + (uInt)thd->to_len); } pthread_mutex_unlock(&thd->data_mutex); diff --git a/extra/mariabackup/ds_encrypt.c b/extra/mariabackup/ds_encrypt.c index f8d62a03e13..19ced416687 100644 --- a/extra/mariabackup/ds_encrypt.c +++ b/extra/mariabackup/ds_encrypt.c @@ -22,7 +22,7 @@ Foundation, Inc., 51 Franklin Street, Fifth Floor, Boston, MA 02110-1301, USA #include #include "common.h" #include "datasink.h" - +#ifdef HAVE_GRYPT #if GCC_VERSION >= 4002 /* Workaround to avoid "gcry_ac_* is deprecated" warnings in gcrypt.h */ # pragma GCC diagnostic ignored "-Wdeprecated-declarations" @@ -615,3 +615,4 @@ encrypt_worker_thread_func(void *arg) return NULL; } +#endif /* HAVE_GCRYPT*/ diff --git a/extra/mariabackup/ds_encrypt.h b/extra/mariabackup/ds_encrypt.h index ed869747d79..3e9e40ad354 100644 --- a/extra/mariabackup/ds_encrypt.h +++ b/extra/mariabackup/ds_encrypt.h @@ -22,7 +22,7 @@ Foundation, Inc., 51 Franklin Street, Fifth Floor, Boston, MA 02110-1301, USA #define DS_ENCRYPT_H #include "datasink.h" - +#ifdef HAVE_GCRYPT extern datasink_t datasink_encrypt; - +#endif #endif diff --git a/extra/mariabackup/ds_local.c b/extra/mariabackup/ds_local.c index dc13ed7595e..3e2b1e0129b 100644 --- a/extra/mariabackup/ds_local.c +++ b/extra/mariabackup/ds_local.c @@ -53,9 +53,9 @@ local_init(const char *root) && my_errno != EEXIST && my_errno != EISDIR) { char errbuf[MYSYS_STRERROR_SIZE]; + my_strerror(errbuf, sizeof(errbuf),my_errno); my_error(EE_CANT_MKDIR, MYF(ME_BELL | ME_WAITTANG), - root, my_errno, my_strerror(errbuf, sizeof(errbuf), - my_errno)); + root, my_errno,errbuf, my_errno); return NULL; } @@ -85,9 +85,9 @@ local_open(ds_ctxt_t *ctxt, const char *path, dirname_part(dirpath, fullpath, &dirpath_len); if (my_mkdir(dirpath, 0777, MYF(0)) < 0 && my_errno != EEXIST) { char errbuf[MYSYS_STRERROR_SIZE]; + my_strerror(errbuf, sizeof(errbuf), my_errno); my_error(EE_CANT_MKDIR, MYF(ME_BELL | ME_WAITTANG), - dirpath, my_errno, my_strerror(errbuf, sizeof(errbuf), - my_errno)); + dirpath, my_errno, errbuf); return NULL; } diff --git a/extra/mariabackup/ds_stdout.c b/extra/mariabackup/ds_stdout.c index 616bcbd831e..91a514ddf64 100644 --- a/extra/mariabackup/ds_stdout.c +++ b/extra/mariabackup/ds_stdout.c @@ -79,7 +79,7 @@ stdout_open(ds_ctxt_t *ctxt __attribute__((unused)), setmode(fileno(stdout), _O_BINARY); #endif - stdout_file->fd = fileno(stdout); + stdout_file->fd = my_fileno(stdout); file->path = (char *) stdout_file + sizeof(ds_stdout_file_t); memcpy(file->path, fullpath, pathlen); diff --git a/extra/mariabackup/ds_tmpfile.c b/extra/mariabackup/ds_tmpfile.c index 915191dcdae..b039d83ba03 100644 --- a/extra/mariabackup/ds_tmpfile.c +++ b/extra/mariabackup/ds_tmpfile.c @@ -53,7 +53,6 @@ datasink_t datasink_tmpfile = { &tmpfile_deinit }; -MY_TMPDIR mysql_tmpdir_list; static ds_ctxt_t * tmpfile_init(const char *root) @@ -90,7 +89,7 @@ tmpfile_open(ds_ctxt_t *ctxt, const char *path, /* Create a temporary file in tmpdir. The file will be automatically removed on close. Code copied from mysql_tmpfile(). */ - fd = create_temp_file(tmp_path, my_tmpdir(&mysql_tmpdir_list), + fd = create_temp_file(tmp_path,xtrabackup_tmpdir, "xbtemp", #ifdef __WIN__ O_BINARY | O_TRUNC | O_SEQUENTIAL | diff --git a/extra/mariabackup/encryption_plugin.cc b/extra/mariabackup/encryption_plugin.cc new file mode 100644 index 00000000000..9f2782d89a1 --- /dev/null +++ b/extra/mariabackup/encryption_plugin.cc @@ -0,0 +1,157 @@ +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include + + +extern struct st_maria_plugin *mysql_optional_plugins[]; +extern struct st_maria_plugin *mysql_mandatory_plugins[]; +static void encryption_plugin_init(int argc, char **argv); + +extern char *xb_plugin_load; +extern char *xb_plugin_dir; + +const int PLUGIN_MAX_ARGS = 1024; +vector backup_plugins_args; + +const char *QUERY_PLUGIN = +"SELECT plugin_name, plugin_library, @@plugin_dir" +" FROM information_schema.plugins WHERE plugin_type='ENCRYPTION'" +" AND plugin_status='ACTIVE'"; + +string encryption_plugin_config; + +static void add_to_plugin_load_list(const char *plugin_def) +{ + opt_plugin_load_list_ptr->push_back(new i_string(plugin_def)); +} + +static char XTRABACKUP_EXE[] = "xtrabackup"; + +void encryption_plugin_backup_init(MYSQL *mysql) +{ + MYSQL_RES *result; + MYSQL_ROW row; + ostringstream oss; + char *argv[PLUGIN_MAX_ARGS]; + int argc; + + result = xb_mysql_query(mysql, QUERY_PLUGIN, true, true); + row = mysql_fetch_row(result); + if (!row) + { + mysql_free_result(result); + return; + } + + char *name= row[0]; + char *library= row[1]; + char *dir= row[2]; + +#ifdef _WIN32 + for (char *p = dir; *p; p++) + if (*p == '\\') *p = '/'; +#endif + + string plugin_load(name); + if (library) + plugin_load += string("=") + library; + + oss << "plugin_load=" << plugin_load << endl; + + /* Required to load the plugin later.*/ + add_to_plugin_load_list(plugin_load.c_str()); + strncpy(opt_plugin_dir, dir, FN_REFLEN); + + oss << "plugin_dir=" << '"' << dir << '"' << endl; + + + /* Read plugin variables. */ + char query[1024]; + snprintf(query, 1024, "SHOW variables like '%s_%%'", name); + mysql_free_result(result); + + result = xb_mysql_query(mysql, query, true, true); + while ((row = mysql_fetch_row(result))) + { + string arg("--"); + arg += row[0]; + arg += "="; + arg += row[1]; + backup_plugins_args.push_back(arg); + oss << row[0] << "=" << row[1] << endl; + } + + mysql_free_result(result); + + /* Check whether to encrypt logs. */ + result = xb_mysql_query(mysql, "select @@innodb_encrypt_log", true, true); + row = mysql_fetch_row(result); + srv_encrypt_log = (row != 0 && row[0][0] == '1'); + oss << "innodb_encrypt_log=" << row[0] << endl; + + mysql_free_result(result); + + encryption_plugin_config = oss.str(); + + argc = 0; + argv[argc++] = XTRABACKUP_EXE; + for(size_t i = 0; i < backup_plugins_args.size(); i++) + { + argv[argc++] = (char *)backup_plugins_args[i].c_str(); + if (argc == PLUGIN_MAX_ARGS - 2) + break; + } + argv[argc] = 0; + + encryption_plugin_init(argc, argv); +} + +const char *encryption_plugin_get_config() +{ + return encryption_plugin_config.c_str(); +} + +extern int finalize_encryption_plugin(st_plugin_int *plugin); + + +void encryption_plugin_prepare_init(int argc, char **argv) +{ + + if (!xb_plugin_load) + { + /* This prevents crashes e.g in --stats with wrong my.cnf*/ + finalize_encryption_plugin(0); + return; + } + + add_to_plugin_load_list(xb_plugin_load); + + if (xb_plugin_dir) + strncpy(opt_plugin_dir, xb_plugin_dir, FN_REFLEN); + + char **new_argv = new char *[argc + 1]; + new_argv[0] = XTRABACKUP_EXE; + memcpy(&new_argv[1], argv, argc*sizeof(char *)); + + encryption_plugin_init(argc+1, new_argv); + + delete[] new_argv; +} + +static void encryption_plugin_init(int argc, char **argv) +{ + /* Patch optional and mandatory plugins, we only need to load the one in xb_plugin_load. */ + mysql_optional_plugins[0] = mysql_mandatory_plugins[0] = 0; + msg("Loading encryption plugin\n"); + for (int i= 1; i < argc; i++) + msg("\t Encryption plugin parameter : '%s'\n", argv[i]); + plugin_init(&argc, argv, PLUGIN_INIT_SKIP_PLUGIN_TABLE); +} + diff --git a/extra/mariabackup/encryption_plugin.h b/extra/mariabackup/encryption_plugin.h new file mode 100644 index 00000000000..16d74790254 --- /dev/null +++ b/extra/mariabackup/encryption_plugin.h @@ -0,0 +1,7 @@ +#include +#include +extern void encryption_plugin_backup_init(MYSQL *mysql); +extern const char* encryption_plugin_get_config(); +extern void encryption_plugin_prepare_init(int argc, char **argv); + +//extern void encryption_plugin_init(int argc, char **argv); diff --git a/extra/mariabackup/fil_cur.cc b/extra/mariabackup/fil_cur.cc index 22ebdb90215..820d8e10c29 100644 --- a/extra/mariabackup/fil_cur.cc +++ b/extra/mariabackup/fil_cur.cc @@ -33,6 +33,7 @@ Foundation, Inc., 51 Franklin Street, Fifth Floor, Boston, MA 02110-1301, USA #include "common.h" #include "read_filt.h" #include "xtrabackup.h" +#include "xb0xb.h" /* Size of read buffer in pages (640 pages = 10M for 16K sized pages) */ #define XB_FIL_CUR_PAGES 640 @@ -167,7 +168,7 @@ xb_fil_cur_open( os_file_create_simple_no_error_handling(0, node->name, OS_FILE_OPEN, OS_FILE_READ_ONLY, - &success); + &success,0); if (!success) { /* The following call prints an error message */ os_file_get_last_error(TRUE); @@ -200,7 +201,7 @@ xb_fil_cur_open( cursor->node = node; cursor->file = node->handle; - if (my_fstat(cursor->file, &cursor->statinfo, MYF(MY_WME))) { + if (stat(cursor->abs_path, &cursor->statinfo)) { msg("[%02u] xtrabackup: error: cannot stat %s\n", thread_n, cursor->abs_path); @@ -253,7 +254,7 @@ xb_fil_cur_open( cursor->buf_page_no = 0; cursor->thread_n = thread_n; - cursor->space_size = cursor->statinfo.st_size / page_size; + cursor->space_size = (ulint)(cursor->statinfo.st_size / page_size); cursor->read_filter = read_filter; cursor->read_filter->init(&cursor->read_filter_ctxt, cursor, @@ -327,26 +328,32 @@ read_retry: cursor->buf_read = 0; cursor->buf_npages = 0; cursor->buf_offset = offset; - cursor->buf_page_no = (ulint) (offset >> cursor->page_size_shift); + cursor->buf_page_no = (ulint)(offset >> cursor->page_size_shift); success = os_file_read(cursor->file, cursor->buf, offset, - to_read); + (ulint)to_read); if (!success) { return(XB_FIL_CUR_ERROR); } + fil_system_enter(); + fil_space_t *space = fil_space_get_by_id(cursor->space_id); + fil_system_exit(); + /* check pages for corruption and re-read if necessary. i.e. in case of partially written pages */ for (page = cursor->buf, i = 0; i < npages; page += cursor->page_size, i++) { + ib_int64_t page_no = cursor->buf_page_no + i; - if (buf_page_is_corrupted(TRUE, page, cursor->zip_size)) { + bool checksum_ok = fil_space_verify_crypt_checksum(page, cursor->zip_size,space, (ulint)page_no); - ulint page_no = cursor->buf_page_no + i; + if (!checksum_ok && + buf_page_is_corrupted(true, page, cursor->zip_size,space)) { if (cursor->is_system && - page_no >= FSP_EXTENT_SIZE && - page_no < FSP_EXTENT_SIZE * 3) { + page_no >= (ib_int64_t)FSP_EXTENT_SIZE && + page_no < (ib_int64_t) FSP_EXTENT_SIZE * 3) { /* skip doublewrite buffer pages */ xb_a(cursor->page_size == UNIV_PAGE_SIZE); msg("[%02u] xtrabackup: " diff --git a/extra/mariabackup/fil_cur.h b/extra/mariabackup/fil_cur.h index 2057765dab5..88239efd2bb 100644 --- a/extra/mariabackup/fil_cur.h +++ b/extra/mariabackup/fil_cur.h @@ -49,10 +49,10 @@ struct xb_fil_cur_t { /*!< read filter context */ byte* orig_buf; /*!< read buffer */ byte* buf; /*!< aligned pointer for orig_buf */ - ulint buf_size; /*!< buffer size in bytes */ - ulint buf_read; /*!< number of read bytes in buffer + size_t buf_size; /*!< buffer size in bytes */ + size_t buf_read; /*!< number of read bytes in buffer after the last cursor read */ - ulint buf_npages; /*!< number of pages in buffer after the + size_t buf_npages; /*!< number of pages in buffer after the last cursor read */ ib_int64_t buf_offset; /*!< file offset of the first page in buffer */ diff --git a/extra/mariabackup/innobackupex.cc b/extra/mariabackup/innobackupex.cc index ec697bb56ff..59fb8fb5565 100644 --- a/extra/mariabackup/innobackupex.cc +++ b/extra/mariabackup/innobackupex.cc @@ -52,14 +52,12 @@ Place, Suite 330, Boston, MA 02111-1307 USA #include #include #include -#include #include #include #include #include "common.h" #include "innobackupex.h" #include "xtrabackup.h" -#include "xtrabackup_version.h" #include "xbstream.h" #include "fil_cur.h" #include "write_filt.h" @@ -667,22 +665,6 @@ static struct my_option ibx_long_options[] = (uchar*) &ibx_xtrabackup_parallel, (uchar*) &ibx_xtrabackup_parallel, 0, GET_INT, REQUIRED_ARG, 1, 1, INT_MAX, 0, 0, 0}, - {"rebuild-indexes", OPT_REBUILD_INDEXES, - "This option only has effect when used together with the --apply-log " - "option and is passed directly to xtrabackup. When used, makes " - "xtrabackup rebuild all secondary indexes after applying the log. " - "This option is normally used to prepare compact backups. See the " - "XtraBackup manual for more information.", - (uchar*) &ibx_xtrabackup_rebuild_indexes, - (uchar*) &ibx_xtrabackup_rebuild_indexes, - 0, GET_BOOL, NO_ARG, 0, 0, 0, 0, 0, 0}, - - {"rebuild-threads", OPT_REBUILD_THREADS, - "Use this number of threads to rebuild indexes in a compact backup. " - "Only has effect with --prepare and --rebuild-indexes.", - (uchar*) &ibx_xtrabackup_rebuild_threads, - (uchar*) &ibx_xtrabackup_rebuild_threads, - 0, GET_UINT, REQUIRED_ARG, 1, 1, UINT_MAX, 0, 0, 0}, {"stream", OPT_STREAM, "This option specifies the format in which to " "do the streamed backup. The option accepts a string argument. The " @@ -845,9 +827,9 @@ ibx_get_one_option(int optid, exit(0); break; case 'v': - msg("innobackupex version %s %s (%s) (revision id: %s)\n", - XTRABACKUP_VERSION, - SYSTEM_TYPE, MACHINE_TYPE, XTRABACKUP_REVISION); + msg("innobackupex version %s %s (%s)\n", + MYSQL_SERVER_VERSION, + SYSTEM_TYPE, MACHINE_TYPE); exit(0); break; case OPT_HISTORY: @@ -1039,7 +1021,6 @@ ibx_init() /* setup xtrabackup options */ xb_close_files = ibx_xb_close_files; - xtrabackup_compact = ibx_xtrabackup_compact; xtrabackup_compress_alg = ibx_xtrabackup_compress_alg; xtrabackup_compress_threads = ibx_xtrabackup_compress_threads; xtrabackup_compress_chunk_size = ibx_xtrabackup_compress_chunk_size; @@ -1057,8 +1038,6 @@ ibx_init() xtrabackup_log_copy_interval = ibx_xtrabackup_log_copy_interval; xtrabackup_incremental = ibx_xtrabackup_incremental; xtrabackup_parallel = ibx_xtrabackup_parallel; - xtrabackup_rebuild_indexes = ibx_xtrabackup_rebuild_indexes; - xtrabackup_rebuild_threads = ibx_xtrabackup_rebuild_threads; xtrabackup_stream_str = ibx_xtrabackup_stream_str; xtrabackup_tables_file = ibx_xtrabackup_tables_file; xtrabackup_throttle = ibx_xtrabackup_throttle; diff --git a/extra/mariabackup/read_filt.cc b/extra/mariabackup/read_filt.cc index 8ebc735e99e..05e6b7c86c7 100644 --- a/extra/mariabackup/read_filt.cc +++ b/extra/mariabackup/read_filt.cc @@ -75,7 +75,7 @@ rf_pass_through_get_next_batch( *read_batch_start = ctxt->offset; *read_batch_len = ctxt->data_file_size - ctxt->offset; - if (*read_batch_len > ctxt->buffer_capacity) { + if (*read_batch_len > (ib_int64_t)ctxt->buffer_capacity) { *read_batch_len = ctxt->buffer_capacity; } @@ -128,7 +128,7 @@ rf_bitmap_get_next_batch( { ulint start_page_id; - start_page_id = ctxt->offset / ctxt->page_size; + start_page_id = (ulint)(ctxt->offset / ctxt->page_size); xb_a (ctxt->offset % ctxt->page_size == 0); @@ -170,7 +170,7 @@ rf_bitmap_get_next_batch( buffer capacity. The subsequent invocations will continue returning the current block in buffer-sized pieces until ctxt->filter_batch_end is reached, trigerring the next bitmap query. */ - if (*read_batch_len > ctxt->buffer_capacity) { + if (*read_batch_len > (ib_int64_t)ctxt->buffer_capacity) { *read_batch_len = ctxt->buffer_capacity; } diff --git a/extra/mariabackup/read_filt.h b/extra/mariabackup/read_filt.h index 73fef06a288..d16f4e1093d 100644 --- a/extra/mariabackup/read_filt.h +++ b/extra/mariabackup/read_filt.h @@ -33,14 +33,14 @@ struct xb_fil_cur_t; struct xb_read_filt_ctxt_t { ib_int64_t offset; /*!< current file offset */ ib_int64_t data_file_size; /*!< data file size */ - ib_int64_t buffer_capacity;/*!< read buffer capacity */ - ulint space_id; /*!< space id */ + size_t buffer_capacity;/*!< read buffer capacity */ + ib_int64_t space_id; /*!< space id */ /* The following fields used only in bitmap filter */ /* Move these to union if any other filters are added in future */ xb_page_bitmap_range *bitmap_range; /*!< changed page bitmap range iterator for space_id */ - ulint page_size; /*!< page size */ - ulint filter_batch_end;/*!< the ending page id of the + size_t page_size; /*!< page size */ + ulint filter_batch_end;/*!< the ending page id of the current changed page block in the bitmap */ }; diff --git a/extra/mariabackup/version_check.pl b/extra/mariabackup/version_check.pl deleted file mode 100644 index 865e2eacb4a..00000000000 --- a/extra/mariabackup/version_check.pl +++ /dev/null @@ -1,1373 +0,0 @@ -use warnings FATAL => 'all'; -use strict; -use English qw(-no_match_vars); -use POSIX "strftime"; - -my @required_perl_version = (5, 0, 5); -my $required_perl_version_old_style = 5.005; - -# check existence of DBD::mysql module -eval { - require DBD::mysql; -}; -my $dbd_mysql_installed = $EVAL_ERROR ? 0 : 1; - -my $now; -my %mysql; -my $prefix = "version_check"; - - -# ########################################################################### -# HTTPMicro package -# This package is a copy without comments from the original. The original -# with comments and its test file can be found in the Bazaar repository at, -# lib/HTTPMicro.pm -# t/lib/HTTPMicro.t -# See https://launchpad.net/percona-toolkit for more information. -# ########################################################################### -{ - -package HTTPMicro; -BEGIN { - $HTTPMicro::VERSION = '0.001'; -} -use strict; -use warnings; - -use Carp (); - - -my @attributes; -BEGIN { - @attributes = qw(agent timeout); - no strict 'refs'; - for my $accessor ( @attributes ) { - *{$accessor} = sub { - @_ > 1 ? $_[0]->{$accessor} = $_[1] : $_[0]->{$accessor}; - }; - } -} - -sub new { - my($class, %args) = @_; - (my $agent = $class) =~ s{::}{-}g; - my $self = { - agent => $agent . "/" . ($class->VERSION || 0), - timeout => 60, - }; - for my $key ( @attributes ) { - $self->{$key} = $args{$key} if exists $args{$key} - } - return bless $self, $class; -} - -my %DefaultPort = ( - http => 80, - https => 443, -); - -sub request { - my ($self, $method, $url, $args) = @_; - @_ == 3 || (@_ == 4 && ref $args eq 'HASH') - or Carp::croak(q/Usage: $http->request(METHOD, URL, [HASHREF])/); - $args ||= {}; # we keep some state in this during _request - - my $response; - for ( 0 .. 1 ) { - $response = eval { $self->_request($method, $url, $args) }; - last unless $@ && $method eq 'GET' - && $@ =~ m{^(?:Socket closed|Unexpected end)}; - } - - if (my $e = "$@") { - $response = { - success => q{}, - status => 599, - reason => 'Internal Exception', - content => $e, - headers => { - 'content-type' => 'text/plain', - 'content-length' => length $e, - } - }; - } - return $response; -} - -sub _request { - my ($self, $method, $url, $args) = @_; - - my ($scheme, $host, $port, $path_query) = $self->_split_url($url); - - my $request = { - method => $method, - scheme => $scheme, - host_port => ($port == $DefaultPort{$scheme} ? $host : "$host:$port"), - uri => $path_query, - headers => {}, - }; - - my $handle = HTTPMicro::Handle->new(timeout => $self->{timeout}); - - $handle->connect($scheme, $host, $port); - - $self->_prepare_headers_and_cb($request, $args); - $handle->write_request_header(@{$request}{qw/method uri headers/}); - $handle->write_content_body($request) if $request->{content}; - - my $response; - do { $response = $handle->read_response_header } - until (substr($response->{status},0,1) ne '1'); - - if (!($method eq 'HEAD' || $response->{status} =~ /^[23]04/)) { - $response->{content} = ''; - $handle->read_content_body(sub { $_[1]->{content} .= $_[0] }, $response); - } - - $handle->close; - $response->{success} = substr($response->{status},0,1) eq '2'; - return $response; -} - -sub _prepare_headers_and_cb { - my ($self, $request, $args) = @_; - - for ($args->{headers}) { - next unless defined; - while (my ($k, $v) = each %$_) { - $request->{headers}{lc $k} = $v; - } - } - $request->{headers}{'host'} = $request->{host_port}; - $request->{headers}{'connection'} = "close"; - $request->{headers}{'user-agent'} ||= $self->{agent}; - - if (defined $args->{content}) { - $request->{headers}{'content-type'} ||= "application/octet-stream"; - utf8::downgrade($args->{content}, 1) - or Carp::croak(q/Wide character in request message body/); - $request->{headers}{'content-length'} = length $args->{content}; - $request->{content} = $args->{content}; - } - return; -} - -sub _split_url { - my $url = pop; - - my ($scheme, $authority, $path_query) = $url =~ m<\A([^:/?#]+)://([^/?#]*)([^#]*)> - or Carp::croak(qq/Cannot parse URL: '$url'/); - - $scheme = lc $scheme; - $path_query = "/$path_query" unless $path_query =~ m<\A/>; - - my $host = (length($authority)) ? lc $authority : 'localhost'; - $host =~ s/\A[^@]*@//; # userinfo - my $port = do { - $host =~ s/:([0-9]*)\z// && length $1 - ? $1 - : $DefaultPort{$scheme} - }; - - return ($scheme, $host, $port, $path_query); -} - -package - HTTPMicro::Handle; # hide from PAUSE/indexers -use strict; -use warnings; - -use Carp qw[croak]; -use Errno qw[EINTR EPIPE]; -use IO::Socket qw[SOCK_STREAM]; - -sub BUFSIZE () { 32768 } - -my $Printable = sub { - local $_ = shift; - s/\r/\\r/g; - s/\n/\\n/g; - s/\t/\\t/g; - s/([^\x20-\x7E])/sprintf('\\x%.2X', ord($1))/ge; - $_; -}; - -sub new { - my ($class, %args) = @_; - return bless { - rbuf => '', - timeout => 60, - max_line_size => 16384, - %args - }, $class; -} - -my $ssl_verify_args = { - check_cn => "when_only", - wildcards_in_alt => "anywhere", - wildcards_in_cn => "anywhere" -}; - -sub connect { - @_ == 4 || croak(q/Usage: $handle->connect(scheme, host, port)/); - my ($self, $scheme, $host, $port) = @_; - - if ( $scheme eq 'https' ) { - eval "require IO::Socket::SSL" - unless exists $INC{'IO/Socket/SSL.pm'}; - croak(qq/IO::Socket::SSL must be installed for https support\n/) - unless $INC{'IO/Socket/SSL.pm'}; - } - elsif ( $scheme ne 'http' ) { - croak(qq/Unsupported URL scheme '$scheme'\n/); - } - - $self->{fh} = 'IO::Socket::INET'->new( - PeerHost => $host, - PeerPort => $port, - Proto => 'tcp', - Type => SOCK_STREAM, - Timeout => $self->{timeout} - ) or croak(qq/Could not connect to '$host:$port': $@/); - - binmode($self->{fh}) - or croak(qq/Could not binmode() socket: '$!'/); - - if ( $scheme eq 'https') { - IO::Socket::SSL->start_SSL($self->{fh}); - ref($self->{fh}) eq 'IO::Socket::SSL' - or die(qq/SSL connection failed for $host\n/); - if ( $self->{fh}->can("verify_hostname") ) { - $self->{fh}->verify_hostname( $host, $ssl_verify_args ) - or die(qq/SSL certificate not valid for $host\n/); - } - else { - my $fh = $self->{fh}; - _verify_hostname_of_cert($host, _peer_certificate($fh), $ssl_verify_args) - or die(qq/SSL certificate not valid for $host\n/); - } - } - - $self->{host} = $host; - $self->{port} = $port; - - return $self; -} - -sub close { - @_ == 1 || croak(q/Usage: $handle->close()/); - my ($self) = @_; - CORE::close($self->{fh}) - or croak(qq/Could not close socket: '$!'/); -} - -sub write { - @_ == 2 || croak(q/Usage: $handle->write(buf)/); - my ($self, $buf) = @_; - - my $len = length $buf; - my $off = 0; - - local $SIG{PIPE} = 'IGNORE'; - - while () { - $self->can_write - or croak(q/Timed out while waiting for socket to become ready for writing/); - my $r = syswrite($self->{fh}, $buf, $len, $off); - if (defined $r) { - $len -= $r; - $off += $r; - last unless $len > 0; - } - elsif ($! == EPIPE) { - croak(qq/Socket closed by remote server: $!/); - } - elsif ($! != EINTR) { - croak(qq/Could not write to socket: '$!'/); - } - } - return $off; -} - -sub read { - @_ == 2 || @_ == 3 || croak(q/Usage: $handle->read(len)/); - my ($self, $len) = @_; - - my $buf = ''; - my $got = length $self->{rbuf}; - - if ($got) { - my $take = ($got < $len) ? $got : $len; - $buf = substr($self->{rbuf}, 0, $take, ''); - $len -= $take; - } - - while ($len > 0) { - $self->can_read - or croak(q/Timed out while waiting for socket to become ready for reading/); - my $r = sysread($self->{fh}, $buf, $len, length $buf); - if (defined $r) { - last unless $r; - $len -= $r; - } - elsif ($! != EINTR) { - croak(qq/Could not read from socket: '$!'/); - } - } - if ($len) { - croak(q/Unexpected end of stream/); - } - return $buf; -} - -sub readline { - @_ == 1 || croak(q/Usage: $handle->readline()/); - my ($self) = @_; - - while () { - if ($self->{rbuf} =~ s/\A ([^\x0D\x0A]* \x0D?\x0A)//x) { - return $1; - } - $self->can_read - or croak(q/Timed out while waiting for socket to become ready for reading/); - my $r = sysread($self->{fh}, $self->{rbuf}, BUFSIZE, length $self->{rbuf}); - if (defined $r) { - last unless $r; - } - elsif ($! != EINTR) { - croak(qq/Could not read from socket: '$!'/); - } - } - croak(q/Unexpected end of stream while looking for line/); -} - -sub read_header_lines { - @_ == 1 || @_ == 2 || croak(q/Usage: $handle->read_header_lines([headers])/); - my ($self, $headers) = @_; - $headers ||= {}; - my $lines = 0; - my $val; - - while () { - my $line = $self->readline; - - if ($line =~ /\A ([^\x00-\x1F\x7F:]+) : [\x09\x20]* ([^\x0D\x0A]*)/x) { - my ($field_name) = lc $1; - $val = \($headers->{$field_name} = $2); - } - elsif ($line =~ /\A [\x09\x20]+ ([^\x0D\x0A]*)/x) { - $val - or croak(q/Unexpected header continuation line/); - next unless length $1; - $$val .= ' ' if length $$val; - $$val .= $1; - } - elsif ($line =~ /\A \x0D?\x0A \z/x) { - last; - } - else { - croak(q/Malformed header line: / . $Printable->($line)); - } - } - return $headers; -} - -sub write_header_lines { - (@_ == 2 && ref $_[1] eq 'HASH') || croak(q/Usage: $handle->write_header_lines(headers)/); - my($self, $headers) = @_; - - my $buf = ''; - while (my ($k, $v) = each %$headers) { - my $field_name = lc $k; - $field_name =~ /\A [\x21\x23-\x27\x2A\x2B\x2D\x2E\x30-\x39\x41-\x5A\x5E-\x7A\x7C\x7E]+ \z/x - or croak(q/Invalid HTTP header field name: / . $Printable->($field_name)); - $field_name =~ s/\b(\w)/\u$1/g; - $buf .= "$field_name: $v\x0D\x0A"; - } - $buf .= "\x0D\x0A"; - return $self->write($buf); -} - -sub read_content_body { - @_ == 3 || @_ == 4 || croak(q/Usage: $handle->read_content_body(callback, response, [read_length])/); - my ($self, $cb, $response, $len) = @_; - $len ||= $response->{headers}{'content-length'}; - - croak("No content-length in the returned response, and this " - . "UA doesn't implement chunking") unless defined $len; - - while ($len > 0) { - my $read = ($len > BUFSIZE) ? BUFSIZE : $len; - $cb->($self->read($read), $response); - $len -= $read; - } - - return; -} - -sub write_content_body { - @_ == 2 || croak(q/Usage: $handle->write_content_body(request)/); - my ($self, $request) = @_; - my ($len, $content_length) = (0, $request->{headers}{'content-length'}); - - $len += $self->write($request->{content}); - - $len == $content_length - or croak(qq/Content-Length missmatch (got: $len expected: $content_length)/); - - return $len; -} - -sub read_response_header { - @_ == 1 || croak(q/Usage: $handle->read_response_header()/); - my ($self) = @_; - - my $line = $self->readline; - - $line =~ /\A (HTTP\/(0*\d+\.0*\d+)) [\x09\x20]+ ([0-9]{3}) [\x09\x20]+ ([^\x0D\x0A]*) \x0D?\x0A/x - or croak(q/Malformed Status-Line: / . $Printable->($line)); - - my ($protocol, $version, $status, $reason) = ($1, $2, $3, $4); - - return { - status => $status, - reason => $reason, - headers => $self->read_header_lines, - protocol => $protocol, - }; -} - -sub write_request_header { - @_ == 4 || croak(q/Usage: $handle->write_request_header(method, request_uri, headers)/); - my ($self, $method, $request_uri, $headers) = @_; - - return $self->write("$method $request_uri HTTP/1.1\x0D\x0A") - + $self->write_header_lines($headers); -} - -sub _do_timeout { - my ($self, $type, $timeout) = @_; - $timeout = $self->{timeout} - unless defined $timeout && $timeout >= 0; - - my $fd = fileno $self->{fh}; - defined $fd && $fd >= 0 - or croak(q/select(2): 'Bad file descriptor'/); - - my $initial = time; - my $pending = $timeout; - my $nfound; - - vec(my $fdset = '', $fd, 1) = 1; - - while () { - $nfound = ($type eq 'read') - ? select($fdset, undef, undef, $pending) - : select(undef, $fdset, undef, $pending) ; - if ($nfound == -1) { - $! == EINTR - or croak(qq/select(2): '$!'/); - redo if !$timeout || ($pending = $timeout - (time - $initial)) > 0; - $nfound = 0; - } - last; - } - $! = 0; - return $nfound; -} - -sub can_read { - @_ == 1 || @_ == 2 || croak(q/Usage: $handle->can_read([timeout])/); - my $self = shift; - return $self->_do_timeout('read', @_) -} - -sub can_write { - @_ == 1 || @_ == 2 || croak(q/Usage: $handle->can_write([timeout])/); - my $self = shift; - return $self->_do_timeout('write', @_) -} - -my $prog = <<'EOP'; -BEGIN { - if ( defined &IO::Socket::SSL::CAN_IPV6 ) { - *CAN_IPV6 = \*IO::Socket::SSL::CAN_IPV6; - } - else { - constant->import( CAN_IPV6 => '' ); - } - my %const = ( - NID_CommonName => 13, - GEN_DNS => 2, - GEN_IPADD => 7, - ); - while ( my ($name,$value) = each %const ) { - no strict 'refs'; - *{$name} = UNIVERSAL::can( 'Net::SSLeay', $name ) || sub { $value }; - } -} -{ - my %dispatcher = ( - issuer => sub { Net::SSLeay::X509_NAME_oneline( Net::SSLeay::X509_get_issuer_name( shift )) }, - subject => sub { Net::SSLeay::X509_NAME_oneline( Net::SSLeay::X509_get_subject_name( shift )) }, - ); - if ( $Net::SSLeay::VERSION >= 1.30 ) { - $dispatcher{commonName} = sub { - my $cn = Net::SSLeay::X509_NAME_get_text_by_NID( - Net::SSLeay::X509_get_subject_name( shift ), NID_CommonName); - $cn =~s{\0$}{}; # work around Bug in Net::SSLeay <1.33 - $cn; - } - } else { - $dispatcher{commonName} = sub { - croak "you need at least Net::SSLeay version 1.30 for getting commonName" - } - } - - if ( $Net::SSLeay::VERSION >= 1.33 ) { - $dispatcher{subjectAltNames} = sub { Net::SSLeay::X509_get_subjectAltNames( shift ) }; - } else { - $dispatcher{subjectAltNames} = sub { - return; - }; - } - - $dispatcher{authority} = $dispatcher{issuer}; - $dispatcher{owner} = $dispatcher{subject}; - $dispatcher{cn} = $dispatcher{commonName}; - - sub _peer_certificate { - my ($self, $field) = @_; - my $ssl = $self->_get_ssl_object or return; - - my $cert = ${*$self}{_SSL_certificate} - ||= Net::SSLeay::get_peer_certificate($ssl) - or return $self->error("Could not retrieve peer certificate"); - - if ($field) { - my $sub = $dispatcher{$field} or croak - "invalid argument for peer_certificate, valid are: ".join( " ",keys %dispatcher ). - "\nMaybe you need to upgrade your Net::SSLeay"; - return $sub->($cert); - } else { - return $cert - } - } - - - my %scheme = ( - ldap => { - wildcards_in_cn => 0, - wildcards_in_alt => 'leftmost', - check_cn => 'always', - }, - http => { - wildcards_in_cn => 'anywhere', - wildcards_in_alt => 'anywhere', - check_cn => 'when_only', - }, - smtp => { - wildcards_in_cn => 0, - wildcards_in_alt => 0, - check_cn => 'always' - }, - none => {}, # do not check - ); - - $scheme{www} = $scheme{http}; # alias - $scheme{xmpp} = $scheme{http}; # rfc 3920 - $scheme{pop3} = $scheme{ldap}; # rfc 2595 - $scheme{imap} = $scheme{ldap}; # rfc 2595 - $scheme{acap} = $scheme{ldap}; # rfc 2595 - $scheme{nntp} = $scheme{ldap}; # rfc 4642 - $scheme{ftp} = $scheme{http}; # rfc 4217 - - - sub _verify_hostname_of_cert { - my $identity = shift; - my $cert = shift; - my $scheme = shift || 'none'; - if ( ! ref($scheme) ) { - $scheme = $scheme{$scheme} or croak "scheme $scheme not defined"; - } - - return 1 if ! %$scheme; # 'none' - - my $commonName = $dispatcher{cn}->($cert); - my @altNames = $dispatcher{subjectAltNames}->($cert); - - if ( my $sub = $scheme->{callback} ) { - return $sub->($identity,$commonName,@altNames); - } - - - my $ipn; - if ( CAN_IPV6 and $identity =~m{:} ) { - $ipn = IO::Socket::SSL::inet_pton(IO::Socket::SSL::AF_INET6,$identity) - or croak "'$identity' is not IPv6, but neither IPv4 nor hostname"; - } elsif ( $identity =~m{^\d+\.\d+\.\d+\.\d+$} ) { - $ipn = IO::Socket::SSL::inet_aton( $identity ) or croak "'$identity' is not IPv4, but neither IPv6 nor hostname"; - } else { - if ( $identity =~m{[^a-zA-Z0-9_.\-]} ) { - $identity =~m{\0} and croak("name '$identity' has \\0 byte"); - $identity = IO::Socket::SSL::idn_to_ascii($identity) or - croak "Warning: Given name '$identity' could not be converted to IDNA!"; - } - } - - my $check_name = sub { - my ($name,$identity,$wtyp) = @_; - $wtyp ||= ''; - my $pattern; - if ( $wtyp eq 'anywhere' and $name =~m{^([a-zA-Z0-9_\-]*)\*(.+)} ) { - $pattern = qr{^\Q$1\E[a-zA-Z0-9_\-]*\Q$2\E$}i; - } elsif ( $wtyp eq 'leftmost' and $name =~m{^\*(\..+)$} ) { - $pattern = qr{^[a-zA-Z0-9_\-]*\Q$1\E$}i; - } else { - $pattern = qr{^\Q$name\E$}i; - } - return $identity =~ $pattern; - }; - - my $alt_dnsNames = 0; - while (@altNames) { - my ($type, $name) = splice (@altNames, 0, 2); - if ( $ipn and $type == GEN_IPADD ) { - return 1 if $ipn eq $name; - - } elsif ( ! $ipn and $type == GEN_DNS ) { - $name =~s/\s+$//; $name =~s/^\s+//; - $alt_dnsNames++; - $check_name->($name,$identity,$scheme->{wildcards_in_alt}) - and return 1; - } - } - - if ( ! $ipn and ( - $scheme->{check_cn} eq 'always' or - $scheme->{check_cn} eq 'when_only' and !$alt_dnsNames)) { - $check_name->($commonName,$identity,$scheme->{wildcards_in_cn}) - and return 1; - } - - return 0; # no match - } -} -EOP - -eval { require IO::Socket::SSL }; -if ( $INC{"IO/Socket/SSL.pm"} ) { - eval $prog; - die $@ if $@; -} - -1; -} -# ########################################################################### -# End HTTPMicro package -# ########################################################################### - -# ########################################################################### -# VersionCheck package -# This package is a copy without comments from the original. The original -# with comments and its test file can be found in the Bazaar repository at, -# lib/VersionCheck.pm -# t/lib/VersionCheck.t -# See https://launchpad.net/percona-toolkit for more information. -# ########################################################################### -{ -package VersionCheck; - - -use strict; -use warnings FATAL => 'all'; -use English qw(-no_match_vars); - -use constant PTDEBUG => $ENV{PTDEBUG} || 0; - -use Data::Dumper; -local $Data::Dumper::Indent = 1; -local $Data::Dumper::Sortkeys = 1; -local $Data::Dumper::Quotekeys = 0; - -use Digest::MD5 qw(md5_hex); -use Sys::Hostname qw(hostname); -use File::Basename qw(); -use File::Spec; -use FindBin qw(); - -eval { - require Percona::Toolkit; - require HTTPMicro; -}; - -{ - my $file = 'percona-version-check'; - my $home = $ENV{HOME} || $ENV{HOMEPATH} || $ENV{USERPROFILE} || '.'; - my @vc_dirs = ( - '/etc/percona', - '/etc/percona-toolkit', - '/tmp', - "$home", - ); - - if ($ENV{PTDEBUG_VERSION_CHECK_HOME}) { - @vc_dirs = ( $ENV{PTDEBUG_VERSION_CHECK_HOME} ); - } - - sub version_check_file { - foreach my $dir ( @vc_dirs ) { - if ( -d $dir && -w $dir ) { - PTDEBUG && _d('Version check file', $file, 'in', $dir); - return $dir . '/' . $file; - } - } - PTDEBUG && _d('Version check file', $file, 'in', $ENV{PWD}); - return $file; # in the CWD - } -} - -sub version_check_time_limit { - return 60 * 60 * 24; # one day -} - - -sub version_check { - my (%args) = @_; - - my $instances = $args{instances} || []; - my $instances_to_check; - - PTDEBUG && _d('FindBin::Bin:', $FindBin::Bin); - if ( !$args{force} ) { - if ( $FindBin::Bin - && (-d "$FindBin::Bin/../.bzr" || -d "$FindBin::Bin/../../.bzr") ) { - PTDEBUG && _d("$FindBin::Bin/../.bzr disables --version-check"); - return; - } - } - - eval { - foreach my $instance ( @$instances ) { - my ($name, $id) = get_instance_id($instance); - $instance->{name} = $name; - $instance->{id} = $id; - } - - push @$instances, { name => 'system', id => 0 }; - - $instances_to_check = get_instances_to_check( - instances => $instances, - vc_file => $args{vc_file}, # testing - now => $args{now}, # testing - ); - PTDEBUG && _d(scalar @$instances_to_check, 'instances to check'); - return unless @$instances_to_check; - - my $protocol = 'https'; - eval { require IO::Socket::SSL; }; - if ( $EVAL_ERROR ) { - PTDEBUG && _d($EVAL_ERROR); - PTDEBUG && _d("SSL not available, won't run version_check"); - return; - } - PTDEBUG && _d('Using', $protocol); - - my $advice = pingback( - instances => $instances_to_check, - protocol => $protocol, - url => $args{url} # testing - || $ENV{PERCONA_VERSION_CHECK_URL} # testing - || "$protocol://v.percona.com", - ); - if ( $advice ) { - PTDEBUG && _d('Advice:', Dumper($advice)); - if ( scalar @$advice > 1) { - print "\n# " . scalar @$advice . " software updates are " - . "available:\n"; - } - else { - print "\n# A software update is available:\n"; - } - print join("\n", map { "# * $_" } @$advice), "\n\n"; - } - }; - if ( $EVAL_ERROR ) { - PTDEBUG && _d('Version check failed:', $EVAL_ERROR); - } - - if ( @$instances_to_check ) { - eval { - update_check_times( - instances => $instances_to_check, - vc_file => $args{vc_file}, # testing - now => $args{now}, # testing - ); - }; - if ( $EVAL_ERROR ) { - PTDEBUG && _d('Error updating version check file:', $EVAL_ERROR); - } - } - - if ( $ENV{PTDEBUG_VERSION_CHECK} ) { - warn "Exiting because the PTDEBUG_VERSION_CHECK " - . "environment variable is defined.\n"; - exit 255; - } - - return; -} - -sub get_instances_to_check { - my (%args) = @_; - - my $instances = $args{instances}; - my $now = $args{now} || int(time); - my $vc_file = $args{vc_file} || version_check_file(); - - if ( !-f $vc_file ) { - PTDEBUG && _d('Version check file', $vc_file, 'does not exist;', - 'version checking all instances'); - return $instances; - } - - open my $fh, '<', $vc_file or die "Cannot open $vc_file: $OS_ERROR"; - chomp(my $file_contents = do { local $/ = undef; <$fh> }); - PTDEBUG && _d('Version check file', $vc_file, 'contents:', $file_contents); - close $fh; - my %last_check_time_for = $file_contents =~ /^([^,]+),(.+)$/mg; - - my $check_time_limit = version_check_time_limit(); - my @instances_to_check; - foreach my $instance ( @$instances ) { - my $last_check_time = $last_check_time_for{ $instance->{id} }; - PTDEBUG && _d('Intsance', $instance->{id}, 'last checked', - $last_check_time, 'now', $now, 'diff', $now - ($last_check_time || 0), - 'hours until next check', - sprintf '%.2f', - ($check_time_limit - ($now - ($last_check_time || 0))) / 3600); - if ( !defined $last_check_time - || ($now - $last_check_time) >= $check_time_limit ) { - PTDEBUG && _d('Time to check', Dumper($instance)); - push @instances_to_check, $instance; - } - } - - return \@instances_to_check; -} - -sub update_check_times { - my (%args) = @_; - - my $instances = $args{instances}; - my $now = $args{now} || int(time); - my $vc_file = $args{vc_file} || version_check_file(); - PTDEBUG && _d('Updating last check time:', $now); - - my %all_instances = map { - $_->{id} => { name => $_->{name}, ts => $now } - } @$instances; - - if ( -f $vc_file ) { - open my $fh, '<', $vc_file or die "Cannot read $vc_file: $OS_ERROR"; - my $contents = do { local $/ = undef; <$fh> }; - close $fh; - - foreach my $line ( split("\n", ($contents || '')) ) { - my ($id, $ts) = split(',', $line); - if ( !exists $all_instances{$id} ) { - $all_instances{$id} = { ts => $ts }; # original ts, not updated - } - } - } - - open my $fh, '>', $vc_file or die "Cannot write to $vc_file: $OS_ERROR"; - foreach my $id ( sort keys %all_instances ) { - PTDEBUG && _d('Updated:', $id, Dumper($all_instances{$id})); - print { $fh } $id . ',' . $all_instances{$id}->{ts} . "\n"; - } - close $fh; - - return; -} - -sub get_instance_id { - my ($instance) = @_; - - my $dbh = $instance->{dbh}; - my $dsn = $instance->{dsn}; - - my $sql = q{SELECT CONCAT(@@hostname, @@port)}; - PTDEBUG && _d($sql); - my ($name) = eval { $dbh->selectrow_array($sql) }; - if ( $EVAL_ERROR ) { - PTDEBUG && _d($EVAL_ERROR); - $sql = q{SELECT @@hostname}; - PTDEBUG && _d($sql); - ($name) = eval { $dbh->selectrow_array($sql) }; - if ( $EVAL_ERROR ) { - PTDEBUG && _d($EVAL_ERROR); - $name = ($dsn->{h} || 'localhost') . ($dsn->{P} || 3306); - } - else { - $sql = q{SHOW VARIABLES LIKE 'port'}; - PTDEBUG && _d($sql); - my (undef, $port) = eval { $dbh->selectrow_array($sql) }; - PTDEBUG && _d('port:', $port); - $name .= $port || ''; - } - } - my $id = md5_hex($name); - - PTDEBUG && _d('MySQL instance:', $id, $name, Dumper($dsn)); - - return $name, $id; -} - - -sub pingback { - my (%args) = @_; - my @required_args = qw(url instances); - foreach my $arg ( @required_args ) { - die "I need a $arg arugment" unless $args{$arg}; - } - my $url = $args{url}; - my $instances = $args{instances}; - - my $ua = $args{ua} || HTTPMicro->new( timeout => 3 ); - - my $response = $ua->request('GET', $url); - PTDEBUG && _d('Server response:', Dumper($response)); - die "No response from GET $url" - if !$response; - die("GET on $url returned HTTP status $response->{status}; expected 200\n", - ($response->{content} || '')) if $response->{status} != 200; - die("GET on $url did not return any programs to check") - if !$response->{content}; - - my $items = parse_server_response( - response => $response->{content} - ); - die "Failed to parse server requested programs: $response->{content}" - if !scalar keys %$items; - - my $versions = get_versions( - items => $items, - instances => $instances, - ); - die "Failed to get any program versions; should have at least gotten Perl" - if !scalar keys %$versions; - - my $client_content = encode_client_response( - items => $items, - versions => $versions, - general_id => md5_hex( hostname() ), - ); - - my $client_response = { - headers => { "X-Percona-Toolkit-Tool" => File::Basename::basename($0) }, - content => $client_content, - }; - PTDEBUG && _d('Client response:', Dumper($client_response)); - - $response = $ua->request('POST', $url, $client_response); - PTDEBUG && _d('Server suggestions:', Dumper($response)); - die "No response from POST $url $client_response" - if !$response; - die "POST $url returned HTTP status $response->{status}; expected 200" - if $response->{status} != 200; - - return unless $response->{content}; - - $items = parse_server_response( - response => $response->{content}, - split_vars => 0, - ); - die "Failed to parse server suggestions: $response->{content}" - if !scalar keys %$items; - my @suggestions = map { $_->{vars} } - sort { $a->{item} cmp $b->{item} } - values %$items; - - return \@suggestions; -} - -sub encode_client_response { - my (%args) = @_; - my @required_args = qw(items versions general_id); - foreach my $arg ( @required_args ) { - die "I need a $arg arugment" unless $args{$arg}; - } - my ($items, $versions, $general_id) = @args{@required_args}; - - my @lines; - foreach my $item ( sort keys %$items ) { - next unless exists $versions->{$item}; - if ( ref($versions->{$item}) eq 'HASH' ) { - my $mysql_versions = $versions->{$item}; - for my $id ( sort keys %$mysql_versions ) { - push @lines, join(';', $id, $item, $mysql_versions->{$id}); - } - } - else { - push @lines, join(';', $general_id, $item, $versions->{$item}); - } - } - - my $client_response = join("\n", @lines) . "\n"; - return $client_response; -} - -sub parse_server_response { - my (%args) = @_; - my @required_args = qw(response); - foreach my $arg ( @required_args ) { - die "I need a $arg arugment" unless $args{$arg}; - } - my ($response) = @args{@required_args}; - - my %items = map { - my ($item, $type, $vars) = split(";", $_); - if ( !defined $args{split_vars} || $args{split_vars} ) { - $vars = [ split(",", ($vars || '')) ]; - } - $item => { - item => $item, - type => $type, - vars => $vars, - }; - } split("\n", $response); - - PTDEBUG && _d('Items:', Dumper(\%items)); - - return \%items; -} - -my %sub_for_type = ( - os_version => \&get_os_version, - perl_version => \&get_perl_version, - perl_module_version => \&get_perl_module_version, - mysql_variable => \&get_mysql_variable, -); - -sub valid_item { - my ($item) = @_; - return unless $item; - if ( !exists $sub_for_type{ $item->{type} } ) { - PTDEBUG && _d('Invalid type:', $item->{type}); - return 0; - } - return 1; -} - -sub get_versions { - my (%args) = @_; - my @required_args = qw(items); - foreach my $arg ( @required_args ) { - die "I need a $arg arugment" unless $args{$arg}; - } - my ($items) = @args{@required_args}; - - my %versions; - foreach my $item ( values %$items ) { - next unless valid_item($item); - eval { - my $version = $sub_for_type{ $item->{type} }->( - item => $item, - instances => $args{instances}, - ); - if ( $version ) { - chomp $version unless ref($version); - $versions{$item->{item}} = $version; - } - }; - if ( $EVAL_ERROR ) { - PTDEBUG && _d('Error getting version for', Dumper($item), $EVAL_ERROR); - } - } - - return \%versions; -} - - -sub get_os_version { - if ( $OSNAME eq 'MSWin32' ) { - require Win32; - return Win32::GetOSDisplayName(); - } - - chomp(my $platform = `uname -s`); - PTDEBUG && _d('platform:', $platform); - return $OSNAME unless $platform; - - chomp(my $lsb_release - = `which lsb_release 2>/dev/null | awk '{print \$1}'` || ''); - PTDEBUG && _d('lsb_release:', $lsb_release); - - my $release = ""; - - if ( $platform eq 'Linux' ) { - if ( -f "/etc/fedora-release" ) { - $release = `cat /etc/fedora-release`; - } - elsif ( -f "/etc/redhat-release" ) { - $release = `cat /etc/redhat-release`; - } - elsif ( -f "/etc/system-release" ) { - $release = `cat /etc/system-release`; - } - elsif ( $lsb_release ) { - $release = `$lsb_release -ds`; - } - elsif ( -f "/etc/lsb-release" ) { - $release = `grep DISTRIB_DESCRIPTION /etc/lsb-release`; - $release =~ s/^\w+="([^"]+)".+/$1/; - } - elsif ( -f "/etc/debian_version" ) { - chomp(my $rel = `cat /etc/debian_version`); - $release = "Debian $rel"; - if ( -f "/etc/apt/sources.list" ) { - chomp(my $code_name = `awk '/^deb/ {print \$3}' /etc/apt/sources.list | awk -F/ '{print \$1}'| awk 'BEGIN {FS="|"} {print \$1}' | sort | uniq -c | sort -rn | head -n1 | awk '{print \$2}'`); - $release .= " ($code_name)" if $code_name; - } - } - elsif ( -f "/etc/os-release" ) { # openSUSE - chomp($release = `grep PRETTY_NAME /etc/os-release`); - $release =~ s/^PRETTY_NAME="(.+)"$/$1/; - } - elsif ( `ls /etc/*release 2>/dev/null` ) { - if ( `grep DISTRIB_DESCRIPTION /etc/*release 2>/dev/null` ) { - $release = `grep DISTRIB_DESCRIPTION /etc/*release | head -n1`; - } - else { - $release = `cat /etc/*release | head -n1`; - } - } - } - elsif ( $platform =~ m/(?:BSD|^Darwin)$/ ) { - my $rel = `uname -r`; - $release = "$platform $rel"; - } - elsif ( $platform eq "SunOS" ) { - my $rel = `head -n1 /etc/release` || `uname -r`; - $release = "$platform $rel"; - } - - if ( !$release ) { - PTDEBUG && _d('Failed to get the release, using platform'); - $release = $platform; - } - chomp($release); - - $release =~ s/^"|"$//g; - - PTDEBUG && _d('OS version =', $release); - return $release; -} - -sub get_perl_version { - my (%args) = @_; - my $item = $args{item}; - return unless $item; - - my $version = sprintf '%vd', $PERL_VERSION; - PTDEBUG && _d('Perl version', $version); - return $version; -} - -sub get_perl_module_version { - my (%args) = @_; - my $item = $args{item}; - return unless $item; - - my $var = '$' . $item->{item} . '::VERSION'; - my $version = eval "use $item->{item}; $var;"; - PTDEBUG && _d('Perl version for', $var, '=', $version); - return $version; -} - -sub get_mysql_variable { - return get_from_mysql( - show => 'VARIABLES', - @_, - ); -} - -sub get_from_mysql { - my (%args) = @_; - my $show = $args{show}; - my $item = $args{item}; - my $instances = $args{instances}; - return unless $show && $item; - - if ( !$instances || !@$instances ) { - PTDEBUG && _d('Cannot check', $item, - 'because there are no MySQL instances'); - return; - } - - if ($item->{item} eq 'MySQL' && $item->{type} eq 'mysql_variable') { - $item->{vars} = ['version_comment', 'version']; - } - - my @versions; - my %version_for; - foreach my $instance ( @$instances ) { - next unless $instance->{id}; # special system instance has id=0 - my $dbh = $instance->{dbh}; - local $dbh->{FetchHashKeyName} = 'NAME_lc'; - my $sql = qq/SHOW $show/; - PTDEBUG && _d($sql); - my $rows = $dbh->selectall_hashref($sql, 'variable_name'); - - my @versions; - foreach my $var ( @{$item->{vars}} ) { - $var = lc($var); - my $version = $rows->{$var}->{value}; - PTDEBUG && _d('MySQL version for', $item->{item}, '=', $version, - 'on', $instance->{name}); - push @versions, $version; - } - $version_for{ $instance->{id} } = join(' ', @versions); - } - - return \%version_for; -} - -sub _d { - my ($package, undef, $line) = caller 0; - @_ = map { (my $temp = $_) =~ s/\n/\n# /g; $temp; } - map { defined $_ ? $_ : 'undef' } - @_; - print STDERR "# $package:$line $PID ", join(' ', @_), "\n"; -} - -1; -} -# ########################################################################### -# End VersionCheck package -# ########################################################################### - -# -# parse_connection_options() subroutine parses connection-related command line -# options -# -sub parse_connection_options { - my $con = shift; - - $con->{dsn} = 'dbi:mysql:'; - - # this option has to be first - if ($ENV{option_defaults_file}) { - $con->{dsn} .= ";mysql_read_default_file=$ENV{option_defaults_file}"; - } - - if ($ENV{option_defaults_extra_file}) { - $con->{dsn} .= ";mysql_read_default_file=$ENV{option_defaults_extra_file}"; - } - - $con->{dsn} .= ";mysql_read_default_group=xtrabackup"; - - if ($ENV{option_mysql_password}) { - $con->{dsn_password} = "$ENV{option_mysql_password}"; - } - if ($ENV{option_mysql_user}) { - $con->{dsn_user} = "$ENV{option_mysql_user}"; - } - if ($ENV{option_mysql_host}) { - $con->{dsn} .= ";host=$ENV{option_mysql_host}"; - } - if ($ENV{option_mysql_port}) { - $con->{dsn} .= ";port=$ENV{option_mysql_port}"; - } - if ($ENV{option_mysql_socket}) { - $con->{dsn} .= ";mysql_socket=$ENV{option_mysql_socket}"; - } -} - -# -# mysql_connect subroutine connects to MySQL server -# -sub mysql_connect { - my %con; - my %args = ( - # Defaults - abort_on_error => 1, - @_ - ); - - $con{abort_on_error} = $args{abort_on_error}; - - parse_connection_options(\%con); - - $now = current_time(); - print STDERR "$now $prefix Connecting to MySQL server with DSN '$con{dsn}'" . - (defined($con{dsn_user}) ? " as '$con{dsn_user}' " : "") . - " (using password: "; - if (defined($con{dsn_password})) { - print STDERR "YES).\n"; - } else { - print STDERR "NO).\n"; - } - - eval { - $con{dbh}=DBI->connect($con{dsn}, $con{dsn_user}, - $con{dsn_password}, { RaiseError => 1 }); - }; - - if ($EVAL_ERROR) { - $con{connect_error}=$EVAL_ERROR; - } else { - $now = current_time(); - print STDERR "$now $prefix Connected to MySQL server\n"; - } - - if ($args{abort_on_error}) { - if (!$dbd_mysql_installed) { - die "Failed to connect to MySQL server as " . - "DBD::mysql module is not installed"; - } else { - if (!$con{dbh}) { - die "Failed to connect to MySQL server: " . - $con{connect_error}; - } - } - } - - if ($con{dbh}) { - $con{dbh}->do("SET SESSION wait_timeout=2147483"); - } - - return %con; -} - -# -# return current local time as string in form "070816 12:23:15" -# -sub current_time { - return strftime("%y%m%d %H:%M:%S", localtime()); -} - - -%mysql = mysql_connect(abort_on_error => 1); - -$now = current_time(); -print STDERR - "$now $prefix Executing a version check against the server...\n"; - -# Redirect STDOUT to STDERR, as VersionCheck prints alerts to STDOUT -select STDERR; - -VersionCheck::version_check( - force => 1, - instances => [ { - dbh => $mysql{dbh}, - dsn => $mysql{dsn} - } - ] - ); -# Restore STDOUT as the default filehandle -select STDOUT; - -$now = current_time(); -print STDERR "$now $prefix Done.\n"; diff --git a/extra/mariabackup/write_filt.cc b/extra/mariabackup/write_filt.cc index 129302d7fa0..cf7753bf380 100644 --- a/extra/mariabackup/write_filt.cc +++ b/extra/mariabackup/write_filt.cc @@ -75,7 +75,7 @@ wf_incremental_init(xb_write_filt_ctxt_t *ctxt, char *dst_name, ctxt->cursor = cursor; /* allocate buffer for incremental backup (4096 pages) */ - buf_size = (UNIV_PAGE_SIZE_MAX / 4 + 1) * UNIV_PAGE_SIZE_MAX; + buf_size = (cursor->page_size / 4 + 1) * cursor->page_size; cp->delta_buf_base = static_cast(ut_malloc(buf_size)); memset(cp->delta_buf_base, 0, buf_size); cp->delta_buf = static_cast diff --git a/extra/mariabackup/write_filt.h b/extra/mariabackup/write_filt.h index 20213b6f523..bcab263f1dd 100644 --- a/extra/mariabackup/write_filt.h +++ b/extra/mariabackup/write_filt.h @@ -27,7 +27,6 @@ Foundation, Inc., 51 Franklin Street, Fifth Floor, Boston, MA 02110-1301, USA #include "fil_cur.h" #include "datasink.h" -#include "compact.h" /* Incremental page filter context */ typedef struct { @@ -41,7 +40,6 @@ typedef struct { xb_fil_cur_t *cursor; union { xb_wf_incremental_ctxt_t wf_incremental_ctxt; - xb_wf_compact_ctxt_t wf_compact_ctxt; } u; } xb_write_filt_ctxt_t; @@ -56,6 +54,5 @@ typedef struct { extern xb_write_filt_t wf_write_through; extern xb_write_filt_t wf_incremental; -extern xb_write_filt_t wf_compact; #endif /* XB_WRITE_FILT_H */ diff --git a/extra/mariabackup/wsrep.cc b/extra/mariabackup/wsrep.cc index 420ada75f36..be11e058255 100644 --- a/extra/mariabackup/wsrep.cc +++ b/extra/mariabackup/wsrep.cc @@ -46,7 +46,7 @@ permission notice: #include #include "common.h" - +#ifdef WITH_WSREP #define WSREP_XID_PREFIX "WSREPXid" #define WSREP_XID_PREFIX_LEN MYSQL_XID_PREFIX_LEN #define WSREP_XID_UUID_OFFSET 8 @@ -61,11 +61,11 @@ permission notice: /* Galera UUID type - for all unique IDs */ typedef struct wsrep_uuid { - uint8_t data[16]; + unsigned char data[16]; } wsrep_uuid_t; /* sequence number of a writeset, etc. */ -typedef int64_t wsrep_seqno_t; +typedef long long wsrep_seqno_t; /* Undefined UUID */ static const wsrep_uuid_t WSREP_UUID_UNDEFINED = {{0,}}; @@ -217,3 +217,4 @@ xb_write_galera_info(bool incremental_prepare) fclose(fp); } +#endif diff --git a/extra/mariabackup/xb0xb.h b/extra/mariabackup/xb0xb.h new file mode 100644 index 00000000000..659ab8ea5d0 --- /dev/null +++ b/extra/mariabackup/xb0xb.h @@ -0,0 +1,78 @@ +/****************************************************** +Copyright (c) 2012 Percona LLC and/or its affiliates. + +Declarations of XtraBackup functions called by InnoDB code. + +This program is free software; you can redistribute it and/or modify +it under the terms of the GNU General Public License as published by +the Free Software Foundation; version 2 of the License. + +This program is distributed in the hope that it will be useful, +but WITHOUT ANY WARRANTY; without even the implied warranty of +MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the +GNU General Public License for more details. + +You should have received a copy of the GNU General Public License +along with this program; if not, write to the Free Software +Foundation, Inc., 51 Franklin Street, Fifth Floor, Boston, MA 02110-1301, USA + +*******************************************************/ + +#ifndef xb0xb_h +#define xb0xb_h + + +extern void os_io_init_simple(void); +extern os_file_t files[1000]; +extern const char *innodb_checksum_algorithm_names[]; +extern TYPELIB innodb_checksum_algorithm_typelib; +extern dberr_t open_or_create_data_files( + ibool* create_new_db, +#ifdef UNIV_LOG_ARCHIVE + lsn_t* min_arch_log_no, + lsn_t* max_arch_log_no, +#endif + lsn_t* min_flushed_lsn, + lsn_t* max_flushed_lsn, + ulint* sum_of_new_sizes) + ; +int +fil_file_readdir_next_file( +/*=======================*/ +dberr_t* err, /*!< out: this is set to DB_ERROR if an error + was encountered, otherwise not changed */ + const char* dirname,/*!< in: directory name or path */ + os_file_dir_t dir, /*!< in: directory stream */ + os_file_stat_t* info) /*!< in/out: buffer where the + info is returned */; +buf_block_t* btr_node_ptr_get_child( + const rec_t* node_ptr,/*!< in: node pointer */ + dict_index_t* index, /*!< in: index */ + const ulint* offsets,/*!< in: array returned by rec_get_offsets() */ + mtr_t* mtr) /*!< in: mtr */; + +buf_block_t* +btr_root_block_get( +/*===============*/ +const dict_index_t* index, /*!< in: index tree */ +ulint mode, /*!< in: either RW_S_LATCH + or RW_X_LATCH */ + mtr_t* mtr) /*!< in: mtr */; +fil_space_t* +fil_space_get_by_name(const char *); +ibool +recv_check_cp_is_consistent(const byte* buf); +void +innodb_log_checksum_func_update( +/*============================*/ +ulint algorithm) /*!< in: algorithm */; +dberr_t recv_find_max_checkpoint(log_group_t** max_group, ulint* max_field); +dberr_t +srv_undo_tablespaces_init( +/*======================*/ +ibool create_new_db, +ibool backup_mode, +const ulint n_conf_tablespaces, +ulint* n_opened); + +#endif diff --git a/extra/mariabackup/xb_regex.h b/extra/mariabackup/xb_regex.h index 94c5e2a5fa7..2e07e434e27 100644 --- a/extra/mariabackup/xb_regex.h +++ b/extra/mariabackup/xb_regex.h @@ -22,50 +22,27 @@ my_regex is used on Windows and native calls are used on POSIX platforms. */ #ifndef XB_REGEX_H #define XB_REGEX_H -#ifdef _WIN32 +#ifdef HAVE_SYSTEM_REGEX +#include +#else +#include +#endif -#include +typedef regex_t* xb_regex_t; -typedef my_regex_t xb_regex_t; -typedef my_regmatch_t xb_regmatch_t; - -#define xb_regex_init() my_regex_init(&my_charset_latin1) +#define xb_regex_init() #define xb_regexec(preg,string,nmatch,pmatch,eflags) \ - my_regexec(preg, string, nmatch, pmatch, eflags) - -#define xb_regerror(errcode,preg,errbuf,errbuf_size) \ - my_regerror(errcode, preg, errbuf, errbuf_size) - -#define xb_regcomp(preg,regex,cflags) \ - my_regcomp(preg, regex, cflags, &my_charset_latin1) - -#define xb_regfree(preg) my_regfree(preg) - -#define xb_regex_end() my_regex_end() - -#else /* ! _WIN32 */ - -#include - -typedef regex_t xb_regex_t; -typedef regmatch_t xb_regmatch_t; - -#define xb_regex_init() do { } while(0) - -#define xb_regexec(preg,string,nmatch,pmatch,eflags) \ regexec(preg, string, nmatch, pmatch, eflags) -#define xb_regerror(errcode,preg,errbuf,errbuf_size) \ +#define xb_regerror(errcode,preg,errbuf,errbuf_size) \ regerror(errcode, preg, errbuf, errbuf_size) -#define xb_regcomp(preg,regex,cflags) \ +#define xb_regcomp(preg,regex,cflags) \ regcomp(preg, regex, cflags) #define xb_regfree(preg) regfree(preg) -#define xb_regex_end() do { } while (0) - -#endif /* _WIN32 */ +#define xb_regex_end() #endif /* XB_REGEX_H */ diff --git a/extra/mariabackup/xbcrypt_common.c b/extra/mariabackup/xbcrypt_common.c index fe34fcb3bb0..52fa2ce3589 100644 --- a/extra/mariabackup/xbcrypt_common.c +++ b/extra/mariabackup/xbcrypt_common.c @@ -27,6 +27,7 @@ Foundation, Inc., 51 Franklin Street, Fifth Floor, Boston, MA 02110-1301, USA # pragma GCC diagnostic ignored "-Wdeprecated-declarations" #endif +#ifdef HAVE_GRYPT #include #if GCC_VERSION >= 4002 @@ -58,3 +59,4 @@ xb_crypt_create_iv(void* ivbuf, size_t ivlen) { gcry_create_nonce(ivbuf, ivlen); } +#endif \ No newline at end of file diff --git a/extra/mariabackup/xbcrypt_write.c b/extra/mariabackup/xbcrypt_write.c index 5cbeb67f227..6cc30852215 100644 --- a/extra/mariabackup/xbcrypt_write.c +++ b/extra/mariabackup/xbcrypt_write.c @@ -73,7 +73,7 @@ int xb_crypt_write_chunk(xb_wcrypt_t *crypt, const void *buf, size_t olen, int8store(ptr, (ulonglong)elen); /* encrypted (actual) size */ ptr += 8; - checksum = crc32(0, buf, elen); + checksum = crc32(0, buf, (uint)elen); int4store(ptr, checksum); /* checksum */ ptr += 4; diff --git a/extra/mariabackup/xbstream.c b/extra/mariabackup/xbstream.c index ba3412a359b..9990b00ea4b 100644 --- a/extra/mariabackup/xbstream.c +++ b/extra/mariabackup/xbstream.c @@ -208,15 +208,15 @@ int stream_one_file(File file, xb_wstream_file_t *xbfile) { uchar *buf; - size_t bytes; - size_t offset; + ssize_t bytes; + my_off_t offset; posix_fadvise(file, 0, 0, POSIX_FADV_SEQUENTIAL); offset = my_tell(file, MYF(MY_WME)); buf = (uchar*)(my_malloc(XBSTREAM_BUFFER_SIZE, MYF(MY_FAE))); - while ((bytes = my_read(file, buf, XBSTREAM_BUFFER_SIZE, + while ((bytes = (ssize_t)my_read(file, buf, XBSTREAM_BUFFER_SIZE, MYF(MY_WME))) > 0) { if (xb_stream_write_data(xbfile, buf, bytes)) { msg("%s: xb_stream_write_data() failed.\n", @@ -232,7 +232,7 @@ stream_one_file(File file, xb_wstream_file_t *xbfile) my_free(buf); - if (bytes == (size_t) -1) { + if (bytes < 0) { return 1; } diff --git a/extra/mariabackup/xbstream_read.c b/extra/mariabackup/xbstream_read.c index 0ffcabd9270..f8d88926ec2 100644 --- a/extra/mariabackup/xbstream_read.c +++ b/extra/mariabackup/xbstream_read.c @@ -48,13 +48,13 @@ xb_stream_read_new(void) stream->buffer = my_malloc(INIT_BUFFER_LEN, MYF(MY_FAE)); stream->buflen = INIT_BUFFER_LEN; - stream->fd = fileno(stdin); - stream->offset = 0; - #ifdef __WIN__ - setmode(stream->fd, _O_BINARY); + setmode(fileno(stdin), _O_BINARY); #endif + stream->fd = my_fileno(stdin); + stream->offset = 0; + return stream; } diff --git a/extra/mariabackup/xbstream_write.c b/extra/mariabackup/xbstream_write.c index 7b042eea49c..a11811dc375 100644 --- a/extra/mariabackup/xbstream_write.c +++ b/extra/mariabackup/xbstream_write.c @@ -34,7 +34,7 @@ struct xb_wstream_struct { struct xb_wstream_file_struct { xb_wstream_t *stream; char *path; - ulong path_len; + size_t path_len; char chunk[XB_STREAM_MIN_CHUNK_SIZE]; char *chunk_ptr; size_t chunk_free; @@ -54,7 +54,7 @@ xb_stream_default_write_callback(xb_wstream_file_t *file __attribute__((unused)) void *userdata __attribute__((unused)), const void *buf, size_t len) { - if (my_write(fileno(stdout), buf, len, MYF(MY_WME | MY_NABP))) + if (my_write(my_fileno(stdout), buf, len, MYF(MY_WME | MY_NABP))) return -1; return len; } @@ -77,7 +77,7 @@ xb_stream_write_open(xb_wstream_t *stream, const char *path, xb_stream_write_callback *onwrite) { xb_wstream_file_t *file; - ulong path_len; + size_t path_len; path_len = strlen(path); @@ -90,7 +90,19 @@ xb_stream_write_open(xb_wstream_t *stream, const char *path, path_len + 1, MYF(MY_FAE)); file->path = (char *) (file + 1); +#ifdef _WIN32 + /* Normalize path on Windows, so we can restore elsewhere.*/ + { + int i; + for (i = 0; ; i++) { + file->path[i] = (path[i] == '\\') ? '/' : path[i]; + if (!path[i]) + break; + } + } +#else memcpy(file->path, path, path_len + 1); +#endif file->path_len = path_len; file->stream = stream; @@ -208,7 +220,7 @@ xb_stream_write_chunk(xb_wstream_file_t *file, const void *buf, size_t len) int8store(ptr, file->offset); /* Payload offset */ ptr += 8; - checksum = crc32(0, buf, len); /* checksum */ + checksum = crc32(0, buf, (uint)len); /* checksum */ int4store(ptr, checksum); ptr += 4; diff --git a/extra/mariabackup/xtrabackup.cc b/extra/mariabackup/xtrabackup.cc index d24c915bb48..3ca6efaafcf 100644 --- a/extra/mariabackup/xtrabackup.cc +++ b/extra/mariabackup/xtrabackup.cc @@ -53,9 +53,9 @@ Place, Suite 330, Boston, MA 02111-1307 USA #ifdef __linux__ # include +#include #endif -#include #include #include @@ -74,7 +74,6 @@ Place, Suite 330, Boston, MA 02111-1307 USA #define G_PTR uchar* #include "common.h" -#include "xtrabackup_version.h" #include "datasink.h" #include "xb_regex.h" @@ -91,6 +90,10 @@ Place, Suite 330, Boston, MA 02111-1307 USA #include "backup_mysql.h" #include "backup_copy.h" #include "backup_mysql.h" +#include "xb0xb.h" +#include "encryption_plugin.h" +#include +#include /* TODO: replace with appropriate macros used in InnoDB 5.6 */ #define PAGE_ZIP_MIN_SIZE_SHIFT 10 @@ -143,15 +146,16 @@ lsn_t xtrabackup_archived_to_lsn = 0; /* for --archived-to-lsn */ char *xtrabackup_tables = NULL; +char *xtrabackup_tmpdir; + /* List of regular expressions for filtering */ typedef struct xb_regex_list_node_struct xb_regex_list_node_t; struct xb_regex_list_node_struct { UT_LIST_NODE_T(xb_regex_list_node_t) regex_list; - xb_regex_t regex; + regex_t regex; }; static UT_LIST_BASE_NODE_T(xb_regex_list_node_t) regex_list; - -static xb_regmatch_t tables_regmatch[1]; +static regmatch_t tables_regmatch[1]; char *xtrabackup_tables_file = NULL; static hash_table_t* tables_hash = NULL; @@ -210,6 +214,7 @@ ulint xtrabackup_rebuild_threads = 1; /* sleep interval beetween log copy iterations in log copying thread in milliseconds (default is 1 second) */ ulint xtrabackup_log_copy_interval = 1000; +static ulong max_buf_pool_modified_pct; /* Ignored option (--log) for MySQL option compatibility */ char* log_ignored_opt = NULL; @@ -294,8 +299,6 @@ it every INNOBASE_WAKE_INTERVAL'th step. */ #define INNOBASE_WAKE_INTERVAL 32 ulong innobase_active_counter = 0; -ibool srv_compact_backup = FALSE; -ibool srv_rebuild_indexes = FALSE; static char *xtrabackup_debug_sync = NULL; @@ -309,13 +312,15 @@ lsn_t min_flushed_lsn= 0; lsn_t max_flushed_lsn= 0; /* The size of archived log file */ -size_t xtrabackup_arch_file_size = 0ULL; +ib_int64_t xtrabackup_arch_file_size = 0ULL; /* The minimal LSN of found archived log files */ lsn_t xtrabackup_arch_first_file_lsn = 0ULL; /* The maximum LSN of found archived log files */ lsn_t xtrabackup_arch_last_file_lsn = 0ULL; ulong xb_open_files_limit= 0; +char *xb_plugin_dir; +char *xb_plugin_load; my_bool xb_close_files= FALSE; /* Datasinks */ @@ -574,6 +579,9 @@ enum options_xtrabackup OPT_XTRA_INCREMENTAL_FORCE_SCAN, OPT_DEFAULTS_GROUP, OPT_OPEN_FILES_LIMIT, + OPT_PLUGIN_DIR, + OPT_PLUGIN_LOAD, + OPT_INNODB_ENCRYPT_LOG, OPT_CLOSE_FILES, OPT_CORE_FILE, @@ -682,7 +690,13 @@ struct my_option xb_client_options[] = 0, GET_BOOL, NO_ARG, 0, 0, 0, 0, 0, 0}, {"stream", OPT_XTRA_STREAM, "Stream all backup files to the standard output " - "in the specified format. Currently the only supported format is 'tar'.", + "in the specified format." +#ifdef HAVE_LIBARCHIVE + "Supported formats are 'tar' and 'xbstream'." +#else + "Supported format is 'xbstream'." +#endif + , (G_PTR*) &xtrabackup_stream_str, (G_PTR*) &xtrabackup_stream_str, 0, GET_STR, REQUIRED_ARG, 0, 0, 0, 0, 0, 0}, @@ -1001,12 +1015,6 @@ struct my_option xb_client_options[] = #include "sslopt-longopts.h" -#if !defined(HAVE_YASSL) - {"server-public-key-path", OPT_SERVER_PUBLIC_KEY, - "File path to the server public RSA key in PEM format.", - &opt_server_public_key, &opt_server_public_key, 0, - GET_STR, REQUIRED_ARG, 0, 0, 0, 0, 0, 0}, -#endif { 0, 0, 0, 0, 0, 0, GET_NO_ARG, NO_ARG, 0, 0, 0, 0, 0, 0} }; @@ -1098,11 +1106,7 @@ Disable with --skip-innodb-doublewrite.", (G_PTR*) &innobase_use_doublewrite, (G_PTR*) &innobase_file_per_table, (G_PTR*) &innobase_file_per_table, 0, GET_BOOL, NO_ARG, FALSE, 0, 0, 0, 0, 0}, - {"innodb_flush_log_at_trx_commit", OPT_INNODB_FLUSH_LOG_AT_TRX_COMMIT, - "Set to 0 (write and flush once per second), 1 (write and flush at each commit) or 2 (write at commit, flush once per second).", - (G_PTR*) &srv_flush_log_at_trx_commit, - (G_PTR*) &srv_flush_log_at_trx_commit, - 0, GET_ULONG, OPT_ARG, 1, 0, 2, 0, 0, 0}, + {"innodb_flush_method", OPT_INNODB_FLUSH_METHOD, "With which method to flush data.", (G_PTR*) &innobase_unix_file_flush_method, (G_PTR*) &innobase_unix_file_flush_method, 0, GET_STR, REQUIRED_ARG, 0, 0, 0, @@ -1203,6 +1207,18 @@ Disable with --skip-innodb-doublewrite.", (G_PTR*) &innobase_use_doublewrite, (G_PTR*) &defaults_group, (G_PTR*) &defaults_group, 0, GET_STR, REQUIRED_ARG, 0, 0, 0, 0, 0, 0}, + {"plugin-dir", OPT_PLUGIN_DIR, "Server plugin directory", + &xb_plugin_dir, &xb_plugin_dir, + 0, GET_STR, REQUIRED_ARG, 0, 0, 0, 0, 0, 0 }, + + { "plugin-load", OPT_PLUGIN_LOAD, "encrypton plugin to load", + &xb_plugin_load, &xb_plugin_load, + 0, GET_STR, REQUIRED_ARG, 0, 0, 0, 0, 0, 0 }, + + { "innodb-encrypt-log", OPT_INNODB_ENCRYPT_LOG, "encrypton plugin to load", + &srv_encrypt_log, &srv_encrypt_log, + 0, GET_BOOL, REQUIRED_ARG, 0, 0, 0, 0, 0, 0 }, + {"open_files_limit", OPT_OPEN_FILES_LIMIT, "the maximum number of file " "descriptors to reserve with setrlimit().", (G_PTR*) &xb_open_files_limit, (G_PTR*) &xb_open_files_limit, 0, GET_ULONG, @@ -1276,9 +1292,8 @@ static const char *xb_server_default_groups[]= static void print_version(void) { - msg("%s version %s based on MySQL server %s %s (%s) (revision id: %s)\n", - my_progname, XTRABACKUP_VERSION, MYSQL_SERVER_VERSION, SYSTEM_TYPE, - MACHINE_TYPE, XTRABACKUP_REVISION); + msg("%s based on MariaDB server %s %s (%s) \n", + my_progname, MYSQL_SERVER_VERSION, SYSTEM_TYPE, MACHINE_TYPE); } static void usage(void) @@ -1515,10 +1530,10 @@ xb_init_log_block_size(void) { srv_log_block_size = 0; if (innobase_log_block_size != 512) { - uint n_shift = get_bit_shift(innobase_log_block_size);; + uint n_shift = (uint)get_bit_shift(innobase_log_block_size);; if (n_shift > 0) { - srv_log_block_size = (1 << n_shift); + srv_log_block_size = (ulint)(1LL << n_shift); msg("InnoDB: The log block size is set to %lu.\n", srv_log_block_size); } @@ -1541,13 +1556,13 @@ innodb_init_param(void) static char current_dir[3]; /* Set if using current lib */ my_bool ret; char *default_path; - + srv_is_being_started = TRUE; /* === some variables from mysqld === */ memset((G_PTR) &mysql_tmpdir_list, 0, sizeof(mysql_tmpdir_list)); if (init_tmpdir(&mysql_tmpdir_list, opt_mysql_tmpdir)) exit(EXIT_FAILURE); - + xtrabackup_tmpdir = my_tmpdir(&mysql_tmpdir_list); /* dummy for initialize all_charsets[] */ get_charset_name(0); @@ -1555,7 +1570,7 @@ innodb_init_param(void) srv_page_size_shift = 0; if (innobase_page_size != (1LL << 14)) { - int n_shift = get_bit_shift((ulint) innobase_page_size); + int n_shift = (int)get_bit_shift((ulint) innobase_page_size); if (n_shift >= 12 && n_shift <= UNIV_PAGE_SIZE_SHIFT_MAX) { srv_page_size_shift = n_shift; @@ -1576,8 +1591,6 @@ innodb_init_param(void) goto error; } - srv_fast_checksum = (ibool) innobase_fast_checksum; - /* Check that values don't overflow on 32-bit systems. */ if (sizeof(ulint) == 4) { if (xtrabackup_use_memory > UINT_MAX32) { @@ -1738,10 +1751,15 @@ mem_free_and_error: } btr_search_enabled = (char) innobase_adaptive_hash_index; + btr_search_index_num = 1; os_use_large_pages = (ibool) innobase_use_large_pages; os_large_page_size = (ulint) innobase_large_page_size; + if (!innobase_log_arch_dir) { + static char default_dir[3] = "./"; + srv_arch_dir = default_dir; + } row_rollback_on_timeout = (ibool) innobase_rollback_on_timeout; srv_file_per_table = (my_bool) innobase_file_per_table; @@ -1848,7 +1866,7 @@ static my_bool innodb_init(void) { int err; - + srv_is_being_started = TRUE; err = innobase_start_or_create_for_mysql(); if (err != DB_SUCCESS) { @@ -1947,12 +1965,6 @@ xtrabackup_read_metadata(char *filename) } /* Optional fields */ - if (fscanf(fp, "compact = %d\n", &t) == 1) { - xtrabackup_compact = (t == 1); - } else { - xtrabackup_compact = 0; - } - if (fscanf(fp, "recover_binlog_info = %d\n", &t) == 1) { recover_binlog_info = (t == 1); } @@ -1981,7 +1993,7 @@ xtrabackup_print_metadata(char *buf, size_t buf_len) metadata_from_lsn, metadata_to_lsn, metadata_last_lsn, - MY_TEST(xtrabackup_compact == TRUE), + MY_TEST(false), MY_TEST(opt_binlog_info == BINLOG_INFO_LOCKLESS)); } @@ -2166,7 +2178,7 @@ check_if_table_matches_filters(const char *name) /* Check against regular expressions list */ for (node = UT_LIST_GET_FIRST(regex_list); node; node = UT_LIST_GET_NEXT(regex_list, node)) { - regres = xb_regexec(&node->regex, name, 1, + regres = regexec(&node->regex, name, 1, tables_regmatch, 0); if (regres != REG_NOMATCH) { @@ -2213,7 +2225,7 @@ check_if_skip_table( dbname = NULL; tbname = name; - while ((ptr = strchr(tbname, SRV_PATH_SEPARATOR)) != NULL) { + while ((ptr = strchr(tbname, '/')) != NULL) { dbname = tbname; tbname = ptr + 1; } @@ -2287,10 +2299,10 @@ xb_get_zip_size(os_file_t file) ibool success; ulint space; - buf = static_cast(ut_malloc(2 * UNIV_PAGE_SIZE_MAX)); - page = static_cast(ut_align(buf, UNIV_PAGE_SIZE_MAX)); + buf = static_cast(ut_malloc(2 * UNIV_PAGE_SIZE)); + page = static_cast(ut_align(buf, UNIV_PAGE_SIZE)); - success = os_file_read(file, page, 0, UNIV_PAGE_SIZE_MAX); + success = os_file_read(file, page, 0, UNIV_PAGE_SIZE); if (!success) { goto end; } @@ -2392,8 +2404,6 @@ xtrabackup_copy_datafile(fil_node_t* node, uint thread_n) /* Setup the page write filter */ if (xtrabackup_incremental) { write_filter = &wf_incremental; - } else if (xtrabackup_compact) { - write_filter = &wf_compact; } else { write_filter = &wf_write_through; } @@ -2484,6 +2494,7 @@ static void xtrabackup_choose_lsn_offset(lsn_t start_lsn) { +#if SUPPORT_PERCONA_5_5 ulint no, alt_no, expected_no; ulint blocks_in_group; lsn_t tmp_offset, end_lsn; @@ -2566,8 +2577,11 @@ xtrabackup_choose_lsn_offset(lsn_t start_lsn) decision to choose one over the other. Die just like a Buridan's ass */ ut_a(lsn_chosen == 1); +#endif } +extern ibool log_block_checksum_is_ok_or_old_format(const byte* block); + /*******************************************************//** Scans log from a buffer and writes new log data to the outpud datasinc. @return true if success */ @@ -2704,13 +2718,22 @@ xtrabackup_scan_log_recs( if (!*finished) { write_size = RECV_SCAN_SIZE; } else { - write_size = ut_uint64_align_up(scanned_lsn, - OS_FILE_LOG_BLOCK_SIZE) - start_lsn; + write_size = (ulint)(ut_uint64_align_up(scanned_lsn, + OS_FILE_LOG_BLOCK_SIZE) - start_lsn); if (!is_last && scanned_lsn % OS_FILE_LOG_BLOCK_SIZE) { write_size -= OS_FILE_LOG_BLOCK_SIZE; } } + if (write_size == 0) { + return(true); + } + + if (srv_encrypt_log) { + log_encrypt_before_write(scanned_checkpoint_no, + log_sys->buf, write_size); + } + if (ds_write(dst_log_file, log_sys->buf, write_size)) { msg("xtrabackup: Error: " "write to logfile failed\n"); @@ -2756,7 +2779,7 @@ xtrabackup_copy_logfile(lsn_t from_lsn, my_bool is_last) mutex_enter(&log_sys->mutex); log_group_read_log_seg(LOG_RECOVER, log_sys->buf, - group, start_lsn, end_lsn); + group, start_lsn, end_lsn, false); if (!xtrabackup_scan_log_recs(group, is_last, start_lsn, &contiguous_lsn, &group_scanned_lsn, @@ -3133,6 +3156,7 @@ xb_load_tablespaces(void) ibool create_new_db; ulint err; ulint sum_of_new_sizes; + lsn_t min_arch_logno, max_arch_logno; for (i = 0; i < srv_n_file_io_threads; i++) { thread_nr[i] = i; @@ -3144,6 +3168,7 @@ xb_load_tablespaces(void) os_thread_sleep(200000); /*0.2 sec*/ err = open_or_create_data_files(&create_new_db, + &min_arch_logno, &max_arch_logno, &min_flushed_lsn, &max_flushed_lsn, &sum_of_new_sizes); if (err != DB_SUCCESS) { @@ -3225,19 +3250,9 @@ xb_data_files_close(void) for (i = 0; i < 1000; i++) { os_aio_wake_all_threads_at_shutdown(); - os_mutex_enter(os_sync_mutex); - if (os_thread_count == 0) { - - os_mutex_exit(os_sync_mutex); - - os_thread_sleep(10000); - break; } - - os_mutex_exit(os_sync_mutex); - os_thread_sleep(10000); } @@ -3411,7 +3426,7 @@ xb_register_regex( node = static_cast (ut_malloc(sizeof(xb_regex_list_node_t))); - ret = xb_regcomp(&node->regex, regex, REG_EXTENDED); + ret = regcomp(&node->regex, regex, REG_EXTENDED); if (ret != 0) { xb_regerror(ret, &node->regex, errbuf, sizeof(errbuf)); msg("xtrabackup: error: tables regcomp(%s): %s\n", @@ -3545,7 +3560,7 @@ xb_filters_free() while (UT_LIST_GET_LEN(regex_list) > 0) { xb_regex_list_node_t* node = UT_LIST_GET_FIRST(regex_list); UT_LIST_REMOVE(regex_list, regex_list, node); - xb_regfree(&node->regex); + regfree(&node->regex); ut_free(node); } @@ -3602,7 +3617,7 @@ open_or_create_log_file( files[i] = os_file_create(innodb_file_log_key, name, OS_FILE_OPEN, OS_FILE_NORMAL, - OS_LOG_FILE, &ret); + OS_LOG_FILE, &ret,0); if (ret == FALSE) { fprintf(stderr, "InnoDB: Error in opening %s\n", name); @@ -3631,12 +3646,12 @@ open_or_create_log_file( which is for this log group */ fil_space_create(name, - 2 * k + SRV_LOG_SPACE_FIRST_ID, 0, FIL_LOG); + 2 * k + SRV_LOG_SPACE_FIRST_ID, 0, FIL_LOG, 0, 0); } ut_a(fil_validate()); - ut_a(fil_node_create(name, srv_log_file_size, + ut_a(fil_node_create(name, (ulint)srv_log_file_size, 2 * k + SRV_LOG_SPACE_FIRST_ID, FALSE)); if (i == 0) { log_group_init(k, srv_n_log_files, @@ -3765,7 +3780,7 @@ xtrabackup_backup_func(void) srv_read_only_mode = TRUE; srv_backup_mode = TRUE; - srv_close_files = xb_close_files; + srv_close_files = (bool)xb_close_files; if (srv_close_files) msg("xtrabackup: warning: close-files specified. Use it " @@ -3779,7 +3794,7 @@ xtrabackup_backup_func(void) xb_normalize_init_values(); -#ifndef __WIN__ + if (srv_file_flush_method_str == NULL) { /* These are the default options */ srv_unix_file_flush_method = SRV_UNIX_FSYNC; @@ -3808,13 +3823,14 @@ xtrabackup_backup_func(void) "innodb_flush_method\n", srv_file_flush_method_str); exit(EXIT_FAILURE); } -#else /* __WIN__ */ + /* We can only use synchronous unbuffered IO on Windows for now */ if (srv_file_flush_method_str != NULL) { msg("xtrabackupp: Warning: " - "ignoring innodb_flush_method = %s on Windows.\n"); + "ignoring innodb_flush_method = %s on Windows.\n", srv_file_flush_method_str); } +#ifdef _WIN32 srv_win_file_flush_method = SRV_WIN_IO_UNBUFFERED; srv_use_native_aio = FALSE; #endif @@ -3836,10 +3852,14 @@ xtrabackup_backup_func(void) computers */ } - os_sync_mutex = NULL; srv_general_init(); ut_crc32_init(); +#ifdef WITH_INNODB_DISALLOW_WRITES + srv_allow_writes_event = os_event_create(); + os_event_set(srv_allow_writes_event); +#endif + xb_filters_init(); { @@ -3901,7 +3921,7 @@ xtrabackup_backup_func(void) } /* create target dir if not exist */ - if (!my_stat(xtrabackup_target_dir,&stat_info,MYF(0)) + if (!xtrabackup_stream_str && !my_stat(xtrabackup_target_dir,&stat_info,MYF(0)) && (my_mkdir(xtrabackup_target_dir,0777,MYF(0)) < 0)){ msg("xtrabackup: Error: cannot mkdir %d: %s\n", my_errno, xtrabackup_target_dir); @@ -3950,10 +3970,10 @@ xtrabackup_backup_func(void) mutex_exit(&log_sys->mutex); reread_log_header: - fil_io(OS_FILE_READ | OS_FILE_LOG, TRUE, max_cp_group->space_id, + fil_io(OS_FILE_READ | OS_FILE_LOG, true, max_cp_group->space_id, 0, 0, 0, LOG_FILE_HDR_SIZE, - log_hdr_buf, max_cp_group); + log_hdr_buf, max_cp_group, NULL); /* check consistency of log file header to copy */ mutex_enter(&log_sys->mutex); @@ -4427,7 +4447,7 @@ loop: } static void -xtrabackup_stats_func(void) +xtrabackup_stats_func(int argc, char **argv) { ulint n; @@ -4439,7 +4459,7 @@ xtrabackup_stats_func(void) exit(EXIT_FAILURE); } msg("xtrabackup: cd to %s\n", mysql_real_data_home); - + encryption_plugin_prepare_init(argc, argv); mysql_data_home= mysql_data_home_buff; mysql_data_home[0]=FN_CURLIB; // all paths are relative from here mysql_data_home[1]=0; @@ -4654,7 +4674,7 @@ xtrabackup_init_temp_log(void) ibool success; ulint field; - byte log_buf[UNIV_PAGE_SIZE_MAX * 128]; /* 2 MB */ + byte* log_buf= (byte *)malloc(UNIV_PAGE_SIZE_MAX * 128); /* 2 MB */ ib_int64_t file_size; @@ -4668,6 +4688,10 @@ xtrabackup_init_temp_log(void) max_no = 0; + if (!log_buf) { + goto error; + } + if (!xb_init_log_block_size()) { goto error; } @@ -4688,7 +4712,7 @@ retry: src_file = os_file_create_simple_no_error_handling(0, src_path, OS_FILE_OPEN, OS_FILE_READ_WRITE, - &success); + &success,0); if (!success) { /* The following call prints an error message */ os_file_get_last_error(TRUE); @@ -4700,7 +4724,7 @@ retry: src_file = os_file_create_simple_no_error_handling(0, dst_path, OS_FILE_OPEN, OS_FILE_READ_WRITE, - &success); + &success,0); if (!success) { os_file_get_last_error(TRUE); msg(" xtrabackup: Fatal error: cannot find %s.\n", @@ -4798,7 +4822,7 @@ not_consistent: mach_write_to_4(log_buf + LOG_CHECKPOINT_1 + LOG_CHECKPOINT_OFFSET_LOW32, LOG_FILE_HDR_SIZE + - (max_lsn - + (ulint)(max_lsn - ut_uint64_align_down(max_lsn, OS_FILE_LOG_BLOCK_SIZE))); mach_write_to_4(log_buf + LOG_CHECKPOINT_1 @@ -4815,7 +4839,7 @@ not_consistent: mach_write_to_4(log_buf + LOG_CHECKPOINT_2 + LOG_CHECKPOINT_OFFSET_LOW32, LOG_FILE_HDR_SIZE + - (max_lsn - + (ulint)(max_lsn - ut_uint64_align_down(max_lsn, OS_FILE_LOG_BLOCK_SIZE))); mach_write_to_4(log_buf + LOG_CHECKPOINT_2 @@ -4916,15 +4940,17 @@ not_consistent: goto error; } xtrabackup_logfile_is_renamed = TRUE; - + free(log_buf); return(FALSE); skip_modify: + free(log_buf); os_file_close(src_file); src_file = XB_FILE_UNDEFINED; return(FALSE); error: + free(log_buf); if (src_file != XB_FILE_UNDEFINED) os_file_close(src_file); msg("xtrabackup: Error: xtrabackup_init_temp_log() failed.\n"); @@ -4977,7 +5003,7 @@ xb_space_create_file( *file = os_file_create_simple_no_error_handling(0, path, OS_FILE_CREATE, OS_FILE_READ_WRITE, - &ret); + &ret,0); if (!ret) { msg("xtrabackup: cannot create file %s\n", path); return ret; @@ -5171,7 +5197,7 @@ xb_delta_open_matching_space( /* No matching space found. create the new one. */ if (!fil_space_create(dest_space_name, space_id, 0, - FIL_TABLESPACE)) { + FIL_TABLESPACE, 0, false)) { msg("xtrabackup: Cannot create tablespace %s\n", dest_space_name); goto exit; @@ -5201,7 +5227,7 @@ found: file = os_file_create_simple_no_error_handling(0, real_name, OS_FILE_OPEN, OS_FILE_READ_WRITE, - &ok); + &ok,0); if (ok) { *success = TRUE; @@ -5290,7 +5316,7 @@ xtrabackup_apply_delta( src_file = os_file_create_simple_no_error_handling(0, src_path, OS_FILE_OPEN, OS_FILE_READ_WRITE, - &success); + &success,0); if (!success) { os_file_get_last_error(TRUE); msg("xtrabackup: error: cannot open %s\n", src_path); @@ -5315,11 +5341,11 @@ xtrabackup_apply_delta( /* allocate buffer for incremental backup (4096 pages) */ incremental_buffer_base = static_cast - (ut_malloc((UNIV_PAGE_SIZE_MAX / 4 + 1) * - UNIV_PAGE_SIZE_MAX)); + (ut_malloc((page_size / 4 + 1) * + page_size)); incremental_buffer = static_cast (ut_align(incremental_buffer_base, - UNIV_PAGE_SIZE_MAX)); + page_size)); msg("Applying %s to %s...\n", src_path, dst_path); @@ -5637,7 +5663,7 @@ xtrabackup_close_temp_log(my_bool clear_flag) src_file = os_file_create_simple_no_error_handling(0, src_path, OS_FILE_OPEN, OS_FILE_READ_WRITE, - &success); + &success,0); if (!success) { goto error; } @@ -5701,7 +5727,7 @@ xb_export_cfg_write_index_fields( } /* Include the NUL byte in the length. */ - ib_uint32_t len = strlen(field->name) + 1; + ib_uint32_t len = (ib_uint32_t)strlen(field->name) + 1; ut_a(len > 1); mach_write_to_4(row, len); @@ -5791,7 +5817,7 @@ xb_export_cfg_write_indexes( /* Write the length of the index name. NUL byte is included in the length. */ - ib_uint32_t len = strlen(index->name) + 1; + ib_uint32_t len = (ib_uint32_t)strlen(index->name) + 1; ut_a(len > 1); mach_write_to_4(row, len); @@ -5865,7 +5891,7 @@ xb_export_cfg_write_table( col_name = dict_table_get_col_name(table, dict_col_get_no(col)); /* Include the NUL byte in the length. */ - len = strlen(col_name) + 1; + len = (ib_uint32_t)strlen(col_name) + 1; ut_a(len > 1); mach_write_to_4(row, len); @@ -5909,7 +5935,7 @@ xb_export_cfg_write_header( const char* hostname = "Hostname unknown"; /* The server hostname includes the NUL byte. */ - len = strlen(hostname) + 1; + len = (ib_uint32_t)strlen(hostname) + 1; mach_write_to_4(value, len); if (fwrite(&value, 1, sizeof(value), file) != sizeof(value) @@ -5922,7 +5948,7 @@ xb_export_cfg_write_header( /* The table name includes the NUL byte. */ ut_a(table->name != 0); - len = strlen(table->name) + 1; + len = (ib_uint32_t)strlen(table->name) + 1; /* Write the table name. */ mach_write_to_4(value, len); @@ -6122,7 +6148,7 @@ store_binlog_info( } static void -xtrabackup_prepare_func(void) +xtrabackup_prepare_func(int argc, char ** argv) { ulint err; datafiles_iter_t *it; @@ -6140,6 +6166,8 @@ xtrabackup_prepare_func(void) } msg("xtrabackup: cd to %s\n", xtrabackup_real_target_dir); + encryption_plugin_prepare_init(argc, argv); + xtrabackup_target_dir= mysql_data_home_buff; xtrabackup_target_dir[0]=FN_CURLIB; // all paths are relative from here xtrabackup_target_dir[1]=0; @@ -6193,7 +6221,6 @@ skip_check: /* Create logfiles for recovery from 'xtrabackup_logfile', before start InnoDB */ srv_max_n_threads = 1000; - os_sync_mutex = NULL; ut_mem_init(); /* temporally dummy value to avoid crash */ srv_page_size_shift = 14; @@ -6204,6 +6231,11 @@ skip_check: mem_init(srv_mem_pool_size); ut_crc32_init(); +#ifdef WITH_INNODB_DISALLOW_WRITES + srv_allow_writes_event = os_event_create(); + os_event_set(srv_allow_writes_event); +#endif + xb_filters_init(); if(!innobase_log_arch_dir && xtrabackup_init_temp_log()) @@ -6213,25 +6245,6 @@ skip_check: goto error_cleanup; } - /* Expand compacted datafiles */ - - if (xtrabackup_compact) { - srv_compact_backup = TRUE; - - if (!xb_expand_datafiles()) { - goto error_cleanup; - } - - /* Reset the 'compact' flag in xtrabackup_checkpoints so we - don't expand on subsequent invocations. */ - xtrabackup_compact = FALSE; - if (!xtrabackup_write_metadata(metadata_path)) { - msg("xtrabackup: error: xtrabackup_write_metadata() " - "failed\n"); - goto error_cleanup; - } - } - xb_normalize_init_values(); if (xtrabackup_incremental || innobase_log_arch_dir) { @@ -6262,17 +6275,16 @@ skip_check: xb_filter_hash_free(inc_dir_tables_hash); } - sync_close(); - sync_initialized = FALSE; if (fil_system) { fil_close(); } - os_sync_free(); + mem_close(); - os_sync_mutex = NULL; ut_free_all_mem(); innodb_free_param(); + sync_close(); + sync_initialized = FALSE; /* Reset the configuration as it might have been changed by xb_data_files_init(). */ @@ -6280,8 +6292,7 @@ skip_check: goto error_cleanup; } - srv_apply_log_only = (ibool) xtrabackup_apply_log_only; - srv_rebuild_indexes = (ibool) xtrabackup_rebuild_indexes; + srv_apply_log_only = (bool) xtrabackup_apply_log_only; /* increase IO threads */ if(srv_n_file_io_threads < 10) { @@ -6291,7 +6302,7 @@ skip_check: if (innobase_log_arch_dir) { srv_arch_dir = innobase_log_arch_dir; - srv_archive_recovery = TRUE; + srv_archive_recovery = true; if (xtrabackup_archived_to_lsn) { if (xtrabackup_archived_to_lsn < metadata_last_lsn) { msg("xtrabackup: warning: logs applying lsn " @@ -6317,7 +6328,7 @@ skip_check: Unfinished transactions are not rolled back during log applying as they can be finished at the firther files applyings. */ - srv_apply_log_only = xtrabackup_apply_log_only = TRUE; + srv_apply_log_only = xtrabackup_apply_log_only = true; if (!xtrabackup_arch_search_files(min_flushed_lsn)) { goto error_cleanup; @@ -6340,6 +6351,12 @@ skip_check: "xtrabackup: Using %lld bytes for buffer pool " "(set by --use-memory parameter)\n", xtrabackup_use_memory); + srv_max_buf_pool_modified_pct = (double)max_buf_pool_modified_pct; + + if (srv_max_dirty_pages_pct_lwm > srv_max_buf_pool_modified_pct) { + srv_max_dirty_pages_pct_lwm = srv_max_buf_pool_modified_pct; + } + if(innodb_init()) goto error_cleanup; @@ -6423,15 +6440,20 @@ skip_check: /* node exist == file exist, here */ strcpy(info_file_path, node->name); +#ifdef _WIN32 + for (int i = 0; info_file_path[i]; i++) + if (info_file_path[i] == '\\') + info_file_path[i]= '/'; +#endif strcpy(info_file_path + strlen(info_file_path) - 4, ".exp"); - len = strlen(info_file_path); + len =(ib_uint32_t)strlen(info_file_path); p = info_file_path; prev = NULL; - while ((next = strchr(p, SRV_PATH_SEPARATOR)) != NULL) + while ((next = strchr(p, '/')) != NULL) { prev = p; p = next + 1; @@ -6503,7 +6525,7 @@ skip_check: info_file_path, OS_FILE_OVERWRITE, OS_FILE_NORMAL, OS_DATA_FILE, - &success); + &success,0); if (!success) { os_file_get_last_error(TRUE); goto next_node; @@ -6569,8 +6591,9 @@ next_node: } exit(EXIT_FAILURE); } - +#ifdef WITH_WSREP xb_write_galera_info(xtrabackup_incremental); +#endif if(innodb_end()) goto error_cleanup; @@ -6578,7 +6601,6 @@ next_node: innodb_free_param(); sync_initialized = FALSE; - os_sync_mutex = NULL; /* re-init necessary components */ ut_mem_init(); @@ -6630,9 +6652,7 @@ next_node: if (fil_system) { fil_close(); } - os_sync_free(); - // mem_close(); - os_sync_mutex = NULL; + ut_free_all_mem(); /* start InnoDB once again to create log files */ @@ -6643,8 +6663,7 @@ next_node: goto error; } - srv_apply_log_only = FALSE; - srv_rebuild_indexes = FALSE; + srv_apply_log_only = false; /* increase IO threads */ if(srv_n_file_io_threads < 10) { @@ -6676,40 +6695,6 @@ error: exit(EXIT_FAILURE); } -/************************************************************************** -Signals-related setup. */ -static -void -setup_signals() -/*===========*/ -{ - struct sigaction sa; - - /* Print a stacktrace on some signals */ - sa.sa_flags = SA_RESETHAND | SA_NODEFER; - sigemptyset(&sa.sa_mask); - sigprocmask(SIG_SETMASK,&sa.sa_mask,NULL); -#ifdef HAVE_STACKTRACE - my_init_stacktrace(); -#endif - sa.sa_handler = handle_fatal_signal; - sigaction(SIGSEGV, &sa, NULL); - sigaction(SIGABRT, &sa, NULL); - sigaction(SIGBUS, &sa, NULL); - sigaction(SIGILL, &sa, NULL); - sigaction(SIGFPE, &sa, NULL); - -#ifdef __linux__ - /* Ensure xtrabackup process is killed when the parent one - (innobackupex) is terminated with an unhandled signal */ - - if (prctl(PR_SET_PDEATHSIG, SIGKILL)) { - msg("prctl() failed with errno = %d\n", errno); - exit(EXIT_FAILURE); - } -#endif -} - /************************************************************************** Append group name to xb_load_default_groups list. */ static @@ -6778,11 +6763,6 @@ xb_init() } if (xtrabackup_backup) { - - if (!opt_noversioncheck) { - version_check(); - } - if ((mysql_connection = xb_mysql_connect()) == NULL) { return(false); } @@ -6791,6 +6771,7 @@ xb_init() return(false); } + encryption_plugin_backup_init(mysql_connection); history_start_time = time(NULL); } @@ -6798,9 +6779,74 @@ xb_init() return(true); } + +extern void init_signals(void); + +#include + +/* Messages . Avoid loading errmsg.sys file */ +void setup_error_messages() +{ + static const char *all_msgs[ER_ERROR_LAST - ER_ERROR_FIRST +1]; + my_default_lc_messages = &my_locale_en_US; + my_default_lc_messages->errmsgs->errmsgs = all_msgs; + + /* Populate the necessary error messages */ + struct { + int id; + const char *fmt; + } + xb_msgs[] = + { + { ER_DATABASE_NAME,"Database" }, + { ER_TABLE_NAME,"Table"}, + { ER_PARTITION_NAME, "Partition" }, + { ER_SUBPARTITION_NAME, "Subpartition" }, + { ER_TEMPORARY_NAME, "Temporary"}, + { ER_RENAMED_NAME, "Renamed"}, + { ER_CANT_FIND_DL_ENTRY, "Can't find symbol '%-.128s' in library"}, + { ER_CANT_OPEN_LIBRARY, "Can't open shared library '%-.192s' (errno: %d, %-.128s)" }, + { ER_OUTOFMEMORY, "Out of memory; restart server and try again (needed %d bytes)" }, + { ER_CANT_OPEN_LIBRARY, "Can't open shared library '%-.192s' (errno: %d, %-.128s)" }, + { ER_UDF_NO_PATHS, "No paths allowed for shared library" }, + { ER_CANT_INITIALIZE_UDF,"Can't initialize function '%-.192s'; %-.80s"}, + { ER_PLUGIN_IS_NOT_LOADED,"Plugin '%-.192s' is not loaded" } + }; + + for (int i = 0; i < (int)array_elements(all_msgs); i++) + all_msgs[i] = "Unknown error"; + + for (int i = 0; i < (int)array_elements(xb_msgs); i++) + all_msgs[xb_msgs[i].id - ER_ERROR_FIRST] = xb_msgs[i].fmt; +} + +extern my_bool(*dict_check_if_skip_table)(const char* name) ; + void handle_options(int argc, char **argv, char ***argv_client, char ***argv_server) { + /* Setup some variables for Innodb.*/ + + srv_xtrabackup = true; + + + files_charset_info = &my_charset_utf8_general_ci; + dict_check_if_skip_table = check_if_skip_table; + + setup_error_messages(); + sys_var_init(); + plugin_mutex_init(); + mysql_rwlock_init(key_rwlock_LOCK_system_variables_hash, &LOCK_system_variables_hash); + opt_stack_trace = 1; + test_flags |= TEST_SIGINT; + init_signals(); +#ifndef _WIN32 + /* Exit process on SIGINT. */ + my_sigset(SIGINT, SIG_DFL); +#endif + + sf_leaking_memory = 0; /* don't report memory leaks on early exist */ + int i; int ho_error; @@ -6811,9 +6857,6 @@ handle_options(int argc, char **argv, char ***argv_client, char ***argv_server) int argc_client = argc; int argc_server = argc; - *argv_client = argv; - *argv_server = argv; - /* scan options for group and config file to load defaults from */ for (i = 1; i < argc; i++) { @@ -6859,12 +6902,27 @@ handle_options(int argc, char **argv, char ***argv_client, char ***argv_server) if (prepare && target_dir) { snprintf(conf_file, sizeof(conf_file), "%s/backup-my.cnf", target_dir); + if (!strncmp(argv[1], "--defaults-file=", 16)) { + /* Remove defaults-file*/ + for (int i = 2; ; i++) { + if ((argv[i-1]= argv[i]) == 0) + break; + } + argc--; + } } + + *argv_client = argv; + *argv_server = argv; if (load_defaults(conf_file, xb_server_default_groups, &argc_server, argv_server)) { exit(EXIT_FAILURE); } + int n; + for (n = 0; (*argv_server)[n]; n++) {}; + argc_server = n; + print_param_str << "# This MySQL options file was generated by XtraBackup.\n" "[" << defaults_group << "]\n"; @@ -6879,6 +6937,7 @@ handle_options(int argc, char **argv, char ***argv_client, char ***argv_server) optp->u_max_value = (G_PTR *) &global_max_value; } + /* Throw a descriptive error if --defaults-file or --defaults-extra-file is not the first command line argument */ for (int i = 2 ; i < argc ; i++) { @@ -6913,6 +6972,9 @@ handle_options(int argc, char **argv, char ***argv_client, char ***argv_server) exit(EXIT_FAILURE); } + for (n = 0; (*argv_client)[n]; n++) {}; + argc_client = n; + if (strcmp(base_name(my_progname), INNOBACKUPEX_BIN_NAME) == 0 && argc_client > 0) { /* emulate innobackupex script */ @@ -6959,10 +7021,16 @@ int main(int argc, char **argv) { char **client_defaults, **server_defaults; char cwd[FN_REFLEN]; - my_bool is_symdir; - setup_signals(); + if (argc > 1 && (strcmp(argv[1], "--innobackupex") == 0)) + { + argv++; + argc--; + argv[0] = "innobackupex"; + innobackupex_mode = true; + } + init_signals(); MY_INIT(argv[0]); pthread_key_create(&THR_THD, NULL); @@ -6977,11 +7045,18 @@ int main(int argc, char **argv) exit(EXIT_FAILURE); } - system_charset_info= &my_charset_utf8_general_ci; + system_charset_info = &my_charset_utf8_general_ci; key_map_full.set_all(); handle_options(argc, argv, &client_defaults, &server_defaults); + int argc_server; + for (argc_server = 0; server_defaults[argc_server]; argc_server++) {} + + int argc_client; + for (argc_client = 0; client_defaults[argc_client]; argc_client++) {} + + if (innobackupex_mode) { if (!ibx_init()) { exit(EXIT_FAILURE); @@ -7002,14 +7077,14 @@ int main(int argc, char **argv) my_load_path(xtrabackup_real_target_dir, xtrabackup_target_dir, cwd); unpack_dirname(xtrabackup_real_target_dir, - xtrabackup_real_target_dir, &is_symdir); + xtrabackup_real_target_dir); xtrabackup_target_dir= xtrabackup_real_target_dir; if (xtrabackup_incremental_basedir) { my_load_path(xtrabackup_real_incremental_basedir, xtrabackup_incremental_basedir, cwd); unpack_dirname(xtrabackup_real_incremental_basedir, - xtrabackup_real_incremental_basedir, &is_symdir); + xtrabackup_real_incremental_basedir); xtrabackup_incremental_basedir = xtrabackup_real_incremental_basedir; } @@ -7018,7 +7093,7 @@ int main(int argc, char **argv) my_load_path(xtrabackup_real_incremental_dir, xtrabackup_incremental_dir, cwd); unpack_dirname(xtrabackup_real_incremental_dir, - xtrabackup_real_incremental_dir, &is_symdir); + xtrabackup_real_incremental_dir); xtrabackup_incremental_dir = xtrabackup_real_incremental_dir; } @@ -7026,7 +7101,7 @@ int main(int argc, char **argv) my_load_path(xtrabackup_real_extra_lsndir, xtrabackup_extra_lsndir, cwd); unpack_dirname(xtrabackup_real_extra_lsndir, - xtrabackup_real_extra_lsndir, &is_symdir); + xtrabackup_real_extra_lsndir); xtrabackup_extra_lsndir = xtrabackup_real_extra_lsndir; } @@ -7181,11 +7256,12 @@ int main(int argc, char **argv) /* --stats */ if (xtrabackup_stats) - xtrabackup_stats_func(); + xtrabackup_stats_func(argc_server,server_defaults); /* --prepare */ - if (xtrabackup_prepare) - xtrabackup_prepare_func(); + if (xtrabackup_prepare) { + xtrabackup_prepare_func(argc_server, server_defaults); + } if (xtrabackup_copy_back || xtrabackup_move_back) { if (!check_if_param_set("datadir")) { @@ -7206,7 +7282,6 @@ int main(int argc, char **argv) ibx_cleanup(); } - xb_regex_end(); free_defaults(client_defaults); free_defaults(server_defaults); diff --git a/extra/mariabackup/xtrabackup.h b/extra/mariabackup/xtrabackup.h index b31028175c9..de21da31741 100644 --- a/extra/mariabackup/xtrabackup.h +++ b/extra/mariabackup/xtrabackup.h @@ -63,8 +63,6 @@ extern lsn_t checkpoint_lsn_start; extern xb_page_bitmap *changed_page_bitmap; -extern ulint xtrabackup_rebuild_threads; - extern char *xtrabackup_incremental; extern my_bool xtrabackup_incremental_force_scan; @@ -80,7 +78,6 @@ extern char *xtrabackup_tables_file; extern char *xtrabackup_databases; extern char *xtrabackup_databases_file; -extern my_bool xtrabackup_compact; extern ibool xtrabackup_compress; extern ibool xtrabackup_encrypt; @@ -106,8 +103,14 @@ extern int xtrabackup_parallel; extern my_bool xb_close_files; extern const char *xtrabackup_compress_alg; -extern uint xtrabackup_compress_threads; -extern ulonglong xtrabackup_compress_chunk_size; +#ifdef __cplusplus +extern "C"{ +#endif + extern uint xtrabackup_compress_threads; + extern ulonglong xtrabackup_compress_chunk_size; +#ifdef __cplusplus +} +#endif extern ulong xtrabackup_encrypt_algo; extern uint xtrabackup_encrypt_threads; extern ulonglong xtrabackup_encrypt_chunk_size; @@ -116,7 +119,6 @@ extern char *xtrabackup_incremental_basedir; extern char *xtrabackup_extra_lsndir; extern char *xtrabackup_incremental_dir; extern ulint xtrabackup_log_copy_interval; -extern my_bool xtrabackup_rebuild_indexes; extern char *xtrabackup_stream_str; extern long xtrabackup_throttle; extern longlong xtrabackup_use_memory; @@ -165,14 +167,6 @@ extern uint opt_safe_slave_backup_timeout; extern const char *opt_history; extern my_bool opt_decrypt; -#if defined(HAVE_OPENSSL) -extern my_bool opt_use_ssl; -extern my_bool opt_ssl_verify_server_cert; -#if !defined(HAVE_YASSL) -extern char *opt_server_public_key; -#endif -#endif - enum binlog_info_enum { BINLOG_INFO_OFF, BINLOG_INFO_LOCKLESS, BINLOG_INFO_ON, BINLOG_INFO_AUTO}; @@ -217,6 +211,14 @@ Check if parameter is set in defaults file or via command line argument bool check_if_param_set(const char *param); +#if defined(HAVE_OPENSSL) +extern my_bool opt_use_ssl; +extern my_bool opt_ssl_verify_server_cert; +#if !defined(HAVE_YASSL) +extern char *opt_server_public_key; +#endif +#endif + void xtrabackup_backup_func(void); diff --git a/extra/mariabackup/xtrabackup_version.h.in b/extra/mariabackup/xtrabackup_version.h.in deleted file mode 100644 index dc4c7992f8f..00000000000 --- a/extra/mariabackup/xtrabackup_version.h.in +++ /dev/null @@ -1,27 +0,0 @@ -/****************************************************** -Copyright (c) 2013 Percona LLC and/or its affiliates. - -Version numbers definitions. - -This program is free software; you can redistribute it and/or modify -it under the terms of the GNU General Public License as published by -the Free Software Foundation; version 2 of the License. - -This program is distributed in the hope that it will be useful, -but WITHOUT ANY WARRANTY; without even the implied warranty of -MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the -GNU General Public License for more details. - -You should have received a copy of the GNU General Public License -along with this program; if not, write to the Free Software -Foundation, Inc., 51 Franklin Street, Fifth Floor, Boston, MA 02110-1301, USA - -*******************************************************/ - -#ifndef XB_VERSION_H -#define XB_VERSION_H - -#define XTRABACKUP_VERSION "@XB_VERSION@" -#define XTRABACKUP_REVISION "@XB_REVISION@" - -#endif /* XB_VERSION_H */ diff --git a/storage/xtradb/CMakeLists.txt b/storage/xtradb/CMakeLists.txt index 8d1bd543d0c..77d8ca7fdbc 100644 --- a/storage/xtradb/CMakeLists.txt +++ b/storage/xtradb/CMakeLists.txt @@ -506,3 +506,6 @@ MYSQL_ADD_PLUGIN(xtradb ${INNOBASE_SOURCES} STORAGE_ENGINE IF(TARGET xtradb AND NOT XTRADB_OK) MESSAGE(FATAL_ERROR "Percona XtraDB is not supported on this platform") ENDIF() + +ADD_SUBDIRECTORY(${CMAKE_SOURCE_DIR}/extra/mariabackup ${CMAKE_BINARY_DIR}/extra/mariabackup) + From 1991411f16366f5acda5c9478ea544c56c3974f0 Mon Sep 17 00:00:00 2001 From: Vladislav Vaintroub Date: Tue, 18 Apr 2017 19:35:48 +0000 Subject: [PATCH 11/28] MDEV-9566 MariaBackup packaging --- cmake/cpack_rpm.cmake | 7 ++++++- debian/control | 8 ++++++++ debian/mariadb-backup-10.1.files | 2 ++ win/packaging/CPackWixConfig.cmake | 7 ++++++- 4 files changed, 22 insertions(+), 2 deletions(-) create mode 100644 debian/mariadb-backup-10.1.files diff --git a/cmake/cpack_rpm.cmake b/cmake/cpack_rpm.cmake index f1aab2c353c..d6f43e3f730 100644 --- a/cmake/cpack_rpm.cmake +++ b/cmake/cpack_rpm.cmake @@ -23,10 +23,14 @@ SET(CPACK_COMPONENT_SHAREDLIBRARIES_GROUP "shared") SET(CPACK_COMPONENT_COMMON_GROUP "common") SET(CPACK_COMPONENT_CLIENTPLUGINS_GROUP "common") SET(CPACK_COMPONENT_COMPAT_GROUP "compat") +SET(CPACK_COMPONENT_BACKUP_GROUP "backup") + SET(CPACK_COMPONENTS_ALL Server ManPagesServer IniFiles Server_Scripts SupportFiles Development ManPagesDevelopment ManPagesTest Readme ManPagesClient Test - Common Client SharedLibraries ClientPlugins) + Common Client SharedLibraries ClientPlugins + backup +) SET(CPACK_RPM_PACKAGE_NAME ${CPACK_PACKAGE_NAME}) SET(CPACK_PACKAGE_FILE_NAME "${CPACK_RPM_PACKAGE_NAME}-${VERSION}-${RPM}-${CMAKE_SYSTEM_PROCESSOR}") @@ -98,6 +102,7 @@ SET(CPACK_RPM_client_USER_FILELIST ${ignored} "%config(noreplace) ${INSTALL_SYSC SET(CPACK_RPM_compat_USER_FILELIST ${ignored}) SET(CPACK_RPM_devel_USER_FILELIST ${ignored}) SET(CPACK_RPM_test_USER_FILELIST ${ignored}) +SET(CPACK_RPM_backup_USER_FILELIST ${ignored}) # "set/append array" - append a set of strings, separated by a space MACRO(SETA var) diff --git a/debian/control b/debian/control index cb602510d95..c25ce6dd20f 100644 --- a/debian/control +++ b/debian/control @@ -10,6 +10,7 @@ Build-Depends: bison, dh-apparmor, dpatch, libaio-dev, + libarchive-dev, libboost-dev, libjudy-dev, libkrb5-dev, @@ -443,3 +444,10 @@ Section: database Architecture: any Depends: libgssapi-krb5-2, mariadb-client-10.1 Description: GSSAPI authentication plugin for MariaDB client + + +Package: mariadb-backup-10.1 +Section: database +Architecture: any +Depends: libarchive12 | libarchive13 +Description: Backup tool for MariaDB server \ No newline at end of file diff --git a/debian/mariadb-backup-10.1.files b/debian/mariadb-backup-10.1.files new file mode 100644 index 00000000000..734117c92e2 --- /dev/null +++ b/debian/mariadb-backup-10.1.files @@ -0,0 +1,2 @@ +usr/bin/mariabackup +usr/bin/mbstream diff --git a/win/packaging/CPackWixConfig.cmake b/win/packaging/CPackWixConfig.cmake index 6df49678d52..6683020c179 100644 --- a/win/packaging/CPackWixConfig.cmake +++ b/win/packaging/CPackWixConfig.cmake @@ -9,7 +9,7 @@ IF(ESSENTIALS) ENDIF() ELSE() SET(CPACK_COMPONENTS_USED - "Server;Client;Development;SharedLibraries;Embedded;Documentation;IniFiles;Readme;Debuginfo;Common;connect-engine;ClientPlugins;gssapi-server;gssapi-client;aws-key-management") + "Server;Client;Development;SharedLibraries;Embedded;Documentation;IniFiles;Readme;Debuginfo;Common;connect-engine;ClientPlugins;gssapi-server;gssapi-client;aws-key-management;backup") ENDIF() SET( WIX_FEATURE_MySQLServer_EXTRA_FEATURES "DBInstance;SharedClientServerComponents") @@ -55,6 +55,11 @@ SET(CPACK_COMPONENT_GROUP_MYSQLSERVER_DESCRIPTION "Install server") "Debug/trace versions of executables and libraries" ) #SET(CPACK_COMPONENT_DEBUGBINARIES_WIX_LEVEL 2) + # Subfeature "Backup" + SET(CPACK_COMPONENT_BACKUP_GROUP "MySQLServer") + SET(CPACK_COMPONENT_BACKUP_DISPLAY_NAME "Backup utilities") + SET(CPACK_COMPONENT_BACKUP_DESCRIPTION "Installs backup utilities(mariabackup and mbstream)") + #Miscellaneous (hidden) components, part of server / or client programs FOREACH(comp connect-engine ClientPlugins gssapi-server gssapi-client aws-key-management) From ca24f35b67c8cc26b7135a677dada8b563b561b4 Mon Sep 17 00:00:00 2001 From: Vladislav Vaintroub Date: Tue, 18 Apr 2017 19:52:03 +0000 Subject: [PATCH 12/28] MDEV-9566 MariaBackup test suite --- .../suite/mariabackup/bug1509812-master.opt | 1 + .../suite/mariabackup/filekeys-data.enc | Bin 0 -> 416 bytes .../suite/mariabackup/filekeys-data.key | 2 + .../suite/mariabackup/full_backup.result | 13 +++ mysql-test/suite/mariabackup/full_backup.test | 23 +++++ .../include/have_file_key_management.inc | 4 + .../include/restart_and_restore.inc | 15 +++ .../mariabackup/incremental_backup.result | 20 ++++ .../suite/mariabackup/incremental_backup.test | 35 +++++++ .../mariabackup/incremental_encrypted.opt | 3 + .../mariabackup/incremental_encrypted.result | 20 ++++ .../mariabackup/incremental_encrypted.test | 45 +++++++++ mysql-test/suite/mariabackup/partial.result | 13 +++ mysql-test/suite/mariabackup/partial.test | 31 +++++++ mysql-test/suite/mariabackup/small_ibd.result | 1 + mysql-test/suite/mariabackup/small_ibd.test | 18 ++++ mysql-test/suite/mariabackup/suite.opt | 1 + mysql-test/suite/mariabackup/suite.pm | 38 ++++++++ mysql-test/suite/mariabackup/tar.result | 12 +++ mysql-test/suite/mariabackup/tar.test | 30 ++++++ .../mariabackup/xb_aws_key_management.opt | 3 + .../mariabackup/xb_aws_key_management.result | 11 +++ .../mariabackup/xb_aws_key_management.test | 22 +++++ .../mariabackup/xb_compressed_encrypted.opt | 9 ++ .../xb_compressed_encrypted.result | 25 +++++ .../mariabackup/xb_compressed_encrypted.test | 35 +++++++ .../mariabackup/xb_file_key_management.opt | 6 ++ .../mariabackup/xb_file_key_management.result | 17 ++++ .../mariabackup/xb_file_key_management.test | 40 ++++++++ .../mariabackup/xb_fulltext_encrypted.opt | 8 ++ .../mariabackup/xb_fulltext_encrypted.result | 14 +++ .../mariabackup/xb_fulltext_encrypted.test | 23 +++++ .../suite/mariabackup/xb_history.result | 5 + mysql-test/suite/mariabackup/xb_history.test | 8 ++ .../suite/mariabackup/xb_page_compress.result | 28 ++++++ .../suite/mariabackup/xb_page_compress.test | 44 +++++++++ .../suite/mariabackup/xb_partition.result | 64 +++++++++++++ .../suite/mariabackup/xb_partition.test | 87 ++++++++++++++++++ mysql-test/suite/mariabackup/xbstream.result | 13 +++ mysql-test/suite/mariabackup/xbstream.test | 22 +++++ 40 files changed, 809 insertions(+) create mode 100644 mysql-test/suite/mariabackup/bug1509812-master.opt create mode 100644 mysql-test/suite/mariabackup/filekeys-data.enc create mode 100644 mysql-test/suite/mariabackup/filekeys-data.key create mode 100644 mysql-test/suite/mariabackup/full_backup.result create mode 100644 mysql-test/suite/mariabackup/full_backup.test create mode 100644 mysql-test/suite/mariabackup/include/have_file_key_management.inc create mode 100644 mysql-test/suite/mariabackup/include/restart_and_restore.inc create mode 100644 mysql-test/suite/mariabackup/incremental_backup.result create mode 100644 mysql-test/suite/mariabackup/incremental_backup.test create mode 100644 mysql-test/suite/mariabackup/incremental_encrypted.opt create mode 100644 mysql-test/suite/mariabackup/incremental_encrypted.result create mode 100644 mysql-test/suite/mariabackup/incremental_encrypted.test create mode 100644 mysql-test/suite/mariabackup/partial.result create mode 100644 mysql-test/suite/mariabackup/partial.test create mode 100644 mysql-test/suite/mariabackup/small_ibd.result create mode 100644 mysql-test/suite/mariabackup/small_ibd.test create mode 100644 mysql-test/suite/mariabackup/suite.opt create mode 100644 mysql-test/suite/mariabackup/suite.pm create mode 100644 mysql-test/suite/mariabackup/tar.result create mode 100644 mysql-test/suite/mariabackup/tar.test create mode 100644 mysql-test/suite/mariabackup/xb_aws_key_management.opt create mode 100644 mysql-test/suite/mariabackup/xb_aws_key_management.result create mode 100644 mysql-test/suite/mariabackup/xb_aws_key_management.test create mode 100644 mysql-test/suite/mariabackup/xb_compressed_encrypted.opt create mode 100644 mysql-test/suite/mariabackup/xb_compressed_encrypted.result create mode 100644 mysql-test/suite/mariabackup/xb_compressed_encrypted.test create mode 100644 mysql-test/suite/mariabackup/xb_file_key_management.opt create mode 100644 mysql-test/suite/mariabackup/xb_file_key_management.result create mode 100644 mysql-test/suite/mariabackup/xb_file_key_management.test create mode 100644 mysql-test/suite/mariabackup/xb_fulltext_encrypted.opt create mode 100644 mysql-test/suite/mariabackup/xb_fulltext_encrypted.result create mode 100644 mysql-test/suite/mariabackup/xb_fulltext_encrypted.test create mode 100644 mysql-test/suite/mariabackup/xb_history.result create mode 100644 mysql-test/suite/mariabackup/xb_history.test create mode 100644 mysql-test/suite/mariabackup/xb_page_compress.result create mode 100644 mysql-test/suite/mariabackup/xb_page_compress.test create mode 100644 mysql-test/suite/mariabackup/xb_partition.result create mode 100644 mysql-test/suite/mariabackup/xb_partition.test create mode 100644 mysql-test/suite/mariabackup/xbstream.result create mode 100644 mysql-test/suite/mariabackup/xbstream.test diff --git a/mysql-test/suite/mariabackup/bug1509812-master.opt b/mysql-test/suite/mariabackup/bug1509812-master.opt new file mode 100644 index 00000000000..de29ed41f59 --- /dev/null +++ b/mysql-test/suite/mariabackup/bug1509812-master.opt @@ -0,0 +1 @@ +--loose-skip-log-bin \ No newline at end of file diff --git a/mysql-test/suite/mariabackup/filekeys-data.enc b/mysql-test/suite/mariabackup/filekeys-data.enc new file mode 100644 index 0000000000000000000000000000000000000000..a8adb2f939c72baf5eb9d94f8f272e898095c5e2 GIT binary patch literal 416 zcmV;R0bl-8VQh3|WM5y;+RZcbTB{aaj+zq$1rn3EX$|^vch87eL%!a%{lGF59BZbb z?ktd2=)WMn(+gx0Xx=EMYqjoD^5jSV<6E*Qn*tW#XkR;+ejj;Ur$6B>8pHpV%Wr!l zLheWidQGBh7t@q?^=fSC;q1qbHPuKrhmnqT28KR}uZ(B*&te4na9?$ipP~X(Xxx4| z9+Y;!%%S)vprk2P{Yffmz{Y52c1TsIzcZBEE+kD-bkyIkZkb89dzjF6NC@Wv2L)D) zJnk$~DR}9p!M9zTg3t@(N_wRcjKdJrg(z^c!Y5Qs7{2i2Crs4nJ0tiDRl{!Pyq&em zb11D04Kt;JS&KbI>e(+o*&hZqden7;m7>9#{`r;|vW?t`*J`U*Pa5P5sYrVzqvWuY z`JBjW)}MlrLEA*^mKwjq`MW1NNeYiOc{TdG!51{9BfV|SZr`{)EyJ{OZkl)1v7$E2 zxrhL(Pp!$cRC!NDplPKv=)_6LB*d!tIu225T6YVUVsdJC=q*neWUffzCs>-e>EG $_expect_file_name; +echo # shutdown server; +shutdown_server; +echo # remove datadir; +rmdir $_datadir; +echo # xtrabackup move back; +exec $XTRABACKUP --copy-back --datadir=$_datadir --target-dir=$targetdir --parallel=2; +echo # restart server; +exec echo "restart" > $_expect_file_name; +enable_reconnect; +source include/wait_until_connected_again.inc; +disable_reconnect; diff --git a/mysql-test/suite/mariabackup/incremental_backup.result b/mysql-test/suite/mariabackup/incremental_backup.result new file mode 100644 index 00000000000..eeedc751d83 --- /dev/null +++ b/mysql-test/suite/mariabackup/incremental_backup.result @@ -0,0 +1,20 @@ +call mtr.add_suppression("InnoDB: New log files created"); +CREATE TABLE t(i INT) ENGINE INNODB; +INSERT INTO t VALUES(1); +# Create full backup , modify table, then create incremental/differential backup +INSERT INTO t VALUES(2); +SELECT * FROM t; +i +1 +2 +# Prepare full backup, apply incremental one +# Restore and check results +# shutdown server +# remove datadir +# xtrabackup move back +# restart server +SELECT * FROM t; +i +1 +2 +DROP TABLE t; diff --git a/mysql-test/suite/mariabackup/incremental_backup.test b/mysql-test/suite/mariabackup/incremental_backup.test new file mode 100644 index 00000000000..b92d7b323ac --- /dev/null +++ b/mysql-test/suite/mariabackup/incremental_backup.test @@ -0,0 +1,35 @@ +call mtr.add_suppression("InnoDB: New log files created"); + + +let $basedir=$MYSQLTEST_VARDIR/tmp/backup; +let $incremental_dir=$MYSQLTEST_VARDIR/tmp/backup_inc1; + + +CREATE TABLE t(i INT) ENGINE INNODB; +INSERT INTO t VALUES(1); + +echo # Create full backup , modify table, then create incremental/differential backup; +--disable_result_log +exec $XTRABACKUP --defaults-file=$MYSQLTEST_VARDIR/my.cnf --backup --target-dir=$basedir; +--enable_result_log +INSERT INTO t VALUES(2); +SELECT * FROM t; +exec $XTRABACKUP --defaults-file=$MYSQLTEST_VARDIR/my.cnf --backup --target-dir=$incremental_dir --incremental-basedir=$basedir; + +--disable_result_log +echo # Prepare full backup, apply incremental one; +exec $XTRABACKUP --prepare --apply-log-only --target-dir=$basedir; +exec $XTRABACKUP --prepare --target-dir=$basedir --incremental-dir=$incremental_dir ; + +echo # Restore and check results; +let $targetdir=$basedir; +-- source include/restart_and_restore.inc +--enable_result_log + +SELECT * FROM t; +DROP TABLE t; + +# Cleanup +rmdir $basedir; +rmdir $incremental_dir; + diff --git a/mysql-test/suite/mariabackup/incremental_encrypted.opt b/mysql-test/suite/mariabackup/incremental_encrypted.opt new file mode 100644 index 00000000000..ea644cef6ce --- /dev/null +++ b/mysql-test/suite/mariabackup/incremental_encrypted.opt @@ -0,0 +1,3 @@ +--innodb-tablespaces-encryption +--plugin-load-add=$DEBUG_KEY_MANAGEMENT_SO +--loose-debug_key_management_version=2 diff --git a/mysql-test/suite/mariabackup/incremental_encrypted.result b/mysql-test/suite/mariabackup/incremental_encrypted.result new file mode 100644 index 00000000000..e8f81e9fa49 --- /dev/null +++ b/mysql-test/suite/mariabackup/incremental_encrypted.result @@ -0,0 +1,20 @@ +call mtr.add_suppression("InnoDB: New log files created"); +CREATE TABLE t(i INT) ENGINE INNODB ENCRYPTED=YES; +INSERT INTO t VALUES(1); +# Create full backup , modify table, then create incremental/differential backup +INSERT INTO t VALUES(2); +SELECT * FROM t; +i +1 +2 +# Prepare full backup, apply incremental one +# Restore and check results +# shutdown server +# remove datadir +# xtrabackup move back +# restart server +SELECT * FROM t; +i +1 +2 +DROP TABLE t; diff --git a/mysql-test/suite/mariabackup/incremental_encrypted.test b/mysql-test/suite/mariabackup/incremental_encrypted.test new file mode 100644 index 00000000000..c379b3c8165 --- /dev/null +++ b/mysql-test/suite/mariabackup/incremental_encrypted.test @@ -0,0 +1,45 @@ +if (!$EXAMPLE_KEY_MANAGEMENT_SO) +{ + --skip needs example_key_management plugin +} +call mtr.add_suppression("InnoDB: New log files created"); + + +let $basedir=$MYSQLTEST_VARDIR/tmp/backup; +let $incremental_dir=$MYSQLTEST_VARDIR/tmp/backup_inc1; + + +CREATE TABLE t(i INT) ENGINE INNODB ENCRYPTED=YES; +INSERT INTO t VALUES(1); + + +echo # Create full backup , modify table, then create incremental/differential backup; +--disable_result_log +exec $XTRABACKUP --defaults-file=$MYSQLTEST_VARDIR/my.cnf --backup --target-dir=$basedir; +--enable_result_log + +INSERT INTO t VALUES(2); +SELECT * FROM t; + +--disable_result_log +exec $XTRABACKUP --defaults-file=$MYSQLTEST_VARDIR/my.cnf --backup --target-dir=$incremental_dir --incremental-basedir=$basedir; +echo # Prepare full backup, apply incremental one; +exec $XTRABACKUP --prepare --apply-log-only --target-dir=$basedir; +exec $XTRABACKUP --prepare --target-dir=$basedir --incremental-dir=$incremental_dir; + +# stats also can support encryption, but needs plugin-load and plugin variables, they are stored in backup-my.cnf +# We need to prepare again to create log files though. +exec $XTRABACKUP --prepare --target-dir=$basedir; +exec $XTRABACKUP --defaults-file=$basedir/backup-my.cnf --stats --datadir=$basedir; + +echo # Restore and check results; +let $targetdir=$basedir; +-- source include/restart_and_restore.inc +--enable_result_log + +SELECT * FROM t; +DROP TABLE t; + +# Cleanup +rmdir $basedir; +rmdir $incremental_dir; diff --git a/mysql-test/suite/mariabackup/partial.result b/mysql-test/suite/mariabackup/partial.result new file mode 100644 index 00000000000..98c59be91bb --- /dev/null +++ b/mysql-test/suite/mariabackup/partial.result @@ -0,0 +1,13 @@ +CREATE TABLE t1(i INT) ENGINE INNODB; +INSERT INTO t1 VALUES(1); +CREATE TABLE t2(i int) ENGINE INNODB; +# xtrabackup backup +t1.ibd +# xtrabackup prepare +ALTER TABLE t1 DISCARD TABLESPACE; +ALTER TABLE t1 IMPORT TABLESPACE; +SELECT * FROM t1; +i +1 +DROP TABLE t1; +DROP TABLE t2; diff --git a/mysql-test/suite/mariabackup/partial.test b/mysql-test/suite/mariabackup/partial.test new file mode 100644 index 00000000000..3b1de8ebc25 --- /dev/null +++ b/mysql-test/suite/mariabackup/partial.test @@ -0,0 +1,31 @@ +# Export single table from backup +# (xtrabackup with --prepare --export) + +CREATE TABLE t1(i INT) ENGINE INNODB; +INSERT INTO t1 VALUES(1); +CREATE TABLE t2(i int) ENGINE INNODB; + +echo # xtrabackup backup; + +let $targetdir=$MYSQLTEST_VARDIR/tmp/backup; +--disable_result_log +exec $XTRABACKUP --defaults-file=$MYSQLTEST_VARDIR/my.cnf --backup "--tables=test.*1" --target-dir=$targetdir; +--enable_result_log +list_files $targetdir/test *.ibd; + +echo # xtrabackup prepare; +--disable_result_log +exec $XTRABACKUP --prepare --export --target-dir=$targetdir; +--enable_result_log + +ALTER TABLE t1 DISCARD TABLESPACE; +let $MYSQLD_DATADIR= `select @@datadir`; +copy_file $targetdir/test/t1.ibd $MYSQLD_DATADIR/test/t1.ibd; +copy_file $targetdir/test/t1.cfg $MYSQLD_DATADIR/test/t1.cfg; +ALTER TABLE t1 IMPORT TABLESPACE; + +SELECT * FROM t1; +DROP TABLE t1; +DROP TABLE t2; +rmdir $targetdir; + diff --git a/mysql-test/suite/mariabackup/small_ibd.result b/mysql-test/suite/mariabackup/small_ibd.result new file mode 100644 index 00000000000..1c840a7a1b6 --- /dev/null +++ b/mysql-test/suite/mariabackup/small_ibd.result @@ -0,0 +1 @@ +#backup diff --git a/mysql-test/suite/mariabackup/small_ibd.test b/mysql-test/suite/mariabackup/small_ibd.test new file mode 100644 index 00000000000..23d6b08c047 --- /dev/null +++ b/mysql-test/suite/mariabackup/small_ibd.test @@ -0,0 +1,18 @@ +# Check if ibd smaller than page size are skipped +# It is possible, due to race conditions that new file +# is created by server while xtrabackup is running +# The first page in this file does not yet exist. +# xtrabackup should skip such file. + +let $_datadir= `SELECT @@datadir`; +write_file $_datadir/test/small.ibd; +EOF +echo #backup; + +let $targetdir=$MYSQLTEST_VARDIR/tmp/backup; +--disable_result_log +exec $XTRABACKUP --defaults-file=$MYSQLTEST_VARDIR/my.cnf --backup --target-dir=$targetdir; +--enable_result_log +remove_file $_datadir/test/small.ibd; +rmdir $targetdir; + diff --git a/mysql-test/suite/mariabackup/suite.opt b/mysql-test/suite/mariabackup/suite.opt new file mode 100644 index 00000000000..ec00a407620 --- /dev/null +++ b/mysql-test/suite/mariabackup/suite.opt @@ -0,0 +1 @@ +--innodb --changed_page_bitmaps --innodb-file-format=Barracuda \ No newline at end of file diff --git a/mysql-test/suite/mariabackup/suite.pm b/mysql-test/suite/mariabackup/suite.pm new file mode 100644 index 00000000000..8eecd4e8018 --- /dev/null +++ b/mysql-test/suite/mariabackup/suite.pm @@ -0,0 +1,38 @@ +package My::Suite::MariaBackup; + +@ISA = qw(My::Suite); +use My::Find; +use File::Basename; +use strict; + +return "Not run for embedded server" if $::opt_embedded_server; + +my $mariabackup_exe= +::mtr_exe_maybe_exists( + "$::bindir/extra/mariabackup$::opt_vs_config/mariabackup", + "$::path_client_bindir/mariabackup"); + +return "No mariabackup" if !$mariabackup_exe; + + +$ENV{XTRABACKUP}= $mariabackup_exe; + +$ENV{XBSTREAM}= ::mtr_exe_maybe_exists( + "$::bindir/extra/mariabackup/$::opt_vs_config/mbstream", + "$::path_client_bindir/mbstream"); + +my $tar_version = `tar --version 2>&1`; +$ENV{HAVE_TAR} = $! ? 0: 1; +my $mariabackup_help=`$mariabackup_exe --help 2>&1`; +$ENV{HAVE_XTRABACKUP_TAR_SUPPORT} = (index($mariabackup_help,"'tar'") == -1) ? 0 : 1; + +$ENV{INNOBACKUPEX}= "$mariabackup_exe --innobackupex"; + +sub skip_combinations { + my %skip; + $skip{'include/have_file_key_management.inc'} = 'needs file_key_management plugin' unless $ENV{FILE_KEY_MANAGEMENT_SO}; + %skip; +} + +bless { }; + diff --git a/mysql-test/suite/mariabackup/tar.result b/mysql-test/suite/mariabackup/tar.result new file mode 100644 index 00000000000..bbb546d7add --- /dev/null +++ b/mysql-test/suite/mariabackup/tar.result @@ -0,0 +1,12 @@ +CREATE TABLE t(i INT) ENGINE INNODB; +INSERT INTO t VALUES(1); +# xtrabackup backup +# xtrabackup prepare +# shutdown server +# remove datadir +# xtrabackup move back +# restart server +SELECT * FROM t; +i +1 +DROP TABLE t; diff --git a/mysql-test/suite/mariabackup/tar.test b/mysql-test/suite/mariabackup/tar.test new file mode 100644 index 00000000000..3938d597e05 --- /dev/null +++ b/mysql-test/suite/mariabackup/tar.test @@ -0,0 +1,30 @@ +if (`select $HAVE_TAR = 0`) +{ + --skip No tar +} +if (`select $HAVE_XTRABACKUP_TAR_SUPPORT = 0`) +{ + --skip Compiled without libarchive +} + + +CREATE TABLE t(i INT) ENGINE INNODB; +INSERT INTO t VALUES(1); + +echo # xtrabackup backup; +let $targetdir=$MYSQLTEST_VARDIR/tmp/backup; +let $streamfile=$MYSQLTEST_VARDIR/tmp/backup.tar; +mkdir $targetdir; + + +exec $XTRABACKUP "--defaults-file=$MYSQLTEST_VARDIR/my.cnf" --backup --stream=tar > $streamfile 2>$targetdir/backup_stream.log; +--disable_result_log +exec tar -C $targetdir -x < $streamfile; +echo # xtrabackup prepare; +exec $XTRABACKUP --prepare --target-dir=$targetdir; + +-- source include/restart_and_restore.inc +--enable_result_log +SELECT * FROM t; +DROP TABLE t; +rmdir $targetdir; diff --git a/mysql-test/suite/mariabackup/xb_aws_key_management.opt b/mysql-test/suite/mariabackup/xb_aws_key_management.opt new file mode 100644 index 00000000000..62d4f864073 --- /dev/null +++ b/mysql-test/suite/mariabackup/xb_aws_key_management.opt @@ -0,0 +1,3 @@ +--plugin-load-add=$AWS_KEY_MANAGEMENT_SO +--loose-aws-key-management +--loose-aws-key-management-master-key-id=$AWS_KEY_MANAGEMENT_MASTER_KEY_ID diff --git a/mysql-test/suite/mariabackup/xb_aws_key_management.result b/mysql-test/suite/mariabackup/xb_aws_key_management.result new file mode 100644 index 00000000000..ccad423f631 --- /dev/null +++ b/mysql-test/suite/mariabackup/xb_aws_key_management.result @@ -0,0 +1,11 @@ +CREATE TABLE t(c VARCHAR(10)) ENGINE INNODB encrypted=yes; +INSERT INTO t VALUES('foobar1'); +# xtrabackup backup +# shutdown server +# remove datadir +# xtrabackup move back +# restart server +SELECT * from t; +c +foobar1 +DROP TABLE t; diff --git a/mysql-test/suite/mariabackup/xb_aws_key_management.test b/mysql-test/suite/mariabackup/xb_aws_key_management.test new file mode 100644 index 00000000000..ca01be607c7 --- /dev/null +++ b/mysql-test/suite/mariabackup/xb_aws_key_management.test @@ -0,0 +1,22 @@ +if (`SELECT COUNT(*)=0 FROM INFORMATION_SCHEMA.PLUGINS WHERE PLUGIN_NAME = 'aws_key_management' AND PLUGIN_STATUS='ACTIVE'`) +{ + --skip needs aws_key_management plugin plugin +} + +if (`SELECT @@aws_key_management_master_key_id=''`) +{ + --skip Test requires AWS_KEY_MANAGEMEMENT_MASTER_KEY_ID env. variable +} + +CREATE TABLE t(c VARCHAR(10)) ENGINE INNODB encrypted=yes; +INSERT INTO t VALUES('foobar1'); +echo # xtrabackup backup; +let $targetdir=$MYSQLTEST_VARDIR/tmp/backup; +--disable_result_log +exec $XTRABACKUP --defaults-file=$MYSQLTEST_VARDIR/my.cnf --backup --target-dir=$targetdir; +exec $XTRABACKUP --prepare --target-dir=$targetdir; +-- source include/restart_and_restore.inc +--enable_result_log +SELECT * from t; +DROP TABLE t; +rmdir $targetdir; diff --git a/mysql-test/suite/mariabackup/xb_compressed_encrypted.opt b/mysql-test/suite/mariabackup/xb_compressed_encrypted.opt new file mode 100644 index 00000000000..e6cbe00bb7c --- /dev/null +++ b/mysql-test/suite/mariabackup/xb_compressed_encrypted.opt @@ -0,0 +1,9 @@ +--innodb-encryption-rotate-key-age=2 +--innodb-encryption-threads=4 +--innodb-tablespaces-encryption +--plugin-load-add=$FILE_KEY_MANAGEMENT_SO +--loose-file-key-management +--loose-file-key-management-filename=$MYSQL_TEST_DIR/std_data/logkey.txt +--innodb_strict_mode +--innodb_file_per_table +--innodb_file_format=Barracuda diff --git a/mysql-test/suite/mariabackup/xb_compressed_encrypted.result b/mysql-test/suite/mariabackup/xb_compressed_encrypted.result new file mode 100644 index 00000000000..c0eb0e70631 --- /dev/null +++ b/mysql-test/suite/mariabackup/xb_compressed_encrypted.result @@ -0,0 +1,25 @@ +CREATE TABLE t1(c1 INT, b VARCHAR(2400), index(b(100),c1)) ENGINE=INNODB ROW_FORMAT=compressed ENCRYPTED=YES; +CREATE PROCEDURE innodb_insert_proc (REPEAT_COUNT INT) +BEGIN +DECLARE CURRENT_NUM INT; +SET CURRENT_NUM = 0; +WHILE CURRENT_NUM < REPEAT_COUNT DO +INSERT INTO t1 VALUES(CURRENT_NUM, concat(uuid(), CURRENT_NUM, repeat('ab', floor(rand()*100) ), uuid())); +SET CURRENT_NUM = CURRENT_NUM + 1; +END WHILE; +END// +COMMIT; +SET AUTOCOMMIT=0; +CALL innodb_insert_proc(50000); +COMMIT; +# xtrabackup backup +drop table t1; +# shutdown server +# remove datadir +# xtrabackup move back +# restart server +select sum(c1) from t1; +sum(c1) +1249975000 +DROP TABLE t1; +drop procedure innodb_insert_proc; diff --git a/mysql-test/suite/mariabackup/xb_compressed_encrypted.test b/mysql-test/suite/mariabackup/xb_compressed_encrypted.test new file mode 100644 index 00000000000..11f63eb0330 --- /dev/null +++ b/mysql-test/suite/mariabackup/xb_compressed_encrypted.test @@ -0,0 +1,35 @@ +source include/have_file_key_management.inc; + +CREATE TABLE t1(c1 INT, b VARCHAR(2400), index(b(100),c1)) ENGINE=INNODB ROW_FORMAT=compressed ENCRYPTED=YES; + +DELIMITER //; +CREATE PROCEDURE innodb_insert_proc (REPEAT_COUNT INT) +BEGIN + DECLARE CURRENT_NUM INT; + SET CURRENT_NUM = 0; + WHILE CURRENT_NUM < REPEAT_COUNT DO + INSERT INTO t1 VALUES(CURRENT_NUM, concat(uuid(), CURRENT_NUM, repeat('ab', floor(rand()*100) ), uuid())); + SET CURRENT_NUM = CURRENT_NUM + 1; + END WHILE; +END// +DELIMITER ;// +COMMIT; + +SET AUTOCOMMIT=0; +CALL innodb_insert_proc(50000); +COMMIT; + + +echo # xtrabackup backup; +--disable_result_log +let $targetdir=$MYSQLTEST_VARDIR/tmp/backup; +exec $INNOBACKUPEX --defaults-file=$MYSQLTEST_VARDIR/my.cnf --no-timestamp $targetdir; +drop table t1; +exec $INNOBACKUPEX --apply-log $targetdir; + +-- source include/restart_and_restore.inc +--enable_result_log +select sum(c1) from t1; +DROP TABLE t1; +drop procedure innodb_insert_proc; +rmdir $targetdir; diff --git a/mysql-test/suite/mariabackup/xb_file_key_management.opt b/mysql-test/suite/mariabackup/xb_file_key_management.opt new file mode 100644 index 00000000000..74a6450a1ef --- /dev/null +++ b/mysql-test/suite/mariabackup/xb_file_key_management.opt @@ -0,0 +1,6 @@ +--innodb-encrypt-log=ON +--plugin-load-add=$FILE_KEY_MANAGEMENT_SO +--loose-file-key-management +--loose-file-key-management-filekey=FILE:$MTR_SUITE_DIR/filekeys-data.key +--loose-file-key-management-filename=$MTR_SUITE_DIR/filekeys-data.enc +--loose-file-key-management-encryption-algorithm=aes_cbc diff --git a/mysql-test/suite/mariabackup/xb_file_key_management.result b/mysql-test/suite/mariabackup/xb_file_key_management.result new file mode 100644 index 00000000000..30aa530698b --- /dev/null +++ b/mysql-test/suite/mariabackup/xb_file_key_management.result @@ -0,0 +1,17 @@ +CREATE TABLE t(c VARCHAR(10)) ENGINE INNODB encrypted=yes; +INSERT INTO t VALUES('foobar1'); +# xtrabackup backup +NOT FOUND /foobar1/ in xtrabackup_logfile +# expect NOT FOUND +INSERT INTO t VALUES('foobar2'); +# xtrabackup prepare +# shutdown server +# remove datadir +# xtrabackup move back +# restart server +NOT FOUND /foobar1/ in xtrabackup_logfile +# expect NOT FOUND +SELECT * FROM t; +c +foobar1 +DROP TABLE t; diff --git a/mysql-test/suite/mariabackup/xb_file_key_management.test b/mysql-test/suite/mariabackup/xb_file_key_management.test new file mode 100644 index 00000000000..bc975a7cdc4 --- /dev/null +++ b/mysql-test/suite/mariabackup/xb_file_key_management.test @@ -0,0 +1,40 @@ +source include/have_file_key_management.inc; + +CREATE TABLE t(c VARCHAR(10)) ENGINE INNODB encrypted=yes; +INSERT INTO t VALUES('foobar1'); +echo # xtrabackup backup; +let $targetdir=$MYSQLTEST_VARDIR/tmp/backup; +--disable_result_log +exec $XTRABACKUP --defaults-file=$MYSQLTEST_VARDIR/my.cnf --backup --target-dir=$targetdir; +--enable_result_log + +--let SEARCH_RANGE = 10000000 +--let SEARCH_PATTERN=foobar1 +--let SEARCH_FILE=$targetdir/xtrabackup_logfile +--source include/search_pattern_in_file.inc +--echo # expect NOT FOUND + +INSERT INTO t VALUES('foobar2'); +echo # xtrabackup prepare; + +--disable_result_log +exec $XTRABACKUP --prepare --target-dir=$targetdir; +exec $XTRABACKUP --defaults-file=$targetdir/backup-my.cnf --stats --datadir=$targetdir ; +-- source include/restart_and_restore.inc +--enable_result_log + +# +# Recheck that plain text data ( +# in not in the log, after prepare +# (MDEV-11538) + +--let SEARCH_RANGE = 10000000 +--let SEARCH_PATTERN=foobar1 +--let SEARCH_FILE=$targetdir/xtrabackup_logfile +--source include/search_pattern_in_file.inc +--echo # expect NOT FOUND + +SELECT * FROM t; +DROP TABLE t; +rmdir $targetdir; + diff --git a/mysql-test/suite/mariabackup/xb_fulltext_encrypted.opt b/mysql-test/suite/mariabackup/xb_fulltext_encrypted.opt new file mode 100644 index 00000000000..b3ef366361a --- /dev/null +++ b/mysql-test/suite/mariabackup/xb_fulltext_encrypted.opt @@ -0,0 +1,8 @@ +--plugin-load-add=$FILE_KEY_MANAGEMENT_SO +--innodb_strict_mode +--innodb_file_per_table +--innodb-encryption-rotate-key-age=2 +--innodb-encryption-threads=4 +--innodb-tablespaces-encryption +--loose-file-key-management +--loose-file-key-management-filename=$MYSQL_TEST_DIR/std_data/logkey.txt \ No newline at end of file diff --git a/mysql-test/suite/mariabackup/xb_fulltext_encrypted.result b/mysql-test/suite/mariabackup/xb_fulltext_encrypted.result new file mode 100644 index 00000000000..01a99e59200 --- /dev/null +++ b/mysql-test/suite/mariabackup/xb_fulltext_encrypted.result @@ -0,0 +1,14 @@ +CREATE TABLE film_text ( +film_id SMALLINT NOT NULL, +title VARCHAR(255) NOT NULL, +description TEXT, +PRIMARY KEY (film_id), +FULLTEXT KEY idx_title_description (title,description), +FULLTEXT KEY (description), +FULLTEXT KEY (title) +)ENGINE=InnoDB DEFAULT CHARSET=utf8 ENCRYPTED=YES; +# shutdown server +# remove datadir +# xtrabackup move back +# restart server +drop table film_text; diff --git a/mysql-test/suite/mariabackup/xb_fulltext_encrypted.test b/mysql-test/suite/mariabackup/xb_fulltext_encrypted.test new file mode 100644 index 00000000000..a98d7802d76 --- /dev/null +++ b/mysql-test/suite/mariabackup/xb_fulltext_encrypted.test @@ -0,0 +1,23 @@ +source include/have_file_key_management.inc; + +CREATE TABLE film_text ( +film_id SMALLINT NOT NULL, +title VARCHAR(255) NOT NULL, +description TEXT, +PRIMARY KEY (film_id), +FULLTEXT KEY idx_title_description (title,description), +FULLTEXT KEY (description), +FULLTEXT KEY (title) +)ENGINE=InnoDB DEFAULT CHARSET=utf8 ENCRYPTED=YES; + +let $targetdir=$MYSQLTEST_VARDIR/tmp/backup; + +--disable_result_log + +exec $INNOBACKUPEX --defaults-file=$MYSQLTEST_VARDIR/my.cnf --no-timestamp $targetdir; +exec $INNOBACKUPEX --apply-log --rebuild-indexes --rebuild-threads=2 $targetdir; +--source include/restart_and_restore.inc + +--enable_result_log + +drop table film_text; diff --git a/mysql-test/suite/mariabackup/xb_history.result b/mysql-test/suite/mariabackup/xb_history.result new file mode 100644 index 00000000000..b6a1ac21147 --- /dev/null +++ b/mysql-test/suite/mariabackup/xb_history.result @@ -0,0 +1,5 @@ +SELECT COUNT(*) FROM PERCONA_SCHEMA.xtrabackup_history; +COUNT(*) +1 +DROP TABLE PERCONA_SCHEMA.xtrabackup_history; +DROP DATABASE PERCONA_SCHEMA; diff --git a/mysql-test/suite/mariabackup/xb_history.test b/mysql-test/suite/mariabackup/xb_history.test new file mode 100644 index 00000000000..28de50127c6 --- /dev/null +++ b/mysql-test/suite/mariabackup/xb_history.test @@ -0,0 +1,8 @@ +let $targetdir=$MYSQLTEST_VARDIR/tmp/backup; +--disable_result_log +exec $XTRABACKUP --defaults-file=$MYSQLTEST_VARDIR/my.cnf --history=foo --backup --target-dir=$targetdir; +--enable_result_log +SELECT COUNT(*) FROM PERCONA_SCHEMA.xtrabackup_history; +DROP TABLE PERCONA_SCHEMA.xtrabackup_history; +DROP DATABASE PERCONA_SCHEMA; +rmdir $targetdir; diff --git a/mysql-test/suite/mariabackup/xb_page_compress.result b/mysql-test/suite/mariabackup/xb_page_compress.result new file mode 100644 index 00000000000..7380856f394 --- /dev/null +++ b/mysql-test/suite/mariabackup/xb_page_compress.result @@ -0,0 +1,28 @@ +CREATE TABLE t1(c1 INT, b CHAR(20)) ENGINE=INNODB PAGE_COMPRESSED=1; +CREATE PROCEDURE innodb_insert_proc (REPEAT_COUNT INT) +BEGIN +DECLARE CURRENT_NUM INT; +SET CURRENT_NUM = 0; +WHILE CURRENT_NUM < REPEAT_COUNT DO +INSERT INTO t1 VALUES(CURRENT_NUM,'TESTING..'); +SET CURRENT_NUM = CURRENT_NUM + 1; +END WHILE; +END// +COMMIT; +SET AUTOCOMMIT=0; +CALL innodb_insert_proc(5000); +COMMIT; +SELECT (VARIABLE_VALUE >= 0) AS HAVE_COMPRESSED_PAGES +FROM INFORMATION_SCHEMA.GLOBAL_STATUS +WHERE VARIABLE_NAME = 'INNODB_NUM_PAGES_PAGE_COMPRESSED'; +HAVE_COMPRESSED_PAGES +1 +# xtrabackup backup +# xtrabackup prepare +ALTER TABLE t1 DISCARD TABLESPACE; +ALTER TABLE t1 IMPORT TABLESPACE; +SELECT COUNT(*) FROM t1; +COUNT(*) +5000 +DROP PROCEDURE innodb_insert_proc; +DROP TABLE t1; diff --git a/mysql-test/suite/mariabackup/xb_page_compress.test b/mysql-test/suite/mariabackup/xb_page_compress.test new file mode 100644 index 00000000000..876aa1a2791 --- /dev/null +++ b/mysql-test/suite/mariabackup/xb_page_compress.test @@ -0,0 +1,44 @@ +CREATE TABLE t1(c1 INT, b CHAR(20)) ENGINE=INNODB PAGE_COMPRESSED=1; + +DELIMITER //; +CREATE PROCEDURE innodb_insert_proc (REPEAT_COUNT INT) +BEGIN + DECLARE CURRENT_NUM INT; + SET CURRENT_NUM = 0; + WHILE CURRENT_NUM < REPEAT_COUNT DO + INSERT INTO t1 VALUES(CURRENT_NUM,'TESTING..'); + SET CURRENT_NUM = CURRENT_NUM + 1; + END WHILE; +END// +DELIMITER ;// +COMMIT; + +SET AUTOCOMMIT=0; +CALL innodb_insert_proc(5000); +COMMIT; + +SELECT (VARIABLE_VALUE >= 0) AS HAVE_COMPRESSED_PAGES + FROM INFORMATION_SCHEMA.GLOBAL_STATUS + WHERE VARIABLE_NAME = 'INNODB_NUM_PAGES_PAGE_COMPRESSED'; + +echo # xtrabackup backup; +let $targetdir=$MYSQLTEST_VARDIR/tmp/backup; + +--disable_result_log +exec $XTRABACKUP --defaults-file=$MYSQLTEST_VARDIR/my.cnf --backup "--tables=test.*1" --target-dir=$targetdir; +echo # xtrabackup prepare; +exec $XTRABACKUP --prepare --export --target-dir=$targetdir; +--enable_result_log + +ALTER TABLE t1 DISCARD TABLESPACE; +let $MYSQLD_DATADIR= `select @@datadir`; +copy_file $targetdir/test/t1.ibd $MYSQLD_DATADIR/test/t1.ibd; +copy_file $targetdir/test/t1.cfg $MYSQLD_DATADIR/test/t1.cfg; +ALTER TABLE t1 IMPORT TABLESPACE; + +SELECT COUNT(*) FROM t1; + +DROP PROCEDURE innodb_insert_proc; +DROP TABLE t1; +rmdir $targetdir; + diff --git a/mysql-test/suite/mariabackup/xb_partition.result b/mysql-test/suite/mariabackup/xb_partition.result new file mode 100644 index 00000000000..f5b6ae0b24d --- /dev/null +++ b/mysql-test/suite/mariabackup/xb_partition.result @@ -0,0 +1,64 @@ +CREATE TABLE t1(a INT) ENGINE=InnoDB; +INSERT INTO t1 VALUES (1), (2), (3); +CREATE TABLE t2(a INT) ENGINE=InnoDB; +INSERT INTO t2 VALUES (4), (5), (6); +CREATE TABLE p ( +a int +) ENGINE=InnoDB +PARTITION BY RANGE (a) +(PARTITION p0 VALUES LESS THAN (100), +PARTITION p1 VALUES LESS THAN (200), +PARTITION p2 VALUES LESS THAN (300), +PARTITION p3 VALUES LESS THAN (400)); +INSERT INTO p VALUES (1), (101), (201), (301); +CREATE TABLE isam_t1(a INT) ENGINE=MyISAM; +INSERT INTO isam_t1 VALUES (1), (2), (3); +CREATE TABLE isam_t2(a INT) ENGINE=MyISAM; +INSERT INTO isam_t2 VALUES (4), (5), (6); +CREATE TABLE isam_p ( +a int +) ENGINE=MyISAM +PARTITION BY RANGE (a) +(PARTITION p0 VALUES LESS THAN (100), +PARTITION p1 VALUES LESS THAN (200), +PARTITION p2 VALUES LESS THAN (300), +PARTITION p3 VALUES LESS THAN (400)); +INSERT INTO isam_p VALUES (1), (101), (201), (301); +DROP TABLE t1; +DROP TABLE t2; +CREATE TABLE t2(a INT) ENGINE=InnoDB; +INSERT INTO t2 VALUES (40), (50), (60); +ALTER TABLE p DROP PARTITION p0; +ALTER TABLE p DROP PARTITION p1; +ALTER TABLE p ADD PARTITION (PARTITION p4 VALUES LESS THAN (500)); +ALTER TABLE p ADD PARTITION (PARTITION p5 VALUES LESS THAN (600)); +INSERT INTO p VALUES (401), (501); +DROP TABLE isam_t1; +DROP TABLE isam_t2; +CREATE TABLE isam_t2(a INT) ENGINE=MyISAM; +INSERT INTO isam_t2 VALUES (40), (50), (60); +ALTER TABLE isam_p DROP PARTITION p0; +ALTER TABLE isam_p DROP PARTITION p1; +ALTER TABLE isam_p ADD PARTITION (PARTITION p4 VALUES LESS THAN (500)); +ALTER TABLE isam_p ADD PARTITION (PARTITION p5 VALUES LESS THAN (600)); +INSERT INTO isam_p VALUES (401), (501); +# shutdown server +# remove datadir +# xtrabackup move back +# restart server +SELECT * from p; +a +201 +301 +401 +501 +SELECT * from isam_p; +a +201 +301 +401 +501 +DROP TABLE isam_p; +DROP TABLE isam_t2; +DROP TABLE p; +DROP TABLE t2; diff --git a/mysql-test/suite/mariabackup/xb_partition.test b/mysql-test/suite/mariabackup/xb_partition.test new file mode 100644 index 00000000000..f051a52edbc --- /dev/null +++ b/mysql-test/suite/mariabackup/xb_partition.test @@ -0,0 +1,87 @@ +--source include/have_partition.inc + +CREATE TABLE t1(a INT) ENGINE=InnoDB; +INSERT INTO t1 VALUES (1), (2), (3); + +CREATE TABLE t2(a INT) ENGINE=InnoDB; +INSERT INTO t2 VALUES (4), (5), (6); + +CREATE TABLE p ( + a int +) ENGINE=InnoDB +PARTITION BY RANGE (a) +(PARTITION p0 VALUES LESS THAN (100), + PARTITION p1 VALUES LESS THAN (200), + PARTITION p2 VALUES LESS THAN (300), + PARTITION p3 VALUES LESS THAN (400)); + +INSERT INTO p VALUES (1), (101), (201), (301); + +CREATE TABLE isam_t1(a INT) ENGINE=MyISAM; +INSERT INTO isam_t1 VALUES (1), (2), (3); + +CREATE TABLE isam_t2(a INT) ENGINE=MyISAM; +INSERT INTO isam_t2 VALUES (4), (5), (6); + +CREATE TABLE isam_p ( + a int +) ENGINE=MyISAM +PARTITION BY RANGE (a) +(PARTITION p0 VALUES LESS THAN (100), + PARTITION p1 VALUES LESS THAN (200), + PARTITION p2 VALUES LESS THAN (300), + PARTITION p3 VALUES LESS THAN (400)); + +INSERT INTO isam_p VALUES (1), (101), (201), (301); + +let $targetdir=$MYSQLTEST_VARDIR/tmp; + +--disable_result_log +exec $XTRABACKUP --defaults-file=$MYSQLTEST_VARDIR/my.cnf --no-timestamp --backup --target-dir=$targetdir/full; +--enable_result_log + +DROP TABLE t1; +DROP TABLE t2; +CREATE TABLE t2(a INT) ENGINE=InnoDB; +INSERT INTO t2 VALUES (40), (50), (60); + +ALTER TABLE p DROP PARTITION p0; +ALTER TABLE p DROP PARTITION p1; +ALTER TABLE p ADD PARTITION (PARTITION p4 VALUES LESS THAN (500)); +ALTER TABLE p ADD PARTITION (PARTITION p5 VALUES LESS THAN (600)); + +INSERT INTO p VALUES (401), (501); + + +DROP TABLE isam_t1; +DROP TABLE isam_t2; +CREATE TABLE isam_t2(a INT) ENGINE=MyISAM; + +INSERT INTO isam_t2 VALUES (40), (50), (60); + +ALTER TABLE isam_p DROP PARTITION p0; +ALTER TABLE isam_p DROP PARTITION p1; +ALTER TABLE isam_p ADD PARTITION (PARTITION p4 VALUES LESS THAN (500)); +ALTER TABLE isam_p ADD PARTITION (PARTITION p5 VALUES LESS THAN (600)); + +INSERT INTO isam_p VALUES (401), (501); + +--disable_result_log +exec $INNOBACKUPEX --defaults-file=$MYSQLTEST_VARDIR/my.cnf --incremental --no-timestamp --incremental-basedir=$targetdir/full $targetdir/inc; +exec $INNOBACKUPEX --defaults-file=$MYSQLTEST_VARDIR/my.cnf --apply-log --redo-only $targetdir/full; +exec $INNOBACKUPEX --defaults-file=$MYSQLTEST_VARDIR/my.cnf --apply-log --redo-only --incremental-dir=$targetdir/inc $targetdir/full; +exec $INNOBACKUPEX --defaults-file=$MYSQLTEST_VARDIR/my.cnf --apply-log $targetdir/full; + +let $targetdir=$targetdir/full; +-- source include/restart_and_restore.inc +--enable_result_log + +SELECT * from p; +SELECT * from isam_p; + +DROP TABLE isam_p; +DROP TABLE isam_t2; +DROP TABLE p; +DROP TABLE t2; +rmdir $MYSQLTEST_VARDIR/tmp/full; +rmdir $MYSQLTEST_VARDIR/tmp/inc; diff --git a/mysql-test/suite/mariabackup/xbstream.result b/mysql-test/suite/mariabackup/xbstream.result new file mode 100644 index 00000000000..f340fedb861 --- /dev/null +++ b/mysql-test/suite/mariabackup/xbstream.result @@ -0,0 +1,13 @@ +CREATE TABLE t(i INT) ENGINE INNODB; +INSERT INTO t VALUES(1); +# xtrabackup backup to stream +# xbstream extract +# xtrabackup prepare +# shutdown server +# remove datadir +# xtrabackup move back +# restart server +SELECT * FROM t; +i +1 +DROP TABLE t; diff --git a/mysql-test/suite/mariabackup/xbstream.test b/mysql-test/suite/mariabackup/xbstream.test new file mode 100644 index 00000000000..f2b4704a87e --- /dev/null +++ b/mysql-test/suite/mariabackup/xbstream.test @@ -0,0 +1,22 @@ +CREATE TABLE t(i INT) ENGINE INNODB; +INSERT INTO t VALUES(1); + +let $targetdir=$MYSQLTEST_VARDIR/tmp/backup; +mkdir $targetdir; +let $streamfile=$MYSQLTEST_VARDIR/tmp/backup.xb; + +echo # xtrabackup backup to stream; +exec $XTRABACKUP --defaults-file=$MYSQLTEST_VARDIR/my.cnf --backup --stream=xbstream > $streamfile 2>$targetdir/backup_stream.log; +echo # xbstream extract; +--disable_result_log +exec $XBSTREAM -x -C $targetdir < $streamfile; + +echo # xtrabackup prepare; +exec $XTRABACKUP --prepare --target-dir=$targetdir; + +-- source include/restart_and_restore.inc +--enable_result_log +SELECT * FROM t; +DROP TABLE t; +rmdir $targetdir; + From 3d8aacba861044843417b710fff208b8bd6a53c4 Mon Sep 17 00:00:00 2001 From: Nirbhay Choubey Date: Wed, 22 Feb 2017 15:58:45 -0500 Subject: [PATCH 13/28] SST script for mariabackup. Execute mariabackup in innobackupex mode to avoid "unknown argument" error. --- debian/mariadb-server-10.1.files.in | 1 + scripts/CMakeLists.txt | 1 + scripts/wsrep_sst_mariabackup.sh | 1040 +++++++++++++++++++++++++++ 3 files changed, 1042 insertions(+) create mode 100644 scripts/wsrep_sst_mariabackup.sh diff --git a/debian/mariadb-server-10.1.files.in b/debian/mariadb-server-10.1.files.in index f3c048ad3d7..3baf73c5e50 100644 --- a/debian/mariadb-server-10.1.files.in +++ b/debian/mariadb-server-10.1.files.in @@ -57,6 +57,7 @@ usr/bin/wsrep_sst_mysqldump usr/bin/wsrep_sst_rsync usr/bin/wsrep_sst_xtrabackup usr/bin/wsrep_sst_xtrabackup-v2 +usr/bin/wsrep_sst_mariabackup usr/share/doc/mariadb-server-10.1/mysqld.sym.gz usr/share/doc/mariadb-server-10.1/INFO_SRC usr/share/doc/mariadb-server-10.1/INFO_BIN diff --git a/scripts/CMakeLists.txt b/scripts/CMakeLists.txt index de612c2e312..17db2b0ba9d 100644 --- a/scripts/CMakeLists.txt +++ b/scripts/CMakeLists.txt @@ -295,6 +295,7 @@ ELSE() wsrep_sst_rsync wsrep_sst_xtrabackup wsrep_sst_xtrabackup-v2 + wsrep_sst_mariabackup ) ENDIF() IF (NOT WITHOUT_SERVER) diff --git a/scripts/wsrep_sst_mariabackup.sh b/scripts/wsrep_sst_mariabackup.sh new file mode 100644 index 00000000000..9e3fc54290d --- /dev/null +++ b/scripts/wsrep_sst_mariabackup.sh @@ -0,0 +1,1040 @@ +#!/bin/bash -ue +# Copyright (C) 2013 Percona Inc +# Copyright (C) 2017 MariaDB +# +# This program is free software; you can redistribute it and/or modify +# it under the terms of the GNU General Public License as published by +# the Free Software Foundation; version 2 of the License. +# +# This program is distributed in the hope that it will be useful, +# but WITHOUT ANY WARRANTY; without even the implied warranty of +# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the +# GNU General Public License for more details. +# +# You should have received a copy of the GNU General Public License +# along with this program; see the file COPYING. If not, write to the +# Free Software Foundation, Inc., 51 Franklin St, Fifth Floor, Boston +# MA 02110-1301 USA. + +# Documentation: +# http://www.percona.com/doc/percona-xtradb-cluster/manual/xtrabackup_sst.html +# Make sure to read that before proceeding! + +. $(dirname $0)/wsrep_sst_common + +ealgo="" +ekey="" +ekeyfile="" +encrypt=0 +nproc=1 +ecode=0 +ssyslog="" +ssystag="" +XTRABACKUP_PID="" +SST_PORT="" +REMOTEIP="" +REMOTEHOST="" +tcert="" +tpem="" +tkey="" +sockopt="" +progress="" +ttime=0 +totime=0 +lsn="" +ecmd="" +rlimit="" +# Initially +stagemsg="${WSREP_SST_OPT_ROLE}" +cpat="" +speciald=1 +ib_home_dir="" +ib_log_dir="" +ib_undo_dir="" + +sfmt="tar" +strmcmd="" +tfmt="" +tcmd="" +rebuild=0 +rebuildcmd="" +payload=0 +pvformat="-F '%N => Rate:%r Avg:%a Elapsed:%t %e Bytes: %b %p' " +pvopts="-f -i 10 -N $WSREP_SST_OPT_ROLE " +STATDIR="" +uextra=0 +disver="" + +tmpopts="" +itmpdir="" +xtmpdir="" + +scomp="" +sdecomp="" + +# Required for backup locks +# For backup locks it is 1 sent by joiner +# 5.6.21 PXC and later can't donate to an older joiner +sst_ver=1 + +if which pv &>/dev/null && pv --help | grep -q FORMAT;then + pvopts+=$pvformat +fi +pcmd="pv $pvopts" +declare -a RC + +INNOBACKUPEX_BIN=mariabackup +XBSTREAM_BIN=mbstream +XBCRYPT_BIN=xbcrypt # Not available in MariaBackup + +DATA="${WSREP_SST_OPT_DATA}" +INFO_FILE="xtrabackup_galera_info" +IST_FILE="xtrabackup_ist" +MAGIC_FILE="${DATA}/${INFO_FILE}" + +# Setting the path for ss and ip +export PATH="/usr/sbin:/sbin:$PATH" + +timeit(){ + local stage=$1 + shift + local cmd="$@" + local x1 x2 took extcode + + if [[ $ttime -eq 1 ]];then + x1=$(date +%s) + wsrep_log_info "Evaluating $cmd" + eval "$cmd" + extcode=$? + x2=$(date +%s) + took=$(( x2-x1 )) + wsrep_log_info "NOTE: $stage took $took seconds" + totime=$(( totime+took )) + else + wsrep_log_info "Evaluating $cmd" + eval "$cmd" + extcode=$? + fi + return $extcode +} + +get_keys() +{ + # $encrypt -eq 1 is for internal purposes only + if [[ $encrypt -ge 2 || $encrypt -eq -1 ]];then + return + fi + + if [[ $encrypt -eq 0 ]];then + if $MY_PRINT_DEFAULTS xtrabackup | grep -q encrypt;then + wsrep_log_error "Unexpected option combination. SST may fail. Refer to http://www.percona.com/doc/percona-xtradb-cluster/manual/xtrabackup_sst.html " + fi + return + fi + + if [[ $sfmt == 'tar' ]];then + wsrep_log_info "NOTE: Xtrabackup-based encryption - encrypt=1 - cannot be enabled with tar format" + encrypt=-1 + return + fi + + wsrep_log_info "Xtrabackup based encryption enabled in my.cnf - Supported only from Xtrabackup 2.1.4" + + if [[ -z $ealgo ]];then + wsrep_log_error "FATAL: Encryption algorithm empty from my.cnf, bailing out" + exit 3 + fi + + if [[ -z $ekey && ! -r $ekeyfile ]];then + wsrep_log_error "FATAL: Either key or keyfile must be readable" + exit 3 + fi + + if [[ -z $ekey ]];then + ecmd="${XBCRYPT_BIN} --encrypt-algo=$ealgo --encrypt-key-file=$ekeyfile" + else + ecmd="${XBCRYPT_BIN} --encrypt-algo=$ealgo --encrypt-key=$ekey" + fi + + if [[ "$WSREP_SST_OPT_ROLE" == "joiner" ]];then + ecmd+=" -d" + fi + + stagemsg+="-XB-Encrypted" +} + +get_transfer() +{ + if [[ -z $SST_PORT ]];then + TSST_PORT=4444 + else + TSST_PORT=$SST_PORT + fi + + if [[ $tfmt == 'nc' ]];then + if [[ ! -x `which nc` ]];then + wsrep_log_error "nc(netcat) not found in path: $PATH" + exit 2 + fi + wsrep_log_info "Using netcat as streamer" + if [[ "$WSREP_SST_OPT_ROLE" == "joiner" ]];then + if nc -h 2>&1 | grep -q ncat;then + tcmd="nc -l ${TSST_PORT}" + else + tcmd="nc -dl ${TSST_PORT}" + fi + else + tcmd="nc ${REMOTEIP} ${TSST_PORT}" + fi + else + tfmt='socat' + wsrep_log_info "Using socat as streamer" + if [[ ! -x `which socat` ]];then + wsrep_log_error "socat not found in path: $PATH" + exit 2 + fi + + if [[ $encrypt -eq 2 || $encrypt -eq 3 ]] && ! socat -V | grep -q "WITH_OPENSSL 1";then + wsrep_log_error "Encryption requested, but socat is not OpenSSL enabled (encrypt=$encrypt)" + exit 2 + fi + + if [[ $encrypt -eq 2 ]];then + wsrep_log_info "Using openssl based encryption with socat: with crt and pem" + if [[ -z $tpem || -z $tcert ]];then + wsrep_log_error "Both PEM and CRT files required" + exit 22 + fi + stagemsg+="-OpenSSL-Encrypted-2" + if [[ "$WSREP_SST_OPT_ROLE" == "joiner" ]];then + wsrep_log_info "Decrypting with cert=${tpem}, cafile=${tcert}" + tcmd="socat -u openssl-listen:${TSST_PORT},reuseaddr,cert=${tpem},cafile=${tcert}${sockopt} stdio" + else + wsrep_log_info "Encrypting with cert=${tpem}, cafile=${tcert}" + tcmd="socat -u stdio openssl-connect:${REMOTEHOST}:${TSST_PORT},cert=${tpem},cafile=${tcert}${sockopt}" + fi + elif [[ $encrypt -eq 3 ]];then + wsrep_log_info "Using openssl based encryption with socat: with key and crt" + if [[ -z $tpem || -z $tkey ]];then + wsrep_log_error "Both certificate and key files required" + exit 22 + fi + stagemsg+="-OpenSSL-Encrypted-3" + if [[ "$WSREP_SST_OPT_ROLE" == "joiner" ]];then + if [[ -z $tcert ]];then + wsrep_log_info "Decrypting with cert=${tpem}, key=${tkey}, verify=0" + tcmd="socat -u openssl-listen:${TSST_PORT},reuseaddr,cert=${tpem},key=${tkey},verify=0${sockopt} stdio" + else + wsrep_log_info "Decrypting with cert=${tpem}, key=${tkey}, cafile=${tcert}" + tcmd="socat -u openssl-listen:${TSST_PORT},reuseaddr,cert=${tpem},key=${tkey},cafile=${tcert}${sockopt} stdio" + fi + else + if [[ -z $tcert ]];then + wsrep_log_info "Encrypting with cert=${tpem}, key=${tkey}, verify=0" + tcmd="socat -u stdio openssl-connect:${REMOTEIP}:${TSST_PORT},cert=${tpem},key=${tkey},verify=0${sockopt}" + else + wsrep_log_info "Encrypting with cert=${tpem}, key=${tkey}, cafile=${tcert}" + tcmd="socat -u stdio openssl-connect:${REMOTEHOST}:${TSST_PORT},cert=${tpem},key=${tkey},cafile=${tcert}${sockopt}" + fi + fi + + else + if [[ "$WSREP_SST_OPT_ROLE" == "joiner" ]];then + tcmd="socat -u TCP-LISTEN:${TSST_PORT},reuseaddr${sockopt} stdio" + else + tcmd="socat -u stdio TCP:${REMOTEIP}:${TSST_PORT}${sockopt}" + fi + fi + fi + +} + +parse_cnf() +{ + local group=$1 + local var=$2 + # print the default settings for given group using my_print_default. + # normalize the variable names specified in cnf file (user can use _ or - for example log-bin or log_bin) + # then grep for needed variable + # finally get the variable value (if variables has been specified multiple time use the last value only) + reval=$($MY_PRINT_DEFAULTS $group | awk -F= '{if ($1 ~ /_/) { gsub(/_/,"-",$1); print $1"="$2 } else { print $0 }}' | grep -- "--$var=" | cut -d= -f2- | tail -1) + if [[ -z $reval ]];then + [[ -n $3 ]] && reval=$3 + fi + echo $reval +} + +get_footprint() +{ + pushd $WSREP_SST_OPT_DATA 1>/dev/null + payload=$(find . -regex '.*\.ibd$\|.*\.MYI$\|.*\.MYD$\|.*ibdata1$' -type f -print0 | du --files0-from=- --block-size=1 -c | awk 'END { print $1 }') + if $MY_PRINT_DEFAULTS xtrabackup | grep -q -- "--compress";then + # QuickLZ has around 50% compression ratio + # When compression/compaction used, the progress is only an approximate. + payload=$(( payload*1/2 )) + fi + popd 1>/dev/null + pcmd+=" -s $payload" + adjust_progress +} + +adjust_progress() +{ + + if [[ ! -x `which pv` ]];then + wsrep_log_error "pv not found in path: $PATH" + wsrep_log_error "Disabling all progress/rate-limiting" + pcmd="" + rlimit="" + progress="" + return + fi + + if [[ -n $progress && $progress != '1' ]];then + if [[ -e $progress ]];then + pcmd+=" 2>>$progress" + else + pcmd+=" 2>$progress" + fi + elif [[ -z $progress && -n $rlimit ]];then + # When rlimit is non-zero + pcmd="pv -q" + fi + + if [[ -n $rlimit && "$WSREP_SST_OPT_ROLE" == "donor" ]];then + wsrep_log_info "Rate-limiting SST to $rlimit" + pcmd+=" -L \$rlimit" + fi +} + +read_cnf() +{ + sfmt=$(parse_cnf sst streamfmt "xbstream") + tfmt=$(parse_cnf sst transferfmt "socat") + tcert=$(parse_cnf sst tca "") + tpem=$(parse_cnf sst tcert "") + tkey=$(parse_cnf sst tkey "") + encrypt=$(parse_cnf sst encrypt 0) + sockopt=$(parse_cnf sst sockopt "") + progress=$(parse_cnf sst progress "") + rebuild=$(parse_cnf sst rebuild 0) + ttime=$(parse_cnf sst time 0) + cpat=$(parse_cnf sst cpat '.*galera\.cache$\|.*sst_in_progress$\|.*\.sst$\|.*gvwstate\.dat$\|.*grastate\.dat$\|.*\.err$\|.*\.log$\|.*RPM_UPGRADE_MARKER$\|.*RPM_UPGRADE_HISTORY$') + ealgo=$(parse_cnf xtrabackup encrypt "") + ekey=$(parse_cnf xtrabackup encrypt-key "") + ekeyfile=$(parse_cnf xtrabackup encrypt-key-file "") + scomp=$(parse_cnf sst compressor "") + sdecomp=$(parse_cnf sst decompressor "") + + # Refer to http://www.percona.com/doc/percona-xtradb-cluster/manual/xtrabackup_sst.html + if [[ -z $ealgo ]];then + ealgo=$(parse_cnf sst encrypt-algo "") + ekey=$(parse_cnf sst encrypt-key "") + ekeyfile=$(parse_cnf sst encrypt-key-file "") + fi + + rlimit=$(parse_cnf sst rlimit "") + uextra=$(parse_cnf sst use-extra 0) + speciald=$(parse_cnf sst sst-special-dirs 1) + iopts=$(parse_cnf sst inno-backup-opts "") + iapts=$(parse_cnf sst inno-apply-opts "") + impts=$(parse_cnf sst inno-move-opts "") + stimeout=$(parse_cnf sst sst-initial-timeout 100) + ssyslog=$(parse_cnf sst sst-syslog 0) + ssystag=$(parse_cnf mysqld_safe syslog-tag "${SST_SYSLOG_TAG:-}") + ssystag+="-" + + if [[ $speciald -eq 0 ]];then + wsrep_log_error "sst-special-dirs equal to 0 is not supported, falling back to 1" + speciald=1 + fi + + if [[ $ssyslog -ne -1 ]];then + if $MY_PRINT_DEFAULTS mysqld_safe | tr '_' '-' | grep -q -- "--syslog";then + ssyslog=1 + fi + fi + + if [[ $encrypt -eq 1 ]]; then + wsrep_log_error "Xtrabackup-based encryption is currently not" \ + "supported with MariaBackup" + exit 2 + fi +} + +get_stream() +{ + if [[ $sfmt == 'xbstream' ]];then + wsrep_log_info "Streaming with xbstream" + if [[ "$WSREP_SST_OPT_ROLE" == "joiner" ]];then + strmcmd="${XBSTREAM_BIN} -x" + else + strmcmd="${XBSTREAM_BIN} -c \${INFO_FILE}" + fi + else + sfmt="tar" + wsrep_log_info "Streaming with tar" + if [[ "$WSREP_SST_OPT_ROLE" == "joiner" ]];then + strmcmd="tar xfi - " + else + strmcmd="tar cf - \${INFO_FILE} " + fi + + fi +} + +get_proc() +{ + set +e + nproc=$(grep -c processor /proc/cpuinfo) + [[ -z $nproc || $nproc -eq 0 ]] && nproc=1 + set -e +} + +sig_joiner_cleanup() +{ + wsrep_log_error "Removing $MAGIC_FILE file due to signal" + rm -f "$MAGIC_FILE" +} + +cleanup_joiner() +{ + # Since this is invoked just after exit NNN + local estatus=$? + if [[ $estatus -ne 0 ]];then + wsrep_log_error "Cleanup after exit with status:$estatus" + elif [ "${WSREP_SST_OPT_ROLE}" = "joiner" ];then + wsrep_log_info "Removing the sst_in_progress file" + wsrep_cleanup_progress_file + fi + if [[ -n $progress && -p $progress ]];then + wsrep_log_info "Cleaning up fifo file $progress" + rm $progress + fi + if [[ -n ${STATDIR:-} ]];then + [[ -d $STATDIR ]] && rm -rf $STATDIR + fi + + # Final cleanup + pgid=$(ps -o pgid= $$ | grep -o '[0-9]*') + + # This means no setsid done in mysqld. + # We don't want to kill mysqld here otherwise. + if [[ $$ -eq $pgid ]];then + + # This means a signal was delivered to the process. + # So, more cleanup. + if [[ $estatus -ge 128 ]];then + kill -KILL -$$ || true + fi + + fi + + exit $estatus +} + +check_pid() +{ + local pid_file="$1" + [ -r "$pid_file" ] && ps -p $(cat "$pid_file") >/dev/null 2>&1 +} + +cleanup_donor() +{ + # Since this is invoked just after exit NNN + local estatus=$? + if [[ $estatus -ne 0 ]];then + wsrep_log_error "Cleanup after exit with status:$estatus" + fi + + if [[ -n ${XTRABACKUP_PID:-} ]];then + if check_pid $XTRABACKUP_PID + then + wsrep_log_error "xtrabackup process is still running. Killing... " + kill_xtrabackup + fi + + fi + rm -f ${DATA}/${IST_FILE} || true + + if [[ -n $progress && -p $progress ]];then + wsrep_log_info "Cleaning up fifo file $progress" + rm -f $progress || true + fi + + wsrep_log_info "Cleaning up temporary directories" + + if [[ -n $xtmpdir ]];then + [[ -d $xtmpdir ]] && rm -rf $xtmpdir || true + fi + + if [[ -n $itmpdir ]];then + [[ -d $itmpdir ]] && rm -rf $itmpdir || true + fi + + # Final cleanup + pgid=$(ps -o pgid= $$ | grep -o '[0-9]*') + + # This means no setsid done in mysqld. + # We don't want to kill mysqld here otherwise. + if [[ $$ -eq $pgid ]];then + + # This means a signal was delivered to the process. + # So, more cleanup. + if [[ $estatus -ge 128 ]];then + kill -KILL -$$ || true + fi + + fi + + exit $estatus + +} + +kill_xtrabackup() +{ + local PID=$(cat $XTRABACKUP_PID) + [ -n "$PID" -a "0" != "$PID" ] && kill $PID && (kill $PID && kill -9 $PID) || : + wsrep_log_info "Removing xtrabackup pid file $XTRABACKUP_PID" + rm -f "$XTRABACKUP_PID" || true +} + +setup_ports() +{ + if [[ "$WSREP_SST_OPT_ROLE" == "donor" ]];then + SST_PORT=$(echo $WSREP_SST_OPT_ADDR | awk -F '[:/]' '{ print $2 }') + REMOTEIP=$(echo $WSREP_SST_OPT_ADDR | awk -F ':' '{ print $1 }') + REMOTEHOST=$(getent hosts $REMOTEIP | awk '{ print $2 }') + if [[ -z $REMOTEHOST ]];then + REMOTEHOST=$REMOTEIP + fi + lsn=$(echo $WSREP_SST_OPT_ADDR | awk -F '[:/]' '{ print $4 }') + sst_ver=$(echo $WSREP_SST_OPT_ADDR | awk -F '[:/]' '{ print $5 }') + else + SST_PORT=$(echo ${WSREP_SST_OPT_ADDR} | awk -F ':' '{ print $2 }') + fi +} + +# waits ~10 seconds for nc to open the port and then reports ready +# (regardless of timeout) +wait_for_listen() +{ + local PORT=$1 + local ADDR=$2 + local MODULE=$3 + for i in {1..50} + do + ss -p state listening "( sport = :$PORT )" | grep -qE 'socat|nc' && break + sleep 0.2 + done + echo "ready ${ADDR}/${MODULE}//$sst_ver" +} + +check_extra() +{ + local use_socket=1 + if [[ $uextra -eq 1 ]];then + if $MY_PRINT_DEFAULTS --mysqld | tr '_' '-' | grep -- "--thread-handling=" | grep -q 'pool-of-threads';then + local eport=$($MY_PRINT_DEFAULTS --mysqld | tr '_' '-' | grep -- "--extra-port=" | cut -d= -f2) + if [[ -n $eport ]];then + # Xtrabackup works only locally. + # Hence, setting host to 127.0.0.1 unconditionally. + wsrep_log_info "SST through extra_port $eport" + INNOEXTRA+=" --host=127.0.0.1 --port=$eport " + use_socket=0 + else + wsrep_log_error "Extra port $eport null, failing" + exit 1 + fi + else + wsrep_log_info "Thread pool not set, ignore the option use_extra" + fi + fi + if [[ $use_socket -eq 1 ]] && [[ -n "${WSREP_SST_OPT_SOCKET}" ]];then + INNOEXTRA+=" --socket=${WSREP_SST_OPT_SOCKET}" + fi +} + +recv_joiner() +{ + local dir=$1 + local msg=$2 + local tmt=$3 + local checkf=$4 + local ltcmd + + if [[ ! -d ${dir} ]];then + # This indicates that IST is in progress + return + fi + + pushd ${dir} 1>/dev/null + set +e + + if [[ $tmt -gt 0 && -x `which timeout` ]];then + if timeout --help | grep -q -- '-k';then + ltcmd="timeout -k $(( tmt+10 )) $tmt $tcmd" + else + ltcmd="timeout -s9 $tmt $tcmd" + fi + timeit "$msg" "$ltcmd | $strmcmd; RC=( "\${PIPESTATUS[@]}" )" + else + timeit "$msg" "$tcmd | $strmcmd; RC=( "\${PIPESTATUS[@]}" )" + fi + + set -e + popd 1>/dev/null + + if [[ ${RC[0]} -eq 124 ]];then + wsrep_log_error "Possible timeout in receving first data from donor in gtid stage" + exit 32 + fi + + for ecode in "${RC[@]}";do + if [[ $ecode -ne 0 ]];then + wsrep_log_error "Error while getting data from donor node: " \ + "exit codes: ${RC[@]}" + exit 32 + fi + done + + if [[ $checkf -eq 1 && ! -r "${MAGIC_FILE}" ]];then + # this message should cause joiner to abort + wsrep_log_error "xtrabackup process ended without creating '${MAGIC_FILE}'" + wsrep_log_info "Contents of datadir" + wsrep_log_info "$(ls -l ${dir}/*)" + exit 32 + fi +} + + +send_donor() +{ + local dir=$1 + local msg=$2 + + pushd ${dir} 1>/dev/null + set +e + timeit "$msg" "$strmcmd | $tcmd; RC=( "\${PIPESTATUS[@]}" )" + set -e + popd 1>/dev/null + + + for ecode in "${RC[@]}";do + if [[ $ecode -ne 0 ]];then + wsrep_log_error "Error while getting data from donor node: " \ + "exit codes: ${RC[@]}" + exit 32 + fi + done + +} + +if [[ ! -x `which $INNOBACKUPEX_BIN` ]];then + wsrep_log_error "${INNOBACKUPEX_BIN} not in path: $PATH" + exit 2 +fi + +rm -f "${MAGIC_FILE}" + +if [[ ! ${WSREP_SST_OPT_ROLE} == 'joiner' && ! ${WSREP_SST_OPT_ROLE} == 'donor' ]];then + wsrep_log_error "Invalid role ${WSREP_SST_OPT_ROLE}" + exit 22 +fi + +read_cnf +setup_ports + +if ${INNOBACKUPEX_BIN} /tmp --help 2>/dev/null | grep -q -- '--version-check'; then + disver="--no-version-check" +fi + +if [[ ${FORCE_FTWRL:-0} -eq 1 ]];then + wsrep_log_info "Forcing FTWRL due to environment variable FORCE_FTWRL equal to $FORCE_FTWRL" + iopts+=" --no-backup-locks " +fi + + +INNOEXTRA="" + +if [[ $ssyslog -eq 1 ]];then + + if [[ ! -x `which logger` ]];then + wsrep_log_error "logger not in path: $PATH. Ignoring" + else + + wsrep_log_info "Logging all stderr of SST/Innobackupex to syslog" + + exec 2> >(logger -p daemon.err -t ${ssystag}wsrep-sst-$WSREP_SST_OPT_ROLE) + + wsrep_log_error() + { + logger -p daemon.err -t ${ssystag}wsrep-sst-$WSREP_SST_OPT_ROLE "$@" + } + + wsrep_log_info() + { + logger -p daemon.info -t ${ssystag}wsrep-sst-$WSREP_SST_OPT_ROLE "$@" + } + + INNOAPPLY="${INNOBACKUPEX_BIN} --innobackupex $disver $iapts --apply-log \$rebuildcmd \${DATA} 2>&1 | logger -p daemon.err -t ${ssystag}innobackupex-apply " + INNOMOVE="${INNOBACKUPEX_BIN} --innobackupex ${WSREP_SST_OPT_CONF} $disver $impts --move-back --force-non-empty-directories \${DATA} 2>&1 | logger -p daemon.err -t ${ssystag}innobackupex-move " + INNOBACKUP="${INNOBACKUPEX_BIN} --innobackupex ${WSREP_SST_OPT_CONF} $disver $iopts \$tmpopts \$INNOEXTRA --galera-info --stream=\$sfmt \$itmpdir 2> >(logger -p daemon.err -t ${ssystag}innobackupex-backup)" + fi + +else + INNOAPPLY="${INNOBACKUPEX_BIN} --innobackupex $disver $iapts --apply-log \$rebuildcmd \${DATA} &>\${DATA}/innobackup.prepare.log" + INNOMOVE="${INNOBACKUPEX_BIN} --innobackupex ${WSREP_SST_OPT_CONF} $disver $impts --move-back --force-non-empty-directories \${DATA} &>\${DATA}/innobackup.move.log" + INNOBACKUP="${INNOBACKUPEX_BIN} --innobackupex ${WSREP_SST_OPT_CONF} $disver $iopts \$tmpopts \$INNOEXTRA --galera-info --stream=\$sfmt \$itmpdir 2>\${DATA}/innobackup.backup.log" +fi + +get_stream +get_transfer + +if [ "$WSREP_SST_OPT_ROLE" = "donor" ] +then + trap cleanup_donor EXIT + + if [ $WSREP_SST_OPT_BYPASS -eq 0 ] + then + usrst=0 + if [[ -z $sst_ver ]];then + wsrep_log_error "Upgrade joiner to 5.6.21 or higher for backup locks support" + wsrep_log_error "The joiner is not supported for this version of donor" + exit 93 + fi + + if [[ -z $(parse_cnf mysqld tmpdir "") && -z $(parse_cnf xtrabackup tmpdir "") ]];then + xtmpdir=$(mktemp -d) + tmpopts=" --tmpdir=$xtmpdir " + wsrep_log_info "Using $xtmpdir as xtrabackup temporary directory" + fi + + itmpdir=$(mktemp -d) + wsrep_log_info "Using $itmpdir as innobackupex temporary directory" + + if [[ -n "${WSREP_SST_OPT_USER:-}" && "$WSREP_SST_OPT_USER" != "(null)" ]]; then + INNOEXTRA+=" --user=$WSREP_SST_OPT_USER" + usrst=1 + fi + + if [ -n "${WSREP_SST_OPT_PSWD:-}" ]; then + INNOEXTRA+=" --password=$WSREP_SST_OPT_PSWD" + elif [[ $usrst -eq 1 ]];then + # Empty password, used for testing, debugging etc. + INNOEXTRA+=" --password=" + fi + + get_keys + if [[ $encrypt -eq 1 ]];then + if [[ -n $ekey ]];then + INNOEXTRA+=" --encrypt=$ealgo --encrypt-key=$ekey " + else + INNOEXTRA+=" --encrypt=$ealgo --encrypt-key-file=$ekeyfile " + fi + fi + + + check_extra + + wsrep_log_info "Streaming GTID file before SST" + + # Store donor's wsrep GTID (state ID) and wsrep_gtid_domain_id + # (separated by a space). + echo "${WSREP_SST_OPT_GTID} ${WSREP_SST_OPT_GTID_DOMAIN_ID}" > "${MAGIC_FILE}" + + ttcmd="$tcmd" + + if [[ $encrypt -eq 1 ]];then + if [[ -n $scomp ]];then + tcmd=" $ecmd | $scomp | $tcmd " + else + tcmd=" $ecmd | $tcmd " + fi + elif [[ -n $scomp ]];then + tcmd=" $scomp | $tcmd " + fi + + + send_donor $DATA "${stagemsg}-gtid" + + tcmd="$ttcmd" + if [[ -n $progress ]];then + get_footprint + tcmd="$pcmd | $tcmd" + elif [[ -n $rlimit ]];then + adjust_progress + tcmd="$pcmd | $tcmd" + fi + + wsrep_log_info "Sleeping before data transfer for SST" + sleep 10 + + wsrep_log_info "Streaming the backup to joiner at ${REMOTEIP} ${SST_PORT:-4444}" + + if [[ -n $scomp ]];then + tcmd="$scomp | $tcmd" + fi + + set +e + timeit "${stagemsg}-SST" "$INNOBACKUP | $tcmd; RC=( "\${PIPESTATUS[@]}" )" + set -e + + if [ ${RC[0]} -ne 0 ]; then + wsrep_log_error "${INNOBACKUPEX_BIN} finished with error: ${RC[0]}. " \ + "Check ${DATA}/innobackup.backup.log" + exit 22 + elif [[ ${RC[$(( ${#RC[@]}-1 ))]} -eq 1 ]];then + wsrep_log_error "$tcmd finished with error: ${RC[1]}" + exit 22 + fi + + # innobackupex implicitly writes PID to fixed location in $xtmpdir + XTRABACKUP_PID="$xtmpdir/xtrabackup_pid" + + + else # BYPASS FOR IST + + wsrep_log_info "Bypassing the SST for IST" + echo "continue" # now server can resume updating data + + # Store donor's wsrep GTID (state ID) and wsrep_gtid_domain_id + # (separated by a space). + echo "${WSREP_SST_OPT_GTID} ${WSREP_SST_OPT_GTID_DOMAIN_ID}" > "${MAGIC_FILE}" + echo "1" > "${DATA}/${IST_FILE}" + get_keys + if [[ $encrypt -eq 1 ]];then + if [[ -n $scomp ]];then + tcmd=" $ecmd | $scomp | $tcmd " + else + tcmd=" $ecmd | $tcmd " + fi + elif [[ -n $scomp ]];then + tcmd=" $scomp | $tcmd " + fi + strmcmd+=" \${IST_FILE}" + + send_donor $DATA "${stagemsg}-IST" + + fi + + echo "done ${WSREP_SST_OPT_GTID}" + wsrep_log_info "Total time on donor: $totime seconds" + +elif [ "${WSREP_SST_OPT_ROLE}" = "joiner" ] +then + [[ -e $SST_PROGRESS_FILE ]] && wsrep_log_info "Stale sst_in_progress file: $SST_PROGRESS_FILE" + [[ -n $SST_PROGRESS_FILE ]] && touch $SST_PROGRESS_FILE + + ib_home_dir=$(parse_cnf mysqld innodb-data-home-dir "") + ib_log_dir=$(parse_cnf mysqld innodb-log-group-home-dir "") + ib_undo_dir=$(parse_cnf mysqld innodb-undo-directory "") + + stagemsg="Joiner-Recv" + + + sencrypted=1 + nthreads=1 + + MODULE="xtrabackup_sst" + + rm -f "${DATA}/${IST_FILE}" + + # May need xtrabackup_checkpoints later on + rm -f ${DATA}/xtrabackup_binary ${DATA}/xtrabackup_galera_info ${DATA}/xtrabackup_logfile + + ADDR=${WSREP_SST_OPT_ADDR} + if [ -z "${SST_PORT}" ] + then + SST_PORT=4444 + ADDR="$(echo ${WSREP_SST_OPT_ADDR} | awk -F ':' '{ print $1 }'):${SST_PORT}" + fi + + wait_for_listen ${SST_PORT} ${ADDR} ${MODULE} & + + trap sig_joiner_cleanup HUP PIPE INT TERM + trap cleanup_joiner EXIT + + if [[ -n $progress ]];then + adjust_progress + tcmd+=" | $pcmd" + fi + + get_keys + if [[ $encrypt -eq 1 && $sencrypted -eq 1 ]];then + if [[ -n $sdecomp ]];then + strmcmd=" $sdecomp | $ecmd | $strmcmd" + else + strmcmd=" $ecmd | $strmcmd" + fi + elif [[ -n $sdecomp ]];then + strmcmd=" $sdecomp | $strmcmd" + fi + + STATDIR=$(mktemp -d) + MAGIC_FILE="${STATDIR}/${INFO_FILE}" + recv_joiner $STATDIR "${stagemsg}-gtid" $stimeout 1 + + + if ! ps -p ${WSREP_SST_OPT_PARENT} &>/dev/null + then + wsrep_log_error "Parent mysqld process (PID:${WSREP_SST_OPT_PARENT}) terminated unexpectedly." + exit 32 + fi + + if [ ! -r "${STATDIR}/${IST_FILE}" ] + then + + if [[ -d ${DATA}/.sst ]];then + wsrep_log_info "WARNING: Stale temporary SST directory: ${DATA}/.sst from previous state transfer. Removing" + rm -rf ${DATA}/.sst + fi + mkdir -p ${DATA}/.sst + (recv_joiner $DATA/.sst "${stagemsg}-SST" 0 0) & + jpid=$! + wsrep_log_info "Proceeding with SST" + + + wsrep_log_info "Cleaning the existing datadir and innodb-data/log directories" + find $ib_home_dir $ib_log_dir $ib_undo_dir $DATA -mindepth 1 -regex $cpat -prune -o -exec rm -rfv {} 1>&2 \+ + + tempdir=$(parse_cnf mysqld log-bin "") + if [[ -n ${tempdir:-} ]];then + binlog_dir=$(dirname $tempdir) + binlog_file=$(basename $tempdir) + if [[ -n ${binlog_dir:-} && $binlog_dir != '.' && $binlog_dir != $DATA ]];then + pattern="$binlog_dir/$binlog_file\.[0-9]+$" + wsrep_log_info "Cleaning the binlog directory $binlog_dir as well" + find $binlog_dir -maxdepth 1 -type f -regex $pattern -exec rm -fv {} 1>&2 \+ || true + rm $binlog_dir/*.index || true + fi + fi + + + + TDATA=${DATA} + DATA="${DATA}/.sst" + + + MAGIC_FILE="${DATA}/${INFO_FILE}" + wsrep_log_info "Waiting for SST streaming to complete!" + wait $jpid + + get_proc + + if [[ ! -s ${DATA}/xtrabackup_checkpoints ]];then + wsrep_log_error "xtrabackup_checkpoints missing, failed innobackupex/SST on donor" + exit 2 + fi + + # Rebuild indexes for compact backups + if grep -q 'compact = 1' ${DATA}/xtrabackup_checkpoints;then + wsrep_log_info "Index compaction detected" + rebuild=1 + fi + + if [[ $rebuild -eq 1 ]];then + nthreads=$(parse_cnf xtrabackup rebuild-threads $nproc) + wsrep_log_info "Rebuilding during prepare with $nthreads threads" + rebuildcmd="--rebuild-indexes --rebuild-threads=$nthreads" + fi + + if test -n "$(find ${DATA} -maxdepth 1 -type f -name '*.qp' -print -quit)";then + + wsrep_log_info "Compressed qpress files found" + + if [[ ! -x `which qpress` ]];then + wsrep_log_error "qpress not found in path: $PATH" + exit 22 + fi + + if [[ -n $progress ]] && pv --help | grep -q 'line-mode';then + count=$(find ${DATA} -type f -name '*.qp' | wc -l) + count=$(( count*2 )) + if pv --help | grep -q FORMAT;then + pvopts="-f -s $count -l -N Decompression -F '%N => Rate:%r Elapsed:%t %e Progress: [%b/$count]'" + else + pvopts="-f -s $count -l -N Decompression" + fi + pcmd="pv $pvopts" + adjust_progress + dcmd="$pcmd | xargs -n 2 qpress -T${nproc}d" + else + dcmd="xargs -n 2 qpress -T${nproc}d" + fi + + + # Decompress the qpress files + wsrep_log_info "Decompression with $nproc threads" + timeit "Joiner-Decompression" "find ${DATA} -type f -name '*.qp' -printf '%p\n%h\n' | $dcmd" + extcode=$? + + if [[ $extcode -eq 0 ]];then + wsrep_log_info "Removing qpress files after decompression" + find ${DATA} -type f -name '*.qp' -delete + if [[ $? -ne 0 ]];then + wsrep_log_error "Something went wrong with deletion of qpress files. Investigate" + fi + else + wsrep_log_error "Decompression failed. Exit code: $extcode" + exit 22 + fi + fi + + + if [[ ! -z $WSREP_SST_OPT_BINLOG ]];then + + BINLOG_DIRNAME=$(dirname $WSREP_SST_OPT_BINLOG) + BINLOG_FILENAME=$(basename $WSREP_SST_OPT_BINLOG) + + # To avoid comparing data directory and BINLOG_DIRNAME + mv $DATA/${BINLOG_FILENAME}.* $BINLOG_DIRNAME/ 2>/dev/null || true + + pushd $BINLOG_DIRNAME &>/dev/null + for bfiles in $(ls -1 ${BINLOG_FILENAME}.[0-9]*);do + echo ${BINLOG_DIRNAME}/${bfiles} >> ${BINLOG_FILENAME}.index + done + popd &> /dev/null + + fi + + wsrep_log_info "Preparing the backup at ${DATA}" + timeit "Xtrabackup prepare stage" "$INNOAPPLY" + + if [ $? -ne 0 ]; + then + wsrep_log_error "${INNOBACKUPEX_BIN} apply finished with errors. Check ${DATA}/innobackup.prepare.log" + exit 22 + fi + + MAGIC_FILE="${TDATA}/${INFO_FILE}" + set +e + rm $TDATA/innobackup.prepare.log $TDATA/innobackup.move.log + set -e + wsrep_log_info "Moving the backup to ${TDATA}" + timeit "Xtrabackup move stage" "$INNOMOVE" + if [[ $? -eq 0 ]];then + wsrep_log_info "Move successful, removing ${DATA}" + rm -rf $DATA + DATA=${TDATA} + else + wsrep_log_error "Move failed, keeping ${DATA} for further diagnosis" + wsrep_log_error "Check ${DATA}/innobackup.move.log for details" + exit 22 + fi + + + else + wsrep_log_info "${IST_FILE} received from donor: Running IST" + fi + + if [[ ! -r ${MAGIC_FILE} ]];then + wsrep_log_error "SST magic file ${MAGIC_FILE} not found/readable" + exit 2 + fi + wsrep_log_info "Galera co-ords from recovery: $(cat ${MAGIC_FILE})" + cat "${MAGIC_FILE}" # Output : UUID:seqno wsrep_gtid_domain_id + wsrep_log_info "Total time on joiner: $totime seconds" +fi + +exit 0 From 64b3427b89f7e7ffb08b4720e1f206ed44c72651 Mon Sep 17 00:00:00 2001 From: Vladislav Vaintroub Date: Tue, 18 Apr 2017 19:58:06 +0000 Subject: [PATCH 14/28] Run mariabackup test on Windows on buildbot --- mysql-test/collections/buildbot_suites.bat | 1 + 1 file changed, 1 insertion(+) create mode 100644 mysql-test/collections/buildbot_suites.bat diff --git a/mysql-test/collections/buildbot_suites.bat b/mysql-test/collections/buildbot_suites.bat new file mode 100644 index 00000000000..a62d535d951 --- /dev/null +++ b/mysql-test/collections/buildbot_suites.bat @@ -0,0 +1 @@ +perl mysql-test-run.pl --verbose-restart --force --testcase-timeout=45 --suite-timeout=600 --max-test-fail=500 --retry=3 --suite=main,innodb,plugins,mariabackup --parallel=4 From d894f7ceedbe3992ec52d22634cba43787faa7dc Mon Sep 17 00:00:00 2001 From: Vladislav Vaintroub Date: Tue, 18 Apr 2017 20:01:30 +0000 Subject: [PATCH 15/28] update gitignore with commonly used (by me) out-of-source builddir names --- .gitignore | 4 ++++ 1 file changed, 4 insertions(+) diff --git a/.gitignore b/.gitignore index 44832c99c6b..05d3b87f6d1 100644 --- a/.gitignore +++ b/.gitignore @@ -238,6 +238,10 @@ storage/mroonga/vendor/groonga/src/groonga-benchmark storage/mroonga/vendor/groonga/src/suggest/groonga-suggest-create-dataset storage/mroonga/mysql-test/mroonga/storage/r/information_schema_plugins.result storage/mroonga/mysql-test/mroonga/storage/r/variable_version.result +xxx/* +yyy/* +zzz/* + # C and C++ # Compiled Object files From 7228b9985f29e1d1362312b25219bb5488d69e17 Mon Sep 17 00:00:00 2001 From: Vladislav Vaintroub Date: Tue, 18 Apr 2017 20:18:16 +0000 Subject: [PATCH 16/28] MDEV-9566 build mariabackup with libarchive for release --- cmake/build_configurations/mysql_release.cmake | 4 ++++ 1 file changed, 4 insertions(+) diff --git a/cmake/build_configurations/mysql_release.cmake b/cmake/build_configurations/mysql_release.cmake index e561144bbd5..c07de46fc16 100644 --- a/cmake/build_configurations/mysql_release.cmake +++ b/cmake/build_configurations/mysql_release.cmake @@ -85,20 +85,24 @@ ENDIF() OPTION(ENABLED_LOCAL_INFILE "" ON) SET(WITH_INNODB_SNAPPY OFF CACHE STRING "") IF(WIN32) + SET(WITH_LIBARCHIVE STATIC CACHE STRING "") ELSEIF(RPM) SET(WITH_SSL system CACHE STRING "") SET(WITH_ZLIB system CACHE STRING "") SET(CHECKMODULE /usr/bin/checkmodule CACHE STRING "") SET(SEMODULE_PACKAGE /usr/bin/semodule_package CACHE STRING "") + SET(WITH_LIBARCHIVE ON CACHE STRING "") ELSEIF(DEB) SET(WITH_SSL system CACHE STRING "") SET(WITH_ZLIB system CACHE STRING "") SET(WITH_LIBWRAP ON) SET(HAVE_EMBEDDED_PRIVILEGE_CONTROL ON) + SET(WITH_LIBARCHIVE ON CACHE STRING "") ELSE() SET(WITH_SSL bundled CACHE STRING "") SET(WITH_ZLIB bundled CACHE STRING "") SET(WITH_JEMALLOC static CACHE STRING "") + SET(WITH_LIBARCHIVE STATIC CACHE STRING "") ENDIF() IF(NOT COMPILATION_COMMENT) From f344d7ec61679fa6912fbf846408b9f4097158a6 Mon Sep 17 00:00:00 2001 From: Vladislav Vaintroub Date: Tue, 18 Apr 2017 20:59:33 +0000 Subject: [PATCH 17/28] Make Ninja generator happy with BUILD_BYPRODUCTS. --- extra/mariabackup/CMakeLists.txt | 17 +++++++++++++---- 1 file changed, 13 insertions(+), 4 deletions(-) diff --git a/extra/mariabackup/CMakeLists.txt b/extra/mariabackup/CMakeLists.txt index f3369b49cee..6f5a17d3614 100644 --- a/extra/mariabackup/CMakeLists.txt +++ b/extra/mariabackup/CMakeLists.txt @@ -56,6 +56,7 @@ IF((NOT WITH_LIBARCHIVE STREQUAL "OFF") AND (NOT LibArchive_FOUND)) ELSE() # Build a local version INCLUDE(ExternalProject) + SET(LIBARCHIVE_DIR ${CMAKE_CURRENT_BINARY_DIR}/libarchive) SET(libarchive_PREFIX ${CMAKE_CURRENT_BINARY_DIR}/libarchive) SET(libarchive_CMAKE_ARGS -DCMAKE_INSTALL_PREFIX:PATH= @@ -71,27 +72,35 @@ IF((NOT WITH_LIBARCHIVE STREQUAL "OFF") AND (NOT LibArchive_FOUND)) ) IF(WIN32) SET(libarchive_CMAKE_ARGS ${libarchive_CMAKE_ARGS} -DWINDOWS_VERSION=WIN7 -DCMAKE_DEBUG_POSTFIX=d) + SET(LIBARCHIVE_RELEASE_LIB ${LIBARCHIVE_DIR}/lib/${CMAKE_STATIC_LIBRARY_PREFIX}archive_static${CMAKE_STATIC_LIBRARY_SUFFIX}) + SET(LIBARCHIVE_DEBUG_LIB ${LIBARCHIVE_DIR}/lib/${CMAKE_STATIC_LIBRARY_PREFIX}archive_staticd${CMAKE_STATIC_LIBRARY_SUFFIX}) + SET(byproducts ${LIBARCHIVE_RELEASE_LIB} ${LIBARCHIVE_DEBUG_LIB}) + ELSE() + SET(LIBARCHIVE_LIB ${LIBARCHIVE_DIR}/lib/${CMAKE_STATIC_LIBRARY_PREFIX}archive${CMAKE_STATIC_LIBRARY_SUFFIX}) + SET(byproducts ${LIBARCHIVE_LIB}) + ENDIF() + + IF(CMAKE_VERSION VERSION_GREATER "3.1") + SET(byproducts BUILD_BYPRODUCTS ${byproducts}) ENDIF() - SET(LIBARCHIVE_DIR ${CMAKE_CURRENT_BINARY_DIR}/libarchive) ExternalProject_Add(libarchive PREFIX ${libarchive_PREFIX} DOWNLOAD_DIR ${LIBARCHIVE_DIR} URL http://www.libarchive.org/downloads/libarchive-3.2.2.tar.gz INSTALL_DIR ${LIBARCHIVE_DIR} CMAKE_ARGS ${libarchive_CMAKE_ARGS} + ${byproducts} ) ADD_LIBRARY(archive_static STATIC IMPORTED) ADD_DEPENDENCIES(archive_static libarchive) IF(WIN32) - SET(LIBARCHIVE_RELEASE_LIB ${LIBARCHIVE_DIR}/lib/${CMAKE_STATIC_LIBRARY_PREFIX}archive_static${CMAKE_STATIC_LIBRARY_SUFFIX}) - SET(LIBARCHIVE_DEBUG_LIB ${LIBARCHIVE_DIR}/lib/${CMAKE_STATIC_LIBRARY_PREFIX}archive_staticd${CMAKE_STATIC_LIBRARY_SUFFIX}) SET_PROPERTY(TARGET archive_static PROPERTY IMPORTED_LOCATION_RELWITHDEBINFO ${LIBARCHIVE_RELEASE_LIB}) SET_PROPERTY(TARGET archive_static PROPERTY IMPORTED_LOCATION_RELEASE ${LIBARCHIVE_RELEASE_LIB}) SET_PROPERTY(TARGET archive_static PROPERTY IMPORTED_LOCATION_DEBUG ${LIBARCHIVE_DEBUG_LIB}) SET_PROPERTY(TARGET archive_static PROPERTY IMPORTED_LOCATION_MINSIZEREL ${LIBARCHIVE_RELEASE_LIB}) ELSE() - SET_PROPERTY(TARGET archive_static PROPERTY IMPORTED_LOCATION ${LIBARCHIVE_DIR}/lib/${CMAKE_STATIC_LIBRARY_PREFIX}archive${CMAKE_STATIC_LIBRARY_SUFFIX}) + SET_PROPERTY(TARGET archive_static PROPERTY IMPORTED_LOCATION ${LIBARCHIVE_LIB}) ENDIF() SET(LibArchive_FOUND ON ) From c8ac0244a843252d59985ad270cad2f074aebfb4 Mon Sep 17 00:00:00 2001 From: Vladislav Vaintroub Date: Wed, 19 Apr 2017 07:45:24 +0000 Subject: [PATCH 18/28] add mariabackup to default suites --- mysql-test/mysql-test-run.pl | 1 + 1 file changed, 1 insertion(+) diff --git a/mysql-test/mysql-test-run.pl b/mysql-test/mysql-test-run.pl index 57fdb10e9d7..c7e8712c9fc 100755 --- a/mysql-test/mysql-test-run.pl +++ b/mysql-test/mysql-test-run.pl @@ -182,6 +182,7 @@ my @DEFAULT_SUITES= qw( innodb_fts- innodb_zip- maria- + mariabackup- multi_source- optimizer_unfixed_bugs- parts- From ecb25df21b2cc9bcbb038dd9b82b8469acd18fd9 Mon Sep 17 00:00:00 2001 From: Vladislav Vaintroub Date: Wed, 19 Apr 2017 13:09:03 +0000 Subject: [PATCH 19/28] Xtrabackup 2.3.8 --- extra/mariabackup/CMakeLists.txt | 15 +- extra/mariabackup/backup_copy.cc | 7 +- extra/mariabackup/backup_mysql.cc | 5 +- extra/mariabackup/crc/CMakeLists.txt | 33 + extra/mariabackup/crc/config.h.cmake | 21 + extra/mariabackup/crc/crc-intel-pclmul.c | 511 ++++++++++++++ extra/mariabackup/crc/crc-intel-pclmul.h | 25 + extra/mariabackup/crc/crc_glue.c | 72 ++ extra/mariabackup/crc/crc_glue.h | 31 + extra/mariabackup/datasink.c | 2 +- extra/mariabackup/datasink.h | 1 + extra/mariabackup/ds_decrypt.c | 665 ++++++++++++++++++ extra/mariabackup/ds_decrypt.h | 30 + extra/mariabackup/ds_encrypt.c | 220 +----- extra/mariabackup/ds_encrypt.h | 5 + extra/mariabackup/xbcrypt.c | 12 +- extra/mariabackup/xbcrypt.h | 7 +- extra/mariabackup/xbcrypt_common.c | 286 +++++++- extra/mariabackup/xbcrypt_common.h | 64 ++ extra/mariabackup/xbcrypt_read.c | 3 +- extra/mariabackup/xbcrypt_write.c | 3 +- extra/mariabackup/xbstream.c | 261 +++++-- extra/mariabackup/xbstream.h | 6 +- extra/mariabackup/xbstream_read.c | 53 +- extra/mariabackup/xbstream_write.c | 6 +- extra/mariabackup/xtrabackup.cc | 435 ++++++++---- extra/mariabackup/xtrabackup.h | 13 + .../suite/mariabackup/partial_exclude.result | 12 + .../suite/mariabackup/partial_exclude.test | 30 + mysql-test/suite/mariabackup/xbstream.test | 2 +- storage/xtradb/fil/fil0fil.cc | 18 +- 31 files changed, 2423 insertions(+), 431 deletions(-) create mode 100644 extra/mariabackup/crc/CMakeLists.txt create mode 100644 extra/mariabackup/crc/config.h.cmake create mode 100644 extra/mariabackup/crc/crc-intel-pclmul.c create mode 100644 extra/mariabackup/crc/crc-intel-pclmul.h create mode 100644 extra/mariabackup/crc/crc_glue.c create mode 100644 extra/mariabackup/crc/crc_glue.h create mode 100644 extra/mariabackup/ds_decrypt.c create mode 100644 extra/mariabackup/ds_decrypt.h create mode 100644 extra/mariabackup/xbcrypt_common.h create mode 100644 mysql-test/suite/mariabackup/partial_exclude.result create mode 100644 mysql-test/suite/mariabackup/partial_exclude.test diff --git a/extra/mariabackup/CMakeLists.txt b/extra/mariabackup/CMakeLists.txt index 6f5a17d3614..693082b765a 100644 --- a/extra/mariabackup/CMakeLists.txt +++ b/extra/mariabackup/CMakeLists.txt @@ -1,14 +1,14 @@ -# Copyright (c) 2013 Percona LLC and/or its affiliates. +# Copyright (c) 2013, 2017 Percona LLC and/or its affiliates. # # This program is free software; you can redistribute it and/or modify # it under the terms of the GNU General Public License as published by # the Free Software Foundation; version 2 of the License. - +# # This program is distributed in the hope that it will be useful, # but WITHOUT ANY WARRANTY; without even the implied warranty of # MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the # GNU General Public License for more details. - +# # You should have received a copy of the GNU General Public License # along with this program; if not, write to the Free Software # Foundation, Inc., 51 Franklin Street, Fifth Floor, Boston, MA 02110-1301, USA @@ -129,6 +129,7 @@ INCLUDE_DIRECTORIES( ${CMAKE_SOURCE_DIR}/sql ${CMAKE_CURRENT_SOURCE_DIR}/quicklz ${CMAKE_CURRENT_SOURCE_DIR} + ${CMAKE_CURRENT_SOURCE_DIR}/crc ) IF(NOT HAVE_SYSTEM_REGEX) @@ -156,7 +157,6 @@ MYSQL_ADD_EXECUTABLE(mariabackup ${DS_ARCHIVE_SOURCE} ds_buffer.c ds_compress.c - ds_encrypt.c ds_local.c ds_stdout.c ds_tmpfile.c @@ -166,8 +166,6 @@ MYSQL_ADD_EXECUTABLE(mariabackup read_filt.cc write_filt.cc wsrep.cc - xbcrypt_common.c - xbcrypt_write.c xbstream_write.c backup_mysql.cc backup_copy.cc @@ -181,9 +179,10 @@ MYSQL_ADD_EXECUTABLE(mariabackup # Export all symbols on Unix, for better crash callstacks SET_TARGET_PROPERTIES(mariabackup PROPERTIES ENABLE_EXPORTS TRUE) +ADD_SUBDIRECTORY(crc) -TARGET_LINK_LIBRARIES(mariabackup sql) +TARGET_LINK_LIBRARIES(mariabackup sql crc) IF(NOT HAVE_SYSTEM_REGEX) TARGET_LINK_LIBRARIES(mariabackup pcreposix) @@ -201,13 +200,13 @@ MYSQL_ADD_EXECUTABLE(mbstream xbstream.c xbstream_read.c xbstream_write.c - COMPONENT backup ) TARGET_LINK_LIBRARIES(mbstream mysys + crc ) IF(MSVC) diff --git a/extra/mariabackup/backup_copy.cc b/extra/mariabackup/backup_copy.cc index 478a6a66b97..67c96b4b648 100644 --- a/extra/mariabackup/backup_copy.cc +++ b/extra/mariabackup/backup_copy.cc @@ -265,6 +265,11 @@ datadir_iter_next_database(datadir_iter_t *it) return(true); } + if (check_if_skip_database_by_path(it->dbpath)) { + msg("Skipping db: %s\n", it->dbpath); + continue; + } + /* We want wrong directory permissions to be a fatal error for XtraBackup. */ it->dbdir = os_file_opendir(it->dbpath, TRUE); @@ -1704,7 +1709,7 @@ copy_back() for (i = 1; i <= srv_undo_tablespaces; i++) { char filename[20]; - sprintf(filename, "undo%03lu", i); + sprintf(filename, "undo%03u", (uint)i); if (!(ret = copy_or_move_file(filename, filename, dst_dir, 1))) { goto cleanup; diff --git a/extra/mariabackup/backup_mysql.cc b/extra/mariabackup/backup_mysql.cc index ed5446b4806..52aaf138fc8 100644 --- a/extra/mariabackup/backup_mysql.cc +++ b/extra/mariabackup/backup_mysql.cc @@ -1414,7 +1414,10 @@ write_xtrabackup_info(MYSQL *connection) bool is_partial = (xtrabackup_tables || xtrabackup_tables_file || xtrabackup_databases - || xtrabackup_databases_file); + || xtrabackup_databases_file + || xtrabackup_tables_exclude + || xtrabackup_databases_exclude + ); backup_file_printf(XTRABACKUP_INFO, "uuid = %s\n" diff --git a/extra/mariabackup/crc/CMakeLists.txt b/extra/mariabackup/crc/CMakeLists.txt new file mode 100644 index 00000000000..577cab6080c --- /dev/null +++ b/extra/mariabackup/crc/CMakeLists.txt @@ -0,0 +1,33 @@ +# Copyright (c) 2017 Percona LLC and/or its affiliates. +# +# This program is free software; you can redistribute it and/or modify +# it under the terms of the GNU General Public License as published by +# the Free Software Foundation; version 2 of the License. + +# This program is distributed in the hope that it will be useful, +# but WITHOUT ANY WARRANTY; without even the implied warranty of +# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the +# GNU General Public License for more details. + +# You should have received a copy of the GNU General Public License +# along with this program; if not, write to the Free Software +# Foundation, Inc., 51 Franklin Street, Fifth Floor, Boston, MA 02110-1301, USA + +PROJECT(crc C) + +IF(NOT CMAKE_CROSSCOMPILING AND NOT MSVC) + STRING(TOLOWER ${CMAKE_SYSTEM_PROCESSOR} processor) + IF(processor MATCHES "86" OR processor MATCHES "amd64" OR processor MATCHES "x64") + # Check for PCLMUL instruction + CHECK_C_SOURCE_RUNS(" + int main() + { + asm volatile (\"pclmulqdq \\$0x00, %%xmm1, %%xmm0\":::\"cc\"); + return 0; + }" HAVE_CLMUL_INSTRUCTION) + ENDIF() +ENDIF() +IF(HAVE_CLMUL_INSTRUCTION) + ADD_DEFINITIONS(-DHAVE_CLMUL_INSTRUCTION) +ENDIF() +ADD_LIBRARY(crc crc_glue.c crc-intel-pclmul.c) diff --git a/extra/mariabackup/crc/config.h.cmake b/extra/mariabackup/crc/config.h.cmake new file mode 100644 index 00000000000..fe81c1859ae --- /dev/null +++ b/extra/mariabackup/crc/config.h.cmake @@ -0,0 +1,21 @@ +/****************************************************** +Copyright (c) 2017 Percona LLC and/or its affiliates. + +Zlib compatible CRC-32 implementation. + +This program is free software; you can redistribute it and/or modify +it under the terms of the GNU General Public License as published by +the Free Software Foundation; version 2 of the License. + +This program is distributed in the hope that it will be useful, +but WITHOUT ANY WARRANTY; without even the implied warranty of +MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the +GNU General Public License for more details. + +You should have received a copy of the GNU General Public License +along with this program; if not, write to the Free Software +Foundation, Inc., 51 Franklin Street, Fifth Floor, Boston, MA 02110-1301, USA + +*******************************************************/ + +#cmakedefine HAVE_CLMUL_INSTRUCTION 1 diff --git a/extra/mariabackup/crc/crc-intel-pclmul.c b/extra/mariabackup/crc/crc-intel-pclmul.c new file mode 100644 index 00000000000..d470c2bee43 --- /dev/null +++ b/extra/mariabackup/crc/crc-intel-pclmul.c @@ -0,0 +1,511 @@ +/****************************************************** +Copyright (c) 2017 Percona LLC and/or its affiliates. + +CRC32 using Intel's PCLMUL instruction. + +This program is free software; you can redistribute it and/or modify +it under the terms of the GNU General Public License as published by +the Free Software Foundation; version 2 of the License. + +This program is distributed in the hope that it will be useful, +but WITHOUT ANY WARRANTY; without even the implied warranty of +MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the +GNU General Public License for more details. + +You should have received a copy of the GNU General Public License +along with this program; if not, write to the Free Software +Foundation, Inc., 51 Franklin Street, Fifth Floor, Boston, MA 02110-1301, USA + +*******************************************************/ + +/* crc-intel-pclmul.c - Intel PCLMUL accelerated CRC implementation + * Copyright (C) 2016 Jussi Kivilinna + * + * This file is part of Libgcrypt. + * + * Libgcrypt is free software; you can redistribute it and/or modify + * it under the terms of the GNU Lesser General Public License as + * published by the Free Software Foundation; either version 2.1 of + * the License, or (at your option) any later version. + * + * Libgcrypt is distributed in the hope that it will be useful, + * but WITHOUT ANY WARRANTY; without even the implied warranty of + * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the + * GNU Lesser General Public License for more details. + * + * You should have received a copy of the GNU Lesser General Public + * License along with this program; if not, write to the Free Software + * Foundation, Inc., 59 Temple Place - Suite 330, Boston, MA 02111-1307, USA + * + */ + +#include +#include +#include +#include + +# define U64_C(c) (c ## UL) + +typedef uint32_t u32; +typedef uint16_t u16; +typedef uint64_t u64; +#ifndef byte +typedef uint8_t byte; +#endif + +# define _gcry_bswap32 __builtin_bswap32 + +#if __GNUC__ >= 4 && defined(__x86_64__) && defined(HAVE_CLMUL_INSTRUCTION) + +#if _GCRY_GCC_VERSION >= 40400 /* 4.4 */ +/* Prevent compiler from issuing SSE instructions between asm blocks. */ +# pragma GCC target("no-sse") +#endif + + +#define ALIGNED_16 __attribute__ ((aligned (16))) + + +struct u16_unaligned_s +{ + u16 a; +} __attribute__((packed, aligned (1), may_alias)); + + +/* Constants structure for generic reflected/non-reflected CRC32 CLMUL + * functions. */ +struct crc32_consts_s +{ + /* k: { x^(32*17), x^(32*15), x^(32*5), x^(32*3), x^(32*2), 0 } mod P(x) */ + u64 k[6]; + /* my_p: { floor(x^64 / P(x)), P(x) } */ + u64 my_p[2]; +}; + + +/* CLMUL constants for CRC32 and CRC32RFC1510. */ +static const struct crc32_consts_s crc32_consts ALIGNED_16 = +{ + { /* k[6] = reverse_33bits( x^(32*y) mod P(x) ) */ + U64_C(0x154442bd4), U64_C(0x1c6e41596), /* y = { 17, 15 } */ + U64_C(0x1751997d0), U64_C(0x0ccaa009e), /* y = { 5, 3 } */ + U64_C(0x163cd6124), 0 /* y = 2 */ + }, + { /* my_p[2] = reverse_33bits ( { floor(x^64 / P(x)), P(x) } ) */ + U64_C(0x1f7011641), U64_C(0x1db710641) + } +}; + +/* Common constants for CRC32 algorithms. */ +static const byte crc32_refl_shuf_shift[3 * 16] ALIGNED_16 = + { + 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, + 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, + 0x00, 0x01, 0x02, 0x03, 0x04, 0x05, 0x06, 0x07, + 0x08, 0x09, 0x0a, 0x0b, 0x0c, 0x0d, 0x0e, 0x0f, + 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, + 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, + }; +static const byte crc32_partial_fold_input_mask[16 + 16] ALIGNED_16 = + { + 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, + 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, + 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, + 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, + }; +static const u64 crc32_merge9to15_shuf[15 - 9 + 1][2] ALIGNED_16 = + { + { U64_C(0x0706050403020100), U64_C(0xffffffffffffff0f) }, /* 9 */ + { U64_C(0x0706050403020100), U64_C(0xffffffffffff0f0e) }, + { U64_C(0x0706050403020100), U64_C(0xffffffffff0f0e0d) }, + { U64_C(0x0706050403020100), U64_C(0xffffffff0f0e0d0c) }, + { U64_C(0x0706050403020100), U64_C(0xffffff0f0e0d0c0b) }, + { U64_C(0x0706050403020100), U64_C(0xffff0f0e0d0c0b0a) }, + { U64_C(0x0706050403020100), U64_C(0xff0f0e0d0c0b0a09) }, /* 15 */ + }; +static const u64 crc32_merge5to7_shuf[7 - 5 + 1][2] ALIGNED_16 = + { + { U64_C(0xffffff0703020100), U64_C(0xffffffffffffffff) }, /* 5 */ + { U64_C(0xffff070603020100), U64_C(0xffffffffffffffff) }, + { U64_C(0xff07060503020100), U64_C(0xffffffffffffffff) }, /* 7 */ + }; + +/* PCLMUL functions for reflected CRC32. */ +static inline void +crc32_reflected_bulk (u32 *pcrc, const byte *inbuf, size_t inlen, + const struct crc32_consts_s *consts) +{ + if (inlen >= 8 * 16) + { + asm volatile ("movd %[crc], %%xmm4\n\t" + "movdqu %[inbuf_0], %%xmm0\n\t" + "movdqu %[inbuf_1], %%xmm1\n\t" + "movdqu %[inbuf_2], %%xmm2\n\t" + "movdqu %[inbuf_3], %%xmm3\n\t" + "pxor %%xmm4, %%xmm0\n\t" + : + : [inbuf_0] "m" (inbuf[0 * 16]), + [inbuf_1] "m" (inbuf[1 * 16]), + [inbuf_2] "m" (inbuf[2 * 16]), + [inbuf_3] "m" (inbuf[3 * 16]), + [crc] "m" (*pcrc) + ); + + inbuf += 4 * 16; + inlen -= 4 * 16; + + asm volatile ("movdqa %[k1k2], %%xmm4\n\t" + : + : [k1k2] "m" (consts->k[1 - 1]) + ); + + /* Fold by 4. */ + while (inlen >= 4 * 16) + { + asm volatile ("movdqu %[inbuf_0], %%xmm5\n\t" + "movdqa %%xmm0, %%xmm6\n\t" + "pclmulqdq $0x00, %%xmm4, %%xmm0\n\t" + "pclmulqdq $0x11, %%xmm4, %%xmm6\n\t" + "pxor %%xmm5, %%xmm0\n\t" + "pxor %%xmm6, %%xmm0\n\t" + + "movdqu %[inbuf_1], %%xmm5\n\t" + "movdqa %%xmm1, %%xmm6\n\t" + "pclmulqdq $0x00, %%xmm4, %%xmm1\n\t" + "pclmulqdq $0x11, %%xmm4, %%xmm6\n\t" + "pxor %%xmm5, %%xmm1\n\t" + "pxor %%xmm6, %%xmm1\n\t" + + "movdqu %[inbuf_2], %%xmm5\n\t" + "movdqa %%xmm2, %%xmm6\n\t" + "pclmulqdq $0x00, %%xmm4, %%xmm2\n\t" + "pclmulqdq $0x11, %%xmm4, %%xmm6\n\t" + "pxor %%xmm5, %%xmm2\n\t" + "pxor %%xmm6, %%xmm2\n\t" + + "movdqu %[inbuf_3], %%xmm5\n\t" + "movdqa %%xmm3, %%xmm6\n\t" + "pclmulqdq $0x00, %%xmm4, %%xmm3\n\t" + "pclmulqdq $0x11, %%xmm4, %%xmm6\n\t" + "pxor %%xmm5, %%xmm3\n\t" + "pxor %%xmm6, %%xmm3\n\t" + : + : [inbuf_0] "m" (inbuf[0 * 16]), + [inbuf_1] "m" (inbuf[1 * 16]), + [inbuf_2] "m" (inbuf[2 * 16]), + [inbuf_3] "m" (inbuf[3 * 16]) + ); + + inbuf += 4 * 16; + inlen -= 4 * 16; + } + + asm volatile ("movdqa %[k3k4], %%xmm6\n\t" + "movdqa %[my_p], %%xmm5\n\t" + : + : [k3k4] "m" (consts->k[3 - 1]), + [my_p] "m" (consts->my_p[0]) + ); + + /* Fold 4 to 1. */ + + asm volatile ("movdqa %%xmm0, %%xmm4\n\t" + "pclmulqdq $0x00, %%xmm6, %%xmm0\n\t" + "pclmulqdq $0x11, %%xmm6, %%xmm4\n\t" + "pxor %%xmm1, %%xmm0\n\t" + "pxor %%xmm4, %%xmm0\n\t" + + "movdqa %%xmm0, %%xmm4\n\t" + "pclmulqdq $0x00, %%xmm6, %%xmm0\n\t" + "pclmulqdq $0x11, %%xmm6, %%xmm4\n\t" + "pxor %%xmm2, %%xmm0\n\t" + "pxor %%xmm4, %%xmm0\n\t" + + "movdqa %%xmm0, %%xmm4\n\t" + "pclmulqdq $0x00, %%xmm6, %%xmm0\n\t" + "pclmulqdq $0x11, %%xmm6, %%xmm4\n\t" + "pxor %%xmm3, %%xmm0\n\t" + "pxor %%xmm4, %%xmm0\n\t" + : + : + ); + } + else + { + asm volatile ("movd %[crc], %%xmm1\n\t" + "movdqu %[inbuf], %%xmm0\n\t" + "movdqa %[k3k4], %%xmm6\n\t" + "pxor %%xmm1, %%xmm0\n\t" + "movdqa %[my_p], %%xmm5\n\t" + : + : [inbuf] "m" (*inbuf), + [crc] "m" (*pcrc), + [k3k4] "m" (consts->k[3 - 1]), + [my_p] "m" (consts->my_p[0]) + ); + + inbuf += 16; + inlen -= 16; + } + + /* Fold by 1. */ + if (inlen >= 16) + { + while (inlen >= 16) + { + /* Load next block to XMM2. Fold XMM0 to XMM0:XMM1. */ + asm volatile ("movdqu %[inbuf], %%xmm2\n\t" + "movdqa %%xmm0, %%xmm1\n\t" + "pclmulqdq $0x00, %%xmm6, %%xmm0\n\t" + "pclmulqdq $0x11, %%xmm6, %%xmm1\n\t" + "pxor %%xmm2, %%xmm0\n\t" + "pxor %%xmm1, %%xmm0\n\t" + : + : [inbuf] "m" (*inbuf) + ); + + inbuf += 16; + inlen -= 16; + } + } + + /* Partial fold. */ + if (inlen) + { + /* Load last input and add padding zeros. */ + asm volatile ("movdqu %[shr_shuf], %%xmm3\n\t" + "movdqu %[shl_shuf], %%xmm4\n\t" + "movdqu %[mask], %%xmm2\n\t" + + "movdqa %%xmm0, %%xmm1\n\t" + "pshufb %%xmm4, %%xmm0\n\t" + "movdqu %[inbuf], %%xmm4\n\t" + "pshufb %%xmm3, %%xmm1\n\t" + "pand %%xmm4, %%xmm2\n\t" + "por %%xmm1, %%xmm2\n\t" + + "movdqa %%xmm0, %%xmm1\n\t" + "pclmulqdq $0x00, %%xmm6, %%xmm0\n\t" + "pclmulqdq $0x11, %%xmm6, %%xmm1\n\t" + "pxor %%xmm2, %%xmm0\n\t" + "pxor %%xmm1, %%xmm0\n\t" + : + : [inbuf] "m" (*(inbuf - 16 + inlen)), + [mask] "m" (crc32_partial_fold_input_mask[inlen]), + [shl_shuf] "m" (crc32_refl_shuf_shift[inlen]), + [shr_shuf] "m" (crc32_refl_shuf_shift[inlen + 16]) + ); + + inbuf += inlen; + inlen -= inlen; + } + + /* Final fold. */ + asm volatile (/* reduce 128-bits to 96-bits */ + "movdqa %%xmm0, %%xmm1\n\t" + "pclmulqdq $0x10, %%xmm6, %%xmm0\n\t" + "psrldq $8, %%xmm1\n\t" + "pxor %%xmm1, %%xmm0\n\t" + + /* reduce 96-bits to 64-bits */ + "pshufd $0xfc, %%xmm0, %%xmm1\n\t" /* [00][00][00][x] */ + "pshufd $0xf9, %%xmm0, %%xmm0\n\t" /* [00][00][x>>64][x>>32] */ + "pclmulqdq $0x00, %[k5], %%xmm1\n\t" /* [00][00][xx][xx] */ + "pxor %%xmm1, %%xmm0\n\t" /* top 64-bit are zero */ + + /* barrett reduction */ + "pshufd $0xf3, %%xmm0, %%xmm1\n\t" /* [00][00][x>>32][00] */ + "pslldq $4, %%xmm0\n\t" /* [??][x>>32][??][??] */ + "pclmulqdq $0x00, %%xmm5, %%xmm1\n\t" /* [00][xx][xx][00] */ + "pclmulqdq $0x10, %%xmm5, %%xmm1\n\t" /* [00][xx][xx][00] */ + "pxor %%xmm1, %%xmm0\n\t" + + /* store CRC */ + "pextrd $2, %%xmm0, %[out]\n\t" + : [out] "=m" (*pcrc) + : [k5] "m" (consts->k[5 - 1]) + ); +} + +static inline void +crc32_reflected_less_than_16 (u32 *pcrc, const byte *inbuf, size_t inlen, + const struct crc32_consts_s *consts) +{ + if (inlen < 4) + { + u32 crc = *pcrc; + u32 data; + + asm volatile ("movdqa %[my_p], %%xmm5\n\t" + : + : [my_p] "m" (consts->my_p[0]) + ); + + if (inlen == 1) + { + data = inbuf[0]; + data ^= crc; + data <<= 24; + crc >>= 8; + } + else if (inlen == 2) + { + data = ((const struct u16_unaligned_s *)inbuf)->a; + data ^= crc; + data <<= 16; + crc >>= 16; + } + else + { + data = ((const struct u16_unaligned_s *)inbuf)->a; + data |= inbuf[2] << 16; + data ^= crc; + data <<= 8; + crc >>= 24; + } + + /* Barrett reduction */ + asm volatile ("movd %[in], %%xmm0\n\t" + "movd %[crc], %%xmm1\n\t" + + "pclmulqdq $0x00, %%xmm5, %%xmm0\n\t" /* [00][00][xx][xx] */ + "psllq $32, %%xmm1\n\t" + "pshufd $0xfc, %%xmm0, %%xmm0\n\t" /* [00][00][00][x] */ + "pclmulqdq $0x10, %%xmm5, %%xmm0\n\t" /* [00][00][xx][xx] */ + "pxor %%xmm1, %%xmm0\n\t" + + "pextrd $1, %%xmm0, %[out]\n\t" + : [out] "=m" (*pcrc) + : [in] "rm" (data), + [crc] "rm" (crc) + ); + } + else if (inlen == 4) + { + /* Barrett reduction */ + asm volatile ("movd %[crc], %%xmm1\n\t" + "movd %[in], %%xmm0\n\t" + "movdqa %[my_p], %%xmm5\n\t" + "pxor %%xmm1, %%xmm0\n\t" + + "pclmulqdq $0x00, %%xmm5, %%xmm0\n\t" /* [00][00][xx][xx] */ + "pshufd $0xfc, %%xmm0, %%xmm0\n\t" /* [00][00][00][x] */ + "pclmulqdq $0x10, %%xmm5, %%xmm0\n\t" /* [00][00][xx][xx] */ + + "pextrd $1, %%xmm0, %[out]\n\t" + : [out] "=m" (*pcrc) + : [in] "m" (*inbuf), + [crc] "m" (*pcrc), + [my_p] "m" (consts->my_p[0]) + ); + } + else + { + asm volatile ("movdqu %[shuf], %%xmm4\n\t" + "movd %[crc], %%xmm1\n\t" + "movdqa %[my_p], %%xmm5\n\t" + "movdqa %[k3k4], %%xmm6\n\t" + : + : [shuf] "m" (crc32_refl_shuf_shift[inlen]), + [crc] "m" (*pcrc), + [my_p] "m" (consts->my_p[0]), + [k3k4] "m" (consts->k[3 - 1]) + ); + + if (inlen >= 8) + { + asm volatile ("movq %[inbuf], %%xmm0\n\t" + : + : [inbuf] "m" (*inbuf) + ); + if (inlen > 8) + { + asm volatile (/*"pinsrq $1, %[inbuf_tail], %%xmm0\n\t"*/ + "movq %[inbuf_tail], %%xmm2\n\t" + "punpcklqdq %%xmm2, %%xmm0\n\t" + "pshufb %[merge_shuf], %%xmm0\n\t" + : + : [inbuf_tail] "m" (inbuf[inlen - 8]), + [merge_shuf] "m" + (*crc32_merge9to15_shuf[inlen - 9]) + ); + } + } + else + { + asm volatile ("movd %[inbuf], %%xmm0\n\t" + "pinsrd $1, %[inbuf_tail], %%xmm0\n\t" + "pshufb %[merge_shuf], %%xmm0\n\t" + : + : [inbuf] "m" (*inbuf), + [inbuf_tail] "m" (inbuf[inlen - 4]), + [merge_shuf] "m" + (*crc32_merge5to7_shuf[inlen - 5]) + ); + } + + /* Final fold. */ + asm volatile ("pxor %%xmm1, %%xmm0\n\t" + "pshufb %%xmm4, %%xmm0\n\t" + + /* reduce 128-bits to 96-bits */ + "movdqa %%xmm0, %%xmm1\n\t" + "pclmulqdq $0x10, %%xmm6, %%xmm0\n\t" + "psrldq $8, %%xmm1\n\t" + "pxor %%xmm1, %%xmm0\n\t" /* top 32-bit are zero */ + + /* reduce 96-bits to 64-bits */ + "pshufd $0xfc, %%xmm0, %%xmm1\n\t" /* [00][00][00][x] */ + "pshufd $0xf9, %%xmm0, %%xmm0\n\t" /* [00][00][x>>64][x>>32] */ + "pclmulqdq $0x00, %[k5], %%xmm1\n\t" /* [00][00][xx][xx] */ + "pxor %%xmm1, %%xmm0\n\t" /* top 64-bit are zero */ + + /* barrett reduction */ + "pshufd $0xf3, %%xmm0, %%xmm1\n\t" /* [00][00][x>>32][00] */ + "pslldq $4, %%xmm0\n\t" /* [??][x>>32][??][??] */ + "pclmulqdq $0x00, %%xmm5, %%xmm1\n\t" /* [00][xx][xx][00] */ + "pclmulqdq $0x10, %%xmm5, %%xmm1\n\t" /* [00][xx][xx][00] */ + "pxor %%xmm1, %%xmm0\n\t" + + /* store CRC */ + "pextrd $2, %%xmm0, %[out]\n\t" + : [out] "=m" (*pcrc) + : [k5] "m" (consts->k[5 - 1]) + ); + } +} + +void +crc32_intel_pclmul (u32 *pcrc, const byte *inbuf, size_t inlen) +{ + const struct crc32_consts_s *consts = &crc32_consts; +#if defined(__x86_64__) && defined(__WIN64__) + char win64tmp[2 * 16]; + + /* XMM6-XMM7 need to be restored after use. */ + asm volatile ("movdqu %%xmm6, 0*16(%0)\n\t" + "movdqu %%xmm7, 1*16(%0)\n\t" + : + : "r" (win64tmp) + : "memory"); +#endif + + if (!inlen) + return; + + if (inlen >= 16) + crc32_reflected_bulk(pcrc, inbuf, inlen, consts); + else + crc32_reflected_less_than_16(pcrc, inbuf, inlen, consts); + +#if defined(__x86_64__) && defined(__WIN64__) + /* Restore used registers. */ + asm volatile("movdqu 0*16(%0), %%xmm6\n\t" + "movdqu 1*16(%0), %%xmm7\n\t" + : + : "r" (win64tmp) + : "memory"); +#endif +} + +#endif diff --git a/extra/mariabackup/crc/crc-intel-pclmul.h b/extra/mariabackup/crc/crc-intel-pclmul.h new file mode 100644 index 00000000000..120058165a0 --- /dev/null +++ b/extra/mariabackup/crc/crc-intel-pclmul.h @@ -0,0 +1,25 @@ +/****************************************************** +Copyright (c) 2017 Percona LLC and/or its affiliates. + +CRC32 using Intel's PCLMUL instruction. + +This program is free software; you can redistribute it and/or modify +it under the terms of the GNU General Public License as published by +the Free Software Foundation; version 2 of the License. + +This program is distributed in the hope that it will be useful, +but WITHOUT ANY WARRANTY; without even the implied warranty of +MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the +GNU General Public License for more details. + +You should have received a copy of the GNU General Public License +along with this program; if not, write to the Free Software +Foundation, Inc., 51 Franklin Street, Fifth Floor, Boston, MA 02110-1301, USA + +*******************************************************/ + +#include +#include + +void +crc32_intel_pclmul(uint32_t *pcrc, const uint8_t *inbuf, size_t inlen); diff --git a/extra/mariabackup/crc/crc_glue.c b/extra/mariabackup/crc/crc_glue.c new file mode 100644 index 00000000000..f42910d8344 --- /dev/null +++ b/extra/mariabackup/crc/crc_glue.c @@ -0,0 +1,72 @@ +/****************************************************** +Copyright (c) 2017 Percona LLC and/or its affiliates. + +Zlib compatible CRC-32 implementation. + +This program is free software; you can redistribute it and/or modify +it under the terms of the GNU General Public License as published by +the Free Software Foundation; version 2 of the License. + +This program is distributed in the hope that it will be useful, +but WITHOUT ANY WARRANTY; without even the implied warranty of +MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the +GNU General Public License for more details. + +You should have received a copy of the GNU General Public License +along with this program; if not, write to the Free Software +Foundation, Inc., 51 Franklin Street, Fifth Floor, Boston, MA 02110-1301, USA + +*******************************************************/ + +#include +#include +#include +#include "crc_glue.h" +#include "crc-intel-pclmul.h" + +#if __GNUC__ >= 4 && defined(__x86_64__) +static int pclmul_enabled = 0; +#endif + +#if defined(__GNUC__) && defined(__x86_64__) +static +uint32_t +cpuid(uint32_t* ecx, uint32_t* edx) +{ + uint32_t level; + + asm("cpuid" : "=a" (level) : "a" (0) : "ebx", "ecx", "edx"); + + if (level < 1) { + return level; + } + + asm("cpuid" : "=c" (*ecx), "=d" (*edx) + : "a" (1) + : "ebx"); + + return level; +} +#endif + +void crc_init() { +#if defined(__GNUC__) && defined(__x86_64__) + uint32_t ecx, edx; + + if (cpuid(&ecx, &edx) > 0) { + pclmul_enabled = ((ecx >> 19) & 1) && ((ecx >> 1) & 1); + } +#endif +} + +unsigned long crc32_iso3309(unsigned long crc, const unsigned char *buf, unsigned int len) +{ +#if __GNUC__ >= 4 && defined(__x86_64__) && defined(HAVE_CLMUL_INSTRUCTION) + if (pclmul_enabled) { + uint32_t crc_accum = crc ^ 0xffffffffL; + crc32_intel_pclmul(&crc_accum, buf, len); + return crc_accum ^ 0xffffffffL; + } +#endif + return crc32(crc, buf, len); +} diff --git a/extra/mariabackup/crc/crc_glue.h b/extra/mariabackup/crc/crc_glue.h new file mode 100644 index 00000000000..e287fa4a7aa --- /dev/null +++ b/extra/mariabackup/crc/crc_glue.h @@ -0,0 +1,31 @@ +/****************************************************** +Copyright (c) 2017 Percona LLC and/or its affiliates. + +Zlib compatible CRC-32 implementation. + +This program is free software; you can redistribute it and/or modify +it under the terms of the GNU General Public License as published by +the Free Software Foundation; version 2 of the License. + +This program is distributed in the hope that it will be useful, +but WITHOUT ANY WARRANTY; without even the implied warranty of +MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the +GNU General Public License for more details. + +You should have received a copy of the GNU General Public License +along with this program; if not, write to the Free Software +Foundation, Inc., 51 Franklin Street, Fifth Floor, Boston, MA 02110-1301, USA + +*******************************************************/ + + +#ifdef __cplusplus +extern "C" { +#endif + +void crc_init(); +unsigned long crc32_iso3309(unsigned long crc, const unsigned char *buf, unsigned int len); + +#ifdef __cplusplus +} +#endif diff --git a/extra/mariabackup/datasink.c b/extra/mariabackup/datasink.c index 199eb77ad55..460e0e8ca19 100644 --- a/extra/mariabackup/datasink.c +++ b/extra/mariabackup/datasink.c @@ -27,7 +27,6 @@ Foundation, Inc., 51 Franklin Street, Fifth Floor, Boston, MA 02110-1301, USA #include "ds_local.h" #include "ds_stdout.h" #include "ds_tmpfile.h" -#include "ds_encrypt.h" #include "ds_buffer.h" /************************************************************************ @@ -60,6 +59,7 @@ ds_create(const char *root, ds_type_t type) ds = &datasink_compress; break; case DS_TYPE_ENCRYPT: + case DS_TYPE_DECRYPT: msg("Error : mariabackup does not support encrypted backups."); exit(EXIT_FAILURE); break; diff --git a/extra/mariabackup/datasink.h b/extra/mariabackup/datasink.h index e378a806441..8bf1321aad1 100644 --- a/extra/mariabackup/datasink.h +++ b/extra/mariabackup/datasink.h @@ -61,6 +61,7 @@ typedef enum { DS_TYPE_XBSTREAM, DS_TYPE_COMPRESS, DS_TYPE_ENCRYPT, + DS_TYPE_DECRYPT, DS_TYPE_TMPFILE, DS_TYPE_BUFFER } ds_type_t; diff --git a/extra/mariabackup/ds_decrypt.c b/extra/mariabackup/ds_decrypt.c new file mode 100644 index 00000000000..e897ca101e5 --- /dev/null +++ b/extra/mariabackup/ds_decrypt.c @@ -0,0 +1,665 @@ +/****************************************************** +Copyright (c) 2017 Percona LLC and/or its affiliates. + +Encryption datasink implementation for XtraBackup. + +This program is free software; you can redistribute it and/or modify +it under the terms of the GNU General Public License as published by +the Free Software Foundation; version 2 of the License. + +This program is distributed in the hope that it will be useful, +but WITHOUT ANY WARRANTY; without even the implied warranty of +MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the +GNU General Public License for more details. + +You should have received a copy of the GNU General Public License +along with this program; if not, write to the Free Software +Foundation, Inc., 51 Franklin Street, Fifth Floor, Boston, MA 02110-1301, USA + +*******************************************************/ + + +#include +#include "common.h" +#include "datasink.h" +#include "xbcrypt.h" +#include "xbcrypt_common.h" +#include "crc_glue.h" + +typedef struct { + pthread_t id; + uint num; + pthread_mutex_t ctrl_mutex; + pthread_cond_t ctrl_cond; + pthread_mutex_t data_mutex; + pthread_cond_t data_cond; + my_bool started; + my_bool data_avail; + my_bool cancelled; + my_bool failed; + const uchar *from; + size_t from_len; + uchar *to; + size_t to_len; + size_t to_size; + const uchar *iv; + size_t iv_len; + unsigned long long offset; + my_bool hash_appended; + gcry_cipher_hd_t cipher_handle; + xb_rcrypt_result_t parse_result; +} crypt_thread_ctxt_t; + +typedef struct { + crypt_thread_ctxt_t *threads; + uint nthreads; + int encrypt_algo; + size_t chunk_size; + char *encrypt_key; + char *encrypt_key_file; +} ds_decrypt_ctxt_t; + +typedef struct { + ds_decrypt_ctxt_t *crypt_ctxt; + size_t bytes_processed; + ds_file_t *dest_file; + uchar *buf; + size_t buf_len; + size_t buf_size; +} ds_decrypt_file_t; + +int ds_decrypt_encrypt_threads = 1; + +static ds_ctxt_t *decrypt_init(const char *root); +static ds_file_t *decrypt_open(ds_ctxt_t *ctxt, const char *path, + MY_STAT *mystat); +static int decrypt_write(ds_file_t *file, const void *buf, size_t len); +static int decrypt_close(ds_file_t *file); +static void decrypt_deinit(ds_ctxt_t *ctxt); + +datasink_t datasink_decrypt = { + &decrypt_init, + &decrypt_open, + &decrypt_write, + &decrypt_close, + &decrypt_deinit +}; + +static crypt_thread_ctxt_t *create_worker_threads(uint n); +static void destroy_worker_threads(crypt_thread_ctxt_t *threads, uint n); +static void *decrypt_worker_thread_func(void *arg); + +static +ds_ctxt_t * +decrypt_init(const char *root) +{ + ds_ctxt_t *ctxt; + ds_decrypt_ctxt_t *decrypt_ctxt; + crypt_thread_ctxt_t *threads; + + if (xb_crypt_init(NULL)) { + return NULL; + } + + /* Create and initialize the worker threads */ + threads = create_worker_threads(ds_decrypt_encrypt_threads); + if (threads == NULL) { + msg("decrypt: failed to create worker threads.\n"); + return NULL; + } + + ctxt = (ds_ctxt_t *) my_malloc(sizeof(ds_ctxt_t) + + sizeof(ds_decrypt_ctxt_t), + MYF(MY_FAE)); + + decrypt_ctxt = (ds_decrypt_ctxt_t *) (ctxt + 1); + decrypt_ctxt->threads = threads; + decrypt_ctxt->nthreads = ds_decrypt_encrypt_threads; + + ctxt->ptr = decrypt_ctxt; + ctxt->root = my_strdup(root, MYF(MY_FAE)); + + return ctxt; +} + +static +ds_file_t * +decrypt_open(ds_ctxt_t *ctxt, const char *path, MY_STAT *mystat) +{ + ds_ctxt_t *dest_ctxt; + + ds_decrypt_ctxt_t *crypt_ctxt; + ds_decrypt_file_t *crypt_file; + + char new_name[FN_REFLEN]; + ds_file_t *file; + + xb_ad(ctxt->pipe_ctxt != NULL); + dest_ctxt = ctxt->pipe_ctxt; + + crypt_ctxt = (ds_decrypt_ctxt_t *) ctxt->ptr; + + + file = (ds_file_t *) my_malloc(sizeof(ds_file_t) + + sizeof(ds_decrypt_file_t), + MYF(MY_FAE|MY_ZEROFILL)); + + crypt_file = (ds_decrypt_file_t *) (file + 1); + + /* Remove the .xbcrypt extension from the filename */ + strncpy(new_name, path, FN_REFLEN); + new_name[strlen(new_name) - 8] = 0; + crypt_file->dest_file = ds_open(dest_ctxt, new_name, mystat); + if (crypt_file->dest_file == NULL) { + msg("decrypt: ds_open(\"%s\") failed.\n", new_name); + goto err; + } + + crypt_file->crypt_ctxt = crypt_ctxt; + crypt_file->buf = NULL; + crypt_file->buf_size = 0; + crypt_file->buf_len = 0; + + file->ptr = crypt_file; + file->path = crypt_file->dest_file->path; + + return file; + +err: + if (crypt_file->dest_file) { + ds_close(crypt_file->dest_file); + } + my_free(file); + return NULL; +} + +#define CHECK_BUF_SIZE(ptr, size, buf, len) \ + if (ptr + size - buf > (ssize_t) len) { \ + result = XB_CRYPT_READ_INCOMPLETE; \ + goto exit; \ + } + +static +xb_rcrypt_result_t +parse_xbcrypt_chunk(crypt_thread_ctxt_t *thd, const uchar *buf, size_t len, + size_t *bytes_processed) +{ + const uchar *ptr; + uint version; + ulong checksum, checksum_exp; + ulonglong tmp; + xb_rcrypt_result_t result = XB_CRYPT_READ_CHUNK; + + *bytes_processed = 0; + ptr = buf; + + CHECK_BUF_SIZE(ptr, XB_CRYPT_CHUNK_MAGIC_SIZE, buf, len); + if (memcmp(ptr, XB_CRYPT_CHUNK_MAGIC3, + XB_CRYPT_CHUNK_MAGIC_SIZE) == 0) { + version = 3; + } else if (memcmp(ptr, XB_CRYPT_CHUNK_MAGIC2, + XB_CRYPT_CHUNK_MAGIC_SIZE) == 0) { + version = 2; + } else if (memcmp(ptr, XB_CRYPT_CHUNK_MAGIC1, + XB_CRYPT_CHUNK_MAGIC_SIZE) == 0) { + version = 1; + } else { + msg("%s:%s: wrong chunk magic at offset 0x%llx.\n", + my_progname, __FUNCTION__, thd->offset); + result = XB_CRYPT_READ_ERROR; + goto exit; + } + + ptr += XB_CRYPT_CHUNK_MAGIC_SIZE; + thd->offset += XB_CRYPT_CHUNK_MAGIC_SIZE; + + CHECK_BUF_SIZE(ptr, 8, buf, len); + tmp = uint8korr(ptr); /* reserved */ + ptr += 8; + thd->offset += 8; + + CHECK_BUF_SIZE(ptr, 8, buf, len); + tmp = uint8korr(ptr); /* original size */ + ptr += 8; + if (tmp > INT_MAX) { + msg("%s:%s: invalid original size at offset 0x%llx.\n", + my_progname, __FUNCTION__, thd->offset); + result = XB_CRYPT_READ_ERROR; + goto exit; + } + thd->offset += 8; + thd->to_len = (size_t)tmp; + + if (thd->to_size < thd->to_len + XB_CRYPT_HASH_LEN) { + thd->to = (uchar *) my_realloc( + thd->to, + thd->to_len + XB_CRYPT_HASH_LEN, + MYF(MY_FAE | MY_ALLOW_ZERO_PTR)); + thd->to_size = thd->to_len; + } + + CHECK_BUF_SIZE(ptr, 8, buf, len); + tmp = uint8korr(ptr); /* encrypted size */ + ptr += 8; + if (tmp > INT_MAX) { + msg("%s:%s: invalid encrypted size at offset 0x%llx.\n", + my_progname, __FUNCTION__, thd->offset); + result = XB_CRYPT_READ_ERROR; + goto exit; + } + thd->offset += 8; + thd->from_len = (size_t)tmp; + + xb_a(thd->from_len <= thd->to_len + XB_CRYPT_HASH_LEN); + + CHECK_BUF_SIZE(ptr, 4, buf, len); + checksum_exp = uint4korr(ptr); /* checksum */ + ptr += 4; + thd->offset += 4; + + /* iv size */ + if (version == 1) { + thd->iv_len = 0; + thd->iv = NULL; + } else { + CHECK_BUF_SIZE(ptr, 8, buf, len); + + tmp = uint8korr(ptr); + if (tmp > INT_MAX) { + msg("%s:%s: invalid iv size at offset 0x%llx.\n", + my_progname, __FUNCTION__, thd->offset); + result = XB_CRYPT_READ_ERROR; + goto exit; + } + ptr += 8; + thd->offset += 8; + thd->iv_len = (size_t)tmp; + } + + if (thd->iv_len > 0) { + CHECK_BUF_SIZE(ptr, thd->iv_len, buf, len); + thd->iv = ptr; + ptr += thd->iv_len; + } + + /* for version euqals 2 we need to read in the iv data but do not init + CTR with it */ + if (version == 2) { + thd->iv_len = 0; + thd->iv = 0; + } + + if (thd->from_len > 0) { + CHECK_BUF_SIZE(ptr, thd->from_len, buf, len); + thd->from = ptr; + ptr += thd->from_len; + } + + xb_ad(thd->from_len <= thd->to_len); + + checksum = crc32_iso3309(0, thd->from, thd->from_len); + if (checksum != checksum_exp) { + msg("%s:%s invalid checksum at offset 0x%llx, " + "expected 0x%lx, actual 0x%lx.\n", my_progname, + __FUNCTION__, thd->offset, checksum_exp, checksum); + result = XB_CRYPT_READ_ERROR; + goto exit; + } + + thd->offset += thd->from_len; + + thd->hash_appended = version > 2; + +exit: + + *bytes_processed = (size_t) (ptr - buf); + + return result; +} + +static +int +decrypt_write(ds_file_t *file, const void *buf, size_t len) +{ + ds_decrypt_file_t *crypt_file; + ds_decrypt_ctxt_t *crypt_ctxt; + crypt_thread_ctxt_t *threads; + crypt_thread_ctxt_t *thd; + uint nthreads; + uint i; + size_t bytes_processed; + xb_rcrypt_result_t parse_result = XB_CRYPT_READ_CHUNK; + my_bool err = FALSE; + + crypt_file = (ds_decrypt_file_t *) file->ptr; + crypt_ctxt = crypt_file->crypt_ctxt; + + threads = crypt_ctxt->threads; + nthreads = crypt_ctxt->nthreads; + + if (crypt_file->buf_len > 0) { + thd = threads; + + pthread_mutex_lock(&thd->ctrl_mutex); + + do { + if (parse_result == XB_CRYPT_READ_INCOMPLETE) { + crypt_file->buf_size = crypt_file->buf_size * 2; + crypt_file->buf = (uchar *) my_realloc( + crypt_file->buf, + crypt_file->buf_size, + MYF(MY_FAE|MY_ALLOW_ZERO_PTR)); + } + + memcpy(crypt_file->buf + crypt_file->buf_len, + buf, MY_MIN(crypt_file->buf_size - + crypt_file->buf_len, len)); + + parse_result = parse_xbcrypt_chunk( + thd, crypt_file->buf, + crypt_file->buf_size, &bytes_processed); + + if (parse_result == XB_CRYPT_READ_ERROR) { + pthread_mutex_unlock(&thd->ctrl_mutex); + return 1; + } + + } while (parse_result == XB_CRYPT_READ_INCOMPLETE && + crypt_file->buf_size < len); + + if (parse_result != XB_CRYPT_READ_CHUNK) { + msg("decrypt: incomplete data.\n"); + pthread_mutex_unlock(&thd->ctrl_mutex); + return 1; + } + + pthread_mutex_lock(&thd->data_mutex); + thd->data_avail = TRUE; + pthread_cond_signal(&thd->data_cond); + pthread_mutex_unlock(&thd->data_mutex); + + len -= bytes_processed - crypt_file->buf_len; + buf += bytes_processed - crypt_file->buf_len; + + /* reap */ + + pthread_mutex_lock(&thd->data_mutex); + while (thd->data_avail == TRUE) { + pthread_cond_wait(&thd->data_cond, + &thd->data_mutex); + } + + if (thd->failed) { + msg("decrypt: failed to decrypt chunk.\n"); + err = TRUE; + } + + xb_a(thd->to_len > 0); + + if (!err && + ds_write(crypt_file->dest_file, thd->to, thd->to_len)) { + msg("decrypt: write to destination failed.\n"); + err = TRUE; + } + + crypt_file->bytes_processed += thd->from_len; + + pthread_mutex_unlock(&thd->data_mutex); + pthread_mutex_unlock(&thd->ctrl_mutex); + + crypt_file->buf_len = 0; + + if (err) { + return 1; + } + } + + while (parse_result == XB_CRYPT_READ_CHUNK && len > 0) { + uint max_thread; + + for (i = 0; i < nthreads; i++) { + thd = threads + i; + + pthread_mutex_lock(&thd->ctrl_mutex); + + parse_result = parse_xbcrypt_chunk( + thd, buf, len, &bytes_processed); + + if (parse_result == XB_CRYPT_READ_ERROR) { + pthread_mutex_unlock(&thd->ctrl_mutex); + err = TRUE; + break; + } + + thd->parse_result = parse_result; + + if (parse_result != XB_CRYPT_READ_CHUNK) { + pthread_mutex_unlock(&thd->ctrl_mutex); + break; + } + + pthread_mutex_lock(&thd->data_mutex); + thd->data_avail = TRUE; + pthread_cond_signal(&thd->data_cond); + pthread_mutex_unlock(&thd->data_mutex); + + len -= bytes_processed; + buf += bytes_processed; + } + + max_thread = (i < nthreads) ? i : nthreads - 1; + + /* Reap and write decrypted data */ + for (i = 0; i <= max_thread; i++) { + thd = threads + i; + + if (thd->parse_result != XB_CRYPT_READ_CHUNK) { + break; + } + + pthread_mutex_lock(&thd->data_mutex); + while (thd->data_avail == TRUE) { + pthread_cond_wait(&thd->data_cond, + &thd->data_mutex); + } + + if (thd->failed) { + msg("decrypt: failed to decrypt chunk.\n"); + err = TRUE; + } + + xb_a(thd->to_len > 0); + + if (!err && ds_write(crypt_file->dest_file, thd->to, + thd->to_len)) { + msg("decrypt: write to destination failed.\n"); + err = TRUE; + } + + crypt_file->bytes_processed += thd->from_len; + + pthread_mutex_unlock(&thd->data_mutex); + pthread_mutex_unlock(&thd->ctrl_mutex); + } + + if (err) { + return 1; + } + } + + if (parse_result == XB_CRYPT_READ_INCOMPLETE && len > 0) { + crypt_file->buf_len = len; + if (crypt_file->buf_size < len) { + crypt_file->buf = (uchar *) my_realloc( + crypt_file->buf, + crypt_file->buf_len, + MYF(MY_FAE | MY_ALLOW_ZERO_PTR)); + crypt_file->buf_size = len; + } + memcpy(crypt_file->buf, buf, len); + } + + return 0; +} + +static +int +decrypt_close(ds_file_t *file) +{ + ds_decrypt_file_t *crypt_file; + ds_file_t *dest_file; + int rc = 0; + + crypt_file = (ds_decrypt_file_t *) file->ptr; + dest_file = crypt_file->dest_file; + + if (ds_close(dest_file)) { + rc = 1; + } + + my_free(crypt_file->buf); + my_free(file); + + return rc; +} + +static +void +decrypt_deinit(ds_ctxt_t *ctxt) +{ + ds_decrypt_ctxt_t *crypt_ctxt; + + xb_ad(ctxt->pipe_ctxt != NULL); + + crypt_ctxt = (ds_decrypt_ctxt_t *) ctxt->ptr; + + destroy_worker_threads(crypt_ctxt->threads, crypt_ctxt->nthreads); + + my_free(ctxt->root); + my_free(ctxt); +} + +static +crypt_thread_ctxt_t * +create_worker_threads(uint n) +{ + crypt_thread_ctxt_t *threads; + uint i; + + threads = (crypt_thread_ctxt_t *) + my_malloc(sizeof(crypt_thread_ctxt_t) * n, + MYF(MY_FAE | MY_ZEROFILL)); + + for (i = 0; i < n; i++) { + crypt_thread_ctxt_t *thd = threads + i; + + thd->num = i + 1; + + /* Initialize the control mutex and condition var */ + if (pthread_mutex_init(&thd->ctrl_mutex, NULL) || + pthread_cond_init(&thd->ctrl_cond, NULL)) { + goto err; + } + + /* Initialize and data mutex and condition var */ + if (pthread_mutex_init(&thd->data_mutex, NULL) || + pthread_cond_init(&thd->data_cond, NULL)) { + goto err; + } + + xb_crypt_cipher_open(&thd->cipher_handle); + + pthread_mutex_lock(&thd->ctrl_mutex); + + if (pthread_create(&thd->id, NULL, decrypt_worker_thread_func, + thd)) { + msg("decrypt: pthread_create() failed: " + "errno = %d\n", errno); + goto err; + } + } + + /* Wait for the threads to start */ + for (i = 0; i < n; i++) { + crypt_thread_ctxt_t *thd = threads + i; + + while (thd->started == FALSE) + pthread_cond_wait(&thd->ctrl_cond, &thd->ctrl_mutex); + pthread_mutex_unlock(&thd->ctrl_mutex); + } + + return threads; + +err: + return NULL; +} + +static +void +destroy_worker_threads(crypt_thread_ctxt_t *threads, uint n) +{ + uint i; + + for (i = 0; i < n; i++) { + crypt_thread_ctxt_t *thd = threads + i; + + pthread_mutex_lock(&thd->data_mutex); + threads[i].cancelled = TRUE; + pthread_cond_signal(&thd->data_cond); + pthread_mutex_unlock(&thd->data_mutex); + + pthread_join(thd->id, NULL); + + pthread_cond_destroy(&thd->data_cond); + pthread_mutex_destroy(&thd->data_mutex); + pthread_cond_destroy(&thd->ctrl_cond); + pthread_mutex_destroy(&thd->ctrl_mutex); + + xb_crypt_cipher_close(thd->cipher_handle); + + my_free(thd->to); + } + + my_free(threads); +} + +static +void * +decrypt_worker_thread_func(void *arg) +{ + crypt_thread_ctxt_t *thd = (crypt_thread_ctxt_t *) arg; + + pthread_mutex_lock(&thd->ctrl_mutex); + + pthread_mutex_lock(&thd->data_mutex); + + thd->started = TRUE; + pthread_cond_signal(&thd->ctrl_cond); + + pthread_mutex_unlock(&thd->ctrl_mutex); + + while (1) { + thd->data_avail = FALSE; + pthread_cond_signal(&thd->data_cond); + + while (!thd->data_avail && !thd->cancelled) { + pthread_cond_wait(&thd->data_cond, &thd->data_mutex); + } + + if (thd->cancelled) + break; + + if (xb_crypt_decrypt(thd->cipher_handle, thd->from, + thd->from_len, thd->to, &thd->to_len, + thd->iv, thd->iv_len, + thd->hash_appended)) { + thd->failed = TRUE; + continue; + } + + } + + pthread_mutex_unlock(&thd->data_mutex); + + return NULL; +} diff --git a/extra/mariabackup/ds_decrypt.h b/extra/mariabackup/ds_decrypt.h new file mode 100644 index 00000000000..3bb4de55f54 --- /dev/null +++ b/extra/mariabackup/ds_decrypt.h @@ -0,0 +1,30 @@ +/****************************************************** +Copyright (c) 2017 Percona LLC and/or its affiliates. + +Encryption interface for XtraBackup. + +This program is free software; you can redistribute it and/or modify +it under the terms of the GNU General Public License as published by +the Free Software Foundation; version 2 of the License. + +This program is distributed in the hope that it will be useful, +but WITHOUT ANY WARRANTY; without even the implied warranty of +MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the +GNU General Public License for more details. + +You should have received a copy of the GNU General Public License +along with this program; if not, write to the Free Software +Foundation, Inc., 51 Franklin Street, Fifth Floor, Boston, MA 02110-1301, USA + +*******************************************************/ + +#ifndef DS_DECRYPT_H +#define DS_DECRYPT_H + +#include "datasink.h" + +extern datasink_t datasink_decrypt; + +extern int ds_decrypt_encrypt_threads; + +#endif diff --git a/extra/mariabackup/ds_encrypt.c b/extra/mariabackup/ds_encrypt.c index 19ced416687..576ea207eb1 100644 --- a/extra/mariabackup/ds_encrypt.c +++ b/extra/mariabackup/ds_encrypt.c @@ -22,25 +22,11 @@ Foundation, Inc., 51 Franklin Street, Fifth Floor, Boston, MA 02110-1301, USA #include #include "common.h" #include "datasink.h" +#include "xbcrypt_common.h" #ifdef HAVE_GRYPT -#if GCC_VERSION >= 4002 -/* Workaround to avoid "gcry_ac_* is deprecated" warnings in gcrypt.h */ -# pragma GCC diagnostic ignored "-Wdeprecated-declarations" -#endif - -#include - -#if GCC_VERSION >= 4002 -# pragma GCC diagnostic warning "-Wdeprecated-declarations" -#endif - #include "xbcrypt.h" -#if !defined(GCRYPT_VERSION_NUMBER) || (GCRYPT_VERSION_NUMBER < 0x010600) -GCRY_THREAD_OPTION_PTHREAD_IMPL; -#endif - -#define XB_CRYPT_CHUNK_SIZE ((size_t) (xtrabackup_encrypt_chunk_size)) +#define XB_CRYPT_CHUNK_SIZE ((size_t) (ds_encrypt_encrypt_chunk_size)) typedef struct { pthread_t id; @@ -52,10 +38,10 @@ typedef struct { my_bool started; my_bool data_avail; my_bool cancelled; - const char *from; + const uchar *from; size_t from_len; - char *to; - char *iv; + uchar *to; + uchar *iv; size_t to_len; gcry_cipher_hd_t cipher_handle; } crypt_thread_ctxt_t; @@ -73,11 +59,8 @@ typedef struct { } ds_encrypt_file_t; /* Encryption options */ -extern ulong xtrabackup_encrypt_algo; -extern char *xtrabackup_encrypt_key; -extern char *xtrabackup_encrypt_key_file; -extern uint xtrabackup_encrypt_threads; -extern ulonglong xtrabackup_encrypt_chunk_size; +uint ds_encrypt_encrypt_threads; +ulonglong ds_encrypt_encrypt_chunk_size; static ds_ctxt_t *encrypt_init(const char *root); static ds_file_t *encrypt_open(ds_ctxt_t *ctxt, const char *path, @@ -98,12 +81,7 @@ static crypt_thread_ctxt_t *create_worker_threads(uint n); static void destroy_worker_threads(crypt_thread_ctxt_t *threads, uint n); static void *encrypt_worker_thread_func(void *arg); -static uint encrypt_algos[] = { GCRY_CIPHER_NONE, GCRY_CIPHER_AES128, - GCRY_CIPHER_AES192, GCRY_CIPHER_AES256 }; -static uint encrypt_algo; -static const uint encrypt_mode = GCRY_CIPHER_MODE_CTR; -static uint encrypt_key_len = 0; -static size_t encrypt_iv_len = 0; +static uint encrypt_iv_len = 0; static ssize_t @@ -129,87 +107,13 @@ encrypt_init(const char *root) ds_ctxt_t *ctxt; ds_encrypt_ctxt_t *encrypt_ctxt; crypt_thread_ctxt_t *threads; - gcry_error_t gcry_error; - /* Acording to gcrypt docs (and my testing), setting up the threading - callbacks must be done first, so, lets give it a shot */ -#if !defined(GCRYPT_VERSION_NUMBER) || (GCRYPT_VERSION_NUMBER < 0x010600) - gcry_error = gcry_control(GCRYCTL_SET_THREAD_CBS, &gcry_threads_pthread); - if (gcry_error) { - msg("encrypt: unable to set libgcrypt thread cbs - " - "%s : %s\n", - gcry_strsource(gcry_error), - gcry_strerror(gcry_error)); - return NULL; - } -#endif - - /* Version check should be the very next call because it - makes sure that important subsystems are intialized. */ - if (!gcry_control(GCRYCTL_ANY_INITIALIZATION_P)) { - const char *gcrypt_version; - gcrypt_version = gcry_check_version(NULL); - /* No other library has already initialized libgcrypt. */ - if (!gcrypt_version) { - msg("encrypt: failed to initialize libgcrypt\n"); - return NULL; - } else { - msg("encrypt: using gcrypt %s\n", gcrypt_version); - } - } - - /* Disable the gcry secure memory, not dealing with this for now */ - gcry_error = gcry_control(GCRYCTL_DISABLE_SECMEM, 0); - if (gcry_error) { - msg("encrypt: unable to disable libgcrypt secmem - " - "%s : %s\n", - gcry_strsource(gcry_error), - gcry_strerror(gcry_error)); - return NULL; - } - - /* Finalize gcry initialization. */ - gcry_error = gcry_control(GCRYCTL_INITIALIZATION_FINISHED, 0); - if (gcry_error) { - msg("encrypt: unable to finish libgcrypt initialization - " - "%s : %s\n", - gcry_strsource(gcry_error), - gcry_strerror(gcry_error)); - return NULL; - } - - /* Determine the algorithm */ - encrypt_algo = encrypt_algos[xtrabackup_encrypt_algo]; - - /* Set up the iv length */ - encrypt_iv_len = gcry_cipher_get_algo_blklen(encrypt_algo); - xb_a(encrypt_iv_len > 0); - - /* Now set up the key */ - if (xtrabackup_encrypt_key == NULL && - xtrabackup_encrypt_key_file == NULL) { - msg("encrypt: no encryption key or key file specified.\n"); - return NULL; - } else if (xtrabackup_encrypt_key && xtrabackup_encrypt_key_file) { - msg("encrypt: both encryption key and key file specified.\n"); - return NULL; - } else if (xtrabackup_encrypt_key_file) { - if (!xb_crypt_read_key_file(xtrabackup_encrypt_key_file, - (void**)&xtrabackup_encrypt_key, - &encrypt_key_len)) { - msg("encrypt: unable to read encryption key file" - " \"%s\".\n", xtrabackup_encrypt_key_file); - return NULL; - } - } else if (xtrabackup_encrypt_key) { - encrypt_key_len = strlen(xtrabackup_encrypt_key); - } else { - msg("encrypt: no encryption key or key file specified.\n"); + if (xb_crypt_init(&encrypt_iv_len)) { return NULL; } /* Create and initialize the worker threads */ - threads = create_worker_threads(xtrabackup_encrypt_threads); + threads = create_worker_threads(ds_encrypt_encrypt_threads); if (threads == NULL) { msg("encrypt: failed to create worker threads.\n"); return NULL; @@ -221,7 +125,7 @@ encrypt_init(const char *root) encrypt_ctxt = (ds_encrypt_ctxt_t *) (ctxt + 1); encrypt_ctxt->threads = threads; - encrypt_ctxt->nthreads = xtrabackup_encrypt_threads; + encrypt_ctxt->nthreads = ds_encrypt_encrypt_threads; ctxt->ptr = encrypt_ctxt; ctxt->root = my_strdup(root, MYF(MY_FAE)); @@ -294,7 +198,7 @@ encrypt_write(ds_file_t *file, const void *buf, size_t len) crypt_thread_ctxt_t *thd; uint nthreads; uint i; - const char *ptr; + const uchar *ptr; crypt_file = (ds_encrypt_file_t *) file->ptr; crypt_ctxt = crypt_file->crypt_ctxt; @@ -302,7 +206,7 @@ encrypt_write(ds_file_t *file, const void *buf, size_t len) threads = crypt_ctxt->threads; nthreads = crypt_ctxt->nthreads; - ptr = (const char *) buf; + ptr = (const uchar *) buf; while (len > 0) { uint max_thread; @@ -403,10 +307,6 @@ encrypt_deinit(ds_ctxt_t *ctxt) my_free(ctxt->root); my_free(ctxt); - if (xtrabackup_encrypt_key) - my_free(xtrabackup_encrypt_key); - if (xtrabackup_encrypt_key_file) - my_free(xtrabackup_encrypt_key_file); } static @@ -427,11 +327,10 @@ create_worker_threads(uint n) thd->cancelled = FALSE; thd->data_avail = FALSE; - thd->to = (char *) my_malloc(XB_CRYPT_CHUNK_SIZE + - XB_CRYPT_HASH_LEN, MYF(MY_FAE)); + thd->to = (uchar *) my_malloc(XB_CRYPT_CHUNK_SIZE + + XB_CRYPT_HASH_LEN, MYF(MY_FAE)); - thd->iv = (char *) my_malloc(encrypt_iv_len, - MYF(MY_FAE)); + thd->iv = (uchar *) my_malloc(encrypt_iv_len, MYF(MY_FAE)); /* Initialize the control mutex and condition var */ if (pthread_mutex_init(&thd->ctrl_mutex, NULL) || @@ -445,32 +344,8 @@ create_worker_threads(uint n) goto err; } - if (encrypt_algo != GCRY_CIPHER_NONE) { - gcry_error_t gcry_error; - - gcry_error = gcry_cipher_open(&thd->cipher_handle, - encrypt_algo, - encrypt_mode, 0); - if (gcry_error) { - msg("encrypt: unable to open libgcrypt" - " cipher - %s : %s\n", - gcry_strsource(gcry_error), - gcry_strerror(gcry_error)); - gcry_cipher_close(thd->cipher_handle); - goto err; - } - - gcry_error = gcry_cipher_setkey(thd->cipher_handle, - xtrabackup_encrypt_key, - encrypt_key_len); - if (gcry_error) { - msg("encrypt: unable to set libgcrypt" - " cipher key - %s : %s\n", - gcry_strsource(gcry_error), - gcry_strerror(gcry_error)); - gcry_cipher_close(thd->cipher_handle); - goto err; - } + if (xb_crypt_cipher_open(&thd->cipher_handle)) { + goto err; } pthread_mutex_lock(&thd->ctrl_mutex); @@ -519,8 +394,7 @@ destroy_worker_threads(crypt_thread_ctxt_t *threads, uint n) pthread_cond_destroy(&thd->ctrl_cond); pthread_mutex_destroy(&thd->ctrl_mutex); - if (encrypt_algo != GCRY_CIPHER_NONE) - gcry_cipher_close(thd->cipher_handle); + xb_crypt_cipher_close(thd->cipher_handle); my_free(thd->to); my_free(thd->iv); @@ -555,60 +429,14 @@ encrypt_worker_thread_func(void *arg) if (thd->cancelled) break; - /* ensure that XB_CRYPT_HASH_LEN is the correct length - of XB_CRYPT_HASH hashing algorithm output */ - assert(gcry_md_get_algo_dlen(XB_CRYPT_HASH) == - XB_CRYPT_HASH_LEN); - - memcpy(thd->to, thd->from, thd->from_len); - gcry_md_hash_buffer(XB_CRYPT_HASH, thd->to + thd->from_len, - thd->from, thd->from_len); thd->to_len = thd->from_len; - if (encrypt_algo != GCRY_CIPHER_NONE) { - gcry_error_t gcry_error; - - gcry_error = gcry_cipher_reset(thd->cipher_handle); - if (gcry_error) { - msg("encrypt: unable to reset cipher - " - "%s : %s\n", - gcry_strsource(gcry_error), - gcry_strerror(gcry_error)); - thd->to_len = 0; - continue; - } - - xb_crypt_create_iv(thd->iv, encrypt_iv_len); - gcry_error = gcry_cipher_setctr(thd->cipher_handle, - thd->iv, - encrypt_iv_len); - if (gcry_error) { - msg("encrypt: unable to set cipher ctr - " - "%s : %s\n", - gcry_strsource(gcry_error), - gcry_strerror(gcry_error)); - thd->to_len = 0; - continue; - } - - gcry_error = gcry_cipher_encrypt(thd->cipher_handle, - thd->to, - thd->to_len + - XB_CRYPT_HASH_LEN, - thd->to, - thd->from_len + - XB_CRYPT_HASH_LEN); - if (gcry_error) { - msg("encrypt: unable to encrypt buffer - " - "%s : %s\n", gcry_strsource(gcry_error), - gcry_strerror(gcry_error)); - thd->to_len = 0; - } - } else { - memcpy(thd->to, thd->from, - thd->from_len + XB_CRYPT_HASH_LEN); + if (xb_crypt_encrypt(thd->cipher_handle, thd->from, + thd->from_len, thd->to, &thd->to_len, + thd->iv)) { + thd->to_len = 0; + continue; } - thd->to_len += XB_CRYPT_HASH_LEN; } pthread_mutex_unlock(&thd->data_mutex); diff --git a/extra/mariabackup/ds_encrypt.h b/extra/mariabackup/ds_encrypt.h index 3e9e40ad354..c4d8d7f8427 100644 --- a/extra/mariabackup/ds_encrypt.h +++ b/extra/mariabackup/ds_encrypt.h @@ -25,4 +25,9 @@ Foundation, Inc., 51 Franklin Street, Fifth Floor, Boston, MA 02110-1301, USA #ifdef HAVE_GCRYPT extern datasink_t datasink_encrypt; #endif +/* Encryption options */ +extern uint ds_encrypt_encrypt_threads; +extern ulonglong ds_encrypt_encrypt_chunk_size; + + #endif diff --git a/extra/mariabackup/xbcrypt.c b/extra/mariabackup/xbcrypt.c index 255da875de4..3da70e171f7 100644 --- a/extra/mariabackup/xbcrypt.c +++ b/extra/mariabackup/xbcrypt.c @@ -22,7 +22,8 @@ Foundation, Inc., 51 Franklin Street, Fifth Floor, Boston, MA 02110-1301, USA #include #include "common.h" #include "xbcrypt.h" -#include +#include "xbcrypt_common.h" +#include "crc_glue.h" #if !defined(GCRYPT_VERSION_NUMBER) || (GCRYPT_VERSION_NUMBER < 0x010600) GCRY_THREAD_OPTION_PTHREAD_IMPL; @@ -138,6 +139,8 @@ main(int argc, char **argv) MY_INIT(argv[0]); + crc_init(); + if (get_options(&argc, &argv)) { goto err; } @@ -402,8 +405,8 @@ mode_decrypt(File filein, File fileout) /* ensure that XB_CRYPT_HASH_LEN is the correct length of XB_CRYPT_HASH hashing algorithm output */ - assert(gcry_md_get_algo_dlen(XB_CRYPT_HASH) == - XB_CRYPT_HASH_LEN); + xb_a(gcry_md_get_algo_dlen(XB_CRYPT_HASH) == + XB_CRYPT_HASH_LEN); gcry_md_hash_buffer(XB_CRYPT_HASH, hash, decryptbuf, originalsize); if (memcmp(hash, (char *) decryptbuf + originalsize, @@ -529,8 +532,7 @@ mode_encrypt(File filein, File fileout) /* ensure that XB_CRYPT_HASH_LEN is the correct length of XB_CRYPT_HASH hashing algorithm output */ - assert(XB_CRYPT_HASH_LEN == - gcry_md_get_algo_dlen(XB_CRYPT_HASH)); + xb_a(XB_CRYPT_HASH_LEN == gcry_md_get_algo_dlen(XB_CRYPT_HASH)); gcry_md_hash_buffer(XB_CRYPT_HASH, chunkbuf + bytesread, chunkbuf, bytesread); diff --git a/extra/mariabackup/xbcrypt.h b/extra/mariabackup/xbcrypt.h index cdabf56a21a..0e832266847 100644 --- a/extra/mariabackup/xbcrypt.h +++ b/extra/mariabackup/xbcrypt.h @@ -65,6 +65,7 @@ xb_rcrypt_t *xb_crypt_read_open(void *userdata, typedef enum { XB_CRYPT_READ_CHUNK, + XB_CRYPT_READ_INCOMPLETE, XB_CRYPT_READ_EOF, XB_CRYPT_READ_ERROR } xb_rcrypt_result_t; @@ -75,10 +76,4 @@ xb_rcrypt_result_t xb_crypt_read_chunk(xb_rcrypt_t *crypt, void **buf, int xb_crypt_read_close(xb_rcrypt_t *crypt); -/****************************************************************************** -Utility interface */ -my_bool xb_crypt_read_key_file(const char *filename, - void** key, uint *keylength); - -void xb_crypt_create_iv(void* ivbuf, size_t ivlen); #endif diff --git a/extra/mariabackup/xbcrypt_common.c b/extra/mariabackup/xbcrypt_common.c index 52fa2ce3589..0cdb54dc66d 100644 --- a/extra/mariabackup/xbcrypt_common.c +++ b/extra/mariabackup/xbcrypt_common.c @@ -1,5 +1,5 @@ /****************************************************** -Copyright (c) 2013 Percona LLC and/or its affiliates. +Copyright (c) 2013, 2017 Percona LLC and/or its affiliates. Encryption configuration file interface for XtraBackup. @@ -21,18 +21,26 @@ Foundation, Inc., 51 Franklin Street, Fifth Floor, Boston, MA 02110-1301, USA #include #include "common.h" #include "xbcrypt.h" +#include "xbcrypt_common.h" -#if GCC_VERSION >= 4002 -/* Workaround to avoid "gcry_ac_* is deprecated" warnings in gcrypt.h */ -# pragma GCC diagnostic ignored "-Wdeprecated-declarations" +/* Encryption options */ +char *ds_encrypt_key = NULL; +char *ds_encrypt_key_file = NULL; +ulong ds_encrypt_algo; + +static uint encrypt_key_len; +static uint encrypt_iv_len; + +static const uint encrypt_mode = GCRY_CIPHER_MODE_CTR; + +static uint encrypt_algos[] = { GCRY_CIPHER_NONE, GCRY_CIPHER_AES128, + GCRY_CIPHER_AES192, GCRY_CIPHER_AES256 }; +static uint encrypt_algo; + +#if !defined(GCRYPT_VERSION_NUMBER) || (GCRYPT_VERSION_NUMBER < 0x010600) +GCRY_THREAD_OPTION_PTHREAD_IMPL; #endif -#ifdef HAVE_GRYPT -#include - -#if GCC_VERSION >= 4002 -# pragma GCC diagnostic warning "-Wdeprecated-declarations" -#endif my_bool xb_crypt_read_key_file(const char *filename, void** key, uint *keylength) @@ -59,4 +67,262 @@ xb_crypt_create_iv(void* ivbuf, size_t ivlen) { gcry_create_nonce(ivbuf, ivlen); } + +gcry_error_t +xb_crypt_init(uint *iv_len) +{ + gcry_error_t gcry_error; + + /* Acording to gcrypt docs (and my testing), setting up the threading + callbacks must be done first, so, lets give it a shot */ +#if !defined(GCRYPT_VERSION_NUMBER) || (GCRYPT_VERSION_NUMBER < 0x010600) + gcry_error = gcry_control(GCRYCTL_SET_THREAD_CBS, &gcry_threads_pthread); + if (gcry_error) { + msg("encryption: unable to set libgcrypt thread cbs - " + "%s : %s\n", + gcry_strsource(gcry_error), + gcry_strerror(gcry_error)); + return gcry_error; + } +#endif + + /* Version check should be the very next call because it + makes sure that important subsystems are intialized. */ + if (!gcry_control(GCRYCTL_ANY_INITIALIZATION_P)) { + const char *gcrypt_version; + gcrypt_version = gcry_check_version(NULL); + /* No other library has already initialized libgcrypt. */ + if (!gcrypt_version) { + msg("encryption: failed to initialize libgcrypt\n"); + return 1; + } else { + msg("encryption: using gcrypt %s\n", gcrypt_version); + } + } + + /* Disable the gcry secure memory, not dealing with this for now */ + gcry_error = gcry_control(GCRYCTL_DISABLE_SECMEM, 0); + if (gcry_error) { + msg("encryption: unable to disable libgcrypt secmem - " + "%s : %s\n", + gcry_strsource(gcry_error), + gcry_strerror(gcry_error)); + return gcry_error; + } + + /* Finalize gcry initialization. */ + gcry_error = gcry_control(GCRYCTL_INITIALIZATION_FINISHED, 0); + if (gcry_error) { + msg("encryption: unable to finish libgcrypt initialization - " + "%s : %s\n", + gcry_strsource(gcry_error), + gcry_strerror(gcry_error)); + return gcry_error; + } + + /* Determine the algorithm */ + encrypt_algo = encrypt_algos[ds_encrypt_algo]; + + /* Set up the iv length */ + encrypt_iv_len = gcry_cipher_get_algo_blklen(encrypt_algo); + xb_a(encrypt_iv_len > 0); + if (iv_len != NULL) { + *iv_len = encrypt_iv_len; + } + + /* Now set up the key */ + if (ds_encrypt_key == NULL && + ds_encrypt_key_file == NULL) { + msg("encryption: no encryption key or key file specified.\n"); + return gcry_error; + } else if (ds_encrypt_key && ds_encrypt_key_file) { + msg("encryption: both encryption key and key file specified.\n"); + return gcry_error; + } else if (ds_encrypt_key_file) { + if (!xb_crypt_read_key_file(ds_encrypt_key_file, + (void**)&ds_encrypt_key, + &encrypt_key_len)) { + msg("encryption: unable to read encryption key file" + " \"%s\".\n", ds_encrypt_key_file); + return gcry_error; + } + } else if (ds_encrypt_key) { + encrypt_key_len = strlen(ds_encrypt_key); + } else { + msg("encryption: no encryption key or key file specified.\n"); + return gcry_error; + } + + return 0; +} + +gcry_error_t +xb_crypt_cipher_open(gcry_cipher_hd_t *cipher_handle) +{ + if (encrypt_algo != GCRY_CIPHER_NONE) { + gcry_error_t gcry_error; + + gcry_error = gcry_cipher_open(cipher_handle, + encrypt_algo, + encrypt_mode, 0); + if (gcry_error) { + msg("encryption: unable to open libgcrypt" + " cipher - %s : %s\n", + gcry_strsource(gcry_error), + gcry_strerror(gcry_error)); + gcry_cipher_close(*cipher_handle); + return gcry_error; + } + + gcry_error = gcry_cipher_setkey(*cipher_handle, + ds_encrypt_key, + encrypt_key_len); + if (gcry_error) { + msg("encryption: unable to set libgcrypt" + " cipher key - %s : %s\n", + gcry_strsource(gcry_error), + gcry_strerror(gcry_error)); + gcry_cipher_close(*cipher_handle); + return gcry_error; + } + return gcry_error; + } + return 0; +} + +void +xb_crypt_cipher_close(gcry_cipher_hd_t cipher_handle) +{ + if (encrypt_algo != GCRY_CIPHER_NONE) + gcry_cipher_close(cipher_handle); +} + +gcry_error_t +xb_crypt_decrypt(gcry_cipher_hd_t cipher_handle, const uchar *from, + size_t from_len, uchar *to, size_t *to_len, + const uchar *iv, size_t iv_len, my_bool hash_appended) +{ + *to_len = from_len; + + if (encrypt_algo != GCRY_CIPHER_NONE) { + + gcry_error_t gcry_error; + + gcry_error = gcry_cipher_reset(cipher_handle); + if (gcry_error) { + msg("%s:encryption: unable to reset libgcrypt" + " cipher - %s : %s\n", my_progname, + gcry_strsource(gcry_error), + gcry_strerror(gcry_error)); + return gcry_error; + } + + if (iv_len > 0) { + gcry_error = gcry_cipher_setctr(cipher_handle, + iv, iv_len); + } + if (gcry_error) { + msg("%s:encryption: unable to set cipher iv - " + "%s : %s\n", my_progname, + gcry_strsource(gcry_error), + gcry_strerror(gcry_error)); + return gcry_error; + } + + /* Try to decrypt it */ + gcry_error = gcry_cipher_decrypt(cipher_handle, to, *to_len, + from, from_len); + if (gcry_error) { + msg("%s:encryption: unable to decrypt chunk - " + "%s : %s\n", my_progname, + gcry_strsource(gcry_error), + gcry_strerror(gcry_error)); + gcry_cipher_close(cipher_handle); + return gcry_error; + } + + if (hash_appended) { + uchar hash[XB_CRYPT_HASH_LEN]; + + *to_len -= XB_CRYPT_HASH_LEN; + + /* ensure that XB_CRYPT_HASH_LEN is the correct length + of XB_CRYPT_HASH hashing algorithm output */ + xb_ad(gcry_md_get_algo_dlen(XB_CRYPT_HASH) == + XB_CRYPT_HASH_LEN); + gcry_md_hash_buffer(XB_CRYPT_HASH, hash, to, + *to_len); + if (memcmp(hash, (char *) to + *to_len, + XB_CRYPT_HASH_LEN) != 0) { + msg("%s:%s invalid plaintext hash. " + "Wrong encrytion key specified?\n", + my_progname, __FUNCTION__); + return 1; + } + } + + } else { + memcpy(to, from, *to_len); + } + + return 0; +} + +gcry_error_t +xb_crypt_encrypt(gcry_cipher_hd_t cipher_handle, const uchar *from, + size_t from_len, uchar *to, size_t *to_len, uchar *iv) +{ + gcry_error_t gcry_error; + + /* ensure that XB_CRYPT_HASH_LEN is the correct length + of XB_CRYPT_HASH hashing algorithm output */ + xb_ad(gcry_md_get_algo_dlen(XB_CRYPT_HASH) == + XB_CRYPT_HASH_LEN); + + memcpy(to, from, from_len); + gcry_md_hash_buffer(XB_CRYPT_HASH, to + from_len, + from, from_len); + + *to_len = from_len; + + if (encrypt_algo != GCRY_CIPHER_NONE) { + + gcry_error = gcry_cipher_reset(cipher_handle); + if (gcry_error) { + msg("encrypt: unable to reset cipher - " + "%s : %s\n", + gcry_strsource(gcry_error), + gcry_strerror(gcry_error)); + return gcry_error; + } + + xb_crypt_create_iv(iv, encrypt_iv_len); + gcry_error = gcry_cipher_setctr(cipher_handle, iv, + encrypt_iv_len); + if (gcry_error) { + msg("encrypt: unable to set cipher ctr - " + "%s : %s\n", + gcry_strsource(gcry_error), + gcry_strerror(gcry_error)); + return gcry_error; + } + + gcry_error = gcry_cipher_encrypt(cipher_handle, to, + *to_len + XB_CRYPT_HASH_LEN, + to, + from_len + XB_CRYPT_HASH_LEN); + if (gcry_error) { + msg("encrypt: unable to encrypt buffer - " + "%s : %s\n", gcry_strsource(gcry_error), + gcry_strerror(gcry_error)); + return gcry_error; + } + } else { + memcpy(to, from, from_len + XB_CRYPT_HASH_LEN); + } + + *to_len += XB_CRYPT_HASH_LEN; + + return 0; +} #endif \ No newline at end of file diff --git a/extra/mariabackup/xbcrypt_common.h b/extra/mariabackup/xbcrypt_common.h new file mode 100644 index 00000000000..85d13c01fc4 --- /dev/null +++ b/extra/mariabackup/xbcrypt_common.h @@ -0,0 +1,64 @@ +/****************************************************** +Copyright (c) 2017 Percona LLC and/or its affiliates. + +Encryption datasink implementation for XtraBackup. + +This program is free software; you can redistribute it and/or modify +it under the terms of the GNU General Public License as published by +the Free Software Foundation; version 2 of the License. + +This program is distributed in the hope that it will be useful, +but WITHOUT ANY WARRANTY; without even the implied warranty of +MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the +GNU General Public License for more details. + +You should have received a copy of the GNU General Public License +along with this program; if not, write to the Free Software +Foundation, Inc., 51 Franklin Street, Fifth Floor, Boston, MA 02110-1301, USA + +*******************************************************/ + +#include +#if HAVE_GCRYPT +#if GCC_VERSION >= 4002 +/* Workaround to avoid "gcry_ac_* is deprecated" warnings in gcrypt.h */ +# pragma GCC diagnostic ignored "-Wdeprecated-declarations" +#endif + +#include + +extern char *ds_encrypt_key; +extern char *ds_encrypt_key_file; +extern int ds_encrypt_threads; +extern ulong ds_encrypt_algo; + +/****************************************************************************** +Utility interface */ +my_bool xb_crypt_read_key_file(const char *filename, + void** key, uint *keylength); + +void xb_crypt_create_iv(void* ivbuf, size_t ivlen); + +/* Initialize gcrypt and setup encryption key and IV lengths */ +gcry_error_t +xb_crypt_init(uint *iv_len); + +/* Setup gcrypt cipher */ +gcry_error_t +xb_crypt_cipher_open(gcry_cipher_hd_t *cipher_handle); + +/* Close gcrypt cipher */ +void +xb_crypt_cipher_close(gcry_cipher_hd_t cipher_handle); + +/* Decrypt buffer */ +gcry_error_t +xb_crypt_decrypt(gcry_cipher_hd_t cipher_handle, const uchar *from, + size_t from_len, uchar *to, size_t *to_len, const uchar *iv, + size_t iv_len, my_bool hash_appended); + +/* Encrypt buffer */ +gcry_error_t +xb_crypt_encrypt(gcry_cipher_hd_t cipher_handle, const uchar *from, + size_t from_len, uchar *to, size_t *to_len, uchar *iv); +#endif diff --git a/extra/mariabackup/xbcrypt_read.c b/extra/mariabackup/xbcrypt_read.c index 6522333f023..41790c7035d 100644 --- a/extra/mariabackup/xbcrypt_read.c +++ b/extra/mariabackup/xbcrypt_read.c @@ -19,6 +19,7 @@ Foundation, Inc., 51 Franklin Street, Fifth Floor, Boston, MA 02110-1301, USA *******************************************************/ #include "xbcrypt.h" +#include "crc_glue.h" struct xb_rcrypt_struct { void *userdata; @@ -212,7 +213,7 @@ xb_crypt_read_chunk(xb_rcrypt_t *crypt, void **buf, size_t *olen, size_t *elen, } } - checksum = crc32(0, crypt->buffer, *elen); + checksum = crc32_iso3309(0, crypt->buffer, *elen); if (checksum != checksum_exp) { msg("%s:%s invalid checksum at offset 0x%llx, " "expected 0x%lx, actual 0x%lx.\n", my_progname, __FUNCTION__, diff --git a/extra/mariabackup/xbcrypt_write.c b/extra/mariabackup/xbcrypt_write.c index 6cc30852215..91dbfc4eb29 100644 --- a/extra/mariabackup/xbcrypt_write.c +++ b/extra/mariabackup/xbcrypt_write.c @@ -19,6 +19,7 @@ Foundation, Inc., 51 Franklin Street, Fifth Floor, Boston, MA 02110-1301, USA *******************************************************/ #include "xbcrypt.h" +#include "crc_glue.h" struct xb_wcrypt_struct { void *userdata; @@ -73,7 +74,7 @@ int xb_crypt_write_chunk(xb_wcrypt_t *crypt, const void *buf, size_t olen, int8store(ptr, (ulonglong)elen); /* encrypted (actual) size */ ptr += 8; - checksum = crc32(0, buf, (uint)elen); + checksum = crc32_iso3309(0, buf, elen); int4store(ptr, checksum); /* checksum */ ptr += 4; diff --git a/extra/mariabackup/xbstream.c b/extra/mariabackup/xbstream.c index 9990b00ea4b..2cc47ec7273 100644 --- a/extra/mariabackup/xbstream.c +++ b/extra/mariabackup/xbstream.c @@ -22,10 +22,13 @@ Foundation, Inc., 51 Franklin Street, Fifth Floor, Boston, MA 02110-1301, USA #include #include #include +#include #include "common.h" #include "xbstream.h" -#include "ds_local.h" -#include "ds_stdout.h" +#include "xbcrypt_common.h" +#include "datasink.h" +#include "ds_decrypt.h" +#include "crc_glue.h" #define XBSTREAM_VERSION "1.0" #define XBSTREAM_BUFFER_SIZE (10 * 1024 * 1024UL) @@ -38,6 +41,12 @@ typedef enum { RUN_MODE_EXTRACT } run_mode_t; +const char *xbstream_encrypt_algo_names[] = +{ "NONE", "AES128", "AES192", "AES256", NullS}; +TYPELIB xbstream_encrypt_algo_typelib= +{array_elements(xbstream_encrypt_algo_names)-1,"", + xbstream_encrypt_algo_names, NULL}; + /* Need the following definitions to avoid linking with ds_*.o and their link dependencies */ datasink_t datasink_archive; @@ -47,9 +56,18 @@ datasink_t datasink_tmpfile; datasink_t datasink_encrypt; datasink_t datasink_buffer; -static run_mode_t opt_mode; +static run_mode_t opt_mode; static char * opt_directory = NULL; static my_bool opt_verbose = 0; +static int opt_parallel = 1; +static ulong opt_encrypt_algo; +static char *opt_encrypt_key_file = NULL; +static void *opt_encrypt_key = NULL; +static int opt_encrypt_threads = 1; + +enum { + OPT_ENCRYPT_THREADS = 256 +}; static struct my_option my_long_options[] = { @@ -65,21 +83,46 @@ static struct my_option my_long_options[] = GET_STR_ALLOC, REQUIRED_ARG, 0, 0, 0, 0, 0, 0}, {"verbose", 'v', "Print verbose output.", &opt_verbose, &opt_verbose, 0, GET_BOOL, NO_ARG, 0, 0, 0, 0, 0, 0}, + {"parallel", 'p', "Number of worker threads for reading / writing.", + &opt_parallel, &opt_parallel, 0, GET_INT, REQUIRED_ARG, + 1, 1, INT_MAX, 0, 0, 0}, + {"decrypt", 'd', "Decrypt files ending with .xbcrypt.", + &opt_encrypt_algo, &opt_encrypt_algo, &xbstream_encrypt_algo_typelib, + GET_ENUM, REQUIRED_ARG, 0, 0, 0, 0, 0, 0}, + {"encrypt-key", 'k', "Encryption key.", + &opt_encrypt_key, &opt_encrypt_key, 0, + GET_STR_ALLOC, REQUIRED_ARG, 0, 0, 0, 0, 0, 0}, + {"encrypt-key-file", 'f', "File which contains encryption key.", + &opt_encrypt_key_file, &opt_encrypt_key_file, 0, + GET_STR_ALLOC, REQUIRED_ARG, 0, 0, 0, 0, 0, 0}, + {"encrypt-threads", OPT_ENCRYPT_THREADS, + "Number of threads for parallel data encryption. " + "The default value is 1.", + &opt_encrypt_threads, &opt_encrypt_threads, + 0, GET_INT, REQUIRED_ARG, 1, 1, INT_MAX, 0, 0, 0}, {0, 0, 0, 0, 0, 0, GET_NO_ARG, NO_ARG, 0, 0, 0, 0, 0, 0} }; +typedef struct { + HASH *filehash; + xb_rstream_t *stream; + ds_ctxt_t *ds_ctxt; + ds_ctxt_t *ds_decrypt_ctxt; + pthread_mutex_t *mutex; +} extract_ctxt_t; + typedef struct { char *path; uint pathlen; my_off_t offset; - ds_ctxt_t *ds_ctxt; ds_file_t *file; + pthread_mutex_t mutex; } file_entry_t; static int get_options(int *argc, char ***argv); static int mode_create(int argc, char **argv); -static int mode_extract(int argc, char **argv); +static int mode_extract(int n_threads, int argc, char **argv); static my_bool get_one_option(int optid, const struct my_option *opt, char *argument); @@ -88,6 +131,8 @@ main(int argc, char **argv) { MY_INIT(argv[0]); + crc_init(); + if (get_options(&argc, &argv)) { goto err; } @@ -104,7 +149,8 @@ main(int argc, char **argv) if (opt_mode == RUN_MODE_CREATE && mode_create(argc, argv)) { goto err; - } else if (opt_mode == RUN_MODE_EXTRACT && mode_extract(argc, argv)) { + } else if (opt_mode == RUN_MODE_EXTRACT && + mode_extract(opt_parallel, argc, argv)) { goto err; } @@ -302,9 +348,22 @@ err: return 1; } +/************************************************************************ +Check if string ends with given suffix. +@return true if string ends with given suffix. */ +static +my_bool +ends_with(const char *str, const char *suffix) +{ + size_t suffix_len = strlen(suffix); + size_t str_len = strlen(str); + return(str_len >= suffix_len + && strcmp(str + str_len - suffix_len, suffix) == 0); +} + static file_entry_t * -file_entry_new(ds_ctxt_t *ds_ctxt, const char *path, uint pathlen) +file_entry_new(extract_ctxt_t *ctxt, const char *path, uint pathlen) { file_entry_t *entry; ds_file_t *file; @@ -321,7 +380,11 @@ file_entry_new(ds_ctxt_t *ds_ctxt, const char *path, uint pathlen) } entry->pathlen = pathlen; - file = ds_open(ds_ctxt, path, NULL); + if (ctxt->ds_decrypt_ctxt && ends_with(path, ".xbcrypt")) { + file = ds_open(ctxt->ds_decrypt_ctxt, path, NULL); + } else { + file = ds_open(ctxt->ds_ctxt, path, NULL); + } if (file == NULL) { msg("%s: failed to create file.\n", my_progname); goto err; @@ -332,7 +395,8 @@ file_entry_new(ds_ctxt_t *ds_ctxt, const char *path, uint pathlen) } entry->file = file; - entry->ds_ctxt = ds_ctxt; + + pthread_mutex_init(&entry->mutex, NULL); return entry; @@ -358,68 +422,77 @@ static void file_entry_free(file_entry_t *entry) { + pthread_mutex_destroy(&entry->mutex); ds_close(entry->file); my_free(entry->path); my_free(entry); } static -int -mode_extract(int argc __attribute__((unused)), - char **argv __attribute__((unused))) +void * +extract_worker_thread_func(void *arg) { - xb_rstream_t *stream; - xb_rstream_result_t res; xb_rstream_chunk_t chunk; - HASH filehash; file_entry_t *entry; - ds_ctxt_t *ds_ctxt; + xb_rstream_result_t res; - stream = xb_stream_read_new(); - if (stream == NULL) { - msg("%s: xb_stream_read_new() failed.\n", my_progname); - return 1; - } + extract_ctxt_t *ctxt = (extract_ctxt_t *) arg; - /* If --directory is specified, it is already set as CWD by now. */ - ds_ctxt = ds_create(".", DS_TYPE_LOCAL); + my_thread_init(); - if (my_hash_init(&filehash, &my_charset_bin, START_FILE_HASH_SIZE, - 0, 0, (my_hash_get_key) get_file_entry_key, - (my_hash_free_key) file_entry_free, MYF(0))) { - msg("%s: failed to initialize file hash.\n", my_progname); - goto err; - } + memset(&chunk, 0, sizeof(chunk)); + + while (1) { + + pthread_mutex_lock(ctxt->mutex); + res = xb_stream_read_chunk(ctxt->stream, &chunk); + + if (res != XB_STREAM_READ_CHUNK) { + pthread_mutex_unlock(ctxt->mutex); + break; + } - while ((res = xb_stream_read_chunk(stream, &chunk)) == - XB_STREAM_READ_CHUNK) { /* If unknown type and ignorable flag is set, skip this chunk */ if (chunk.type == XB_CHUNK_TYPE_UNKNOWN && \ !(chunk.flags & XB_STREAM_FLAG_IGNORABLE)) { + pthread_mutex_unlock(ctxt->mutex); continue; } /* See if we already have this file open */ - entry = (file_entry_t *) my_hash_search(&filehash, + entry = (file_entry_t *) my_hash_search(ctxt->filehash, (uchar *) chunk.path, chunk.pathlen); if (entry == NULL) { - entry = file_entry_new(ds_ctxt, chunk.path, + entry = file_entry_new(ctxt, + chunk.path, chunk.pathlen); if (entry == NULL) { - goto err; + pthread_mutex_unlock(ctxt->mutex); + break; } - if (my_hash_insert(&filehash, (uchar *) entry)) { + if (my_hash_insert(ctxt->filehash, (uchar *) entry)) { msg("%s: my_hash_insert() failed.\n", my_progname); - goto err; + pthread_mutex_unlock(ctxt->mutex); + break; } } - if (chunk.type == XB_CHUNK_TYPE_EOF) { - my_hash_delete(&filehash, (uchar *) entry); + pthread_mutex_lock(&entry->mutex); + pthread_mutex_unlock(ctxt->mutex); + + res = xb_stream_validate_checksum(&chunk); + + if (res != XB_STREAM_READ_CHUNK) { + pthread_mutex_unlock(&entry->mutex); + break; + } + + if (chunk.type == XB_CHUNK_TYPE_EOF) { + pthread_mutex_unlock(&entry->mutex); continue; } @@ -427,30 +500,114 @@ mode_extract(int argc __attribute__((unused)), msg("%s: out-of-order chunk: real offset = 0x%llx, " "expected offset = 0x%llx\n", my_progname, chunk.offset, entry->offset); - goto err; + pthread_mutex_unlock(&entry->mutex); + res = XB_STREAM_READ_ERROR; + break; } if (ds_write(entry->file, chunk.data, chunk.length)) { msg("%s: my_write() failed.\n", my_progname); - goto err; + pthread_mutex_unlock(&entry->mutex); + res = XB_STREAM_READ_ERROR; + break; } entry->offset += chunk.length; - }; - if (res == XB_STREAM_READ_ERROR) { - goto err; + pthread_mutex_unlock(&entry->mutex); } - my_hash_free(&filehash); - ds_destroy(ds_ctxt); - xb_stream_read_done(stream); + if (chunk.data) + my_free(chunk.data); - return 0; -err: - my_hash_free(&filehash); - ds_destroy(ds_ctxt); - xb_stream_read_done(stream); + my_thread_end(); - return 1; + return (void *)(res); +} + + +static +int +mode_extract(int n_threads, int argc __attribute__((unused)), + char **argv __attribute__((unused))) +{ + xb_rstream_t *stream = NULL; + HASH filehash; + ds_ctxt_t *ds_ctxt = NULL; + ds_ctxt_t *ds_decrypt_ctxt = NULL; + extract_ctxt_t ctxt; + int i; + pthread_t *tids = NULL; + void **retvals = NULL; + pthread_mutex_t mutex; + int ret = 0; + + if (my_hash_init(&filehash, &my_charset_bin, START_FILE_HASH_SIZE, + 0, 0, (my_hash_get_key) get_file_entry_key, + (my_hash_free_key) file_entry_free, MYF(0))) { + msg("%s: failed to initialize file hash.\n", my_progname); + return 1; + } + + if (pthread_mutex_init(&mutex, NULL)) { + msg("%s: failed to initialize mutex.\n", my_progname); + my_hash_free(&filehash); + return 1; + } + + /* If --directory is specified, it is already set as CWD by now. */ + ds_ctxt = ds_create(".", DS_TYPE_LOCAL); + if (ds_ctxt == NULL) { + ret = 1; + goto exit; + } + + + stream = xb_stream_read_new(); + if (stream == NULL) { + msg("%s: xb_stream_read_new() failed.\n", my_progname); + pthread_mutex_destroy(&mutex); + ret = 1; + goto exit; + } + + ctxt.stream = stream; + ctxt.filehash = &filehash; + ctxt.ds_ctxt = ds_ctxt; + ctxt.ds_decrypt_ctxt = ds_decrypt_ctxt; + ctxt.mutex = &mutex; + + tids = malloc(sizeof(pthread_t) * n_threads); + retvals = malloc(sizeof(void*) * n_threads); + + for (i = 0; i < n_threads; i++) + pthread_create(tids + i, NULL, extract_worker_thread_func, + &ctxt); + + for (i = 0; i < n_threads; i++) + pthread_join(tids[i], retvals + i); + + for (i = 0; i < n_threads; i++) { + if ((ulong)retvals[i] == XB_STREAM_READ_ERROR) { + ret = 1; + goto exit; + } + } + +exit: + pthread_mutex_destroy(&mutex); + + free(tids); + free(retvals); + + my_hash_free(&filehash); + if (ds_ctxt != NULL) { + ds_destroy(ds_ctxt); + } + if (ds_decrypt_ctxt) { + ds_destroy(ds_decrypt_ctxt); + } + xb_stream_read_done(stream); + + return ret; } diff --git a/extra/mariabackup/xbstream.h b/extra/mariabackup/xbstream.h index e9f1468e58d..ac1bf05e321 100644 --- a/extra/mariabackup/xbstream.h +++ b/extra/mariabackup/xbstream.h @@ -1,5 +1,5 @@ /****************************************************** -Copyright (c) 2011-2013 Percona LLC and/or its affiliates. +Copyright (c) 2011-2017 Percona LLC and/or its affiliates. The xbstream format interface. @@ -89,8 +89,10 @@ typedef struct { char path[FN_REFLEN]; size_t length; my_off_t offset; + my_off_t checksum_offset; void *data; ulong checksum; + size_t buflen; } xb_rstream_chunk_t; xb_rstream_t *xb_stream_read_new(void); @@ -100,4 +102,6 @@ xb_rstream_result_t xb_stream_read_chunk(xb_rstream_t *stream, int xb_stream_read_done(xb_rstream_t *stream); +int xb_stream_validate_checksum(xb_rstream_chunk_t *chunk); + #endif diff --git a/extra/mariabackup/xbstream_read.c b/extra/mariabackup/xbstream_read.c index f8d88926ec2..8d19242301b 100644 --- a/extra/mariabackup/xbstream_read.c +++ b/extra/mariabackup/xbstream_read.c @@ -1,5 +1,5 @@ /****************************************************** -Copyright (c) 2011-2013 Percona LLC and/or its affiliates. +Copyright (c) 2011-2017 Percona LLC and/or its affiliates. The xbstream format reader implementation. @@ -23,6 +23,7 @@ Foundation, Inc., 51 Franklin Street, Fifth Floor, Boston, MA 02110-1301, USA #include #include "common.h" #include "xbstream.h" +#include "crc_glue.h" /* Allocate 1 MB for the payload buffer initially */ #define INIT_BUFFER_LEN (1024 * 1024) @@ -34,8 +35,6 @@ Foundation, Inc., 51 Franklin Street, Fifth Floor, Boston, MA 02110-1301, USA struct xb_rstream_struct { my_off_t offset; File fd; - void *buffer; - size_t buflen; }; xb_rstream_t * @@ -45,9 +44,6 @@ xb_stream_read_new(void) stream = (xb_rstream_t *) my_malloc(sizeof(xb_rstream_t), MYF(MY_FAE)); - stream->buffer = my_malloc(INIT_BUFFER_LEN, MYF(MY_FAE)); - stream->buflen = INIT_BUFFER_LEN; - #ifdef __WIN__ setmode(fileno(stdin), _O_BINARY); #endif @@ -71,6 +67,23 @@ validate_chunk_type(uchar code) } } +int +xb_stream_validate_checksum(xb_rstream_chunk_t *chunk) +{ + ulong checksum; + + checksum = crc32_iso3309(0, chunk->data, (uint)chunk->length); + if (checksum != chunk->checksum) { + msg("xb_stream_read_chunk(): invalid checksum at offset " + "0x%llx: expected 0x%lx, read 0x%lx.\n", + (ulonglong) chunk->checksum_offset, chunk->checksum, + checksum); + return XB_STREAM_READ_ERROR; + } + + return XB_STREAM_READ_CHUNK; +} + #define F_READ(buf,len) \ do { \ if (xb_read_full(fd, buf, len) < len) { \ @@ -87,8 +100,6 @@ xb_stream_read_chunk(xb_rstream_t *stream, xb_rstream_chunk_t *chunk) uint pathlen; size_t tbytes; ulonglong ullval; - ulong checksum_exp; - ulong checksum; File fd = stream->fd; xb_ad(sizeof(tmpbuf) >= CHUNK_HEADER_CONSTANT_LEN); @@ -178,39 +189,30 @@ xb_stream_read_chunk(xb_rstream_t *stream, xb_rstream_chunk_t *chunk) stream->offset += 8; /* Reallocate the buffer if needed */ - if (chunk->length > stream->buflen) { - stream->buffer = my_realloc(stream->buffer, chunk->length, - MYF(MY_WME)); - if (stream->buffer == NULL) { + if (chunk->length > chunk->buflen) { + chunk->data = my_realloc(chunk->data, chunk->length, + MYF(MY_WME | MY_ALLOW_ZERO_PTR)); + if (chunk->data == NULL) { msg("xb_stream_read_chunk(): failed to increase buffer " "to %lu bytes.\n", (ulong) chunk->length); goto err; } - stream->buflen = chunk->length; + chunk->buflen = chunk->length; } /* Checksum */ F_READ(tmpbuf, 4); - checksum_exp = uint4korr(tmpbuf); + chunk->checksum = uint4korr(tmpbuf); + chunk->checksum_offset = stream->offset; /* Payload */ if (chunk->length > 0) { - F_READ(stream->buffer, chunk->length); + F_READ(chunk->data, chunk->length); stream->offset += chunk->length; } - checksum = crc32(0, stream->buffer, chunk->length); - if (checksum != checksum_exp) { - msg("xb_stream_read_chunk(): invalid checksum at offset " - "0x%llx: expected 0x%lx, read 0x%lx.\n", - (ulonglong) stream->offset, checksum_exp, checksum); - goto err; - } stream->offset += 4; - chunk->data = stream->buffer; - chunk->checksum = checksum; - return XB_STREAM_READ_CHUNK; err: @@ -220,7 +222,6 @@ err: int xb_stream_read_done(xb_rstream_t *stream) { - my_free(stream->buffer); my_free(stream); return 0; diff --git a/extra/mariabackup/xbstream_write.c b/extra/mariabackup/xbstream_write.c index a11811dc375..978be71e7dd 100644 --- a/extra/mariabackup/xbstream_write.c +++ b/extra/mariabackup/xbstream_write.c @@ -1,5 +1,5 @@ /****************************************************** -Copyright (c) 2011-2013 Percona LLC and/or its affiliates. +Copyright (c) 2011-2017 Percona LLC and/or its affiliates. The xbstream format writer implementation. @@ -23,6 +23,7 @@ Foundation, Inc., 51 Franklin Street, Fifth Floor, Boston, MA 02110-1301, USA #include #include "common.h" #include "xbstream.h" +#include "crc_glue.h" /* Group writes smaller than this into a single chunk */ #define XB_STREAM_MIN_CHUNK_SIZE (10 * 1024 * 1024) @@ -215,12 +216,13 @@ xb_stream_write_chunk(xb_wstream_file_t *file, const void *buf, size_t len) int8store(ptr, len); /* Payload length */ ptr += 8; + checksum = crc32_iso3309(0, buf, (uint)len); /* checksum */ + pthread_mutex_lock(&stream->mutex); int8store(ptr, file->offset); /* Payload offset */ ptr += 8; - checksum = crc32(0, buf, (uint)len); /* checksum */ int4store(ptr, checksum); ptr += 4; diff --git a/extra/mariabackup/xtrabackup.cc b/extra/mariabackup/xtrabackup.cc index 3ca6efaafcf..d16687926aa 100644 --- a/extra/mariabackup/xtrabackup.cc +++ b/extra/mariabackup/xtrabackup.cc @@ -1,6 +1,6 @@ /****************************************************** XtraBackup: hot backup tool for InnoDB -(c) 2009-2015 Percona LLC and/or its affiliates +(c) 2009-2017 Percona LLC and/or its affiliates Originally Created 3/3/2009 Yasufumi Kinoshita Written by Alexey Kopytov, Aleksandr Kuzminsky, Stewart Smith, Vadim Tkachenko, Yasufumi Kinoshita, Ignacio Nin and Baron Schwartz. @@ -67,6 +67,7 @@ Place, Suite 330, Boston, MA 02111-1307 USA #include #include +#include #include #include #include @@ -94,6 +95,7 @@ Place, Suite 330, Boston, MA 02111-1307 USA #include "encryption_plugin.h" #include #include +#include /* TODO: replace with appropriate macros used in InnoDB 5.6 */ #define PAGE_ZIP_MIN_SIZE_SHIFT 10 @@ -144,25 +146,24 @@ char xtrabackup_real_incremental_dir[FN_REFLEN]; lsn_t xtrabackup_archived_to_lsn = 0; /* for --archived-to-lsn */ -char *xtrabackup_tables = NULL; - char *xtrabackup_tmpdir; -/* List of regular expressions for filtering */ -typedef struct xb_regex_list_node_struct xb_regex_list_node_t; -struct xb_regex_list_node_struct { - UT_LIST_NODE_T(xb_regex_list_node_t) regex_list; - regex_t regex; -}; -static UT_LIST_BASE_NODE_T(xb_regex_list_node_t) regex_list; -static regmatch_t tables_regmatch[1]; - +char *xtrabackup_tables = NULL; char *xtrabackup_tables_file = NULL; -static hash_table_t* tables_hash = NULL; +char *xtrabackup_tables_exclude = NULL; + +typedef std::list regex_list_t; +static regex_list_t regex_include_list; +static regex_list_t regex_exclude_list; + +static hash_table_t* tables_include_hash = NULL; +static hash_table_t* tables_exclude_hash = NULL; char *xtrabackup_databases = NULL; char *xtrabackup_databases_file = NULL; -static hash_table_t* databases_hash = NULL; +char *xtrabackup_databases_exclude = NULL; +static hash_table_t* databases_include_hash = NULL; +static hash_table_t* databases_exclude_hash = NULL; static hash_table_t* inc_dir_tables_hash; @@ -615,6 +616,8 @@ enum options_xtrabackup OPT_SSL_VERIFY_SERVER_CERT, OPT_SERVER_PUBLIC_KEY, + OPT_XTRA_TABLES_EXCLUDE, + OPT_XTRA_DATABASES_EXCLUDE, }; struct my_option xb_client_options[] = @@ -685,6 +688,16 @@ struct my_option xb_client_options[] = "filtering by list of databases in the file.", (G_PTR*) &xtrabackup_databases_file, (G_PTR*) &xtrabackup_databases_file, 0, GET_STR, REQUIRED_ARG, 0, 0, 0, 0, 0, 0}, + {"tables-exclude", OPT_XTRA_TABLES_EXCLUDE, "filtering by regexp for table names. " + "Operates the same way as --tables, but matched names are excluded from backup. " + "Note that this option has a higher priority than --tables.", + (G_PTR*) &xtrabackup_tables_exclude, (G_PTR*) &xtrabackup_tables_exclude, + 0, GET_STR, REQUIRED_ARG, 0, 0, 0, 0, 0, 0}, + {"databases-exclude", OPT_XTRA_DATABASES_EXCLUDE, "Excluding databases based on name, " + "Operates the same way as --databases, but matched names are excluded from backup. " + "Note that this option has a higher priority than --databases.", + (G_PTR*) &xtrabackup_databases_exclude, (G_PTR*) &xtrabackup_databases_exclude, + 0, GET_STR, REQUIRED_ARG, 0, 0, 0, 0, 0, 0}, {"create-ib-logfile", OPT_XTRA_CREATE_IB_LOGFILE, "** not work for now** creates ib_logfile* also after '--prepare'. ### If you want create ib_logfile*, only re-execute this command in same options. ###", (G_PTR*) &xtrabackup_create_ib_logfile, (G_PTR*) &xtrabackup_create_ib_logfile, 0, GET_BOOL, NO_ARG, 0, 0, 0, 0, 0, 0}, @@ -1036,8 +1049,8 @@ struct my_option xb_server_options[] = (G_PTR*) &opt_mysql_tmpdir, (G_PTR*) &opt_mysql_tmpdir, 0, GET_STR, REQUIRED_ARG, 0, 0, 0, 0, 0, 0}, {"parallel", OPT_XTRA_PARALLEL, - "Number of threads to use for parallel datafiles transfer. Does not have " - "any effect in the stream mode. The default value is 1.", + "Number of threads to use for parallel datafiles transfer. " + "The default value is 1.", (G_PTR*) &xtrabackup_parallel, (G_PTR*) &xtrabackup_parallel, 0, GET_INT, REQUIRED_ARG, 1, 1, INT_MAX, 0, 0, 0}, @@ -2162,45 +2175,141 @@ xtrabackup_io_throttling(void) } } -/************************************************************************ -Checks if a given table name matches any of specifications in the --tables or ---tables-file options. - -@return TRUE on match. */ -static my_bool -check_if_table_matches_filters(const char *name) +static +my_bool regex_list_check_match( + const regex_list_t& list, + const char* name) { - int regres; - xb_filter_entry_t* table; - xb_regex_list_node_t* node; - - if (UT_LIST_GET_LEN(regex_list)) { - /* Check against regular expressions list */ - for (node = UT_LIST_GET_FIRST(regex_list); node; - node = UT_LIST_GET_NEXT(regex_list, node)) { - regres = regexec(&node->regex, name, 1, - tables_regmatch, 0); - if (regres != REG_NOMATCH) { - - return(TRUE); - } - } - } - - if (tables_hash) { - HASH_SEARCH(name_hash, tables_hash, ut_fold_string(name), - xb_filter_entry_t*, - table, (void) 0, - !strcmp(table->name, name)); - if (table) { + regmatch_t tables_regmatch[1]; + for (regex_list_t::const_iterator i = list.begin(), end = list.end(); + i != end; ++i) { + const regex_t& regex = *i; + int regres = regexec(®ex, name, 1, tables_regmatch, 0); + if (regres != REG_NOMATCH) { return(TRUE); } } - return(FALSE); } +static +my_bool +find_filter_in_hashtable( + const char* name, + hash_table_t* table, + xb_filter_entry_t** result +) +{ + xb_filter_entry_t* found = NULL; + HASH_SEARCH(name_hash, table, ut_fold_string(name), + xb_filter_entry_t*, + found, (void) 0, + !strcmp(found->name, name)); + + if (found && result) { + *result = found; + } + return (found != NULL); +} + +/************************************************************************ +Checks if a given table name matches any of specifications given in +regex_list or tables_hash. + +@return TRUE on match or both regex_list and tables_hash are empty.*/ +static my_bool +check_if_table_matches_filters(const char *name, + const regex_list_t& regex_list, + hash_table_t* tables_hash) +{ + if (regex_list.empty() && !tables_hash) { + return(FALSE); + } + + if (regex_list_check_match(regex_list, name)) { + return(TRUE); + } + + if (tables_hash && find_filter_in_hashtable(name, tables_hash, NULL)) { + return(TRUE); + } + + return FALSE; +} + +enum skip_database_check_result { + DATABASE_SKIP, + DATABASE_SKIP_SOME_TABLES, + DATABASE_DONT_SKIP, + DATABASE_DONT_SKIP_UNLESS_EXPLICITLY_EXCLUDED, +}; + +/************************************************************************ +Checks if a database specified by name should be skipped from backup based on +the --databases, --databases_file or --databases_exclude options. + +@return TRUE if entire database should be skipped, + FALSE otherwise. +*/ +static +skip_database_check_result +check_if_skip_database( + const char* name /*!< in: path to the database */ +) +{ + /* There are some filters for databases, check them */ + xb_filter_entry_t* database = NULL; + + if (databases_exclude_hash && + find_filter_in_hashtable(name, databases_exclude_hash, + &database) && + !database->has_tables) { + /* Database is found and there are no tables specified, + skip entire db. */ + return DATABASE_SKIP; + } + + if (databases_include_hash) { + if (!find_filter_in_hashtable(name, databases_include_hash, + &database)) { + /* Database isn't found, skip the database */ + return DATABASE_SKIP; + } else if (database->has_tables) { + return DATABASE_SKIP_SOME_TABLES; + } else { + return DATABASE_DONT_SKIP_UNLESS_EXPLICITLY_EXCLUDED; + } + } + + return DATABASE_DONT_SKIP; +} + +/************************************************************************ +Checks if a database specified by path should be skipped from backup based on +the --databases, --databases_file or --databases_exclude options. + +@return TRUE if the table should be skipped. */ +my_bool +check_if_skip_database_by_path( + const char* path /*!< in: path to the db directory. */ +) +{ + if (databases_include_hash == NULL && + databases_exclude_hash == NULL) { + return(FALSE); + } + + const char* db_name = strrchr(path, SRV_PATH_SEPARATOR); + if (db_name == NULL) { + db_name = path; + } else { + ++db_name; + } + + return check_if_skip_database(db_name) == DATABASE_SKIP; +} + /************************************************************************ Checks if a table specified as a name in the form "database/name" (InnoDB 5.6) or "./database/name.ibd" (InnoDB 5.5-) should be skipped from backup based on @@ -2217,9 +2326,12 @@ check_if_skip_table( const char *ptr; char *eptr; - if (UT_LIST_GET_LEN(regex_list) == 0 && - tables_hash == NULL && - databases_hash == NULL) { + if (regex_exclude_list.empty() && + regex_include_list.empty() && + tables_include_hash == NULL && + tables_exclude_hash == NULL && + databases_include_hash == NULL && + databases_exclude_hash == NULL) { return(FALSE); } @@ -2237,23 +2349,10 @@ check_if_skip_table( strncpy(buf, dbname, FN_REFLEN); buf[tbname - 1 - dbname] = 0; - if (databases_hash) { - /* There are some filters for databases, check them */ - xb_filter_entry_t* database; - - HASH_SEARCH(name_hash, databases_hash, ut_fold_string(buf), - xb_filter_entry_t*, - database, (void) 0, - !strcmp(database->name, buf)); - /* Table's database isn't found, skip the table */ - if (!database) { - return(TRUE); - } - /* There aren't tables specified for the database, - it should be backed up entirely */ - if (!database->has_tables) { - return(FALSE); - } + const skip_database_check_result skip_database = + check_if_skip_database(buf); + if (skip_database == DATABASE_SKIP) { + return (TRUE); } buf[FN_REFLEN - 1] = '\0'; @@ -2270,21 +2369,43 @@ check_if_skip_table( /* For partitioned tables first try to match against the regexp without truncating the #P#... suffix so we can backup individual partitions with regexps like '^test[.]t#P#p5' */ - if (check_if_table_matches_filters(buf)) { - + if (check_if_table_matches_filters(buf, regex_exclude_list, + tables_exclude_hash)) { + return(TRUE); + } + if (check_if_table_matches_filters(buf, regex_include_list, + tables_include_hash)) { return(FALSE); } if ((eptr = strstr(buf, "#P#")) != NULL) { - *eptr = 0; - if (check_if_table_matches_filters(buf)) { - + if (check_if_table_matches_filters(buf, regex_exclude_list, + tables_exclude_hash)) { + return (TRUE); + } + if (check_if_table_matches_filters(buf, regex_include_list, + tables_include_hash)) { return(FALSE); } } - return(TRUE); + if (skip_database == DATABASE_DONT_SKIP_UNLESS_EXPLICITLY_EXCLUDED) { + /* Database is in include-list, and qualified name wasn't + found in any of exclusion filters.*/ + return (FALSE); + } + + if (skip_database == DATABASE_SKIP_SOME_TABLES || + !regex_include_list.empty() || + tables_include_hash) { + + /* Include lists are present, but qualified name + failed to match any.*/ + return(TRUE); + } + + return(FALSE); } /*********************************************************************** @@ -3040,6 +3161,8 @@ xtrabackup_init_datasinks(void) if (xtrabackup_encrypt) { ds_ctxt_t *ds; + + ds = ds_create(xtrabackup_target_dir, DS_TYPE_ENCRYPT); xtrabackup_add_datasink(ds); @@ -3360,7 +3483,10 @@ static void xb_register_filter_entry( /*=====================*/ - const char* name) /*!< in: name */ + const char* name, /*!< in: name */ + hash_table_t** databases_hash, + hash_table_t** tables_hash + ) { const char* p; size_t namelen; @@ -3376,25 +3502,45 @@ xb_register_filter_entry( strncpy(dbname, name, p - name); dbname[p - name] = 0; - if (databases_hash) { - HASH_SEARCH(name_hash, databases_hash, + if (*databases_hash) { + HASH_SEARCH(name_hash, (*databases_hash), ut_fold_string(dbname), xb_filter_entry_t*, db_entry, (void) 0, !strcmp(db_entry->name, dbname)); } if (!db_entry) { - db_entry = xb_add_filter(dbname, &databases_hash); + db_entry = xb_add_filter(dbname, databases_hash); } db_entry->has_tables = TRUE; - xb_add_filter(name, &tables_hash); + xb_add_filter(name, tables_hash); } else { xb_validate_name(name, namelen); - xb_add_filter(name, &databases_hash); + xb_add_filter(name, databases_hash); } } +static +void +xb_register_include_filter_entry( + const char* name +) +{ + xb_register_filter_entry(name, &databases_include_hash, + &tables_include_hash); +} + +static +void +xb_register_exclude_filter_entry( + const char* name +) +{ + xb_register_filter_entry(name, &databases_exclude_hash, + &tables_exclude_hash); +} + /*********************************************************************** Register new table for the filter. */ static @@ -3408,33 +3554,52 @@ xb_register_table( exit(EXIT_FAILURE); } - xb_register_filter_entry(name); + xb_register_include_filter_entry(name); } -/*********************************************************************** -Register new regex for the filter. */ static void -xb_register_regex( -/*==============*/ - const char* regex) /*!< in: regex */ +xb_add_regex_to_list( + const char* regex, /*!< in: regex */ + const char* error_context, /*!< in: context to error message */ + regex_list_t* list) /*! in: list to put new regex to */ { - xb_regex_list_node_t* node; char errbuf[100]; int ret; - node = static_cast - (ut_malloc(sizeof(xb_regex_list_node_t))); + regex_t compiled_regex; + ret = regcomp(&compiled_regex, regex, REG_EXTENDED); - ret = regcomp(&node->regex, regex, REG_EXTENDED); if (ret != 0) { - xb_regerror(ret, &node->regex, errbuf, sizeof(errbuf)); - msg("xtrabackup: error: tables regcomp(%s): %s\n", - regex, errbuf); + regerror(ret, &compiled_regex, errbuf, sizeof(errbuf)); + msg("xtrabackup: error: %s regcomp(%s): %s\n", + error_context, regex, errbuf); exit(EXIT_FAILURE); } - UT_LIST_ADD_LAST(regex_list, regex_list, node); + list->push_back(compiled_regex); +} + +/*********************************************************************** +Register new regex for the include filter. */ +static +void +xb_register_include_regex( +/*==============*/ + const char* regex) /*!< in: regex */ +{ + xb_add_regex_to_list(regex, "tables", ®ex_include_list); +} + +/*********************************************************************** +Register new regex for the exclude filter. */ +static +void +xb_register_exclude_regex( +/*==============*/ + const char* regex) /*!< in: regex */ +{ + xb_add_regex_to_list(regex, "tables-exclude", ®ex_exclude_list); } typedef void (*insert_entry_func_t)(const char*); @@ -3500,26 +3665,34 @@ static void xb_filters_init() { - UT_LIST_INIT(regex_list); - if (xtrabackup_databases) { xb_load_list_string(xtrabackup_databases, " \t", - xb_register_filter_entry); + xb_register_include_filter_entry); } if (xtrabackup_databases_file) { xb_load_list_file(xtrabackup_databases_file, - xb_register_filter_entry); + xb_register_include_filter_entry); + } + + if (xtrabackup_databases_exclude) { + xb_load_list_string(xtrabackup_databases_exclude, " \t", + xb_register_exclude_filter_entry); } if (xtrabackup_tables) { xb_load_list_string(xtrabackup_tables, ",", - xb_register_regex); + xb_register_include_regex); } if (xtrabackup_tables_file) { xb_load_list_file(xtrabackup_tables_file, xb_register_table); } + + if (xtrabackup_tables_exclude) { + xb_load_list_string(xtrabackup_tables_exclude, ",", + xb_register_exclude_regex); + } } static @@ -3551,25 +3724,37 @@ xb_filter_hash_free(hash_table_t* hash) hash_table_free(hash); } +static void xb_regex_list_free(regex_list_t* list) +{ + while (list->size() > 0) { + xb_regfree(&list->front()); + list->pop_front(); + } +} + /************************************************************************ Destroy table filters for partial backup. */ static void xb_filters_free() { - while (UT_LIST_GET_LEN(regex_list) > 0) { - xb_regex_list_node_t* node = UT_LIST_GET_FIRST(regex_list); - UT_LIST_REMOVE(regex_list, regex_list, node); - regfree(&node->regex); - ut_free(node); + xb_regex_list_free(®ex_include_list); + xb_regex_list_free(®ex_exclude_list); + + if (tables_include_hash) { + xb_filter_hash_free(tables_include_hash); } - if (tables_hash) { - xb_filter_hash_free(tables_hash); + if (tables_exclude_hash) { + xb_filter_hash_free(tables_exclude_hash); } - if (databases_hash) { - xb_filter_hash_free(databases_hash); + if (databases_include_hash) { + xb_filter_hash_free(databases_include_hash); + } + + if (databases_exclude_hash) { + xb_filter_hash_free(databases_exclude_hash); } } @@ -3854,6 +4039,7 @@ xtrabackup_backup_func(void) srv_general_init(); ut_crc32_init(); + crc_init(); #ifdef WITH_INNODB_DISALLOW_WRITES srv_allow_writes_event = os_event_create(); @@ -6014,7 +6200,7 @@ xb_export_cfg_write( file = fopen(file_path, "w+b"); if (file == NULL) { - msg("xtrabackup: Error: cannot close %s\n", node->name); + msg("xtrabackup: Error: cannot open %s\n", node->name); success = false; } else { @@ -6473,20 +6659,23 @@ skip_check: table_name); goto next_node; } - index = dict_table_get_first_index(table); - n_index = UT_LIST_GET_LEN(table->indexes); - if (n_index > 31) { - msg("xtrabackup: error: " - "sorry, cannot export over " - "31 indexes for now.\n"); - goto next_node; - } /* Write MySQL 5.6 .cfg file */ if (!xb_export_cfg_write(node, table)) { goto next_node; } + index = dict_table_get_first_index(table); + n_index = UT_LIST_GET_LEN(table->indexes); + if (n_index > 31) { + msg("xtrabackup: warning: table '%s' has more " + "than 31 indexes, .exp file was not " + "generated. Table will fail to import " + "on server version prior to 5.6.\n", + table->name); + goto next_node; + } + /* init exp file */ memset(page, 0, UNIV_PAGE_SIZE); mach_write_to_4(page , 0x78706f72UL); @@ -6659,6 +6848,12 @@ next_node: if (!xtrabackup_apply_log_only) { + /* xtrabackup_incremental_dir is used to indicate that + we are going to apply incremental backup. Here we already + applied incremental backup and are about to do final prepare + of the full backup */ + xtrabackup_incremental_dir = NULL; + if(innodb_init_param()) { goto error; } @@ -7016,20 +7211,24 @@ handle_options(int argc, char **argv, char ***argv_client, char ***argv_server) } /* ================= main =================== */ +extern my_bool(*fil_check_if_skip_database_by_path)(const char* name); int main(int argc, char **argv) { char **client_defaults, **server_defaults; char cwd[FN_REFLEN]; - + static char INNOBACKUPEX_EXE[]= "innobackupex"; if (argc > 1 && (strcmp(argv[1], "--innobackupex") == 0)) { argv++; argc--; - argv[0] = "innobackupex"; + argv[0] = INNOBACKUPEX_EXE; innobackupex_mode = true; } + /* Setup skip fil_load_single_tablespaces callback.*/ + fil_check_if_skip_database_by_path = check_if_skip_database_by_path; + init_signals(); MY_INIT(argv[0]); diff --git a/extra/mariabackup/xtrabackup.h b/extra/mariabackup/xtrabackup.h index de21da31741..51491ce1f00 100644 --- a/extra/mariabackup/xtrabackup.h +++ b/extra/mariabackup/xtrabackup.h @@ -77,6 +77,8 @@ extern char *xtrabackup_tables; extern char *xtrabackup_tables_file; extern char *xtrabackup_databases; extern char *xtrabackup_databases_file; +extern char *xtrabackup_tables_exclude; +extern char *xtrabackup_databases_exclude; extern ibool xtrabackup_compress; extern ibool xtrabackup_encrypt; @@ -205,6 +207,17 @@ check_if_skip_table( /******************/ const char* name); /*!< in: path to the table */ + +/************************************************************************ +Checks if a database specified by path should be skipped from backup based on +the --databases, --databases_file or --databases_exclude options. + +@return TRUE if the table should be skipped. */ +my_bool +check_if_skip_database_by_path( + const char* path /*!< in: path to the db directory. */ +); + /************************************************************************ Check if parameter is set in defaults file or via command line argument @return true if parameter is set. */ diff --git a/mysql-test/suite/mariabackup/partial_exclude.result b/mysql-test/suite/mariabackup/partial_exclude.result new file mode 100644 index 00000000000..0da9b547caa --- /dev/null +++ b/mysql-test/suite/mariabackup/partial_exclude.result @@ -0,0 +1,12 @@ +CREATE TABLE t1(i INT) ENGINE INNODB; +INSERT INTO t1 VALUES(1); +CREATE TABLE t2(i int) ENGINE INNODB; +CREATE DATABASE db2; +USE db2; +CREATE TABLE t1(i INT) ENGINE INNODB; +USE test; +# xtrabackup backup +t1.ibd +DROP TABLE t1; +DROP TABLE t2; +DROP DATABASE db2; diff --git a/mysql-test/suite/mariabackup/partial_exclude.test b/mysql-test/suite/mariabackup/partial_exclude.test new file mode 100644 index 00000000000..631f9d7ee71 --- /dev/null +++ b/mysql-test/suite/mariabackup/partial_exclude.test @@ -0,0 +1,30 @@ +# Test --databases-exclude and --tables-exclude feature of xtrabackup 2.3.8 + +CREATE TABLE t1(i INT) ENGINE INNODB; +INSERT INTO t1 VALUES(1); +CREATE TABLE t2(i int) ENGINE INNODB; + +CREATE DATABASE db2; +USE db2; +CREATE TABLE t1(i INT) ENGINE INNODB; + +USE test; + +echo # xtrabackup backup; + +let $targetdir=$MYSQLTEST_VARDIR/tmp/backup; +--disable_result_log +exec $XTRABACKUP --defaults-file=$MYSQLTEST_VARDIR/my.cnf --backup "--tables-exclude=test.*2" "--databases-exclude=db2" --target-dir=$targetdir; +--enable_result_log + +# check that only t1 table is in backup (t2 is excluded) +list_files $targetdir/test *.ibd; +# check that db2 database is not in the backup (excluded) +--error 1 +list_files $targetdir/db2 *.ibd; + +DROP TABLE t1; +DROP TABLE t2; +DROP DATABASE db2; +rmdir $targetdir; + diff --git a/mysql-test/suite/mariabackup/xbstream.test b/mysql-test/suite/mariabackup/xbstream.test index f2b4704a87e..06e5685276c 100644 --- a/mysql-test/suite/mariabackup/xbstream.test +++ b/mysql-test/suite/mariabackup/xbstream.test @@ -9,7 +9,7 @@ echo # xtrabackup backup to stream; exec $XTRABACKUP --defaults-file=$MYSQLTEST_VARDIR/my.cnf --backup --stream=xbstream > $streamfile 2>$targetdir/backup_stream.log; echo # xbstream extract; --disable_result_log -exec $XBSTREAM -x -C $targetdir < $streamfile; +exec $XBSTREAM -x -C $targetdir --parallel=16 < $streamfile; echo # xtrabackup prepare; exec $XTRABACKUP --prepare --target-dir=$targetdir; diff --git a/storage/xtradb/fil/fil0fil.cc b/storage/xtradb/fil/fil0fil.cc index 216a36e2e82..c95fd872c67 100644 --- a/storage/xtradb/fil/fil0fil.cc +++ b/storage/xtradb/fil/fil0fil.cc @@ -71,6 +71,7 @@ static ulint srv_data_read, srv_data_written; MYSQL_PLUGIN_IMPORT extern my_bool lower_case_file_system; + /* IMPLEMENTATION OF THE TABLESPACE MEMORY CACHE ============================================= @@ -5379,6 +5380,9 @@ fil_file_readdir_next_file( return(-1); } + +my_bool(*fil_check_if_skip_database_by_path)(const char* name); + #define CHECK_TIME_EVERY_N_FILES 10 /********************************************************************//** At the server startup, if we need crash recovery, scans the database @@ -5449,7 +5453,19 @@ fil_load_single_table_tablespaces(ibool (*pred)(const char*, const char*)) "%s/%s", fil_path_to_mysql_datadir, dbinfo.name); srv_normalize_path_for_win(dbpath); - dbdir = os_file_opendir(dbpath, FALSE); + if (IS_XTRABACKUP()) { + ut_a(fil_check_if_skip_database_by_path); + if (fil_check_if_skip_database_by_path(dbpath)) { + fprintf(stderr, "Skipping db: %s\n", dbpath); + dbdir = NULL; + } else { + /* We want wrong directory permissions to be a fatal + error for XtraBackup. */ + dbdir = os_file_opendir(dbpath, TRUE); + } + } else { + dbdir = os_file_opendir(dbpath, FALSE); + } if (dbdir != NULL) { From e8bc838eb9ee4aa8de11bfa2472c9784df78590f Mon Sep 17 00:00:00 2001 From: Vladislav Vaintroub Date: Thu, 20 Apr 2017 19:23:07 +0000 Subject: [PATCH 20/28] mariabackup - do not extend innodb tablespaces in prepare, unless incremental prepare is running. --- extra/mariabackup/xtrabackup.cc | 4 ++++ 1 file changed, 4 insertions(+) diff --git a/extra/mariabackup/xtrabackup.cc b/extra/mariabackup/xtrabackup.cc index d16687926aa..c6962069315 100644 --- a/extra/mariabackup/xtrabackup.cc +++ b/extra/mariabackup/xtrabackup.cc @@ -6546,6 +6546,8 @@ skip_check: if(innodb_init()) goto error_cleanup; + if (xtrabackup_incremental) { + it = datafiles_iter_new(fil_system); if (it == NULL) { msg("xtrabackup: Error: datafiles_iter_new() failed.\n"); @@ -6588,6 +6590,8 @@ skip_check: datafiles_iter_free(it); + } /* if (xtrabackup_incremental) */ + if (xtrabackup_export) { msg("xtrabackup: export option is specified.\n"); os_file_t info_file = XB_FILE_UNDEFINED; From 4e07fc0ab5cae18ace570b534904c24e208e31e0 Mon Sep 17 00:00:00 2001 From: Sergei Golubchik Date: Fri, 21 Apr 2017 00:50:53 +0200 Subject: [PATCH 21/28] test failure use --defaults-file to avoid reading ~/.my.cnf and don't crash if some ibdata file couldn't be opened --- extra/mariabackup/backup_copy.cc | 2 +- mysql-test/suite/mariabackup/include/restart_and_restore.inc | 2 +- 2 files changed, 2 insertions(+), 2 deletions(-) diff --git a/extra/mariabackup/backup_copy.cc b/extra/mariabackup/backup_copy.cc index 67c96b4b648..1565e20d732 100644 --- a/extra/mariabackup/backup_copy.cc +++ b/extra/mariabackup/backup_copy.cc @@ -977,7 +977,7 @@ copy_file(ds_ctxt_t *datasink, const char *action; if (!datafile_open(src_file_path, &cursor, thread_n)) { - goto error; + goto error_close; } strncpy(dst_name, cursor.rel_path, sizeof(dst_name)); diff --git a/mysql-test/suite/mariabackup/include/restart_and_restore.inc b/mysql-test/suite/mariabackup/include/restart_and_restore.inc index 46eb5eee4a8..39616cc6f15 100644 --- a/mysql-test/suite/mariabackup/include/restart_and_restore.inc +++ b/mysql-test/suite/mariabackup/include/restart_and_restore.inc @@ -7,7 +7,7 @@ shutdown_server; echo # remove datadir; rmdir $_datadir; echo # xtrabackup move back; -exec $XTRABACKUP --copy-back --datadir=$_datadir --target-dir=$targetdir --parallel=2; +exec $XTRABACKUP --defaults-file=$MYSQLTEST_VARDIR/my.cnf --copy-back --datadir=$_datadir --target-dir=$targetdir --parallel=2; echo # restart server; exec echo "restart" > $_expect_file_name; enable_reconnect; From f21dcd9933b4a8138e1f6c3a9d7cabb65198e0b5 Mon Sep 17 00:00:00 2001 From: Sergei Golubchik Date: Thu, 20 Apr 2017 20:12:16 +0200 Subject: [PATCH 22/28] .gitignore mariabackup --- .gitignore | 3 +++ 1 file changed, 3 insertions(+) diff --git a/.gitignore b/.gitignore index 05d3b87f6d1..c3c140bb99a 100644 --- a/.gitignore +++ b/.gitignore @@ -46,6 +46,8 @@ extra/comp_err extra/innochecksum extra/jemalloc/build/ extra/jemalloc/tmp/ +extra/mariabackup/mariabackup +extra/mariabackup/mbstream extra/my_print_defaults extra/mysql_waitpid extra/mysqld_safe_helper @@ -118,6 +120,7 @@ scripts/mytop scripts/wsrep_sst_common scripts/wsrep_sst_mysqldump scripts/wsrep_sst_rsync +scripts/wsrep_sst_mariabackup scripts/wsrep_sst_xtrabackup scripts/wsrep_sst_xtrabackup-v2 scripts/maria_add_gis_sp.sql From c0e24cd0e8e283f5b1c90fc4bd58d0c2be639d57 Mon Sep 17 00:00:00 2001 From: Sergei Golubchik Date: Thu, 20 Apr 2017 22:43:58 +0200 Subject: [PATCH 23/28] compiler warnings --- extra/mariabackup/backup_mysql.cc | 1 - extra/mariabackup/crc/crc_glue.c | 6 +++--- extra/mariabackup/ds_archive.c | 11 ++++++++--- extra/mariabackup/xtrabackup.cc | 2 +- storage/xtradb/handler/ha_innodb.cc | 4 ++-- 5 files changed, 14 insertions(+), 10 deletions(-) diff --git a/extra/mariabackup/backup_mysql.cc b/extra/mariabackup/backup_mysql.cc index 52aaf138fc8..6299afffc6e 100644 --- a/extra/mariabackup/backup_mysql.cc +++ b/extra/mariabackup/backup_mysql.cc @@ -1395,7 +1395,6 @@ write_xtrabackup_info(MYSQL *connection) char buf_start_time[100]; char buf_end_time[100]; tm tm; - my_bool null = TRUE; ostringstream oss; const char *xb_stream_name[] = {"file", "tar", "xbstream"}; diff --git a/extra/mariabackup/crc/crc_glue.c b/extra/mariabackup/crc/crc_glue.c index f42910d8344..ae3fa91c1b0 100644 --- a/extra/mariabackup/crc/crc_glue.c +++ b/extra/mariabackup/crc/crc_glue.c @@ -18,11 +18,11 @@ Foundation, Inc., 51 Franklin Street, Fifth Floor, Boston, MA 02110-1301, USA *******************************************************/ -#include -#include -#include #include "crc_glue.h" #include "crc-intel-pclmul.h" +#include +#include +#include #if __GNUC__ >= 4 && defined(__x86_64__) static int pclmul_enabled = 0; diff --git a/extra/mariabackup/ds_archive.c b/extra/mariabackup/ds_archive.c index aa38b2f9530..50afcce4bc7 100644 --- a/extra/mariabackup/ds_archive.c +++ b/extra/mariabackup/ds_archive.c @@ -24,6 +24,11 @@ Foundation, Inc., 51 Franklin Street, Fifth Floor, Boston, MA 02110-1301, USA #include "common.h" #include "datasink.h" +#if ARCHIVE_VERSION_NUMBER < 3000000 +#define archive_write_add_filter_none(X) archive_write_set_compression_none(X) +#define archive_write_free(X) archive_write_finish(X) +#endif + typedef struct { struct archive *archive; ds_file_t *dest_file; @@ -114,14 +119,14 @@ archive_init(const char *root __attribute__((unused))) archive_ctxt->archive = a; archive_ctxt->dest_file = NULL; - if (archive_write_set_compression_none(a) != ARCHIVE_OK || + if(archive_write_add_filter_none(a) != ARCHIVE_OK || archive_write_set_format_pax_restricted(a) != ARCHIVE_OK || /* disable internal buffering so we don't have to flush the output in xtrabackup */ archive_write_set_bytes_per_block(a, 0) != ARCHIVE_OK) { msg("failed to set libarchive archive options: %s\n", archive_error_string(a)); - archive_write_finish(a); + archive_write_free(a); goto err; } @@ -262,7 +267,7 @@ archive_deinit(ds_ctxt_t *ctxt) if (archive_write_close(a) != ARCHIVE_OK) { msg("archive_write_close() failed.\n"); } - archive_write_finish(a); + archive_write_free(a); if (archive_ctxt->dest_file) { ds_close(archive_ctxt->dest_file); diff --git a/extra/mariabackup/xtrabackup.cc b/extra/mariabackup/xtrabackup.cc index c6962069315..c116d119cee 100644 --- a/extra/mariabackup/xtrabackup.cc +++ b/extra/mariabackup/xtrabackup.cc @@ -6514,7 +6514,7 @@ skip_check: Unfinished transactions are not rolled back during log applying as they can be finished at the firther files applyings. */ - srv_apply_log_only = xtrabackup_apply_log_only = true; + xtrabackup_apply_log_only = srv_apply_log_only = true; if (!xtrabackup_arch_search_files(min_flushed_lsn)) { goto error_cleanup; diff --git a/storage/xtradb/handler/ha_innodb.cc b/storage/xtradb/handler/ha_innodb.cc index 18ecee4aee9..8c6d2ef5cd6 100644 --- a/storage/xtradb/handler/ha_innodb.cc +++ b/storage/xtradb/handler/ha_innodb.cc @@ -17401,11 +17401,11 @@ innodb_log_archive_update( if (in_val) { /* turn archiving on */ - srv_log_archive_on = innobase_log_archive = 1; + innobase_log_archive = srv_log_archive_on = 1; log_archive_archivelog(); } else { /* turn archivng off */ - srv_log_archive_on = innobase_log_archive = 0; + innobase_log_archive = srv_log_archive_on = 0; log_archive_noarchivelog(); } } From 0636637e37a4471ae2ef44b4932c6dcb4d9a77cf Mon Sep 17 00:00:00 2001 From: Sergei Golubchik Date: Tue, 25 Apr 2017 22:55:27 +0200 Subject: [PATCH 24/28] regenerate SSL certificates again and make sure that private ca key is not deleted at the end of the procedure, so that we could generate additional certificates any time without regenerating everything --- mysql-test/lib/generate-ssl-certs.sh | 14 +- mysql-test/r/ssl.result | 4 +- mysql-test/std_data/cacert.pem | 115 ++++++------ mysql-test/std_data/cakey.pem | 28 +++ mysql-test/std_data/client-cert.pem | 90 +++++----- mysql-test/std_data/client-key.pem | 26 +-- mysql-test/std_data/server-cert.pem | 90 +++++----- mysql-test/std_data/server-key.pem | 26 +-- mysql-test/std_data/server8k-cert.pem | 248 +++++++++++++------------- mysql-test/std_data/server8k-key.pem | 194 ++++++++++---------- 10 files changed, 433 insertions(+), 402 deletions(-) create mode 100644 mysql-test/std_data/cakey.pem diff --git a/mysql-test/lib/generate-ssl-certs.sh b/mysql-test/lib/generate-ssl-certs.sh index cc919dfe32e..e5e995489a0 100755 --- a/mysql-test/lib/generate-ssl-certs.sh +++ b/mysql-test/lib/generate-ssl-certs.sh @@ -1,30 +1,32 @@ -#!/bin/sh -xe +#!/bin/sh + +set -xe # simply run me from mysql-test/ cd std_data/ # boilerplace for "openssl ca" and /etc/ssl/openssl.cnf rm -rf demoCA -mkdir demoCA demoCA/private demoCA/newcerts +mkdir demoCA demoCA/newcerts touch demoCA/index.txt echo 01 > demoCA/serial # CA certificate, self-signed -openssl req -x509 -newkey rsa:2048 -keyout demoCA/private/cakey.pem -out cacert.pem -days 7300 -nodes -subj '/CN=cacert/C=FI/ST=Helsinki/L=Helsinki/O=MariaDB' -text +openssl req -x509 -newkey rsa:2048 -keyout cakey.pem -out cacert.pem -days 7300 -nodes -subj '/CN=cacert/C=FI/ST=Helsinki/L=Helsinki/O=MariaDB' -text # server certificate signing request and private key. Note the very long subject (for MDEV-7859) openssl req -newkey rsa:1024 -keyout server-key.pem -out demoCA/server-req.pem -days 7300 -nodes -subj '/CN=localhost/C=FI/ST=state or province within country, in other certificates in this file it is the same as L/L=location, usually an address but often ambiguously used/OU=organizational unit name, a division name within an organization/O=organization name, typically a company name' # convert the key to yassl compatible format openssl rsa -in server-key.pem -out server-key.pem # sign the server certificate with CA certificate -openssl ca -days 7300 -batch -cert cacert.pem -policy policy_anything -out server-cert.pem -infiles demoCA/server-req.pem +openssl ca -keyfile cakey.pem -days 7300 -batch -cert cacert.pem -policy policy_anything -out server-cert.pem -infiles demoCA/server-req.pem openssl req -newkey rsa:8192 -keyout server8k-key.pem -out demoCA/server8k-req.pem -days 7300 -nodes -subj '/CN=server8k/C=FI/ST=Helsinki/L=Helsinki/O=MariaDB' openssl rsa -in server8k-key.pem -out server8k-key.pem -openssl ca -days 7300 -batch -cert cacert.pem -policy policy_anything -out server8k-cert.pem -infiles demoCA/server8k-req.pem +openssl ca -keyfile cakey.pem -days 7300 -batch -cert cacert.pem -policy policy_anything -out server8k-cert.pem -infiles demoCA/server8k-req.pem openssl req -newkey rsa:1024 -keyout client-key.pem -out demoCA/client-req.pem -days 7300 -nodes -subj '/CN=client/C=FI/ST=Helsinki/L=Helsinki/O=MariaDB' openssl rsa -in client-key.pem -out client-key.pem -openssl ca -days 7300 -batch -cert cacert.pem -policy policy_anything -out client-cert.pem -infiles demoCA/client-req.pem +openssl ca -keyfile cakey.pem -days 7300 -batch -cert cacert.pem -policy policy_anything -out client-cert.pem -infiles demoCA/client-req.pem rm -rf demoCA diff --git a/mysql-test/r/ssl.result b/mysql-test/r/ssl.result index e74aef8b940..dac0e75d59d 100644 --- a/mysql-test/r/ssl.result +++ b/mysql-test/r/ssl.result @@ -3,10 +3,10 @@ have_ssl 1 SHOW STATUS LIKE 'Ssl_server_not_before'; Variable_name Value -Ssl_server_not_before Apr 25 14:55:05 2015 GMT +Ssl_server_not_before Apr 25 20:52:21 2017 GMT SHOW STATUS LIKE 'Ssl_server_not_after'; Variable_name Value -Ssl_server_not_after Apr 20 14:55:05 2035 GMT +Ssl_server_not_after Apr 20 20:52:21 2037 GMT drop table if exists t1,t2,t3,t4; CREATE TABLE t1 ( Period smallint(4) unsigned zerofill DEFAULT '0000' NOT NULL, diff --git a/mysql-test/std_data/cacert.pem b/mysql-test/std_data/cacert.pem index e934823eea3..cc5d9100e30 100644 --- a/mysql-test/std_data/cacert.pem +++ b/mysql-test/std_data/cacert.pem @@ -1,78 +1,79 @@ Certificate: Data: Version: 3 (0x2) - Serial Number: 11580370790696127632 (0xa0b5bde0f2c08c90) - Signature Algorithm: sha1WithRSAEncryption + Serial Number: + e5:b1:e3:71:e9:6f:a9:e1 + Signature Algorithm: sha256WithRSAEncryption Issuer: CN=cacert, C=FI, ST=Helsinki, L=Helsinki, O=MariaDB Validity - Not Before: Apr 25 14:55:05 2015 GMT - Not After : Apr 20 14:55:05 2035 GMT + Not Before: Apr 25 20:52:21 2017 GMT + Not After : Apr 20 20:52:21 2037 GMT Subject: CN=cacert, C=FI, ST=Helsinki, L=Helsinki, O=MariaDB Subject Public Key Info: Public Key Algorithm: rsaEncryption Public-Key: (2048 bit) Modulus: - 00:c0:1f:90:7c:2b:c2:ea:01:93:ce:e0:c5:72:e8: - 1c:06:bd:63:4e:b6:d2:c6:00:32:13:27:42:9e:c9: - 3c:91:33:4d:15:90:67:7d:9d:d8:be:9b:12:e2:f6: - 1b:46:81:4a:8b:10:c5:b8:14:53:ab:6a:2c:c3:7f: - 66:87:6c:0e:18:51:4e:9c:93:7a:6d:a1:d4:06:47: - 58:61:a6:04:21:2c:bd:74:7a:e4:68:45:fe:91:fe: - fb:a6:29:47:ec:c5:c3:88:c8:c9:e7:d7:c6:1a:0d: - b8:f5:c5:02:57:25:01:cc:d5:8c:37:46:58:c6:71: - 30:ee:63:38:99:84:5e:9e:3c:af:40:d4:f0:f2:12: - 44:6e:2f:4d:cd:f9:da:4d:0e:1f:a6:fe:35:c3:9d: - 40:08:82:5e:6f:7d:4d:09:16:7d:a1:78:d6:9f:9f: - 44:d6:b1:ad:e7:50:25:1a:f3:4e:16:92:4a:17:5e: - 0b:e1:c8:9f:62:22:c4:e2:01:96:63:ed:37:a2:e5: - 70:b9:dc:c8:8e:c4:fe:00:21:f5:b9:48:c0:43:55: - 4a:d8:0c:9d:ce:d6:60:30:bb:81:31:c8:e9:0e:aa: - 1c:18:3d:e4:10:47:42:17:c0:4d:fb:f5:d9:c2:e4: - 07:33:f7:15:94:63:6d:11:ad:4f:d4:1d:11:41:c1: - e2:dd + 00:a0:ad:d5:b1:ec:45:6f:d6:33:fc:5a:03:29:14: + f1:8e:78:d5:27:53:79:e0:92:7c:10:3b:79:a0:d7: + b6:9d:a8:5c:4d:fa:68:11:b3:03:9e:ee:5e:20:79: + 23:d8:9c:49:34:9c:1d:c4:6e:53:1f:9a:92:1f:08: + c1:15:e2:ad:cf:59:cd:1e:55:84:79:f9:09:ca:36: + 8a:50:83:c6:38:48:c6:d3:fa:f6:f2:2a:4f:bd:5d: + 60:9d:eb:21:c4:8c:f2:dd:2d:49:10:63:46:47:de: + 2d:59:a0:4a:e0:58:e6:c0:ae:d8:d4:5e:9a:f8:f5: + 68:1d:ea:80:8a:d6:01:b0:d5:5f:30:4d:88:5a:c5: + 1f:81:92:c1:40:54:c8:bb:a6:a1:43:de:81:3c:4b: + 79:95:82:bb:52:da:a3:a4:a0:69:ff:7e:00:8c:86: + 85:ec:af:03:68:a8:83:48:a0:e4:1d:31:a9:5c:47: + 99:9d:3a:3f:b5:3e:12:7c:4d:47:15:72:f1:11:5c: + 4a:ef:08:1c:7b:8f:e6:03:06:07:4f:94:21:b0:5e: + 27:fa:93:8c:b4:cc:56:34:3b:6d:c4:4a:14:57:b2: + 21:1a:3e:2f:c5:9e:47:1a:59:05:22:0e:56:b1:a7: + e8:80:9b:82:c3:54:57:12:05:94:79:a2:03:d9:64: + 3c:63 Exponent: 65537 (0x10001) X509v3 extensions: X509v3 Subject Key Identifier: - C7:2C:01:95:1A:F5:3E:CD:04:A6:24:35:35:04:D9:A7:16:01:2A:79 + 1C:C7:2B:AA:1B:B1:BB:2E:9A:F4:0F:B1:86:60:57:38:C2:41:05:12 X509v3 Authority Key Identifier: - keyid:C7:2C:01:95:1A:F5:3E:CD:04:A6:24:35:35:04:D9:A7:16:01:2A:79 + keyid:1C:C7:2B:AA:1B:B1:BB:2E:9A:F4:0F:B1:86:60:57:38:C2:41:05:12 X509v3 Basic Constraints: CA:TRUE - Signature Algorithm: sha1WithRSAEncryption - 40:6f:6a:54:f3:29:30:48:46:bd:da:46:71:64:52:14:a7:c2: - 34:b7:5e:1e:42:3d:e7:47:92:cd:87:e7:9d:5d:1a:82:77:82: - 62:32:d4:9d:b6:44:11:dc:88:78:38:a5:d3:1f:1e:be:c2:d6: - 14:b0:58:35:cd:66:22:43:97:ba:bb:e3:44:4f:9d:75:14:9f: - 6f:37:d3:50:07:09:36:bc:58:92:e8:fe:c0:a8:ba:29:55:65: - e2:6f:8f:ab:a5:1d:4f:56:37:de:c7:b4:39:20:4c:a8:4c:db: - 56:51:12:7e:e7:7f:83:9d:c4:c7:72:8f:6f:83:f0:af:e3:37: - 1c:40:fe:5e:38:26:2f:05:46:a7:0c:a5:81:79:d6:9c:9c:d7: - 56:eb:96:fe:c7:ae:8e:4f:5e:4a:6c:3a:fa:68:be:65:60:a2: - d3:3f:07:76:45:b3:95:3e:11:ef:3a:0e:6f:73:47:4c:90:dd: - 0b:36:b4:22:df:62:8d:58:d2:a6:34:5b:f0:06:5d:cd:bf:52: - fa:ee:9b:4f:e8:79:18:6e:1c:6e:5f:96:10:6d:2f:02:1b:dd: - bf:14:c9:32:3c:83:a5:6e:56:56:78:9d:ce:84:50:a4:df:cc: - b5:a9:b1:ec:09:07:74:02:27:7a:9d:d2:96:a9:80:95:9a:f2: - 8c:e9:ef:99 + Signature Algorithm: sha256WithRSAEncryption + 0d:4b:21:52:fa:49:34:56:14:db:83:ae:1c:3d:a7:4d:3e:ea: + 55:7e:1a:37:7a:65:89:ee:19:05:94:9d:3a:ad:59:c4:38:16: + b2:bd:02:ee:5a:a6:7e:e2:b1:21:a3:ad:af:8c:ae:c3:30:71: + ad:d7:d2:24:0f:c4:d9:47:80:c5:95:05:1d:7c:8a:49:0a:7d: + 8b:61:ca:b5:68:3d:3e:4e:f1:c7:45:62:c8:cc:a9:2f:f3:12: + f1:3f:92:34:7f:07:ab:d3:ac:ab:af:2d:c9:69:63:8a:b2:e5: + 35:ea:7d:b8:17:38:72:82:5f:96:3d:dc:8d:e5:11:bb:ae:f3: + 02:2d:20:77:5c:64:59:18:a6:e7:fa:c7:89:e8:30:12:14:04: + 40:5b:e9:b1:8f:86:81:b9:0d:6c:b6:fc:98:f9:b7:52:ab:8f: + 7e:53:c8:a0:05:e4:cd:0d:6b:d2:74:9f:17:7a:a1:c3:76:5e: + f3:29:1c:c6:be:56:ab:02:f7:5d:e1:c9:21:27:6d:66:7a:41: + 29:49:a3:f8:f5:2a:e7:03:2a:7c:52:4b:f5:46:58:45:be:a4: + 4c:a0:65:37:1d:d8:ac:f8:1f:81:ca:9c:79:f0:ff:22:8c:1d: + ce:2b:d0:1e:ce:99:f2:db:fa:66:84:e6:86:6f:19:3b:10:f1: + 92:ac:57:b2 -----BEGIN CERTIFICATE----- -MIIDfzCCAmegAwIBAgIJAKC1veDywIyQMA0GCSqGSIb3DQEBBQUAMFYxDzANBgNV +MIIDfzCCAmegAwIBAgIJAOWx43Hpb6nhMA0GCSqGSIb3DQEBCwUAMFYxDzANBgNV BAMMBmNhY2VydDELMAkGA1UEBhMCRkkxETAPBgNVBAgMCEhlbHNpbmtpMREwDwYD -VQQHDAhIZWxzaW5raTEQMA4GA1UECgwHTWFyaWFEQjAeFw0xNTA0MjUxNDU1MDVa -Fw0zNTA0MjAxNDU1MDVaMFYxDzANBgNVBAMMBmNhY2VydDELMAkGA1UEBhMCRkkx +VQQHDAhIZWxzaW5raTEQMA4GA1UECgwHTWFyaWFEQjAeFw0xNzA0MjUyMDUyMjFa +Fw0zNzA0MjAyMDUyMjFaMFYxDzANBgNVBAMMBmNhY2VydDELMAkGA1UEBhMCRkkx ETAPBgNVBAgMCEhlbHNpbmtpMREwDwYDVQQHDAhIZWxzaW5raTEQMA4GA1UECgwH -TWFyaWFEQjCCASIwDQYJKoZIhvcNAQEBBQADggEPADCCAQoCggEBAMAfkHwrwuoB -k87gxXLoHAa9Y0620sYAMhMnQp7JPJEzTRWQZ32d2L6bEuL2G0aBSosQxbgUU6tq -LMN/ZodsDhhRTpyTem2h1AZHWGGmBCEsvXR65GhF/pH++6YpR+zFw4jIyefXxhoN -uPXFAlclAczVjDdGWMZxMO5jOJmEXp48r0DU8PISRG4vTc352k0OH6b+NcOdQAiC -Xm99TQkWfaF41p+fRNaxredQJRrzThaSShdeC+HIn2IixOIBlmPtN6LlcLncyI7E -/gAh9blIwENVStgMnc7WYDC7gTHI6Q6qHBg95BBHQhfATfv12cLkBzP3FZRjbRGt -T9QdEUHB4t0CAwEAAaNQME4wHQYDVR0OBBYEFMcsAZUa9T7NBKYkNTUE2acWASp5 -MB8GA1UdIwQYMBaAFMcsAZUa9T7NBKYkNTUE2acWASp5MAwGA1UdEwQFMAMBAf8w -DQYJKoZIhvcNAQEFBQADggEBAEBvalTzKTBIRr3aRnFkUhSnwjS3Xh5CPedHks2H -551dGoJ3gmIy1J22RBHciHg4pdMfHr7C1hSwWDXNZiJDl7q740RPnXUUn28301AH -CTa8WJLo/sCouilVZeJvj6ulHU9WN97HtDkgTKhM21ZREn7nf4OdxMdyj2+D8K/j -NxxA/l44Ji8FRqcMpYF51pyc11brlv7Hro5PXkpsOvpovmVgotM/B3ZFs5U+Ee86 -Dm9zR0yQ3Qs2tCLfYo1Y0qY0W/AGXc2/Uvrum0/oeRhuHG5flhBtLwIb3b8UyTI8 -g6VuVlZ4nc6EUKTfzLWpsewJB3QCJ3qd0papgJWa8ozp75k= +TWFyaWFEQjCCASIwDQYJKoZIhvcNAQEBBQADggEPADCCAQoCggEBAKCt1bHsRW/W +M/xaAykU8Y541SdTeeCSfBA7eaDXtp2oXE36aBGzA57uXiB5I9icSTScHcRuUx+a +kh8IwRXirc9ZzR5VhHn5Cco2ilCDxjhIxtP69vIqT71dYJ3rIcSM8t0tSRBjRkfe +LVmgSuBY5sCu2NRemvj1aB3qgIrWAbDVXzBNiFrFH4GSwUBUyLumoUPegTxLeZWC +u1Lao6Sgaf9+AIyGheyvA2iog0ig5B0xqVxHmZ06P7U+EnxNRxVy8RFcSu8IHHuP +5gMGB0+UIbBeJ/qTjLTMVjQ7bcRKFFeyIRo+L8WeRxpZBSIOVrGn6ICbgsNUVxIF +lHmiA9lkPGMCAwEAAaNQME4wHQYDVR0OBBYEFBzHK6obsbsumvQPsYZgVzjCQQUS +MB8GA1UdIwQYMBaAFBzHK6obsbsumvQPsYZgVzjCQQUSMAwGA1UdEwQFMAMBAf8w +DQYJKoZIhvcNAQELBQADggEBAA1LIVL6STRWFNuDrhw9p00+6lV+Gjd6ZYnuGQWU +nTqtWcQ4FrK9Au5apn7isSGjra+MrsMwca3X0iQPxNlHgMWVBR18ikkKfYthyrVo +PT5O8cdFYsjMqS/zEvE/kjR/B6vTrKuvLclpY4qy5TXqfbgXOHKCX5Y93I3lEbuu +8wItIHdcZFkYpuf6x4noMBIUBEBb6bGPhoG5DWy2/Jj5t1Krj35TyKAF5M0Na9J0 +nxd6ocN2XvMpHMa+VqsC913hySEnbWZ6QSlJo/j1KucDKnxSS/VGWEW+pEygZTcd +2Kz4H4HKnHnw/yKMHc4r0B7OmfLb+maE5oZvGTsQ8ZKsV7I= -----END CERTIFICATE----- diff --git a/mysql-test/std_data/cakey.pem b/mysql-test/std_data/cakey.pem new file mode 100644 index 00000000000..88e251f00e2 --- /dev/null +++ b/mysql-test/std_data/cakey.pem @@ -0,0 +1,28 @@ +-----BEGIN PRIVATE KEY----- +MIIEvQIBADANBgkqhkiG9w0BAQEFAASCBKcwggSjAgEAAoIBAQCgrdWx7EVv1jP8 +WgMpFPGOeNUnU3ngknwQO3mg17adqFxN+mgRswOe7l4geSPYnEk0nB3EblMfmpIf +CMEV4q3PWc0eVYR5+QnKNopQg8Y4SMbT+vbyKk+9XWCd6yHEjPLdLUkQY0ZH3i1Z +oErgWObArtjUXpr49Wgd6oCK1gGw1V8wTYhaxR+BksFAVMi7pqFD3oE8S3mVgrtS +2qOkoGn/fgCMhoXsrwNoqINIoOQdMalcR5mdOj+1PhJ8TUcVcvERXErvCBx7j+YD +BgdPlCGwXif6k4y0zFY0O23EShRXsiEaPi/FnkcaWQUiDlaxp+iAm4LDVFcSBZR5 +ogPZZDxjAgMBAAECggEAWmy6AGFpSmEP7IpzkOEaeAWEX5dY1YtaioAOGPiM6vje +yXuMqblG5mBbVIcYJ0T85cCd9/fmi7ifVxvEHh7tle2Bw/p4jXQbkFNVT655FR/P +1Wg9JVeufHFaeETlQgnYe6SKo9BaswNUHkZZHRyq7/D2Ub3UFRt2tq9MG9YIKY1m +rP9s7E+EDuH9UhYmaWdQfNm8muIXWK8WjicI5+PX0CQ1NtUy6vS7qBzcBzvT0chC +Jtja29S6Nvg12A96nHsRmQyUaQjRlqosSwiagpc5mZmNeCEUoY+3deIdYIUMSQnf +judZOKVPq0GOW5Y1U068LGODWaifPkinGBj+04VH0QKBgQDOp/jVCOUdEeqFJ/8m +wEsfsRIrXvtGJHgbDXcVJ69FwlX+yaKGEuC+4f21uyxPn6GoFw+NKAyTmGKH7VAX +OFQLrMQ/DMlNbZrCAAFcXMqrnLaVwqMeIIoVNfKAa8u15K40qc+B0it61Nlay5wq +wvXoSZrdqXSgsI29pav20+8pTQKBgQDHC3l1+gMZ1rCar+5KdVBN1Wq4Xh7cwZw6 +FxEvyrDCJePEU2L7FpH1pFuB4WpXdBu3CPo70ZgwfqBXn4qLOOI3gTtDHActyiUm ++WRG62O+5Ye7aLB4xy0MfnKNA2g/yHj1ozwM8kA5JRptAzDnzWfVE0k47/pVAVzt +E2bZuSykbwKBgQCL6SkMgjMr1T9j20phn/q8gBN/DZUtTe+K0Tj4N5/wqLuz/its +fkdutG4ipZBAcCDwPnym4qBxJNBAmqiIr/gm11ceILgBFd2azoodUC1etoDfL6Fj ++j/CUH3X+CM5CJPwz67Pg80wIf7t+7/FK611ELAqtllhmWa9KPcd6yqWWQKBgHh5 +Xnvk5kmWY3BNOgrBNOjXWu/asA1n9lpGqfVmVlQ8wL6MxiU5xQCMCYL0X/ws37WK +boMUWmxHyF8gxqd7t5hm1OrKpSG274PGgUZXpRjfLqdlNyLzUzXztvvY6xloCqaK +tYcUfYDZD0SaINi8v7L9KF2ZCsi2uXsZOjBf30BrAoGAXPPotkw/CkcPQBS13cha +ZWeeH5NDKBADWXfLfcRUs108c9xw4BYr5yGilSPscN2ZP0/iWONKp/c6/STS54t5 +lkOKKUbkAFbQu8UKa1J7zrnHZv+Mr4I/iBBy6VkN8Spp2vBI3Ng6jhPIJg3Gum9p +943wWtAnIhe/UqCRT3a/GZg= +-----END PRIVATE KEY----- diff --git a/mysql-test/std_data/client-cert.pem b/mysql-test/std_data/client-cert.pem index cbe8bc2c677..9f6f0cfde7c 100644 --- a/mysql-test/std_data/client-cert.pem +++ b/mysql-test/std_data/client-cert.pem @@ -2,25 +2,25 @@ Certificate: Data: Version: 3 (0x2) Serial Number: 3 (0x3) - Signature Algorithm: sha1WithRSAEncryption + Signature Algorithm: sha256WithRSAEncryption Issuer: CN=cacert, C=FI, ST=Helsinki, L=Helsinki, O=MariaDB Validity - Not Before: Apr 25 14:55:16 2015 GMT - Not After : Apr 20 14:55:16 2035 GMT + Not Before: Apr 25 20:52:33 2017 GMT + Not After : Apr 20 20:52:33 2037 GMT Subject: C=FI, ST=Helsinki, L=Helsinki, O=MariaDB, CN=client Subject Public Key Info: Public Key Algorithm: rsaEncryption Public-Key: (1024 bit) Modulus: - 00:ce:a0:3d:3c:a4:bb:4f:a1:4f:91:0d:05:ac:5b: - 8a:15:7f:d7:aa:0c:a3:a7:9f:b2:c7:26:9d:65:28: - b1:84:d3:a0:ef:9e:b1:45:0f:33:df:98:6e:71:ff: - 2b:66:9c:9c:c1:25:13:27:42:b6:20:46:e7:e7:47: - a1:88:47:c2:9e:e2:45:25:99:9f:f9:28:1a:9a:13: - 67:5d:3e:b3:b8:fe:40:25:ac:26:49:46:2c:03:43: - 83:67:d8:0f:41:ae:2e:f4:d8:71:60:3c:8e:e7:91: - d0:bb:2c:ca:12:da:71:1a:7b:e3:fa:8c:8f:c3:bb: - 62:55:89:b3:bf:85:45:01:61 + 00:a1:10:ea:cc:8e:2c:73:6b:33:1a:5e:26:19:b6: + 4b:4c:bc:04:b8:c2:e2:33:eb:67:a2:7a:27:af:3f: + f7:ef:49:5f:c1:d2:b9:d9:71:fe:17:a0:93:da:dc: + f1:47:de:fa:1f:c3:c1:d1:a5:2a:06:cb:b3:e8:9a: + c1:bd:78:77:68:45:c1:55:cd:b1:c1:d3:df:8c:12: + 4f:c2:3a:0d:b7:58:dc:ca:13:08:b9:fb:12:24:90: + aa:b7:4e:04:eb:43:0d:45:be:1c:17:d6:a8:b1:af: + 10:3c:39:d6:08:45:ed:a9:7e:3a:69:ae:70:22:86: + 7e:71:1f:f1:0e:d0:0d:32:c3 Exponent: 65537 (0x10001) X509v3 extensions: X509v3 Basic Constraints: @@ -28,42 +28,42 @@ Certificate: Netscape Comment: OpenSSL Generated Certificate X509v3 Subject Key Identifier: - 5A:73:74:8E:14:29:C3:FB:B4:19:0F:97:8F:AA:6F:E1:E1:A8:F7:5B + 0C:20:76:A1:80:9C:2F:30:3D:F7:AB:8D:31:19:AD:E2:F7:E2:8D:12 X509v3 Authority Key Identifier: - keyid:C7:2C:01:95:1A:F5:3E:CD:04:A6:24:35:35:04:D9:A7:16:01:2A:79 + keyid:1C:C7:2B:AA:1B:B1:BB:2E:9A:F4:0F:B1:86:60:57:38:C2:41:05:12 - Signature Algorithm: sha1WithRSAEncryption - 32:42:4b:36:44:a5:6c:fb:70:d8:08:2b:cb:16:34:15:db:39: - 60:7b:7e:b4:4a:bc:fb:e5:16:04:97:0d:eb:f5:68:95:da:2f: - 23:57:4c:c9:29:2b:d1:1b:1b:9f:bd:f4:79:75:df:62:7f:63: - b4:84:7a:95:5c:c4:ee:f3:77:16:e4:0b:8a:5e:c9:64:bd:7c: - 04:50:ac:ff:9a:41:6b:b1:6a:9f:cd:45:10:72:83:10:8a:26: - 1d:7f:6c:84:34:5a:41:79:72:91:ee:87:5d:1d:3a:55:ff:91: - 7e:52:85:ff:42:41:eb:76:56:23:e5:bc:bc:79:b1:aa:4e:4c: - bf:7b:df:63:8b:1a:3c:4b:01:72:89:35:bb:0d:92:97:16:6e: - ae:50:cb:89:ee:c6:7a:d0:d3:32:22:0f:19:33:1e:ee:ff:41: - a5:a1:25:c5:4c:ce:8f:98:4c:b5:2c:1f:ec:cc:f1:21:e2:3a: - ff:7d:6a:87:fe:89:fd:2c:20:3e:fb:9b:b8:c0:f9:09:99:ce: - 45:63:82:09:1c:bb:79:d8:a8:40:21:46:c7:ae:3e:dd:89:9d: - 56:46:4a:f4:ed:7d:5b:a6:1e:a6:1b:26:f9:ec:26:b4:51:3a: - 87:b6:50:13:84:33:22:1a:8a:20:c5:44:64:b8:bb:de:32:ec: - 6b:58:db:17 + Signature Algorithm: sha256WithRSAEncryption + 39:c0:90:13:19:85:47:9d:c6:ab:8c:c6:c9:0f:33:11:19:f7: + 01:2c:1b:08:f6:81:98:11:ab:48:05:d9:b2:29:56:32:9c:ba: + e5:40:df:85:5e:6d:fd:6e:36:9a:14:eb:90:50:57:de:2f:ed: + 2d:89:a6:8a:40:1c:41:84:9b:da:e1:6d:e6:7c:46:b2:e0:90: + 93:02:1c:52:2e:af:b4:d4:a1:d8:9d:19:cf:0a:67:bf:c3:3e: + 2e:02:f4:3e:bc:2e:59:57:30:85:8a:32:ab:22:88:72:37:6e: + ee:ed:f8:53:72:c9:28:87:50:47:81:1b:80:4c:f8:80:ce:2f: + 47:ca:78:ce:38:51:70:ec:df:ee:fc:ea:5a:40:1e:4d:1c:fd: + 4e:f6:74:d0:22:a4:7e:57:df:16:1a:a0:8d:be:fe:ee:f2:07: + 2e:39:a1:97:40:19:f9:3b:b8:e7:c4:98:6e:1d:1a:27:d3:19: + 4c:5c:c9:c3:31:98:c1:3c:27:0e:6a:de:cf:88:72:cf:e4:65: + c9:0d:33:32:f1:ea:f7:dd:5b:9d:42:6d:ee:c7:a8:b7:85:d2: + 41:e0:84:38:ce:86:81:ba:6e:7d:d5:ad:7a:00:58:d7:c5:83: + 9e:5c:1d:38:32:72:49:f5:42:4b:e7:c6:5c:12:6d:e1:5d:51: + 2c:f5:52:f0 -----BEGIN CERTIFICATE----- -MIIDHjCCAgagAwIBAgIBAzANBgkqhkiG9w0BAQUFADBWMQ8wDQYDVQQDDAZjYWNl +MIIDHjCCAgagAwIBAgIBAzANBgkqhkiG9w0BAQsFADBWMQ8wDQYDVQQDDAZjYWNl cnQxCzAJBgNVBAYTAkZJMREwDwYDVQQIDAhIZWxzaW5raTERMA8GA1UEBwwISGVs -c2lua2kxEDAOBgNVBAoMB01hcmlhREIwHhcNMTUwNDI1MTQ1NTE2WhcNMzUwNDIw -MTQ1NTE2WjBWMQswCQYDVQQGEwJGSTERMA8GA1UECAwISGVsc2lua2kxETAPBgNV +c2lua2kxEDAOBgNVBAoMB01hcmlhREIwHhcNMTcwNDI1MjA1MjMzWhcNMzcwNDIw +MjA1MjMzWjBWMQswCQYDVQQGEwJGSTERMA8GA1UECAwISGVsc2lua2kxETAPBgNV BAcMCEhlbHNpbmtpMRAwDgYDVQQKDAdNYXJpYURCMQ8wDQYDVQQDDAZjbGllbnQw -gZ8wDQYJKoZIhvcNAQEBBQADgY0AMIGJAoGBAM6gPTyku0+hT5ENBaxbihV/16oM -o6efsscmnWUosYTToO+esUUPM9+YbnH/K2acnMElEydCtiBG5+dHoYhHwp7iRSWZ -n/koGpoTZ10+s7j+QCWsJklGLANDg2fYD0GuLvTYcWA8jueR0LssyhLacRp74/qM -j8O7YlWJs7+FRQFhAgMBAAGjezB5MAkGA1UdEwQCMAAwLAYJYIZIAYb4QgENBB8W -HU9wZW5TU0wgR2VuZXJhdGVkIENlcnRpZmljYXRlMB0GA1UdDgQWBBRac3SOFCnD -+7QZD5ePqm/h4aj3WzAfBgNVHSMEGDAWgBTHLAGVGvU+zQSmJDU1BNmnFgEqeTAN -BgkqhkiG9w0BAQUFAAOCAQEAMkJLNkSlbPtw2AgryxY0Fds5YHt+tEq8++UWBJcN -6/VoldovI1dMySkr0Rsbn730eXXfYn9jtIR6lVzE7vN3FuQLil7JZL18BFCs/5pB -a7Fqn81FEHKDEIomHX9shDRaQXlyke6HXR06Vf+RflKF/0JB63ZWI+W8vHmxqk5M -v3vfY4saPEsBcok1uw2SlxZurlDLie7GetDTMiIPGTMe7v9BpaElxUzOj5hMtSwf -7MzxIeI6/31qh/6J/SwgPvubuMD5CZnORWOCCRy7edioQCFGx64+3YmdVkZK9O19 -W6Yephsm+ewmtFE6h7ZQE4QzIhqKIMVEZLi73jLsa1jbFw== +gZ8wDQYJKoZIhvcNAQEBBQADgY0AMIGJAoGBAKEQ6syOLHNrMxpeJhm2S0y8BLjC +4jPrZ6J6J68/9+9JX8HSudlx/hegk9rc8Ufe+h/DwdGlKgbLs+iawb14d2hFwVXN +scHT34wST8I6DbdY3MoTCLn7EiSQqrdOBOtDDUW+HBfWqLGvEDw51ghF7al+Ommu +cCKGfnEf8Q7QDTLDAgMBAAGjezB5MAkGA1UdEwQCMAAwLAYJYIZIAYb4QgENBB8W +HU9wZW5TU0wgR2VuZXJhdGVkIENlcnRpZmljYXRlMB0GA1UdDgQWBBQMIHahgJwv +MD33q40xGa3i9+KNEjAfBgNVHSMEGDAWgBQcxyuqG7G7Lpr0D7GGYFc4wkEFEjAN +BgkqhkiG9w0BAQsFAAOCAQEAOcCQExmFR53Gq4zGyQ8zERn3ASwbCPaBmBGrSAXZ +silWMpy65UDfhV5t/W42mhTrkFBX3i/tLYmmikAcQYSb2uFt5nxGsuCQkwIcUi6v +tNSh2J0Zzwpnv8M+LgL0PrwuWVcwhYoyqyKIcjdu7u34U3LJKIdQR4EbgEz4gM4v +R8p4zjhRcOzf7vzqWkAeTRz9TvZ00CKkflffFhqgjb7+7vIHLjmhl0AZ+Tu458SY +bh0aJ9MZTFzJwzGYwTwnDmrez4hyz+RlyQ0zMvHq991bnUJt7seot4XSQeCEOM6G +gbpufdWtegBY18WDnlwdODJySfVCS+fGXBJt4V1RLPVS8A== -----END CERTIFICATE----- diff --git a/mysql-test/std_data/client-key.pem b/mysql-test/std_data/client-key.pem index 5037c6e2728..b6b5507cd15 100644 --- a/mysql-test/std_data/client-key.pem +++ b/mysql-test/std_data/client-key.pem @@ -1,15 +1,15 @@ -----BEGIN RSA PRIVATE KEY----- -MIICXAIBAAKBgQDOoD08pLtPoU+RDQWsW4oVf9eqDKOnn7LHJp1lKLGE06DvnrFF -DzPfmG5x/ytmnJzBJRMnQrYgRufnR6GIR8Ke4kUlmZ/5KBqaE2ddPrO4/kAlrCZJ -RiwDQ4Nn2A9Bri702HFgPI7nkdC7LMoS2nEae+P6jI/Du2JVibO/hUUBYQIDAQAB -AoGAa/FgLFcul3oA9BjmdtVXfMXNp8N0l3QhVFLC9P7eRjK8p5GysA4yHkQmpp0U -UkXMykYRDHiYZqJEMhnEtEowzBmodi7go+gpwAR2eUKwESmJoBhPvqDJAbS/fL5D -H2Wk6FGsdKoPhEpigWefu6ZqlX5GCGa601eMYLMR9i+6bbUCQQDspD4j2q8oihTU -RQt/XpF1l+5ZRHjQOokwRekuHdq0powtNxZ+X3V8Qy8JbDRNCM2YtfKMX4gXAfZp -JWs7HoPvAkEA34doY3AKxZSpXD84m4dnJ0/Ubfk3+tcC1EPYyDJ1DHpfz7fy6aoX -z8TWCQXtSBGaEa9Dgbz+EFXuctLbUR8/rwJACDjIo+xEK69oe9uOQ7WgbiqCMH3N -iMaP36p+KIkHAUHMGwIP+QIODewzpSsqQgbtRcIElFX5X3tE+XBAYoRz5wJAKH3/ -CwRg7ynfBDbvqjz9EsIDWWisG2SXvpwLyThau8fvU1GfT3Tgm2Ks4zWPpl6J6mo1 -cGssGwl2CJbp4+glQQJBAJAwvKufpB+M6OjvKh89GGsCEaV1ENJ41FPcQwJ2pjed -Fcq28ZP59v7bfBH2IkNu3pfEzmvQnmRlTEtXGjNn+i8= +MIICXQIBAAKBgQChEOrMjixzazMaXiYZtktMvAS4wuIz62eieievP/fvSV/B0rnZ +cf4XoJPa3PFH3vofw8HRpSoGy7PomsG9eHdoRcFVzbHB09+MEk/COg23WNzKEwi5 ++xIkkKq3TgTrQw1FvhwX1qixrxA8OdYIRe2pfjpprnAihn5xH/EO0A0ywwIDAQAB +AoGAaJMqT+vTcpDcmjcct78DPuwoiKmH4rvqCrUTRRbcbJGQSbD/F+6KUl7hAM5J +Hifo8GzST8LT+ZuS0OiB9/naVGJjcLzpV+us+keMctB/cjmsPAAsRdeA8Xk0jTWv +v+5a5ZWSSbRXycuUtywtoESY1RLEyB0k3Dcxvk6SD3LnNMkCQQDTyDJfP+eirhgC +mKiNvHJjLtlRT11IMLMVTELEntsyKqzVgficZCWGkFRoF9iEO1OyCJaug7RZsuxs +Z3lk/gq1AkEAwrHwzVPU+dTqsZ8tYHlq+d0xy+6eTtYy7e/5qH3AUz10us6BG/LY +XVTwRFAOKQOciKZ/zPQa4oYeAc0oozSalwJBAIuMbyS0Rz262bdcQDSk5/rS8//P ++/eFn3t5NMW6p1T3KcvGSLtEgjWZBQVSMSlwrkWxwxhbUIuKip8jz6nse8ECQDgm +g5FkLRdEfc9uXfLl8aFQVu0+y29nPVb8D9+1LMOVBNZfekLqPdZlCcpZ4EuZ3ApZ +IqCkgiB0l7DjT2YKZM8CQQCp8z1JvoNRwhIuojh7JajY5QBkXtvHootXr+vDUHNf +SgR5BRmqZb0F5/BK4/7JbumiHI11QUULxhOnNtlS7DDt -----END RSA PRIVATE KEY----- diff --git a/mysql-test/std_data/server-cert.pem b/mysql-test/std_data/server-cert.pem index 1cc1519ada9..b874f129ce3 100644 --- a/mysql-test/std_data/server-cert.pem +++ b/mysql-test/std_data/server-cert.pem @@ -2,25 +2,25 @@ Certificate: Data: Version: 3 (0x2) Serial Number: 1 (0x1) - Signature Algorithm: sha1WithRSAEncryption + Signature Algorithm: sha256WithRSAEncryption Issuer: CN=cacert, C=FI, ST=Helsinki, L=Helsinki, O=MariaDB Validity - Not Before: Apr 25 14:55:05 2015 GMT - Not After : Apr 20 14:55:05 2035 GMT + Not Before: Apr 25 20:52:21 2017 GMT + Not After : Apr 20 20:52:21 2037 GMT Subject: C=FI, ST=state or province within country, in other certificates in this file it is the same as L, L=location, usually an address but often ambiguously used, O=organization name, typically a company name, OU=organizational unit name, a division name within an organization, CN=localhost Subject Public Key Info: Public Key Algorithm: rsaEncryption Public-Key: (1024 bit) Modulus: - 00:aa:e6:54:bd:dd:52:1e:16:f7:24:52:37:58:2b: - a7:af:49:e1:cd:75:2a:18:52:e1:48:f0:59:82:c0: - 7a:d9:66:b3:97:04:b3:77:f4:39:fd:d1:c0:1a:c5: - a6:ab:44:84:d2:17:39:53:25:63:9b:c3:24:78:51: - 5c:77:6b:df:b4:82:1d:e4:43:f4:67:0a:5d:89:a2: - fe:b0:ea:64:3a:1d:9d:49:78:c8:7f:79:a5:cd:45: - 4b:0c:ad:ae:4f:e2:d4:5d:ec:e8:73:06:ed:98:92: - 85:49:b2:9c:31:3b:44:38:5f:bb:5a:f1:68:84:a9: - c3:5b:31:39:d4:47:98:38:55 + 00:c9:fe:83:3f:0b:38:89:1d:43:15:93:5c:26:b9: + 80:65:41:bd:2f:63:66:5e:db:fa:33:20:d9:c7:e1: + 35:f3:14:3c:c8:b4:f7:09:d8:f5:b5:44:8f:6b:7e: + a0:a4:3b:45:5f:e6:f4:0f:08:67:f8:5a:4c:49:e4: + e5:39:31:69:8c:cf:25:78:93:a6:7f:58:e9:90:9c: + 61:cb:2e:14:b1:57:b2:15:9c:ea:8d:6f:96:20:fe: + 29:ed:2c:71:b8:4f:1f:e0:05:6c:04:b1:7e:e0:bc: + 42:8e:bf:95:5e:a9:5e:c9:c9:a4:64:c2:1f:59:94: + 14:c2:06:44:79:bc:d2:65:2d Exponent: 65537 (0x10001) X509v3 extensions: X509v3 Basic Constraints: @@ -28,47 +28,47 @@ Certificate: Netscape Comment: OpenSSL Generated Certificate X509v3 Subject Key Identifier: - E5:72:8F:57:72:D6:75:63:28:7F:E2:BF:00:B7:1D:B8:AA:FE:94:59 + CB:22:3B:E6:DA:B8:3D:7E:39:61:18:38:50:C8:4D:B4:C8:9A:3E:2B X509v3 Authority Key Identifier: - keyid:C7:2C:01:95:1A:F5:3E:CD:04:A6:24:35:35:04:D9:A7:16:01:2A:79 + keyid:1C:C7:2B:AA:1B:B1:BB:2E:9A:F4:0F:B1:86:60:57:38:C2:41:05:12 - Signature Algorithm: sha1WithRSAEncryption - 88:44:46:fa:7d:16:ae:9d:16:5b:95:26:03:3c:71:f4:29:3d: - df:cb:f4:14:20:9f:87:24:b4:29:17:2d:7a:12:48:76:ac:00: - 44:26:ba:93:83:ad:58:7e:b7:77:e4:b0:32:0d:e5:dd:fb:cc: - 0e:9b:88:e0:24:82:e4:41:43:47:5a:4e:d3:b4:5b:47:4b:57: - eb:67:02:63:bb:dd:05:12:f5:95:01:0b:89:81:ca:c2:91:14: - 21:9a:9e:c9:84:91:46:35:0e:26:44:1e:91:88:74:4f:fe:d3: - 19:9e:65:fa:46:e2:46:04:ad:91:79:4c:70:1b:68:b2:49:e9: - 6c:f4:58:44:3b:43:15:85:56:64:1b:84:74:49:95:9f:cd:93: - 9d:8e:69:ab:ca:46:97:b6:74:e9:2a:83:85:62:cd:e5:be:c3: - 52:bd:cf:90:cc:60:27:76:ee:1b:3c:da:69:73:e2:11:68:14: - dc:7d:9f:b8:6f:20:a2:0c:b7:8e:33:40:89:d1:a3:89:e2:60: - 6a:ec:b5:9f:e8:c5:55:10:40:b2:95:5e:54:8a:10:8e:d5:90: - d9:98:86:d8:f9:b6:01:41:8c:d7:0d:0e:86:0e:50:6d:a2:64: - 00:2a:91:5e:35:64:15:e3:86:34:3a:39:eb:0f:4f:56:c7:15: - 4c:74:2e:91 + Signature Algorithm: sha256WithRSAEncryption + 8e:b7:3d:cc:2b:e5:27:49:49:5a:d4:3c:83:9b:2f:7d:11:de: + 6b:0f:b4:51:02:e4:37:d0:c4:b5:7b:4b:e3:42:93:75:32:d1: + eb:41:a2:27:fe:4d:91:ae:2b:a0:8b:3b:7f:e9:1b:47:85:73: + 9f:b7:05:74:34:eb:62:12:d8:f2:24:6a:b4:24:58:7a:6a:55: + 3e:ba:54:f9:51:de:54:0e:19:06:f6:01:23:32:0a:6c:81:e2: + 57:8a:9e:71:c5:ba:ca:c3:0f:18:d4:ef:54:4b:e1:ee:7c:3a: + f3:4d:c3:a3:44:21:9f:c8:ef:85:01:ab:fe:a9:7b:36:05:10: + 25:5e:61:b8:1d:7c:40:8a:5d:5b:c1:bb:7c:79:45:6e:84:63: + b5:3f:51:e9:9c:57:01:de:2d:1e:85:cb:83:f0:16:6d:78:bb: + 12:01:a5:e5:a2:a7:80:fa:54:8f:9a:5c:de:1c:52:a2:bd:00: + 49:e4:04:65:30:f9:b9:fc:4f:94:e2:d8:39:89:b6:a5:a5:2d: + db:25:a3:0d:5b:f9:e1:2a:13:19:5e:d7:1a:33:89:5b:ac:bd: + 29:26:da:1a:90:7e:08:19:dd:59:4b:50:fd:46:c3:0b:91:33: + 8b:c6:70:d1:22:18:37:a3:8b:d3:8f:f8:68:cf:38:8e:e1:ef: + ac:17:88:ca -----BEGIN CERTIFICATE----- -MIIEETCCAvmgAwIBAgIBATANBgkqhkiG9w0BAQUFADBWMQ8wDQYDVQQDDAZjYWNl +MIIEETCCAvmgAwIBAgIBATANBgkqhkiG9w0BAQsFADBWMQ8wDQYDVQQDDAZjYWNl cnQxCzAJBgNVBAYTAkZJMREwDwYDVQQIDAhIZWxzaW5raTERMA8GA1UEBwwISGVs -c2lua2kxEDAOBgNVBAoMB01hcmlhREIwHhcNMTUwNDI1MTQ1NTA1WhcNMzUwNDIw -MTQ1NTA1WjCCAUcxCzAJBgNVBAYTAkZJMWEwXwYDVQQIDFhzdGF0ZSBvciBwcm92 +c2lua2kxEDAOBgNVBAoMB01hcmlhREIwHhcNMTcwNDI1MjA1MjIxWhcNMzcwNDIw +MjA1MjIxWjCCAUcxCzAJBgNVBAYTAkZJMWEwXwYDVQQIDFhzdGF0ZSBvciBwcm92 aW5jZSB3aXRoaW4gY291bnRyeSwgaW4gb3RoZXIgY2VydGlmaWNhdGVzIGluIHRo aXMgZmlsZSBpdCBpcyB0aGUgc2FtZSBhcyBMMUAwPgYDVQQHDDdsb2NhdGlvbiwg dXN1YWxseSBhbiBhZGRyZXNzIGJ1dCBvZnRlbiBhbWJpZ3VvdXNseSB1c2VkMTQw MgYDVQQKDCtvcmdhbml6YXRpb24gbmFtZSwgdHlwaWNhbGx5IGEgY29tcGFueSBu YW1lMUkwRwYDVQQLDEBvcmdhbml6YXRpb25hbCB1bml0IG5hbWUsIGEgZGl2aXNp b24gbmFtZSB3aXRoaW4gYW4gb3JnYW5pemF0aW9uMRIwEAYDVQQDDAlsb2NhbGhv -c3QwgZ8wDQYJKoZIhvcNAQEBBQADgY0AMIGJAoGBAKrmVL3dUh4W9yRSN1grp69J -4c11KhhS4UjwWYLAetlms5cEs3f0Of3RwBrFpqtEhNIXOVMlY5vDJHhRXHdr37SC -HeRD9GcKXYmi/rDqZDodnUl4yH95pc1FSwytrk/i1F3s6HMG7ZiShUmynDE7RDhf -u1rxaISpw1sxOdRHmDhVAgMBAAGjezB5MAkGA1UdEwQCMAAwLAYJYIZIAYb4QgEN -BB8WHU9wZW5TU0wgR2VuZXJhdGVkIENlcnRpZmljYXRlMB0GA1UdDgQWBBTlco9X -ctZ1Yyh/4r8Atx24qv6UWTAfBgNVHSMEGDAWgBTHLAGVGvU+zQSmJDU1BNmnFgEq -eTANBgkqhkiG9w0BAQUFAAOCAQEAiERG+n0Wrp0WW5UmAzxx9Ck938v0FCCfhyS0 -KRctehJIdqwARCa6k4OtWH63d+SwMg3l3fvMDpuI4CSC5EFDR1pO07RbR0tX62cC -Y7vdBRL1lQELiYHKwpEUIZqeyYSRRjUOJkQekYh0T/7TGZ5l+kbiRgStkXlMcBto -sknpbPRYRDtDFYVWZBuEdEmVn82TnY5pq8pGl7Z06SqDhWLN5b7DUr3PkMxgJ3bu -GzzaaXPiEWgU3H2fuG8gogy3jjNAidGjieJgauy1n+jFVRBAspVeVIoQjtWQ2ZiG -2Pm2AUGM1w0Ohg5QbaJkACqRXjVkFeOGNDo56w9PVscVTHQukQ== +c3QwgZ8wDQYJKoZIhvcNAQEBBQADgY0AMIGJAoGBAMn+gz8LOIkdQxWTXCa5gGVB +vS9jZl7b+jMg2cfhNfMUPMi09wnY9bVEj2t+oKQ7RV/m9A8IZ/haTEnk5TkxaYzP +JXiTpn9Y6ZCcYcsuFLFXshWc6o1vliD+Ke0scbhPH+AFbASxfuC8Qo6/lV6pXsnJ +pGTCH1mUFMIGRHm80mUtAgMBAAGjezB5MAkGA1UdEwQCMAAwLAYJYIZIAYb4QgEN +BB8WHU9wZW5TU0wgR2VuZXJhdGVkIENlcnRpZmljYXRlMB0GA1UdDgQWBBTLIjvm +2rg9fjlhGDhQyE20yJo+KzAfBgNVHSMEGDAWgBQcxyuqG7G7Lpr0D7GGYFc4wkEF +EjANBgkqhkiG9w0BAQsFAAOCAQEAjrc9zCvlJ0lJWtQ8g5svfRHeaw+0UQLkN9DE +tXtL40KTdTLR60GiJ/5Nka4roIs7f+kbR4Vzn7cFdDTrYhLY8iRqtCRYempVPrpU ++VHeVA4ZBvYBIzIKbIHiV4qeccW6ysMPGNTvVEvh7nw6803Do0Qhn8jvhQGr/ql7 +NgUQJV5huB18QIpdW8G7fHlFboRjtT9R6ZxXAd4tHoXLg/AWbXi7EgGl5aKngPpU +j5pc3hxSor0ASeQEZTD5ufxPlOLYOYm2paUt2yWjDVv54SoTGV7XGjOJW6y9KSba +GpB+CBndWUtQ/UbDC5Ezi8Zw0SIYN6OL04/4aM84juHvrBeIyg== -----END CERTIFICATE----- diff --git a/mysql-test/std_data/server-key.pem b/mysql-test/std_data/server-key.pem index 3125ae88a8a..feb7fbaf535 100644 --- a/mysql-test/std_data/server-key.pem +++ b/mysql-test/std_data/server-key.pem @@ -1,15 +1,15 @@ -----BEGIN RSA PRIVATE KEY----- -MIICXAIBAAKBgQCq5lS93VIeFvckUjdYK6evSeHNdSoYUuFI8FmCwHrZZrOXBLN3 -9Dn90cAaxaarRITSFzlTJWObwyR4UVx3a9+0gh3kQ/RnCl2Jov6w6mQ6HZ1JeMh/ -eaXNRUsMra5P4tRd7OhzBu2YkoVJspwxO0Q4X7ta8WiEqcNbMTnUR5g4VQIDAQAB -AoGAblQWXyBzdBN1Z5BgRF6ieYpj6OT70QoogJMR5lRmutUPma4iQo17pr3znBT/ -nU+1w3/UtTXNEXCwqbA01q/gkbP2PaW/sbHLVow1B7u/o42WW6I3Btnl3ClnCNjD -Mo7/Gj027hhp7mC61r81JeJVh8fJUgxdNqoH7AkDnA+FJAECQQDjIl3k6W2P+bHb -bp+8eyY7ITQbppZh+3hFJKRL7DZKFYL5J6gejiBURnG9DKnhoSP2nqzqdrRhWZhB -ZHr+ciEBAkEAwJ5rMpFoIwRzgPD4Q4iSqHcBbFcJE7dK1XLq6MYUVNQGfDU8pBvI -EocXphpsJ8CbR35dGDY19rmO2LjG3RBDVQJAetRN9Inrjw2YCjNzvKjYTuew1zcq -YghszO94zfoKjdu+PWEdwJBZmVmTDoo3oGXVHfxHRHA3MeISvWJKRSmRAQJAHL9H -9msXJKrEZkkQdFvMr5HbR4UR2LxxUbvt7UGqxSJDuYPkggWXbZR15hdpbuFjC1+D -m1pz4Ve+RwAExfdoZQJBANfmuWtlLU+SMpDG4zOyC7u4dz+TtnEOfDUECFNZtqvU -MWz98MIXAjiBDYU1Z0BrA7b0/FVsPR3t6JZFQWWI2y8= +MIICXQIBAAKBgQDJ/oM/CziJHUMVk1wmuYBlQb0vY2Ze2/ozINnH4TXzFDzItPcJ +2PW1RI9rfqCkO0Vf5vQPCGf4WkxJ5OU5MWmMzyV4k6Z/WOmQnGHLLhSxV7IVnOqN +b5Yg/intLHG4Tx/gBWwEsX7gvEKOv5VeqV7JyaRkwh9ZlBTCBkR5vNJlLQIDAQAB +AoGAZzoeMJW3arr9kYUhTtj1+MlDuXf+1PNuRbrAERnSzErHVXrF1M5/owdKlBC+ +X6+6oGwSxavyFkVvP0QusK/D1DxSs60Mk1p3Ax5nVpNYiBdeBULcYiXCSUbKItYL +vzyECIc02t+auWf/wp3Wy3g6sU6FoT2QCPSsy0qRC48VBzECQQD79P9w0XjJGk8y +3zPc4JOr3a1UUu4VLjFdFDR2eZZRE02NcXfLHE0+Idk3TYnNDRLoWDJjdfZ0thZZ +KuJ58wIDAkEAzTxGYmT/aieDC3G8sHMqLUvjN30TfTocv7mWlpxWo7zbRbQm7jsB +S5weRKtu3yVmdb6rW+5IZSCazc/j4T5tDwJAUgDRSpTCrSFE+Zevt4nYRi2mBjXf +i0E3i8XRtWWpSMXxjcGKba7ObRRzMA6qdPR2XOynqbtw9Vd2Ops8jthKpQJBAJJm +8tZxsXlqIiLhyXYdUPLq9XS5tlNYRvXFT9S0RWGb8NbyQesjEDN9dGIL4JUxurs3 +fkmf2ui4r3UtXSNqQqUCQQDjatAhBdibh5UawO5VpI87OJMzATCkY/mX3R+TnbOm +MkdydjF022P0M4N24DiM+2nBTDp+F45LwtQLa4vmB2No -----END RSA PRIVATE KEY----- diff --git a/mysql-test/std_data/server8k-cert.pem b/mysql-test/std_data/server8k-cert.pem index 7e41195cea4..df27c77e8e0 100644 --- a/mysql-test/std_data/server8k-cert.pem +++ b/mysql-test/std_data/server8k-cert.pem @@ -2,85 +2,85 @@ Certificate: Data: Version: 3 (0x2) Serial Number: 2 (0x2) - Signature Algorithm: sha1WithRSAEncryption + Signature Algorithm: sha256WithRSAEncryption Issuer: CN=cacert, C=FI, ST=Helsinki, L=Helsinki, O=MariaDB Validity - Not Before: Apr 25 14:55:16 2015 GMT - Not After : Apr 20 14:55:16 2035 GMT + Not Before: Apr 25 20:52:33 2017 GMT + Not After : Apr 20 20:52:33 2037 GMT Subject: C=FI, ST=Helsinki, L=Helsinki, O=MariaDB, CN=server8k Subject Public Key Info: Public Key Algorithm: rsaEncryption Public-Key: (8192 bit) Modulus: - 00:e6:24:79:3d:3a:58:6a:12:1c:13:6a:43:d9:c1: - 65:ec:55:c1:4f:7d:fc:f2:a6:56:a5:ab:c5:48:2c: - ce:9c:9d:47:3d:94:93:d5:3b:a3:d0:09:a8:8f:e0: - 4b:36:a0:95:ae:2d:ad:7f:a2:a3:c5:f6:87:80:4c: - 6a:26:15:47:73:20:47:e1:f8:5f:49:b7:13:20:f7: - 32:9f:db:7d:62:41:1b:60:26:7b:41:26:16:0c:92: - 22:ef:b4:2e:b3:db:90:e9:09:ca:fe:1a:1b:e4:f3: - 78:69:7d:ca:6e:c5:39:4b:46:72:09:51:08:40:78: - 8e:04:2a:23:cb:d5:50:cf:96:dd:56:43:10:1b:d2: - 71:28:8f:10:a9:e4:44:1c:39:8a:06:a7:fa:37:48: - bd:5a:dd:37:7f:7a:00:cf:84:8a:48:a3:75:a5:67: - fb:7b:47:2d:26:00:2e:65:ed:63:4e:b7:94:18:3f: - 5a:08:74:54:e5:af:dd:86:1b:34:9b:4c:de:fe:d8: - 69:3d:72:90:c7:5c:83:82:78:ea:ab:06:d4:f1:06: - 20:ad:fd:24:bb:e9:94:e0:c6:32:f5:df:af:b4:14: - 4d:9c:ce:0d:62:3a:3a:2b:11:f9:9e:d7:8e:63:2b: - 57:35:10:7e:d5:44:64:8c:61:9b:4e:f8:e8:a5:fb: - 47:bb:85:33:ea:c6:e0:3d:e2:2c:e8:41:e1:15:e2: - a7:45:23:df:d0:f9:93:01:97:89:95:77:4d:d5:d4: - c1:db:61:ca:e2:84:36:9c:01:9b:33:ec:53:83:2f: - dd:d7:d8:20:c0:6b:4a:73:92:d2:6c:22:a6:a2:68: - 46:8b:aa:3f:aa:fe:47:b7:98:70:fd:ba:59:88:af: - 9f:0d:d7:cb:a0:42:44:f9:f0:54:39:c1:cd:fb:b4: - e4:c6:d6:7e:1d:f5:ed:b9:1a:0f:d7:e6:a2:ab:a2: - 25:1f:6c:f9:ab:9c:d8:a3:b9:da:32:72:51:6d:61: - f1:3b:7d:06:2b:3d:43:d5:52:1f:cb:62:14:53:69: - 4d:91:12:22:f0:55:f9:fc:4f:de:ee:e3:fc:fd:40: - 57:50:eb:0e:7a:45:cc:52:0f:24:6f:45:02:72:6b: - 6d:90:94:1c:d4:fb:34:f6:4b:4d:25:17:6f:df:4a: - 64:f8:ad:1e:6e:df:ad:6c:b3:1d:1a:e6:0e:59:7b: - f8:a7:13:77:78:85:bf:3f:7c:12:d4:8e:34:ff:01: - 90:03:42:85:60:e4:99:d6:19:32:46:41:e8:50:ca: - 2a:03:61:cd:c5:68:f4:92:0f:6e:48:89:41:9e:53: - bc:41:62:ed:4a:92:64:b5:cb:3d:55:6c:d9:87:87: - 9a:ab:fc:22:50:66:92:2b:b9:d7:9e:3b:ed:80:e3: - fa:19:69:38:87:b6:25:3a:db:b5:d0:f2:80:4d:af: - 6d:7d:92:90:de:aa:df:be:80:26:1c:69:ee:7a:e3: - 45:c9:a4:4e:6a:e0:56:5f:1f:61:44:3f:62:34:c9: - 1e:21:5f:f6:7f:68:c0:6e:bf:d2:35:1e:53:99:e4: - e1:bd:64:a4:49:3c:c3:ce:b6:e8:a9:3d:27:54:ea: - f1:3e:a1:fc:7b:bf:8d:71:60:90:c5:66:24:85:de: - 7d:47:1f:62:83:e2:63:8e:10:5c:14:cd:d0:7e:86: - 44:4d:df:05:10:43:b8:3c:87:64:69:ec:ea:fb:49: - 9a:c6:76:c1:8f:ea:49:98:0a:d3:97:af:64:ef:da: - 5f:a9:57:03:e3:a0:15:d8:68:c5:40:d8:7b:0f:26: - 0d:5f:f0:be:5a:4c:fd:af:9e:bf:2d:31:40:71:25: - de:d0:73:19:2d:ae:a6:cb:7c:f0:b8:a4:a9:5c:50: - 80:41:4e:dc:f7:20:a4:a6:66:65:fb:92:d1:43:2d: - bf:30:b2:0d:db:9b:a3:ac:28:08:c4:81:99:0c:0d: - 45:e9:a9:e5:6c:da:bf:10:bb:a7:3e:5a:5f:b7:93: - 4a:20:15:29:69:74:78:d1:eb:53:a8:88:49:cb:de: - 0c:e2:9b:31:e2:2f:56:95:cf:55:92:a3:8e:a9:ef: - 68:cb:00:11:d4:71:06:4b:e5:89:0c:b6:e7:2b:2f: - 98:65:21:8e:2a:a3:86:73:bb:1b:76:e1:94:02:d8: - a1:51:97:15:60:a2:39:d5:fd:dc:a8:be:30:12:44: - b1:49:0b:94:82:cf:5f:93:61:1c:3c:eb:05:5b:a4: - 17:ee:30:cd:7c:db:3f:ee:79:02:da:14:20:98:fd: - 9a:0a:f1:39:c8:59:5b:4e:a3:ad:f8:04:e6:0b:b6: - 81:7e:41:00:af:f7:37:ec:6e:bc:28:a3:3d:76:b6: - fd:e9:88:c7:1e:78:79:27:62:a2:83:34:15:61:b8: - e4:c3:ac:f5:7f:3e:4e:5f:5f:31:5b:e8:91:1b:80: - 5e:cb:74:b2:e5:a3:8a:d5:5a:89:fa:63:f1:ff:67: - bd:59:d0:70:77:b7:75:b5:34:74:3d:2e:99:46:0b: - 4b:c4:64:2f:93:48:fe:47:92:6a:0b:42:5e:ef:c9: - 06:64:84:60:89:2b:84:1f:31:0d:36:15:4e:6b:cd: - 14:f7:a0:d1:b2:b8:ff:53:f5:aa:b9:ed:63:50:7a: - 6f:62:e7:c7:7f:bc:f5:e2:0c:f8:28:a4:0d:ba:75: - d0:b8:c7:9b:e3:94:62:66:1c:d8:6c:02:2e:a5:a2: - 62:50:fe:cf:31 + 00:b3:a7:81:cf:a3:9b:3c:57:2a:8e:de:13:08:a6: + 15:6a:68:08:ce:a1:a0:ef:fc:32:95:5a:9a:b5:cc: + 84:bb:a3:15:a0:fc:29:bb:71:0e:f3:40:20:97:35: + 92:92:89:36:6a:5f:be:a5:24:5a:c8:0e:f7:53:a8: + e0:74:05:90:38:21:a5:25:72:2b:56:70:74:aa:d6: + 30:25:e0:95:dd:3d:4a:48:4b:25:a7:a8:c4:eb:e5: + d6:10:4f:95:42:91:b0:cb:68:2f:bf:96:0f:0a:9b: + 2b:01:0d:03:ae:3f:b7:7e:f6:1e:c1:14:42:04:7f: + 09:21:e8:3e:87:c0:b3:d7:dd:4f:7e:95:b6:83:33: + 8f:cc:f4:bb:11:8e:66:5f:b4:32:22:8b:8e:34:93: + 82:0f:02:d1:6f:85:b2:a7:7e:36:0c:f3:0d:91:46: + f9:a5:91:dc:60:4a:0d:eb:e5:37:e3:76:d8:13:bb: + 55:33:23:4c:b3:90:df:6a:b6:80:3d:f6:bd:9d:bd: + ef:d3:8d:7a:9a:61:ac:02:3d:10:b5:c2:53:d2:7e: + ee:1c:90:a5:b0:e8:db:71:8f:d3:53:e4:80:b0:b6: + cc:b0:f2:eb:46:c3:d9:3c:48:01:1d:3a:5c:d0:f2: + 17:9c:9a:8e:fc:2a:36:e7:1c:59:41:66:ad:e1:d3: + 82:d8:95:ee:ae:dc:ab:72:94:cb:0b:8a:df:ef:70: + f7:f9:e2:77:76:64:67:2d:02:dc:1a:db:02:e9:5e: + 63:3f:3e:07:e9:65:e4:02:78:62:55:e1:52:18:5a: + a3:9c:56:dd:cd:d3:f7:f2:55:45:62:20:ce:fd:41: + 93:b9:c2:20:90:23:a4:41:4f:30:5c:5e:a0:97:6f: + a4:1f:87:6a:97:87:70:f9:d5:56:b7:6a:cc:f1:28: + d2:26:34:57:65:9c:2d:41:43:8b:6c:bd:15:6f:ec: + ce:3e:9f:4d:2f:a6:21:d8:f7:4d:d1:63:4f:f0:9e: + 04:c3:82:33:58:34:20:4b:81:0d:42:0d:44:62:48: + 1b:4c:cf:e0:69:e4:c8:f8:9c:98:1f:74:37:3a:3f: + 23:40:3a:88:80:c2:9c:9b:ba:6a:b9:2c:3a:67:a6: + b6:5c:83:fd:93:3f:6e:53:19:48:e3:3d:fa:f5:8e: + 03:40:8f:e6:af:c5:de:aa:f8:70:77:36:2b:6e:3a: + df:bc:d7:f6:c7:4f:02:7f:a1:ad:2f:3c:e9:08:45: + 2f:57:1a:f1:5b:9c:27:07:3a:44:43:3e:05:b1:4a: + 1b:42:a4:90:ab:ce:4b:f7:c1:9b:d7:dd:6f:05:11: + f9:e8:a3:a1:99:6c:2e:27:3d:b1:54:6f:eb:b2:e8: + f8:e6:12:6f:8a:69:05:cc:c6:f6:c4:2d:b0:06:b6: + 21:50:28:50:dc:8c:b2:8c:65:ce:d9:2b:51:b8:62: + 56:70:b4:46:4d:e4:4c:a1:86:31:3d:e3:07:7f:73: + e3:09:6e:ee:a7:29:2a:80:23:8a:b6:0c:22:9e:43: + 1c:a9:8c:47:2c:02:6d:13:ef:88:a7:d7:d1:60:d9: + 6b:28:80:97:8a:b7:ff:62:6b:28:54:0d:ab:63:3a: + ac:57:9e:2f:95:2b:f4:a0:3d:a1:60:a0:57:1a:09: + 23:ee:fc:4d:9b:45:83:f2:0a:1f:b3:77:0e:98:20: + b3:16:fc:97:03:30:fc:ab:3e:7e:fc:3e:76:64:cb: + 4b:4d:c9:6c:8f:aa:70:ec:03:c5:81:31:43:11:9a: + fc:2d:ba:25:94:0b:73:7e:c1:39:01:ca:68:70:1a: + ef:a4:ba:dc:13:d6:37:17:09:0d:ca:aa:21:94:bb: + 3e:43:fd:8c:80:e6:58:fd:75:23:5a:ca:1f:0b:2e: + 35:c2:cd:77:0c:73:b3:2f:9a:e4:03:68:58:c8:3f: + f3:76:8d:05:53:d2:34:db:09:af:70:ed:67:0b:ff: + 2f:64:35:2d:71:22:7a:f2:f2:2b:7d:6d:be:80:bd: + 18:38:37:01:67:a4:5f:48:29:52:78:9f:b5:b8:80: + 70:ca:10:6a:e1:27:3b:c0:61:4e:96:f8:e8:0f:42: + b1:fa:bc:05:ef:31:91:38:ab:0f:a7:ef:2c:86:c1: + ab:85:36:7c:10:9d:c7:0a:6b:e6:2b:11:dd:fa:45: + e7:d2:c3:3a:c1:cd:60:27:68:26:d7:87:49:5f:e3: + 7f:26:20:1f:99:a8:f9:00:a6:0d:d5:13:81:6e:fc: + 27:8a:03:58:6b:0e:3a:c7:20:a1:5c:24:d1:08:a6: + 5a:1e:e7:81:3d:21:05:2e:4b:5d:18:14:f1:4f:ab: + 5c:e5:81:da:c1:2c:f6:12:ff:25:09:97:1e:9d:51: + 6d:1f:63:2e:9c:ed:7d:86:45:b0:35:73:f6:9f:82: + 65:bc:7f:46:88:e1:a1:60:a1:37:50:26:9c:d7:76: + f3:dc:94:89:6d:d7:be:a6:17:a1:2b:71:14:37:17: + 5f:ed:5a:39:99:fc:69:af:bb:63:55:c6:60:7b:64: + d8:bb:b7:4a:9f:b9:aa:89:4b:30:1f:9c:ef:23:7a: + af:7e:31:dd:fb:dd:0a:d5:04:0a:8c:57:6f:64:80: + eb:f3:ee:6c:33:d2:72:c2:24:68:a2:4e:99:88:1c: + f7:3b:5d:10:cc:4e:ea:a5:cb:00:40:8a:a9:63:2e: + 18:92:eb:b1:28:05:fe:19:ea:7b:32:fb:63:56:2c: + 0b:20:01:92:a3 Exponent: 65537 (0x10001) X509v3 extensions: X509v3 Basic Constraints: @@ -88,61 +88,61 @@ Certificate: Netscape Comment: OpenSSL Generated Certificate X509v3 Subject Key Identifier: - B3:A9:00:04:C7:9C:2B:CD:C0:5F:D1:28:5C:5C:C6:1F:26:F7:17:5A + 43:3B:26:15:A5:60:62:3F:07:2A:8C:8C:DA:F8:74:BE:36:B2:33:E4 X509v3 Authority Key Identifier: - keyid:C7:2C:01:95:1A:F5:3E:CD:04:A6:24:35:35:04:D9:A7:16:01:2A:79 + keyid:1C:C7:2B:AA:1B:B1:BB:2E:9A:F4:0F:B1:86:60:57:38:C2:41:05:12 - Signature Algorithm: sha1WithRSAEncryption - 01:43:2c:d5:1d:b5:36:05:31:ca:b2:f3:9e:71:ce:62:fe:b2: - e5:e3:2d:30:23:99:51:3d:50:ee:d6:ce:76:e8:43:8d:5d:9d: - 28:9b:43:90:0c:d4:24:54:3c:53:3c:18:92:ec:93:86:87:7a: - d6:7f:5d:3b:56:cd:9d:96:7a:06:c8:16:19:8b:ed:c8:21:cf: - 15:b9:fb:06:7d:cb:5c:46:9c:c5:2e:8e:de:77:21:e5:9d:35: - 95:00:c7:ad:d2:57:36:65:1a:43:6c:ee:75:ad:a2:d8:c0:60: - d5:07:d7:5c:5d:8e:ae:af:4a:e6:fa:6a:13:78:98:b8:11:c7: - dc:d2:a7:dd:29:b5:76:fa:ef:13:62:5f:9a:0d:e2:87:6a:04: - 3c:5f:72:44:d0:d0:7a:70:c6:09:2b:bf:92:91:6d:f4:2e:53: - f3:b4:c5:23:61:d6:87:c1:30:ef:fe:92:6f:97:78:f2:ea:3c: - ff:e8:54:3a:9f:49:ac:a9:2b:46:c3:76:b1:f9:eb:31:a3:4f: - 40:58:16:90:77:b6:3d:6f:85:95:12:a9:ca:70:0a:9a:cd:61: - 46:27:84:ce:9e:33:54:8f:9a:41:6d:4d:11:bd:14:7f:ff:32: - e9:06:bc:36:38:11:5c:0f:e9:a5:5a:0f:5c:7c:fa:f5:73:5b: - 4c:47:f2:f2 + Signature Algorithm: sha256WithRSAEncryption + 5c:c7:4e:56:eb:21:1b:f3:5d:c4:56:fa:bf:6d:e1:0d:fa:cc: + b0:7c:13:58:b9:d4:47:b3:de:b4:ae:f2:45:02:88:65:43:89: + cf:ea:9c:d0:ea:45:42:35:d2:ec:bc:b4:73:03:db:85:7b:c7: + f0:68:d7:dc:2b:70:71:63:ed:33:68:9f:29:ff:67:0a:69:3b: + c4:65:7c:25:00:cf:7f:ee:aa:fe:44:19:70:f2:74:db:da:9f: + 52:92:1a:03:e6:0a:49:85:2e:2a:02:c1:81:d4:6d:cf:98:d1: + e9:6a:2f:bb:fa:a6:d5:4b:55:38:c2:aa:8c:f5:d3:f9:e6:74: + db:00:d6:29:f9:d1:9c:7a:0e:98:c5:e5:8c:16:49:d8:cf:ee: + a0:cb:c2:2c:31:cf:62:2c:b1:7f:6d:60:b0:ce:d5:07:45:e8: + 44:17:7c:c4:12:fb:00:57:24:57:e8:17:78:3b:1e:0d:91:aa: + 67:98:d0:e5:9a:86:4b:88:4a:2f:55:5d:2e:13:ec:f5:4a:7d: + e7:13:a7:57:03:41:2f:f6:6c:08:8e:58:ef:b0:7a:79:32:e3: + bb:4b:eb:4d:42:cd:42:96:2d:67:f6:4c:c1:f6:62:fb:c0:3e: + 09:69:8f:36:7e:fa:c8:cd:ff:72:3e:df:92:f7:8f:44:cf:77: + fe:6e:74:de -----BEGIN CERTIFICATE----- -MIIGpDCCBYygAwIBAgIBAjANBgkqhkiG9w0BAQUFADBWMQ8wDQYDVQQDDAZjYWNl +MIIGpDCCBYygAwIBAgIBAjANBgkqhkiG9w0BAQsFADBWMQ8wDQYDVQQDDAZjYWNl cnQxCzAJBgNVBAYTAkZJMREwDwYDVQQIDAhIZWxzaW5raTERMA8GA1UEBwwISGVs -c2lua2kxEDAOBgNVBAoMB01hcmlhREIwHhcNMTUwNDI1MTQ1NTE2WhcNMzUwNDIw -MTQ1NTE2WjBYMQswCQYDVQQGEwJGSTERMA8GA1UECAwISGVsc2lua2kxETAPBgNV +c2lua2kxEDAOBgNVBAoMB01hcmlhREIwHhcNMTcwNDI1MjA1MjMzWhcNMzcwNDIw +MjA1MjMzWjBYMQswCQYDVQQGEwJGSTERMA8GA1UECAwISGVsc2lua2kxETAPBgNV BAcMCEhlbHNpbmtpMRAwDgYDVQQKDAdNYXJpYURCMREwDwYDVQQDDAhzZXJ2ZXI4 -azCCBCIwDQYJKoZIhvcNAQEBBQADggQPADCCBAoCggQBAOYkeT06WGoSHBNqQ9nB -ZexVwU99/PKmVqWrxUgszpydRz2Uk9U7o9AJqI/gSzagla4trX+io8X2h4BMaiYV -R3MgR+H4X0m3EyD3Mp/bfWJBG2Ame0EmFgySIu+0LrPbkOkJyv4aG+TzeGl9ym7F -OUtGcglRCEB4jgQqI8vVUM+W3VZDEBvScSiPEKnkRBw5igan+jdIvVrdN396AM+E -ikijdaVn+3tHLSYALmXtY063lBg/Wgh0VOWv3YYbNJtM3v7YaT1ykMdcg4J46qsG -1PEGIK39JLvplODGMvXfr7QUTZzODWI6OisR+Z7XjmMrVzUQftVEZIxhm0746KX7 -R7uFM+rG4D3iLOhB4RXip0Uj39D5kwGXiZV3TdXUwdthyuKENpwBmzPsU4Mv3dfY -IMBrSnOS0mwipqJoRouqP6r+R7eYcP26WYivnw3Xy6BCRPnwVDnBzfu05MbWfh31 -7bkaD9fmoquiJR9s+auc2KO52jJyUW1h8Tt9Bis9Q9VSH8tiFFNpTZESIvBV+fxP -3u7j/P1AV1DrDnpFzFIPJG9FAnJrbZCUHNT7NPZLTSUXb99KZPitHm7frWyzHRrm -Dll7+KcTd3iFvz98EtSONP8BkANChWDkmdYZMkZB6FDKKgNhzcVo9JIPbkiJQZ5T -vEFi7UqSZLXLPVVs2YeHmqv8IlBmkiu515477YDj+hlpOIe2JTrbtdDygE2vbX2S -kN6q376AJhxp7nrjRcmkTmrgVl8fYUQ/YjTJHiFf9n9owG6/0jUeU5nk4b1kpEk8 -w8626Kk9J1Tq8T6h/Hu/jXFgkMVmJIXefUcfYoPiY44QXBTN0H6GRE3fBRBDuDyH -ZGns6vtJmsZ2wY/qSZgK05evZO/aX6lXA+OgFdhoxUDYew8mDV/wvlpM/a+evy0x -QHEl3tBzGS2upst88LikqVxQgEFO3PcgpKZmZfuS0UMtvzCyDdubo6woCMSBmQwN -Remp5WzavxC7pz5aX7eTSiAVKWl0eNHrU6iIScveDOKbMeIvVpXPVZKjjqnvaMsA -EdRxBkvliQy25ysvmGUhjiqjhnO7G3bhlALYoVGXFWCiOdX93Ki+MBJEsUkLlILP -X5NhHDzrBVukF+4wzXzbP+55AtoUIJj9mgrxOchZW06jrfgE5gu2gX5BAK/3N+xu -vCijPXa2/emIxx54eSdiooM0FWG45MOs9X8+Tl9fMVvokRuAXst0suWjitVaifpj -8f9nvVnQcHe3dbU0dD0umUYLS8RkL5NI/keSagtCXu/JBmSEYIkrhB8xDTYVTmvN -FPeg0bK4/1P1qrntY1B6b2Lnx3+89eIM+CikDbp10LjHm+OUYmYc2GwCLqWiYlD+ -zzECAwEAAaN7MHkwCQYDVR0TBAIwADAsBglghkgBhvhCAQ0EHxYdT3BlblNTTCBH -ZW5lcmF0ZWQgQ2VydGlmaWNhdGUwHQYDVR0OBBYEFLOpAATHnCvNwF/RKFxcxh8m -9xdaMB8GA1UdIwQYMBaAFMcsAZUa9T7NBKYkNTUE2acWASp5MA0GCSqGSIb3DQEB -BQUAA4IBAQABQyzVHbU2BTHKsvOecc5i/rLl4y0wI5lRPVDu1s526EONXZ0om0OQ -DNQkVDxTPBiS7JOGh3rWf107Vs2dlnoGyBYZi+3IIc8VufsGfctcRpzFLo7edyHl -nTWVAMet0lc2ZRpDbO51raLYwGDVB9dcXY6ur0rm+moTeJi4Ecfc0qfdKbV2+u8T -Yl+aDeKHagQ8X3JE0NB6cMYJK7+SkW30LlPztMUjYdaHwTDv/pJvl3jy6jz/6FQ6 -n0msqStGw3ax+esxo09AWBaQd7Y9b4WVEqnKcAqazWFGJ4TOnjNUj5pBbU0RvRR/ -/zLpBrw2OBFcD+mlWg9cfPr1c1tMR/Ly +azCCBCIwDQYJKoZIhvcNAQEBBQADggQPADCCBAoCggQBALOngc+jmzxXKo7eEwim +FWpoCM6hoO/8MpVamrXMhLujFaD8KbtxDvNAIJc1kpKJNmpfvqUkWsgO91Oo4HQF +kDghpSVyK1ZwdKrWMCXgld09SkhLJaeoxOvl1hBPlUKRsMtoL7+WDwqbKwENA64/ +t372HsEUQgR/CSHoPofAs9fdT36VtoMzj8z0uxGOZl+0MiKLjjSTgg8C0W+Fsqd+ +NgzzDZFG+aWR3GBKDevlN+N22BO7VTMjTLOQ32q2gD32vZ2979ONepphrAI9ELXC +U9J+7hyQpbDo23GP01PkgLC2zLDy60bD2TxIAR06XNDyF5yajvwqNuccWUFmreHT +gtiV7q7cq3KUywuK3+9w9/nid3ZkZy0C3BrbAuleYz8+B+ll5AJ4YlXhUhhao5xW +3c3T9/JVRWIgzv1Bk7nCIJAjpEFPMFxeoJdvpB+HapeHcPnVVrdqzPEo0iY0V2Wc +LUFDi2y9FW/szj6fTS+mIdj3TdFjT/CeBMOCM1g0IEuBDUINRGJIG0zP4GnkyPic +mB90Nzo/I0A6iIDCnJu6arksOmemtlyD/ZM/blMZSOM9+vWOA0CP5q/F3qr4cHc2 +K24637zX9sdPAn+hrS886QhFL1ca8VucJwc6REM+BbFKG0KkkKvOS/fBm9fdbwUR ++eijoZlsLic9sVRv67Lo+OYSb4ppBczG9sQtsAa2IVAoUNyMsoxlztkrUbhiVnC0 +Rk3kTKGGMT3jB39z4wlu7qcpKoAjirYMIp5DHKmMRywCbRPviKfX0WDZayiAl4q3 +/2JrKFQNq2M6rFeeL5Ur9KA9oWCgVxoJI+78TZtFg/IKH7N3Dpggsxb8lwMw/Ks+ +fvw+dmTLS03JbI+qcOwDxYExQxGa/C26JZQLc37BOQHKaHAa76S63BPWNxcJDcqq +IZS7PkP9jIDmWP11I1rKHwsuNcLNdwxzsy+a5ANoWMg/83aNBVPSNNsJr3DtZwv/ +L2Q1LXEievLyK31tvoC9GDg3AWekX0gpUniftbiAcMoQauEnO8BhTpb46A9Csfq8 +Be8xkTirD6fvLIbBq4U2fBCdxwpr5isR3fpF59LDOsHNYCdoJteHSV/jfyYgH5mo ++QCmDdUTgW78J4oDWGsOOscgoVwk0QimWh7ngT0hBS5LXRgU8U+rXOWB2sEs9hL/ +JQmXHp1RbR9jLpztfYZFsDVz9p+CZbx/RojhoWChN1AmnNd289yUiW3XvqYXoStx +FDcXX+1aOZn8aa+7Y1XGYHtk2Lu3Sp+5qolLMB+c7yN6r34x3fvdCtUECoxXb2SA +6/PubDPScsIkaKJOmYgc9ztdEMxO6qXLAECKqWMuGJLrsSgF/hnqezL7Y1YsCyAB +kqMCAwEAAaN7MHkwCQYDVR0TBAIwADAsBglghkgBhvhCAQ0EHxYdT3BlblNTTCBH +ZW5lcmF0ZWQgQ2VydGlmaWNhdGUwHQYDVR0OBBYEFEM7JhWlYGI/ByqMjNr4dL42 +sjPkMB8GA1UdIwQYMBaAFBzHK6obsbsumvQPsYZgVzjCQQUSMA0GCSqGSIb3DQEB +CwUAA4IBAQBcx05W6yEb813EVvq/beEN+sywfBNYudRHs960rvJFAohlQ4nP6pzQ +6kVCNdLsvLRzA9uFe8fwaNfcK3BxY+0zaJ8p/2cKaTvEZXwlAM9/7qr+RBlw8nTb +2p9SkhoD5gpJhS4qAsGB1G3PmNHpai+7+qbVS1U4wqqM9dP55nTbANYp+dGceg6Y +xeWMFknYz+6gy8IsMc9iLLF/bWCwztUHRehEF3zEEvsAVyRX6Bd4Ox4NkapnmNDl +moZLiEovVV0uE+z1Sn3nE6dXA0Ev9mwIjljvsHp5MuO7S+tNQs1Cli1n9kzB9mL7 +wD4JaY82fvrIzf9yPt+S949Ez3f+bnTe -----END CERTIFICATE----- diff --git a/mysql-test/std_data/server8k-key.pem b/mysql-test/std_data/server8k-key.pem index 72d2756477c..a383b359275 100644 --- a/mysql-test/std_data/server8k-key.pem +++ b/mysql-test/std_data/server8k-key.pem @@ -1,99 +1,99 @@ -----BEGIN RSA PRIVATE KEY----- -MIISKgIBAAKCBAEA5iR5PTpYahIcE2pD2cFl7FXBT3388qZWpavFSCzOnJ1HPZST -1Tuj0Amoj+BLNqCVri2tf6KjxfaHgExqJhVHcyBH4fhfSbcTIPcyn9t9YkEbYCZ7 -QSYWDJIi77Qus9uQ6QnK/hob5PN4aX3KbsU5S0ZyCVEIQHiOBCojy9VQz5bdVkMQ -G9JxKI8QqeREHDmKBqf6N0i9Wt03f3oAz4SKSKN1pWf7e0ctJgAuZe1jTreUGD9a -CHRU5a/dhhs0m0ze/thpPXKQx1yDgnjqqwbU8QYgrf0ku+mU4MYy9d+vtBRNnM4N -Yjo6KxH5nteOYytXNRB+1URkjGGbTvjopftHu4Uz6sbgPeIs6EHhFeKnRSPf0PmT -AZeJlXdN1dTB22HK4oQ2nAGbM+xTgy/d19ggwGtKc5LSbCKmomhGi6o/qv5Ht5hw -/bpZiK+fDdfLoEJE+fBUOcHN+7TkxtZ+HfXtuRoP1+aiq6IlH2z5q5zYo7naMnJR -bWHxO30GKz1D1VIfy2IUU2lNkRIi8FX5/E/e7uP8/UBXUOsOekXMUg8kb0UCcmtt -kJQc1Ps09ktNJRdv30pk+K0ebt+tbLMdGuYOWXv4pxN3eIW/P3wS1I40/wGQA0KF -YOSZ1hkyRkHoUMoqA2HNxWj0kg9uSIlBnlO8QWLtSpJktcs9VWzZh4eaq/wiUGaS -K7nXnjvtgOP6GWk4h7YlOtu10PKATa9tfZKQ3qrfvoAmHGnueuNFyaROauBWXx9h -RD9iNMkeIV/2f2jAbr/SNR5TmeThvWSkSTzDzrboqT0nVOrxPqH8e7+NcWCQxWYk -hd59Rx9ig+JjjhBcFM3QfoZETd8FEEO4PIdkaezq+0maxnbBj+pJmArTl69k79pf -qVcD46AV2GjFQNh7DyYNX/C+Wkz9r56/LTFAcSXe0HMZLa6my3zwuKSpXFCAQU7c -9yCkpmZl+5LRQy2/MLIN25ujrCgIxIGZDA1F6anlbNq/ELunPlpft5NKIBUpaXR4 -0etTqIhJy94M4psx4i9Wlc9VkqOOqe9oywAR1HEGS+WJDLbnKy+YZSGOKqOGc7sb -duGUAtihUZcVYKI51f3cqL4wEkSxSQuUgs9fk2EcPOsFW6QX7jDNfNs/7nkC2hQg -mP2aCvE5yFlbTqOt+ATmC7aBfkEAr/c37G68KKM9drb96YjHHnh5J2KigzQVYbjk -w6z1fz5OX18xW+iRG4Bey3Sy5aOK1VqJ+mPx/2e9WdBwd7d1tTR0PS6ZRgtLxGQv -k0j+R5JqC0Je78kGZIRgiSuEHzENNhVOa80U96DRsrj/U/Wque1jUHpvYufHf7z1 -4gz4KKQNunXQuMeb45RiZhzYbAIupaJiUP7PMQIDAQABAoIEAQDdg63OaSJAtj2f -0mCMb8ISwFfYk4Osar5rp/Gzjq0vwZKYizHfxA/gZeuA0HqUkeyAQicE+x53pNq3 -etWQ4lprTV7i+ZV99mDLEiQACdudft1Cpsdr5aTDZMWLwvpQ072fEHX6Llc6/72e -jB0UkXCcK6oHnZ87rs3C5Gyf/SpTJPrV1KbkoKGaUFnRrIyCPj/EOFp2a+UWWGba -pCuzkfcoA21xT6yW8+NY+EOwh9VWJwy8af3WtWIh0ix+sCDqegsJcHObWXJQ8ZMD -Oi9lfqXnd+ZskYOR+zn5P8w9LJiJ1CEAFLR9H15tpleFtCSw/z5pLP9ndvTwyeIb -GSmU2VAqBgP6roGYDuL6iq6Dyi0GN4luM0pz9c/PtY2Ni8MrzeziKjAF6OXiDE41 -rxTwdG7RxnNa2q7+tjt9hrCgLbYqd6W/6VKYZY1YKvIE+PhdwGtzxwbKsOf6phqt -6DZr0BKIX7VwMeXRHbhtjw6hR86diH2koLQPfH4crDIL3GQ2J13C/RxjTLAEOli2 -y4paeOYzOe6ANDXXBOhieFw5f6mqD9pTF2PWDTnTJhfMGJ6gXPapDYoycS8huNnM -DnqtMVIOf2zOI8aSkumJyXq2bEpGSNK80IJgyKfpk0b4qOdbG7tklfbhVCxTida0 -qqgmJKLGrNmJNW8KLB1U4/hcowEVFz8MF0OZAnByDHi2axwSRSGD5aG2yhCfs3Sy -C269M25v0WuTeOs02RnHUvLPIhvbOEVTu6AjvAtYIfjEq+xqrjbZqUqJeh/xo2tF -SohWe5aQs5DntsMCkZWlqlXc4CCaCqVACegh5Rn+8JbpZvpzy4HUtThfVbB45a5P -6uRAsoxiIHbPLyIaKHFqqfog98hzD4TZYKpfb4RCgqKgEAHGKh4/KPM9O4iw67Oj -EnT2WduegazemzOxQIuZ1UcNu/2d7uy8v/zPGLEUfqhFLS8PMStEuLdQQZfU2DFM -rHJZsH3UtgSreZuQnlTHYy073UhB44mhcYbwa03jMkLHb1bt9q9l8ZAZ5FjstKJD -vN5hMqT5LBQtvArCB+aIVDQxEyZiqFxBlW658uAIhCkLXNB5BVfEUEB8w5vZXOPn -UpNyM8v0dYOkiSOH6+Kk0i7eC8CSrSQAaLnuFZxepGCGMFksywMv7mDg6yX1sAkE -RMakKaAaZ55QpUv1/heaBVK0xmwpjJBefAoMCyFh/I+o7BkFdWGsGF2Tc5ZxZB3F -DF75An4WV2qFMNEIb2SGBBUHGoG75C7tRC9ll39SqX7X2pXMaHvJvEx76CmhC9ub -TzkRDV0Zi/GqiBQXz5kn4AP3GCuJUQJIvK2l0HPqbqNULrhahwQWBOpIek/SLBFV -3fRQc5ABAoICAQDzqaS5/JEKUGMzVikWYnxl6fXgMp/l3Hq2DyifRqgYVoSr/wOJ -PTOOJrtuvwHK6kbOv4YRfKwgVWlJgoAP9Tz0GGpE7toSk3biHY4tTF4KR3xIIOnI -LQLNDR0LDrJSjQhAH5z8U/E+dpOCBHLfXodZ0/ZuXx9YS/pNDQBW/+HOmDtCfqpb -kLmxjZcvZnAZyrlSjGrHJoPjiFrVA/1xvY68TNWHWu8EXphKO/aBi5/e4wWS7I8q -0lDjjPB3GIm0rhumowoGYDPtchcTIvip9aGSPsw56elV+PiwbZaPHZXwHeI0Q25j -UAdkmkwO4VaYGeR4JSUTjoOwWdXXediRq9SKRwhOoat3GJdDwYByYgZb8MKdQCZV -H9hD0rEF4JOrUnOsCteXMNEczQocfSxduLD7XLie+oXoEggNE/hD9EJ397sOvyUg -rgmJLBPCbIPdnfCwm9rdLMsWUjvd4JFq1dyQ9QM1LkdOGQdSoUixtN8X75BdkWhL -/LUqVV/L5/1DStYniO6k4ry67u4POisAZa4czPAJrBEOakVxkCIoaH+mZ/J3GxWc -DvnV6myvcNXUEENhfMt2mikP5zcPthq1O1nm7RFqiow4s/f0jc6Epr/LToXwhm8j -5ch0TkcM+ioguCyYXWEzRJUt3GBWpHoSvxqKLxaWbo1mtxHpGxHDIIFAoQKCAgEA -8cuU0H0OE3JJaG+9Hp6q5mJfdrAzeIPTCT1jxVDptZaplMVo8X0+LIKYpzWqbGWa -Z3cTmReOZBukuKPKXuZEufYqzwIh7lEv1ls2SJ8Gedf3Dq3C2fCITHzXlsg58mrf -tD6cKE85EmI/GhXtaBc51Qa60w5MyXfzK3EPi8lD1v5Il5K1MG6sv+7+P1LgAQVj -c9mPVT1WmgBFVG2/syJ9L94smwhrtM2NhxMkMtEBrOiv0Wy0bEb1wIo5ndBGBoeF -KXBfz87HDbUCSxItDFZ5mDk2ARnzl/13+wr81aKcxW2LyQ0eiiN+HczmM1xpavU6 -KI/QnIvUNqW9XRSNpz/w+DO9+gs9BPIODMww4KpLVtgFfy5KYre1/MpgZHVlHDCI -r72WaDFHdMs75JN1rPNnYx81LtKp0JnE2bIDda3qlCKTBtwaFEUg1F0v9NK9yiZF -JytwUUIfURayT34hfDAJDaSemWT3b4U2kYBr7Fp+g6R5iAosx/TnCYxtuGhdSLA7 -JkL4gy6Ll6hQkIyxRxC6R38wzwUFg4pbV6qrhjP1qyuU1BsZrBSV1RSbcHghSAZj -UmT/sCKOhnliptnnz3e6ceP8I9Y2/TOeJXPdH6zvfaTpnVLiZC48FBKnXFCM4SaS -v5qejQChBBOyd9ybO40sxC/Db8WsPa1LCMnHzm9YtJECggIBAOitGIYZ7o7A0NnW -7wlXJp9TWN2suhAQCIYN16axHBeRlsN/k2rcLH0T82v/rcCNEuye63GLcfq2VQJD -QfL7JieZoPEOfOO8OYwW2QvcnviF5uIhu9zRTplsak2CzToTPsus5IX3yqjxJk7D -3MX8SJe41tvyrcXtV6l4PtjVJFXZ/My0lA04GHq6UKKM2YNnmKyBNf3Sg6MlNEex -tbWFY+XgIQ9OAGxVwGsy82QlZpa6hzoT92rWxzNkBMRVhDorAHCRxCLufufBSNOX -U4lI6+7xVKgA/DaCt0gSffFQOTu/7aAmvCTN6SFiTjrvZ3d7UjaAkfUo5Avk3VUU -H4CLazxYNFCSBWCG9PtYrkWL4bKuJ8lt1qz26ddqPGz2VU0GQTsKZ3ESxr0P9xXZ -WiJtGWQO2KXHaVOA1HtYlHyfpVFWGSQt11BroOOCWaQRZ6KbDZo4WjlWauB3yn9H -0NUKP8OUlGmWNbUYJWv8Y6R+qYL72wnero/RU1aHjCD0+V3m+rVqBykg80Q2oNGd -pC6O6kLKWHWEAA2Z4ZJeJqAbmbnYDSs7i9koW96wKvUldKs6lH1lZA1vjMKI8oVo -SfzCHklsHHN2tgv3n1HCkHEPopqL808JsNvAsziM/0AfK1dvR/Z/RTBlz2apyL0z -9vYr0zYdXR5tl018h/fnR4Dja0nBAoICAG7z5Ui6adAgnHDQ/W5kjrWDJJ05A5Vk -oF1YD0VG+Md76Ds9TwepWfNgxh3McXSmNvENYi/Y8F0dljun5UAg2B6hSEmPh/Rs -ys/JMSKn3nfg0xyoPYZ2aoT2sJMfM//6bMawCEYy/FRkQuJ8d2FRbxdCrE4W/MNi -SHKBwgl6BKhOLQ9oob4kux6j04VJSUMOrvFFPxF1QR0PzMCOT+4qgFmL0NofRhVE -UZJPBUhQQj2O1Mb749rfwPtmMkQpg0iRvVgjmcOo9gxjKDc8kvFbaRaiAhcFSpNP -G1RWWWjK4rdxqjhKzd3DpaaVFHkzCair+NEOFUMQowOkkD17Jfk+TRwH2kA4EfsH -+8yBklg1rLagD/2pRzb/aAIk95CBkjEaUE6cZ71OB6UfcU9VHyWK18FTW7strgDr -3rmqhw3kDYWGTANU0p7EjekFDfGBFtgBMbBkrXiMKAS0roPts/3hqPXauXBYNXa2 -ITEpRdzCNdXAjogPazciOTZXey2ork2hmLLY/mJr5GBmOvnppZufoGsYgVQt0Lbo -sTuMNIMzqqKuL/AYs3IUTCDoAN/eMB911vPDJtUzNkWcw9kemtHH9gU3AceMD9Wm -bwZuoWRcndUA7LZlhz9DPAxV9vhUeiEC60oC8IhxZnGRfE6mK5FrGcS4yiyq/3uZ -3pmcxhQDYigRAoICAQCzK0SUG9tSzwpyuEb7hbS7isT0XOnVvLDEXL2jTwVqbN8a -pC9jebbzJgxk5EA+np/eZlRU7h4aZjRhtDwi4HsarXzgIni6z1vswAiOSZRCfC17 -8/tjfER9w3pJMRTJ5MFeN7JvNUhWnXY5fX6SBl/Eocwwy8S/eIWUY53aI2k8UZMR -eLUb9I6eL21qZAPp/XD8/3pF5DH0ECOS96C8mDKTGnARUyfkKrlaASSTfe5XToNb -IpDEYtLwpHN0missq+ObqPc5lwot/nBy2fdllksb3mCCK8AxuKEDsVtWbb2maoa9 -lRGxIxr++ZV8n0dAwGCV18CbhBYh/B2YnrexI2ZfYOQbzPjHiuUgSLGBJvIUX+S+ -O8ij0vQBqECfwCq63SrdW340Huob52d/cod9pNolO5edftPNJVJP+44sZtEJoDBW -WgzdpVV9fA8TYsO6B3Dci960fbpWnKZ01uLFWVTkou8IwqcaxJIyMzS3Gmqo4UEF -oxBSrg9rTAMJSnm2YWBHK1Z8DDlLzw3SkNZVKm6Ai4yEcLs1PzGZejsJvsu3zGgq -nM4liRHK2QqY1SJf6PsJGxAI62tlyTRfjxtC3Jq9oJbmJs+jl0gtNKEbkfQc83IL -8wCHOm9G7KK7XJjMCb50Dh80YIDFj5GTQLU1tKdpDBfq6YTKUHqjPA8GRnz8vA== +MIISKAIBAAKCBAEAs6eBz6ObPFcqjt4TCKYVamgIzqGg7/wylVqatcyEu6MVoPwp +u3EO80AglzWSkok2al++pSRayA73U6jgdAWQOCGlJXIrVnB0qtYwJeCV3T1KSEsl +p6jE6+XWEE+VQpGwy2gvv5YPCpsrAQ0Drj+3fvYewRRCBH8JIeg+h8Cz191PfpW2 +gzOPzPS7EY5mX7QyIouONJOCDwLRb4Wyp342DPMNkUb5pZHcYEoN6+U343bYE7tV +MyNMs5DfaraAPfa9nb3v0416mmGsAj0QtcJT0n7uHJClsOjbcY/TU+SAsLbMsPLr +RsPZPEgBHTpc0PIXnJqO/Co25xxZQWat4dOC2JXurtyrcpTLC4rf73D3+eJ3dmRn +LQLcGtsC6V5jPz4H6WXkAnhiVeFSGFqjnFbdzdP38lVFYiDO/UGTucIgkCOkQU8w +XF6gl2+kH4dql4dw+dVWt2rM8SjSJjRXZZwtQUOLbL0Vb+zOPp9NL6Yh2PdN0WNP +8J4Ew4IzWDQgS4ENQg1EYkgbTM/gaeTI+JyYH3Q3Oj8jQDqIgMKcm7pquSw6Z6a2 +XIP9kz9uUxlI4z369Y4DQI/mr8XeqvhwdzYrbjrfvNf2x08Cf6GtLzzpCEUvVxrx +W5wnBzpEQz4FsUobQqSQq85L98Gb191vBRH56KOhmWwuJz2xVG/rsuj45hJvimkF +zMb2xC2wBrYhUChQ3IyyjGXO2StRuGJWcLRGTeRMoYYxPeMHf3PjCW7upykqgCOK +tgwinkMcqYxHLAJtE++Ip9fRYNlrKICXirf/YmsoVA2rYzqsV54vlSv0oD2hYKBX +Ggkj7vxNm0WD8gofs3cOmCCzFvyXAzD8qz5+/D52ZMtLTclsj6pw7APFgTFDEZr8 +LbollAtzfsE5AcpocBrvpLrcE9Y3FwkNyqohlLs+Q/2MgOZY/XUjWsofCy41ws13 +DHOzL5rkA2hYyD/zdo0FU9I02wmvcO1nC/8vZDUtcSJ68vIrfW2+gL0YODcBZ6Rf +SClSeJ+1uIBwyhBq4Sc7wGFOlvjoD0Kx+rwF7zGROKsPp+8shsGrhTZ8EJ3HCmvm +KxHd+kXn0sM6wc1gJ2gm14dJX+N/JiAfmaj5AKYN1ROBbvwnigNYaw46xyChXCTR +CKZaHueBPSEFLktdGBTxT6tc5YHawSz2Ev8lCZcenVFtH2MunO19hkWwNXP2n4Jl +vH9GiOGhYKE3UCac13bz3JSJbde+phehK3EUNxdf7Vo5mfxpr7tjVcZge2TYu7dK +n7mqiUswH5zvI3qvfjHd+90K1QQKjFdvZIDr8+5sM9JywiRook6ZiBz3O10QzE7q +pcsAQIqpYy4YkuuxKAX+Gep7MvtjViwLIAGSowIDAQABAoIEAG18dDwatQx9As62 +wFrQ/NQwOs3S6sXWqO0knoyU639+0Duf8b7uE/Ji7nm4iG8NeEAzwXfbRAgQNuKh +VMjgaxgKSR8dCRRQzIkgp48t46dFJvQP+A7JZ9lr5J1kIs7DAz+zawYTaW9JSVgG +mmujIS0ayXtZ4THkSaiEZTdOVUwE7+FxVB15WvXJYAg/BFzm8HblHfEkxGppiUGx +9ULmRAJ4TZhv7Mzeq7Ny7jEJLJpPXzgHPtE/fgshC0d5mYYpzT1I99OxsTpMtAtZ +M8nReEwcVvjJnnDGOJZo1J2X8GXBwdVE9eXWjZmTODqbytz77S/fcaxz/amzOSIG +dYne4SsllKS0jzfxPhz2saq26ik7vxpD7i0cRQ+I0alvgISWRTQUiHSI8UTtYJ6+ +/l7VAa/isoTZOsOmwv7M7D6U9EtjrUKwBAUHFk+0Fg2w9WELhep7/bEiQWvnOMtX +sruWBm5zv5CxW1MuujKn2t5jHNWhQaeOIrW9V6Hew3PJsLTTHUVX2Ikc8KZLObTP +XTtGE993cADrYYxPH8l40QCHB2t6bM2maBTRy8PlMFMsYaM7rqnftlOuXyRYLm97 +oO7F8InpD1mDND4bBKbL3UOgl1QW9dbb8KOn8pERDXD9Z3K4ewyORiyuONXxBAz2 +QiaQ4gDJWD6sEySPEHcpDjemi6mKJFtDw935JxS2brso/NHoJSVdcNxisd2irA8P +U8jyFTrTm7cO0nKFx9vv9r5dsbkqyQnJ9V0q4/DoWvnkPElKuvTeJMw6CvTjfvc1 +xIf8m3MQJTN8pqPpIOSj8DIhQIJBEVxtcajg+vlMuefR8hTCdLzQV2QEdLGjNGph +OhFuHrHw1b3I4nCyi9MOLzpdUDgZkaw4hPg38u71RJ6bUF6pE89Yj04/svB7SZgA +sE8+/pZcASZl1Y0T29OM3qFJvukBWwsNij5P5SLmTXDev+DXbzUvmVLCk8nJkJ5m +n1peGst0ApHnEzxVhU+9d5FWBdIgN7fp7V2AlFt1r0TRWUYndBn9/B6xiyfd5iyK +LpcbRjLpQ/9b3SYQZL9t65Hc40rP35rY0luQTzVd6E/D041E0NWdpAZai4GrpMKp +NUBXBbmwnCZEzExLi78We6aF5tG9Vh+uxyYUXlrABR98jONgYIvnWaATaw6oLgam +B84RhWUa1f+K70wrTorUO8QNvYO2n4jpwJikn4H9pFvvzzoOB7RaA+EF4HjaVmQr +Jm6S18c7sksYFGIJYsPrXMtgOMQVYiQ3s8LDgsWKoEQYgvgxE24AoXGlXXbALrpo +4WvRN7qLAm3n2OQYFUK0SGoELOADQFQhbNL91i0k4wzQ56ofQCb8aTmm3q76TnbK +4gmIUIECggIBAOm3PhMqIKb3ZEZOWYMKByb7v8seuVh6mwXmgITcZIiA2f8rTfyH +OZanYK72FsuGZEbf1ELwNUX0JobwklA1Uq9NajaJUdfe4wDpghf282APHK1nD3m3 +FX7Rsoof/dW9+bV/IIbQDwy+1ctB20m6dejt3kWyHSXly2y2L3bk1flee8oi+VCO +ROp4u23KJcglIYlL+cL1z76lX41v8JW3deCTL/JStmK6zuvFhY5fTlOGxsLhAY3X +VTZ0MNUpezlBaXTixi7ilgbh7rJSpCmeno5my1hdl/5Q5U/u5F5E7ZxKxlhuVWdA +cf9qbtolSsgJ2w8axPD3dLnQ6wsNnl4yL77TdNr8OlbwIADXHH9VzZbfEgOhP4r9 +Cwz4hIUbylJ9QGZVsEVZhvqR3M6OgqlYOmC72nLPXVYbL/Rr7cTx8t+WJ4wJRP3l +pi6AgDZXKbMKyMIhM1IscSR2swtAFyl6g6rcAjFyG+p1IKMOQ3YOqdVVUf5UYSpo +NBTpJ+ho3CyQJK25yT2Maex5Q91M6Rk9vmXaIs+K40t/zu8cnCIH9j3In0BrpSi7 +WUyLgzXW+7ZrW7cugC93VVgjGNbgBiDimZ5duViI+xY+3OlrsUxzSHN+YY+8H2G2 +TfzT+Mgo8lJXH9tseFBLvRKhoPbbCyCwnOJ6k2k8/iqZ2eE7vQodyW/zAoICAQDE +yK5Mlu0k3oTVasgYRNDYKJ9uyarVLaRcN6UjzNUSta2hABzQe/LG895EucK9yVa0 +4jT7EXILYQpHnQkaYtyHtN/deYWLRObT6RMws7te2MhwzQkaNavgi449dHP38TPS +qy3Vh6v4nMhRjkhLqNiiJoDWThtkB216t+71GxdF2l9trrcvlC6q0Z65PR4LK3B/ +3m8KmlyGN940Ovb/4kEh2EZmNwbMq+3SaXiS4R7ZvKoboyNmwvCh4t5ne+NXz3Vm +7srej0cnOM54dTuNVdXm6pajYjELNVJGsK0vo6CAZiAOyasFkwMFMr5w/ehBaZIo +9ewWjvgDdvQ0+JBMVZJot7e5D4tUgVTaIMpQryzAp/u3r5KC1bkcC1Zie+44CaPr +V2FspEyYQDPJHC2tRj33lbaCWDdfdhYhv6uzlcM9ExPj+RQA0xlCuxh1pxK43H4t +PdWvcIeIKtFRWq9gsH1/wDBaIsB5Kykq4xyvMOj3dyV+dyIO1D3GG2+ZoBt8gklk +ieDIzCMKsXbomgm64/Ewlx9tf1fbTouSWP1+Gh0Wne3sCEtCY72ebYgjtaA0w8Xr +HGTsEdw/rSer46uULYWXYC6VYZwlyTbPh4NSD7ATrQo6HnaKugtuya9NPfYD9S9w +PoXPXpNKvwSq9SA/XsodHet9uU+IlKnYI41hVGaukQKCAgEAu7dAFmh5vWFqmsWH +Q6akQ/1j+KL+v0AKj1yp0Qyreq/zZ+3CP5fk+0oIb76cZG/Wzs4sUG9aeos0/qDv +A5kwhjipnJACul38+diTcugYZgj9QZFLbrlMfIW03xf+6tQhYlNvLpih3dzRHuYQ +WVF4LtQO+O+sVaoSD4js6pO2AQqQZrRchwUd+S1rbS0112FrZDkvrBV+/GMbMiC8 +naOy++N8WSdx8i6Uz+3f7ZBC+vd/YTsT4ncXrBr6tdsRa9VL/GPARhwb3/9LltKl +pAAQ3dNWc+0IjW0wIVmA9u5p4mR0lsZXtgyfA0TbMD9PpwezeJhJojk++ZgBkOkp +UAMDMkPo0ZJQ0U8ghUxBSU0DSbB1aZDz1pCTdaGN3tOJV2Pee9NLNwhHT64kKYtx +Cs8gZF29gQVotCY2CB0/5jVmm4qgzOsNDNiM90CiugKcLX/162Z7L/8eCOmFuP2L +HPeYFX2MDWbXYSMeSZjFOmdgpUZYCux+9m6nljwGn7bJUMmdjCNyrQrAcPydM3v4 +pRtgd7ISz5uAID3RWWAjT3oRn2Ip9rX++44ulTpg8rdviP8FtUWWE4nHdHkMFFnu +0i7ur7Ibki7DbQSkrp6e0watJIZXmfjBQ9wHG5CteybYKqZrofOnVubeP76t6Ffa +wjDYcVd+WsJwaosT4sH2fc+a4V0CggIAX703N+ISHGb6f47FLTGF3jTdZJxPNgpP +teIIwUMtwz2K7RLN8gzr0Cnj0NID0iw7uCN6HVCeiLS8uJzJSgIhQxEX/XvbIify +mdUC1iYLmhikZGUM8ah9J/Ed0f5vUYkTaoKH+SRZ4cn9l9g+ijjqtPFAq4Myxlbq +x9LnOm7kwsJ8vTNMwqQ4lDkKHRuEG3EV+dAxfBofu9lAjW7GrzXX9Gxxy9cnwro/ +KajQ1b0bMloc9PdJCPz87/YduIFI2XcAqoSxRNi2iHmV0ntO7vN87yzg/zqNm+Nk +VSouCEh2lSGapTcusraUuJm4l+agcfq39p+3Lj5+ocjlBUSNh+X2JmaDx1Ctu/50 +Vu+vHtQrVMwlpneQM4Fh3ygQ7jdXbgIasyw/JQlGjqhlyYmhG3VaPrOjKQVcY2Bj +YiYLloLdi0BV5AtR1RjfaFz4iH0xmakE5mP4K7P27KPQVOFnBAa5+EZ7/856kDkU +fV4jnYSRQ5y/LCkv88EFEPZWNIr19Hy60mRWbbsLrW8r5/zRkRDHTpmn27EiAHvV +Ye1T3187Gb7ae4SJgRJftc2ad60s9Z7uEvYsj8Icwwoui+1nk98V6NENuPXVb3jW +5t8KYAlr5BbDLAWYLORZHZyYYl8r6z8FxnlUxtA/nxUSY06BUvN+J+5fnTpkXJHf +lF2vslPK5VECggIANpTlrc5Zp9c0jPOiBQT/92LdtMR9mB76WB8fX9xn1Tm7V/cI +MatRrFzaTvi5HqHxPWJJ5cgMYKQw8kHHk1q9U7jeexZGCICFOUjk3N2Z0aXIWrx+ +hiS1i5DzRaZNI682s436cdaS4GpZIhqjIOR4/JS15Nfu+DVqyHOevWnd0D/tGoJB +JpiePYl2k7b8sirM3bHnRDtBiMHrHyHxzpdTLImDLHXr7C5DTeQAzzo+YMu45mNm +utKq3SRlO2wizj0qpONgJ6cHI7pypAPalliA+hPRRJqaHmTr7LpkP3+g73hf26wK +PoF6CLkdaN945F4Z48/3zEOP3XRIQocMA46ayA+8TsOpx9rm8L1LrF1pgmJi5Ml9 +3yV93ZZ/w4Jy0pqZ28nn3gcX6JP7qtfgioLLeyuuBphpOQXl7Ys3w1d0EEErpATU +tRG+BS16o+FhM1iXAN8c8HrqOtyJzCpYnuoWiNc+G8BIn6ox3xmihwK4hiR6s/OM +Mxaa6T4FF/IWIzuCezYPGgybWMXj3nRHsD7eNP170irBLwS3GdD1HWM2BpmLdJa0 +m/7QQkONhCnNFtjIalfPWMX51z/8GOKRKWH5A8PY6CxVHRg1ZfHT4TlfbdRPIG8d +6JrWt2s4eLv9VRdM+oMifHR4qvLDgg7R8pBr2FSsVXM9U2+j+drIeAhkjHI= -----END RSA PRIVATE KEY----- From b8c840500816c514b6722145a7f307c499793b69 Mon Sep 17 00:00:00 2001 From: Sergei Golubchik Date: Wed, 26 Apr 2017 17:55:27 +0200 Subject: [PATCH 25/28] cleanup: simplify setting of ssl* options in my.cnf templates magic-- --- mysql-test/include/default_client.cnf | 6 +++ mysql-test/include/default_mysqld.cnf | 4 ++ mysql-test/lib/My/ConfigFactory.pm | 56 --------------------------- 3 files changed, 10 insertions(+), 56 deletions(-) diff --git a/mysql-test/include/default_client.cnf b/mysql-test/include/default_client.cnf index bcd6a4849d2..e326b308f5f 100644 --- a/mysql-test/include/default_client.cnf +++ b/mysql-test/include/default_client.cnf @@ -20,3 +20,9 @@ default-character-set=latin1 [mysql_upgrade] default-character-set=latin1 + +[mysqltest] +loose-ssl-ca=@ENV.MYSQL_TEST_DIR/std_data/cacert.pem +loose-ssl-cert=@ENV.MYSQL_TEST_DIR/std_data/client-cert.pem +loose-ssl-key=@ENV.MYSQL_TEST_DIR/std_data/client-key.pem +loose-skip-ssl diff --git a/mysql-test/include/default_mysqld.cnf b/mysql-test/include/default_mysqld.cnf index b5b16461781..44a7fd12d27 100644 --- a/mysql-test/include/default_mysqld.cnf +++ b/mysql-test/include/default_mysqld.cnf @@ -108,6 +108,10 @@ binlog-direct-non-transactional-updates default-storage-engine=myisam +loose-ssl-ca=@ENV.MYSQL_TEST_DIR/std_data/cacert.pem +loose-ssl-cert=@ENV.MYSQL_TEST_DIR/std_data/server-cert.pem +loose-ssl-key=@ENV.MYSQL_TEST_DIR/std_data/server-key.pem + # here, at the end of [mysqld] group mtr will automatically disable # all optional plugins. diff --git a/mysql-test/lib/My/ConfigFactory.pm b/mysql-test/lib/My/ConfigFactory.pm index 8b997e8b096..7584a99c093 100644 --- a/mysql-test/lib/My/ConfigFactory.pm +++ b/mysql-test/lib/My/ConfigFactory.pm @@ -182,55 +182,6 @@ sub fix_log_slow_queries { return "$dir/mysqld-slow.log"; } -sub fix_std_data { - my ($self, $config, $group_name, $group)= @_; - my $testdir= $self->get_testdir($group); - return "$testdir/std_data"; -} - -sub ssl_supported { - my ($self)= @_; - return $self->{ARGS}->{ssl}; -} - -sub fix_skip_ssl { - return if !ssl_supported(@_); - # Add skip-ssl if ssl is supported to avoid - # that mysqltest connects with SSL by default - return 1; -} - -sub fix_ssl_ca { - return if !ssl_supported(@_); - my $std_data= fix_std_data(@_); - return "$std_data/cacert.pem" -} - -sub fix_ssl_server_cert { - return if !ssl_supported(@_); - my $std_data= fix_std_data(@_); - return "$std_data/server-cert.pem" -} - -sub fix_ssl_client_cert { - return if !ssl_supported(@_); - my $std_data= fix_std_data(@_); - return "$std_data/client-cert.pem" -} - -sub fix_ssl_server_key { - return if !ssl_supported(@_); - my $std_data= fix_std_data(@_); - return "$std_data/server-key.pem" -} - -sub fix_ssl_client_key { - return if !ssl_supported(@_); - my $std_data= fix_std_data(@_); - return "$std_data/client-key.pem" -} - - # # Rules to run for each mysqld in the config # - will be run in order listed here @@ -255,9 +206,6 @@ my @mysqld_rules= { '#user' => sub { return shift->{ARGS}->{user} || ""; } }, { '#password' => sub { return shift->{ARGS}->{password} || ""; } }, { 'server-id' => \&fix_server_id, }, - { 'ssl-ca' => \&fix_ssl_ca }, - { 'ssl-cert' => \&fix_ssl_server_cert }, - { 'ssl-key' => \&fix_ssl_server_key }, { 'bind-address' => \&fix_bind_address }, ); @@ -284,10 +232,6 @@ my @client_rules= # my @mysqltest_rules= ( - { 'ssl-ca' => \&fix_ssl_ca }, - { 'ssl-cert' => \&fix_ssl_client_cert }, - { 'ssl-key' => \&fix_ssl_client_key }, - { 'skip-ssl' => \&fix_skip_ssl }, ); From 1b27c254731747756d254f96cd8666dae3f0809b Mon Sep 17 00:00:00 2001 From: Sergei Golubchik Date: Tue, 25 Apr 2017 23:00:58 +0200 Subject: [PATCH 26/28] MDEV-10594 SSL hostname verification fails for SubjectAltNames use X509_check_host for OpenSSL 1.0.2+ This adds: * support for subjectAltNames * wildcards * sub-domain matching --- mysql-test/lib/generate-ssl-certs.sh | 7 +++ mysql-test/std_data/serversan-cert.pem | 60 ++++++++++++++++++++++++++ mysql-test/std_data/serversan-key.pem | 16 +++++++ mysql-test/suite.pm | 4 ++ mysql-test/t/ssl_7937.combinations | 5 +++ sql-common/client.c | 21 ++++----- 6 files changed, 103 insertions(+), 10 deletions(-) create mode 100644 mysql-test/std_data/serversan-cert.pem create mode 100644 mysql-test/std_data/serversan-key.pem diff --git a/mysql-test/lib/generate-ssl-certs.sh b/mysql-test/lib/generate-ssl-certs.sh index e5e995489a0..8f15ba9d521 100755 --- a/mysql-test/lib/generate-ssl-certs.sh +++ b/mysql-test/lib/generate-ssl-certs.sh @@ -29,4 +29,11 @@ openssl req -newkey rsa:1024 -keyout client-key.pem -out demoCA/client-req.pem - openssl rsa -in client-key.pem -out client-key.pem openssl ca -keyfile cakey.pem -days 7300 -batch -cert cacert.pem -policy policy_anything -out client-cert.pem -infiles demoCA/client-req.pem +# with SubjectAltName, only for OpenSSL 1.0.2+ +cat > demoCA/sanext.conf <= 0x10002000L && !defined(HAVE_YASSL) +#include +#define HAVE_X509_check_host +#endif + static int ssl_verify_server_cert(Vio *vio, const char* server_hostname, const char **errptr) { SSL *ssl; X509 *server_cert= NULL; +#ifndef HAVE_X509_check_host char *cn= NULL; int cn_loc= -1; ASN1_STRING *cn_asn1= NULL; X509_NAME_ENTRY *cn_entry= NULL; X509_NAME *subject= NULL; +#endif int ret_validation= 1; DBUG_ENTER("ssl_verify_server_cert"); @@ -1811,14 +1818,9 @@ static int ssl_verify_server_cert(Vio *vio, const char* server_hostname, const c are what we expect. */ - /* - Some notes for future development - We should check host name in alternative name first and then if needed check in common name. - Currently yssl doesn't support alternative name. - openssl 1.0.2 support X509_check_host method for host name validation, we may need to start using - X509_check_host in the future. - */ - +#ifdef HAVE_X509_check_host + ret_validation= X509_check_host(server_cert, server_hostname, 0, 0, 0) != 1; +#else subject= X509_get_subject_name(server_cert); cn_loc= X509_NAME_get_index_by_NID(subject, NID_commonName, -1); if (cn_loc < 0) @@ -1826,7 +1828,6 @@ static int ssl_verify_server_cert(Vio *vio, const char* server_hostname, const c *errptr= "Failed to get CN location in the certificate subject"; goto error; } - cn_entry= X509_NAME_get_entry(subject, cn_loc); if (cn_entry == NULL) { @@ -1855,7 +1856,7 @@ static int ssl_verify_server_cert(Vio *vio, const char* server_hostname, const c /* Success */ ret_validation= 0; } - +#endif *errptr= "SSL certificate validation failure"; error: From 6935d66053b2c9723314d89d5169dcf34ed56c08 Mon Sep 17 00:00:00 2001 From: Sergei Golubchik Date: Wed, 26 Apr 2017 21:58:21 +0200 Subject: [PATCH 27/28] MDEV-12591 MariaDB embedded server refuses start with innodb_plugin innodb calls sd_notify(), it must be linked with libsystemd --- storage/innobase/CMakeLists.txt | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/storage/innobase/CMakeLists.txt b/storage/innobase/CMakeLists.txt index 7ae962880d1..5d1d9d8c551 100644 --- a/storage/innobase/CMakeLists.txt +++ b/storage/innobase/CMakeLists.txt @@ -512,5 +512,5 @@ ENDIF() MYSQL_ADD_PLUGIN(innobase ${INNOBASE_SOURCES} STORAGE_ENGINE MODULE_ONLY MODULE_OUTPUT_NAME ha_innodb - LINK_LIBRARIES ${ZLIB_LIBRARY} ${LINKER_SCRIPT}) + LINK_LIBRARIES ${ZLIB_LIBRARY} ${LIBSYSTEMD} ${LINKER_SCRIPT}) From b82c602db588cfa688278ef772050c004590c124 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Marko=20M=C3=A4kel=C3=A4?= Date: Fri, 28 Apr 2017 03:20:49 +0300 Subject: [PATCH 28/28] MDEV-12602 InnoDB: Failing assertion: space->n_pending_ops == 0 This fixes a regression caused by MDEV-12428. When we introduced a variant of fil_space_acquire() that could increment space->n_pending_ops after space->stop_new_ops was set, the logic of fil_check_pending_operations() was broken. fil_space_t::n_pending_ios: A new field to track read or write access from the buffer pool routines immediately before a block write or after a block read in the file system. fil_space_acquire_for_io(), fil_space_release_for_io(): Similar to fil_space_acquire_silent() and fil_space_release(), but modify fil_space_t::n_pending_ios instead of fil_space_t::n_pending_ops. Adjust a number of places accordingly, and remove some redundant tablespace lookups. The following parts of this fix differ from the 10.2 version of this fix: buf_page_get_corrupt(): Add a tablespace parameter. In 10.2, we already had a two-phase process of freeing fil_space objects (first, fil_space_detach(), then release fil_system->mutex, and finally free the fil_space and fil_node objects). fil_space_free_and_mutex_exit(): Renamed from fil_space_free(). Detach the tablespace from the fil_system cache, release the fil_system->mutex, and then wait for space->n_pending_ios to reach 0, to avoid accessing freed data in a concurrent thread. During the wait, future calls to fil_space_acquire_for_io() will not find this tablespace, and the count can only be decremented to 0, at which point it is safe to free the objects. fil_node_free_part1(), fil_node_free_part2(): Refactored from fil_node_free(). --- storage/innobase/buf/buf0buf.cc | 135 +++++++++++----------- storage/innobase/buf/buf0flu.cc | 11 +- storage/innobase/fil/fil0crypt.cc | 4 +- storage/innobase/fil/fil0fil.cc | 173 +++++++++++++++++++---------- storage/innobase/include/buf0buf.h | 44 ++++---- storage/innobase/include/fil0fil.h | 63 +++++++---- storage/xtradb/buf/buf0buf.cc | 133 +++++++++++----------- storage/xtradb/buf/buf0flu.cc | 11 +- storage/xtradb/fil/fil0crypt.cc | 4 +- storage/xtradb/fil/fil0fil.cc | 173 +++++++++++++++++++---------- storage/xtradb/include/buf0buf.h | 40 ++++--- storage/xtradb/include/fil0fil.h | 63 +++++++---- 12 files changed, 503 insertions(+), 351 deletions(-) diff --git a/storage/innobase/buf/buf0buf.cc b/storage/innobase/buf/buf0buf.cc index 8f8069a0884..30893798d6f 100644 --- a/storage/innobase/buf/buf0buf.cc +++ b/storage/innobase/buf/buf0buf.cc @@ -317,6 +317,15 @@ on the io_type */ ? (counter##_READ) \ : (counter##_WRITTEN)) +/** Decrypt a page. +@param[in,out] bpage Page control block +@param[in,out] space tablespace +@return whether the operation was successful */ +static +bool +buf_page_decrypt_after_read(buf_page_t* bpage, fil_space_t* space) + MY_ATTRIBUTE((nonnull)); + /* prototypes for new functions added to ha_innodb.cc */ trx_t* innobase_get_trx(); @@ -483,16 +492,13 @@ buf_block_alloc( } #endif /* !UNIV_HOTBACKUP */ -/********************************************************************//** -Checks if a page is all zeroes. -@return TRUE if the page is all zeroes */ +/** Check if a page is all zeroes. +@param[in] read_buf database page +@param[in] zip_size ROW_FORMAT=COMPRESSED page size, or 0 +@return whether the page is all zeroes */ UNIV_INTERN bool -buf_page_is_zeroes( -/*===============*/ - const byte* read_buf, /*!< in: a database page */ - const ulint zip_size) /*!< in: size of compressed page; - 0 for uncompressed pages */ +buf_page_is_zeroes(const byte* read_buf, ulint zip_size) { const ulint page_size = zip_size ? zip_size : UNIV_PAGE_SIZE; @@ -607,8 +613,7 @@ buf_page_is_checksum_valid_none( && checksum_field1 == BUF_NO_CHECKSUM_MAGIC); } -/********************************************************************//** -Checks if a page is corrupt. +/** Check if a page is corrupt. @param[in] check_lsn true if LSN should be checked @param[in] read_buf Page to be checked @param[in] zip_size compressed size or 0 @@ -4439,34 +4444,30 @@ buf_mark_space_corrupt( buf_pool_mutex_exit(buf_pool); } -/********************************************************************//** -Check if page is maybe compressed, encrypted or both when we encounter +/** Check if page is maybe compressed, encrypted or both when we encounter corrupted page. Note that we can't be 100% sure if page is corrupted or decrypt/decompress just failed. -@param[in,out] bpage Page -@return DB_SUCCESS if page has been read and is not corrupted, -@retval DB_PAGE_CORRUPTED if page based on checksum check is corrupted, -@retval DB_DECRYPTION_FAILED if page post encryption checksum matches but +@param[in,out] bpage page +@param[in,out] space tablespace from fil_space_acquire_for_io() +@return whether the operation succeeded +@retval DB_SUCCESS if page has been read and is not corrupted +@retval DB_PAGE_CORRUPTED if page based on checksum check is corrupted +@retval DB_DECRYPTION_FAILED if page post encryption checksum matches but after decryption normal page checksum does not match. -@retval DB_TABLESPACE_DELETED if accessed tablespace is not found */ +@retval DB_TABLESPACE_DELETED if accessed tablespace is not found */ static dberr_t -buf_page_check_corrupt(buf_page_t* bpage) +buf_page_check_corrupt(buf_page_t* bpage, fil_space_t* space) { + ut_ad(space->n_pending_ios > 0); + ulint zip_size = buf_page_get_zip_size(bpage); byte* dst_frame = (zip_size) ? bpage->zip.data : ((buf_block_t*) bpage)->frame; - FilSpace space(bpage->space, true); bool still_encrypted = false; dberr_t err = DB_SUCCESS; bool corrupted = false; - fil_space_crypt_t* crypt_data = NULL; - - if (!space()) { - return(DB_TABLESPACE_DELETED); - } - - crypt_data = space()->crypt_data; + fil_space_crypt_t* crypt_data = space->crypt_data; /* In buf_decrypt_after_read we have either decrypted the page if page post encryption checksum matches and used key_id is found @@ -4478,11 +4479,12 @@ buf_page_check_corrupt(buf_page_t* bpage) crypt_data->type != CRYPT_SCHEME_UNENCRYPTED && !bpage->encrypted && fil_space_verify_crypt_checksum(dst_frame, zip_size, - space(), bpage->offset)); + space, bpage->offset)); if (!still_encrypted) { /* If traditional checksums match, we assume that page is not anymore encrypted. */ - corrupted = buf_page_is_corrupted(true, dst_frame, zip_size, space()); + corrupted = buf_page_is_corrupted(true, dst_frame, zip_size, + space); if (!corrupted) { bpage->encrypted = false; @@ -4505,7 +4507,7 @@ buf_page_check_corrupt(buf_page_t* bpage) ", page number=%u]" " in file %s cannot be decrypted.", bpage->space, bpage->offset, - space()->name); + space->name); ib_logf(IB_LOG_LEVEL_INFO, "However key management plugin or used key_version " ULINTPF @@ -4523,28 +4525,24 @@ buf_page_check_corrupt(buf_page_t* bpage) return (err); } -/********************************************************************//** -Completes an asynchronous read or write request of a file page to or from -the buffer pool. +/** Complete a read or write request of a file page to or from the buffer pool. @param[in,out] bpage Page to complete @param[in] evict whether or not to evict the page from LRU list. -@return DB_SUCCESS if page has been read and is not corrupted, -DB_PAGE_CORRUPTED if page based on checksum check is corrupted, -DB_DECRYPTION_FAILED if page post encryption checksum matches but -after decryption normal page checksum does not match. -in write only DB_SUCCESS is possible. */ +@return whether the operation succeeded +@retval DB_SUCCESS always when writing, or if a read page was OK +@retval DB_PAGE_CORRUPTED if the checksum fails on a page read +@retval DB_DECRYPTION_FAILED if page post encryption checksum matches but + after decryption normal page checksum does + not match */ UNIV_INTERN dberr_t -buf_page_io_complete( - buf_page_t* bpage, - bool evict) +buf_page_io_complete(buf_page_t* bpage, bool evict) { enum buf_io_fix io_type; buf_pool_t* buf_pool = buf_pool_from_bpage(bpage); const ibool uncompressed = (buf_page_get_state(bpage) == BUF_BLOCK_FILE_PAGE); - fil_space_t* space = NULL; byte* frame = NULL; dberr_t err = DB_SUCCESS; @@ -4564,7 +4562,13 @@ buf_page_io_complete( ulint read_space_id = 0; uint key_version = 0; - buf_page_decrypt_after_read(bpage); + ut_ad(bpage->zip.data || ((buf_block_t*)bpage)->frame); + fil_space_t* space = fil_space_acquire_for_io(bpage->space); + if (!space) { + return(DB_TABLESPACE_DELETED); + } + + buf_page_decrypt_after_read(bpage, space); if (buf_page_get_zip_size(bpage)) { frame = bpage->zip.data; @@ -4635,7 +4639,7 @@ buf_page_io_complete( bpage->offset); } - err = buf_page_check_corrupt(bpage); + err = buf_page_check_corrupt(bpage, space); database_corrupted: @@ -4647,6 +4651,7 @@ database_corrupted: buf_mark_space_corrupt(bpage); ib_logf(IB_LOG_LEVEL_INFO, "Simulated page corruption"); + fil_space_release_for_io(space); return(err); } err = DB_SUCCESS; @@ -4654,9 +4659,6 @@ database_corrupted: ); if (err == DB_PAGE_CORRUPTED) { - fil_system_enter(); - space = fil_space_get_by_id(bpage->space); - ib_logf(IB_LOG_LEVEL_ERROR, "Database page corruption on disk" " or a failed file read of tablespace %s" @@ -4667,8 +4669,6 @@ database_corrupted: space->name, bpage->space, bpage->offset); - fil_system_exit(); - buf_page_print(frame, buf_page_get_zip_size(bpage), BUF_PAGE_PRINT_NO_CRASH); @@ -4693,6 +4693,7 @@ database_corrupted: table as corrupted instead of crashing server */ if (bpage->space > TRX_SYS_SPACE) { buf_mark_space_corrupt(bpage); + fil_space_release_for_io(space); return(err); } else { ib_logf(IB_LOG_LEVEL_FATAL, @@ -4730,6 +4731,8 @@ database_corrupted: } } + + fil_space_release_for_io(space); } else { /* io_type == BUF_IO_WRITE */ if (bpage->slot) { @@ -6156,16 +6159,17 @@ buf_page_encrypt_before_write( return dst_frame; } -/********************************************************************//** -Decrypt page after it has been read from disk -@param[in,out] bpage Page control block -@return true if successfull, false if something went wrong -*/ -UNIV_INTERN +/** Decrypt a page. +@param[in,out] bpage Page control block +@param[in,out] space tablespace +@return whether the operation was successful */ +static bool -buf_page_decrypt_after_read( - buf_page_t* bpage) +buf_page_decrypt_after_read(buf_page_t* bpage, fil_space_t* space) { + ut_ad(space->n_pending_ios > 0); + ut_ad(space->id == bpage->space); + ulint zip_size = buf_page_get_zip_size(bpage); ulint size = (zip_size) ? zip_size : UNIV_PAGE_SIZE; @@ -6183,12 +6187,10 @@ buf_page_decrypt_after_read( return (true); } - FilSpace space(bpage->space, false, true); - /* Page is encrypted if encryption information is found from tablespace and page contains used key_version. This is true also for pages first compressed and then encrypted. */ - if (!space() || !space()->crypt_data) { + if (!space->crypt_data) { key_version = 0; } @@ -6225,9 +6227,9 @@ buf_page_decrypt_after_read( /* Mark page encrypted in case it should be. */ - if (key_version && space()->crypt_data && - space()->crypt_data->type != CRYPT_SCHEME_UNENCRYPTED) { - bpage->encrypted=true; + if (space->crypt_data->type + != CRYPT_SCHEME_UNENCRYPTED) { + bpage->encrypted = true; } return (false); @@ -6241,12 +6243,8 @@ buf_page_decrypt_after_read( #endif /* decrypt using crypt_buf to dst_frame */ - byte* res = fil_space_decrypt(space(), - slot->crypt_buf, - dst_frame, - &bpage->encrypted); - - if (!res) { + if (!fil_space_decrypt(space, slot->crypt_buf, + dst_frame, &bpage->encrypted)) { success = false; } @@ -6277,5 +6275,6 @@ buf_page_decrypt_after_read( } } + ut_ad(space->n_pending_ios > 0); return (success); } diff --git a/storage/innobase/buf/buf0flu.cc b/storage/innobase/buf/buf0flu.cc index c20dd03c8a9..b41ea4c8f02 100644 --- a/storage/innobase/buf/buf0flu.cc +++ b/storage/innobase/buf/buf0flu.cc @@ -831,7 +831,7 @@ buf_flush_write_block_low( buf_flush_t flush_type, /*!< in: type of flush */ bool sync) /*!< in: true if sync IO request */ { - fil_space_t* space = fil_space_acquire(bpage->space, true); + fil_space_t* space = fil_space_acquire_for_io(bpage->space); if (!space) { return; } @@ -956,6 +956,13 @@ buf_flush_write_block_low( ut_ad(flush_type == BUF_FLUSH_SINGLE_PAGE); fil_flush(space); + /* The tablespace could already have been dropped, + because fil_io(request, sync) would already have + decremented the node->n_pending. However, + buf_page_io_complete() only needs to look up the + tablespace during read requests, not during writes. */ + ut_ad(buf_page_get_io_fix(bpage) == BUF_IO_WRITE); + /* true means we want to evict this page from the LRU list as well. */ #ifdef UNIV_DEBUG @@ -966,7 +973,7 @@ buf_flush_write_block_low( ut_ad(err == DB_SUCCESS); } - fil_space_release(space); + fil_space_release_for_io(space); /* Increment the counter of I/O operations used for selecting LRU policy. */ diff --git a/storage/innobase/fil/fil0crypt.cc b/storage/innobase/fil/fil0crypt.cc index b51c6ac9cdc..c25d3e78f0e 100644 --- a/storage/innobase/fil/fil0crypt.cc +++ b/storage/innobase/fil/fil0crypt.cc @@ -683,7 +683,7 @@ fil_space_encrypt( } fil_space_crypt_t* crypt_data = space->crypt_data; - ut_ad(space->n_pending_ops); + ut_ad(space->n_pending_ios > 0); ulint zip_size = fsp_flags_get_zip_size(space->flags); byte* tmp = fil_encrypt_buf(crypt_data, space->id, offset, lsn, src_frame, zip_size, dst_frame); @@ -860,7 +860,7 @@ fil_space_decrypt( *decrypted = false; ut_ad(space->crypt_data != NULL && space->crypt_data->is_encrypted()); - ut_ad(space->n_pending_ops > 0); + ut_ad(space->n_pending_ios > 0); bool encrypted = fil_space_decrypt( space->crypt_data, diff --git a/storage/innobase/fil/fil0fil.cc b/storage/innobase/fil/fil0fil.cc index 66eb425c58b..7bc784121e5 100644 --- a/storage/innobase/fil/fil0fil.cc +++ b/storage/innobase/fil/fil0fil.cc @@ -1,7 +1,7 @@ /***************************************************************************** Copyright (c) 1995, 2016, Oracle and/or its affiliates. All Rights Reserved. -Copyright (c) 2014, 2017, MariaDB Corporation. All Rights Reserved. +Copyright (c) 2014, 2017, MariaDB Corporation. This program is free software; you can redistribute it and/or modify it under the terms of the GNU General Public License as published by the Free Software @@ -244,18 +244,16 @@ fil_node_complete_io( ulint type); /*!< in: OS_FILE_WRITE or OS_FILE_READ; marks the node as modified if type == OS_FILE_WRITE */ -/*******************************************************************//** -Frees a space object from the tablespace memory cache. Closes the files in -the chain but does not delete them. There must not be any pending i/o's or +/** Free a space object from the tablespace memory cache. Close the files in +the chain but do not delete them. There must not be any pending i/o's or flushes on the files. -@return TRUE on success */ +The fil_system->mutex will be released. +@param[in] id tablespace ID +@param[in] x_latched whether the caller holds exclusive space->latch +@return whether the tablespace existed */ static -ibool -fil_space_free( -/*===========*/ - ulint id, /* in: space id */ - ibool x_latched); /* in: TRUE if caller has space->latch - in X mode */ +bool +fil_space_free_and_mutex_exit(ulint id, bool x_latched); /********************************************************************//** Reads data from a space to a buffer. Remember that the possible incomplete blocks at the end of file are ignored: they are not taken into account when @@ -1360,18 +1358,14 @@ retry: } } -/*******************************************************************//** -Frees a file node object from a tablespace memory cache. */ +/** Prepare a data file object for freeing. +@param[in,out] space tablespace +@param[in,out] node data file */ static void -fil_node_free( -/*==========*/ - fil_node_t* node, /*!< in, own: file node */ - fil_system_t* system, /*!< in: tablespace memory cache */ - fil_space_t* space) /*!< in: space where the file node is chained */ +fil_node_free_part1(fil_space_t* space, fil_node_t* node) { - ut_ad(node && system && space); - ut_ad(mutex_own(&(system->mutex))); + ut_ad(mutex_own(&fil_system->mutex)); ut_a(node->magic_n == FIL_NODE_MAGIC_N); ut_a(node->n_pending == 0); ut_a(!node->being_extended); @@ -1394,12 +1388,22 @@ fil_node_free( space->is_in_unflushed_spaces = false; UT_LIST_REMOVE(unflushed_spaces, - system->unflushed_spaces, + fil_system->unflushed_spaces, space); } - fil_node_close_file(node, system); + fil_node_close_file(node, fil_system); } +} + +/** Free a data file object. +@param[in,out] space tablespace +@param[in] node data file */ +static +void +fil_node_free_part2(fil_space_t* space, fil_node_t* node) +{ + ut_ad(!node->open); space->size -= node->size; @@ -1439,7 +1443,8 @@ fil_space_truncate_start( trunc_len -= node->size * UNIV_PAGE_SIZE; - fil_node_free(node, fil_system, space); + fil_node_free_part1(space, node); + fil_node_free_part2(space, node); } mutex_exit(&fil_system->mutex); @@ -1498,10 +1503,9 @@ fil_space_create( "from the cache with id %lu", name, (ulong) id); - ibool success = fil_space_free(space->id, FALSE); + bool success = fil_space_free_and_mutex_exit( + space->id, false); ut_a(success); - - mutex_exit(&fil_system->mutex); } } while (space != 0); @@ -1656,19 +1660,16 @@ fil_assign_new_space_id( return(success); } -/*******************************************************************//** -Frees a space object from the tablespace memory cache. Closes the files in -the chain but does not delete them. There must not be any pending i/o's or +/** Free a space object from the tablespace memory cache. Close the files in +the chain but do not delete them. There must not be any pending i/o's or flushes on the files. -@return TRUE if success */ +The fil_system->mutex will be released. +@param[in] id tablespace ID +@param[in] x_latched whether the caller holds exclusive space->latch +@return whether the tablespace existed */ static -ibool -fil_space_free( -/*===========*/ - /* out: TRUE if success */ - ulint id, /* in: space id */ - ibool x_latched) /* in: TRUE if caller has space->latch - in X mode */ +bool +fil_space_free_and_mutex_exit(ulint id, bool x_latched) { fil_space_t* space; fil_space_t* fnamespace; @@ -1678,13 +1679,11 @@ fil_space_free( space = fil_space_get_by_id(id); if (!space) { - ut_print_timestamp(stderr); - fprintf(stderr, - " InnoDB: Error: trying to remove tablespace %lu" - " from the cache but\n" - "InnoDB: it is not there.\n", (ulong) id); - - return(FALSE); + ib_logf(IB_LOG_LEVEL_ERROR, + "trying to remove non-existing tablespace " ULINTPF, + id); + mutex_exit(&fil_system->mutex); + return(false); } HASH_DELETE(fil_space_t, hash, fil_system->spaces, id, space); @@ -1716,11 +1715,25 @@ fil_space_free( ut_a(space->magic_n == FIL_SPACE_MAGIC_N); ut_a(0 == space->n_pending_flushes); + for (fil_node_t* node = UT_LIST_GET_FIRST(space->chain); + node != NULL; + node = UT_LIST_GET_NEXT(chain, node)) { + fil_node_free_part1(space, node); + } + + mutex_exit(&fil_system->mutex); + + /* Wait for fil_space_release_for_io(); after + fil_space_detach(), the tablespace cannot be found, so + fil_space_acquire_for_io() would return NULL */ + while (space->n_pending_ios) { + os_thread_sleep(100); + } + for (fil_node_t* fil_node = UT_LIST_GET_FIRST(space->chain); fil_node != NULL; fil_node = UT_LIST_GET_FIRST(space->chain)) { - - fil_node_free(fil_node, fil_system, space); + fil_node_free_part2(space, fil_node); } ut_a(0 == UT_LIST_GET_LEN(space->chain)); @@ -2114,7 +2127,11 @@ fil_close_all_files(void) space = UT_LIST_GET_NEXT(space_list, space); - fil_space_free(prev_space->id, FALSE); + /* This is executed during shutdown. No other thread + can create or remove tablespaces while we are not + holding fil_system->mutex. */ + fil_space_free_and_mutex_exit(prev_space->id, false); + mutex_enter(&fil_system->mutex); } mutex_exit(&fil_system->mutex); @@ -2156,7 +2173,11 @@ fil_close_log_files( space = UT_LIST_GET_NEXT(space_list, space); if (free) { - fil_space_free(prev_space->id, FALSE); + /* This is executed during startup. No other thread + can create or remove tablespaces while we are not + holding fil_system->mutex. */ + fil_space_free_and_mutex_exit(prev_space->id, false); + mutex_enter(&fil_system->mutex); } } @@ -2962,15 +2983,13 @@ fil_close_tablespace( /* If the free is successful, the X lock will be released before the space memory data structure is freed. */ - if (!fil_space_free(id, TRUE)) { + if (!fil_space_free_and_mutex_exit(id, TRUE)) { rw_lock_x_unlock(&space->latch); err = DB_TABLESPACE_NOT_FOUND; } else { err = DB_SUCCESS; } - mutex_exit(&fil_system->mutex); - /* If it is a delete then also delete any generated files, otherwise when we drop the database the remove directory will fail. */ @@ -3079,12 +3098,10 @@ fil_delete_tablespace( ut_a(node->n_pending == 0); } - if (!fil_space_free(id, TRUE)) { + if (!fil_space_free_and_mutex_exit(id, true)) { err = DB_TABLESPACE_NOT_FOUND; } - mutex_exit(&fil_system->mutex); - if (err != DB_SUCCESS) { rw_lock_x_unlock(&space->latch); } else if (!os_file_delete(innodb_file_data_key, path) @@ -5879,7 +5896,7 @@ UNIV_INTERN ulint fil_space_get_block_size(const fil_space_t* space, unsigned offset) { - ut_ad(space->n_pending_ops > 0); + ut_ad(space->n_pending_ios > 0); ulint block_size = 512; @@ -6280,7 +6297,7 @@ UNIV_INTERN void fil_flush(fil_space_t* space) { - ut_ad(space->n_pending_ops > 0); + ut_ad(space->n_pending_ios > 0); if (!space->is_stopping()) { mutex_enter(&fil_system->mutex); @@ -7161,13 +7178,11 @@ Used by background threads that do not necessarily hold proper locks for concurrency control. @param[in] id tablespace ID @param[in] silent whether to silently ignore missing tablespaces -@param[in] for_io whether to look up the tablespace while performing I/O - (possibly executing TRUNCATE) @return the tablespace @retval NULL if missing or being deleted or truncated */ UNIV_INTERN fil_space_t* -fil_space_acquire_low(ulint id, bool silent, bool for_io) +fil_space_acquire_low(ulint id, bool silent) { fil_space_t* space; @@ -7180,7 +7195,7 @@ fil_space_acquire_low(ulint id, bool silent, bool for_io) ib_logf(IB_LOG_LEVEL_WARN, "Trying to access missing" " tablespace " ULINTPF ".", id); } - } else if (!for_io && space->is_stopping()) { + } else if (space->is_stopping()) { space = NULL; } else { space->n_pending_ops++; @@ -7193,6 +7208,7 @@ fil_space_acquire_low(ulint id, bool silent, bool for_io) /** Release a tablespace acquired with fil_space_acquire(). @param[in,out] space tablespace to release */ +UNIV_INTERN void fil_space_release(fil_space_t* space) { @@ -7203,6 +7219,41 @@ fil_space_release(fil_space_t* space) mutex_exit(&fil_system->mutex); } +/** Acquire a tablespace for reading or writing a block, +when it could be dropped concurrently. +@param[in] id tablespace ID +@return the tablespace +@retval NULL if missing */ +UNIV_INTERN +fil_space_t* +fil_space_acquire_for_io(ulint id) +{ + mutex_enter(&fil_system->mutex); + + fil_space_t* space = fil_space_get_by_id(id); + + if (space) { + space->n_pending_ios++; + } + + mutex_exit(&fil_system->mutex); + + return(space); +} + +/** Release a tablespace acquired with fil_space_acquire_for_io(). +@param[in,out] space tablespace to release */ +UNIV_INTERN +void +fil_space_release_for_io(fil_space_t* space) +{ + mutex_enter(&fil_system->mutex); + ut_ad(space->magic_n == FIL_SPACE_MAGIC_N); + ut_ad(space->n_pending_ios > 0); + space->n_pending_ios--; + mutex_exit(&fil_system->mutex); +} + /** Return the next fil_space_t. Once started, the caller must keep calling this until it returns NULL. fil_space_acquire() and fil_space_release() are invoked here which @@ -7211,6 +7262,7 @@ blocks a concurrent operation from dropping the tablespace. If NULL, use the first fil_space_t on fil_system->space_list. @return pointer to the next fil_space_t. @retval NULL if this was the last*/ +UNIV_INTERN fil_space_t* fil_space_next(fil_space_t* prev_space) { @@ -7278,6 +7330,7 @@ blocks a concurrent operation from dropping the tablespace. If NULL, use the first fil_space_t on fil_system->space_list. @return pointer to the next fil_space_t. @retval NULL if this was the last*/ +UNIV_INTERN fil_space_t* fil_space_keyrotate_next( fil_space_t* prev_space) diff --git a/storage/innobase/include/buf0buf.h b/storage/innobase/include/buf0buf.h index 3dc2bfc7e5d..1301e112078 100644 --- a/storage/innobase/include/buf0buf.h +++ b/storage/innobase/include/buf0buf.h @@ -1,7 +1,7 @@ /***************************************************************************** Copyright (c) 1995, 2016, Oracle and/or its affiliates. All Rights Reserved. -Copyright (c) 2013, 2017, MariaDB Corporation. All Rights Reserved. +Copyright (c) 2013, 2017, MariaDB Corporation. This program is free software; you can redistribute it and/or modify it under the terms of the GNU General Public License as published by the Free Software @@ -679,13 +679,13 @@ buf_page_is_checksum_valid_none( ulint checksum_field2) MY_ATTRIBUTE((warn_unused_result)); -/********************************************************************//** -Checks if a page is corrupt. +/** Check if a page is corrupt. @param[in] check_lsn true if LSN should be checked @param[in] read_buf Page to be checked @param[in] zip_size compressed size or 0 @param[in] space Pointer to tablespace @return true if corrupted, false if not */ +UNIV_INTERN bool buf_page_is_corrupted( bool check_lsn, @@ -693,15 +693,13 @@ buf_page_is_corrupted( ulint zip_size, const fil_space_t* space) MY_ATTRIBUTE((warn_unused_result)); -/********************************************************************//** -Checks if a page is all zeroes. -@return TRUE if the page is all zeroes */ +/** Check if a page is all zeroes. +@param[in] read_buf database page +@param[in] zip_size ROW_FORMAT=COMPRESSED page size, or 0 +@return whether the page is all zeroes */ +UNIV_INTERN bool -buf_page_is_zeroes( -/*===============*/ - const byte* read_buf, /*!< in: a database page */ - const ulint zip_size); /*!< in: size of compressed page; - 0 for uncompressed pages */ +buf_page_is_zeroes(const byte* read_buf, ulint zip_size); #ifndef UNIV_HOTBACKUP /**********************************************************************//** Gets the space id, page offset, and byte offset within page of a @@ -1240,20 +1238,20 @@ buf_page_init_for_read( version of the tablespace in case we have done DISCARD + IMPORT */ ulint offset);/*!< in: page number */ -/********************************************************************//** -Completes an asynchronous read or write request of a file page to or from -the buffer pool. -@param[in,out] bpage pointer to the block in question -@param[in] evict true if page should be evicted from LRU -@return DB_SUCCESS if page has been read and is not corrupted, -DB_PAGE_CORRUPTED if page based on checksum check is corrupted, -DB_DECRYPTION_FAILED if page post encryption checksum matches but -after decryption normal page checksum does not match.*/ +/** Complete a read or write request of a file page to or from the buffer pool. +@param[in,out] bpage Page to complete +@param[in] evict whether or not to evict the page + from LRU list. +@return whether the operation succeeded +@retval DB_SUCCESS always when writing, or if a read page was OK +@retval DB_PAGE_CORRUPTED if the checksum fails on a page read +@retval DB_DECRYPTION_FAILED if page post encryption checksum matches but + after decryption normal page checksum does + not match */ UNIV_INTERN dberr_t -buf_page_io_complete( - buf_page_t* bpage, - bool evict = false); +buf_page_io_complete(buf_page_t* bpage, bool evict = false) + MY_ATTRIBUTE((nonnull)); /********************************************************************//** Calculates a folded value of a file page address to use in the page hash diff --git a/storage/innobase/include/fil0fil.h b/storage/innobase/include/fil0fil.h index 0bf0364afba..599abc88fa3 100644 --- a/storage/innobase/include/fil0fil.h +++ b/storage/innobase/include/fil0fil.h @@ -320,13 +320,21 @@ struct fil_space_t { ulint n_pending_flushes; /*!< this is positive when flushing the tablespace to disk; dropping of the tablespace is forbidden if this is positive */ - ulint n_pending_ops;/*!< this is positive when we - have pending operations against this - tablespace. The pending operations can - be ibuf merges or lock validation code - trying to read a block. - Dropping of the tablespace is forbidden - if this is positive */ + /** Number of pending buffer pool operations accessing the tablespace + without holding a table lock or dict_operation_lock S-latch + that would prevent the table (and tablespace) from being + dropped. An example is change buffer merge. + The tablespace cannot be dropped while this is nonzero, + or while fil_node_t::n_pending is nonzero. + Protected by fil_system->mutex. */ + ulint n_pending_ops; + /** Number of pending block read or write operations + (when a write is imminent or a read has recently completed). + The tablespace object cannot be freed while this is nonzero, + but it can be detached from fil_system. + Note that fil_node_t::n_pending tracks actual pending I/O requests. + Protected by fil_system->mutex. */ + ulint n_pending_ios; hash_node_t hash; /*!< hash chain node */ hash_node_t name_hash;/*!< hash chain the name_hash table */ #ifndef UNIV_HOTBACKUP @@ -646,13 +654,11 @@ Used by background threads that do not necessarily hold proper locks for concurrency control. @param[in] id tablespace ID @param[in] silent whether to silently ignore missing tablespaces -@param[in] for_io whether to look up the tablespace while performing I/O - (possibly executing TRUNCATE) @return the tablespace @retval NULL if missing or being deleted or truncated */ UNIV_INTERN fil_space_t* -fil_space_acquire_low(ulint id, bool silent, bool for_io = false) +fil_space_acquire_low(ulint id, bool silent) MY_ATTRIBUTE((warn_unused_result)); /** Acquire a tablespace when it could be dropped concurrently. @@ -665,31 +671,45 @@ for concurrency control. @retval NULL if missing or being deleted or truncated */ inline fil_space_t* -fil_space_acquire(ulint id, bool for_io = false) +fil_space_acquire(ulint id) { - return (fil_space_acquire_low(id, false, for_io)); + return(fil_space_acquire_low(id, false)); } /** Acquire a tablespace that may not exist. Used by background threads that do not necessarily hold proper locks for concurrency control. @param[in] id tablespace ID -@param[in] for_io whether to look up the tablespace while performing I/O - (possibly executing TRUNCATE) @return the tablespace @retval NULL if missing or being deleted */ inline fil_space_t* -fil_space_acquire_silent(ulint id, bool for_io = false) +fil_space_acquire_silent(ulint id) { - return (fil_space_acquire_low(id, true, for_io)); + return(fil_space_acquire_low(id, true)); } /** Release a tablespace acquired with fil_space_acquire(). @param[in,out] space tablespace to release */ +UNIV_INTERN void fil_space_release(fil_space_t* space); +/** Acquire a tablespace for reading or writing a block, +when it could be dropped concurrently. +@param[in] id tablespace ID +@return the tablespace +@retval NULL if missing */ +UNIV_INTERN +fil_space_t* +fil_space_acquire_for_io(ulint id); + +/** Release a tablespace acquired with fil_space_acquire_for_io(). +@param[in,out] space tablespace to release */ +UNIV_INTERN +void +fil_space_release_for_io(fil_space_t* space); + /** Return the next fil_space_t. Once started, the caller must keep calling this until it returns NULL. fil_space_acquire() and fil_space_release() are invoked here which @@ -698,6 +718,7 @@ blocks a concurrent operation from dropping the tablespace. If NULL, use the first fil_space_t on fil_system->space_list. @return pointer to the next fil_space_t. @retval NULL if this was the last */ +UNIV_INTERN fil_space_t* fil_space_next( fil_space_t* prev_space) @@ -711,6 +732,7 @@ blocks a concurrent operation from dropping the tablespace. If NULL, use the first fil_space_t on fil_system->space_list. @return pointer to the next fil_space_t. @retval NULL if this was the last*/ +UNIV_INTERN fil_space_t* fil_space_keyrotate_next( fil_space_t* prev_space) @@ -727,12 +749,9 @@ public: /** Constructor: Look up the tablespace and increment the reference count if found. @param[in] space_id tablespace ID - @param[in] silent whether not print any errors - @param[in] for_io whether to look up the tablespace - while performing I/O - (possibly executing TRUNCATE) */ - explicit FilSpace(ulint space_id, bool silent = false, bool for_io = false) - : m_space(fil_space_acquire_low(space_id, silent, for_io)) {} + @param[in] silent whether not to print any errors */ + explicit FilSpace(ulint space_id, bool silent = false) + : m_space(fil_space_acquire_low(space_id, silent)) {} /** Assignment operator: This assumes that fil_space_acquire() has already been done for the fil_space_t. The caller must diff --git a/storage/xtradb/buf/buf0buf.cc b/storage/xtradb/buf/buf0buf.cc index 4593f84b43b..8873e2bcb42 100644 --- a/storage/xtradb/buf/buf0buf.cc +++ b/storage/xtradb/buf/buf0buf.cc @@ -65,6 +65,15 @@ Created 11/5/1995 Heikki Tuuri #include "fil0pagecompress.h" #include "ha_prototypes.h" +/** Decrypt a page. +@param[in,out] bpage Page control block +@param[in,out] space tablespace +@return whether the operation was successful */ +static +bool +buf_page_decrypt_after_read(buf_page_t* bpage, fil_space_t* space) + MY_ATTRIBUTE((nonnull)); + /* prototypes for new functions added to ha_innodb.cc */ trx_t* innobase_get_trx(); @@ -548,16 +557,13 @@ buf_block_alloc( } #endif /* !UNIV_HOTBACKUP */ -/********************************************************************//** -Checks if a page is all zeroes. -@return TRUE if the page is all zeroes */ +/** Check if a page is all zeroes. +@param[in] read_buf database page +@param[in] zip_size ROW_FORMAT=COMPRESSED page size, or 0 +@return whether the page is all zeroes */ UNIV_INTERN bool -buf_page_is_zeroes( -/*===============*/ - const byte* read_buf, /*!< in: a database page */ - const ulint zip_size) /*!< in: size of compressed page; - 0 for uncompressed pages */ +buf_page_is_zeroes(const byte* read_buf, ulint zip_size) { const ulint page_size = zip_size ? zip_size : UNIV_PAGE_SIZE; @@ -673,8 +679,7 @@ buf_page_is_checksum_valid_none( && checksum_field1 == BUF_NO_CHECKSUM_MAGIC); } -/********************************************************************//** -Checks if a page is corrupt. +/** Check if a page is corrupt. @param[in] check_lsn true if LSN should be checked @param[in] read_buf Page to be checked @param[in] zip_size compressed size or 0 @@ -4526,34 +4531,30 @@ buf_mark_space_corrupt( mutex_exit(&buf_pool->LRU_list_mutex); } -/********************************************************************//** -Check if page is maybe compressed, encrypted or both when we encounter +/** Check if page is maybe compressed, encrypted or both when we encounter corrupted page. Note that we can't be 100% sure if page is corrupted or decrypt/decompress just failed. -@param[in,out] bpage Page -@return DB_SUCCESS if page has been read and is not corrupted, -@retval DB_PAGE_CORRUPTED if page based on checksum check is corrupted, -@retval DB_DECRYPTION_FAILED if page post encryption checksum matches but +@param[in,out] bpage page +@param[in,out] space tablespace from fil_space_acquire_for_io() +@return whether the operation succeeded +@retval DB_SUCCESS if page has been read and is not corrupted +@retval DB_PAGE_CORRUPTED if page based on checksum check is corrupted +@retval DB_DECRYPTION_FAILED if page post encryption checksum matches but after decryption normal page checksum does not match. -@retval DB_TABLESPACE_DELETED if accessed tablespace is not found */ +@retval DB_TABLESPACE_DELETED if accessed tablespace is not found */ static dberr_t -buf_page_check_corrupt(buf_page_t* bpage) +buf_page_check_corrupt(buf_page_t* bpage, fil_space_t* space) { + ut_ad(space->n_pending_ios > 0); + ulint zip_size = buf_page_get_zip_size(bpage); byte* dst_frame = (zip_size) ? bpage->zip.data : ((buf_block_t*) bpage)->frame; - FilSpace space(bpage->space, true); bool still_encrypted = false; dberr_t err = DB_SUCCESS; bool corrupted = false; - fil_space_crypt_t* crypt_data = NULL; - - if (!space()) { - return(DB_TABLESPACE_DELETED); - } - - crypt_data = space()->crypt_data; + fil_space_crypt_t* crypt_data = space->crypt_data; /* In buf_decrypt_after_read we have either decrypted the page if page post encryption checksum matches and used key_id is found @@ -4565,12 +4566,12 @@ buf_page_check_corrupt(buf_page_t* bpage) crypt_data->type != CRYPT_SCHEME_UNENCRYPTED && !bpage->encrypted && fil_space_verify_crypt_checksum(dst_frame, zip_size, - space(), bpage->offset)); - + space, bpage->offset)); if (!still_encrypted) { /* If traditional checksums match, we assume that page is not anymore encrypted. */ - corrupted = buf_page_is_corrupted(true, dst_frame, zip_size, space()); + corrupted = buf_page_is_corrupted(true, dst_frame, zip_size, + space); if (!corrupted) { bpage->encrypted = false; @@ -4593,7 +4594,7 @@ buf_page_check_corrupt(buf_page_t* bpage) ", page number=%u]" " in file %s cannot be decrypted.", bpage->space, bpage->offset, - space()->name); + space->name); ib_logf(IB_LOG_LEVEL_INFO, "However key management plugin or used key_version " ULINTPF @@ -4611,26 +4612,23 @@ buf_page_check_corrupt(buf_page_t* bpage) return (err); } -/********************************************************************//** -Completes an asynchronous read or write request of a file page to or from -the buffer pool. +/** Complete a read or write request of a file page to or from the buffer pool. @param[in,out] bpage Page to complete -@return DB_SUCCESS if page has been read and is not corrupted, -DB_PAGE_CORRUPTED if page based on checksum check is corrupted, -DB_DECRYPTION_FAILED if page post encryption checksum matches but -after decryption normal page checksum does not match. -in write only DB_SUCCESS is possible. */ +@return whether the operation succeeded +@retval DB_SUCCESS always when writing, or if a read page was OK +@retval DB_PAGE_CORRUPTED if the checksum fails on a page read +@retval DB_DECRYPTION_FAILED if page post encryption checksum matches but + after decryption normal page checksum does + not match */ UNIV_INTERN dberr_t -buf_page_io_complete( - buf_page_t* bpage) +buf_page_io_complete(buf_page_t* bpage) { enum buf_io_fix io_type; buf_pool_t* buf_pool = buf_pool_from_bpage(bpage); const ibool uncompressed = (buf_page_get_state(bpage) == BUF_BLOCK_FILE_PAGE); bool have_LRU_mutex = false; - fil_space_t* space = NULL; byte* frame = NULL; dberr_t err = DB_SUCCESS; @@ -4650,7 +4648,13 @@ buf_page_io_complete( ulint read_space_id = 0; uint key_version = 0; - buf_page_decrypt_after_read(bpage); + ut_ad(bpage->zip.data || ((buf_block_t*)bpage)->frame); + fil_space_t* space = fil_space_acquire_for_io(bpage->space); + if (!space) { + return(DB_TABLESPACE_DELETED); + } + + buf_page_decrypt_after_read(bpage, space); if (buf_page_get_zip_size(bpage)) { frame = bpage->zip.data; @@ -4724,7 +4728,7 @@ buf_page_io_complete( if (UNIV_LIKELY(!bpage->is_corrupt || !srv_pass_corrupt_table)) { - err = buf_page_check_corrupt(bpage); + err = buf_page_check_corrupt(bpage, space); } database_corrupted: @@ -4737,6 +4741,7 @@ database_corrupted: buf_mark_space_corrupt(bpage); ib_logf(IB_LOG_LEVEL_INFO, "Simulated page corruption"); + fil_space_release_for_io(space); return(err); } err = DB_SUCCESS; @@ -4744,9 +4749,6 @@ database_corrupted: ); if (err == DB_PAGE_CORRUPTED) { - fil_system_enter(); - space = fil_space_get_by_id(bpage->space); - ib_logf(IB_LOG_LEVEL_ERROR, "Database page corruption on disk" " or a failed file read of tablespace %s" @@ -4757,8 +4759,6 @@ database_corrupted: space->name, bpage->space, bpage->offset); - fil_system_exit(); - buf_page_print(frame, buf_page_get_zip_size(bpage), BUF_PAGE_PRINT_NO_CRASH); @@ -4795,6 +4795,7 @@ database_corrupted: table as corrupted instead of crashing server */ if (bpage->space > TRX_SYS_SPACE) { buf_mark_space_corrupt(bpage); + fil_space_release_for_io(space); return(err); } else { ib_logf(IB_LOG_LEVEL_FATAL, @@ -4833,6 +4834,8 @@ database_corrupted: } } + + fil_space_release_for_io(space); } else { /* io_type == BUF_IO_WRITE */ if (bpage->slot) { @@ -6303,16 +6306,17 @@ buf_page_encrypt_before_write( return dst_frame; } -/********************************************************************//** -Decrypt page after it has been read from disk -@param[in,out] bpage Page control block -@return true if successfull, false if something went wrong -*/ -UNIV_INTERN +/** Decrypt a page. +@param[in,out] bpage Page control block +@param[in,out] space tablespace +@return whether the operation was successful */ +static bool -buf_page_decrypt_after_read( - buf_page_t* bpage) +buf_page_decrypt_after_read(buf_page_t* bpage, fil_space_t* space) { + ut_ad(space->n_pending_ios > 0); + ut_ad(space->id == bpage->space); + ulint zip_size = buf_page_get_zip_size(bpage); ulint size = (zip_size) ? zip_size : UNIV_PAGE_SIZE; @@ -6330,12 +6334,10 @@ buf_page_decrypt_after_read( return (true); } - FilSpace space(bpage->space, false, true); - /* Page is encrypted if encryption information is found from tablespace and page contains used key_version. This is true also for pages first compressed and then encrypted. */ - if (!space() || !space()->crypt_data) { + if (!space->crypt_data) { key_version = 0; } @@ -6372,8 +6374,8 @@ buf_page_decrypt_after_read( /* Mark page encrypted in case it should be. */ - if (key_version && space()->crypt_data && - space()->crypt_data->type != CRYPT_SCHEME_UNENCRYPTED) { + if (space->crypt_data->type + != CRYPT_SCHEME_UNENCRYPTED) { bpage->encrypted = true; } @@ -6388,12 +6390,8 @@ buf_page_decrypt_after_read( #endif /* decrypt using crypt_buf to dst_frame */ - byte* res = fil_space_decrypt(space(), - slot->crypt_buf, - dst_frame, - &bpage->encrypted); - - if (!res) { + if (!fil_space_decrypt(space, slot->crypt_buf, + dst_frame, &bpage->encrypted)) { success = false; } @@ -6424,5 +6422,6 @@ buf_page_decrypt_after_read( } } + ut_ad(space->n_pending_ios > 0); return (success); } diff --git a/storage/xtradb/buf/buf0flu.cc b/storage/xtradb/buf/buf0flu.cc index 9c9823bbc28..6f8bbd75d99 100644 --- a/storage/xtradb/buf/buf0flu.cc +++ b/storage/xtradb/buf/buf0flu.cc @@ -873,7 +873,7 @@ buf_flush_write_block_low( buf_flush_t flush_type, /*!< in: type of flush */ bool sync) /*!< in: true if sync IO request */ { - fil_space_t* space = fil_space_acquire(bpage->space, true); + fil_space_t* space = fil_space_acquire_for_io(bpage->space); if (!space) { return; } @@ -995,6 +995,13 @@ buf_flush_write_block_low( ut_ad(flush_type == BUF_FLUSH_SINGLE_PAGE); fil_flush(space); + /* The tablespace could already have been dropped, + because fil_io(request, sync) would already have + decremented the node->n_pending. However, + buf_page_io_complete() only needs to look up the + tablespace during read requests, not during writes. */ + ut_ad(buf_page_get_io_fix_unlocked(bpage) == BUF_IO_WRITE); + #ifdef UNIV_DEBUG dberr_t err = #endif @@ -1003,7 +1010,7 @@ buf_flush_write_block_low( ut_ad(err == DB_SUCCESS); } - fil_space_release(space); + fil_space_release_for_io(space); /* Increment the counter of I/O operations used for selecting LRU policy. */ diff --git a/storage/xtradb/fil/fil0crypt.cc b/storage/xtradb/fil/fil0crypt.cc index 72a4e713e60..3537ad2c66a 100644 --- a/storage/xtradb/fil/fil0crypt.cc +++ b/storage/xtradb/fil/fil0crypt.cc @@ -683,7 +683,7 @@ fil_space_encrypt( } fil_space_crypt_t* crypt_data = space->crypt_data; - ut_ad(space->n_pending_ops); + ut_ad(space->n_pending_ios > 0); ulint zip_size = fsp_flags_get_zip_size(space->flags); byte* tmp = fil_encrypt_buf(crypt_data, space->id, offset, lsn, src_frame, zip_size, dst_frame); @@ -860,7 +860,7 @@ fil_space_decrypt( *decrypted = false; ut_ad(space->crypt_data != NULL && space->crypt_data->is_encrypted()); - ut_ad(space->n_pending_ops > 0); + ut_ad(space->n_pending_ios > 0); bool encrypted = fil_space_decrypt( space->crypt_data, diff --git a/storage/xtradb/fil/fil0fil.cc b/storage/xtradb/fil/fil0fil.cc index c95fd872c67..e39be46840c 100644 --- a/storage/xtradb/fil/fil0fil.cc +++ b/storage/xtradb/fil/fil0fil.cc @@ -1,7 +1,7 @@ /***************************************************************************** Copyright (c) 1995, 2016, Oracle and/or its affiliates. All Rights Reserved. -Copyright (c) 2014, 2017, MariaDB Corporation. All Rights Reserved. +Copyright (c) 2014, 2017, MariaDB Corporation. This program is free software; you can redistribute it and/or modify it under the terms of the GNU General Public License as published by the Free Software @@ -249,18 +249,16 @@ fil_node_complete_io( ulint type); /*!< in: OS_FILE_WRITE or OS_FILE_READ; marks the node as modified if type == OS_FILE_WRITE */ -/*******************************************************************//** -Frees a space object from the tablespace memory cache. Closes the files in -the chain but does not delete them. There must not be any pending i/o's or +/** Free a space object from the tablespace memory cache. Close the files in +the chain but do not delete them. There must not be any pending i/o's or flushes on the files. -@return TRUE on success */ +The fil_system->mutex will be released. +@param[in] id tablespace ID +@param[in] x_latched whether the caller holds exclusive space->latch +@return whether the tablespace existed */ static -ibool -fil_space_free( -/*===========*/ - ulint id, /* in: space id */ - ibool x_latched); /* in: TRUE if caller has space->latch - in X mode */ +bool +fil_space_free_and_mutex_exit(ulint id, bool x_latched); /********************************************************************//** Reads data from a space to a buffer. Remember that the possible incomplete blocks at the end of file are ignored: they are not taken into account when @@ -1369,18 +1367,14 @@ retry: } } -/*******************************************************************//** -Frees a file node object from a tablespace memory cache. */ +/** Prepare a data file object for freeing. +@param[in,out] space tablespace +@param[in,out] node data file */ static void -fil_node_free( -/*==========*/ - fil_node_t* node, /*!< in, own: file node */ - fil_system_t* system, /*!< in: tablespace memory cache */ - fil_space_t* space) /*!< in: space where the file node is chained */ +fil_node_free_part1(fil_space_t* space, fil_node_t* node) { - ut_ad(node && system && space); - ut_ad(mutex_own(&(system->mutex))); + ut_ad(mutex_own(&fil_system->mutex)); ut_a(node->magic_n == FIL_NODE_MAGIC_N); ut_a(node->n_pending == 0); ut_a(!node->being_extended); @@ -1403,12 +1397,22 @@ fil_node_free( space->is_in_unflushed_spaces = false; UT_LIST_REMOVE(unflushed_spaces, - system->unflushed_spaces, + fil_system->unflushed_spaces, space); } - fil_node_close_file(node, system); + fil_node_close_file(node, fil_system); } +} + +/** Free a data file object. +@param[in,out] space tablespace +@param[in] node data file */ +static +void +fil_node_free_part2(fil_space_t* space, fil_node_t* node) +{ + ut_ad(!node->open); space->size -= node->size; @@ -1448,7 +1452,8 @@ fil_space_truncate_start( trunc_len -= node->size * UNIV_PAGE_SIZE; - fil_node_free(node, fil_system, space); + fil_node_free_part1(space, node); + fil_node_free_part2(space, node); } mutex_exit(&fil_system->mutex); @@ -1540,10 +1545,9 @@ fil_space_create( "from the cache with id %lu", name, (ulong) id); - ibool success = fil_space_free(space->id, FALSE); + bool success = fil_space_free_and_mutex_exit( + space->id, false); ut_a(success); - - mutex_exit(&fil_system->mutex); } } while (space != 0); @@ -1698,19 +1702,16 @@ fil_assign_new_space_id( return(success); } -/*******************************************************************//** -Frees a space object from the tablespace memory cache. Closes the files in -the chain but does not delete them. There must not be any pending i/o's or +/** Free a space object from the tablespace memory cache. Close the files in +the chain but do not delete them. There must not be any pending i/o's or flushes on the files. -@return TRUE if success */ +The fil_system->mutex will be released. +@param[in] id tablespace ID +@param[in] x_latched whether the caller holds exclusive space->latch +@return whether the tablespace existed */ static -ibool -fil_space_free( -/*===========*/ - /* out: TRUE if success */ - ulint id, /* in: space id */ - ibool x_latched) /* in: TRUE if caller has space->latch - in X mode */ +bool +fil_space_free_and_mutex_exit(ulint id, bool x_latched) { fil_space_t* space; fil_space_t* fnamespace; @@ -1720,13 +1721,11 @@ fil_space_free( space = fil_space_get_by_id(id); if (!space) { - ut_print_timestamp(stderr); - fprintf(stderr, - " InnoDB: Error: trying to remove tablespace %lu" - " from the cache but\n" - "InnoDB: it is not there.\n", (ulong) id); - - return(FALSE); + ib_logf(IB_LOG_LEVEL_ERROR, + "trying to remove non-existing tablespace " ULINTPF, + id); + mutex_exit(&fil_system->mutex); + return(false); } HASH_DELETE(fil_space_t, hash, fil_system->spaces, id, space); @@ -1758,11 +1757,25 @@ fil_space_free( ut_a(space->magic_n == FIL_SPACE_MAGIC_N); ut_a(0 == space->n_pending_flushes); + for (fil_node_t* node = UT_LIST_GET_FIRST(space->chain); + node != NULL; + node = UT_LIST_GET_NEXT(chain, node)) { + fil_node_free_part1(space, node); + } + + mutex_exit(&fil_system->mutex); + + /* Wait for fil_space_release_for_io(); after + fil_space_detach(), the tablespace cannot be found, so + fil_space_acquire_for_io() would return NULL */ + while (space->n_pending_ios) { + os_thread_sleep(100); + } + for (fil_node_t* fil_node = UT_LIST_GET_FIRST(space->chain); fil_node != NULL; fil_node = UT_LIST_GET_FIRST(space->chain)) { - - fil_node_free(fil_node, fil_system, space); + fil_node_free_part2(space, fil_node); } ut_a(0 == UT_LIST_GET_LEN(space->chain)); @@ -2162,7 +2175,11 @@ fil_close_all_files(void) space = UT_LIST_GET_NEXT(space_list, space); - fil_space_free(prev_space->id, FALSE); + /* This is executed during shutdown. No other thread + can create or remove tablespaces while we are not + holding fil_system->mutex. */ + fil_space_free_and_mutex_exit(prev_space->id, false); + mutex_enter(&fil_system->mutex); } mutex_exit(&fil_system->mutex); @@ -2210,7 +2227,11 @@ fil_close_log_files( space = UT_LIST_GET_NEXT(space_list, space); if (free) { - fil_space_free(prev_space->id, FALSE); + /* This is executed during startup. No other thread + can create or remove tablespaces while we are not + holding fil_system->mutex. */ + fil_space_free_and_mutex_exit(prev_space->id, false); + mutex_enter(&fil_system->mutex); } } @@ -3020,15 +3041,13 @@ fil_close_tablespace( /* If the free is successful, the X lock will be released before the space memory data structure is freed. */ - if (!fil_space_free(id, TRUE)) { + if (!fil_space_free_and_mutex_exit(id, TRUE)) { rw_lock_x_unlock(&space->latch); err = DB_TABLESPACE_NOT_FOUND; } else { err = DB_SUCCESS; } - mutex_exit(&fil_system->mutex); - /* If it is a delete then also delete any generated files, otherwise when we drop the database the remove directory will fail. */ @@ -3137,12 +3156,10 @@ fil_delete_tablespace( ut_a(node->n_pending == 0); } - if (!fil_space_free(id, TRUE)) { + if (!fil_space_free_and_mutex_exit(id, true)) { err = DB_TABLESPACE_NOT_FOUND; } - mutex_exit(&fil_system->mutex); - if (err != DB_SUCCESS) { rw_lock_x_unlock(&space->latch); } else if (!os_file_delete(innodb_file_data_key, path) @@ -6189,7 +6206,7 @@ UNIV_INTERN ulint fil_space_get_block_size(const fil_space_t* space, unsigned offset) { - ut_ad(space->n_pending_ops > 0); + ut_ad(space->n_pending_ios > 0); ulint block_size = 512; @@ -6626,7 +6643,7 @@ UNIV_INTERN void fil_flush(fil_space_t* space) { - ut_ad(space->n_pending_ops > 0); + ut_ad(space->n_pending_ios > 0); if (!space->is_stopping()) { mutex_enter(&fil_system->mutex); @@ -7555,13 +7572,11 @@ Used by background threads that do not necessarily hold proper locks for concurrency control. @param[in] id tablespace ID @param[in] silent whether to silently ignore missing tablespaces -@param[in] for_io whether to look up the tablespace while performing I/O - (possibly executing TRUNCATE) @return the tablespace @retval NULL if missing or being deleted or truncated */ UNIV_INTERN fil_space_t* -fil_space_acquire_low(ulint id, bool silent, bool for_io) +fil_space_acquire_low(ulint id, bool silent) { fil_space_t* space; @@ -7574,7 +7589,7 @@ fil_space_acquire_low(ulint id, bool silent, bool for_io) ib_logf(IB_LOG_LEVEL_WARN, "Trying to access missing" " tablespace " ULINTPF ".", id); } - } else if (!for_io && space->is_stopping()) { + } else if (space->is_stopping()) { space = NULL; } else { space->n_pending_ops++; @@ -7585,8 +7600,44 @@ fil_space_acquire_low(ulint id, bool silent, bool for_io) return(space); } +/** Acquire a tablespace for reading or writing a block, +when it could be dropped concurrently. +@param[in] id tablespace ID +@return the tablespace +@retval NULL if missing */ +UNIV_INTERN +fil_space_t* +fil_space_acquire_for_io(ulint id) +{ + mutex_enter(&fil_system->mutex); + + fil_space_t* space = fil_space_get_by_id(id); + + if (space) { + space->n_pending_ios++; + } + + mutex_exit(&fil_system->mutex); + + return(space); +} + +/** Release a tablespace acquired with fil_space_acquire_for_io(). +@param[in,out] space tablespace to release */ +UNIV_INTERN +void +fil_space_release_for_io(fil_space_t* space) +{ + mutex_enter(&fil_system->mutex); + ut_ad(space->magic_n == FIL_SPACE_MAGIC_N); + ut_ad(space->n_pending_ios > 0); + space->n_pending_ios--; + mutex_exit(&fil_system->mutex); +} + /** Release a tablespace acquired with fil_space_acquire(). @param[in,out] space tablespace to release */ +UNIV_INTERN void fil_space_release(fil_space_t* space) { @@ -7605,6 +7656,7 @@ blocks a concurrent operation from dropping the tablespace. If NULL, use the first fil_space_t on fil_system->space_list. @return pointer to the next fil_space_t. @retval NULL if this was the last*/ +UNIV_INTERN fil_space_t* fil_space_next(fil_space_t* prev_space) { @@ -7672,6 +7724,7 @@ blocks a concurrent operation from dropping the tablespace. If NULL, use the first fil_space_t on fil_system->space_list. @return pointer to the next fil_space_t. @retval NULL if this was the last*/ +UNIV_INTERN fil_space_t* fil_space_keyrotate_next( fil_space_t* prev_space) diff --git a/storage/xtradb/include/buf0buf.h b/storage/xtradb/include/buf0buf.h index 08c3a765a8b..9b4276efaa8 100644 --- a/storage/xtradb/include/buf0buf.h +++ b/storage/xtradb/include/buf0buf.h @@ -1,7 +1,7 @@ /***************************************************************************** Copyright (c) 1995, 2016, Oracle and/or its affiliates. All Rights Reserved. -Copyright (c) 2013, 2017, MariaDB Corporation. All Rights Reserved. +Copyright (c) 2013, 2017, MariaDB Corporation. This program is free software; you can redistribute it and/or modify it under the terms of the GNU General Public License as published by the Free Software @@ -675,13 +675,13 @@ buf_page_is_checksum_valid_none( ulint checksum_field2) MY_ATTRIBUTE((warn_unused_result)); -/********************************************************************//** -Checks if a page is corrupt. +/** Check if a page is corrupt. @param[in] check_lsn true if LSN should be checked @param[in] read_buf Page to be checked @param[in] zip_size compressed size or 0 @param[in] space Pointer to tablespace @return true if corrupted, false if not */ +UNIV_INTERN bool buf_page_is_corrupted( bool check_lsn, @@ -689,15 +689,13 @@ buf_page_is_corrupted( ulint zip_size, const fil_space_t* space) MY_ATTRIBUTE((warn_unused_result)); -/********************************************************************//** -Checks if a page is all zeroes. -@return TRUE if the page is all zeroes */ +/** Check if a page is all zeroes. +@param[in] read_buf database page +@param[in] zip_size ROW_FORMAT=COMPRESSED page size, or 0 +@return whether the page is all zeroes */ +UNIV_INTERN bool -buf_page_is_zeroes( -/*===============*/ - const byte* read_buf, /*!< in: a database page */ - const ulint zip_size); /*!< in: size of compressed page; - 0 for uncompressed pages */ +buf_page_is_zeroes(const byte* read_buf, ulint zip_size); #ifndef UNIV_HOTBACKUP /**********************************************************************//** Gets the space id, page offset, and byte offset within page of a @@ -1259,18 +1257,18 @@ buf_page_init_for_read( version of the tablespace in case we have done DISCARD + IMPORT */ ulint offset);/*!< in: page number */ -/********************************************************************//** -Completes an asynchronous read or write request of a file page to or from -the buffer pool. -@param[in,out] bpage pointer to the block in question -@return DB_SUCCESS if page has been read and is not corrupted, -DB_PAGE_CORRUPTED if page based on checksum check is corrupted, -DB_DECRYPTION_FAILED if page post encryption checksum matches but -after decryption normal page checksum does not match.*/ +/** Complete a read or write request of a file page to or from the buffer pool. +@param[in,out] bpage Page to complete +@return whether the operation succeeded +@retval DB_SUCCESS always when writing, or if a read page was OK +@retval DB_PAGE_CORRUPTED if the checksum fails on a page read +@retval DB_DECRYPTION_FAILED if page post encryption checksum matches but + after decryption normal page checksum does + not match */ UNIV_INTERN dberr_t -buf_page_io_complete( - buf_page_t* bpage); +buf_page_io_complete(buf_page_t* bpage) + MY_ATTRIBUTE((nonnull)); /********************************************************************//** Calculates a folded value of a file page address to use in the page hash table. diff --git a/storage/xtradb/include/fil0fil.h b/storage/xtradb/include/fil0fil.h index 37269db1960..2f03d2aa0f5 100644 --- a/storage/xtradb/include/fil0fil.h +++ b/storage/xtradb/include/fil0fil.h @@ -314,13 +314,21 @@ struct fil_space_t { ulint n_pending_flushes; /*!< this is positive when flushing the tablespace to disk; dropping of the tablespace is forbidden if this is positive */ - ulint n_pending_ops;/*!< this is positive when we - have pending operations against this - tablespace. The pending operations can - be ibuf merges or lock validation code - trying to read a block. - Dropping of the tablespace is forbidden - if this is positive */ + /** Number of pending buffer pool operations accessing the tablespace + without holding a table lock or dict_operation_lock S-latch + that would prevent the table (and tablespace) from being + dropped. An example is change buffer merge. + The tablespace cannot be dropped while this is nonzero, + or while fil_node_t::n_pending is nonzero. + Protected by fil_system->mutex. */ + ulint n_pending_ops; + /** Number of pending block read or write operations + (when a write is imminent or a read has recently completed). + The tablespace object cannot be freed while this is nonzero, + but it can be detached from fil_system. + Note that fil_node_t::n_pending tracks actual pending I/O requests. + Protected by fil_system->mutex. */ + ulint n_pending_ios; hash_node_t hash; /*!< hash chain node */ hash_node_t name_hash;/*!< hash chain the name_hash table */ #ifndef UNIV_HOTBACKUP @@ -652,13 +660,11 @@ Used by background threads that do not necessarily hold proper locks for concurrency control. @param[in] id tablespace ID @param[in] silent whether to silently ignore missing tablespaces -@param[in] for_io whether to look up the tablespace while performing I/O - (possibly executing TRUNCATE) @return the tablespace @retval NULL if missing or being deleted or truncated */ UNIV_INTERN fil_space_t* -fil_space_acquire_low(ulint id, bool silent, bool for_io = false) +fil_space_acquire_low(ulint id, bool silent) MY_ATTRIBUTE((warn_unused_result)); /** Acquire a tablespace when it could be dropped concurrently. @@ -671,31 +677,45 @@ for concurrency control. @retval NULL if missing or being deleted or truncated */ inline fil_space_t* -fil_space_acquire(ulint id, bool for_io = false) +fil_space_acquire(ulint id) { - return (fil_space_acquire_low(id, false, for_io)); + return(fil_space_acquire_low(id, false)); } /** Acquire a tablespace that may not exist. Used by background threads that do not necessarily hold proper locks for concurrency control. @param[in] id tablespace ID -@param[in] for_io whether to look up the tablespace while performing I/O - (possibly executing TRUNCATE) @return the tablespace @retval NULL if missing or being deleted */ inline fil_space_t* -fil_space_acquire_silent(ulint id, bool for_io = false) +fil_space_acquire_silent(ulint id) { - return (fil_space_acquire_low(id, true, for_io)); + return(fil_space_acquire_low(id, true)); } /** Release a tablespace acquired with fil_space_acquire(). @param[in,out] space tablespace to release */ +UNIV_INTERN void fil_space_release(fil_space_t* space); +/** Acquire a tablespace for reading or writing a block, +when it could be dropped concurrently. +@param[in] id tablespace ID +@return the tablespace +@retval NULL if missing */ +UNIV_INTERN +fil_space_t* +fil_space_acquire_for_io(ulint id); + +/** Release a tablespace acquired with fil_space_acquire_for_io(). +@param[in,out] space tablespace to release */ +UNIV_INTERN +void +fil_space_release_for_io(fil_space_t* space); + /** Return the next fil_space_t. Once started, the caller must keep calling this until it returns NULL. fil_space_acquire() and fil_space_release() are invoked here which @@ -704,6 +724,7 @@ blocks a concurrent operation from dropping the tablespace. If NULL, use the first fil_space_t on fil_system->space_list. @return pointer to the next fil_space_t. @retval NULL if this was the last */ +UNIV_INTERN fil_space_t* fil_space_next( fil_space_t* prev_space) @@ -717,6 +738,7 @@ blocks a concurrent operation from dropping the tablespace. If NULL, use the first fil_space_t on fil_system->space_list. @return pointer to the next fil_space_t. @retval NULL if this was the last*/ +UNIV_INTERN fil_space_t* fil_space_keyrotate_next( fil_space_t* prev_space) @@ -733,12 +755,9 @@ public: /** Constructor: Look up the tablespace and increment the reference count if found. @param[in] space_id tablespace ID - @param[in] silent whether not print any errors - @param[in] for_io whether to look up the tablespace - while performing I/O - (possibly executing TRUNCATE) */ - explicit FilSpace(ulint space_id, bool silent = false, bool for_io = false) - : m_space(fil_space_acquire_low(space_id, silent, for_io)) {} + @param[in] silent whether not to print any errors */ + explicit FilSpace(ulint space_id, bool silent = false) + : m_space(fil_space_acquire_low(space_id, silent)) {} /** Assignment operator: This assumes that fil_space_acquire() has already been done for the fil_space_t. The caller must