From 1c55b845e0fe337e647ba230288ed13e966cb7c7 Mon Sep 17 00:00:00 2001 From: Monty Date: Sun, 3 Dec 2023 14:09:43 +0200 Subject: [PATCH] MDEV-32932 Port backup features from ES Added support to BACKUP STAGE to maria-backup This is a port of the code from ES 10.6 See MDEV-5336 for backup stages description. The following old options are not supported by the new code: --rsync ; This is because rsync will not work on tables that are in used. --no-backup-locks ; This is disabled as mariadb-backup will always use backup locks for better performance. --- extra/mariabackup/CMakeLists.txt | 10 +- extra/mariabackup/aria_backup_client.cc | 1016 +++++++++++++++++ extra/mariabackup/aria_backup_client.h | 38 + extra/mariabackup/backup_copy.cc | 491 +++----- extra/mariabackup/backup_copy.h | 26 +- extra/mariabackup/backup_debug.h | 21 +- extra/mariabackup/backup_mysql.cc | 203 ++-- extra/mariabackup/backup_mysql.h | 22 +- extra/mariabackup/common_engine.cc | 512 +++++++++ extra/mariabackup/common_engine.h | 39 + extra/mariabackup/datasink.cc | 28 +- extra/mariabackup/datasink.h | 17 +- extra/mariabackup/ddl_log.cc | 553 +++++++++ extra/mariabackup/ddl_log.h | 15 + extra/mariabackup/ds_buffer.cc | 9 +- extra/mariabackup/ds_compress.cc | 9 +- extra/mariabackup/ds_local.cc | 105 +- extra/mariabackup/ds_stdout.cc | 8 +- extra/mariabackup/ds_tmpfile.cc | 8 +- extra/mariabackup/ds_xbstream.cc | 61 +- .../{xb_plugin.cc => encryption_plugin.cc} | 44 +- extra/mariabackup/encryption_plugin.h | 7 + extra/mariabackup/innobackupex.cc | 12 - extra/mariabackup/thread_pool.cc | 50 + extra/mariabackup/thread_pool.h | 62 + extra/mariabackup/write_filt.cc | 12 + extra/mariabackup/wsrep.cc | 46 +- extra/mariabackup/xb_plugin.h | 5 - extra/mariabackup/xbstream.cc | 64 +- extra/mariabackup/xbstream.h | 19 +- extra/mariabackup/xbstream_read.cc | 115 +- extra/mariabackup/xbstream_write.cc | 152 ++- extra/mariabackup/xtrabackup.cc | 891 ++++++++++++--- extra/mariabackup/xtrabackup.h | 53 +- mysql-test/include/aria_log_control_load.inc | 11 + .../mariabackup/absolute_ibdata_paths.test | 2 +- .../suite/mariabackup/alter_copy_race.result | 2 +- .../suite/mariabackup/alter_copy_race.test | 4 +- .../mariabackup/apply-log-only-incr.test | 4 +- .../suite/mariabackup/apply-log-only.test | 2 +- mysql-test/suite/mariabackup/aria_backup.opt | 1 + .../suite/mariabackup/aria_backup.result | 780 +++++++++++++ mysql-test/suite/mariabackup/aria_backup.test | 423 +++++++ mysql-test/suite/mariabackup/aria_log.opt | 1 + .../mariabackup/aria_log_dir_path.result | 1 - .../suite/mariabackup/aria_log_dir_path.test | 6 +- .../mariabackup/aria_log_dir_path_rel.result | 1 - .../aria_log_rotate_during_backup.opt | 2 + .../aria_log_rotate_during_backup.result | 58 + .../aria_log_rotate_during_backup.test | 82 ++ .../suite/mariabackup/auth_plugin_win.test | 2 +- .../suite/mariabackup/backup_grants.result | 2 - .../suite/mariabackup/backup_grants.test | 24 +- mysql-test/suite/mariabackup/backup_ssl.test | 2 +- mysql-test/suite/mariabackup/binlog.test | 2 +- .../suite/mariabackup/compress_qpress.test | 2 +- .../mariabackup/create_during_backup.test | 2 +- ...ate_with_data_directory_during_backup.test | 2 +- .../suite/mariabackup/data_directory.test | 5 +- .../mariabackup/ddl_for_common_engine.result | 67 ++ .../mariabackup/ddl_for_common_engine.test | 79 ++ mysql-test/suite/mariabackup/disabled.def | 2 + .../encrypted_page_compressed.test | 2 +- .../encrypted_page_corruption.test | 4 +- .../suite/mariabackup/extra_lsndir.test | 2 +- .../suite/mariabackup/full_backup.result.orig | 38 + .../suite/mariabackup/full_backup.result.rej | 10 + mysql-test/suite/mariabackup/full_backup.test | 2 +- .../suite/mariabackup/full_backup.test.orig | 65 ++ .../suite/mariabackup/full_backup.test.rej | 10 + mysql-test/suite/mariabackup/huge_lsn.test | 2 +- .../suite/mariabackup/huge_lsn.test.orig | 114 ++ .../mariabackup/incremental_encrypted.test | 4 +- .../innodb_ddl_on_intermediate_table.result | 5 + .../innodb_ddl_on_intermediate_table.test | 18 + .../suite/mariabackup/lock_ddl_per_table.test | 2 +- .../mariabackup/log_checksum_mismatch.test | 2 +- ...ile_unexpected_large_number_in_name.result | 20 + ..._file_unexpected_large_number_in_name.test | 47 + .../suite/mariabackup/log_tables.result | 24 + mysql-test/suite/mariabackup/log_tables.test | 49 + mysql-test/suite/mariabackup/mdev-14447.test | 2 +- mysql-test/suite/mariabackup/missing_ibd.test | 2 +- .../nolock_ddl_during_backup_end.test | 2 +- mysql-test/suite/mariabackup/partial.test | 2 +- .../suite/mariabackup/partial_exclude.test | 2 +- .../suite/mariabackup/partition_datadir.test | 2 +- .../suite/mariabackup/partition_partial.test | 2 +- .../mariabackup/rename_during_mdl_lock.test | 2 +- .../suite/mariabackup/skip_innodb.test.orig | 13 + .../suite/mariabackup/skip_innodb.test.rej | 17 + mysql-test/suite/mariabackup/small_ibd.test | 2 +- .../std_data/ment1587_aria_log.00000004 | 0 .../suite/mariabackup/system_versioning.test | 4 +- .../mariabackup/truncate_during_backup.test | 2 +- .../suite/mariabackup/undo_space_id.test | 2 +- .../mariabackup/undo_upgrade.result.orig | 19 + .../suite/mariabackup/undo_upgrade.result.rej | 22 + .../suite/mariabackup/undo_upgrade.test.orig | 50 + .../suite/mariabackup/undo_upgrade.test.rej | 53 + .../unencrypted_page_compressed.result.orig | 10 + .../unencrypted_page_compressed.result.rej | 8 + .../unencrypted_page_compressed.test | 2 +- .../unencrypted_page_compressed.test.orig | 50 + .../unencrypted_page_compressed.test.rej | 8 + .../suite/mariabackup/unsupported_redo.test | 8 +- .../mariabackup/xb_aws_key_management.test | 2 +- .../mariabackup/xb_file_key_management.test | 2 +- .../xb_file_key_management.test.orig | 33 + mysql-test/suite/mariabackup/xb_history.test | 2 +- .../suite/mariabackup/xb_history.test.orig | 27 + .../suite/mariabackup/xb_page_compress.test | 2 +- .../suite/mariabackup/xb_partition.test | 2 +- mysql-test/suite/mariabackup/xb_rocksdb.test | 4 +- .../suite/mariabackup/xb_rocksdb_datadir.test | 2 +- mysql-test/suite/mariabackup/xbstream.test | 2 +- 116 files changed, 6251 insertions(+), 827 deletions(-) create mode 100644 extra/mariabackup/aria_backup_client.cc create mode 100644 extra/mariabackup/aria_backup_client.h create mode 100644 extra/mariabackup/common_engine.cc create mode 100644 extra/mariabackup/common_engine.h create mode 100644 extra/mariabackup/ddl_log.cc create mode 100644 extra/mariabackup/ddl_log.h rename extra/mariabackup/{xb_plugin.cc => encryption_plugin.cc} (83%) create mode 100644 extra/mariabackup/encryption_plugin.h create mode 100644 extra/mariabackup/thread_pool.cc create mode 100644 extra/mariabackup/thread_pool.h delete mode 100644 extra/mariabackup/xb_plugin.h create mode 100644 mysql-test/include/aria_log_control_load.inc create mode 100644 mysql-test/suite/mariabackup/aria_backup.opt create mode 100644 mysql-test/suite/mariabackup/aria_backup.result create mode 100644 mysql-test/suite/mariabackup/aria_backup.test create mode 100644 mysql-test/suite/mariabackup/aria_log.opt create mode 100644 mysql-test/suite/mariabackup/aria_log_rotate_during_backup.opt create mode 100644 mysql-test/suite/mariabackup/aria_log_rotate_during_backup.result create mode 100644 mysql-test/suite/mariabackup/aria_log_rotate_during_backup.test create mode 100644 mysql-test/suite/mariabackup/ddl_for_common_engine.result create mode 100644 mysql-test/suite/mariabackup/ddl_for_common_engine.test create mode 100644 mysql-test/suite/mariabackup/full_backup.result.orig create mode 100644 mysql-test/suite/mariabackup/full_backup.result.rej create mode 100644 mysql-test/suite/mariabackup/full_backup.test.orig create mode 100644 mysql-test/suite/mariabackup/full_backup.test.rej create mode 100644 mysql-test/suite/mariabackup/huge_lsn.test.orig create mode 100644 mysql-test/suite/mariabackup/innodb_ddl_on_intermediate_table.result create mode 100644 mysql-test/suite/mariabackup/innodb_ddl_on_intermediate_table.test create mode 100644 mysql-test/suite/mariabackup/log_file_unexpected_large_number_in_name.result create mode 100644 mysql-test/suite/mariabackup/log_file_unexpected_large_number_in_name.test create mode 100644 mysql-test/suite/mariabackup/log_tables.result create mode 100644 mysql-test/suite/mariabackup/log_tables.test create mode 100644 mysql-test/suite/mariabackup/skip_innodb.test.orig create mode 100644 mysql-test/suite/mariabackup/skip_innodb.test.rej create mode 100644 mysql-test/suite/mariabackup/std_data/ment1587_aria_log.00000004 create mode 100644 mysql-test/suite/mariabackup/undo_upgrade.result.orig create mode 100644 mysql-test/suite/mariabackup/undo_upgrade.result.rej create mode 100644 mysql-test/suite/mariabackup/undo_upgrade.test.orig create mode 100644 mysql-test/suite/mariabackup/undo_upgrade.test.rej create mode 100644 mysql-test/suite/mariabackup/unencrypted_page_compressed.result.orig create mode 100644 mysql-test/suite/mariabackup/unencrypted_page_compressed.result.rej create mode 100644 mysql-test/suite/mariabackup/unencrypted_page_compressed.test.orig create mode 100644 mysql-test/suite/mariabackup/unencrypted_page_compressed.test.rej create mode 100644 mysql-test/suite/mariabackup/xb_file_key_management.test.orig create mode 100644 mysql-test/suite/mariabackup/xb_history.test.orig diff --git a/extra/mariabackup/CMakeLists.txt b/extra/mariabackup/CMakeLists.txt index 9294b390f60..b8d23d4c2e7 100644 --- a/extra/mariabackup/CMakeLists.txt +++ b/extra/mariabackup/CMakeLists.txt @@ -31,6 +31,7 @@ ENDIF() INCLUDE_DIRECTORIES( ${CMAKE_SOURCE_DIR}/include ${CMAKE_SOURCE_DIR}/sql + ${CMAKE_SOURCE_DIR}/storage/maria ${CMAKE_CURRENT_SOURCE_DIR}/quicklz ${CMAKE_CURRENT_SOURCE_DIR} ) @@ -71,8 +72,12 @@ MYSQL_ADD_EXECUTABLE(mariadb-backup xbstream_write.cc backup_mysql.cc backup_copy.cc - xb_plugin.cc + encryption_plugin.cc ${PROJECT_BINARY_DIR}/sql/sql_builtin.cc + aria_backup_client.cc + thread_pool.cc + ddl_log.cc + common_engine.cc ${PROJECT_SOURCE_DIR}/sql/net_serv.cc ${PROJECT_SOURCE_DIR}/libmysqld/libmysql.c COMPONENT backup @@ -81,7 +86,8 @@ MYSQL_ADD_EXECUTABLE(mariadb-backup # Export all symbols on Unix, for better crash callstacks SET_TARGET_PROPERTIES(mariadb-backup PROPERTIES ENABLE_EXPORTS TRUE) -TARGET_LINK_LIBRARIES(mariadb-backup sql sql_builtins) +TARGET_LINK_LIBRARIES(mariadb-backup sql sql_builtins aria) + IF(NOT HAVE_SYSTEM_REGEX) TARGET_LINK_LIBRARIES(mariadb-backup pcre2-posix) ENDIF() diff --git a/extra/mariabackup/aria_backup_client.cc b/extra/mariabackup/aria_backup_client.cc new file mode 100644 index 00000000000..1ea1486d7cb --- /dev/null +++ b/extra/mariabackup/aria_backup_client.cc @@ -0,0 +1,1016 @@ +#include +#include +extern "C" { +#include "maria_def.h" +} +#undef LSN_MAX +#include "aria_backup_client.h" +#include "backup_copy.h" +#include "common.h" +#include "sql_table.h" +#include "ma_checkpoint.h" +#include "ma_recovery.h" +#include "backup_debug.h" +#include "aria_backup.h" +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include + +namespace aria { + +const char *log_preffix = "aria_log."; + + +static std::string log_file_name_only(size_t log_num) { + std::string log_file; + { + std::stringstream ss; + ss << std::setw(8) << std::setfill('0') << log_num; + log_file.append(log_preffix).append(ss.str()); + } + return log_file; +} + + +static std::string log_file_name(const char *datadir_path, size_t log_num) { + std::string log_file(datadir_path); + return log_file.append("/").append(log_file_name_only(log_num)); +} + + +class LogFileCollection +{ + uint32 m_first; + uint32 m_count; +public: + uint32 first() const { return m_first; } + uint32 count() const { return m_count; } + uint32 last() const + { + DBUG_ASSERT(m_count > 0); + return m_first + m_count - 1; + } + + // Initialize by checking existing log files on the disk + LogFileCollection(const char *datadir, uint32 max_log_no) + { + uint32 end= find_greatest_existing_log(datadir, max_log_no); + if (!end) + { + // No log files were found at all + m_first= 0; + m_count= 0; + } + else if (end == 1) + { + // Just the very first one log file (aria_log.00000001) was found. + m_first= 1; + m_count= 1; + } + else + { + // Multiple files were found + m_first= find_greatest_missing_log(datadir, end - 1) + 1; + m_count= 1 + end - m_first; + } + } + + /* + Skip all missing log files and find the greatest existing log file, or + Skip all existing log files and find the greatest missing log file. + + @param datadir - Search files in this directory + @param start - Start searching from this log number and go downto 1. + @param kind - true - search for an existing file + false - search for a missing file. + @returns - [1..start] - the greatest found log file + of the searched kind + - 0 - if no log files of this kind + were found in the range [1..start]. + */ + static uint32 find_greatest_existing_or_missing_log(const char *datadir, + uint32 start, + bool kind) + { + DBUG_ASSERT(start > 0); + for (uint32 i= start; i > 0; i--) + { + if (file_exists(log_file_name(datadir, i).c_str()) == kind) + return i; + } + return 0; // No log files of the searched kind were found + } + + static uint32 find_greatest_existing_log(const char *datadir, uint32 start) + { + return find_greatest_existing_or_missing_log(datadir, start, true); + } + + static uint32 find_greatest_missing_log(const char *datadir, uint32 start) + { + return find_greatest_existing_or_missing_log(datadir, start, false); + } + + /* + In some scenarios (e.g. log rotate) some new log files can appear + outside of the initially assumed [first,last] log number range. + This function adds all extra files behind "last". + */ + void find_logs_after_last(const char *datadir) + { + DBUG_ASSERT(m_count > 0); + for ( ; + file_exists(log_file_name(datadir, last() + 1).c_str()) ; + m_count++) + { } + } + + void report_found(unsigned thread_num) const + { + if (m_count) + msg(thread_num, + "Found %u aria log files, " + "minimum log number %u, " + "maximum log number %u", + m_count, m_first, last()); + } + + void die_if_missing(uint32 logno) const + { + DBUG_ASSERT(logno > 0); + if (!m_count || m_first > logno || last() < logno) + die("Aria log file %u does not exists.", logno); + } +}; + + +class Table { +public: + struct Partition { + std::string m_file_path; + File m_index_file = -1; + MY_STAT m_index_file_stat; + File m_data_file = -1; + MY_STAT m_data_file_stat; + }; + Table() = default; + Table (Table &&other) = delete; + Table & operator= (Table &&other) = delete; + Table(const Table &) = delete; + Table & operator= (const Table &) = delete; + ~Table(); + bool init(const char *data_file_path); + bool open(MYSQL *con, bool opt_no_lock, unsigned thread_num); + bool close(); + bool copy(ds_ctxt_t *ds, unsigned thread_num); + + bool is_online_backup_safe() const { + DBUG_ASSERT(is_opened()); + return m_cap.online_backup_safe; + } + bool is_stats() const { + return is_stats_table(m_db.c_str(), m_table.c_str()); + } + bool is_log() const { + return is_log_table(m_db.c_str(), m_table.c_str()); + } + bool is_opened() const { + return !m_partitions.empty() && + m_partitions[0].m_index_file >= 0 && m_partitions[0].m_data_file >= 0; + }; + std::string &get_full_name() { + return m_full_name; + } + std::string &get_db() { return m_db; } + std::string &get_table() { return m_table; } + std::string &get_version() { return m_table_version; } + bool is_partitioned() const { return m_partitioned; } + void add_partition(const Table &partition) { + DBUG_ASSERT(is_partitioned()); + m_partitions.push_back(partition.m_partitions[0]); + } +#ifndef DBUG_OFF + const std::string& get_sql_name() const { return m_sql_name; } +#endif //DBUG_OFF +private: + + bool copy(ds_ctxt_t *ds, bool is_index, unsigned thread_num); + // frm and par files will be copied under BLOCK_DDL stage in + // backup_copy_non_system() + bool copy_frm_and_par(ds_ctxt_t *ds, unsigned thread_num); + bool read_table_version_id(File file); + + std::string m_db; + std::string m_table; + std::string m_full_name; + std::string m_frm_par_path; + std::string m_table_version; +#ifndef DBUG_OFF + std::string m_sql_name; +#endif //DBUG_OFF + bool m_partitioned = false; + std::vector m_partitions; + ARIA_TABLE_CAPABILITIES m_cap; +}; + +Table::~Table() { + (void)close(); +} + +bool Table::init(const char *data_file_path) { + DBUG_ASSERT(data_file_path); + + const char *ext_pos = strrchr(data_file_path, '.'); + if (!ext_pos) + return false; + + char db_name_orig[FN_REFLEN]; + char table_name_orig[FN_REFLEN]; + parse_db_table_from_file_path( + data_file_path, db_name_orig, table_name_orig); + if (!db_name_orig[0] || !table_name_orig[0]) + return false; + char db_name_conv[FN_REFLEN]; + char table_name_conv[FN_REFLEN]; + filename_to_tablename(db_name_orig, db_name_conv, sizeof(db_name_conv)); + filename_to_tablename( + table_name_orig, table_name_conv, sizeof(table_name_conv)); + if (!db_name_conv[0] || !table_name_conv[0]) + return false; + + if (strstr(data_file_path, "#P#")) + m_partitioned = true; + + const char *table_name_begin = strrchr(data_file_path, FN_LIBCHAR); + if (!table_name_begin) + return false; + m_frm_par_path.assign(data_file_path, table_name_begin + 1). + append(table_name_orig); + + m_db.assign(db_name_conv); + m_table.assign(table_name_conv); + // TODO: find the correct way to represent quoted table/db names + m_full_name.assign("`").append(m_db).append("`.`"). + append(m_table).append("`"); +#ifndef DBUG_OFF + m_sql_name.assign(m_db).append("/").append(m_table); +#endif // DBUG_OFF + Partition partition; + partition.m_file_path.assign(data_file_path, ext_pos - data_file_path); + m_partitions.push_back(std::move(partition)); + return true; +} + +bool Table::read_table_version_id(File file) { + m_table_version = ::read_table_version_id(file); + return m_table_version.empty(); +} + +bool Table::open(MYSQL *con, bool opt_no_lock, unsigned thread_num) { + int error= 1; + bool have_capabilities = false; + File frm_file = -1; + + if (!opt_no_lock && !backup_lock(con, m_full_name.c_str())) { + msg(thread_num, "Error on BACKUP LOCK for aria table %s", + m_full_name.c_str()); + goto exit; + } + + for (Partition &partition : m_partitions) { + std::string file_path = partition.m_file_path + ".MAI"; + if ((partition.m_index_file= my_open(file_path.c_str(), + O_RDONLY | O_SHARE | O_NOFOLLOW | O_CLOEXEC, + MYF(MY_WME))) < 0) { + msg(thread_num, "Error on aria table file open %s", file_path.c_str()); + goto exit; + } + if (!my_stat(file_path.c_str(), &partition.m_index_file_stat, MYF(0))) { + msg(thread_num, "Error on aria table file stat %s", file_path.c_str()); + goto exit; + } + if (!have_capabilities) { + if ((error= aria_get_capabilities(partition.m_index_file, &m_cap))) { + msg(thread_num, "aria_get_capabilities failed: %d", error); + goto exit; + } + have_capabilities = true; + } + + file_path = partition.m_file_path + ".MAD"; + if ((partition.m_data_file= my_open(file_path.c_str(), + O_RDONLY | O_SHARE | O_NOFOLLOW | O_CLOEXEC, MYF(MY_WME))) < 0) { + msg(thread_num, "Error on aria table file open %s", file_path.c_str()); + goto exit; + } + if (!my_stat(file_path.c_str(), &partition.m_data_file_stat, MYF(0))) { + msg(thread_num, "Error on aria table file stat %s", file_path.c_str()); + goto exit; + } + } + + if ((frm_file = mysql_file_open( + key_file_frm, (m_frm_par_path + ".frm").c_str(), + O_RDONLY | O_SHARE, MYF(0))) < 0) { + msg(thread_num, "Error on aria table %s file open", + (m_frm_par_path + ".frm").c_str()); + goto exit; + } + + error = 0; + +exit: + if (!opt_no_lock && !backup_unlock(con)) { + msg(thread_num, "Error on BACKUP UNLOCK for aria table %s", + m_full_name.c_str()); + error = 1; + } + if (error) + (void)close(); + else { + (void)read_table_version_id(frm_file); + mysql_file_close(frm_file, MYF(MY_WME)); + } + return !error; +} + +bool Table::close() { + for (Partition &partition : m_partitions) { + if (partition.m_index_file >= 0) { + my_close(partition.m_index_file, MYF(MY_WME)); + partition.m_index_file = -1; + } + if (partition.m_data_file >= 0) { + my_close(partition.m_data_file, MYF(MY_WME)); + partition.m_data_file = -1; + } + } + return true; +} + +bool Table::copy(ds_ctxt_t *ds, unsigned thread_num) { + DBUG_ASSERT(is_opened()); + DBUG_MARIABACKUP_EVENT_LOCK("before_aria_table_copy", + fil_space_t::name_type(m_sql_name.data(), m_sql_name.size())); + bool result = +// copy_frm_and_par(ds, thread_num) && + copy(ds, true, thread_num) && copy(ds, false, thread_num); + return result; +} + +bool Table::copy(ds_ctxt_t *ds, bool is_index, unsigned thread_num) { + DBUG_ASSERT(ds); + const char *ext = is_index ? ".MAI" : ".MAD"; + int error= 1; + for (const Partition &partition : m_partitions) { + ds_file_t *dst_file = nullptr; + uchar *copy_buffer = nullptr; + std::string full_name = partition.m_file_path + ext; + const char *dst_path = + (xtrabackup_copy_back || xtrabackup_move_back) ? + full_name.c_str() : trim_dotslash(full_name.c_str()); + + dst_file = ds_open(ds, dst_path, + is_index ? &partition.m_index_file_stat : &partition.m_data_file_stat); + if (!dst_file) { + msg(thread_num, "error: cannot open the destination stream for %s", + dst_path); + goto err; + } + + copy_buffer = + reinterpret_cast(my_malloc(PSI_NOT_INSTRUMENTED, + m_cap.block_size, MYF(0))); + + DBUG_MARIABACKUP_EVENT_LOCK( + is_index ? + "before_aria_index_file_copy": + "before_aria_data_file_copy", + fil_space_t::name_type(m_sql_name.data(), + m_sql_name.size())); + + for (ulonglong block= 0 ; ; block++) { + size_t length = m_cap.block_size; + if (is_index) { + if ((error= aria_read_index( + partition.m_index_file, &m_cap, block, copy_buffer) == + HA_ERR_END_OF_FILE)) + break; + } else { + if ((error= aria_read_data( + partition.m_data_file, &m_cap, block, copy_buffer, &length) == + HA_ERR_END_OF_FILE)) + break; + } + if (error) { + msg(thread_num, "error: aria_read %s failed: %d", + is_index ? "index" : "data", error); + goto err; + } + xtrabackup_io_throttling(); + if ((error = ds_write(dst_file, copy_buffer, length))) { + msg(thread_num, "error: aria_write failed: %d", error); + goto err; + } + } + + DBUG_MARIABACKUP_EVENT_LOCK( + is_index ? + "after_aria_index_file_copy": + "after_aria_data_file_copy", + fil_space_t::name_type(m_sql_name.data(), + m_sql_name.size())); + + error = 0; + msg(thread_num, "aria table file %s is copied successfully.", + full_name.c_str()); + + err: + if (dst_file) + ds_close(dst_file); + if (copy_buffer) + my_free(copy_buffer); + if (error) + break; + } + return !error; +} + +class BackupImpl { +public: + BackupImpl( + const char *datadir_path, + const char *aria_log_path, + ds_ctxt_t *datasink, bool opt_no_lock, + std::vector &con_pool, ThreadPool &thread_pool) : + m_datadir_path(datadir_path), + m_aria_log_dir_path(aria_log_path), + m_ds(datasink), m_con_pool(con_pool), + m_tasks_group(thread_pool), m_thread_pool(thread_pool) { } + ~BackupImpl() { destroy(); } + bool init(); + bool start(bool no_lock); + bool wait_for_finish(); + bool copy_offline_tables( + const std::unordered_set *exclude_tables, bool no_lock, + bool copy_stats); + bool finalize(); + void set_post_copy_table_hook(const post_copy_table_hook_t &hook) { + m_table_post_copy_hook = hook; + } + bool copy_log_tail() { return copy_log_tail(0, false); } +private: + void destroy(); + void scan_job(bool no_lock, unsigned thread_num); + bool copy_log_tail(unsigned thread_num, bool finalize); + void copy_log_file_job(size_t log_num, unsigned thread_num); + void destroy_log_tail(); + void process_table_job(Table *table, bool online_only, bool copy_stats, + bool no_lock, unsigned thread_num); + + const char *m_datadir_path; + const char *m_aria_log_dir_path; + std::string aria_log_dir_path() const + { + if (!m_aria_log_dir_path || !m_aria_log_dir_path[0]) + return m_datadir_path; + if (is_absolute_path(m_aria_log_dir_path)) + return m_aria_log_dir_path; + return std::string(m_datadir_path).append("/") + .append(m_aria_log_dir_path); + } + ds_ctxt_t *m_ds; + std::vector &m_con_pool; + + TasksGroup m_tasks_group; + + std::mutex m_offline_tables_mutex; + std::vector> m_offline_tables; + post_copy_table_hook_t m_table_post_copy_hook; + + ThreadPool &m_thread_pool; + + size_t m_last_log_num = 0; + ds_file_t* m_last_log_dst = nullptr; + File m_last_log_src = -1; +}; + +bool BackupImpl::init() { + DBUG_ASSERT(m_tasks_group.is_finished()); + return true; +}; + +void BackupImpl::destroy() { + DBUG_ASSERT(m_tasks_group.is_finished()); + destroy_log_tail(); +} + +bool BackupImpl::start(bool no_lock) { + DBUG_ASSERT(m_tasks_group.is_finished()); + m_tasks_group.push_task( + std::bind(&BackupImpl::scan_job, this, no_lock, std::placeholders::_1)); + return true; +} + +void BackupImpl::process_table_job( + Table *table_ptr, bool online_only, bool copy_stats, bool no_lock, + unsigned thread_num) { + DBUG_ASSERT(table_ptr); + DBUG_ASSERT(thread_num < m_con_pool.size()); + std::unique_ptr table(table_ptr); + bool is_online; + bool is_stats; + bool need_copy; + int result = 1; + + if (!m_tasks_group.get_result()) + goto exit; + + if (!table->open(m_con_pool[thread_num], no_lock, thread_num)) { + // if table can't be opened, it might be removed or renamed, this is not + // error for transactional tables + table->close(); // Close opened table files + goto exit; + } + + is_online = table->is_online_backup_safe(); + is_stats = table->is_stats(); + + need_copy = (!online_only || is_online) && (copy_stats || !is_stats); + + if (need_copy && !table->copy(m_ds, thread_num)) { + table->close(); + DBUG_MARIABACKUP_EVENT_LOCK("after_aria_table_copy", + fil_space_t::name_type(table->get_sql_name().data(), + table->get_sql_name().size())); + // if table is opened, it must be copied, + // the corresponding diagnostic messages must be issued in Table::copy() + result = 0; + goto exit; + } + + if (!table->close()) { + msg(thread_num, "Can't close aria table %s.\n", + table->get_full_name().c_str()); + result = 0; + goto exit; + } + + if (!need_copy) { + std::lock_guard lock(m_offline_tables_mutex); + m_offline_tables.push_back(std::move(table)); + } + else { + DBUG_MARIABACKUP_EVENT_LOCK("after_aria_table_copy", + fil_space_t::name_type(table->get_sql_name().data(), + table->get_sql_name().size())); + if (m_table_post_copy_hook) + m_table_post_copy_hook( + std::move(table->get_db()), + std::move(table->get_table()), + std::move(table->get_version())); + } +exit: + m_tasks_group.finish_task(result); +} + + +void BackupImpl::scan_job(bool no_lock, unsigned thread_num) { + std::unordered_map> partitioned_tables; + + std::string aria_log_dir_path_cache(aria_log_dir_path()); + std::string log_control_file_path(aria_log_dir_path_cache); + log_control_file_path.append("/aria_log_control"); + if (!m_ds->copy_file( + log_control_file_path.c_str(), "aria_log_control", + 0, false)) { + msg("Aria log control file copying error."); + m_tasks_group.finish_task(0); + return; + } + + msg(thread_num, "Loading aria_log_control."); + aria_readonly= 1; + maria_data_root= aria_log_dir_path_cache.c_str(); + if (ma_control_file_open(FALSE, FALSE, FALSE, O_RDONLY)) + die("Can't open Aria control file (%d)", errno); + uint32 aria_log_control_last_log_number= last_logno; + msg(thread_num, "aria_log_control: last_log_number: %d", + aria_log_control_last_log_number); + ma_control_file_end(); + + msg(thread_num, "Start scanning aria tables."); + + foreach_file_in_db_dirs(m_datadir_path, [&](const char *file_path)->bool { + + if (check_if_skip_table(file_path)) { + msg(thread_num, "Skipping %s.", file_path); + return true; + } + + if (!ends_with(file_path, ".MAD")) + return true; + + std::unique_ptr
table(new Table()); + if (!table->init(file_path)) { + msg(thread_num, "Can't init aria table %s.\n", file_path); + return true; + } + + if (table->is_log()) + return true; + + if (table->is_partitioned()) { + auto table_it = partitioned_tables.find(table->get_full_name()); + if (table_it == partitioned_tables.end()) { + partitioned_tables[table->get_full_name()] = std::move(table); + } else { + table_it->second->add_partition(*table); + } + return true; + } + + m_tasks_group.push_task( + std::bind(&BackupImpl::process_table_job, this, table.release(), true, + false, no_lock, std::placeholders::_1)); + return true; + }); + + for (auto &table_it : partitioned_tables) { + m_tasks_group.push_task( + std::bind(&BackupImpl::process_table_job, this, table_it.second.release(), + true, false, no_lock, std::placeholders::_1)); + } + + msg(thread_num, "Start scanning aria log files."); + + LogFileCollection logs(aria_log_dir_path_cache.c_str(), + aria_log_control_last_log_number); + logs.report_found(thread_num); + logs.die_if_missing(aria_log_control_last_log_number); + + m_last_log_num= logs.last(); + + DBUG_MARIABACKUP_EVENT("after_scanning_log_files", {}); + + for (uint32 i= logs.first(); i <= logs.last(); ++i) + m_tasks_group.push_task( + std::bind(&BackupImpl::copy_log_file_job, this, + i, std::placeholders::_1)); + + msg(thread_num, "Stop scanning aria tables."); + + m_tasks_group.finish_task(1); +} + +template +T align_down(T n, ulint align_no) +{ + DBUG_ASSERT(align_no > 0); + DBUG_ASSERT(ut_is_2pow(align_no)); + return n & ~(static_cast(align_no) - 1); +} + +static ssize_t copy_file_chunk(File src, ds_file_t* dst, size_t size) { + size_t bytes_read; + static const size_t max_buf_size = 10 * 1024 * 1024; + size_t buf_size = size ? std::min(size, max_buf_size) : max_buf_size; + std::unique_ptr buf(new uchar[buf_size]); + ssize_t copied_size = 0; + bool unlim = !size; + while((unlim || size) && (bytes_read = my_read(src, buf.get(), + unlim ? buf_size : std::min(buf_size, size), MY_WME))) { + if (bytes_read == size_t(-1)) + return -1; + xtrabackup_io_throttling(); + if (ds_write(dst, buf.get(), bytes_read)) + return -1; + copied_size += bytes_read; + if (!unlim) + size -= bytes_read; + } + return copied_size; +} + +bool BackupImpl::copy_log_tail(unsigned thread_num, bool finalize) { + bool result = false; + std::string log_file = log_file_name(aria_log_dir_path().c_str(), m_last_log_num); + std::string prev_log_file; + ssize_t total_bytes_copied = 0; + MY_STAT stat_info; + my_off_t file_offset = 0; + size_t to_copy_size = 0; + +repeat: + memset(&stat_info, 0, sizeof(MY_STAT)); + if (!m_tasks_group.get_result()) { + msg(thread_num, "Skip copying aria lof file tail %s due to error.", + log_file.c_str()); + result = true; + goto exit; + } + + msg(thread_num, "Start copying aria log file tail: %s", log_file.c_str()); + + if (m_last_log_src < 0 && (m_last_log_src = + my_open(log_file.c_str(), O_RDONLY | O_SHARE | O_NOFOLLOW | O_CLOEXEC, + MYF(MY_WME))) < 0) { + msg("Aria log file %s open failed: %d", log_file.c_str(), my_errno); + goto exit; + } + + if (!m_last_log_dst && + !(m_last_log_dst = ds_open(m_ds, + log_file_name_only(m_last_log_num).c_str(), + &stat_info, false))) { + msg(thread_num, "error: failed to open the target stream for " + "aria log file %s.", + log_file.c_str()); + goto exit; + } + +// If there is no need to finalize log file copying, calculate the size to copy +// without the last page, which can be rewritten by the server +// (see translog_force_current_buffer_to_finish()). + if (!finalize) { + if (my_fstat(m_last_log_src, &stat_info, MYF(0))) { + msg(thread_num, "error: failed to get file size for aria log file: %s.", + log_file.c_str()); + goto exit; + } + if ((file_offset = my_tell(m_last_log_src, MYF(0))) == (my_off_t)(-1)) { + msg(thread_num, "error: failed to get file offset for aria log file: %s.", + log_file.c_str()); + goto exit; + } + DBUG_ASSERT(file_offset <= static_cast(stat_info.st_size)); + to_copy_size = static_cast(stat_info.st_size) - file_offset; + to_copy_size = to_copy_size >= TRANSLOG_PAGE_SIZE ? + (align_down(to_copy_size, TRANSLOG_PAGE_SIZE) - TRANSLOG_PAGE_SIZE) : 0; + } + +// Copy from the last position to the end of file, +// excluding the last page is there is no need to finalize the copy. + if ((to_copy_size || finalize) && + (total_bytes_copied = copy_file_chunk(m_last_log_src, + m_last_log_dst, to_copy_size)) < 0) { + msg(thread_num, "Aria log file %s chunk copy error", log_file.c_str()); + goto exit; + } + + msg(thread_num, "Stop copying aria log file tail: %s, copied %zu bytes", + log_file.c_str(), total_bytes_copied); + +// Check if there is new log file, if yes, then copy the last page of the old +// one, and fix it last LSN in the log header, as it is changed on new +// log file creating by the server (see translog_create_new_file() and +// translog_max_lsn_to_header()). Then close the old log file and repeat +// the copying for the new log file. + prev_log_file = std::move(log_file); + log_file = log_file_name(aria_log_dir_path().c_str(), m_last_log_num + 1); + if (file_exists(log_file.c_str())) { + uchar lsn_buff[LSN_STORE_SIZE]; + msg(thread_num, "Found new aria log tail file: %s, start copy %s tail", + log_file.c_str(), prev_log_file.c_str()); + if ((total_bytes_copied = copy_file_chunk(m_last_log_src, + m_last_log_dst, 0)) < 0) { + msg(thread_num, "Aria log file %s tail copy error", + prev_log_file.c_str()); + goto exit; + } + + if (my_pread(m_last_log_src, lsn_buff, LSN_STORE_SIZE, + (LOG_HEADER_DATA_SIZE - LSN_STORE_SIZE), MYF(0)) < LSN_STORE_SIZE) { + msg(thread_num, "Aria lsn store read error for log file %s", + prev_log_file.c_str()); + goto exit; + } + + if (ds_seek_set(m_last_log_dst, (LOG_HEADER_DATA_SIZE - LSN_STORE_SIZE))) { + msg(thread_num, "Set aria log pointer error for log file %s", + prev_log_file.c_str()); + goto exit; + } + + if (ds_write(m_last_log_dst, lsn_buff, LSN_STORE_SIZE)) { + msg(thread_num, "LSN write error for aria log file %s", + prev_log_file.c_str()); + goto exit; + } + + msg(thread_num, "The last %zu bytes were copied for %s.", + total_bytes_copied, prev_log_file.c_str()); + destroy_log_tail(); + ++m_last_log_num; + goto repeat; + } + + result = true; + +exit: + if (!result) + destroy_log_tail(); + return result; +} + +void BackupImpl::copy_log_file_job(size_t log_num, unsigned thread_num) { + DBUG_ASSERT(log_num <= m_last_log_num); + + if (!m_tasks_group.get_result()) { + msg(thread_num, "Skip copying %zu aria log file due to error", log_num); + m_tasks_group.finish_task(0); + return; + } + +// Copy log file if the file is not the last one. + if (log_num < m_last_log_num) { + std::string log_file = log_file_name(aria_log_dir_path().c_str(), log_num); + if (!m_ds->copy_file(log_file.c_str(), + log_file_name_only(log_num).c_str(), + thread_num, false)) { + msg(thread_num, "Error on copying %s aria log file.", log_file.c_str()); + m_tasks_group.finish_task(0); + } + else + m_tasks_group.finish_task(1); + return; + } +// Copy the last log file. + m_tasks_group.finish_task(copy_log_tail(thread_num, false) ? 1 : 0); +} + +void BackupImpl::destroy_log_tail() { + if (m_last_log_src >= 0) { + my_close(m_last_log_src, MYF(MY_WME)); + m_last_log_src = -1; + } + if (m_last_log_dst) { + ds_close(m_last_log_dst); + m_last_log_dst = nullptr; + } +} + +bool BackupImpl::wait_for_finish() { + return m_tasks_group.wait_for_finish(); +} + +bool BackupImpl::copy_offline_tables( + const std::unordered_set *exclude_tables, bool no_lock, + bool copy_stats) { + DBUG_ASSERT(m_tasks_group.is_finished()); + + std::vector> ignored_tables; + + while (true) { + std::unique_lock lock(m_offline_tables_mutex); + if (m_offline_tables.empty()) + break; + auto table = std::move(m_offline_tables.back()); + m_offline_tables.pop_back(); + lock.unlock(); + if ((exclude_tables && + exclude_tables->count(table_key(table->get_db(), table->get_table()))) || + (!copy_stats && table->is_stats())) { + ignored_tables.push_back(std::move(table)); + continue; + } + m_tasks_group.push_task( + std::bind(&BackupImpl::process_table_job, this, table.release(), false, + copy_stats, no_lock, std::placeholders::_1)); + } + + if (!ignored_tables.empty()) { + std::lock_guard lock(m_offline_tables_mutex); + m_offline_tables = std::move(ignored_tables); + } + + return true; +} + +bool BackupImpl::finalize() { + DBUG_ASSERT(m_tasks_group.is_finished()); + DBUG_ASSERT(!m_con_pool.empty()); + bool result = true; + msg("Start copying statistics aria tables."); + copy_offline_tables(nullptr, true, true); + while (!m_tasks_group.is_finished()) + std::this_thread::sleep_for(std::chrono::milliseconds(1)); + msg("Stop copying statistics aria tables."); + copy_log_tail(0, true); + destroy_log_tail(); + return result; +} + +Backup::Backup(const char *datadir_path, + const char *aria_log_path, + ds_ctxt_t *datasink, + std::vector &con_pool, ThreadPool &thread_pool) : + m_backup_impl( + new BackupImpl(datadir_path, aria_log_path, + datasink, opt_no_lock, con_pool, + thread_pool)) { } + +Backup::~Backup() { + delete m_backup_impl; +} + +bool Backup::init() { + return m_backup_impl->init(); +} + +bool Backup::start(bool no_lock) { + return m_backup_impl->start(no_lock); +} + +bool Backup::wait_for_finish() { + return m_backup_impl->wait_for_finish(); +} + +bool Backup::copy_offline_tables( + const std::unordered_set *exclude_tables, bool no_lock, + bool copy_stats) { + return m_backup_impl->copy_offline_tables(exclude_tables, no_lock, + copy_stats); +} + +bool Backup::finalize() { + return m_backup_impl->finalize(); +} + +bool Backup::copy_log_tail() { + return m_backup_impl->copy_log_tail(); +} + +void Backup::set_post_copy_table_hook(const post_copy_table_hook_t &hook) { + m_backup_impl->set_post_copy_table_hook(hook); +} + +bool prepare(const char *target_dir) { + maria_data_root= (char *)target_dir; + + if (maria_init()) + die("Can't init Aria engine (%d)", errno); + + maria_block_size= 0; /* Use block size from file */ + /* we don't want to create a control file, it MUST exist */ + if (ma_control_file_open(FALSE, TRUE, TRUE, control_file_open_flags)) + die("Can't open Aria control file (%d)", errno); + + if (last_logno == FILENO_IMPOSSIBLE) + die("Can't find any Aria log"); + + LogFileCollection logs(target_dir, last_logno); + logs.die_if_missing(last_logno); // Fatal, a broken backup. + /* + "mariadb-backup --backup" can put extra log files, + with log number greater than last_logno. For example, + this combination of files is possible: + - aria_log_control (with last_logno==1) + - aria_log.00000001 (last_logno) + - aria_log.00000002 (last_logno+1, the extra log file) + This can happen if during the ealier run of + "mariadb-backup --backup" a log rotate happened. + The extra log file is copied to the backup directory, + but last_logno in aria_log_control does not get updated. + This mismatch is probably not good and should eventually be fixed. + But during "mariadb-backup --prepare" this mismatch goes away: + aria_log_control gets fixed to say last_logno==2. + See mysql-test/suite/mariabackup/aria_log_rotate_during_backup.test, + it covers the scenario with one extra file created during --backup. + */ + logs.find_logs_after_last(target_dir); + last_logno= logs.last(); // Update last_logno if extra logs were found + + if (init_pagecache(maria_pagecache, 1024L*1024L, 0, 0, + static_cast(maria_block_size), 0, MY_WME) == 0) + die("Got error in Aria init_pagecache() (errno: %d)", errno); + + if (init_pagecache(maria_log_pagecache, 1024L*1024L, + 0, 0, TRANSLOG_PAGE_SIZE, 0, MY_WME) == 0 || + translog_init(maria_data_root, TRANSLOG_FILE_SIZE, + 0, 0, maria_log_pagecache, TRANSLOG_DEFAULT_FLAGS, FALSE)) + die("Can't init Aria loghandler (%d)", errno); + + if (maria_recovery_from_log()) + die("Aria log apply FAILED"); + + if (maria_recovery_changed_data || recovery_failures) { + if (ma_control_file_write_and_force(last_checkpoint_lsn, last_logno, + max_trid_in_control_file, 0)) + die("Aria control file update error"); +// TODO: find out do we need checkpoint here + } + + maria_end(); + return true; +} + +} // namespace aria diff --git a/extra/mariabackup/aria_backup_client.h b/extra/mariabackup/aria_backup_client.h new file mode 100644 index 00000000000..7a581b5862e --- /dev/null +++ b/extra/mariabackup/aria_backup_client.h @@ -0,0 +1,38 @@ +#pragma once +#include "my_global.h" +#include "datasink.h" +#include "backup_mysql.h" +#include "thread_pool.h" +#include "xtrabackup.h" + +namespace aria { + +bool prepare(const char *target_dir); + +class BackupImpl; + +class Backup { + public: + Backup(const char *datadir_path, + const char *aria_log_path, + ds_ctxt_t *datasink, + std::vector &con_pool, ThreadPool &thread_pool); + ~Backup(); + Backup (Backup &&other) = delete; + Backup & operator= (Backup &&other) = delete; + Backup(const Backup &) = delete; + Backup & operator= (const Backup &) = delete; + bool init(); + bool start(bool no_lock); + bool wait_for_finish(); + bool copy_offline_tables( + const std::unordered_set *exclude_tables, bool no_lock, + bool copy_stats); + bool finalize(); + bool copy_log_tail(); + void set_post_copy_table_hook(const post_copy_table_hook_t &hook); + private: + BackupImpl *m_backup_impl; +}; + +} // namespace aria diff --git a/extra/mariabackup/backup_copy.cc b/extra/mariabackup/backup_copy.cc index f8d315d9eb7..41f7c44bd35 100644 --- a/extra/mariabackup/backup_copy.cc +++ b/extra/mariabackup/backup_copy.cc @@ -41,6 +41,9 @@ Street, Fifth Floor, Boston, MA 02110-1335 USA *******************************************************/ #include +#include +#include +#include #include #include #include @@ -66,19 +69,26 @@ Street, Fifth Floor, Boston, MA 02110-1335 USA #include #endif +#ifdef MYSQL_CLIENT +#define WAS_MYSQL_CLIENT 1 +#undef MYSQL_CLIENT +#endif + +#include "table.h" + +#ifdef WAS_MYSQL_CLIENT +#define MYSQL_CLIENT 1 +#undef WAS_MYSQL_CLIENT +#endif #define ROCKSDB_BACKUP_DIR "#rocksdb" -/* list of files to sync for --rsync mode */ -static std::set rsync_list; /* locations of tablespaces read from .isl files */ static std::map tablespace_locations; /* Whether LOCK BINLOG FOR BACKUP has been issued during backup */ bool binlog_locked; -static void rocksdb_create_checkpoint(); -static bool has_rocksdb_plugin(); static void rocksdb_backup_checkpoint(ds_ctxt *ds_data); static void rocksdb_copy_back(ds_ctxt *ds_data); @@ -135,10 +145,6 @@ struct datadir_thread_ctxt_t { bool ret; }; -static bool backup_files_from_datadir(ds_ctxt_t *ds_data, - const char *dir_path, - const char *prefix); - /************************************************************************ Retirn true if character if file separator */ bool @@ -585,7 +591,6 @@ datafile_read(datafile_cur_t *cursor) Check to see if a file exists. Takes name of the file to check. @return true if file exists. */ -static bool file_exists(const char *filename) { @@ -601,7 +606,6 @@ file_exists(const char *filename) /************************************************************************ Trim leading slashes from absolute path so it becomes relative */ -static const char * trim_dotslash(const char *path) { @@ -634,7 +638,7 @@ ends_with(const char *str, const char *suffix) && strcmp(str + str_len - suffix_len, suffix) == 0); } -static bool starts_with(const char *str, const char *prefix) +bool starts_with(const char *str, const char *prefix) { return strncmp(str, prefix, strlen(prefix)) == 0; } @@ -785,7 +789,6 @@ directory_exists_and_empty(const char *dir, const char *comment) /************************************************************************ Check if file name ends with given set of suffixes. @return true if it does. */ -static bool filename_matches(const char *filename, const char **ext_list) { @@ -800,6 +803,115 @@ filename_matches(const char *filename, const char **ext_list) return(false); } +// TODO: the code can be used to find storage engine of partitions +/* +static +bool is_aria_frm_or_par(const char *path) { + if (!ends_with(path, ".frm") && !ends_with(path, ".par")) + return false; + + const char *frm_path = path; + if (ends_with(path, ".par")) { + size_t frm_path_len = strlen(path); + DBUG_ASSERT(frm_path_len > strlen("frm")); + frm_path = strdup(path); + strcpy(const_cast(frm_path) + frm_path_len - strlen("frm"), "frm"); + } + + bool result = false; + File file; + uchar header[40]; + legacy_db_type dbt; + + if ((file= mysql_file_open(key_file_frm, frm_path, O_RDONLY | O_SHARE, MYF(0))) + < 0) + goto err; + + if (mysql_file_read(file, (uchar*) header, sizeof(header), MYF(MY_NABP))) + goto err; + + if (!strncmp((char*) header, "TYPE=VIEW\n", 10)) + goto err; + + if (!is_binary_frm_header(header)) + goto err; + + dbt = (legacy_db_type)header[3]; + + if (dbt == DB_TYPE_ARIA) { + result = true; + } + else if (dbt == DB_TYPE_PARTITION_DB) { + MY_STAT state; + uchar *frm_image= 0; +// uint n_length; + + if (mysql_file_fstat(file, &state, MYF(MY_WME))) + goto err; + + if (mysql_file_seek(file, 0, SEEK_SET, MYF(MY_WME))) + goto err; + + if (read_string(file, &frm_image, (size_t)state.st_size)) + goto err; + + dbt = (legacy_db_type)frm_image[61]; + if (dbt == DB_TYPE_ARIA) { + result = true; + } + my_free(frm_image); + } + +err: + if (file >= 0) + mysql_file_close(file, MYF(MY_WME)); + if (frm_path != path) + free(const_cast(frm_path)); + return result; +} +*/ + +void parse_db_table_from_file_path( + const char *filepath, char *dbname, char *tablename) { + dbname[0] = '\0'; + tablename[0] = '\0'; + const char *dbname_start = nullptr; + const char *tablename_start = filepath; + const char *const_ptr; + while ((const_ptr = strchr(tablename_start, FN_LIBCHAR)) != NULL) { + dbname_start = tablename_start; + tablename_start = const_ptr + 1; + } + if (!dbname_start) + return; + size_t dbname_len = tablename_start - dbname_start - 1; + if (dbname_len >= FN_REFLEN) + dbname_len = FN_REFLEN-1; + strmake(dbname, dbname_start, dbname_len); + strmake(tablename, tablename_start, FN_REFLEN-1); + char *ptr; + if ((ptr = strchr(tablename, '.'))) + *ptr = '\0'; + if ((ptr = strstr(tablename, "#P#"))) + *ptr = '\0'; +} + +bool is_system_table(const char *dbname, const char *tablename) +{ + DBUG_ASSERT(dbname); + DBUG_ASSERT(tablename); + + LEX_CSTRING lex_dbname; + LEX_CSTRING lex_tablename; + lex_dbname.str = dbname; + lex_dbname.length = strlen(dbname); + lex_tablename.str = tablename; + lex_tablename.length = strlen(tablename); + + TABLE_CATEGORY tg = get_table_category(&lex_dbname, &lex_tablename); + + return (tg == TABLE_CATEGORY_LOG) || (tg == TABLE_CATEGORY_SYSTEM); +} /************************************************************************ Copy data file for backup. Also check if it is allowed to copy by @@ -810,9 +922,8 @@ static bool datafile_copy_backup(ds_ctxt *ds_data, const char *filepath, uint thread_n) { - const char *ext_list[] = {"frm", "isl", "MYD", "MYI", "MAD", "MAI", - "MRG", "TRG", "TRN", "ARM", "ARZ", "CSM", "CSV", "opt", "par", - NULL}; + const char *ext_list[] = {".frm", ".isl", ".TRG", ".TRN", ".opt", ".par", + NULL}; /* Get the name and the path for the tablespace. node->name always contains the path (which may be absolute for remote tablespaces in @@ -830,42 +941,7 @@ datafile_copy_backup(ds_ctxt *ds_data, const char *filepath, uint thread_n) if (filename_matches(filepath, ext_list)) { return ds_data->copy_file(filepath, filepath, thread_n); - } - - return(true); -} - - -/************************************************************************ -Same as datafile_copy_backup, but put file name into the list for -rsync command. */ -static -bool -datafile_rsync_backup(const char *filepath, bool save_to_list, FILE *f) -{ - const char *ext_list[] = {"frm", "isl", "MYD", "MYI", "MAD", "MAI", - "MRG", "TRG", "TRN", "ARM", "ARZ", "CSM", "CSV", "opt", "par", - NULL}; - - /* Get the name and the path for the tablespace. node->name always - contains the path (which may be absolute for remote tablespaces in - 5.6+). space->name contains the tablespace name in the form - "./database/table.ibd" (in 5.5-) or "database/table" (in 5.6+). For a - multi-node shared tablespace, space->name contains the name of the first - node, but that's irrelevant, since we only need node_name to match them - against filters, and the shared tablespace is always copied regardless - of the filters value. */ - - if (check_if_skip_table(filepath)) { - return(true); - } - - if (filename_matches(filepath, ext_list)) { - fprintf(f, "%s\n", filepath); - if (save_to_list) { - rsync_list.insert(filepath); - } - } + } return(true); } @@ -1004,16 +1080,15 @@ Copy file for backup/restore. bool ds_ctxt_t::copy_file(const char *src_file_path, const char *dst_file_path, - uint thread_n) + uint thread_n, + bool rewrite) { char dst_name[FN_REFLEN]; ds_file_t *dstfile = NULL; datafile_cur_t cursor; xb_fil_cur_result_t res; DBUG_ASSERT(datasink->remove); - const char *dst_path = - (xtrabackup_copy_back || xtrabackup_move_back)? - dst_file_path : trim_dotslash(dst_file_path); + const char *dst_path = convert_dst(dst_file_path); if (!datafile_open(src_file_path, &cursor, thread_n)) { goto error_close; @@ -1021,7 +1096,7 @@ ds_ctxt_t::copy_file(const char *src_file_path, strncpy(dst_name, cursor.rel_path, sizeof(dst_name)); - dstfile = ds_open(this, dst_path, &cursor.statinfo); + dstfile = ds_open(this, dst_path, &cursor.statinfo, rewrite); if (dstfile == NULL) { msg(thread_n,"error: " "cannot open the destination stream for %s", dst_name); @@ -1245,278 +1320,45 @@ cleanup: } - - -static bool -backup_files(ds_ctxt *ds_data, const char *from, bool prep_mode) +backup_files(ds_ctxt *ds_data, const char *from) { - char rsync_tmpfile_name[FN_REFLEN]; - FILE *rsync_tmpfile = NULL; datadir_iter_t *it; datadir_node_t node; bool ret = true; - - if (prep_mode && !opt_rsync) { - return(true); - } - - if (opt_rsync) { - snprintf(rsync_tmpfile_name, sizeof(rsync_tmpfile_name), - "%s/%s%d", opt_mysql_tmpdir, - "xtrabackup_rsyncfiles_pass", - prep_mode ? 1 : 2); - rsync_tmpfile = fopen(rsync_tmpfile_name, "w"); - if (rsync_tmpfile == NULL) { - msg("Error: can't create file %s", - rsync_tmpfile_name); - return(false); - } - } - - msg("Starting %s non-InnoDB tables and files", - prep_mode ? "prep copy of" : "to backup"); - + msg("Starting to backup non-InnoDB tables and files"); datadir_node_init(&node); it = datadir_iter_new(from); - while (datadir_iter_next(it, &node)) { - if (!node.is_empty_dir) { - if (opt_rsync) { - ret = datafile_rsync_backup(node.filepath, - !prep_mode, rsync_tmpfile); - } else { - ret = datafile_copy_backup(ds_data, node.filepath, 1); - } + ret = datafile_copy_backup(ds_data, node.filepath, 1); if (!ret) { msg("Failed to copy file %s", node.filepath); goto out; } - } else if (!prep_mode) { + } else { /* backup fake file into empty directory */ char path[FN_REFLEN]; - snprintf(path, sizeof(path), - "%s/db.opt", node.filepath); - if (!(ret = ds_data->backup_file_printf( - trim_dotslash(path), "%s", ""))) { + snprintf(path, sizeof(path), "%s/db.opt", node.filepath); + if (!(ret = ds_data->backup_file_printf(trim_dotslash(path), "%s", ""))) { msg("Failed to create file %s", path); goto out; } } } - - if (opt_rsync) { - std::stringstream cmd; - int err; - - if (buffer_pool_filename && file_exists(buffer_pool_filename)) { - fprintf(rsync_tmpfile, "%s\n", buffer_pool_filename); - rsync_list.insert(buffer_pool_filename); - } - if (file_exists("ib_lru_dump")) { - fprintf(rsync_tmpfile, "%s\n", "ib_lru_dump"); - rsync_list.insert("ib_lru_dump"); - } - - fclose(rsync_tmpfile); - rsync_tmpfile = NULL; - - cmd << "rsync -t . --files-from=" << rsync_tmpfile_name - << " " << xtrabackup_target_dir; - - msg("Starting rsync as: %s", cmd.str().c_str()); - if ((err = system(cmd.str().c_str()) && !prep_mode) != 0) { - msg("Error: rsync failed with error code %d", err); - ret = false; - goto out; - } - msg("rsync finished successfully."); - - if (!prep_mode && !opt_no_lock) { - char path[FN_REFLEN]; - char dst_path[FN_REFLEN]; - char *newline; - - /* Remove files that have been removed between first and - second passes. Cannot use "rsync --delete" because it - does not work with --files-from. */ - snprintf(rsync_tmpfile_name, sizeof(rsync_tmpfile_name), - "%s/%s", opt_mysql_tmpdir, - "xtrabackup_rsyncfiles_pass1"); - - rsync_tmpfile = fopen(rsync_tmpfile_name, "r"); - if (rsync_tmpfile == NULL) { - msg("Error: can't open file %s", - rsync_tmpfile_name); - ret = false; - goto out; - } - - while (fgets(path, sizeof(path), rsync_tmpfile)) { - - newline = strchr(path, '\n'); - if (newline) { - *newline = 0; - } - if (rsync_list.count(path) < 1) { - snprintf(dst_path, sizeof(dst_path), - "%s/%s", xtrabackup_target_dir, - path); - msg("Removing %s", dst_path); - unlink(dst_path); - } - } - - fclose(rsync_tmpfile); - rsync_tmpfile = NULL; - } - } - - msg("Finished %s non-InnoDB tables and files", - prep_mode ? "a prep copy of" : "backing up"); - + msg("Finished backing up non-InnoDB tables and files"); out: datadir_iter_free(it); datadir_node_free(&node); - - if (rsync_tmpfile != NULL) { - fclose(rsync_tmpfile); - } - return(ret); } - -lsn_t get_current_lsn(MYSQL *connection) -{ - static const char lsn_prefix[] = "\nLog sequence number "; - lsn_t lsn = 0; - if (MYSQL_RES *res = xb_mysql_query(connection, - "SHOW ENGINE INNODB STATUS", - true, false)) { - if (MYSQL_ROW row = mysql_fetch_row(res)) { - const char *p= strstr(row[2], lsn_prefix); - DBUG_ASSERT(p); - if (p) { - p += sizeof lsn_prefix - 1; - lsn = lsn_t(strtoll(p, NULL, 10)); - } - } - mysql_free_result(res); - } - return lsn; -} - lsn_t server_lsn_after_lock; extern void backup_wait_for_lsn(lsn_t lsn); -/** Start --backup */ -bool backup_start(ds_ctxt *ds_data, ds_ctxt *ds_meta, - CorruptedPages &corrupted_pages) -{ - if (!opt_no_lock) { - if (opt_safe_slave_backup) { - if (!wait_for_safe_slave(mysql_connection)) { - return(false); - } - } - - if (!backup_files(ds_data, fil_path_to_mysql_datadir, true)) { - return(false); - } - - history_lock_time = time(NULL); - - if (!lock_tables(mysql_connection)) { - return(false); - } - server_lsn_after_lock = get_current_lsn(mysql_connection); - } - - if (!backup_files(ds_data, fil_path_to_mysql_datadir, false)) { - return(false); - } - - if (!backup_files_from_datadir(ds_data, fil_path_to_mysql_datadir, - "aws-kms-key") || - !backup_files_from_datadir(ds_data, - aria_log_dir_path, - "aria_log")) { - return false; - } - - if (has_rocksdb_plugin()) { - rocksdb_create_checkpoint(); - } - - msg("Waiting for log copy thread to read lsn %llu", (ulonglong)server_lsn_after_lock); - backup_wait_for_lsn(server_lsn_after_lock); - DBUG_EXECUTE_FOR_KEY("sleep_after_waiting_for_lsn", {}, - { - ulong milliseconds = strtoul(dbug_val, NULL, 10); - msg("sleep_after_waiting_for_lsn"); - my_sleep(milliseconds*1000UL); - }); - - corrupted_pages.backup_fix_ddl(ds_data, ds_meta); - - // There is no need to stop slave thread before coping non-Innodb data when - // --no-lock option is used because --no-lock option requires that no DDL or - // DML to non-transaction tables can occur. - if (opt_no_lock) { - if (opt_safe_slave_backup) { - if (!wait_for_safe_slave(mysql_connection)) { - return(false); - } - } - } - - if (opt_slave_info) { - lock_binlog_maybe(mysql_connection); - - if (!write_slave_info(ds_data, mysql_connection)) { - return(false); - } - } - - /* The only reason why Galera/binlog info is written before - wait_for_ibbackup_log_copy_finish() is that after that call the xtrabackup - binary will start streamig a temporary copy of REDO log to stdout and - thus, any streaming from innobackupex would interfere. The only way to - avoid that is to have a single process, i.e. merge innobackupex and - xtrabackup. */ - if (opt_galera_info) { - if (!write_galera_info(ds_data, mysql_connection)) { - return(false); - } - } - - if (opt_binlog_info == BINLOG_INFO_ON) { - - lock_binlog_maybe(mysql_connection); - write_binlog_info(ds_data, mysql_connection); - } - - if (!opt_no_lock) { - msg("Executing FLUSH NO_WRITE_TO_BINLOG ENGINE LOGS..."); - xb_mysql_query(mysql_connection, - "FLUSH NO_WRITE_TO_BINLOG ENGINE LOGS", false); - } - - return(true); -} /** Release resources after backup_start() */ void backup_release() { - /* release all locks */ - if (!opt_no_lock) { - unlock_all(mysql_connection); - history_lock_time = 0; - } else { - history_lock_time = time(NULL) - history_lock_time; - } - if (opt_lock_ddl_per_table) { mdl_unlock_all(); } @@ -1534,7 +1376,7 @@ static const char *default_buffer_pool_file = "ib_buffer_pool"; bool backup_finish(ds_ctxt *ds_data) { /* Copy buffer pool dump or LRU dump */ - if (!opt_rsync && opt_galera_info) { + if (opt_galera_info) { if (buffer_pool_filename && file_exists(buffer_pool_filename)) { ds_data->copy_file(buffer_pool_filename, default_buffer_pool_file, 0); } @@ -1922,7 +1764,8 @@ copy_back() dst_dir = dst_dir_buf.make(srv_log_group_home_dir); - /* --backup generates a single ib_logfile0, which we must copy. */ + /* --backup generates a single LOG_FILE_NAME, which we must copy + if it exists. */ ds_tmp = ds_create(dst_dir, DS_TYPE_LOCAL); if (!(ret = copy_or_move_file(ds_tmp, LOG_FILE_NAME, LOG_FILE_NAME, @@ -2169,8 +2012,6 @@ decrypt_decompress() it = datadir_iter_new(".", false); - ut_a(xtrabackup_parallel >= 0); - ret = run_data_threads(it, decrypt_decompress_thread_func, xtrabackup_parallel ? xtrabackup_parallel : 1); @@ -2192,9 +2033,9 @@ decrypt_decompress() Do not copy the Innodb files (ibdata1, redo log files), as this is done in a separate step. */ -static bool backup_files_from_datadir(ds_ctxt_t *ds_data, - const char *dir_path, - const char *prefix) +bool backup_files_from_datadir(ds_ctxt_t *ds_data, + const char *dir_path, + const char *prefix) { os_file_dir_t dir = os_file_opendir(dir_path); if (dir == IF_WIN(INVALID_HANDLE_VALUE, nullptr)) return false; @@ -2218,10 +2059,6 @@ static bool backup_files_from_datadir(ds_ctxt_t *ds_data, pname = info.name; if (!starts_with(pname, prefix)) - /* For ES exchange the above line with the following code: - (!xtrabackup_prepare || !xtrabackup_incremental_dir || - !starts_with(pname, "aria_log"))) - */ continue; if (xtrabackup_prepare && xtrabackup_incremental_dir && @@ -2244,7 +2081,7 @@ static int rocksdb_remove_checkpoint_directory() return 0; } -static bool has_rocksdb_plugin() +bool has_rocksdb_plugin() { static bool first_time = true; static bool has_plugin= false; @@ -2390,7 +2227,7 @@ static void rocksdb_unlock_checkpoint() #define MARIADB_CHECKPOINT_DIR "mariabackup-checkpoint" static char rocksdb_checkpoint_dir[FN_REFLEN]; -static void rocksdb_create_checkpoint() +void rocksdb_create_checkpoint() { MYSQL_RES *result = xb_mysql_query(mysql_connection, "SELECT @@rocksdb_datadir,@@datadir", true, true); MYSQL_ROW row = mysql_fetch_row(result); @@ -2470,3 +2307,39 @@ static void rocksdb_copy_back(ds_ctxt *ds_data) { mkdirp(rocksdb_home_dir, 0777, MYF(0)); ds_data->copy_or_move_dir(ROCKSDB_BACKUP_DIR, rocksdb_home_dir, xtrabackup_copy_back, xtrabackup_copy_back); } + +void foreach_file_in_db_dirs( + const char *dir_path, std::function func) { + DBUG_ASSERT(dir_path); + + datadir_iter_t *it; + datadir_node_t node; + + datadir_node_init(&node); + it = datadir_iter_new(dir_path); + + while (datadir_iter_next(it, &node)) + if (!node.is_empty_dir && !func(node.filepath)) + break; + + datadir_iter_free(it); + datadir_node_free(&node); +} + +void foreach_file_in_datadir( + const char *dir_path, std::function func) +{ + DBUG_ASSERT(dir_path); + os_file_dir_t dir = os_file_opendir(dir_path); + os_file_stat_t info; + while (os_file_readdir_next_file(dir_path, dir, &info) == 0) { + if (info.type != OS_FILE_TYPE_FILE) + continue; + const char *pname = strrchr(info.name, IF_WIN('\\', '/')); + if (!pname) + pname = info.name; + if (!func(pname)) + break; + } + os_file_closedir(dir); +} diff --git a/extra/mariabackup/backup_copy.h b/extra/mariabackup/backup_copy.h index b5aaf3121e9..43b75e19939 100644 --- a/extra/mariabackup/backup_copy.h +++ b/extra/mariabackup/backup_copy.h @@ -2,6 +2,7 @@ #ifndef XTRABACKUP_BACKUP_COPY_H #define XTRABACKUP_BACKUP_COPY_H +#include #include #include #include "datasink.h" @@ -21,8 +22,7 @@ bool equal_paths(const char *first, const char *second); /** Start --backup */ -bool backup_start(ds_ctxt *ds_data, ds_ctxt *ds_meta, - CorruptedPages &corrupted_pages); +bool backup_files(ds_ctxt *ds_data, const char *from); /** Release resources after backup_start() */ void backup_release(); /** Finish after backup_start() and backup_release() */ @@ -38,7 +38,25 @@ is_path_separator(char); bool directory_exists(const char *dir, bool create); -lsn_t -get_current_lsn(MYSQL *connection); +bool has_rocksdb_plugin(); +void rocksdb_create_checkpoint(); +void foreach_file_in_db_dirs( + const char *dir_path, std::function func); +void foreach_file_in_datadir( + const char *dir_path, std::function func); +bool ends_with(const char *str, const char *suffix); +bool starts_with(const char *str, const char *prefix); +void parse_db_table_from_file_path( + const char *filepath, char *dbname, char *tablename); +const char *trim_dotslash(const char *path); +bool backup_files_from_datadir(ds_ctxt_t *ds_data, + const char *dir_path, + const char *prefix); +bool is_system_table(const char *dbname, const char *tablename); +std::unique_ptr> + find_files(const char *dir_path, const char *prefix, const char *suffix); +bool file_exists(const char *filename); +bool +filename_matches(const char *filename, const char **ext_list); #endif diff --git a/extra/mariabackup/backup_debug.h b/extra/mariabackup/backup_debug.h index 777b4f4adeb..9286bc7b4e2 100644 --- a/extra/mariabackup/backup_debug.h +++ b/extra/mariabackup/backup_debug.h @@ -1,5 +1,6 @@ #pragma once #include "my_dbug.h" + #ifndef DBUG_OFF char *dbug_mariabackup_get_val(const char *event, fil_space_t::name_type key); /* @@ -14,11 +15,21 @@ To use this facility, you need to for the variable) 3. start mariabackup with --dbug=+d,debug_mariabackup_events */ -#define DBUG_EXECUTE_FOR_KEY(EVENT, KEY, CODE) \ - DBUG_EXECUTE_IF("mariabackup_inject_code", \ - { char *dbug_val= dbug_mariabackup_get_val(EVENT, KEY); \ - if (dbug_val) CODE }) +extern void dbug_mariabackup_event( + const char *event, const fil_space_t::name_type key, bool need_lock); +#define DBUG_MARIABACKUP_EVENT(A, B) \ + DBUG_EXECUTE_IF("mariabackup_events", \ + dbug_mariabackup_event(A,B,false);); +#define DBUG_MARIABACKUP_EVENT_LOCK(A, B) \ + DBUG_EXECUTE_IF("mariabackup_events", \ + dbug_mariabackup_event(A,B, true);); +#define DBUG_EXECUTE_FOR_KEY(EVENT, KEY, CODE) \ + DBUG_EXECUTE_IF("mariabackup_inject_code", {\ + char *dbug_val = dbug_mariabackup_get_val(EVENT, KEY); \ + if (dbug_val && *dbug_val) CODE \ + }) #else +#define DBUG_MARIABACKUP_EVENT(A,B) +#define DBUG_MARIABACKUP_EVENT_LOCK(A,B) #define DBUG_EXECUTE_FOR_KEY(EVENT, KEY, CODE) #endif - diff --git a/extra/mariabackup/backup_mysql.cc b/extra/mariabackup/backup_mysql.cc index 5b98c630030..2d4b19c20a7 100644 --- a/extra/mariabackup/backup_mysql.cc +++ b/extra/mariabackup/backup_mysql.cc @@ -60,10 +60,11 @@ Street, Fifth Floor, Boston, MA 02110-1335 USA #include "backup_copy.h" #include "backup_mysql.h" #include "mysqld.h" -#include "xb_plugin.h" +#include "encryption_plugin.h" #include #include #include "page0zip.h" +#include "backup_debug.h" char *tool_name; char tool_args[2048]; @@ -71,7 +72,7 @@ char tool_args[2048]; ulong mysql_server_version; /* server capabilities */ -bool have_backup_locks = false; +bool have_changed_page_bitmaps = false; bool have_lock_wait_timeout = false; bool have_galera_enabled = false; bool have_multi_threaded_slave = false; @@ -251,13 +252,14 @@ struct mysql_variable { static -void +uint read_mysql_variables(MYSQL *connection, const char *query, mysql_variable *vars, bool vertical_result) { MYSQL_RES *mysql_result; MYSQL_ROW row; mysql_variable *var; + uint n_values=0; mysql_result = xb_mysql_query(connection, query, true); @@ -271,6 +273,7 @@ read_mysql_variables(MYSQL *connection, const char *query, mysql_variable *vars, if (strcmp(var->name, name) == 0 && value != NULL) { *(var->value) = strdup(value); + n_values++; } } } @@ -287,6 +290,7 @@ read_mysql_variables(MYSQL *connection, const char *query, mysql_variable *vars, if (strcmp(var->name, name) == 0 && value != NULL) { *(var->value) = strdup(value); + n_values++; } } ++i; @@ -295,6 +299,7 @@ read_mysql_variables(MYSQL *connection, const char *query, mysql_variable *vars, } mysql_free_result(mysql_result); + return n_values; } @@ -359,7 +364,6 @@ bool get_mysql_vars(MYSQL *connection) { char *gtid_mode_var= NULL; char *version_var= NULL; - char *have_backup_locks_var= NULL; char *log_bin_var= NULL; char *lock_wait_timeout_var= NULL; char *wsrep_on_var= NULL; @@ -384,7 +388,6 @@ bool get_mysql_vars(MYSQL *connection) bool ret= true; mysql_variable mysql_vars[]= { - {"have_backup_locks", &have_backup_locks_var}, {"log_bin", &log_bin_var}, {"lock_wait_timeout", &lock_wait_timeout_var}, {"gtid_mode", >id_mode_var}, @@ -409,11 +412,6 @@ bool get_mysql_vars(MYSQL *connection) read_mysql_variables(connection, "SHOW VARIABLES", mysql_vars, true); - if (have_backup_locks_var != NULL && !opt_no_backup_locks) - { - have_backup_locks= true; - } - if (opt_binlog_info == BINLOG_INFO_AUTO) { if (log_bin_var != NULL && !strcmp(log_bin_var, "ON")) @@ -867,11 +865,11 @@ static void stop_query_killer() /*********************************************************************//** -Function acquires either a backup tables lock, if supported -by the server, or a global read lock (FLUSH TABLES WITH READ LOCK) -otherwise. +Function acquires backup locks @returns true if lock acquired */ -bool lock_tables(MYSQL *connection) + +bool +lock_for_backup_stage_start(MYSQL *connection) { if (have_lock_wait_timeout || opt_lock_wait_timeout) { @@ -884,12 +882,6 @@ bool lock_tables(MYSQL *connection) xb_mysql_query(connection, buf, false); } - if (have_backup_locks) - { - msg("Executing LOCK TABLES FOR BACKUP..."); - xb_mysql_query(connection, "LOCK TABLES FOR BACKUP", false); - return (true); - } if (opt_lock_wait_timeout) { @@ -914,8 +906,6 @@ bool lock_tables(MYSQL *connection) xb_mysql_query(connection, "BACKUP STAGE START", true); DBUG_MARIABACKUP_EVENT("after_backup_stage_start", {}); - xb_mysql_query(connection, "BACKUP STAGE BLOCK_COMMIT", true); - DBUG_MARIABACKUP_EVENT("after_backup_stage_block_commit", {}); /* Set the maximum supported session value for lock_wait_timeout to prevent unnecessary timeouts when the global value is changed from the default */ @@ -931,24 +921,68 @@ bool lock_tables(MYSQL *connection) return (true); } -/*********************************************************************//** -If backup locks are used, execute LOCK BINLOG FOR BACKUP provided that we are -not in the --no-lock mode and the lock has not been acquired already. -@returns true if lock acquired */ bool -lock_binlog_maybe(MYSQL *connection) -{ - if (have_backup_locks && !opt_no_lock && !binlog_locked) { - msg("Executing LOCK BINLOG FOR BACKUP..."); - xb_mysql_query(connection, "LOCK BINLOG FOR BACKUP", false); - binlog_locked = true; - - return(true); +lock_for_backup_stage_flush(MYSQL *connection) { + if (opt_kill_long_queries_timeout) { + start_query_killer(); } - - return(false); + xb_mysql_query(connection, "BACKUP STAGE FLUSH", true); + if (opt_kill_long_queries_timeout) { + stop_query_killer(); + } + return true; } +bool +lock_for_backup_stage_block_ddl(MYSQL *connection) { + if (opt_kill_long_queries_timeout) { + start_query_killer(); + } + xb_mysql_query(connection, "BACKUP STAGE BLOCK_DDL", true); + DBUG_MARIABACKUP_EVENT("after_backup_stage_block_ddl", {}); + if (opt_kill_long_queries_timeout) { + stop_query_killer(); + } + return true; +} + +bool +lock_for_backup_stage_commit(MYSQL *connection) { + if (opt_kill_long_queries_timeout) { + start_query_killer(); + } + xb_mysql_query(connection, "BACKUP STAGE BLOCK_COMMIT", true); + DBUG_MARIABACKUP_EVENT("after_backup_stage_block_commit", {}); + if (opt_kill_long_queries_timeout) { + stop_query_killer(); + } + return true; +} + +bool backup_lock(MYSQL *con, const char *table_name) { + static const std::string backup_lock_prefix("BACKUP LOCK "); + std::string backup_lock_query = backup_lock_prefix + table_name; + xb_mysql_query(con, backup_lock_query.c_str(), true); + return true; +} + +bool backup_unlock(MYSQL *con) { + xb_mysql_query(con, "BACKUP UNLOCK", true); + return true; +} + +std::unordered_set +get_tables_in_use(MYSQL *con) { + std::unordered_set result; + MYSQL_RES *q_res = + xb_mysql_query(con, "SHOW OPEN TABLES WHERE In_use = 1", true); + while (MYSQL_ROW row = mysql_fetch_row(q_res)) { + auto tk = table_key(row[0], row[1]); + msg("Table %s is in use", tk.c_str()); + result.insert(std::move(tk)); + } + return result; +} /*********************************************************************//** Releases either global read lock acquired with FTWRL and the binlog @@ -1383,77 +1417,12 @@ write_slave_info(ds_ctxt *datasink, MYSQL *connection) /*********************************************************************//** -Retrieves MySQL Galera and -saves it in a file. It also prints it to stdout. */ +Old function, not needed anymore with BACKUP LOCKS +*/ bool write_galera_info(ds_ctxt *datasink, MYSQL *connection) { - char *state_uuid = NULL, *state_uuid55 = NULL; - char *last_committed = NULL, *last_committed55 = NULL; - char *domain_id = NULL, *domain_id55 = NULL; - bool result; - - mysql_variable status[] = { - {"Wsrep_local_state_uuid", &state_uuid}, - {"wsrep_local_state_uuid", &state_uuid55}, - {"Wsrep_last_committed", &last_committed}, - {"wsrep_last_committed", &last_committed55}, - {NULL, NULL} - }; - - mysql_variable value[] = { - {"Wsrep_gtid_domain_id", &domain_id}, - {"wsrep_gtid_domain_id", &domain_id55}, - {NULL, NULL} - }; - - /* When backup locks are supported by the server, we should skip - creating xtrabackup_galera_info file on the backup stage, because - wsrep_local_state_uuid and wsrep_last_committed will be inconsistent - without blocking commits. The state file will be created on the prepare - stage using the WSREP recovery procedure. */ - if (have_backup_locks) { - return(true); - } - - read_mysql_variables(connection, "SHOW STATUS", status, true); - - if ((state_uuid == NULL && state_uuid55 == NULL) - || (last_committed == NULL && last_committed55 == NULL)) { - msg("Warning: failed to get master wsrep state from SHOW STATUS."); - result = true; - goto cleanup; - } - - read_mysql_variables(connection, "SHOW VARIABLES LIKE 'wsrep%'", value, true); - - if (domain_id == NULL && domain_id55 == NULL) { - msg("Warning: failed to get master wsrep state from SHOW VARIABLES."); - result = true; - goto cleanup; - } - - result = datasink->backup_file_printf(XTRABACKUP_GALERA_INFO, - "%s:%s %s\n", state_uuid ? state_uuid : state_uuid55, - last_committed ? last_committed : last_committed55, - domain_id ? domain_id : domain_id55); - - if (result) - { - result= datasink->backup_file_printf(XTRABACKUP_DONOR_GALERA_INFO, - "%s:%s %s\n", state_uuid ? state_uuid : state_uuid55, - last_committed ? last_committed : last_committed55, - domain_id ? domain_id : domain_id55); - } - if (result) - { - write_current_binlog_file(datasink, connection); - } - -cleanup: - free_mysql_variables(status); - - return(result); + return true; // Success } @@ -1496,8 +1465,6 @@ write_current_binlog_file(ds_ctxt *datasink, MYSQL *connection) if (gtid_exists) { size_t log_bin_dir_length; - lock_binlog_maybe(connection); - xb_mysql_query(connection, "FLUSH BINARY LOGS", false); read_mysql_variables(connection, "SHOW MASTER STATUS", @@ -1856,13 +1823,13 @@ bool write_backup_config_file(ds_ctxt *datasink) srv_log_file_size, srv_page_size, srv_undo_dir, - srv_undo_tablespaces, + (uint) srv_undo_tablespaces, page_zip_level, innobase_buffer_pool_filename ? "innodb_buffer_pool_filename=" : "", innobase_buffer_pool_filename ? innobase_buffer_pool_filename : "", - xb_plugin_get_config()); + encryption_plugin_get_config()); return rc; } @@ -1987,3 +1954,23 @@ mdl_unlock_all() mysql_close(mdl_con); spaceid_to_tablename.clear(); } + +ulonglong get_current_lsn(MYSQL *connection) +{ + static const char lsn_prefix[] = "\nLog sequence number "; + ulonglong lsn = 0; + if (MYSQL_RES *res = xb_mysql_query(connection, + "SHOW ENGINE INNODB STATUS", + true, false)) { + if (MYSQL_ROW row = mysql_fetch_row(res)) { + const char *p= strstr(row[2], lsn_prefix); + DBUG_ASSERT(p); + if (p) { + p += sizeof lsn_prefix - 1; + lsn = lsn_t(strtoll(p, NULL, 10)); + } + } + mysql_free_result(res); + } + return lsn; +} diff --git a/extra/mariabackup/backup_mysql.h b/extra/mariabackup/backup_mysql.h index 039934be02d..c87efd21c11 100644 --- a/extra/mariabackup/backup_mysql.h +++ b/extra/mariabackup/backup_mysql.h @@ -2,12 +2,15 @@ #define XTRABACKUP_BACKUP_MYSQL_H #include +#include +#include +#include "datasink.h" /* MariaDB version */ extern ulong mysql_server_version; /* server capabilities */ -extern bool have_backup_locks; +extern bool have_changed_page_bitmaps; extern bool have_lock_wait_timeout; extern bool have_galera_enabled; extern bool have_multi_threaded_slave; @@ -71,7 +74,21 @@ bool lock_binlog_maybe(MYSQL *connection); bool -lock_tables(MYSQL *connection); +lock_for_backup_stage_start(MYSQL *connection); + +bool +lock_for_backup_stage_flush(MYSQL *connection); + +bool +lock_for_backup_stage_block_ddl(MYSQL *connection); + +bool +lock_for_backup_stage_commit(MYSQL *connection); + +bool backup_lock(MYSQL *con, const char *table_name); +bool backup_unlock(MYSQL *con); + +std::unordered_set get_tables_in_use(MYSQL *con); bool wait_for_safe_slave(MYSQL *connection); @@ -82,5 +99,6 @@ write_galera_info(ds_ctxt *datasink, MYSQL *connection); bool write_slave_info(ds_ctxt *datasink, MYSQL *connection); +ulonglong get_current_lsn(MYSQL *connection); #endif diff --git a/extra/mariabackup/common_engine.cc b/extra/mariabackup/common_engine.cc new file mode 100644 index 00000000000..a4a8706243a --- /dev/null +++ b/extra/mariabackup/common_engine.cc @@ -0,0 +1,512 @@ +#include "common_engine.h" +#include "backup_copy.h" +#include "xtrabackup.h" +#include "common.h" +#include "backup_debug.h" + +#include +#include +#include +#include + +namespace common_engine { + +class Table { +public: + Table(std::string &db, std::string &table, std::string &fs_name) : + m_db(std::move(db)), m_table(std::move(table)), + m_fs_name(std::move(fs_name)) {} + virtual ~Table() {} + void add_file_name(const char *file_name) { m_fnames.push_back(file_name); } + virtual bool copy(ds_ctxt_t *ds, MYSQL *con, bool no_lock, + bool finalize, unsigned thread_num); + std::string &get_db() { return m_db; } + std::string &get_table() { return m_table; } + std::string &get_version() { return m_version; } + +protected: + std::string m_db; + std::string m_table; + std::string m_fs_name; + std::string m_version; + std::vector m_fnames; +}; + +bool +Table::copy(ds_ctxt_t *ds, MYSQL *con, bool no_lock, bool, unsigned thread_num) { + static const size_t buf_size = 10 * 1024 * 1024; + std::unique_ptr buf; + bool result = false; + File frm_file = -1; + std::vector files; + bool locked = false; + std::string full_tname("`"); + full_tname.append(m_db).append("`.`").append(m_table).append("`"); + + if (!no_lock && !backup_lock(con, full_tname.c_str())) { + msg(thread_num, "Error on executing BACKUP LOCK for table %s", + full_tname.c_str()); + goto exit; + } + else + locked = !no_lock; + + if ((frm_file = mysql_file_open(key_file_frm, (m_fs_name + ".frm").c_str(), + O_RDONLY | O_SHARE, MYF(0))) < 0 && !m_fnames.empty() && + !ends_with(m_fnames[0].c_str(), ".ARZ") && + !ends_with(m_fnames[0].c_str(), ".ARM")) { + // Don't treat it as error, as the table can be dropped after it + // was added to queue for copying + result = true; + goto exit; + } + + for (const auto &fname : m_fnames) { + File file = mysql_file_open(0, fname.c_str(),O_RDONLY | O_SHARE, MYF(0)); + if (file < 0) { + msg(thread_num, "Error on file %s open during %s table copy", + fname.c_str(), full_tname.c_str()); + goto exit; + } + files.push_back(file); + } + + if (locked && !backup_unlock(con)) { + msg(thread_num, "Error on BACKUP UNLOCK for table %s", full_tname.c_str()); + locked = false; + goto exit; + } + + locked = false; + + buf.reset(new uchar[buf_size]); + + for (size_t i = 0; i < m_fnames.size(); ++i) { + ds_file_t *dst_file = nullptr; + size_t bytes_read; + size_t copied_size = 0; + MY_STAT stat_info; + + if (my_fstat(files[i], &stat_info, MYF(0))) { + msg(thread_num, "error: failed to get stat info for file %s of " + "table %s", m_fnames[i].c_str(), full_tname.c_str()); + goto exit; + } + + const char *dst_path = + (xtrabackup_copy_back || xtrabackup_move_back) ? + m_fnames[i].c_str() : trim_dotslash(m_fnames[i].c_str()); + + dst_file = ds_open(ds, dst_path, &stat_info, false); + if (!dst_file) { + msg(thread_num, "error: cannot open destination stream for %s, table %s", + dst_path, full_tname.c_str()); + goto exit; + } + + while ((bytes_read = my_read(files[i], buf.get(), buf_size, MY_WME))) { + if (bytes_read == size_t(-1)) { + msg(thread_num, "error: file %s read for table %s", + m_fnames[i].c_str(), full_tname.c_str()); + ds_close(dst_file); + goto exit; + } + xtrabackup_io_throttling(); + if (ds_write(dst_file, buf.get(), bytes_read)) { + msg(thread_num, "error: file %s write for table %s", + dst_path, full_tname.c_str()); + ds_close(dst_file); + goto exit; + } + copied_size += bytes_read; + } + mysql_file_close(files[i], MYF(MY_WME)); + files[i] = -1; + ds_close(dst_file); + msg(thread_num, "Copied file %s for table %s, %zu bytes", + m_fnames[i].c_str(), full_tname.c_str(), copied_size); + } + + result = true; + +#ifndef DBUG_OFF + { + std::string sql_name(m_db); + sql_name.append("/").append(m_table); + DBUG_MARIABACKUP_EVENT_LOCK("after_ce_table_copy", fil_space_t::name_type(sql_name.data(), sql_name.size())); + } +#endif // DBUG_OFF +exit: + if (frm_file >= 0) { + m_version = ::read_table_version_id(frm_file); + mysql_file_close(frm_file, MYF(MY_WME)); + } + if (locked && !backup_unlock(con)) { + msg(thread_num, "Error on BACKUP UNLOCK for table %s", full_tname.c_str()); + result = false; + } + for (auto file : files) + if (file >= 0) + mysql_file_close(file, MYF(MY_WME)); + return result; +} + +// Append-only tables +class LogTable : public Table { + public: + LogTable(std::string &db, std::string &table, std::string &fs_name) : + Table(db, table, fs_name) {} + + virtual ~LogTable() { (void)close(); } + bool + copy(ds_ctxt_t *ds, MYSQL *con, bool no_lock, bool finalize, + unsigned thread_num) override; + bool close(); + private: + bool open(ds_ctxt_t *ds, unsigned thread_num); + std::vector m_src; + std::vector m_dst; +}; + +bool +LogTable::open(ds_ctxt_t *ds, unsigned thread_num) { + DBUG_ASSERT(m_src.empty()); + DBUG_ASSERT(m_dst.empty()); + + std::string full_tname("`"); + full_tname.append(m_db).append("`.`").append(m_table).append("`"); + + for (const auto &fname : m_fnames) { + File file = mysql_file_open(0, fname.c_str(),O_RDONLY | O_SHARE, MYF(0)); + if (file < 0) { + msg(thread_num, "Error on file %s open during %s log table copy", + fname.c_str(), full_tname.c_str()); + return false; + } + m_src.push_back(file); + + MY_STAT stat_info; + if (my_fstat(file, &stat_info, MYF(0))) { + msg(thread_num, "error: failed to get stat info for file %s of " + "log table %s", fname.c_str(), full_tname.c_str()); + return false; + } + const char *dst_path = + (xtrabackup_copy_back || xtrabackup_move_back) ? + fname.c_str() : trim_dotslash(fname.c_str()); + ds_file_t *dst_file = ds_open(ds, dst_path, &stat_info, false); + if (!dst_file) { + msg(thread_num, "error: cannot open destination stream for %s, " + "log table %s", dst_path, full_tname.c_str()); + return false; + } + m_dst.push_back(dst_file); + } + + File frm_file; + if ((frm_file = mysql_file_open(key_file_frm, (m_fs_name + ".frm").c_str(), + O_RDONLY | O_SHARE, MYF(0))) < 0 && !m_fnames.empty() && + !ends_with(m_fnames[0].c_str(), ".ARZ") && + !ends_with(m_fnames[0].c_str(), ".ARM")) { + msg(thread_num, "Error on .frm file open for log table %s", + full_tname.c_str()); + return false; + } + + m_version = ::read_table_version_id(frm_file); + mysql_file_close(frm_file, MYF(MY_WME)); + + return true; +} + +bool LogTable::close() { + while (!m_src.empty()) { + auto f = m_src.back(); + m_src.pop_back(); + mysql_file_close(f, MYF(MY_WME)); + } + while (!m_dst.empty()) { + auto f = m_dst.back(); + m_dst.pop_back(); + ds_close(f); + } + return true; +} + +bool +LogTable::copy(ds_ctxt_t *ds, MYSQL *con, bool no_lock, bool finalize, + unsigned thread_num) { + static const size_t buf_size = 10 * 1024 * 1024; + DBUG_ASSERT(ds); + DBUG_ASSERT(con); + if (m_src.empty() && !open(ds, thread_num)) { + close(); + return false; + } + DBUG_ASSERT(m_src.size() == m_dst.size()); + + std::unique_ptr buf(new uchar[buf_size]); + for (size_t i = 0; i < m_src.size(); ++i) { + // .CSM can be rewritten (see write_meta_file() usage in ha_tina.cc) + if (!finalize && ends_with(m_fnames[i].c_str(), ".CSM")) + continue; + size_t bytes_read; + size_t copied_size = 0; + while ((bytes_read = my_read(m_src[i], buf.get(), buf_size, MY_WME))) { + if (bytes_read == size_t(-1)) { + msg(thread_num, "error: file %s read for log table %s", + m_fnames[i].c_str(), + std::string("`").append(m_db).append("`.`"). + append(m_table).append("`").c_str()); + close(); + return false; + } + xtrabackup_io_throttling(); + if (ds_write(m_dst[i], buf.get(), bytes_read)) { + msg(thread_num, "error: file %s write for log table %s", + m_fnames[i].c_str(), std::string("`").append(m_db).append("`.`"). + append(m_table).append("`").c_str()); + close(); + return false; + } + copied_size += bytes_read; + } + msg(thread_num, "Copied file %s for log table %s, %zu bytes", + m_fnames[i].c_str(), std::string("`").append(m_db).append("`.`"). + append(m_table).append("`").c_str(), copied_size); + } + + return true; +} + +class BackupImpl { + public: + BackupImpl( + const char *datadir_path, ds_ctxt_t *datasink, + std::vector &con_pool, ThreadPool &thread_pool) : + m_datadir_path(datadir_path), m_ds(datasink), m_con_pool(con_pool), + m_process_table_jobs(thread_pool) {} + ~BackupImpl() { } + bool scan( + const std::unordered_set &exclude_tables, + std::unordered_set *out_processed_tables, + bool no_lock, bool collect_log_and_stats); + void set_post_copy_table_hook(const post_copy_table_hook_t &hook) { + m_table_post_copy_hook = hook; + } + bool copy_log_tables(bool finalize); + bool copy_stats_tables(); + bool wait_for_finish(); + bool close_log_tables(); + private: + + void process_table_job(Table *table, bool no_lock, bool delete_table, + bool finalize, unsigned thread_num); + + const char *m_datadir_path; + ds_ctxt_t *m_ds; + std::vector &m_con_pool; + TasksGroup m_process_table_jobs; + + post_copy_table_hook_t m_table_post_copy_hook; + std::unordered_map> m_log_tables; + std::unordered_map> m_stats_tables; +}; + +void BackupImpl::process_table_job(Table *table, bool no_lock, + bool delete_table, bool finalize, unsigned thread_num) { + int result = 0; + + if (!m_process_table_jobs.get_result()) + goto exit; + + if (!table->copy(m_ds, m_con_pool[thread_num], no_lock, finalize, thread_num)) + goto exit; + + if (m_table_post_copy_hook) + m_table_post_copy_hook(table->get_db(), table->get_table(), + table->get_version()); + + result = 1; + +exit: + if (delete_table) + delete table; + m_process_table_jobs.finish_task(result); +} + +bool BackupImpl::scan(const std::unordered_set &exclude_tables, + std::unordered_set *out_processed_tables, bool no_lock, + bool collect_log_and_stats) { + + msg("Start scanning common engine tables, need backup locks: %d, " + "collect log and stat tables: %d", no_lock, collect_log_and_stats); + + std::unordered_map> found_tables; + + foreach_file_in_db_dirs(m_datadir_path, + [&](const char *file_path)->bool { + + static const char *ext_list[] = + {".MYD", ".MYI", ".MRG", ".ARM", ".ARZ", ".CSM", ".CSV", NULL}; + + bool is_aria = ends_with(file_path, ".MAD") || ends_with(file_path, ".MAI"); + + if (!collect_log_and_stats && is_aria) + return true; + + if (!is_aria && !filename_matches(file_path, ext_list)) + return true; + + if (check_if_skip_table(file_path)) { + msg("Skipping %s.", file_path); + return true; + } + + auto db_table_fs = convert_filepath_to_tablename(file_path); + auto tk = + table_key(std::get<0>(db_table_fs), std::get<1>(db_table_fs)); + + // log and stats tables are only collected in this function, + // so there is no need to filter out them with exclude_tables. + if (collect_log_and_stats) { + if (is_log_table(std::get<0>(db_table_fs).c_str(), + std::get<1>(db_table_fs).c_str())) { + auto table_it = m_log_tables.find(tk); + if (table_it == m_log_tables.end()) { + msg("Log table found: %s", tk.c_str()); + table_it = m_log_tables.emplace(tk, + std::unique_ptr(new LogTable(std::get<0>(db_table_fs), + std::get<1>(db_table_fs), std::get<2>(db_table_fs)))).first; + } + msg("Collect log table file: %s", file_path); + table_it->second->add_file_name(file_path); + return true; + } + // Aria can handle statistics tables + else if (is_stats_table(std::get<0>(db_table_fs).c_str(), + std::get<1>(db_table_fs).c_str()) && !is_aria) { + auto table_it = m_stats_tables.find(tk); + if (table_it == m_stats_tables.end()) { + msg("Stats table found: %s", tk.c_str()); + table_it = m_stats_tables.emplace(tk, + std::unique_ptr
(new Table(std::get<0>(db_table_fs), + std::get<1>(db_table_fs), std::get<2>(db_table_fs)))).first; + } + msg("Collect stats table file: %s", file_path); + table_it->second->add_file_name(file_path); + return true; + } + } else if (is_log_table(std::get<0>(db_table_fs).c_str(), + std::get<1>(db_table_fs).c_str()) || + is_stats_table(std::get<0>(db_table_fs).c_str(), + std::get<1>(db_table_fs).c_str())) + return true; + + if (is_aria) + return true; + + if (exclude_tables.count(tk)) { + msg("Skip table %s at it is in exclude list", tk.c_str()); + return true; + } + + auto table_it = found_tables.find(tk); + if (table_it == found_tables.end()) { + table_it = found_tables.emplace(tk, + std::unique_ptr
(new Table(std::get<0>(db_table_fs), + std::get<1>(db_table_fs), std::get<2>(db_table_fs)))).first; + } + + table_it->second->add_file_name(file_path); + + return true; + }); + + for (auto &table_it : found_tables) { + m_process_table_jobs.push_task( + std::bind(&BackupImpl::process_table_job, this, table_it.second.release(), + no_lock, true, false, std::placeholders::_1)); + if (out_processed_tables) + out_processed_tables->insert(table_it.first); + } + + msg("Stop scanning common engine tables"); + return true; +} + +bool BackupImpl::copy_log_tables(bool finalize) { + for (auto &table_it : m_log_tables) { + // Do not execute BACKUP LOCK for log tables as it's supposed + // that they must be copied on BLOCK_DDL and BLOCK_COMMIT locks. + m_process_table_jobs.push_task( + std::bind(&BackupImpl::process_table_job, this, table_it.second.get(), + true, false, finalize, std::placeholders::_1)); + } + return true; +} + +bool BackupImpl::copy_stats_tables() { + for (auto &table_it : m_stats_tables) { + // Do not execute BACKUP LOCK for stats tables as it's supposed + // that they must be copied on BLOCK_DDL and BLOCK_COMMIT locks. + // Delete stats table object after copy (see process_table_job()) + m_process_table_jobs.push_task( + std::bind(&BackupImpl::process_table_job, this, table_it.second.release(), + true, true, false, std::placeholders::_1)); + } + m_stats_tables.clear(); + return true; +} + +bool BackupImpl::wait_for_finish() { + /* Wait for threads to exit */ + return m_process_table_jobs.wait_for_finish(); +} + +bool BackupImpl::close_log_tables() { + bool result = wait_for_finish(); + for (auto &table_it : m_log_tables) + table_it.second->close(); + return result; +} + +Backup::Backup(const char *datadir_path, ds_ctxt_t *datasink, + std::vector &con_pool, ThreadPool &thread_pool) : + m_backup_impl( + new BackupImpl(datadir_path, datasink, con_pool, + thread_pool)) { } + +Backup::~Backup() { + delete m_backup_impl; +} + +bool Backup::scan( + const std::unordered_set &exclude_tables, + std::unordered_set *out_processed_tables, + bool no_lock, bool collect_log_and_stats) { + return m_backup_impl->scan(exclude_tables, out_processed_tables, no_lock, + collect_log_and_stats); +} + +bool Backup::copy_log_tables(bool finalize) { + return m_backup_impl->copy_log_tables(finalize); +} + +bool Backup::copy_stats_tables() { + return m_backup_impl->copy_stats_tables(); +} + +bool Backup::wait_for_finish() { + return m_backup_impl->wait_for_finish(); +} + +bool Backup::close_log_tables() { + return m_backup_impl->close_log_tables(); +} + +void Backup::set_post_copy_table_hook(const post_copy_table_hook_t &hook) { + m_backup_impl->set_post_copy_table_hook(hook); +} + +} // namespace common_engine diff --git a/extra/mariabackup/common_engine.h b/extra/mariabackup/common_engine.h new file mode 100644 index 00000000000..6f5d8062e50 --- /dev/null +++ b/extra/mariabackup/common_engine.h @@ -0,0 +1,39 @@ +#pragma once +#include "my_global.h" +#include "backup_mysql.h" +#include "datasink.h" +#include "thread_pool.h" +#include "xtrabackup.h" + +#include +#include +#include + +namespace common_engine { + +class BackupImpl; + +class Backup { + public: + Backup(const char *datadir_path, ds_ctxt_t *datasink, + std::vector &con_pool, ThreadPool &thread_pool); + ~Backup(); + Backup (Backup &&other) = delete; + Backup & operator= (Backup &&other) = delete; + Backup(const Backup &) = delete; + Backup & operator= (const Backup &) = delete; + bool scan( + const std::unordered_set &exclude_tables, + std::unordered_set *out_processed_tables, + bool no_lock, bool collect_log_and_stats); + bool copy_log_tables(bool finalize); + bool copy_stats_tables(); + bool wait_for_finish(); + bool close_log_tables(); + void set_post_copy_table_hook(const post_copy_table_hook_t &hook); + private: + BackupImpl *m_backup_impl; +}; + +} // namespace common_engine + diff --git a/extra/mariabackup/datasink.cc b/extra/mariabackup/datasink.cc index a576526d7ff..132ff3fcfb6 100644 --- a/extra/mariabackup/datasink.cc +++ b/extra/mariabackup/datasink.cc @@ -80,11 +80,11 @@ ds_create(const char *root, ds_type_t type) /************************************************************************ Open a datasink file */ ds_file_t * -ds_open(ds_ctxt_t *ctxt, const char *path, MY_STAT *stat) +ds_open(ds_ctxt_t *ctxt, const char *path, const MY_STAT *stat, bool rewrite) { ds_file_t *file; - file = ctxt->datasink->open(ctxt, path, stat); + file = ctxt->datasink->open(ctxt, path, stat, rewrite); if (file != NULL) { file->datasink = ctxt->datasink; } @@ -104,6 +104,30 @@ ds_write(ds_file_t *file, const void *buf, size_t len) return file->datasink->write(file, (const uchar *)buf, len); } +int ds_seek_set(ds_file_t *file, my_off_t offset) { + DBUG_ASSERT(file); + DBUG_ASSERT(file->datasink); + if (file->datasink->seek_set) + return file->datasink->seek_set(file, offset); + return 0; +} + +int ds_rename(ds_ctxt_t *ctxt, const char *old_path, const char *new_path) { + DBUG_ASSERT(ctxt); + DBUG_ASSERT(ctxt->datasink); + if (ctxt->datasink->rename) + return ctxt->datasink->rename(ctxt, old_path, new_path); + return 0; +} + +int ds_remove(ds_ctxt_t *ctxt, const char *path) { + DBUG_ASSERT(ctxt); + DBUG_ASSERT(ctxt->datasink); + if (ctxt->datasink->remove) + return ctxt->datasink->mremove(ctxt, path); + return 0; +} + /************************************************************************ Close a datasink file. @return 0 on success, 1, on error. */ diff --git a/extra/mariabackup/datasink.h b/extra/mariabackup/datasink.h index 57468e0c9c7..98cbe5252ac 100644 --- a/extra/mariabackup/datasink.h +++ b/extra/mariabackup/datasink.h @@ -43,7 +43,8 @@ typedef struct ds_ctxt { */ bool copy_file(const char *src_file_path, const char *dst_file_path, - uint thread_n); + uint thread_n, + bool rewrite = false); bool move_file(const char *src_file_path, const char *dst_file_path, @@ -76,10 +77,15 @@ typedef struct { struct datasink_struct { ds_ctxt_t *(*init)(const char *root); - ds_file_t *(*open)(ds_ctxt_t *ctxt, const char *path, MY_STAT *stat); + ds_file_t *(*open)(ds_ctxt_t *ctxt, const char *path, + const MY_STAT *stat, bool rewrite); int (*write)(ds_file_t *file, const unsigned char *buf, size_t len); + int (*seek_set)(ds_file_t *file, my_off_t offset); int (*close)(ds_file_t *file); int (*remove)(const char *path); + // TODO: consider to return bool from "rename" and "remove" + int (*rename)(ds_ctxt_t *ctxt, const char *old_path, const char *new_path); + int (*mremove)(ds_ctxt_t *ctxt, const char *path); void (*deinit)(ds_ctxt_t *ctxt); }; @@ -106,12 +112,17 @@ ds_ctxt_t *ds_create(const char *root, ds_type_t type); /************************************************************************ Open a datasink file */ -ds_file_t *ds_open(ds_ctxt_t *ctxt, const char *path, MY_STAT *stat); +ds_file_t *ds_open( + ds_ctxt_t *ctxt, const char *path, const MY_STAT *stat, bool rewrite = false); /************************************************************************ Write to a datasink file. @return 0 on success, 1 on error. */ int ds_write(ds_file_t *file, const void *buf, size_t len); +int ds_seek_set(ds_file_t *file, my_off_t offset); + +int ds_rename(ds_ctxt_t *ctxt, const char *old_path, const char *new_path); +int ds_remove(ds_ctxt_t *ctxt, const char *path); /************************************************************************ Close a datasink file. diff --git a/extra/mariabackup/ddl_log.cc b/extra/mariabackup/ddl_log.cc new file mode 100644 index 00000000000..6af34172dcb --- /dev/null +++ b/extra/mariabackup/ddl_log.cc @@ -0,0 +1,553 @@ +#include "ddl_log.h" +#include "common.h" +#include "my_sys.h" +#include "sql_table.h" +#include "backup_copy.h" +#include "xtrabackup.h" +#include +#include +#include +#include + +namespace ddl_log { + +struct Entry { + enum Type { + CREATE, + ALTER, + RENAME, + REPAIR, + OPTIMIZE, + DROP, + TRUNCATE, + CHANGE_INDEX, + BULK_INSERT + }; + Type type; + std::string date; + std::string engine; + bool partitioned; + std::string db; + std::string table; + std::string id; + std::string new_engine; + bool new_partitioned; + std::string new_db; + std::string new_table; + std::string new_id; +}; + +typedef std::vector> entries_t; +typedef std::function)> store_entry_func_t; + +const char *aria_engine_name = "Aria"; +static const char *frm_ext = ".frm"; +static const char *database_keyword = "DATABASE"; + +const std::unordered_map> engine_exts = +{ + {"Aria", {".MAD", ".MAI"}}, + {"MyISAM", {".MYD", ".MYI"}}, + {"MRG_MyISAM", {".MRG"}}, + {"ARCHIVE", {".ARM", ".ARZ"}}, + {"CSV", {".CSM", ".CSV"}} +}; + +static inline bool known_engine(const std::string &engine) { + return engine_exts.count(engine); +} + +// TODO: add error messages +size_t parse(const uchar *buf, size_t buf_size, bool &error_flag, + store_entry_func_t &store_entry_func) { + DBUG_ASSERT(buf); + static constexpr char token_delimiter = '\t'; + static constexpr char line_delimiter = '\n'; + enum { + TOKEN_FIRST = 0, + TOKEN_DATE = TOKEN_FIRST, + TOKEN_TYPE, + TOKEN_ENGINE, + TOKEN_PARTITIONED, + TOKEN_DB, + TOKEN_TABLE, + TOKEN_ID, + TOKEN_MANDATORY = TOKEN_ID, + TOKEN_NEW_ENGINE, + TOKEN_NEW_PARTITIONED, + TOKEN_NEW_DB, + TOKEN_NEW_TABLE, + TOKEN_NEW_ID, + TOKEN_LAST = TOKEN_NEW_ID + }; + const size_t string_offsets[TOKEN_LAST + 1] = { + offsetof(Entry, date), + offsetof(Entry, type), // not a string, be careful + offsetof(Entry, engine), + offsetof(Entry, partitioned), // not a string, be careful + offsetof(Entry, db), + offsetof(Entry, table), + offsetof(Entry, id), + offsetof(Entry, new_engine), + offsetof(Entry, new_partitioned), // not a string, be careful + offsetof(Entry, new_db), + offsetof(Entry, new_table), + offsetof(Entry, new_id) + }; + const std::unordered_map str_to_type = { + {"CREATE", Entry::CREATE}, + {"ALTER", Entry::ALTER}, + {"RENAME", Entry::RENAME}, + // TODO: fix to use uppercase-only + {"repair", Entry::REPAIR}, + {"optimize", Entry::OPTIMIZE}, + {"DROP", Entry::DROP}, + {"TRUNCATE", Entry::TRUNCATE}, + {"CHANGE_INDEX", Entry::CHANGE_INDEX}, + {"BULK_INSERT", Entry::BULK_INSERT} + }; + + const uchar *new_line = buf; + const uchar *token_start = buf; + unsigned token_num = TOKEN_FIRST; + + error_flag = false; + + std::unique_ptr entry(new Entry()); + + for (const uchar *ptr = buf; ptr < buf + buf_size; ++ptr) { + + if (*ptr != token_delimiter && *ptr != line_delimiter) + continue; + + if (token_start != ptr) { + std::string token(token_start, ptr); + + if (token_num == TOKEN_TYPE) { + const auto type_it = str_to_type.find(token); + if (type_it == str_to_type.end()) { + error_flag = true; + goto exit; + } + entry->type = type_it->second; + } + else if (token_num == TOKEN_PARTITIONED) { + entry->partitioned = token[0] - '0'; + } + else if (token_num == TOKEN_NEW_PARTITIONED) { + entry->new_partitioned = token[0] - '0'; + } + else if (token_num <= TOKEN_LAST) { + DBUG_ASSERT(token_num != TOKEN_TYPE); + DBUG_ASSERT(token_num != TOKEN_PARTITIONED); + DBUG_ASSERT(token_num != TOKEN_NEW_PARTITIONED); + reinterpret_cast + (reinterpret_cast(entry.get()) + string_offsets[token_num])-> + assign(std::move(token)); + } + else { + error_flag = true; + goto exit; + } + } + token_start = ptr + 1; + + if (*ptr == line_delimiter) { + if (token_num < TOKEN_MANDATORY) { + error_flag = true; + goto exit; + } + if (!store_entry_func(std::move(entry))) { + error_flag = true; + goto exit; + } + entry.reset(new Entry()); + token_num = TOKEN_FIRST; + new_line = ptr + 1; + } else + ++token_num; + } + +exit: + return new_line - buf; +} + +bool parse(const char *file_path, store_entry_func_t store_entry_func) { + DBUG_ASSERT(file_path); + DBUG_ASSERT(store_entry_func); + File file= -1; + bool result = true; + uchar buf[1024]; + size_t bytes_read = 0; + size_t buf_read_offset = 0; + + if ((file= my_open(file_path, O_RDONLY | O_SHARE | O_NOFOLLOW | O_CLOEXEC, + MYF(MY_WME))) < 0) { + msg("DDL log file %s open failed: %d", file_path, my_errno); + result = false; + goto exit; + } + + while((bytes_read = my_read( + file, &buf[buf_read_offset], sizeof(buf) - buf_read_offset, MY_WME)) > 0) { + if (bytes_read == size_t(-1)) { + msg("DDL log file %s read error: %d", file_path, my_errno); + result = false; + break; + } + bytes_read += buf_read_offset; + bool parse_error_flag = false; + size_t bytes_parsed = parse( + buf, bytes_read, parse_error_flag, store_entry_func); + if (parse_error_flag) { + result = false; + break; + } + size_t rest_size = bytes_read - bytes_parsed; + if (rest_size) + memcpy(buf, buf + bytes_parsed, rest_size); + buf_read_offset = rest_size; + } + +exit: + if (file >= 0) + my_close(file, MYF(MY_WME)); + return result; +}; + + +static +bool process_database( + const char *datadir_path, + ds_ctxt_t *ds, + const Entry &entry, + std::unordered_set &dropped_databases) { + + if (entry.type == Entry::Type::CREATE || + entry.type == Entry::Type::ALTER) { + std::string opt_file(datadir_path); + opt_file.append("/").append(entry.db).append("/db.opt"); + if (!ds->copy_file(opt_file.c_str(), opt_file.c_str(), 0, true)) { + msg("Failed to re-copy %s.", opt_file.c_str()); + return false; + } + if (entry.type == Entry::Type::CREATE) + dropped_databases.erase(entry.db); + return true; + } + + DBUG_ASSERT(entry.type == Entry::Type::DROP); + + std::string db_path(datadir_path); + db_path.append("/").append(entry.db); + const char *dst_path = convert_dst(db_path.c_str()); + if (!ds_remove(ds, dst_path)) { + dropped_databases.insert(entry.db); + return true; + } + return false; +} + +static +std::unique_ptr> + find_table_files( + const char *dir_path, + const std::string &db, + const std::string &table) { + + std::unique_ptr> + result(new std::vector()); + + std::string prefix = convert_tablename_to_filepath(dir_path, db, table); + foreach_file_in_db_dirs(dir_path, [&](const char *file_name)->bool { + if (!strncmp(file_name, prefix.c_str(), prefix.size())) { + DBUG_ASSERT(strlen(file_name) >= prefix.size()); + if (file_name[prefix.size()] == '.' || + !strncmp(file_name + prefix.size(), "#P#", strlen("#P#"))) + result->push_back(std::string(file_name)); + } + return true; + }); + + return result; +} + +static +bool process_remove( + const char *datadir_path, + ds_ctxt_t *ds, + const Entry &entry, + bool remove_frm) { + + if (check_if_skip_table( + std::string(entry.db).append("/").append(entry.table).c_str())) + return true; + + auto ext_it = engine_exts.find(entry.engine); + if (ext_it == engine_exts.end()) + return true; + + std::string file_preffix = convert_tablename_to_filepath(datadir_path, + entry.db, entry.table); + const char *dst_preffix = convert_dst(file_preffix.c_str()); + + for (const char *ext : ext_it->second) { + std::string old_name(dst_preffix); + if (!entry.partitioned) + old_name.append(ext); + else + old_name.append("#P#*"); + if (ds_remove(ds, old_name.c_str())) { + msg("Failed to remove %s.", old_name.c_str()); + return false; + } + } + + if (remove_frm) { + std::string old_frm_name(dst_preffix); + old_frm_name.append(frm_ext); + if (ds_remove(ds, old_frm_name.c_str())) { + msg("Failed to remove %s.", old_frm_name.c_str()); + return false; + } + } + return true; + +} + +static +bool process_recopy( + const char *datadir_path, + ds_ctxt_t *ds, + const Entry &entry, + const tables_t &tables) { + + if (check_if_skip_table( + std::string(entry.db).append("/").append(entry.table).c_str())) + return true; + + const std::string &new_table_id = + entry.new_id.empty() ? entry.id : entry.new_id; + DBUG_ASSERT(!new_table_id.empty()); + const std::string &new_table = + entry.new_table.empty() ? entry.table : entry.new_table; + DBUG_ASSERT(!new_table.empty()); + const std::string &new_db = + entry.new_db.empty() ? entry.db : entry.new_db; + DBUG_ASSERT(!new_db.empty()); + const std::string &new_engine = + entry.new_engine.empty() ? entry.engine : entry.new_engine; + DBUG_ASSERT(!new_engine.empty()); + + if (entry.type != Entry::Type::BULK_INSERT) { + auto table_it = tables.find(table_key(new_db, new_table)); + if (table_it != tables.end() && + table_it->second == new_table_id) + return true; + } + + if (!entry.new_engine.empty() && + entry.engine != entry.new_engine && + !known_engine(entry.new_engine)) { + return process_remove(datadir_path, ds, entry, false); + } + + if ((entry.partitioned || entry.new_partitioned) && + !process_remove(datadir_path, ds, entry, false)) + return false; + + if (entry.partitioned || entry.new_partitioned) { + auto files = find_table_files(datadir_path, new_db, new_table); + if (!files.get()) + return true; + for (const auto &file : *files) { + const char *dst_path = convert_dst(file.c_str()); + if (!ds->copy_file(file.c_str(), dst_path, 0, true)) { + msg("Failed to re-copy %s.", file.c_str()); + return false; + } + } + return true; + } + + auto ext_it = engine_exts.find(new_engine); + if (ext_it == engine_exts.end()) + return false; + + for (const char *ext : ext_it->second) { + std::string file_name = + convert_tablename_to_filepath(datadir_path, new_db, new_table). + append(ext); + const char *dst_path = convert_dst(file_name.c_str()); + if (file_exists(file_name.c_str()) && + !ds->copy_file(file_name.c_str(), dst_path, 0, true)) { + msg("Failed to re-copy %s.", file_name.c_str()); + return false; + } + } + + std::string frm_file = + convert_tablename_to_filepath(datadir_path, new_db, new_table). + append(frm_ext); + const char *frm_dst_path = convert_dst(frm_file.c_str()); + if (file_exists(frm_file.c_str()) && + !ds->copy_file(frm_file.c_str(), frm_dst_path, 0, true)) { + msg("Failed to re-copy %s.", frm_file.c_str()); + return false; + } + + return true; +} + +static +bool process_rename( + const char *datadir_path, + ds_ctxt_t *ds, + const Entry &entry) { + + if (check_if_skip_table( + std::string(entry.db).append("/").append(entry.table).c_str())) + return true; + + DBUG_ASSERT(entry.db != "partition"); + + auto ext_it = engine_exts.find(entry.engine); + if (ext_it == engine_exts.end()) + return false; + + std::string new_preffix = convert_tablename_to_filepath(datadir_path, + entry.new_db, entry.new_table); + const char *dst_path = convert_dst(new_preffix.c_str()); + + std::string old_preffix = convert_tablename_to_filepath(datadir_path, + entry.db, entry.table); + const char *src_path = convert_dst(old_preffix.c_str()); + + for (const char *ext : ext_it->second) { + std::string old_name(src_path); + old_name.append(ext); + std::string new_name(dst_path); + new_name.append(ext); + if (ds_rename(ds, old_name.c_str(), new_name.c_str())) { + msg("Failed to rename %s to %s.", + old_name.c_str(), new_name.c_str()); + return false; + } + } + + std::string new_frm_file = new_preffix + frm_ext; + const char *new_frm_dst = convert_dst(new_frm_file.c_str()); + if (file_exists(new_frm_file.c_str()) && + !ds->copy_file(new_frm_file.c_str(), new_frm_dst, 0, true)) { + msg("Failed to re-copy %s.", new_frm_file.c_str()); + return false; + } + +// TODO: return this code if .frm is copied not under BLOCK_DDL +/* + std::string old_frm_name(src_path); + old_frm_name.append(frm_ext); + std::string new_frm_name(dst_path); + new_frm_name.append(frm_ext); + if (ds_rename(ds, old_frm_name.c_str(), new_frm_name.c_str())) { + msg("Failed to rename %s to %s.", + old_frm_name.c_str(), new_frm_name.c_str()); + return false; + } +*/ + return true; +} + +bool backup( + const char *datadir_path, + ds_ctxt_t *ds, + const tables_t &tables) { + DBUG_ASSERT(datadir_path); + DBUG_ASSERT(ds); + char ddl_log_path[FN_REFLEN]; + fn_format(ddl_log_path, "ddl", datadir_path, ".log", 0); + std::vector> entries; + + std::unordered_set processed_tables; + std::unordered_set dropped_databases; + + bool parsing_result = + parse(ddl_log_path, [&](std::unique_ptr entry)->bool { + + if (entry->engine == database_keyword) + return process_database(datadir_path, ds, *entry, dropped_databases); + + if (!known_engine(entry->engine) && !known_engine(entry->new_engine)) + return true; + + if (entry->type == Entry::Type::CREATE || + (entry->type == Entry::Type::ALTER && + !entry->new_engine.empty() && + entry->engine != entry->new_engine)) { + if (!process_recopy(datadir_path, ds, *entry, tables)) + return false; + processed_tables.insert(table_key(entry->db, entry->table)); + if (entry->type == Entry::Type::ALTER) + processed_tables.insert(table_key(entry->new_db, entry->new_table)); + return true; + } + + if (entry->type == Entry::Type::DROP) { + if (!process_remove(datadir_path, ds, *entry, true)) + return false; + processed_tables.insert(table_key(entry->db, entry->table)); + return true; + } + if (entry->type == Entry::Type::RENAME) { + if (entry->partitioned) { + if (!process_remove(datadir_path, ds, *entry, true)) + return false; + Entry recopy_entry { + entry->type, + {}, + entry->new_engine.empty() ? entry->engine : entry->new_engine, + true, + entry->new_db, + entry->new_table, + entry->new_id, + {}, true, {}, {}, {} + }; + if (!process_recopy(datadir_path, ds, recopy_entry, tables)) + return false; + } + else if (!process_rename(datadir_path, ds, *entry)) + return false; + processed_tables.insert(table_key(entry->db, entry->table)); + processed_tables.insert(table_key(entry->new_db, entry->new_table)); + return true; + } + + entries.push_back(std::move(entry)); + return true; + + }); + + if (!parsing_result) + return false; + + + while (!entries.empty()) { + auto entry = std::move(entries.back()); + entries.pop_back(); + auto tk = table_key( + entry->new_db.empty() ? entry->db : entry->new_db, + entry->new_table.empty() ? entry->table : entry->new_table); + if (dropped_databases.count(entry->db) || + dropped_databases.count(entry->new_db)) + continue; + if (processed_tables.count(tk)) + continue; + processed_tables.insert(std::move(tk)); + if (!process_recopy(datadir_path, ds, *entry, tables)) + return false; + } + + return true; +} + +} // namespace ddl_log diff --git a/extra/mariabackup/ddl_log.h b/extra/mariabackup/ddl_log.h new file mode 100644 index 00000000000..5cac3e5dcfc --- /dev/null +++ b/extra/mariabackup/ddl_log.h @@ -0,0 +1,15 @@ +#pragma once +#include "my_global.h" +#include "datasink.h" +#include "aria_backup_client.h" +#include +#include +#include +#include + +namespace ddl_log { + +typedef std::unordered_map tables_t; +bool backup(const char *datadir_path, ds_ctxt_t *ds, const tables_t &tables); + +} // namespace ddl_log diff --git a/extra/mariabackup/ds_buffer.cc b/extra/mariabackup/ds_buffer.cc index d6a420951cb..bc1d466350a 100644 --- a/extra/mariabackup/ds_buffer.cc +++ b/extra/mariabackup/ds_buffer.cc @@ -44,7 +44,7 @@ typedef struct { static ds_ctxt_t *buffer_init(const char *root); static ds_file_t *buffer_open(ds_ctxt_t *ctxt, const char *path, - MY_STAT *mystat); + const MY_STAT *mystat, bool rewrite); static int buffer_write(ds_file_t *file, const uchar *buf, size_t len); static int buffer_close(ds_file_t *file); static void buffer_deinit(ds_ctxt_t *ctxt); @@ -53,8 +53,11 @@ datasink_t datasink_buffer = { &buffer_init, &buffer_open, &buffer_write, + nullptr, &buffer_close, &dummy_remove, + nullptr, + nullptr, &buffer_deinit }; @@ -84,8 +87,10 @@ buffer_init(const char *root) } static ds_file_t * -buffer_open(ds_ctxt_t *ctxt, const char *path, MY_STAT *mystat) +buffer_open(ds_ctxt_t *ctxt, const char *path, + const MY_STAT *mystat, bool rewrite) { + DBUG_ASSERT(rewrite == false); ds_buffer_ctxt_t *buffer_ctxt; ds_ctxt_t *pipe_ctxt; ds_file_t *dst_file; diff --git a/extra/mariabackup/ds_compress.cc b/extra/mariabackup/ds_compress.cc index f7a9b7a1fbd..0cb52e978db 100644 --- a/extra/mariabackup/ds_compress.cc +++ b/extra/mariabackup/ds_compress.cc @@ -65,7 +65,7 @@ extern ulonglong xtrabackup_compress_chunk_size; static ds_ctxt_t *compress_init(const char *root); static ds_file_t *compress_open(ds_ctxt_t *ctxt, const char *path, - MY_STAT *mystat); + const MY_STAT *mystat, bool rewrite); static int compress_write(ds_file_t *file, const uchar *buf, size_t len); static int compress_close(ds_file_t *file); static void compress_deinit(ds_ctxt_t *ctxt); @@ -74,8 +74,11 @@ datasink_t datasink_compress = { &compress_init, &compress_open, &compress_write, + nullptr, &compress_close, &dummy_remove, + nullptr, + nullptr, &compress_deinit }; @@ -116,8 +119,10 @@ compress_init(const char *root) static ds_file_t * -compress_open(ds_ctxt_t *ctxt, const char *path, MY_STAT *mystat) +compress_open(ds_ctxt_t *ctxt, const char *path, + const MY_STAT *mystat, bool rewrite) { + DBUG_ASSERT(rewrite == false); ds_compress_ctxt_t *comp_ctxt; ds_ctxt_t *dest_ctxt; ds_file_t *dest_file; diff --git a/extra/mariabackup/ds_local.cc b/extra/mariabackup/ds_local.cc index f86612b951a..ff2021fc035 100644 --- a/extra/mariabackup/ds_local.cc +++ b/extra/mariabackup/ds_local.cc @@ -42,8 +42,9 @@ typedef struct { static ds_ctxt_t *local_init(const char *root); static ds_file_t *local_open(ds_ctxt_t *ctxt, const char *path, - MY_STAT *mystat); + const MY_STAT *mystat, bool rewrite); static int local_write(ds_file_t *file, const uchar *buf, size_t len); +static int local_seek_set(ds_file_t *file, my_off_t offset); static int local_close(ds_file_t *file); static void local_deinit(ds_ctxt_t *ctxt); @@ -52,13 +53,20 @@ static int local_remove(const char *path) return unlink(path); } +static int local_rename( + ds_ctxt_t *ctxt, const char *old_path, const char *new_path); +static int local_mremove(ds_ctxt_t *ctxt, const char *path); + extern "C" { datasink_t datasink_local = { &local_init, &local_open, &local_write, + &local_seek_set, &local_close, &local_remove, + &local_rename, + &local_mremove, &local_deinit }; } @@ -89,7 +97,7 @@ local_init(const char *root) static ds_file_t * local_open(ds_ctxt_t *ctxt, const char *path, - MY_STAT *mystat __attribute__((unused))) + const MY_STAT *mystat __attribute__((unused)), bool rewrite) { char fullpath[FN_REFLEN]; char dirpath[FN_REFLEN]; @@ -111,8 +119,10 @@ local_open(ds_ctxt_t *ctxt, const char *path, return NULL; } - fd = my_create(fullpath, 0, O_WRONLY | O_BINARY | O_EXCL | O_NOFOLLOW, - MYF(MY_WME)); + // TODO: check in Windows and set the corresponding flags on fail + fd = my_create(fullpath, 0, + O_WRONLY | O_BINARY | (rewrite ? O_TRUNC : O_EXCL) | O_NOFOLLOW, + MYF(MY_WME)); if (fd < 0) { return NULL; } @@ -194,8 +204,8 @@ static void init_ibd_data(ds_local_file_t *local_file, const uchar *buf, size_t return; } - auto flags = mach_read_from_4(&buf[FIL_PAGE_DATA + FSP_SPACE_FLAGS]); - auto ssize = FSP_FLAGS_GET_PAGE_SSIZE(flags); + uint32_t flags = mach_read_from_4(&buf[FIL_PAGE_DATA + FSP_SPACE_FLAGS]); + uint32_t ssize = FSP_FLAGS_GET_PAGE_SSIZE(flags); local_file->pagesize= ssize == 0 ? UNIV_PAGE_SIZE_ORIG : ((UNIV_ZIP_SIZE_MIN >> 1) << ssize); local_file->compressed = fil_space_t::full_crc32(flags) ? fil_space_t::is_compressed(flags) @@ -239,6 +249,15 @@ local_write(ds_file_t *file, const uchar *buf, size_t len) return 1; } +static +int +local_seek_set(ds_file_t *file, my_off_t offset) { + ds_local_file_t *local_file= (ds_local_file_t *)file->ptr; + if (my_seek(local_file->fd, offset, SEEK_SET, MYF(0)) == MY_FILEPOS_ERROR) + return 1; + return 0; +} + /* Set EOF at file's current position.*/ static int set_eof(File fd) { @@ -276,3 +295,77 @@ local_deinit(ds_ctxt_t *ctxt) my_free(ctxt->root); my_free(ctxt); } + + +static int local_rename( + ds_ctxt_t *ctxt, const char *old_path, const char *new_path) { + char full_old_path[FN_REFLEN]; + char full_new_path[FN_REFLEN]; + fn_format(full_old_path, old_path, ctxt->root, "", MYF(MY_RELATIVE_PATH)); + fn_format(full_new_path, new_path, ctxt->root, "", MYF(MY_RELATIVE_PATH)); + // Ignore errors as .frm files can me copied separately. + // TODO: return error processing here after the corresponding changes in + // xtrabackup.cc + (void)my_rename(full_old_path, full_new_path, MYF(0)); +// if (my_rename(full_old_path, full_new_path, MYF(0))) { +// msg("Failed to rename file %s to %s", old_path, new_path); +// return 1; +// } + return 0; +} + +// It's ok if destination does not contain the file or folder +static int local_mremove(ds_ctxt_t *ctxt, const char *path) { + char full_path[FN_REFLEN]; + fn_format(full_path, path, ctxt->root, "", MYF(MY_RELATIVE_PATH)); + size_t full_path_len = strlen(full_path); + if (full_path[full_path_len - 1] == '*') { + full_path[full_path_len - 1] = '\0'; + char *preffix = strrchr(full_path, '/'); + const char *full_path_dir = full_path; + size_t preffix_len; + if (preffix) { + preffix_len = (full_path_len - 1) - (preffix - full_path); + *(preffix++) = '\0'; + } + else { + preffix = full_path; + preffix_len = full_path_len - 1; + full_path_dir= IF_WIN(".\\", "./"); + } + if (!preffix_len) + return 0; + MY_DIR *dir= my_dir(full_path_dir, 0); + if (!dir) + return 0; + for (size_t i = 0; i < dir->number_of_files; ++i) { + char full_fpath[FN_REFLEN]; + if (strncmp(dir->dir_entry[i].name, preffix, preffix_len)) + continue; + fn_format(full_fpath, dir->dir_entry[i].name, + full_path_dir, "", MYF(MY_RELATIVE_PATH)); + (void)my_delete(full_fpath, MYF(0)); + } + my_dirend(dir); + } + else { + MY_STAT stat; + if (!my_stat(full_path, &stat, MYF(0))) + return 0; + MY_DIR *dir= my_dir(full_path, 0); + if (!dir) { + // TODO: check for error here if necessary + (void)my_delete(full_path, MYF(0)); + return 0; + } + for (size_t i = 0; i < dir->number_of_files; ++i) { + char full_fpath[FN_REFLEN]; + fn_format(full_fpath, dir->dir_entry[i].name, + full_path, "", MYF(MY_RELATIVE_PATH)); + (void)my_delete(full_fpath, MYF(0)); + } + my_dirend(dir); + (void)my_rmtree(full_path, MYF(0)); + } + return 0; +} diff --git a/extra/mariabackup/ds_stdout.cc b/extra/mariabackup/ds_stdout.cc index a9639ff7739..3fc0873b6ca 100644 --- a/extra/mariabackup/ds_stdout.cc +++ b/extra/mariabackup/ds_stdout.cc @@ -30,7 +30,7 @@ typedef struct { static ds_ctxt_t *stdout_init(const char *root); static ds_file_t *stdout_open(ds_ctxt_t *ctxt, const char *path, - MY_STAT *mystat); + const MY_STAT *mystat, bool rewrite); static int stdout_write(ds_file_t *file, const uchar *buf, size_t len); static int stdout_close(ds_file_t *file); static void stdout_deinit(ds_ctxt_t *ctxt); @@ -39,8 +39,11 @@ datasink_t datasink_stdout = { &stdout_init, &stdout_open, &stdout_write, + nullptr, &stdout_close, &dummy_remove, + nullptr, + nullptr, &stdout_deinit }; @@ -61,8 +64,9 @@ static ds_file_t * stdout_open(ds_ctxt_t *ctxt __attribute__((unused)), const char *path __attribute__((unused)), - MY_STAT *mystat __attribute__((unused))) + const MY_STAT *mystat __attribute__((unused)), bool rewrite) { + DBUG_ASSERT(rewrite == false); ds_stdout_file_t *stdout_file; ds_file_t *file; size_t pathlen; diff --git a/extra/mariabackup/ds_tmpfile.cc b/extra/mariabackup/ds_tmpfile.cc index 80b9d3bb4d0..6bafee25971 100644 --- a/extra/mariabackup/ds_tmpfile.cc +++ b/extra/mariabackup/ds_tmpfile.cc @@ -41,7 +41,7 @@ typedef struct { static ds_ctxt_t *tmpfile_init(const char *root); static ds_file_t *tmpfile_open(ds_ctxt_t *ctxt, const char *path, - MY_STAT *mystat); + const MY_STAT *mystat, bool rewrite); static int tmpfile_write(ds_file_t *file, const uchar *buf, size_t len); static int tmpfile_close(ds_file_t *file); static void tmpfile_deinit(ds_ctxt_t *ctxt); @@ -50,8 +50,11 @@ datasink_t datasink_tmpfile = { &tmpfile_init, &tmpfile_open, &tmpfile_write, + nullptr, &tmpfile_close, &dummy_remove, + nullptr, + nullptr, &tmpfile_deinit }; @@ -80,8 +83,9 @@ tmpfile_init(const char *root) static ds_file_t * tmpfile_open(ds_ctxt_t *ctxt, const char *path, - MY_STAT *mystat) + const MY_STAT *mystat, bool rewrite) { + DBUG_ASSERT(rewrite == false); ds_tmpfile_ctxt_t *tmpfile_ctxt; char tmp_path[FN_REFLEN]; ds_tmp_file_t *tmp_file; diff --git a/extra/mariabackup/ds_xbstream.cc b/extra/mariabackup/ds_xbstream.cc index 3bf8bd086c2..96e0cf7aea3 100644 --- a/extra/mariabackup/ds_xbstream.cc +++ b/extra/mariabackup/ds_xbstream.cc @@ -40,24 +40,31 @@ General streaming interface */ static ds_ctxt_t *xbstream_init(const char *root); static ds_file_t *xbstream_open(ds_ctxt_t *ctxt, const char *path, - MY_STAT *mystat); + const MY_STAT *mystat, bool rewrite); static int xbstream_write(ds_file_t *file, const uchar *buf, size_t len); +static int xbstream_seek_set(ds_file_t *file, my_off_t offset); static int xbstream_close(ds_file_t *file); static void xbstream_deinit(ds_ctxt_t *ctxt); +static int xbstream_rename( + ds_ctxt_t *ctxt, const char *old_path, const char *new_path); +static int xbstream_mremove(ds_ctxt_t *ctxt, const char *path); + datasink_t datasink_xbstream = { &xbstream_init, &xbstream_open, &xbstream_write, + &xbstream_seek_set, &xbstream_close, &dummy_remove, + &xbstream_rename, + &xbstream_mremove, &xbstream_deinit }; static ssize_t -my_xbstream_write_callback(xb_wstream_file_t *f __attribute__((unused)), - void *userdata, const void *buf, size_t len) +my_xbstream_write_callback(void *userdata, const void *buf, size_t len) { ds_stream_ctxt_t *stream_ctxt; @@ -89,7 +96,7 @@ xbstream_init(const char *root __attribute__((unused))) goto err; } - xbstream = xb_stream_write_new(); + xbstream = xb_stream_write_new(my_xbstream_write_callback, stream_ctxt); if (xbstream == NULL) { msg("xb_stream_write_new() failed."); goto err; @@ -108,7 +115,8 @@ err: static ds_file_t * -xbstream_open(ds_ctxt_t *ctxt, const char *path, MY_STAT *mystat) +xbstream_open(ds_ctxt_t *ctxt, const char *path, + const MY_STAT *mystat, bool rewrite) { ds_file_t *file; ds_stream_file_t *stream_file; @@ -144,9 +152,7 @@ xbstream_open(ds_ctxt_t *ctxt, const char *path, MY_STAT *mystat) xbstream = stream_ctxt->xbstream; - xbstream_file = xb_stream_write_open(xbstream, path, mystat, - stream_ctxt, - my_xbstream_write_callback); + xbstream_file = xb_stream_write_open(xbstream, path, mystat, rewrite); if (xbstream_file == NULL) { msg("xb_stream_write_open() failed."); @@ -190,6 +196,45 @@ xbstream_write(ds_file_t *file, const uchar *buf, size_t len) return 0; } +static +int +xbstream_seek_set(ds_file_t *file, my_off_t offset) +{ + ds_stream_file_t *stream_file; + xb_wstream_file_t *xbstream_file; + + + stream_file = (ds_stream_file_t *) file->ptr; + + xbstream_file = stream_file->xbstream_file; + + if (xb_stream_write_seek_set(xbstream_file, offset)) { + msg("xb_stream_write_seek_set() failed."); + return 1; + } + + return 0; +} + +static +int +xbstream_mremove(ds_ctxt_t *ctxt, const char *path) { + ds_stream_ctxt_t *stream_ctxt = + reinterpret_cast(ctxt->ptr); + xb_wstream_t *xbstream = stream_ctxt->xbstream; + return xb_stream_write_remove(xbstream, path); +} + +static +int +xbstream_rename( + ds_ctxt_t *ctxt, const char *old_path, const char *new_path) { + ds_stream_ctxt_t *stream_ctxt = + reinterpret_cast(ctxt->ptr); + xb_wstream_t *xbstream = stream_ctxt->xbstream; + return xb_stream_write_rename(xbstream, old_path, new_path); +} + static int xbstream_close(ds_file_t *file) diff --git a/extra/mariabackup/xb_plugin.cc b/extra/mariabackup/encryption_plugin.cc similarity index 83% rename from extra/mariabackup/xb_plugin.cc rename to extra/mariabackup/encryption_plugin.cc index 7470d376eaa..d71ceaac7f4 100644 --- a/extra/mariabackup/xb_plugin.cc +++ b/extra/mariabackup/encryption_plugin.cc @@ -1,4 +1,4 @@ -/* Copyright (c) 2017, 2022, MariaDB Corporation. +/* Copyright (c) 2017, MariaDB Corporation. This program is free software; you can redistribute it and/or modify it under the terms of the GNU General Public License as published by @@ -17,18 +17,18 @@ #include #include #include -#include +#include #include #include #include #include #include -#include +#include extern struct st_maria_plugin *mysql_optional_plugins[]; extern struct st_maria_plugin *mysql_mandatory_plugins[]; -static void xb_plugin_init(int argc, char **argv); +static void encryption_plugin_init(int argc, char **argv); extern char *xb_plugin_load; extern char *xb_plugin_dir; @@ -42,7 +42,7 @@ const char *QUERY_PLUGIN = " OR (plugin_type = 'DAEMON' AND plugin_name LIKE 'provider\\_%')" " AND plugin_status='ACTIVE'"; -std::string xb_plugin_config; +std::string encryption_plugin_config; static void add_to_plugin_load_list(const char *plugin_def) { @@ -52,16 +52,16 @@ static void add_to_plugin_load_list(const char *plugin_def) static char XTRABACKUP_EXE[] = "xtrabackup"; /* - Read "plugin-load" value from backup-my.cnf during prepare phase. + Read "plugin-load" value (encryption plugin) from backup-my.cnf during + prepare phase. The value is stored during backup phase. */ -static std::string get_plugin_from_cnf(const char *dir) +static std::string get_encryption_plugin_from_cnf() { - std::string path = dir + std::string("/backup-my.cnf"); - FILE *f = fopen(path.c_str(), "r"); + FILE *f = fopen("backup-my.cnf", "r"); if (!f) { - die("Can't open %s for reading", path.c_str()); + die("Can't open backup-my.cnf for reading"); } char line[512]; std::string plugin_load; @@ -80,7 +80,7 @@ static std::string get_plugin_from_cnf(const char *dir) } -void xb_plugin_backup_init(MYSQL *mysql) +void encryption_plugin_backup_init(MYSQL *mysql) { MYSQL_RES *result; MYSQL_ROW row; @@ -163,7 +163,7 @@ void xb_plugin_backup_init(MYSQL *mysql) mysql_free_result(result); } - xb_plugin_config = oss.str(); + encryption_plugin_config = oss.str(); argc = 0; argv[argc++] = XTRABACKUP_EXE; @@ -175,23 +175,23 @@ void xb_plugin_backup_init(MYSQL *mysql) } argv[argc] = 0; - xb_plugin_init(argc, argv); + encryption_plugin_init(argc, argv); } -const char *xb_plugin_get_config() +const char *encryption_plugin_get_config() { - return xb_plugin_config.c_str(); + return encryption_plugin_config.c_str(); } extern int finalize_encryption_plugin(st_plugin_int *plugin); -void xb_plugin_prepare_init(int argc, char **argv, const char *dir) +void encryption_plugin_prepare_init(int argc, char **argv) { - std::string plugin_load= get_plugin_from_cnf(dir ? dir : "."); + std::string plugin_load= get_encryption_plugin_from_cnf(); if (plugin_load.size()) { - msg("Loading plugins from %s", plugin_load.c_str()); + msg("Loading encryption plugin from %s", plugin_load.c_str()); } else { @@ -211,19 +211,19 @@ void xb_plugin_prepare_init(int argc, char **argv, const char *dir) new_argv[0] = XTRABACKUP_EXE; memcpy(&new_argv[1], argv, argc*sizeof(char *)); - xb_plugin_init(argc+1, new_argv); + encryption_plugin_init(argc+1, new_argv); delete[] new_argv; } -static void xb_plugin_init(int argc, char **argv) +static void encryption_plugin_init(int argc, char **argv) { /* Patch optional and mandatory plugins, we only need to load the one in xb_plugin_load. */ mysql_optional_plugins[0] = mysql_mandatory_plugins[0] = 0; plugin_maturity = MariaDB_PLUGIN_MATURITY_UNKNOWN; /* mariabackup accepts all plugins */ - msg("Loading plugins"); + msg("Loading encryption plugin"); for (int i= 1; i < argc; i++) - msg("\t Plugin parameter : '%s'", argv[i]); + msg("\t Encryption plugin parameter : '%s'", argv[i]); plugin_init(&argc, argv, PLUGIN_INIT_SKIP_PLUGIN_TABLE); } diff --git a/extra/mariabackup/encryption_plugin.h b/extra/mariabackup/encryption_plugin.h new file mode 100644 index 00000000000..16d74790254 --- /dev/null +++ b/extra/mariabackup/encryption_plugin.h @@ -0,0 +1,7 @@ +#include +#include +extern void encryption_plugin_backup_init(MYSQL *mysql); +extern const char* encryption_plugin_get_config(); +extern void encryption_plugin_prepare_init(int argc, char **argv); + +//extern void encryption_plugin_init(int argc, char **argv); diff --git a/extra/mariabackup/innobackupex.cc b/extra/mariabackup/innobackupex.cc index b925b41552d..e6cd8a8814e 100644 --- a/extra/mariabackup/innobackupex.cc +++ b/extra/mariabackup/innobackupex.cc @@ -78,7 +78,6 @@ my_bool opt_ibx_galera_info = FALSE; my_bool opt_ibx_slave_info = FALSE; my_bool opt_ibx_no_lock = FALSE; my_bool opt_ibx_safe_slave_backup = FALSE; -my_bool opt_ibx_rsync = FALSE; my_bool opt_ibx_force_non_empty_dirs = FALSE; my_bool opt_ibx_noversioncheck = FALSE; my_bool opt_ibx_no_backup_locks = FALSE; @@ -297,15 +296,6 @@ static struct my_option ibx_long_options[] = (uchar *) &opt_ibx_safe_slave_backup, 0, GET_BOOL, NO_ARG, 0, 0, 0, 0, 0, 0}, - {"rsync", OPT_RSYNC, "Uses the rsync utility to optimize local file " - "transfers. When this option is specified, innobackupex uses rsync " - "to copy all non-InnoDB files instead of spawning a separate cp for " - "each file, which can be much faster for servers with a large number " - "of databases or tables. This option cannot be used together with " - "--stream.", - (uchar *) &opt_ibx_rsync, (uchar *) &opt_ibx_rsync, - 0, GET_BOOL, NO_ARG, 0, 0, 0, 0, 0, 0}, - {"force-non-empty-directories", OPT_FORCE_NON_EMPTY_DIRS, "This " "option, when specified, makes --copy-back or --move-back transfer " "files to non-empty directories. Note that no existing files will be " @@ -864,10 +854,8 @@ ibx_init() opt_slave_info = opt_ibx_slave_info; opt_no_lock = opt_ibx_no_lock; opt_safe_slave_backup = opt_ibx_safe_slave_backup; - opt_rsync = opt_ibx_rsync; opt_force_non_empty_dirs = opt_ibx_force_non_empty_dirs; opt_noversioncheck = opt_ibx_noversioncheck; - opt_no_backup_locks = opt_ibx_no_backup_locks; opt_decompress = opt_ibx_decompress; opt_incremental_history_name = opt_ibx_incremental_history_name; diff --git a/extra/mariabackup/thread_pool.cc b/extra/mariabackup/thread_pool.cc new file mode 100644 index 00000000000..e18581f4b24 --- /dev/null +++ b/extra/mariabackup/thread_pool.cc @@ -0,0 +1,50 @@ +#include "thread_pool.h" +#include "common.h" + +bool ThreadPool::start(size_t threads_count) { + if (!m_stopped) + return false; + m_stopped = false; + for (unsigned i = 0; i < threads_count; ++i) + m_threads.emplace_back(&ThreadPool::thread_func, this, i); + return true; +} + +void ThreadPool::stop() { + if (m_stopped) + return; + m_stop = true; + m_cv.notify_all(); + for (auto &t : m_threads) + t.join(); + m_stopped = true; +}; + +void ThreadPool::push(ThreadPool::job_t &&j) { + std::unique_lock lock(m_mutex); + m_jobs.push(j); + lock.unlock(); + m_cv.notify_one(); +} + +void ThreadPool::thread_func(unsigned thread_num) { + if (my_thread_init()) + die("Can't init mysql thread"); + std::unique_lock lock(m_mutex); + while(true) { + if (m_stop) + goto exit; + while (!m_jobs.empty()) { + if (m_stop) + goto exit; + job_t j = std::move(m_jobs.front()); + m_jobs.pop(); + lock.unlock(); + j(thread_num); + lock.lock(); + } + m_cv.wait(lock, [&] { return m_stop || !m_jobs.empty(); }); + } +exit: + my_thread_end(); +} diff --git a/extra/mariabackup/thread_pool.h b/extra/mariabackup/thread_pool.h new file mode 100644 index 00000000000..10ad74c6220 --- /dev/null +++ b/extra/mariabackup/thread_pool.h @@ -0,0 +1,62 @@ +#pragma once +#include +#include +#include +#include +#include +#include +#include +#include "trx0sys.h" + +class ThreadPool { +public: + typedef std::function job_t; + + ThreadPool() { m_stop = false; m_stopped = true; } + ThreadPool (ThreadPool &&other) = delete; + ThreadPool & operator= (ThreadPool &&other) = delete; + ThreadPool(const ThreadPool &) = delete; + ThreadPool & operator= (const ThreadPool &) = delete; + + bool start(size_t threads_count); + void stop(); + void push(job_t &&j); + size_t threads_count() const { return m_threads.size(); } +private: + void thread_func(unsigned thread_num); + std::mutex m_mutex; + std::condition_variable m_cv; + std::queue m_jobs; + std::atomic m_stop; + std::atomic m_stopped; + std::vector m_threads; +}; + +class TasksGroup { +public: + TasksGroup(ThreadPool &thread_pool) : m_thread_pool(thread_pool) { + m_tasks_count = 0; + m_tasks_result = 1; + } + void push_task(ThreadPool::job_t &&j) { + ++m_tasks_count; + m_thread_pool.push(std::forward(j)); + } + void finish_task(int res) { + --m_tasks_count; + m_tasks_result.fetch_and(res); + } + int get_result() const { return m_tasks_result; } + bool is_finished() const { + return !m_tasks_count; + } + bool wait_for_finish() { + while (!is_finished()) + std::this_thread::sleep_for(std::chrono::milliseconds(1)); + return get_result(); + } +private: + ThreadPool &m_thread_pool; + std::atomic m_tasks_count; + std::atomic m_tasks_result; +}; diff --git a/extra/mariabackup/write_filt.cc b/extra/mariabackup/write_filt.cc index 052cea26ef6..13f19ca6b6a 100644 --- a/extra/mariabackup/write_filt.cc +++ b/extra/mariabackup/write_filt.cc @@ -144,6 +144,18 @@ wf_incremental_process(xb_write_filt_ctxt_t *ctxt, ds_file_t *dstfile) return false; } + /* Check whether TRX_SYS page has been changed */ + if (mach_read_from_4(page + FIL_PAGE_SPACE_ID) + == TRX_SYS_SPACE + && mach_read_from_4(page + FIL_PAGE_OFFSET) + == TRX_SYS_PAGE_NO) { + msg(cursor->thread_n, + "--incremental backup is impossible if " + "the server had been restarted with " + "different innodb_undo_tablespaces."); + return false; + } + /* updated page */ if (cp->npages == page_size / 4) { /* flush buffer */ diff --git a/extra/mariabackup/wsrep.cc b/extra/mariabackup/wsrep.cc index acaf5c50e7c..15463a85476 100644 --- a/extra/mariabackup/wsrep.cc +++ b/extra/mariabackup/wsrep.cc @@ -55,6 +55,9 @@ permission notice: #define XB_GALERA_INFO_FILENAME "xtrabackup_galera_info" #define XB_GALERA_DONOR_INFO_FILENAME "donor_galera_info" +/* backup copy of galera info file as sent by donor */ +#define XB_GALERA_INFO_FILENAME_SST "xtrabackup_galera_info_SST" + /*********************************************************************** Store Galera checkpoint info in the 'xtrabackup_galera_info' file, if that information is present in the trx system header. Otherwise, do nothing. */ @@ -68,19 +71,45 @@ xb_write_galera_info(bool incremental_prepare) long long seqno; MY_STAT statinfo; - /* Do not overwrite an existing file to be compatible with - servers with older server versions */ - if (!incremental_prepare && - my_stat(XB_GALERA_INFO_FILENAME, &statinfo, MYF(0)) != NULL) { + xid.null(); + /* try to read last wsrep XID from innodb rsegs, we will use it + instead of galera info file received from donor + */ + if (!trx_rseg_read_wsrep_checkpoint(xid)) { + /* no worries yet, SST may have brought in galera info file + from some old MariaDB version, which does not support + wsrep XID storing in innodb rsegs + */ return; } - xid.null(); + /* if SST brought in galera info file, copy it as *_SST file + this will not be used, saved just for future reference + */ + if (my_stat(XB_GALERA_INFO_FILENAME, &statinfo, MYF(0)) != NULL) { + FILE* fp_in = fopen(XB_GALERA_INFO_FILENAME, "r"); + FILE* fp_out = fopen(XB_GALERA_INFO_FILENAME_SST, "w"); - if (!trx_rseg_read_wsrep_checkpoint(xid)) { - - return; + char buf[BUFSIZ] = {'\0'}; + size_t size; + while ((size = fread(buf, 1, BUFSIZ, fp_in))) { + if (fwrite(buf, 1, size, fp_out) != strlen(buf)) { + die( + "could not write to " + XB_GALERA_INFO_FILENAME_SST + ", errno = %d\n", + errno); + } + } + if (!feof(fp_in)) { + die( + XB_GALERA_INFO_FILENAME_SST + " not fully copied\n" + ); + } + fclose(fp_out); + fclose(fp_in); } wsrep_uuid_t uuid; @@ -97,7 +126,6 @@ xb_write_galera_info(bool incremental_prepare) "could not create " XB_GALERA_INFO_FILENAME ", errno = %d\n", errno); - exit(EXIT_FAILURE); } seqno = wsrep_xid_seqno(&xid); diff --git a/extra/mariabackup/xb_plugin.h b/extra/mariabackup/xb_plugin.h deleted file mode 100644 index fea24b6b052..00000000000 --- a/extra/mariabackup/xb_plugin.h +++ /dev/null @@ -1,5 +0,0 @@ -#include -#include -extern void xb_plugin_backup_init(MYSQL *mysql); -extern const char* xb_plugin_get_config(); -extern void xb_plugin_prepare_init(int argc, char **argv, const char *dir); diff --git a/extra/mariabackup/xbstream.cc b/extra/mariabackup/xbstream.cc index 6306806b867..d69a0029a2b 100644 --- a/extra/mariabackup/xbstream.cc +++ b/extra/mariabackup/xbstream.cc @@ -262,7 +262,7 @@ mode_create(int argc, char **argv) return 1; } - stream = xb_stream_write_new(); + stream = xb_stream_write_new(nullptr, nullptr); if (stream == NULL) { msg("%s: xb_stream_write_new() failed.", my_progname); return 1; @@ -287,7 +287,7 @@ mode_create(int argc, char **argv) goto err; } - file = xb_stream_write_open(stream, filepath, &mystat, NULL, NULL); + file = xb_stream_write_open(stream, filepath, &mystat, false); if (file == NULL) { goto err; } @@ -314,7 +314,8 @@ err: static file_entry_t * -file_entry_new(extract_ctxt_t *ctxt, const char *path, uint pathlen) +file_entry_new(extract_ctxt_t *ctxt, const char *path, uint pathlen, + uchar chunk_flags) { file_entry_t *entry; ds_file_t *file; @@ -331,7 +332,8 @@ file_entry_new(extract_ctxt_t *ctxt, const char *path, uint pathlen) } entry->pathlen = pathlen; - file = ds_open(ctxt->ds_ctxt, path, NULL); + file = ds_open(ctxt->ds_ctxt, path, NULL, + chunk_flags == XB_STREAM_FLAG_REWRITE); if (file == NULL) { msg("%s: failed to create file.", my_progname); @@ -412,10 +414,50 @@ extract_worker_thread_func(void *arg) (uchar *) chunk.path, chunk.pathlen); + if (entry && (chunk.type == XB_CHUNK_TYPE_REMOVE || + chunk.type == XB_CHUNK_TYPE_RENAME)) { + msg("%s: rename and remove chunks can not be applied to opened file: %s", + my_progname, chunk.path); + pthread_mutex_unlock(ctxt->mutex); + break; + } + + if (chunk.type == XB_CHUNK_TYPE_REMOVE) { + if (ds_remove(ctxt->ds_ctxt, chunk.path)) { + msg("%s: error on file removing: %s", my_progname, chunk.path); + pthread_mutex_unlock(ctxt->mutex); + res = XB_STREAM_READ_ERROR; + break; + } + pthread_mutex_unlock(ctxt->mutex); + continue; + } + + if (chunk.type == XB_CHUNK_TYPE_RENAME) { + if (my_hash_search(ctxt->filehash, + reinterpret_cast(chunk.data), chunk.length)) { + msg("%s: rename chunks can not be applied to opened file: %s", + my_progname, reinterpret_cast(chunk.data)); + pthread_mutex_unlock(ctxt->mutex); + break; + } + if (ds_rename(ctxt->ds_ctxt, chunk.path, + reinterpret_cast(chunk.data))) { + msg("%s: error on file renaming: %s to %s", my_progname, + reinterpret_cast(chunk.data), chunk.path); + pthread_mutex_unlock(ctxt->mutex); + res = XB_STREAM_READ_ERROR; + break; + } + pthread_mutex_unlock(ctxt->mutex); + continue; + } + if (entry == NULL) { entry = file_entry_new(ctxt, chunk.path, - chunk.pathlen); + chunk.pathlen, + chunk.flags); if (entry == NULL) { pthread_mutex_unlock(ctxt->mutex); break; @@ -432,6 +474,18 @@ extract_worker_thread_func(void *arg) pthread_mutex_unlock(ctxt->mutex); + if (chunk.type == XB_CHUNK_TYPE_SEEK) { + if (ds_seek_set(entry->file, chunk.offset)) { + msg("%s: my_seek() failed.", my_progname); + pthread_mutex_unlock(&entry->mutex); + res = XB_STREAM_READ_ERROR; + break; + } + entry->offset = chunk.offset; + pthread_mutex_unlock(&entry->mutex); + continue; + } + res = xb_stream_validate_checksum(&chunk); if (res != XB_STREAM_READ_CHUNK) { diff --git a/extra/mariabackup/xbstream.h b/extra/mariabackup/xbstream.h index 1b36ec249b6..c8b2997d402 100644 --- a/extra/mariabackup/xbstream.h +++ b/extra/mariabackup/xbstream.h @@ -29,6 +29,7 @@ Foundation, Inc., 51 Franklin Street, Fifth Floor, Boston, MA 02110-1335 USA /* Chunk flags */ /* Chunk can be ignored if unknown version/format */ #define XB_STREAM_FLAG_IGNORABLE 0x01 +#define XB_STREAM_FLAG_REWRITE 0x02 /* Magic + flags + type + path len */ #define CHUNK_HEADER_CONSTANT_LEN ((sizeof(XB_STREAM_CHUNK_MAGIC) - 1) + \ @@ -48,18 +49,21 @@ typedef enum { /************************************************************************ Write interface. */ -typedef ssize_t xb_stream_write_callback(xb_wstream_file_t *file, +typedef ssize_t xb_stream_write_callback( void *userdata, const void *buf, size_t len); -xb_wstream_t *xb_stream_write_new(void); - +xb_wstream_t *xb_stream_write_new( + xb_stream_write_callback *write_callback, void *user_data); xb_wstream_file_t *xb_stream_write_open(xb_wstream_t *stream, const char *path, - MY_STAT *mystat, void *userdata, - xb_stream_write_callback *onwrite); + const MY_STAT *mystat, bool rewrite); int xb_stream_write_data(xb_wstream_file_t *file, const void *buf, size_t len); - +int xb_stream_write_seek_set(xb_wstream_file_t *file, my_off_t offset); +int xb_stream_write_remove(xb_wstream_t *stream, const char *path); +int +xb_stream_write_rename( + xb_wstream_t *stream, const char *old_path, const char *new_path); int xb_stream_write_close(xb_wstream_file_t *file); int xb_stream_write_done(xb_wstream_t *stream); @@ -76,6 +80,9 @@ typedef enum { typedef enum { XB_CHUNK_TYPE_UNKNOWN = '\0', XB_CHUNK_TYPE_PAYLOAD = 'P', + XB_CHUNK_TYPE_RENAME = 'R', + XB_CHUNK_TYPE_REMOVE = 'D', + XB_CHUNK_TYPE_SEEK = 'S', XB_CHUNK_TYPE_EOF = 'E' } xb_chunk_type_t; diff --git a/extra/mariabackup/xbstream_read.cc b/extra/mariabackup/xbstream_read.cc index b54a98157ea..d82176ad8be 100644 --- a/extra/mariabackup/xbstream_read.cc +++ b/extra/mariabackup/xbstream_read.cc @@ -59,6 +59,9 @@ validate_chunk_type(uchar code) { switch ((xb_chunk_type_t) code) { case XB_CHUNK_TYPE_PAYLOAD: + case XB_CHUNK_TYPE_RENAME: + case XB_CHUNK_TYPE_REMOVE: + case XB_CHUNK_TYPE_SEEK: case XB_CHUNK_TYPE_EOF: return (xb_chunk_type_t) code; default: @@ -159,57 +162,91 @@ xb_stream_read_chunk(xb_rstream_t *stream, xb_rstream_chunk_t *chunk) } chunk->path[pathlen] = '\0'; - if (chunk->type == XB_CHUNK_TYPE_EOF) { + if (chunk->type == XB_CHUNK_TYPE_EOF || + chunk->type == XB_CHUNK_TYPE_REMOVE) { return XB_STREAM_READ_CHUNK; } - /* Payload length */ - F_READ(tmpbuf, 16); - ullval = uint8korr(tmpbuf); - if (ullval > (ulonglong) SIZE_T_MAX) { - msg("xb_stream_read_chunk(): chunk length is too large at " - "offset 0x%llx: 0x%llx.", (ulonglong) stream->offset, - ullval); - goto err; - } - chunk->length = (size_t) ullval; - stream->offset += 8; - - /* Payload offset */ - ullval = uint8korr(tmpbuf + 8); - if (ullval > (ulonglong) MY_OFF_T_MAX) { - msg("xb_stream_read_chunk(): chunk offset is too large at " - "offset 0x%llx: 0x%llx.", (ulonglong) stream->offset, - ullval); - goto err; - } - chunk->offset = (my_off_t) ullval; - stream->offset += 8; - - /* Reallocate the buffer if needed */ - if (chunk->length > chunk->buflen) { - chunk->data = my_realloc(PSI_NOT_INSTRUMENTED, chunk->data, chunk->length, - MYF(MY_WME | MY_ALLOW_ZERO_PTR)); - if (chunk->data == NULL) { - msg("xb_stream_read_chunk(): failed to increase buffer " - "to %lu bytes.", (ulong) chunk->length); + if (chunk->type == XB_CHUNK_TYPE_RENAME) { + F_READ(tmpbuf, 4); + size_t new_pathlen = uint4korr(tmpbuf); + if (new_pathlen >= FN_REFLEN) { + msg("xb_stream_read_chunk(): path length (%lu) for new name of 'rename'" + " chunk is too large", (ulong) new_pathlen); goto err; } - chunk->buflen = chunk->length; + chunk->length = new_pathlen; + stream->offset +=4; + } + else if (chunk->type == XB_CHUNK_TYPE_SEEK) { + F_READ(tmpbuf, 8); + chunk->offset = uint8korr(tmpbuf); + stream->offset += 8; + return XB_STREAM_READ_CHUNK; + } + else { + /* Payload length */ + F_READ(tmpbuf, 16); + ullval = uint8korr(tmpbuf); + if (ullval > (ulonglong) SIZE_T_MAX) { + msg("xb_stream_read_chunk(): chunk length is too large at " + "offset 0x%llx: 0x%llx.", (ulonglong) stream->offset, + ullval); + goto err; + } + chunk->length = (size_t) ullval; + stream->offset += 8; + + /* Payload offset */ + ullval = uint8korr(tmpbuf + 8); + if (ullval > (ulonglong) MY_OFF_T_MAX) { + msg("xb_stream_read_chunk(): chunk offset is too large at " + "offset 0x%llx: 0x%llx.", (ulonglong) stream->offset, + ullval); + goto err; + } + chunk->offset = (my_off_t) ullval; + stream->offset += 8; } - /* Checksum */ - F_READ(tmpbuf, 4); - chunk->checksum = uint4korr(tmpbuf); - chunk->checksum_offset = stream->offset; + /* Reallocate the buffer if needed, take into account trailing '\0' for + new file name in the case of XB_CHUNK_TYPE_RENAME */ + if (chunk->length + 1 > chunk->buflen) { + chunk->data = my_realloc(PSI_NOT_INSTRUMENTED, chunk->data, + chunk->length + 1, MYF(MY_WME | MY_ALLOW_ZERO_PTR)); + if (chunk->data == NULL) { + msg("xb_stream_read_chunk(): failed to increase buffer " + "to %lu bytes.", (ulong) chunk->length + 1); + goto err; + } + chunk->buflen = chunk->length + 1; + } - /* Payload */ - if (chunk->length > 0) { + if (chunk->type == XB_CHUNK_TYPE_RENAME) { + if (chunk->length == 0) { + msg("xb_stream_read_chunk(): failed to read new name for file to rename " + ": %s", chunk->path); + goto err; + } F_READ(chunk->data, chunk->length); stream->offset += chunk->length; + reinterpret_cast(chunk->data)[chunk->length] = '\0'; + ++chunk->length; } + else { + /* Checksum */ + F_READ(tmpbuf, 4); + chunk->checksum = uint4korr(tmpbuf); + chunk->checksum_offset = stream->offset; - stream->offset += 4; + /* Payload */ + if (chunk->length > 0) { + F_READ(chunk->data, chunk->length); + stream->offset += chunk->length; + } + + stream->offset += 4; + } return XB_STREAM_READ_CHUNK; diff --git a/extra/mariabackup/xbstream_write.cc b/extra/mariabackup/xbstream_write.cc index 5801e867aac..926e091becb 100644 --- a/extra/mariabackup/xbstream_write.cc +++ b/extra/mariabackup/xbstream_write.cc @@ -21,6 +21,7 @@ Foundation, Inc., 51 Franklin Street, Fifth Floor, Boston, MA 02110-1335 USA #include #include #include +#include #include "common.h" #include "xbstream.h" @@ -29,6 +30,8 @@ Foundation, Inc., 51 Franklin Street, Fifth Floor, Boston, MA 02110-1335 USA struct xb_wstream_struct { pthread_mutex_t mutex; + xb_stream_write_callback *write; + void *user_data; }; struct xb_wstream_file_struct { @@ -39,8 +42,7 @@ struct xb_wstream_file_struct { char *chunk_ptr; size_t chunk_free; my_off_t offset; - void *userdata; - xb_stream_write_callback *write; + bool rewrite; }; static int xb_stream_flush(xb_wstream_file_t *file); @@ -50,7 +52,7 @@ static int xb_stream_write_eof(xb_wstream_file_t *file); static ssize_t -xb_stream_default_write_callback(xb_wstream_file_t *file __attribute__((unused)), +xb_stream_default_write_callback( void *userdata __attribute__((unused)), const void *buf, size_t len) { @@ -60,21 +62,31 @@ xb_stream_default_write_callback(xb_wstream_file_t *file __attribute__((unused)) } xb_wstream_t * -xb_stream_write_new(void) +xb_stream_write_new( + xb_stream_write_callback *write_callback, void *user_data) { xb_wstream_t *stream; stream = (xb_wstream_t *) my_malloc(PSI_NOT_INSTRUMENTED, sizeof(xb_wstream_t), MYF(MY_FAE)); pthread_mutex_init(&stream->mutex, NULL); + if (write_callback) { +#ifdef _WIN32 + setmode(fileno(stdout), _O_BINARY); +#endif + stream->write = write_callback; + stream->user_data = user_data; + } + else { + stream->write = xb_stream_default_write_callback; + stream->user_data = user_data; + } return stream;; } xb_wstream_file_t * xb_stream_write_open(xb_wstream_t *stream, const char *path, - MY_STAT *mystat __attribute__((unused)), - void *userdata, - xb_stream_write_callback *onwrite) + const MY_STAT *mystat __attribute__((unused)), bool rewrite) { xb_wstream_file_t *file; size_t path_len; @@ -109,16 +121,7 @@ xb_stream_write_open(xb_wstream_t *stream, const char *path, file->offset = 0; file->chunk_ptr = file->chunk; file->chunk_free = XB_STREAM_MIN_CHUNK_SIZE; - if (onwrite) { -#ifdef _WIN32 - setmode(fileno(stdout), _O_BINARY); -#endif - file->userdata = userdata; - file->write = onwrite; - } else { - file->userdata = NULL; - file->write = xb_stream_default_write_callback; - } + file->rewrite = rewrite; return file; } @@ -202,7 +205,8 @@ xb_stream_write_chunk(xb_wstream_file_t *file, const void *buf, size_t len) memcpy(ptr, XB_STREAM_CHUNK_MAGIC, sizeof(XB_STREAM_CHUNK_MAGIC) - 1); ptr += sizeof(XB_STREAM_CHUNK_MAGIC) - 1; - *ptr++ = 0; /* Chunk flags */ + *ptr++ = + file->rewrite ? XB_STREAM_FLAG_REWRITE : 0; /* Chunk flags */ *ptr++ = (uchar) XB_CHUNK_TYPE_PAYLOAD; /* Chunk type */ @@ -227,11 +231,11 @@ xb_stream_write_chunk(xb_wstream_file_t *file, const void *buf, size_t len) xb_ad(ptr <= tmpbuf + sizeof(tmpbuf)); - if (file->write(file, file->userdata, tmpbuf, ptr-tmpbuf) == -1) + if (stream->write(stream->user_data, tmpbuf, ptr-tmpbuf) == -1) goto err; - if (file->write(file, file->userdata, buf, len) == -1) /* Payload */ + if (stream->write(stream->user_data, buf, len) == -1) /* Payload */ goto err; file->offset+= len; @@ -247,6 +251,38 @@ err: return 1; } +int xb_stream_write_seek_set(xb_wstream_file_t *file, my_off_t offset) +{ + /* Chunk magic + flags + chunk type + path_len + path + offset */ + uchar tmpbuf[sizeof(XB_STREAM_CHUNK_MAGIC) - 1 + 1 + 1 + 4 + + FN_REFLEN + 8]; + int error = 0; + xb_wstream_t *stream = file->stream; + uchar *ptr = tmpbuf; + /* Chunk magic */ + memcpy(ptr, XB_STREAM_CHUNK_MAGIC, sizeof(XB_STREAM_CHUNK_MAGIC) - 1); + ptr += sizeof(XB_STREAM_CHUNK_MAGIC) - 1; + *ptr++ = 0; /* Chunk flags */ + *ptr++ = (uchar) XB_CHUNK_TYPE_SEEK; /* Chunk type */ + int4store(ptr, file->path_len); /* Path length */ + ptr += 4; + memcpy(ptr, file->path, file->path_len); /* Path */ + ptr += file->path_len; + int8store(ptr, static_cast(offset)); /* Offset */ + ptr += 8; + if (xb_stream_flush(file)) + return 1; + pthread_mutex_lock(&stream->mutex); + if (stream->write(stream->user_data, tmpbuf, ptr-tmpbuf) == -1) + error = 1; + if (!error) + file->offset = offset; + pthread_mutex_unlock(&stream->mutex); + if (xb_stream_flush(file)) + return 1; + return error; +} + static int xb_stream_write_eof(xb_wstream_file_t *file) @@ -278,7 +314,7 @@ xb_stream_write_eof(xb_wstream_file_t *file) xb_ad(ptr <= tmpbuf + sizeof(tmpbuf)); - if (file->write(file, file->userdata, tmpbuf, + if (stream->write(stream->user_data, tmpbuf, (ulonglong) (ptr - tmpbuf)) == -1) goto err; @@ -291,3 +327,77 @@ err: return 1; } + + +int +xb_stream_write_remove(xb_wstream_t *stream, const char *path) { + /* Chunk magic + flags + chunk type + path_len + path */ + uchar tmpbuf[sizeof(XB_STREAM_CHUNK_MAGIC) - 1 + 1 + 1 + 4 + FN_REFLEN]; + uchar *ptr = tmpbuf; + /* Chunk magic */ + memcpy(ptr, XB_STREAM_CHUNK_MAGIC, sizeof(XB_STREAM_CHUNK_MAGIC) - 1); + ptr += sizeof(XB_STREAM_CHUNK_MAGIC) - 1; + + *ptr++ = 0; /* Chunk flags */ + + *ptr++ = (uchar) XB_CHUNK_TYPE_REMOVE; /* Chunk type */ + size_t path_len = strlen(path); + int4store(ptr, path_len); /* Path length */ + ptr += 4; + + memcpy(ptr, path, path_len); /* Path */ + ptr += path_len; + + xb_ad(ptr <= tmpbuf + sizeof(tmpbuf)); + + pthread_mutex_lock(&stream->mutex); + + ssize_t result = stream->write(stream->user_data, tmpbuf, + (ulonglong) (ptr - tmpbuf)); + + pthread_mutex_unlock(&stream->mutex); + + return result < 0; + +} + +int +xb_stream_write_rename( + xb_wstream_t *stream, const char *old_path, const char *new_path) { + /* Chunk magic + flags + chunk type + path_len + path + path_len + path*/ + uchar tmpbuf[sizeof(XB_STREAM_CHUNK_MAGIC) - 1 + 1 + 1 + + 4 + FN_REFLEN + 4 + FN_REFLEN]; + uchar *ptr = tmpbuf; + /* Chunk magic */ + memcpy(ptr, XB_STREAM_CHUNK_MAGIC, sizeof(XB_STREAM_CHUNK_MAGIC) - 1); + ptr += sizeof(XB_STREAM_CHUNK_MAGIC) - 1; + + *ptr++ = 0; /* Chunk flags */ + + *ptr++ = (uchar) XB_CHUNK_TYPE_RENAME; /* Chunk type */ + size_t path_len = strlen(old_path); + int4store(ptr, path_len); /* Path length */ + ptr += 4; + + memcpy(ptr, old_path, path_len); /* Path */ + ptr += path_len; + + path_len = strlen(new_path); + int4store(ptr, path_len); /* Path length */ + ptr += 4; + + memcpy(ptr, new_path, path_len); /* Path */ + ptr += path_len; + + xb_ad(ptr <= tmpbuf + sizeof(tmpbuf)); + + pthread_mutex_lock(&stream->mutex); + + ssize_t result = stream->write(stream->user_data, tmpbuf, + (ulonglong) (ptr - tmpbuf)); + + pthread_mutex_unlock(&stream->mutex); + + return result < 0; +} + diff --git a/extra/mariabackup/xtrabackup.cc b/extra/mariabackup/xtrabackup.cc index fa50e6286b4..84580f80c0a 100644 --- a/extra/mariabackup/xtrabackup.cc +++ b/extra/mariabackup/xtrabackup.cc @@ -69,6 +69,7 @@ Street, Fifth Floor, Boston, MA 02110-1335 USA # include #endif +#include "aria_backup_client.h" #include #include @@ -81,6 +82,7 @@ Street, Fifth Floor, Boston, MA 02110-1335 USA #include #include #include "ha_innodb.h" +#include "fts0types.h" #include #include @@ -96,7 +98,6 @@ Street, Fifth Floor, Boston, MA 02110-1335 USA #include "xb_regex.h" #include "fil_cur.h" #include "write_filt.h" -#include "xtrabackup.h" #include "ds_buffer.h" #include "ds_tmpfile.h" #include "xbstream.h" @@ -106,12 +107,17 @@ Street, Fifth Floor, Boston, MA 02110-1335 USA #include "backup_mysql.h" #include "backup_copy.h" #include "backup_mysql.h" -#include "xb_plugin.h" +#include "encryption_plugin.h" #include #include #include #include #include +#include +#include "ddl_log.h" +#include "common_engine.h" +#include "lex_string.h" +#include "sql_table.h" #include "backup_debug.h" #define MB_CORRUPTED_PAGES_FILE "innodb_corrupted_pages" @@ -124,6 +130,9 @@ int sd_notifyf() { return 0; } int sys_var_init(); +extern const char* fts_common_tables[]; +extern const fts_index_selector_t fts_index_selector[]; + /* === xtrabackup specific options === */ #define DEFAULT_TARGET_DIR "./xtrabackup_backupfiles/" char xtrabackup_real_target_dir[FN_REFLEN] = DEFAULT_TARGET_DIR; @@ -192,10 +201,12 @@ struct xb_filter_entry_t{ xb_filter_entry_t *name_hash; }; +lsn_t checkpoint_lsn_start; +lsn_t checkpoint_no_start; /** whether log_copying_thread() is active; protected by recv_sys.mutex */ static bool log_copying_running; -int xtrabackup_parallel; +uint xtrabackup_parallel; char *xtrabackup_stream_str = NULL; xb_stream_fmt_t xtrabackup_stream_fmt = XB_STREAM_FMT_NONE; @@ -252,7 +263,7 @@ recv_sys.mutex. */ static std::set fail_undo_ids; longlong innobase_page_size = (1LL << 14); /* 16KB */ -char* innobase_buffer_pool_filename = NULL; +char *innobase_buffer_pool_filename = NULL; /* The default values for the following char* start-up parameters are determined in innobase_init below: */ @@ -358,10 +369,8 @@ my_bool opt_galera_info = FALSE; my_bool opt_slave_info = FALSE; my_bool opt_no_lock = FALSE; my_bool opt_safe_slave_backup = FALSE; -my_bool opt_rsync = FALSE; my_bool opt_force_non_empty_dirs = FALSE; my_bool opt_noversioncheck = FALSE; -my_bool opt_no_backup_locks = FALSE; my_bool opt_decompress = FALSE; my_bool opt_remove_original; my_bool opt_log_innodb_page_corruption; @@ -695,8 +704,190 @@ typedef void (*process_single_tablespace_func_t)(const char *dirname, uint32_t defer_space_id); static dberr_t enumerate_ibd_files(process_single_tablespace_func_t callback); +const char *convert_dst(const char *dst) { + return + (xtrabackup_copy_back || xtrabackup_move_back) ? + dst : trim_dotslash(dst); +} + +std::string convert_tablename_to_filepath( + const char *data_dir_path, const std::string &db, const std::string &table) { + char dbbuff[FN_REFLEN]; + char tbbuff[FN_REFLEN]; + (void)tablename_to_filename(db.c_str(), dbbuff, sizeof(dbbuff)); + (void)tablename_to_filename(table.c_str(), tbbuff, sizeof(tbbuff)); + std::string result(data_dir_path); + result.append(1, FN_LIBCHAR).append(dbbuff). + append(1, FN_LIBCHAR).append(tbbuff); + return result; +} + +std::tuple +convert_filepath_to_tablename(const char *filepath) { + char db_name_orig[FN_REFLEN]; + char table_name_orig[FN_REFLEN]; + parse_db_table_from_file_path(filepath, db_name_orig, table_name_orig); + if (!db_name_orig[0] || !table_name_orig[0]) + return std::make_tuple("", "", ""); + char db_name_conv[FN_REFLEN]; + char table_name_conv[FN_REFLEN]; + filename_to_tablename(db_name_orig, db_name_conv, sizeof(db_name_conv)); + filename_to_tablename( + table_name_orig, table_name_conv, sizeof(table_name_conv)); + if (!db_name_conv[0] || !table_name_conv[0]) + return std::make_tuple("", "", ""); + return std::make_tuple(db_name_conv, table_name_conv, + std::string(db_name_orig).append("/").append(table_name_orig)); +} + +std::string get_table_version_from_image(const std::vector &frm_image) { + DBUG_ASSERT(frm_image.size() >= 64); + + if (!strncmp((char*) frm_image.data(), "TYPE=VIEW\n", 10)) + return {}; + + if (!is_binary_frm_header(frm_image.data())) + return {}; + + /* Length of the MariaDB extra2 segment in the form file. */ + uint len = uint2korr(frm_image.data() + 4); + const uchar *extra2= frm_image.data() + 64; + + if (*extra2 == '/') // old frm had '/' there + return {}; + + const uchar *e2end= extra2 + len; + while (extra2 + 3 <= e2end) + { + uchar type= *extra2++; + size_t length= *extra2++; + if (!length) + { + if (extra2 + 2 >= e2end) + return {}; + length= uint2korr(extra2); + extra2+= 2; + if (length < 256) + return {}; + } + if (extra2 + length > e2end) + return {}; + if (type == EXTRA2_TABLEDEF_VERSION) { + char buff[MY_UUID_STRING_LENGTH]; + my_uuid2str(extra2, buff, 1); + return std::string(buff, buff + MY_UUID_STRING_LENGTH); + } + extra2+= length; + } + + return {}; +} + +std::pair + get_table_engine_from_image(const std::vector &frm_image) { + + DBUG_ASSERT(frm_image.size() >= 64); + + if (!strncmp((char*) frm_image.data(), "TYPE=VIEW\n", 10)) + return std::make_pair(false, DB_TYPE_UNKNOWN); + + if (!is_binary_frm_header(frm_image.data())) + return std::make_pair(false, DB_TYPE_UNKNOWN); + + legacy_db_type dbt = (legacy_db_type)frm_image[3]; + + if (dbt >= DB_TYPE_FIRST_DYNAMIC) + return std::make_pair(false, DB_TYPE_UNKNOWN); + + if (dbt != DB_TYPE_PARTITION_DB) + return std::make_pair(false, dbt); + + dbt = (legacy_db_type)frm_image[61]; + return std::make_pair(true, + dbt < DB_TYPE_FIRST_DYNAMIC ? dbt : DB_TYPE_UNKNOWN); +} + +std::vector read_frm_image(File file) { + std::vector frm_image; + MY_STAT state; + + if (mysql_file_fstat(file, &state, MYF(MY_WME))) + return frm_image; + + frm_image.resize((size_t)state.st_size, 0); + + if (mysql_file_read( + file, frm_image.data(), (size_t)state.st_size, MYF(MY_NABP))) + frm_image.clear(); + + return frm_image; +} + +std::string read_table_version_id(File file) { + auto frm_image = read_frm_image(file); + if (frm_image.empty()) + return {}; + return get_table_version_from_image(frm_image); +} + +bool is_log_table(const char *dbname, const char *tablename) { + DBUG_ASSERT(dbname); + DBUG_ASSERT(tablename); + + LEX_CSTRING lex_db; + LEX_CSTRING lex_table; + lex_db.str = dbname; + lex_db.length = strlen(dbname); + lex_table.str = tablename; + lex_table.length = strlen(tablename); + + if (!lex_string_eq(&MYSQL_SCHEMA_NAME, &lex_db)) + return false; + + if (lex_string_eq(&GENERAL_LOG_NAME, &lex_table)) + return true; + + if (lex_string_eq(&SLOW_LOG_NAME, &lex_table)) + return true; + + return false; +} + +bool is_stats_table(const char *dbname, const char *tablename) { + DBUG_ASSERT(dbname); + DBUG_ASSERT(tablename); + + LEX_CSTRING lex_db; + LEX_CSTRING lex_table; + lex_db.str = dbname; + lex_db.length = strlen(dbname); + lex_table.str = tablename; + lex_table.length = strlen(tablename); + + if (!lex_string_eq(&MYSQL_SCHEMA_NAME, &lex_db)) + return false; + + CHARSET_INFO *ci= system_charset_info; + + return (lex_table.length > 4 && + /* one of mysql.*_stat tables, but not mysql.innodb* tables*/ + ((my_tolower(ci, lex_table.str[lex_table.length-5]) == 's' && + my_tolower(ci, lex_table.str[lex_table.length-4]) == 't' && + my_tolower(ci, lex_table.str[lex_table.length-3]) == 'a' && + my_tolower(ci, lex_table.str[lex_table.length-2]) == 't' && + my_tolower(ci, lex_table.str[lex_table.length-1]) == 's') && + !(my_tolower(ci, lex_table.str[0]) == 'i' && + my_tolower(ci, lex_table.str[1]) == 'n' && + my_tolower(ci, lex_table.str[2]) == 'n' && + my_tolower(ci, lex_table.str[3]) == 'o'))); +} + /* ======== Datafiles iterator ======== */ struct datafiles_iter_t { + datafiles_iter_t() : space(fil_system.space_list.end()), node(nullptr), started(FALSE) { + } + ~datafiles_iter_t() { + } space_list_t::iterator space = fil_system.space_list.end(); fil_node_t *node = nullptr; bool started = false; @@ -776,8 +967,6 @@ static void *dbug_execute_in_new_connection(void *arg) return nullptr; } -static pthread_t dbug_alter_thread; - /* Execute query from a new connection, in own thread. @@ -788,8 +977,9 @@ Execute query from a new connection, in own thread. otherwise query should return error. @param expected_errno - if not 0, and query finished with error, expected mysql_errno() +@return created thread id */ -static void dbug_start_query_thread( +static pthread_t dbug_start_query_thread( const char *query, const char *wait_state, int expected_err, @@ -801,12 +991,14 @@ static void dbug_start_query_thread( par->expect_err = expected_err; par->expect_errno = expected_errno; par->con = xb_mysql_connect(); - - mysql_thread_create(0, &dbug_alter_thread, nullptr, + if (mysql_set_server_option(par->con, MYSQL_OPTION_MULTI_STATEMENTS_ON)) + die("Can't set multistatement option for query: %s", query); + pthread_t result_thread; + mysql_thread_create(0, &result_thread, nullptr, dbug_execute_in_new_connection, par); if (!wait_state) - return; + return result_thread; char q[256]; snprintf(q, sizeof(q), @@ -828,7 +1020,11 @@ static void dbug_start_query_thread( end: msg("query '%s' on connection %lu reached state '%s'", query, mysql_thread_id(par->con), wait_state); + return result_thread; } + +static pthread_t dbug_alter_thread; +static pthread_t dbug_emulate_ddl_on_intermediate_table_thread; #endif void mdl_lock_all() @@ -951,6 +1147,31 @@ static void backup_file_op(uint32_t space_id, int type, } } +static bool check_if_fts_table(const char *file_name) { + const char *table_name_start = strrchr(file_name, '/'); + if (table_name_start) + ++table_name_start; + else + table_name_start = file_name; + + if (!starts_with(table_name_start,"FTS_")) + return false; + + const char *table_name_end = strrchr(table_name_start, '.'); + if (!table_name_end) + table_name_end = table_name_start + strlen(table_name_start); + ptrdiff_t table_name_len = table_name_end - table_name_end; + + for (const char **suffix = fts_common_tables; *suffix; ++suffix) + if (!strncmp(table_name_start, *suffix, table_name_len)) + return true; + for (size_t i = 0; fts_index_selector[i].suffix; ++i) + if (!strncmp(table_name_start, fts_index_selector[i].suffix, + table_name_len)) + return true; + + return false; +} /* This callback is called if DDL operation is detected, @@ -984,8 +1205,9 @@ static void backup_file_op_fail(uint32_t space_id, int type, break; case FILE_DELETE: fail = !check_if_skip_table( - filename_to_spacename(name, len).c_str()); - msg("DDL tracking : delete %u \"%.*s\"", space_id, int(len), name); + filename_to_spacename(name, len).c_str()) + && !check_if_fts_table(reinterpret_cast(name)); + msg("DDL tracking : delete %u \"%.*s\"", space_id, int(len), name); break; default: ut_ad(0); @@ -1111,6 +1333,7 @@ enum options_xtrabackup OPT_INNODB_LOG_FILE_BUFFERING, #endif OPT_INNODB_LOG_FILE_SIZE, + OPT_INNODB_LOG_FILES_IN_GROUP, OPT_INNODB_OPEN_FILES, OPT_XTRA_DEBUG_SYNC, OPT_INNODB_CHECKSUM_ALGORITHM, @@ -1130,7 +1353,6 @@ enum options_xtrabackup OPT_RSYNC, OPT_FORCE_NON_EMPTY_DIRS, OPT_NO_VERSION_CHECK, - OPT_NO_BACKUP_LOCKS, OPT_DECOMPRESS, OPT_INCREMENTAL_HISTORY_NAME, OPT_INCREMENTAL_HISTORY_UUID, @@ -1371,16 +1593,6 @@ struct my_option xb_client_options[]= { (uchar *) &opt_safe_slave_backup, (uchar *) &opt_safe_slave_backup, 0, GET_BOOL, NO_ARG, 0, 0, 0, 0, 0, 0}, - {"rsync", OPT_RSYNC, - "Uses the rsync utility to optimize local file " - "transfers. When this option is specified, " XB_TOOL_NAME " uses rsync " - "to copy all non-InnoDB files instead of spawning a separate cp for " - "each file, which can be much faster for servers with a large number " - "of databases or tables. This option cannot be used together with " - "--stream.", - (uchar *) &opt_rsync, (uchar *) &opt_rsync, 0, GET_BOOL, NO_ARG, 0, 0, 0, - 0, 0, 0}, - {"force-non-empty-directories", OPT_FORCE_NON_EMPTY_DIRS, "This " "option, when specified, makes --copy-back or --move-back transfer " @@ -1397,15 +1609,6 @@ struct my_option xb_client_options[]= { (uchar *) &opt_noversioncheck, (uchar *) &opt_noversioncheck, 0, GET_BOOL, NO_ARG, 0, 0, 0, 0, 0, 0}, - {"no-backup-locks", OPT_NO_BACKUP_LOCKS, - "This option controls if " - "backup locks should be used instead of FLUSH TABLES WITH READ LOCK " - "on the backup stage. The option has no effect when backup locks are " - "not supported by the server. This option is enabled by default, " - "disable with --no-backup-locks.", - (uchar *) &opt_no_backup_locks, (uchar *) &opt_no_backup_locks, 0, - GET_BOOL, NO_ARG, 0, 0, 0, 0, 0, 0}, - {"decompress", OPT_DECOMPRESS, "Decompresses all files with the .qp " "extension in a backup previously made with the --compress option. " @@ -1594,7 +1797,7 @@ struct my_option xb_server_options[] = {"parallel", OPT_XTRA_PARALLEL, "Number of threads to use for parallel datafiles transfer. " "The default value is 1.", - (G_PTR*) &xtrabackup_parallel, (G_PTR*) &xtrabackup_parallel, 0, GET_INT, + (G_PTR*) &xtrabackup_parallel, (G_PTR*) &xtrabackup_parallel, 0, GET_UINT, REQUIRED_ARG, 1, 1, INT_MAX, 0, 0, 0}, {"extended_validation", OPT_XTRA_EXTENDED_VALIDATION, @@ -1753,10 +1956,10 @@ struct my_option xb_server_options[] = 0, GET_STR, REQUIRED_ARG, 0, 0, 0, 0, 0, 0}, {"plugin-dir", OPT_PLUGIN_DIR, - "Server plugin directory. Used to load plugins during 'prepare' phase." - "Has no effect in the 'backup' phase (plugin directory during backup is the same as server's)", - &xb_plugin_dir, &xb_plugin_dir, - 0, GET_STR, REQUIRED_ARG, 0, 0, 0, 0, 0, 0 }, + "Server plugin directory. Used to load plugins during 'prepare' phase." + "Has no effect in the 'backup' phase (plugin directory during backup is the same as server's)", + &xb_plugin_dir, &xb_plugin_dir, + 0, GET_STR, REQUIRED_ARG, 0, 0, 0, 0, 0, 0 }, {"aria_log_dir_path", OPT_ARIA_LOG_DIR_PATH, "Path to individual files and their sizes.", @@ -1884,7 +2087,7 @@ static int prepare_export() IF_WIN("\"","") "\"%s\" --mysqld \"%s\"" " --defaults-extra-file=./backup-my.cnf --defaults-group-suffix=%s --datadir=." " --innodb --innodb-fast-shutdown=0 --loose-partition" - " --innodb-buffer-pool-size=%llu" + " --innodb_purge_rseg_truncate_frequency=1 --innodb-buffer-pool-size=%llu" " --console --skip-log-error --skip-log-bin --bootstrap %s< " BOOTSTRAP_FILENAME IF_WIN("\"",""), mariabackup_exe, @@ -1898,7 +2101,7 @@ static int prepare_export() IF_WIN("\"","") "\"%s\" --mysqld" " --defaults-file=./backup-my.cnf --defaults-group-suffix=%s --datadir=." " --innodb --innodb-fast-shutdown=0 --loose-partition" - " --innodb-buffer-pool-size=%llu" + " --innodb_purge_rseg_truncate_frequency=1 --innodb-buffer-pool-size=%llu" " --console --log-error= --skip-log-bin --bootstrap %s< " BOOTSTRAP_FILENAME IF_WIN("\"",""), mariabackup_exe, @@ -2020,6 +2223,10 @@ xb_get_one_option(const struct my_option *opt, ADD_PRINT_PARAM_OPT(srv_log_group_home_dir); break; + case OPT_INNODB_LOG_FILES_IN_GROUP: + case OPT_INNODB_LOG_FILE_SIZE: + break; + case OPT_INNODB_FLUSH_METHOD: #ifdef _WIN32 /* From: storage/innobase/handler/ha_innodb.cc:innodb_init_params */ @@ -3354,50 +3561,22 @@ To use this facility, you need to 3. start mariabackup with --dbug=+d,debug_mariabackup_events */ void dbug_mariabackup_event(const char *event, - const fil_space_t::name_type key) + const fil_space_t::name_type key, + bool need_lock) { + static std::mutex dbug_mariabackup_event_mutex; char *sql = dbug_mariabackup_get_val(event, key); if (sql && *sql) { msg("dbug_mariabackup_event : executing '%s'", sql); - xb_mysql_query(mysql_connection, sql, false, true); - } + if (need_lock) { + std::lock_guard lock(dbug_mariabackup_event_mutex); + xb_mysql_query(mysql_connection, sql, false, true); + } else + xb_mysql_query(mysql_connection, sql, false, true); + } } #endif // DBUG_OFF -/** Datafiles copying thread.*/ -static void data_copy_thread_func(data_thread_ctxt_t *ctxt) /* thread context */ -{ - uint num = ctxt->num; - fil_node_t* node; - ut_ad(ctxt->corrupted_pages); - - /* - Initialize mysys thread-specific memory so we can - use mysys functions in this thread. - */ - my_thread_init(); - - while ((node = datafiles_iter_next(ctxt->it)) != NULL) { - DBUG_MARIABACKUP_EVENT("before_copy", node->space->name()); - DBUG_EXECUTE_FOR_KEY("wait_innodb_redo_before_copy", - node->space->name(), - backup_wait_for_lsn(get_current_lsn(mysql_connection));); - /* copy the datafile */ - if (xtrabackup_copy_datafile(ctxt->datasinks->m_data, - ctxt->datasinks->m_meta, node, num, NULL, - xtrabackup_incremental ? wf_incremental : wf_write_through, - *ctxt->corrupted_pages)) - die("failed to copy datafile."); - - DBUG_MARIABACKUP_EVENT("after_copy", node->space->name()); - } - - pthread_mutex_lock(ctxt->count_mutex); - (*ctxt->count)--; - pthread_mutex_unlock(ctxt->count_mutex); - - my_thread_end(); -} /************************************************************************ Initialize the appropriate datasink(s). Both local backups and streaming in the @@ -3552,6 +3731,11 @@ static void xb_load_single_table_tablespace(const char *dirname, } if (file->open_read_only(true) != DB_SUCCESS) { + // Ignore FTS tables, as they can be removed for intermediate tables, + // this code must be executed under stronger or equal to BLOCK_DDL lock, + // so there must not be errors for non-intermediate FTS tables. + if (check_if_fts_table(filname)) + return; die("Can't open datafile %s", name); } @@ -4557,7 +4741,6 @@ bool Backup_datasinks::backup_low() if (failed_ids.size() > 0) { return false; } - if (!xtrabackup_incremental) { safe_strcpy(metadata_type, sizeof(metadata_type), "full-backuped"); @@ -4596,16 +4779,441 @@ bool Backup_datasinks::backup_low() return true; } +class InnodbDataCopier { +public: + InnodbDataCopier(Backup_datasinks &backup_datasinks, + CorruptedPages &corrupted_pages, + ThreadPool &thread_pool) : + m_backup_datasinks(backup_datasinks), + m_corrupted_pages(corrupted_pages), + m_tasks(thread_pool) {} + + ~InnodbDataCopier() { + DBUG_ASSERT(m_tasks.is_finished()); + } + + bool start() { + DBUG_ASSERT(m_tasks.is_finished()); + m_tasks.push_task( + std::bind(&InnodbDataCopier::scan_job, this, std::placeholders::_1)); + return true; + } + + bool wait_for_finish() { + return m_tasks.wait_for_finish(); + } + +private: + void scan_job(unsigned thread_num) { + datafiles_iter_t it; + fil_node_t* node; + while ((node = datafiles_iter_next(&it)) != nullptr) { + m_tasks.push_task( + std::bind(&InnodbDataCopier::copy_job, this, node, + std::placeholders::_1)); + } + m_tasks.finish_task(1); + } + + void copy_job(fil_node_t *node, unsigned thread_num) { + DBUG_ASSERT(node); + // TODO: this came from the old code, where it was not thread-safe + // too, use separate mysql connection per thread here + DBUG_MARIABACKUP_EVENT("before_copy", node->space->name()); + DBUG_EXECUTE_FOR_KEY("wait_innodb_redo_before_copy", + node->space->name(), + backup_wait_for_lsn( + get_current_lsn(mysql_connection));); + /* copy the datafile */ + if(xtrabackup_copy_datafile(m_backup_datasinks.m_data, + m_backup_datasinks.m_meta, + node, thread_num, NULL, + xtrabackup_incremental + ? wf_incremental : wf_write_through, + m_corrupted_pages)) + die("mariabackup: Error: failed to copy datafile."); + // TODO: this came from the old code, where it was not thread-safe + // too, use separate mysql connection per thread here + DBUG_MARIABACKUP_EVENT("after_copy", node->space->name()); + m_tasks.finish_task(1); + } + + Backup_datasinks &m_backup_datasinks; + CorruptedPages &m_corrupted_pages; + TasksGroup m_tasks; +}; + + +class BackupStages { + + public: + + BackupStages(ds_ctxt_t *ds_data) : + m_bs_con(nullptr), + m_aria_backup(fil_path_to_mysql_datadir, + aria_log_dir_path, + ds_data, m_con_pool, m_thread_pool), + m_common_backup(fil_path_to_mysql_datadir, ds_data, m_con_pool, + m_thread_pool) {} + + ~BackupStages() { destroy(); } + + bool init() { + if ((m_bs_con = xb_mysql_connect()) == nullptr) + return false; + + while(m_con_pool.size() < xtrabackup_parallel) { + MYSQL *con = xb_mysql_connect(); + if (con == nullptr) + return false; + m_con_pool.push_back(con); + } + + if (!m_thread_pool.start(xtrabackup_parallel)) + return false; + if (!m_aria_backup.init()) + return false; + m_aria_backup.set_post_copy_table_hook( + std::bind(&BackupStages::store_table_version, this, + std::placeholders::_1, std::placeholders::_2, std::placeholders::_3)); + m_common_backup.set_post_copy_table_hook( + std::bind(&BackupStages::store_table_version, this, + std::placeholders::_1, std::placeholders::_2, std::placeholders::_3)); + return true; + } + + void destroy() { + m_thread_pool.stop(); + while (!m_con_pool.empty()) { + MYSQL *con = m_con_pool.back(); + m_con_pool.pop_back(); + mysql_close(con); + } + if (m_bs_con) + mysql_close(m_bs_con); + m_bs_con = nullptr; + } + + bool stage_start(Backup_datasinks &backup_datasinks, + CorruptedPages &corrupted_pages) { + msg("BACKUP STAGE START"); + if (!opt_no_lock) { + if (opt_safe_slave_backup) { + if (!wait_for_safe_slave(mysql_connection)) { + return(false); + } + } + + history_lock_time = time(NULL); + + if (!lock_for_backup_stage_start(m_bs_con)) { + msg("Error on BACKUP STAGE START query execution"); + return(false); + } + } + + InnodbDataCopier innodb_data_copier(backup_datasinks, + corrupted_pages, + m_thread_pool); + // Start InnoDB data files copy in background + if (!innodb_data_copier.start()) { + msg("Error on starting InnoDB data files backup"); + return false; + } + // Start online non-stats-log Aria tables copying in background + if (!m_aria_backup.start(opt_no_lock)) { + msg("Error on starting Aria data files backup"); + innodb_data_copier.wait_for_finish(); + return false; + } + + // Wait for all innodb data files copy finish + if(!innodb_data_copier.wait_for_finish()) { + msg("InnoDB data files backup process is finished with error"); + return false; + } + // Wait for online non-stats-log Aria tables copy finish + if (!m_aria_backup.wait_for_finish()) { + msg("Aria data files backup process is finished with error"); + return false; + } + + DBUG_MARIABACKUP_EVENT_LOCK("after_aria_background", {}); + + return true; + } + + bool stage_flush() { + msg("BACKUP STAGE FLUSH"); + if (!opt_no_lock && !lock_for_backup_stage_flush(m_bs_con)) { + msg("Error on BACKUP STAGE FLUSH query execution"); + return false; + } + auto tables_in_use = get_tables_in_use(mysql_connection); + // Copy non-stats-log non-in-use tables of non-InnoDB-Aria-RocksDB engines + // in background + if (!m_common_backup.scan(tables_in_use, + &m_copied_common_tables, opt_no_lock, true)) { + msg("Error on scan data directory for common engines"); + return false; + } + // Copy Aria offline non-stats-log non-in-use tables in background + if (!m_aria_backup.copy_offline_tables(&tables_in_use, opt_no_lock, + false)) { + msg("Error on start Aria tables backup"); + return false; + } + + if (!m_aria_backup.copy_log_tail()) { + msg("Error on Aria log tail copy"); + return false; + }; + + // Wait for Aria tables copy finish + if (!m_aria_backup.wait_for_finish()) { + msg("Aria data files backup process is finished with error"); + return false; + } + // Wait for non-InnoDB-Aria-RocksDB engines copy finish + if (!m_common_backup.wait_for_finish()) { + msg("Data files backup process is finished with error"); + return false; + } + + DBUG_EXECUTE_IF("emulate_ddl_on_intermediate_table", + dbug_emulate_ddl_on_intermediate_table_thread = + dbug_start_query_thread( + "SET debug_sync='copy_data_between_tables_after_set_backup_lock " + "SIGNAL copy_started';" + "SET debug_sync='copy_data_between_tables_before_reset_backup_lock " + "SIGNAL before_backup_lock_reset WAIT_FOR backup_lock_reset';" + "SET debug_sync='alter_table_after_temp_table_drop " + "SIGNAL temp_table_dropped';" + "SET SESSION lock_wait_timeout = 1;" + "ALTER TABLE test.t1 ADD COLUMN col1_copy INT, ALGORITHM = COPY;", + NULL, 0, 0); + xb_mysql_query(mysql_connection, + "SET debug_sync='now WAIT_FOR copy_started'", false, true); + ); + + return true; + } + + bool stage_block_ddl(Backup_datasinks &backup_datasinks, + CorruptedPages &corrupted_pages) { + if (!opt_no_lock) { + if (!lock_for_backup_stage_block_ddl(m_bs_con)) { + msg("BACKUP STAGE BLOCK_DDL"); + return false; + } + if (have_galera_enabled) + { + xb_mysql_query(mysql_connection, "SET SESSION wsrep_sync_wait=0", false); + } + } + + ulonglong server_lsn_after_lock = get_current_lsn(mysql_connection); + + // Copy the rest of non-stats-lognon-InnoDB-Aria-RocksDB tables + // Do not execute BACKUP LOCK under BLOCK_DDL stage + if (!m_common_backup.scan(m_copied_common_tables, &m_copied_common_tables, + true, false)) { + msg("Error on scan data directory for common engines"); + return false; + } + // Copy log tables tail + if (!m_common_backup.copy_log_tables(false)) { + msg("Error on copy system tables"); + return false; + } + + // Copy the rest of non-stats Aria tables in background + if (!m_aria_backup.copy_offline_tables(nullptr, true, false)) { + msg("Error on start Aria tables backup"); + return false; + } + + // Copy .frm, .trn and other files + if (!backup_files(backup_datasinks.m_data, + fil_path_to_mysql_datadir)) { + msg("Backup files error"); + return false; + } + + msg("Waiting for log copy thread to read lsn %llu", + server_lsn_after_lock); + backup_wait_for_lsn(server_lsn_after_lock); + corrupted_pages.backup_fix_ddl(backup_datasinks.m_data, + backup_datasinks.m_meta); + + if (!m_aria_backup.copy_log_tail()) { + msg("Error on Aria log tail copy"); + return false; + } + + // Wait for Aria tables copy finish + if (!m_aria_backup.wait_for_finish()) { + msg("Aria data files backup process is finished with error"); + return false; + } + // Wait for non-InnoDB-Aria-RocksDB engines copy finish + if (!m_common_backup.wait_for_finish()) { + msg("Data files backup process is finished with error"); + return false; + } + + ddl_log::backup(fil_path_to_mysql_datadir, + backup_datasinks.m_data, m_tables); + + DBUG_MARIABACKUP_EVENT_LOCK("after_stage_block_ddl", {}); + + return true; + } + + bool stage_block_commit(Backup_datasinks &backup_datasinks) { + msg("BACKUP STAGE BLOCK_COMMIT"); + if (!opt_no_lock && !lock_for_backup_stage_commit(m_bs_con)) { + msg("Error on BACKUP STAGE BLOCK_COMMIT query execution"); + return false; + } + + // Copy log tables tail + if (!m_common_backup.copy_log_tables(true)) { + msg("Error on copy log tables"); + return false; + } + + // Copy stats tables + if (!m_common_backup.copy_stats_tables()) { + msg("Error on copy stats tables"); + return false; + } + + // Copy system Aria files + if (!m_aria_backup.finalize()) { + msg("Error on finalize Aria tables backup"); + return false; + } + + if (!m_common_backup.wait_for_finish()) { + msg("Error on finish common engines backup"); + return false; + } + + if (!m_common_backup.close_log_tables()) { + msg("Error on close log tables"); + return false; + } + + if (!backup_files_from_datadir(backup_datasinks.m_data, + fil_path_to_mysql_datadir, + "aws-kms-key")) { + msg("Error on root data dir files backup"); + return false; + } + + if (has_rocksdb_plugin()) { + rocksdb_create_checkpoint(); + } + + // There is no need to stop slave thread before coping non-Innodb data when + // --no-lock option is used because --no-lock option requires that no DDL or + // DML to non-transaction tables can occur. + if (opt_no_lock) { + if (opt_safe_slave_backup) { + if (!wait_for_safe_slave(mysql_connection)) { + return(false); + } + } + } + + if (opt_slave_info) { + if (!write_slave_info(backup_datasinks.m_data, + mysql_connection)) { + return(false); + } + } + + /* The only reason why Galera/binlog info is written before + wait_for_ibbackup_log_copy_finish() is that after that call the xtrabackup + binary will start streamig a temporary copy of REDO log to stdout and + thus, any streaming from innobackupex would interfere. The only way to + avoid that is to have a single process, i.e. merge innobackupex and + xtrabackup. */ + if (opt_galera_info) { + if (!write_galera_info(backup_datasinks.m_data, + mysql_connection)) { + return(false); + } + } + + bool with_binlogs = opt_binlog_info == BINLOG_INFO_ON; + + if (with_binlogs || opt_galera_info) { + if (!write_binlog_info(backup_datasinks.m_data, + mysql_connection)) { + return(false); + } + } + + if (!opt_no_lock) { + msg("Executing FLUSH NO_WRITE_TO_BINLOG ENGINE LOGS..."); + xb_mysql_query(mysql_connection, + "FLUSH NO_WRITE_TO_BINLOG ENGINE LOGS", false); + } + + return backup_datasinks.backup_low(); + } + + bool stage_end(Backup_datasinks &backup_datasinks) { + msg("BACKUP STAGE END"); + /* release all locks */ + if (!opt_no_lock) { + unlock_all(m_bs_con); + history_lock_time = 0; + } else { + history_lock_time = time(NULL) - history_lock_time; + } + backup_release(); + DBUG_EXECUTE_IF("check_mdl_lock_works", + pthread_join(dbug_alter_thread, nullptr); + ); + + DBUG_EXECUTE_IF("emulate_ddl_on_intermediate_table", + pthread_join( + dbug_emulate_ddl_on_intermediate_table_thread, + nullptr); + ); + + backup_finish(backup_datasinks.m_data); + return true; + } + + void store_table_version( + std::string db, std::string table, std::string table_version) { + auto tk = table_key(db, table); + std::lock_guard lock(m_tables_mutex); + m_tables[std::move(tk)] = std::move(table_version); + } + + private: + Backup_datasinks *backup_datasinks; + MYSQL *m_bs_con; + ThreadPool m_thread_pool; + std::vector m_con_pool; + std::mutex m_tables_mutex; + ddl_log::tables_t m_tables; + aria::Backup m_aria_backup; + common_engine::Backup m_common_backup; + std::unordered_set m_copied_common_tables; +}; + /** Implement --backup @return whether the operation succeeded */ static bool xtrabackup_backup_func() { MY_STAT stat_info; - uint i; - uint count; - pthread_mutex_t count_mutex; CorruptedPages corrupted_pages; - data_thread_ctxt_t *data_threads; Backup_datasinks backup_datasinks; pthread_cond_init(&scanned_lsn_cond, NULL); @@ -4621,7 +5229,7 @@ static bool xtrabackup_backup_func() return(false); } msg("cd to %s", mysql_real_data_home); - xb_plugin_backup_init(mysql_connection); + encryption_plugin_backup_init(mysql_connection); msg("open files limit requested %lu, set to %lu", xb_open_files_limit, xb_set_max_open_files(xb_open_files_limit)); @@ -4806,67 +5414,35 @@ fail: mdl_lock_all(); DBUG_EXECUTE_IF("check_mdl_lock_works", - dbug_start_query_thread("ALTER TABLE test.t ADD COLUMN mdl_lock_column int", + dbug_alter_thread = + dbug_start_query_thread("ALTER TABLE test.t ADD COLUMN mdl_lock_column int", "Waiting for table metadata lock", 0, 0);); } - datafiles_iter_t it; + BackupStages stages(backup_datasinks.m_data); - /* Create data copying threads */ - data_threads = (data_thread_ctxt_t *) - malloc(sizeof(data_thread_ctxt_t) * xtrabackup_parallel); - count = xtrabackup_parallel; - pthread_mutex_init(&count_mutex, NULL); - - for (i = 0; i < (uint) xtrabackup_parallel; i++) { - data_threads[i].it = ⁢ - data_threads[i].num = i+1; - data_threads[i].count = &count; - data_threads[i].count_mutex = &count_mutex; - data_threads[i].corrupted_pages = &corrupted_pages; - data_threads[i].datasinks= &backup_datasinks; - std::thread(data_copy_thread_func, data_threads + i).detach(); - } - - /* Wait for threads to exit */ - while (1) { - std::this_thread::sleep_for(std::chrono::seconds(1)); - pthread_mutex_lock(&count_mutex); - bool stop = count == 0; - pthread_mutex_unlock(&count_mutex); - if (stop) { - break; - } - } - - pthread_mutex_destroy(&count_mutex); - free(data_threads); - - DBUG_ASSERT(backup_datasinks.m_data); - DBUG_ASSERT(backup_datasinks.m_meta); - bool ok = backup_start(backup_datasinks.m_data, - backup_datasinks.m_meta, corrupted_pages); - - if (ok) { - ok = backup_datasinks.backup_low(); - - backup_release(); - - DBUG_EXECUTE_IF("check_mdl_lock_works", - pthread_join(dbug_alter_thread, nullptr);); - - if (ok) { - backup_finish(backup_datasinks.m_data); - } - } - - if (opt_log_innodb_page_corruption) - ok = corrupted_pages.print_to_file(backup_datasinks.m_data, - MB_CORRUPTED_PAGES_FILE); - - if (!ok) { + if (!stages.init()) + goto fail; + + if (!stages.stage_start(backup_datasinks, corrupted_pages)) + goto fail; + + if (!stages.stage_flush()) + goto fail; + + if (!stages.stage_block_ddl(backup_datasinks, corrupted_pages)) + goto fail; + + if (!stages.stage_block_commit(backup_datasinks)) + goto fail; + + if (!stages.stage_end(backup_datasinks)) + goto fail; + + if (opt_log_innodb_page_corruption + && !corrupted_pages.print_to_file(backup_datasinks.m_data, + MB_CORRUPTED_PAGES_FILE)) goto fail; - } backup_datasinks.destroy(); @@ -4929,6 +5505,12 @@ void CorruptedPages::backup_fix_ddl(ds_ctxt *ds_data, ds_ctxt *ds_meta) DBUG_MARIABACKUP_EVENT("backup_fix_ddl", {}); + DBUG_EXECUTE_IF("emulate_ddl_on_intermediate_table", + xb_mysql_query(mysql_connection, + "SET debug_sync='now SIGNAL backup_lock_reset " + "WAIT_FOR temp_table_dropped'", false, true); + ); + for (space_id_to_name_t::iterator iter = ddl_tracker.tables_in_backup.begin(); iter != ddl_tracker.tables_in_backup.end(); iter++) { @@ -5073,6 +5655,7 @@ void CorruptedPages::backup_fix_ddl(ds_ctxt *ds_data, ds_ctxt *ds_meta) } } + /* ================= prepare ================= */ /*********************************************************************** @@ -5559,7 +6142,7 @@ std::string change_extension(std::string filename, std::string new_ext) { } -static void rename_file(const char *from,const char *to) { +void rename_file(const char *from,const char *to) { msg("Renaming %s to %s\n", from, to); if (my_rename(from, to, MY_WME)) { die("Can't rename %s to %s errno %d", from, to, errno); @@ -5581,7 +6164,7 @@ typedef ibool (*handle_datadir_entry_func_t)( void* arg); /*! /dev/null 2>&1") == 0)); - if (!have_rsync) - { - msg("Error: rsync executable not found, cannot run backup with --rsync\n"); - return false; - } - } - n_mixed_options = 0; if (opt_decompress) { @@ -6484,6 +7054,7 @@ xb_init() if (opt_check_privileges && !check_all_privileges()) { return(false); } + history_start_time = time(NULL); } diff --git a/extra/mariabackup/xtrabackup.h b/extra/mariabackup/xtrabackup.h index 14036b7a92c..38d7e5fdd03 100644 --- a/extra/mariabackup/xtrabackup.h +++ b/extra/mariabackup/xtrabackup.h @@ -26,6 +26,13 @@ Foundation, Inc., 51 Franklin Street, Fifth Floor, Boston, MA 02110-1335 USA #include "xbstream.h" #include "fil0fil.h" #include +#include "handler.h" + +#include +#include +#include +#include + #define XB_TOOL_NAME "mariadb-backup" #define XB_HISTORY_TABLE "mysql.mariadb_backup_history" @@ -110,7 +117,7 @@ extern my_bool xtrabackup_decrypt_decompress; extern char *innobase_data_file_path; extern longlong innobase_page_size; -extern int xtrabackup_parallel; +extern uint xtrabackup_parallel; extern my_bool xb_close_files; extern const char *xtrabackup_compress_alg; @@ -129,7 +136,6 @@ extern my_bool opt_galera_info; extern my_bool opt_slave_info; extern my_bool opt_no_lock; extern my_bool opt_safe_slave_backup; -extern my_bool opt_rsync; extern my_bool opt_force_non_empty_dirs; extern my_bool opt_noversioncheck; extern my_bool opt_no_backup_locks; @@ -286,15 +292,40 @@ fil_file_readdir_next_file( os_file_stat_t* info); /*!< in/out: buffer where the info is returned */ -#ifndef DBUG_OFF -#include -extern void dbug_mariabackup_event(const char *event, - const fil_space_t::name_type key); +const char *convert_dst(const char *dst); -#define DBUG_MARIABACKUP_EVENT(A, B) \ - DBUG_EXECUTE_IF("mariabackup_events", dbug_mariabackup_event(A, B);) -#else -#define DBUG_MARIABACKUP_EVENT(A, B) /* empty */ -#endif // DBUG_OFF +std::string get_table_version_from_image(const std::vector &frm_image); +std::pair + get_table_engine_from_image(const std::vector &frm_image); +std::string read_table_version_id(File file); +std::string convert_tablename_to_filepath( + const char *data_dir_path, const std::string &db, const std::string &table); + +std::tuple +convert_filepath_to_tablename(const char *filepath); + +typedef std::string table_key_t; + +inline table_key_t table_key(const std::string &db, const std::string &table) { + return std::string(db).append(".").append(table); +}; + +inline table_key_t table_key(const char *db, const char *table) { + return std::string(db).append(".").append(table); +}; + +typedef std::function + post_copy_table_hook_t; + +my_bool +check_if_skip_table( +/******************/ + const char* name); /*!< in: path to the table */ + +bool is_log_table(const char *dbname, const char *tablename); +bool is_stats_table(const char *dbname, const char *tablename); + +extern my_bool xtrabackup_copy_back; +extern my_bool xtrabackup_move_back; #endif /* XB_XTRABACKUP_H */ diff --git a/mysql-test/include/aria_log_control_load.inc b/mysql-test/include/aria_log_control_load.inc new file mode 100644 index 00000000000..34db3aeb4f5 --- /dev/null +++ b/mysql-test/include/aria_log_control_load.inc @@ -0,0 +1,11 @@ +# +# This file loads aria_log_control file into a user variable @aria_log_control. +# Set $ARIA_DATADIR before including this file +# + +--disable_query_log +--copy_file $ARIA_DATADIR/aria_log_control $MYSQLTEST_VARDIR/aria_log_control_tmp +--chmod 0777 $MYSQLTEST_VARDIR/aria_log_control_tmp +--eval SET @aria_log_control=(SELECT LOAD_FILE('$MYSQLTEST_VARDIR/aria_log_control_tmp')) +--remove_file $MYSQLTEST_VARDIR/aria_log_control_tmp +--enable_query_log diff --git a/mysql-test/suite/mariabackup/absolute_ibdata_paths.test b/mysql-test/suite/mariabackup/absolute_ibdata_paths.test index fa304f0bc55..f4d0e2a5bd5 100644 --- a/mysql-test/suite/mariabackup/absolute_ibdata_paths.test +++ b/mysql-test/suite/mariabackup/absolute_ibdata_paths.test @@ -30,7 +30,7 @@ let $_innodb_data_file_path=`select @@innodb_data_file_path`; let $_innodb_data_home_dir=`select @@innodb_data_home_dir`; --disable_result_log -exec $XTRABACKUP --defaults-file=$MYSQLTEST_VARDIR/my.cnf --backup --target-dir=$targetdir; +exec $XTRABACKUP --defaults-file=$MYSQLTEST_VARDIR/my.cnf --backup --parallel=10 --target-dir=$targetdir; --enable_result_log exec $XTRABACKUP --prepare --target-dir=$targetdir; diff --git a/mysql-test/suite/mariabackup/alter_copy_race.result b/mysql-test/suite/mariabackup/alter_copy_race.result index 82202249f81..ae58ac28881 100644 --- a/mysql-test/suite/mariabackup/alter_copy_race.result +++ b/mysql-test/suite/mariabackup/alter_copy_race.result @@ -4,7 +4,7 @@ INSERT into t1 values(1); connect con2, localhost, root,,; connection con2; set lock_wait_timeout=1; -SET debug_sync='copy_data_between_tables_before_reset_backup_lock SIGNAL go WAIT_FOR after_backup_stage_block_commit'; +SET debug_sync='copy_data_between_tables_before_reset_backup_lock SIGNAL go WAIT_FOR after_backup_stage_block_ddl'; SET debug_sync='alter_table_after_temp_table_drop SIGNAL temp_table_dropped'; SET debug_sync='now WAIT_FOR after_backup_stage_start';ALTER TABLE test.t1 FORCE, algorithm=COPY;| connection default; diff --git a/mysql-test/suite/mariabackup/alter_copy_race.test b/mysql-test/suite/mariabackup/alter_copy_race.test index 553643bf667..cfabd76c513 100644 --- a/mysql-test/suite/mariabackup/alter_copy_race.test +++ b/mysql-test/suite/mariabackup/alter_copy_race.test @@ -18,7 +18,7 @@ INSERT into t1 values(1); connect con2, localhost, root,,; connection con2; set lock_wait_timeout=1; -SET debug_sync='copy_data_between_tables_before_reset_backup_lock SIGNAL go WAIT_FOR after_backup_stage_block_commit'; +SET debug_sync='copy_data_between_tables_before_reset_backup_lock SIGNAL go WAIT_FOR after_backup_stage_block_ddl'; SET debug_sync='alter_table_after_temp_table_drop SIGNAL temp_table_dropped'; DELIMITER |; send SET debug_sync='now WAIT_FOR after_backup_stage_start';ALTER TABLE test.t1 FORCE, algorithm=COPY;| @@ -27,7 +27,7 @@ connection default; # setup mariabackup events let after_backup_stage_start=SET debug_sync='now SIGNAL after_backup_stage_start WAIT_FOR go'; -let after_backup_stage_block_commit=SET debug_sync='now SIGNAL after_backup_stage_block_commit'; +let after_backup_stage_block_ddl=SET debug_sync='now SIGNAL after_backup_stage_block_ddl'; let backup_fix_ddl=SET debug_sync='now WAIT_FOR temp_table_dropped'; --disable_result_log exec $XTRABACKUP --defaults-file=$MYSQLTEST_VARDIR/my.cnf --backup --target-dir=$targetdir --dbug=+d,mariabackup_events; diff --git a/mysql-test/suite/mariabackup/apply-log-only-incr.test b/mysql-test/suite/mariabackup/apply-log-only-incr.test index 01b74af2211..1da2be090d6 100644 --- a/mysql-test/suite/mariabackup/apply-log-only-incr.test +++ b/mysql-test/suite/mariabackup/apply-log-only-incr.test @@ -19,7 +19,7 @@ dec $n; } --disable_result_log -exec $XTRABACKUP --defaults-file=$MYSQLTEST_VARDIR/my.cnf --backup --target-dir=$basedir; +exec $XTRABACKUP --defaults-file=$MYSQLTEST_VARDIR/my.cnf --backup --parallel=10 --target-dir=$basedir; --enable_result_log let $n=100; while ($n) { @@ -36,7 +36,7 @@ disconnect flush_log; connection default; --disable_result_log -exec $XTRABACKUP --defaults-file=$MYSQLTEST_VARDIR/my.cnf --backup --ftwrl-wait-timeout=5 --ftwrl-wait-threshold=300 --ftwrl-wait-query-type=all --target-dir=$incremental_dir --incremental-basedir=$basedir ; +exec $XTRABACKUP --defaults-file=$MYSQLTEST_VARDIR/my.cnf --backup --parallel=10 --ftwrl-wait-timeout=5 --ftwrl-wait-threshold=300 --ftwrl-wait-query-type=all --target-dir=$incremental_dir --incremental-basedir=$basedir ; exec $XTRABACKUP --prepare --verbose --target-dir=$basedir ; --enable_result_log diff --git a/mysql-test/suite/mariabackup/apply-log-only.test b/mysql-test/suite/mariabackup/apply-log-only.test index 7ffed0719f0..dbae77b7c9a 100644 --- a/mysql-test/suite/mariabackup/apply-log-only.test +++ b/mysql-test/suite/mariabackup/apply-log-only.test @@ -8,7 +8,7 @@ start transaction; INSERT INTO t VALUES(1); --disable_result_log -exec $XTRABACKUP --defaults-file=$MYSQLTEST_VARDIR/my.cnf --backup --target-dir=$basedir; +exec $XTRABACKUP --defaults-file=$MYSQLTEST_VARDIR/my.cnf --backup --parallel=10 --target-dir=$basedir; --enable_result_log exec $XTRABACKUP --prepare --target-dir=$basedir ; diff --git a/mysql-test/suite/mariabackup/aria_backup.opt b/mysql-test/suite/mariabackup/aria_backup.opt new file mode 100644 index 00000000000..3565e3f1023 --- /dev/null +++ b/mysql-test/suite/mariabackup/aria_backup.opt @@ -0,0 +1 @@ +--loose-partition --loose-aria-log-file-size=8388608 diff --git a/mysql-test/suite/mariabackup/aria_backup.result b/mysql-test/suite/mariabackup/aria_backup.result new file mode 100644 index 00000000000..e8c73f0ab1e --- /dev/null +++ b/mysql-test/suite/mariabackup/aria_backup.result @@ -0,0 +1,780 @@ +### +# Test for backup to stream +##### +### +# Test for mix of online/offline backup tables +##### +CREATE TABLE t_default(i INT PRIMARY KEY) +ENGINE ARIA; +INSERT INTO t_default VALUES (1); +CREATE TABLE t_tr_p_ch(i INT PRIMARY KEY) +ENGINE ARIA TRANSACTIONAL=1 ROW_FORMAT=PAGE PAGE_CHECKSUM=1; +INSERT INTO t_tr_p_ch VALUES (1); +CREATE TABLE t_tr_p_nch(i INT PRIMARY KEY) +ENGINE ARIA TRANSACTIONAL=1 ROW_FORMAT=PAGE PAGE_CHECKSUM=0; +INSERT INTO t_tr_p_nch VALUES (1); +CREATE TABLE t_p_ch(i INT PRIMARY KEY) +ENGINE ARIA TRANSACTIONAL=0 ROW_FORMAT=PAGE PAGE_CHECKSUM=1; +INSERT INTO t_p_ch VALUES (1); +CREATE TABLE t_p_nch(i INT PRIMARY KEY) +ENGINE ARIA TRANSACTIONAL=0 ROW_FORMAT=PAGE PAGE_CHECKSUM=0; +INSERT INTO t_p_nch VALUES (1); +CREATE TABLE t_fixed(i INT PRIMARY KEY) +ENGINE ARIA TRANSACTIONAL=0 ROW_FORMAT=FIXED PAGE_CHECKSUM=1; +INSERT INTO t_fixed VALUES (1); +CREATE TABLE t_dyn(i INT PRIMARY KEY) +ENGINE ARIA TRANSACTIONAL=0 ROW_FORMAT=DYNAMIC PAGE_CHECKSUM=1; +INSERT INTO t_dyn VALUES (1); +# Test for partitioned table +CREATE TABLE t_part_online(i INT PRIMARY KEY) +ENGINE ARIA TRANSACTIONAL = 1 PAGE_CHECKSUM = 1 +PARTITION BY RANGE( i ) ( +PARTITION p0 VALUES LESS THAN (10), +PARTITION p1 VALUES LESS THAN (20), +PARTITION p2 VALUES LESS THAN (30) +); +INSERT INTO t_part_online VALUES(5); +INSERT INTO t_part_online VALUES(15); +INSERT INTO t_part_online VALUES(25); +SELECT * FROM t_part_online; +i +5 +15 +25 +CREATE TABLE t_part_offline(i INT) +ENGINE ARIA TRANSACTIONAL = 0 PAGE_CHECKSUM = 0 +PARTITION BY RANGE( i ) ( +PARTITION p0 VALUES LESS THAN (10), +PARTITION p1 VALUES LESS THAN (20), +PARTITION p2 VALUES LESS THAN (30) +); +INSERT INTO t_part_offline VALUES(5); +INSERT INTO t_part_offline VALUES(15); +INSERT INTO t_part_offline VALUES(25); +# Test for filename to tablename mapping +CREATE TABLE `t 1 t-1`(i INT PRIMARY KEY) +ENGINE ARIA TRANSACTIONAL=1 ROW_FORMAT=PAGE PAGE_CHECKSUM=1; +INSERT INTO `t 1 t-1` VALUES (1); +CREATE TABLE `t-part online`(i INT PRIMARY KEY) +ENGINE ARIA TRANSACTIONAL = 1 PAGE_CHECKSUM = 1 +PARTITION BY RANGE( i ) ( +PARTITION p0 VALUES LESS THAN (10), +PARTITION p1 VALUES LESS THAN (20), +PARTITION p2 VALUES LESS THAN (30) +); +INSERT INTO `t-part online` VALUES(5); +INSERT INTO `t-part online` VALUES(15); +INSERT INTO `t-part online` VALUES(25); +### +# Test for redo log files backup; +##### +CREATE TABLE t_logs_1(i INT) +ENGINE ARIA TRANSACTIONAL=1 ROW_FORMAT=PAGE PAGE_CHECKSUM=1; +CREATE TABLE t_logs_2 LIKE t_logs_1; +CREATE TABLE t_bulk_ins LIKE t_logs_1; +INSERT INTO t_logs_1 VALUES +(0), (1), (2), (3), (4), (5), (6), (7), (8), (9), +(0), (1), (2), (3), (4), (5), (6), (7), (8), (9), +(0), (1), (2), (3), (4), (5), (6), (7), (8), (9), +(0), (1), (2), (3), (4), (5), (6), (7), (8), (9), +(0), (1), (2), (3), (4), (5), (6), (7), (8), (9), +(0), (1), (2), (3), (4), (5), (6), (7), (8), (9), +(0), (1), (2), (3), (4), (5), (6), (7), (8), (9), +(0), (1), (2), (3), (4), (5), (6), (7), (8), (9), +(0), (1), (2), (3), (4), (5), (6), (7), (8), (9), +(0), (1), (2), (3), (4), (5), (6), (7), (8), (9); +# Generate several log files +INSERT INTO t_logs_1 SELECT * FROM t_logs_1; +INSERT INTO t_logs_1 SELECT * FROM t_logs_1; +INSERT INTO t_logs_1 SELECT * FROM t_logs_1; +INSERT INTO t_logs_1 SELECT * FROM t_logs_1; +INSERT INTO t_logs_1 SELECT * FROM t_logs_1; +INSERT INTO t_logs_1 SELECT * FROM t_logs_1; +INSERT INTO t_logs_1 SELECT * FROM t_logs_1; +INSERT INTO t_logs_1 SELECT * FROM t_logs_1; +INSERT INTO t_logs_1 SELECT * FROM t_logs_1; +INSERT INTO t_logs_1 SELECT * FROM t_logs_1; +INSERT INTO t_logs_1 SELECT * FROM t_logs_1; +INSERT INTO t_logs_1 SELECT * FROM t_logs_1; +INSERT INTO t_logs_1 SELECT * FROM t_logs_1; +INSERT INTO t_logs_1 SELECT * FROM t_logs_1; +### +# Test for DML during backup for online backup +##### +CREATE TABLE t_dml_ins(i INT PRIMARY KEY) +ENGINE ARIA TRANSACTIONAL=1 ROW_FORMAT=PAGE PAGE_CHECKSUM=1; +INSERT INTO t_dml_ins VALUES(1); +CREATE TABLE t_dml_upd(i INT PRIMARY KEY) +ENGINE ARIA TRANSACTIONAL=1 ROW_FORMAT=PAGE PAGE_CHECKSUM=1; +INSERT INTO t_dml_upd VALUES(1); +CREATE TABLE t_dml_del(i INT PRIMARY KEY) +ENGINE ARIA TRANSACTIONAL=1 ROW_FORMAT=PAGE PAGE_CHECKSUM=1; +INSERT INTO t_dml_del VALUES(1); +### +# Test for DDL during backup for online backup +##### +CREATE DATABASE test_for_db_drop; +CREATE TABLE test_for_db_drop.t(i INT PRIMARY KEY) ENGINE ARIA; +CREATE TABLE t_db_create(i INT PRIMARY KEY) ENGINE ARIA; +SHOW DATABASES; +Database +information_schema +mtr +mysql +performance_schema +sys +test +test_for_db_drop +CREATE TABLE t_alter(i INT PRIMARY KEY) ENGINE ARIA; +INSERT INTO t_alter VALUES (1); +CREATE TABLE t_trunc(i INT PRIMARY KEY) ENGINE ARIA; +INSERT INTO t_trunc VALUES (1); +CREATE TABLE t_ch_i (i int(10), index(i) ) ENGINE=Aria; +INSERT INTO t_ch_i VALUES(1); +CREATE TABLE t_change_engine(i INT PRIMARY KEY) ENGINE InnoDB; +INSERT INTO t_change_engine VALUES (1); +CREATE TABLE t_rename(i INT PRIMARY KEY) ENGINE ARIA; +CREATE DATABASE test_for_rename; +CREATE TABLE t_rename_2(i INT PRIMARY KEY) ENGINE ARIA; +CREATE TABLE t_rename_3(i INT PRIMARY KEY) ENGINE ARIA; +CREATE TABLE t_rename_4(i INT PRIMARY KEY) ENGINE ARIA; +CREATE TABLE t_delete(i INT PRIMARY KEY) ENGINE ARIA; +CREATE TABLE t_delete_2(i INT PRIMARY KEY) ENGINE ARIA; +CREATE TABLE t_rename_alter(i INT PRIMARY KEY) ENGINE ARIA; +CREATE TABLE t_rename_create(i INT PRIMARY KEY) ENGINE ARIA; +CREATE TABLE t_part_create(i INT PRIMARY KEY) ENGINE ARIA; +CREATE TABLE t_part_add_part(i INT PRIMARY KEY) ENGINE ARIA; +CREATE TABLE t_part_change_eng(i INT PRIMARY KEY) ENGINE ARIA PARTITION BY HASH(i) PARTITIONS 2; +CREATE TABLE t_part_change_eng_2(i INT PRIMARY KEY) ENGINE InnoDB PARTITION BY HASH(i) PARTITIONS 2; +CREATE TABLE t_part_change_eng_3(i INT PRIMARY KEY) ENGINE Aria; +CREATE TABLE t_part_alter(i INT PRIMARY KEY) ENGINE Aria PARTITION BY HASH(i) PARTITIONS 2; +CREATE TABLE t_part_alter_2(i INT PRIMARY KEY) ENGINE Aria PARTITION BY HASH(i) PARTITIONS 3; +CREATE TABLE t_part_drop(i INT PRIMARY KEY) ENGINE Aria PARTITION BY HASH(i) PARTITIONS 2; +CREATE TABLE t_part_rename(i INT PRIMARY KEY) ENGINE Aria PARTITION BY HASH(i) PARTITIONS 2; +CREATE TABLE t_part_rename_3(i INT PRIMARY KEY) ENGINE Aria PARTITION BY HASH(i) PARTITIONS 2; +CREATE TABLE t_part_rm_part(i INT PRIMARY KEY) ENGINE Aria PARTITION BY HASH(i) PARTITIONS 2; +SET SESSION debug_dbug="+d,maria_flush_whole_log"; +SET GLOBAL aria_checkpoint_interval=10000; +### Backup to stream +# xtrabackup prepare +# shutdown server +# remove datadir +# xtrabackup move back +# restart +### Result for DDL test +SHOW CREATE TABLE t_alter; +Table Create Table +t_alter CREATE TABLE `t_alter` ( + `i` int(11) NOT NULL, + `c` int(11) DEFAULT NULL, + PRIMARY KEY (`i`) +) ENGINE=Aria DEFAULT CHARSET=latin1 COLLATE=latin1_swedish_ci PAGE_CHECKSUM=1 +SELECT * FROM t_alter; +i c +1 NULL +SHOW CREATE TABLE t_change_engine; +Table Create Table +t_change_engine CREATE TABLE `t_change_engine` ( + `i` int(11) NOT NULL, + PRIMARY KEY (`i`) +) ENGINE=Aria DEFAULT CHARSET=latin1 COLLATE=latin1_swedish_ci PAGE_CHECKSUM=1 +SELECT * FROM t_change_engine; +i +1 +SELECT * FROM t_trunc; +i +1 +SELECT * FROM t_ch_i; +i +1 +SELECT * FROM t_rename_new; +i +SELECT * FROM test_for_rename.t_rename_new_2; +i +SELECT * FROM t_rename_new_new_3; +i +SELECT * FROM t_rename_new_4; +i +SELECT * FROM t_delete; +ERROR 42S02: Table 'test.t_delete' doesn't exist +SHOW CREATE TABLE t_delete_2; +Table Create Table +t_delete_2 CREATE TABLE `t_delete_2` ( + `i` int(11) NOT NULL, + PRIMARY KEY (`i`) +) ENGINE=InnoDB DEFAULT CHARSET=latin1 COLLATE=latin1_swedish_ci PAGE_CHECKSUM=1 +SELECT * FROM t_rename_alter_2; +i c +SELECT * FROM t_rename_create; +d +SELECT * FROM t_rename_create_new; +i +SHOW CREATE TABLE t_part_create_2; +Table Create Table +t_part_create_2 CREATE TABLE `t_part_create_2` ( + `i` int(11) DEFAULT NULL +) ENGINE=Aria DEFAULT CHARSET=latin1 COLLATE=latin1_swedish_ci + PARTITION BY HASH (`i`) +PARTITIONS 2 +SELECT * FROM t_part_create_2; +i +SHOW CREATE TABLE t_part_add_part; +Table Create Table +t_part_add_part CREATE TABLE `t_part_add_part` ( + `i` int(11) NOT NULL, + PRIMARY KEY (`i`) +) ENGINE=Aria DEFAULT CHARSET=latin1 COLLATE=latin1_swedish_ci PAGE_CHECKSUM=1 + PARTITION BY HASH (`i`) +PARTITIONS 2 +SELECT * FROM t_part_add_part; +i +SHOW CREATE TABLE t_part_change_eng; +Table Create Table +t_part_change_eng CREATE TABLE `t_part_change_eng` ( + `i` int(11) NOT NULL, + PRIMARY KEY (`i`) +) ENGINE=InnoDB DEFAULT CHARSET=latin1 COLLATE=latin1_swedish_ci + PARTITION BY HASH (`i`) +PARTITIONS 2 +SELECT * FROM t_part_change_eng; +i +SHOW CREATE TABLE t_part_change_eng_2; +Table Create Table +t_part_change_eng_2 CREATE TABLE `t_part_change_eng_2` ( + `i` int(11) NOT NULL, + PRIMARY KEY (`i`) +) ENGINE=Aria DEFAULT CHARSET=latin1 COLLATE=latin1_swedish_ci + PARTITION BY HASH (`i`) +PARTITIONS 2 +SELECT * FROM t_part_change_eng_2; +i +SELECT * FROM t_part_alter; +i c +SHOW CREATE TABLE t_part_alter_2; +Table Create Table +t_part_alter_2 CREATE TABLE `t_part_alter_2` ( + `i` int(11) NOT NULL, + PRIMARY KEY (`i`) +) ENGINE=Aria DEFAULT CHARSET=latin1 COLLATE=latin1_swedish_ci + PARTITION BY HASH (`i`) +PARTITIONS 2 +SELECT * FROM t_part_alter_2; +i +SELECT * FROM t_part_drop; +ERROR 42S02: Table 'test.t_part_drop' doesn't exist +SELECT * FROM t_part_rename; +ERROR 42S02: Table 'test.t_part_rename' doesn't exist +SELECT * FROM t_part_rename_2; +i +SELECT * FROM t_part_rename_3; +ERROR 42S02: Table 'test.t_part_rename_3' doesn't exist +SELECT * FROM test_for_rename.t_part_rename_4; +i +SHOW CREATE TABLE t_part_rm_part; +Table Create Table +t_part_rm_part CREATE TABLE `t_part_rm_part` ( + `i` int(11) NOT NULL, + `c` int(11) DEFAULT NULL, + PRIMARY KEY (`i`) +) ENGINE=Aria DEFAULT CHARSET=latin1 COLLATE=latin1_swedish_ci PAGE_CHECKSUM=1 +SELECT * FROM t_part_rm_part; +i c +SHOW DATABASES; +Database +information_schema +mtr +mysql +performance_schema +sys +test +test_for_db_create +test_for_rename +### Clean up for DDL test +DROP DATABASE test_for_db_create; +DROP TABLE t_db_create; +DROP TABLE t_change_engine; +DROP TABLE t_alter; +DROP TABLE t_trunc; +DROP TABLE t_ch_i; +DROP TABLE t_rename_new; +DROP TABLE t_rename_new_new_3; +DROP TABLE t_rename_new_4; +DROP TABLE t_delete_2; +DROP TABLE t_rename_alter_2; +DROP TABLE t_rename_create; +DROP TABLE t_rename_create_new; +DROP TABLE t_part_create; +DROP TABLE t_part_create_2; +DROP TABLE t_part_add_part; +DROP TABLE t_part_change_eng; +DROP TABLE t_part_change_eng_2; +DROP TABLE t_part_change_eng_3; +DROP TABLE t_part_alter; +DROP TABLE t_part_alter_2; +DROP TABLE t_part_rename_2; +DROP TABLE t_part_rm_part; +DROP DATABASE test_for_rename; +### Result for DML test +SELECT * FROM t_dml_ins; +i +1 +2 +SELECT * FROM t_dml_upd; +i +2 +SELECT * FROM t_dml_del; +i +### Clean up for DML test +DROP TABLE t_dml_ins; +DROP TABLE t_dml_upd; +DROP TABLE t_dml_del; +### Result for redo log files backup +# ok +# ok +# ok +### Cleanup for redo log files backup +DROP TABLE t_logs_1; +DROP TABLE t_logs_2; +DROP TABLE t_bulk_ins; +### Result for online/offline tables test +SELECT * FROM t_default; +i +1 +SELECT * FROM t_tr_p_ch; +i +1 +SELECT * FROM t_tr_p_nch; +i +1 +SELECT * FROM t_p_ch; +i +1 +SELECT * FROM t_p_nch; +i +1 +SELECT * FROM t_fixed; +i +1 +SELECT * FROM t_dyn; +i +1 +SELECT * FROM t_part_online; +i +5 +15 +25 +SELECT * FROM t_part_offline; +i +5 +15 +25 +SELECT * FROM `t 1 t-1`; +i +1 +SELECT * FROM `t-part online`; +i +5 +15 +25 +### Cleanup for online/offline tables test +DROP TABLE t_default; +DROP TABLE t_tr_p_ch; +DROP TABLE t_tr_p_nch; +DROP TABLE t_p_ch; +DROP TABLE t_p_nch; +DROP TABLE t_fixed; +DROP TABLE t_dyn; +DROP TABLE t_part_online; +DROP TABLE t_part_offline; +DROP TABLE `t 1 t-1`; +DROP TABLE `t-part online`; +### +# Test for backup to directory +##### +### +# Test for mix of online/offline backup tables +##### +CREATE TABLE t_default(i INT PRIMARY KEY) +ENGINE ARIA; +INSERT INTO t_default VALUES (1); +CREATE TABLE t_tr_p_ch(i INT PRIMARY KEY) +ENGINE ARIA TRANSACTIONAL=1 ROW_FORMAT=PAGE PAGE_CHECKSUM=1; +INSERT INTO t_tr_p_ch VALUES (1); +CREATE TABLE t_tr_p_nch(i INT PRIMARY KEY) +ENGINE ARIA TRANSACTIONAL=1 ROW_FORMAT=PAGE PAGE_CHECKSUM=0; +INSERT INTO t_tr_p_nch VALUES (1); +CREATE TABLE t_p_ch(i INT PRIMARY KEY) +ENGINE ARIA TRANSACTIONAL=0 ROW_FORMAT=PAGE PAGE_CHECKSUM=1; +INSERT INTO t_p_ch VALUES (1); +CREATE TABLE t_p_nch(i INT PRIMARY KEY) +ENGINE ARIA TRANSACTIONAL=0 ROW_FORMAT=PAGE PAGE_CHECKSUM=0; +INSERT INTO t_p_nch VALUES (1); +CREATE TABLE t_fixed(i INT PRIMARY KEY) +ENGINE ARIA TRANSACTIONAL=0 ROW_FORMAT=FIXED PAGE_CHECKSUM=1; +INSERT INTO t_fixed VALUES (1); +CREATE TABLE t_dyn(i INT PRIMARY KEY) +ENGINE ARIA TRANSACTIONAL=0 ROW_FORMAT=DYNAMIC PAGE_CHECKSUM=1; +INSERT INTO t_dyn VALUES (1); +# Test for partitioned table +CREATE TABLE t_part_online(i INT PRIMARY KEY) +ENGINE ARIA TRANSACTIONAL = 1 PAGE_CHECKSUM = 1 +PARTITION BY RANGE( i ) ( +PARTITION p0 VALUES LESS THAN (10), +PARTITION p1 VALUES LESS THAN (20), +PARTITION p2 VALUES LESS THAN (30) +); +INSERT INTO t_part_online VALUES(5); +INSERT INTO t_part_online VALUES(15); +INSERT INTO t_part_online VALUES(25); +SELECT * FROM t_part_online; +i +5 +15 +25 +CREATE TABLE t_part_offline(i INT) +ENGINE ARIA TRANSACTIONAL = 0 PAGE_CHECKSUM = 0 +PARTITION BY RANGE( i ) ( +PARTITION p0 VALUES LESS THAN (10), +PARTITION p1 VALUES LESS THAN (20), +PARTITION p2 VALUES LESS THAN (30) +); +INSERT INTO t_part_offline VALUES(5); +INSERT INTO t_part_offline VALUES(15); +INSERT INTO t_part_offline VALUES(25); +# Test for filename to tablename mapping +CREATE TABLE `t 1 t-1`(i INT PRIMARY KEY) +ENGINE ARIA TRANSACTIONAL=1 ROW_FORMAT=PAGE PAGE_CHECKSUM=1; +INSERT INTO `t 1 t-1` VALUES (1); +CREATE TABLE `t-part online`(i INT PRIMARY KEY) +ENGINE ARIA TRANSACTIONAL = 1 PAGE_CHECKSUM = 1 +PARTITION BY RANGE( i ) ( +PARTITION p0 VALUES LESS THAN (10), +PARTITION p1 VALUES LESS THAN (20), +PARTITION p2 VALUES LESS THAN (30) +); +INSERT INTO `t-part online` VALUES(5); +INSERT INTO `t-part online` VALUES(15); +INSERT INTO `t-part online` VALUES(25); +### +# Test for redo log files backup; +##### +CREATE TABLE t_logs_1(i INT) +ENGINE ARIA TRANSACTIONAL=1 ROW_FORMAT=PAGE PAGE_CHECKSUM=1; +CREATE TABLE t_logs_2 LIKE t_logs_1; +CREATE TABLE t_bulk_ins LIKE t_logs_1; +INSERT INTO t_logs_1 VALUES +(0), (1), (2), (3), (4), (5), (6), (7), (8), (9), +(0), (1), (2), (3), (4), (5), (6), (7), (8), (9), +(0), (1), (2), (3), (4), (5), (6), (7), (8), (9), +(0), (1), (2), (3), (4), (5), (6), (7), (8), (9), +(0), (1), (2), (3), (4), (5), (6), (7), (8), (9), +(0), (1), (2), (3), (4), (5), (6), (7), (8), (9), +(0), (1), (2), (3), (4), (5), (6), (7), (8), (9), +(0), (1), (2), (3), (4), (5), (6), (7), (8), (9), +(0), (1), (2), (3), (4), (5), (6), (7), (8), (9), +(0), (1), (2), (3), (4), (5), (6), (7), (8), (9); +# Generate several log files +INSERT INTO t_logs_1 SELECT * FROM t_logs_1; +INSERT INTO t_logs_1 SELECT * FROM t_logs_1; +INSERT INTO t_logs_1 SELECT * FROM t_logs_1; +INSERT INTO t_logs_1 SELECT * FROM t_logs_1; +INSERT INTO t_logs_1 SELECT * FROM t_logs_1; +INSERT INTO t_logs_1 SELECT * FROM t_logs_1; +INSERT INTO t_logs_1 SELECT * FROM t_logs_1; +INSERT INTO t_logs_1 SELECT * FROM t_logs_1; +INSERT INTO t_logs_1 SELECT * FROM t_logs_1; +INSERT INTO t_logs_1 SELECT * FROM t_logs_1; +INSERT INTO t_logs_1 SELECT * FROM t_logs_1; +INSERT INTO t_logs_1 SELECT * FROM t_logs_1; +INSERT INTO t_logs_1 SELECT * FROM t_logs_1; +INSERT INTO t_logs_1 SELECT * FROM t_logs_1; +### +# Test for DML during backup for online backup +##### +CREATE TABLE t_dml_ins(i INT PRIMARY KEY) +ENGINE ARIA TRANSACTIONAL=1 ROW_FORMAT=PAGE PAGE_CHECKSUM=1; +INSERT INTO t_dml_ins VALUES(1); +CREATE TABLE t_dml_upd(i INT PRIMARY KEY) +ENGINE ARIA TRANSACTIONAL=1 ROW_FORMAT=PAGE PAGE_CHECKSUM=1; +INSERT INTO t_dml_upd VALUES(1); +CREATE TABLE t_dml_del(i INT PRIMARY KEY) +ENGINE ARIA TRANSACTIONAL=1 ROW_FORMAT=PAGE PAGE_CHECKSUM=1; +INSERT INTO t_dml_del VALUES(1); +### +# Test for DDL during backup for online backup +##### +CREATE DATABASE test_for_db_drop; +CREATE TABLE test_for_db_drop.t(i INT PRIMARY KEY) ENGINE ARIA; +CREATE TABLE t_db_create(i INT PRIMARY KEY) ENGINE ARIA; +SHOW DATABASES; +Database +information_schema +mtr +mysql +performance_schema +sys +test +test_for_db_drop +CREATE TABLE t_alter(i INT PRIMARY KEY) ENGINE ARIA; +INSERT INTO t_alter VALUES (1); +CREATE TABLE t_trunc(i INT PRIMARY KEY) ENGINE ARIA; +INSERT INTO t_trunc VALUES (1); +CREATE TABLE t_ch_i (i int(10), index(i) ) ENGINE=Aria; +INSERT INTO t_ch_i VALUES(1); +CREATE TABLE t_change_engine(i INT PRIMARY KEY) ENGINE InnoDB; +INSERT INTO t_change_engine VALUES (1); +CREATE TABLE t_rename(i INT PRIMARY KEY) ENGINE ARIA; +CREATE DATABASE test_for_rename; +CREATE TABLE t_rename_2(i INT PRIMARY KEY) ENGINE ARIA; +CREATE TABLE t_rename_3(i INT PRIMARY KEY) ENGINE ARIA; +CREATE TABLE t_rename_4(i INT PRIMARY KEY) ENGINE ARIA; +CREATE TABLE t_delete(i INT PRIMARY KEY) ENGINE ARIA; +CREATE TABLE t_delete_2(i INT PRIMARY KEY) ENGINE ARIA; +CREATE TABLE t_rename_alter(i INT PRIMARY KEY) ENGINE ARIA; +CREATE TABLE t_rename_create(i INT PRIMARY KEY) ENGINE ARIA; +CREATE TABLE t_part_create(i INT PRIMARY KEY) ENGINE ARIA; +CREATE TABLE t_part_add_part(i INT PRIMARY KEY) ENGINE ARIA; +CREATE TABLE t_part_change_eng(i INT PRIMARY KEY) ENGINE ARIA PARTITION BY HASH(i) PARTITIONS 2; +CREATE TABLE t_part_change_eng_2(i INT PRIMARY KEY) ENGINE InnoDB PARTITION BY HASH(i) PARTITIONS 2; +CREATE TABLE t_part_change_eng_3(i INT PRIMARY KEY) ENGINE Aria; +CREATE TABLE t_part_alter(i INT PRIMARY KEY) ENGINE Aria PARTITION BY HASH(i) PARTITIONS 2; +CREATE TABLE t_part_alter_2(i INT PRIMARY KEY) ENGINE Aria PARTITION BY HASH(i) PARTITIONS 3; +CREATE TABLE t_part_drop(i INT PRIMARY KEY) ENGINE Aria PARTITION BY HASH(i) PARTITIONS 2; +CREATE TABLE t_part_rename(i INT PRIMARY KEY) ENGINE Aria PARTITION BY HASH(i) PARTITIONS 2; +CREATE TABLE t_part_rename_3(i INT PRIMARY KEY) ENGINE Aria PARTITION BY HASH(i) PARTITIONS 2; +CREATE TABLE t_part_rm_part(i INT PRIMARY KEY) ENGINE Aria PARTITION BY HASH(i) PARTITIONS 2; +SET SESSION debug_dbug="+d,maria_flush_whole_log"; +SET GLOBAL aria_checkpoint_interval=10000; +### Backup to dir +# xtrabackup prepare +# shutdown server +# remove datadir +# xtrabackup move back +# restart +### Result for DDL test +SHOW CREATE TABLE t_alter; +Table Create Table +t_alter CREATE TABLE `t_alter` ( + `i` int(11) NOT NULL, + `c` int(11) DEFAULT NULL, + PRIMARY KEY (`i`) +) ENGINE=Aria DEFAULT CHARSET=latin1 COLLATE=latin1_swedish_ci PAGE_CHECKSUM=1 +SELECT * FROM t_alter; +i c +1 NULL +SHOW CREATE TABLE t_change_engine; +Table Create Table +t_change_engine CREATE TABLE `t_change_engine` ( + `i` int(11) NOT NULL, + PRIMARY KEY (`i`) +) ENGINE=Aria DEFAULT CHARSET=latin1 COLLATE=latin1_swedish_ci PAGE_CHECKSUM=1 +SELECT * FROM t_change_engine; +i +1 +SELECT * FROM t_trunc; +i +1 +SELECT * FROM t_ch_i; +i +1 +SELECT * FROM t_rename_new; +i +SELECT * FROM test_for_rename.t_rename_new_2; +i +SELECT * FROM t_rename_new_new_3; +i +SELECT * FROM t_rename_new_4; +i +SELECT * FROM t_delete; +ERROR 42S02: Table 'test.t_delete' doesn't exist +SHOW CREATE TABLE t_delete_2; +Table Create Table +t_delete_2 CREATE TABLE `t_delete_2` ( + `i` int(11) NOT NULL, + PRIMARY KEY (`i`) +) ENGINE=InnoDB DEFAULT CHARSET=latin1 COLLATE=latin1_swedish_ci PAGE_CHECKSUM=1 +SELECT * FROM t_rename_alter_2; +i c +SELECT * FROM t_rename_create; +d +SELECT * FROM t_rename_create_new; +i +SHOW CREATE TABLE t_part_create_2; +Table Create Table +t_part_create_2 CREATE TABLE `t_part_create_2` ( + `i` int(11) DEFAULT NULL +) ENGINE=Aria DEFAULT CHARSET=latin1 COLLATE=latin1_swedish_ci + PARTITION BY HASH (`i`) +PARTITIONS 2 +SELECT * FROM t_part_create_2; +i +SHOW CREATE TABLE t_part_add_part; +Table Create Table +t_part_add_part CREATE TABLE `t_part_add_part` ( + `i` int(11) NOT NULL, + PRIMARY KEY (`i`) +) ENGINE=Aria DEFAULT CHARSET=latin1 COLLATE=latin1_swedish_ci PAGE_CHECKSUM=1 + PARTITION BY HASH (`i`) +PARTITIONS 2 +SELECT * FROM t_part_add_part; +i +SHOW CREATE TABLE t_part_change_eng; +Table Create Table +t_part_change_eng CREATE TABLE `t_part_change_eng` ( + `i` int(11) NOT NULL, + PRIMARY KEY (`i`) +) ENGINE=InnoDB DEFAULT CHARSET=latin1 COLLATE=latin1_swedish_ci + PARTITION BY HASH (`i`) +PARTITIONS 2 +SELECT * FROM t_part_change_eng; +i +SHOW CREATE TABLE t_part_change_eng_2; +Table Create Table +t_part_change_eng_2 CREATE TABLE `t_part_change_eng_2` ( + `i` int(11) NOT NULL, + PRIMARY KEY (`i`) +) ENGINE=Aria DEFAULT CHARSET=latin1 COLLATE=latin1_swedish_ci + PARTITION BY HASH (`i`) +PARTITIONS 2 +SELECT * FROM t_part_change_eng_2; +i +SELECT * FROM t_part_alter; +i c +SHOW CREATE TABLE t_part_alter_2; +Table Create Table +t_part_alter_2 CREATE TABLE `t_part_alter_2` ( + `i` int(11) NOT NULL, + PRIMARY KEY (`i`) +) ENGINE=Aria DEFAULT CHARSET=latin1 COLLATE=latin1_swedish_ci + PARTITION BY HASH (`i`) +PARTITIONS 2 +SELECT * FROM t_part_alter_2; +i +SELECT * FROM t_part_drop; +ERROR 42S02: Table 'test.t_part_drop' doesn't exist +SELECT * FROM t_part_rename; +ERROR 42S02: Table 'test.t_part_rename' doesn't exist +SELECT * FROM t_part_rename_2; +i +SELECT * FROM t_part_rename_3; +ERROR 42S02: Table 'test.t_part_rename_3' doesn't exist +SELECT * FROM test_for_rename.t_part_rename_4; +i +SHOW CREATE TABLE t_part_rm_part; +Table Create Table +t_part_rm_part CREATE TABLE `t_part_rm_part` ( + `i` int(11) NOT NULL, + `c` int(11) DEFAULT NULL, + PRIMARY KEY (`i`) +) ENGINE=Aria DEFAULT CHARSET=latin1 COLLATE=latin1_swedish_ci PAGE_CHECKSUM=1 +SELECT * FROM t_part_rm_part; +i c +SHOW DATABASES; +Database +information_schema +mtr +mysql +performance_schema +sys +test +test_for_db_create +test_for_rename +### Clean up for DDL test +DROP DATABASE test_for_db_create; +DROP TABLE t_db_create; +DROP TABLE t_change_engine; +DROP TABLE t_alter; +DROP TABLE t_trunc; +DROP TABLE t_ch_i; +DROP TABLE t_rename_new; +DROP TABLE t_rename_new_new_3; +DROP TABLE t_rename_new_4; +DROP TABLE t_delete_2; +DROP TABLE t_rename_alter_2; +DROP TABLE t_rename_create; +DROP TABLE t_rename_create_new; +DROP TABLE t_part_create; +DROP TABLE t_part_create_2; +DROP TABLE t_part_add_part; +DROP TABLE t_part_change_eng; +DROP TABLE t_part_change_eng_2; +DROP TABLE t_part_change_eng_3; +DROP TABLE t_part_alter; +DROP TABLE t_part_alter_2; +DROP TABLE t_part_rename_2; +DROP TABLE t_part_rm_part; +DROP DATABASE test_for_rename; +### Result for DML test +SELECT * FROM t_dml_ins; +i +1 +2 +SELECT * FROM t_dml_upd; +i +2 +SELECT * FROM t_dml_del; +i +### Clean up for DML test +DROP TABLE t_dml_ins; +DROP TABLE t_dml_upd; +DROP TABLE t_dml_del; +### Result for redo log files backup +# ok +# ok +# ok +### Cleanup for redo log files backup +DROP TABLE t_logs_1; +DROP TABLE t_logs_2; +DROP TABLE t_bulk_ins; +### Result for online/offline tables test +SELECT * FROM t_default; +i +1 +SELECT * FROM t_tr_p_ch; +i +1 +SELECT * FROM t_tr_p_nch; +i +1 +SELECT * FROM t_p_ch; +i +1 +SELECT * FROM t_p_nch; +i +1 +SELECT * FROM t_fixed; +i +1 +SELECT * FROM t_dyn; +i +1 +SELECT * FROM t_part_online; +i +5 +15 +25 +SELECT * FROM t_part_offline; +i +5 +15 +25 +SELECT * FROM `t 1 t-1`; +i +1 +SELECT * FROM `t-part online`; +i +5 +15 +25 +### Cleanup for online/offline tables test +DROP TABLE t_default; +DROP TABLE t_tr_p_ch; +DROP TABLE t_tr_p_nch; +DROP TABLE t_p_ch; +DROP TABLE t_p_nch; +DROP TABLE t_fixed; +DROP TABLE t_dyn; +DROP TABLE t_part_online; +DROP TABLE t_part_offline; +DROP TABLE `t 1 t-1`; +DROP TABLE `t-part online`; diff --git a/mysql-test/suite/mariabackup/aria_backup.test b/mysql-test/suite/mariabackup/aria_backup.test new file mode 100644 index 00000000000..5375f6e6f53 --- /dev/null +++ b/mysql-test/suite/mariabackup/aria_backup.test @@ -0,0 +1,423 @@ +--source include/have_aria.inc +--source include/have_partition.inc +--source include/have_debug.inc +--source include/big_test.inc + +--let $targetdir=$MYSQLTEST_VARDIR/tmp/backup +--let $backup_stream=2 +--let $backup_dir=1 +--let $backup_variant=$backup_stream + +while ($backup_variant) { +if ($backup_variant == $backup_stream) { +--echo ### +--echo # Test for backup to stream +--echo ##### +} +if ($backup_variant == $backup_dir) { +--echo ### +--echo # Test for backup to directory +--echo ##### +} + +--echo ### +--echo # Test for mix of online/offline backup tables +--echo ##### + +CREATE TABLE t_default(i INT PRIMARY KEY) + ENGINE ARIA; +INSERT INTO t_default VALUES (1); + +CREATE TABLE t_tr_p_ch(i INT PRIMARY KEY) + ENGINE ARIA TRANSACTIONAL=1 ROW_FORMAT=PAGE PAGE_CHECKSUM=1; +INSERT INTO t_tr_p_ch VALUES (1); + +CREATE TABLE t_tr_p_nch(i INT PRIMARY KEY) + ENGINE ARIA TRANSACTIONAL=1 ROW_FORMAT=PAGE PAGE_CHECKSUM=0; +INSERT INTO t_tr_p_nch VALUES (1); + +CREATE TABLE t_p_ch(i INT PRIMARY KEY) + ENGINE ARIA TRANSACTIONAL=0 ROW_FORMAT=PAGE PAGE_CHECKSUM=1; +INSERT INTO t_p_ch VALUES (1); + +CREATE TABLE t_p_nch(i INT PRIMARY KEY) + ENGINE ARIA TRANSACTIONAL=0 ROW_FORMAT=PAGE PAGE_CHECKSUM=0; +INSERT INTO t_p_nch VALUES (1); + +CREATE TABLE t_fixed(i INT PRIMARY KEY) + ENGINE ARIA TRANSACTIONAL=0 ROW_FORMAT=FIXED PAGE_CHECKSUM=1; +INSERT INTO t_fixed VALUES (1); + +CREATE TABLE t_dyn(i INT PRIMARY KEY) + ENGINE ARIA TRANSACTIONAL=0 ROW_FORMAT=DYNAMIC PAGE_CHECKSUM=1; +INSERT INTO t_dyn VALUES (1); + +--echo # Test for partitioned table +CREATE TABLE t_part_online(i INT PRIMARY KEY) + ENGINE ARIA TRANSACTIONAL = 1 PAGE_CHECKSUM = 1 + PARTITION BY RANGE( i ) ( + PARTITION p0 VALUES LESS THAN (10), + PARTITION p1 VALUES LESS THAN (20), + PARTITION p2 VALUES LESS THAN (30) + ); + +INSERT INTO t_part_online VALUES(5); +INSERT INTO t_part_online VALUES(15); +INSERT INTO t_part_online VALUES(25); +SELECT * FROM t_part_online; + +CREATE TABLE t_part_offline(i INT) + ENGINE ARIA TRANSACTIONAL = 0 PAGE_CHECKSUM = 0 + PARTITION BY RANGE( i ) ( + PARTITION p0 VALUES LESS THAN (10), + PARTITION p1 VALUES LESS THAN (20), + PARTITION p2 VALUES LESS THAN (30) + ); + +INSERT INTO t_part_offline VALUES(5); +INSERT INTO t_part_offline VALUES(15); +INSERT INTO t_part_offline VALUES(25); + +--echo # Test for filename to tablename mapping +CREATE TABLE `t 1 t-1`(i INT PRIMARY KEY) + ENGINE ARIA TRANSACTIONAL=1 ROW_FORMAT=PAGE PAGE_CHECKSUM=1; +INSERT INTO `t 1 t-1` VALUES (1); + +CREATE TABLE `t-part online`(i INT PRIMARY KEY) + ENGINE ARIA TRANSACTIONAL = 1 PAGE_CHECKSUM = 1 + PARTITION BY RANGE( i ) ( + PARTITION p0 VALUES LESS THAN (10), + PARTITION p1 VALUES LESS THAN (20), + PARTITION p2 VALUES LESS THAN (30) + ); + +INSERT INTO `t-part online` VALUES(5); +INSERT INTO `t-part online` VALUES(15); +INSERT INTO `t-part online` VALUES(25); + + +--echo ### +--echo # Test for redo log files backup; +--echo ##### +CREATE TABLE t_logs_1(i INT) + ENGINE ARIA TRANSACTIONAL=1 ROW_FORMAT=PAGE PAGE_CHECKSUM=1; +CREATE TABLE t_logs_2 LIKE t_logs_1; +CREATE TABLE t_bulk_ins LIKE t_logs_1; +INSERT INTO t_logs_1 VALUES + (0), (1), (2), (3), (4), (5), (6), (7), (8), (9), + (0), (1), (2), (3), (4), (5), (6), (7), (8), (9), + (0), (1), (2), (3), (4), (5), (6), (7), (8), (9), + (0), (1), (2), (3), (4), (5), (6), (7), (8), (9), + (0), (1), (2), (3), (4), (5), (6), (7), (8), (9), + (0), (1), (2), (3), (4), (5), (6), (7), (8), (9), + (0), (1), (2), (3), (4), (5), (6), (7), (8), (9), + (0), (1), (2), (3), (4), (5), (6), (7), (8), (9), + (0), (1), (2), (3), (4), (5), (6), (7), (8), (9), + (0), (1), (2), (3), (4), (5), (6), (7), (8), (9); +--echo # Generate several log files +--let $i = 0 +while ($i < 14) { +INSERT INTO t_logs_1 SELECT * FROM t_logs_1; +--inc $i +} + +--echo ### +--echo # Test for DML during backup for online backup +--echo ##### +CREATE TABLE t_dml_ins(i INT PRIMARY KEY) + ENGINE ARIA TRANSACTIONAL=1 ROW_FORMAT=PAGE PAGE_CHECKSUM=1; +INSERT INTO t_dml_ins VALUES(1); +--let after_aria_table_copy_test_t_dml_ins=INSERT INTO test.t_dml_ins VALUES(2) +CREATE TABLE t_dml_upd(i INT PRIMARY KEY) + ENGINE ARIA TRANSACTIONAL=1 ROW_FORMAT=PAGE PAGE_CHECKSUM=1; +INSERT INTO t_dml_upd VALUES(1); +--let after_aria_table_copy_test_t_dml_upd=UPDATE test.t_dml_upd SET i = 2 +CREATE TABLE t_dml_del(i INT PRIMARY KEY) + ENGINE ARIA TRANSACTIONAL=1 ROW_FORMAT=PAGE PAGE_CHECKSUM=1; +INSERT INTO t_dml_del VALUES(1); +--let after_aria_table_copy_test_t_dml_del=DELETE FROM test.t_dml_del + +--echo ### +--echo # Test for DDL during backup for online backup +--echo ##### +CREATE DATABASE test_for_db_drop; +CREATE TABLE test_for_db_drop.t(i INT PRIMARY KEY) ENGINE ARIA; +--let after_aria_table_copy_test_for_db_drop_t=DROP DATABASE test_for_db_drop +CREATE TABLE t_db_create(i INT PRIMARY KEY) ENGINE ARIA; +--let after_aria_table_copy_test_t_db_create=CREATE DATABASE test_for_db_create +--sorted_result +SHOW DATABASES; + +CREATE TABLE t_alter(i INT PRIMARY KEY) ENGINE ARIA; +INSERT INTO t_alter VALUES (1); +--let after_aria_table_copy_test_t_alter=ALTER TABLE test.t_alter ADD COLUMN c INT + +CREATE TABLE t_trunc(i INT PRIMARY KEY) ENGINE ARIA; +INSERT INTO t_trunc VALUES (1); +--let after_aria_table_copy_test_t_trunc=TRUNCATE TABLE test.t_trunc + +CREATE TABLE t_ch_i (i int(10), index(i) ) ENGINE=Aria; +INSERT INTO t_ch_i VALUES(1); +--let after_aria_table_copy_test_t_ch_i=ALTER TABLE test.t_ch_i DISABLE KEYS + +CREATE TABLE t_change_engine(i INT PRIMARY KEY) ENGINE InnoDB; +INSERT INTO t_change_engine VALUES (1); +--let after_aria_background=begin not atomic ALTER TABLE test.t_change_engine ENGINE = ARIA; INSERT INTO test.t_logs_1 SELECT * FROM test.t_logs_1; INSERT INTO test.t_bulk_ins SELECT * FROM test.t_logs_1; INSERT INTO test.t_logs_2 SET i = 1; end + +CREATE TABLE t_rename(i INT PRIMARY KEY) ENGINE ARIA; +--let after_aria_table_copy_test_t_rename=RENAME TABLE test.t_rename TO test.t_rename_new +CREATE DATABASE test_for_rename; +CREATE TABLE t_rename_2(i INT PRIMARY KEY) ENGINE ARIA; +--let after_aria_table_copy_test_t_rename_2=RENAME TABLE test.t_rename_2 TO test_for_rename.t_rename_new_2 + +CREATE TABLE t_rename_3(i INT PRIMARY KEY) ENGINE ARIA; +--let after_aria_table_copy_test_t_rename_3=begin not atomic RENAME TABLE test.t_rename_3 TO test.t_rename_new_3; RENAME TABLE test.t_rename_new_3 TO test.t_rename_new_new_3; end + +CREATE TABLE t_rename_4(i INT PRIMARY KEY) ENGINE ARIA; +--let after_aria_table_copy_test_t_rename_4=begin not atomic RENAME TABLE test.t_rename_4 TO test.t_rename_new_4; RENAME TABLE test.t_rename_new_4 TO test.t_rename_new_new_4; RENAME TABLE test.t_rename_new_new_4 TO test.t_rename_new_4; end + +CREATE TABLE t_delete(i INT PRIMARY KEY) ENGINE ARIA; +--let after_aria_table_copy_test_t_delete=DROP TABLE test.t_delete + +CREATE TABLE t_delete_2(i INT PRIMARY KEY) ENGINE ARIA; +--let after_aria_table_copy_test_t_delete_2=ALTER TABLE test.t_delete_2 ENGINE=Innodb + +CREATE TABLE t_rename_alter(i INT PRIMARY KEY) ENGINE ARIA; +--let after_aria_table_copy_test_t_rename_alter=begin not atomic RENAME TABLE test.t_rename_alter TO test.t_rename_alter_2; ALTER TABLE test.t_rename_alter_2 ADD COLUMN c INT; end + +CREATE TABLE t_rename_create(i INT PRIMARY KEY) ENGINE ARIA; +--let after_aria_table_copy_test_t_rename_create=begin not atomic RENAME TABLE test.t_rename_create TO test.t_rename_create_new; CREATE TABLE test.t_rename_create(d INT PRIMARY KEY) ENGINE ARIA; end + +CREATE TABLE t_part_create(i INT PRIMARY KEY) ENGINE ARIA; +--let after_aria_table_copy_test_t_part_create=create table test.t_part_create_2 (i int) engine=Aria PARTITION BY HASH(i) PARTITIONS 2 + +CREATE TABLE t_part_add_part(i INT PRIMARY KEY) ENGINE ARIA; +--let after_aria_table_copy_test_t_part_add_part=alter table test.t_part_add_part PARTITION BY HASH(i) PARTITIONS 2 + +CREATE TABLE t_part_change_eng(i INT PRIMARY KEY) ENGINE ARIA PARTITION BY HASH(i) PARTITIONS 2; +--let after_aria_table_copy_test_t_part_change_eng=alter table test.t_part_change_eng ENGINE=InnoDB + +CREATE TABLE t_part_change_eng_2(i INT PRIMARY KEY) ENGINE InnoDB PARTITION BY HASH(i) PARTITIONS 2; +CREATE TABLE t_part_change_eng_3(i INT PRIMARY KEY) ENGINE Aria; +--let after_aria_table_copy_test_t_part_change_eng_3=alter table test.t_part_change_eng_2 ENGINE=Aria + +CREATE TABLE t_part_alter(i INT PRIMARY KEY) ENGINE Aria PARTITION BY HASH(i) PARTITIONS 2; +--let after_aria_table_copy_test_t_part_alter=alter table test.t_part_alter ADD COLUMN c INT + +CREATE TABLE t_part_alter_2(i INT PRIMARY KEY) ENGINE Aria PARTITION BY HASH(i) PARTITIONS 3; +--let after_aria_table_copy_test_t_part_alter_2=alter table test.t_part_alter_2 COALESCE PARTITION 1 + +CREATE TABLE t_part_drop(i INT PRIMARY KEY) ENGINE Aria PARTITION BY HASH(i) PARTITIONS 2; +--let after_aria_table_copy_test_t_part_drop=DROP table test.t_part_drop + +CREATE TABLE t_part_rename(i INT PRIMARY KEY) ENGINE Aria PARTITION BY HASH(i) PARTITIONS 2; +--let after_aria_table_copy_test_t_part_rename=RENAME TABLE test.t_part_rename TO test.t_part_rename_2 + +CREATE TABLE t_part_rename_3(i INT PRIMARY KEY) ENGINE Aria PARTITION BY HASH(i) PARTITIONS 2; +--let after_aria_table_copy_test_t_part_rename_3=RENAME TABLE test.t_part_rename_3 TO test_for_rename.t_part_rename_4 + +CREATE TABLE t_part_rm_part(i INT PRIMARY KEY) ENGINE Aria PARTITION BY HASH(i) PARTITIONS 2; +--let after_aria_table_copy_test_t_part_rm_part=begin not atomic ALTER TABLE test.t_part_rm_part REMOVE PARTITIONING; ALTER TABLE test.t_part_rm_part ADD COLUMN c INT; end + +SET SESSION debug_dbug="+d,maria_flush_whole_log"; +SET GLOBAL aria_checkpoint_interval=10000; + +--mkdir $targetdir + +if ($backup_variant == $backup_stream) { +--echo ### Backup to stream +--let $streamfile=$MYSQLTEST_VARDIR/tmp/backup.xb +exec $XTRABACKUP --defaults-file=$MYSQLTEST_VARDIR/my.cnf --backup --parallel=10 --target-dir=$targetdir --dbug=+d,mariabackup_events --stream=xbstream > $streamfile 2>$targetdir/backup_stream.log; +--disable_result_log +exec $XBSTREAM -x -C $targetdir < $streamfile; +--enable_result_log +} + +if ($backup_variant == $backup_dir) { +--echo ### Backup to dir +--disable_result_log +--exec $XTRABACKUP --defaults-file=$MYSQLTEST_VARDIR/my.cnf --backup --parallel=10 --target-dir=$targetdir --dbug=+d,mariabackup_events +--enable_result_log +} + +--let $t_logs_1_records_count_before_backup=`SELECT COUNT(*) FROM t_logs_1` +--let $t_logs_2_records_count_before_backup=`SELECT COUNT(*) FROM t_logs_2` +--let $t_bulk_ins_records_count_before_backup=`SELECT COUNT(*) FROM t_bulk_ins` + +--echo # xtrabackup prepare +--disable_result_log +--exec $XTRABACKUP --prepare --target-dir=$targetdir +--source include/restart_and_restore.inc +--enable_result_log + +--echo ### Result for DDL test +SHOW CREATE TABLE t_alter; +SELECT * FROM t_alter; +SHOW CREATE TABLE t_change_engine; +SELECT * FROM t_change_engine; +SELECT * FROM t_trunc; +SELECT * FROM t_ch_i; +SELECT * FROM t_rename_new; +SELECT * FROM test_for_rename.t_rename_new_2; +SELECT * FROM t_rename_new_new_3; +SELECT * FROM t_rename_new_4; +--error ER_NO_SUCH_TABLE +SELECT * FROM t_delete; +SHOW CREATE TABLE t_delete_2; +SELECT * FROM t_rename_alter_2; +SELECT * FROM t_rename_create; +SELECT * FROM t_rename_create_new; +SHOW CREATE TABLE t_part_create_2; +SELECT * FROM t_part_create_2; +SHOW CREATE TABLE t_part_add_part; +SELECT * FROM t_part_add_part; +SHOW CREATE TABLE t_part_change_eng; +SELECT * FROM t_part_change_eng; +SHOW CREATE TABLE t_part_change_eng_2; +SELECT * FROM t_part_change_eng_2; +SELECT * FROM t_part_alter; +SHOW CREATE TABLE t_part_alter_2; +SELECT * FROM t_part_alter_2; +--error ER_NO_SUCH_TABLE +SELECT * FROM t_part_drop; +--error ER_NO_SUCH_TABLE +SELECT * FROM t_part_rename; +SELECT * FROM t_part_rename_2; +--error ER_NO_SUCH_TABLE +SELECT * FROM t_part_rename_3; +SELECT * FROM test_for_rename.t_part_rename_4; +SHOW CREATE TABLE t_part_rm_part; +SELECT * FROM t_part_rm_part; +--sorted_result +SHOW DATABASES; + +--echo ### Clean up for DDL test +DROP DATABASE test_for_db_create; +DROP TABLE t_db_create; +DROP TABLE t_change_engine; +DROP TABLE t_alter; +DROP TABLE t_trunc; +DROP TABLE t_ch_i; +DROP TABLE t_rename_new; +DROP TABLE t_rename_new_new_3; +DROP TABLE t_rename_new_4; +DROP TABLE t_delete_2; +DROP TABLE t_rename_alter_2; +DROP TABLE t_rename_create; +DROP TABLE t_rename_create_new; +DROP TABLE t_part_create; +DROP TABLE t_part_create_2; +DROP TABLE t_part_add_part; +DROP TABLE t_part_change_eng; +DROP TABLE t_part_change_eng_2; +DROP TABLE t_part_change_eng_3; +DROP TABLE t_part_alter; +DROP TABLE t_part_alter_2; +DROP TABLE t_part_rename_2; +DROP TABLE t_part_rm_part; +DROP DATABASE test_for_rename; +--let after_aria_table_copy_test_for_db_drop_t= +--let after_aria_table_copy_test_t_db_create= +--let after_aria_table_copy_test_t_alter= +--let after_aria_background= +--let after_aria_table_copy_test_t_trunc= +--let after_aria_table_copy_test_t_ch_i= +--let after_aria_table_copy_test_t_rename= +--let after_aria_table_copy_test_t_rename_2= +--let after_aria_table_copy_test_t_rename_3= +--let after_aria_table_copy_test_t_rename_4= +--let after_aria_table_copy_test_t_delete= +--let after_aria_table_copy_test_t_delete_2= +--let after_aria_table_copy_test_t_rename_alter= +--let after_aria_table_copy_test_t_rename_create= +--let after_aria_table_copy_test_t_part_create= +--let after_aria_table_copy_test_t_part_add_part= +--let after_aria_table_copy_test_t_part_change_eng= +--let after_aria_table_copy_test_t_part_change_eng_3= +--let after_aria_table_copy_test_t_part_alter= +--let after_aria_table_copy_test_t_part_alter_2= +--let after_aria_table_copy_test_t_part_drop= +--let after_aria_table_copy_test_t_part_rename= +--let after_aria_table_copy_test_t_part_rename_3= +--let after_aria_table_copy_test_t_part_rm_part= + +--echo ### Result for DML test +SELECT * FROM t_dml_ins; +SELECT * FROM t_dml_upd; +SELECT * FROM t_dml_del; + +--echo ### Clean up for DML test +DROP TABLE t_dml_ins; +DROP TABLE t_dml_upd; +DROP TABLE t_dml_del; +--let after_aria_table_copy_test_t_dml_ins= +--let after_aria_table_copy_test_t_dml_upd= +--let after_aria_table_copy_test_t_dml_del= + +--echo ### Result for redo log files backup +--let $t_logs_1_records_count_after_backup=`SELECT COUNT(*) FROM t_logs_1` +--let $t_logs_2_records_count_after_backup=`SELECT COUNT(*) FROM t_logs_2` +--let $t_bulk_ins_records_count_after_backup=`SELECT COUNT(*) FROM t_bulk_ins` +if ($t_logs_1_records_count_after_backup == $t_logs_1_records_count_before_backup) { +--echo # ok +} +if ($t_logs_1_records_count_after_backup != $t_logs_1_records_count_before_backup) { +--echo # failed +} +if ($t_logs_2_records_count_after_backup == $t_logs_2_records_count_before_backup) { +--echo # ok +} +if ($t_logs_2_records_count_after_backup != $t_logs_2_records_count_before_backup) { +--echo # failed +} +if ($t_bulk_ins_records_count_after_backup == $t_bulk_ins_records_count_before_backup) { +--echo # ok +} +if ($t_bulk_ins_records_count_after_backup != $t_bulk_ins_records_count_before_backup) { +--echo # failed +} + +--echo ### Cleanup for redo log files backup +DROP TABLE t_logs_1; +DROP TABLE t_logs_2; +DROP TABLE t_bulk_ins; +--let $t_logs_1_records_count_before_backup= +--let $t_logs_1_records_count_after_backup= +--let $t_logs_2_records_count_before_backup= +--let $t_logs_2_records_count_after_backup= +--let $t_bulk_ins_records_count_before_backup= +--let $t_bulk_ins_records_count_after_backup= + +--echo ### Result for online/offline tables test +SELECT * FROM t_default; +SELECT * FROM t_tr_p_ch; +SELECT * FROM t_tr_p_nch; +SELECT * FROM t_p_ch; +SELECT * FROM t_p_nch; +SELECT * FROM t_fixed; +SELECT * FROM t_dyn; +SELECT * FROM t_part_online; +SELECT * FROM t_part_offline; +SELECT * FROM `t 1 t-1`; +SELECT * FROM `t-part online`; + +--echo ### Cleanup for online/offline tables test +DROP TABLE t_default; +DROP TABLE t_tr_p_ch; +DROP TABLE t_tr_p_nch; +DROP TABLE t_p_ch; +DROP TABLE t_p_nch; +DROP TABLE t_fixed; +DROP TABLE t_dyn; +DROP TABLE t_part_online; +DROP TABLE t_part_offline; +DROP TABLE `t 1 t-1`; +DROP TABLE `t-part online`; + +if ($backup_variant == $backup_stream) { +--remove_file $streamfile +} +--rmdir $targetdir +--dec $backup_variant +} + diff --git a/mysql-test/suite/mariabackup/aria_log.opt b/mysql-test/suite/mariabackup/aria_log.opt new file mode 100644 index 00000000000..f226499f5dd --- /dev/null +++ b/mysql-test/suite/mariabackup/aria_log.opt @@ -0,0 +1 @@ +--loose-aria-log-file-size=8388608 diff --git a/mysql-test/suite/mariabackup/aria_log_dir_path.result b/mysql-test/suite/mariabackup/aria_log_dir_path.result index 1a877321bbe..ead4b836682 100644 --- a/mysql-test/suite/mariabackup/aria_log_dir_path.result +++ b/mysql-test/suite/mariabackup/aria_log_dir_path.result @@ -35,7 +35,6 @@ DROP TABLE t1; SET @@global.aria_checkpoint_interval=DEFAULT /*Force checkpoint*/; SHOW ENGINE aria logs; Type Name Status -Aria aria_log.00000001 free Aria aria_log.00000002 in use # Restarting mariadbd with default parameters # restart diff --git a/mysql-test/suite/mariabackup/aria_log_dir_path.test b/mysql-test/suite/mariabackup/aria_log_dir_path.test index 0178cd4eae5..40bc39446bf 100644 --- a/mysql-test/suite/mariabackup/aria_log_dir_path.test +++ b/mysql-test/suite/mariabackup/aria_log_dir_path.test @@ -48,7 +48,6 @@ SET @@global.aria_checkpoint_interval=DEFAULT /*Force checkpoint*/; --replace_regex /Size +[0-9]+ ; .+aria_log/aria_log/ SHOW ENGINE aria logs; - --echo # mariadb-backup --backup --disable_result_log --mkdir $targetdir @@ -61,7 +60,6 @@ SHOW ENGINE aria logs; --exec $XTRABACKUP --prepare --target-dir=$targetdir --enable_result_log - --echo # shutdown server --disable_result_log --source include/shutdown_mysqld.inc @@ -70,12 +68,14 @@ SHOW ENGINE aria logs; --echo # remove aria-log-dir-path --rmdir $ARIA_LOGDIR_FS + --echo # mariadb-backup --copy-back --let $mariadb_backup_parameters=--defaults-file=$MYSQLTEST_VARDIR/my.cnf --copy-back --datadir=$datadir --target-dir=$targetdir --parallel=2 --throttle=1 --aria-log-dir-path=$ARIA_LOGDIR_MARIADB --replace_result $MYSQL_TEST_DIR MYSQL_TEST_DIR $MYSQLTEST_VARDIR MYSQLTEST_VARDIR --exec echo "# with parameters: $mariadb_backup_parameters" --exec $XTRABACKUP $mariadb_backup_parameters + --echo # starting server --let $restart_parameters=$server_parameters --source include/start_mysqld.inc @@ -91,7 +91,7 @@ DROP TABLE t1; --echo # Testing aria log files after --copy-back SET @@global.aria_checkpoint_interval=DEFAULT /*Force checkpoint*/; --file_exists $ARIA_LOGDIR_FS/aria_log_control ---file_exists $ARIA_LOGDIR_FS/aria_log.00000001 +#--file_exists $ARIA_LOGDIR_FS/aria_log.00000001 --file_exists $ARIA_LOGDIR_FS/aria_log.00000002 --error 1 --file_exists $ARIA_LOGDIR_FS/aria_log.00000003 diff --git a/mysql-test/suite/mariabackup/aria_log_dir_path_rel.result b/mysql-test/suite/mariabackup/aria_log_dir_path_rel.result index 7fef26096e0..736bc5564e8 100644 --- a/mysql-test/suite/mariabackup/aria_log_dir_path_rel.result +++ b/mysql-test/suite/mariabackup/aria_log_dir_path_rel.result @@ -35,7 +35,6 @@ DROP TABLE t1; SET @@global.aria_checkpoint_interval=DEFAULT /*Force checkpoint*/; SHOW ENGINE aria logs; Type Name Status -Aria aria_log.00000001 free Aria aria_log.00000002 in use # Restarting mariadbd with default parameters # restart diff --git a/mysql-test/suite/mariabackup/aria_log_rotate_during_backup.opt b/mysql-test/suite/mariabackup/aria_log_rotate_during_backup.opt new file mode 100644 index 00000000000..7c3ebe422c3 --- /dev/null +++ b/mysql-test/suite/mariabackup/aria_log_rotate_during_backup.opt @@ -0,0 +1,2 @@ +--loose-aria-log-file-size=8388608 +--loose-restart-for-aria_log_rotate_during_backup="This is needed to recreate datadir, to have Aria start logs from aria_log.00000001" diff --git a/mysql-test/suite/mariabackup/aria_log_rotate_during_backup.result b/mysql-test/suite/mariabackup/aria_log_rotate_during_backup.result new file mode 100644 index 00000000000..0691bce8554 --- /dev/null +++ b/mysql-test/suite/mariabackup/aria_log_rotate_during_backup.result @@ -0,0 +1,58 @@ +SHOW VARIABLES LIKE 'aria_log_file_size'; +Variable_name Value +aria_log_file_size 8388608 +CREATE PROCEDURE display_aria_log_control(ctrl BLOB) +BEGIN +SELECT HEX(REVERSE(SUBSTRING(ctrl, 42, 4))) AS last_logno; +END; +$$ +CREATE PROCEDURE populate_t1() +BEGIN +FOR id IN 0..9 DO +INSERT INTO test.t1 (id, txt) VALUES (id, REPEAT(id,1024*1024)); +END FOR; +END; +$$ +CREATE TABLE test.t1(id INT, txt LONGTEXT) ENGINE=Aria; +# MYSQLD_DATADIR/aria_log_control before --backup +CALL display_aria_log_control(@aria_log_control); +last_logno +00000001 +# Running --backup +# MYSQLD_DATADIR/aria_log_control after --backup +CALL display_aria_log_control(@aria_log_control); +last_logno +00000002 +# targetdir/aria_log_control after --backup +CALL display_aria_log_control(@aria_log_control); +last_logno +00000001 +# Running --prepare +# targetdir/aria_log_control after --prepare +CALL display_aria_log_control(@aria_log_control); +last_logno +00000002 +# shutdown server +# remove datadir +# xtrabackup move back +# restart +# MYSQLD_DATADIR/aria_log_control after --copy-back +CALL display_aria_log_control(@aria_log_control); +last_logno +00000002 +# Checking that after --restore all t1 data is there +SELECT id, LENGTH(txt) FROM t1 ORDER BY id; +id LENGTH(txt) +0 1048576 +1 1048576 +2 1048576 +3 1048576 +4 1048576 +5 1048576 +6 1048576 +7 1048576 +8 1048576 +9 1048576 +DROP TABLE t1; +DROP PROCEDURE populate_t1; +DROP PROCEDURE display_aria_log_control; diff --git a/mysql-test/suite/mariabackup/aria_log_rotate_during_backup.test b/mysql-test/suite/mariabackup/aria_log_rotate_during_backup.test new file mode 100644 index 00000000000..172ade338d5 --- /dev/null +++ b/mysql-test/suite/mariabackup/aria_log_rotate_during_backup.test @@ -0,0 +1,82 @@ +--source include/have_debug.inc +--source include/have_aria.inc + +SHOW VARIABLES LIKE 'aria_log_file_size'; + +--let $MYSQLD_DATADIR= `select @@datadir` +--let $targetdir=$MYSQLTEST_VARDIR/tmp/backup +mkdir $targetdir; + + +DELIMITER $$; +CREATE PROCEDURE display_aria_log_control(ctrl BLOB) +BEGIN + SELECT HEX(REVERSE(SUBSTRING(ctrl, 42, 4))) AS last_logno; +END; +$$ +DELIMITER ;$$ + +DELIMITER $$; +CREATE PROCEDURE populate_t1() +BEGIN + FOR id IN 0..9 DO + INSERT INTO test.t1 (id, txt) VALUES (id, REPEAT(id,1024*1024)); + END FOR; +END; +$$ +DELIMITER ;$$ + + +CREATE TABLE test.t1(id INT, txt LONGTEXT) ENGINE=Aria; + +--echo # MYSQLD_DATADIR/aria_log_control before --backup +--let ARIA_DATADIR=$MYSQLD_DATADIR +--source include/aria_log_control_load.inc +CALL display_aria_log_control(@aria_log_control); + + +--echo # Running --backup +--let after_scanning_log_files=CALL test.populate_t1 +--disable_result_log +--exec $XTRABACKUP --defaults-file=$MYSQLTEST_VARDIR/my.cnf --backup --parallel=10 --target-dir=$targetdir --dbug=+d,mariabackup_events 2>&1 +--let after_scanning_log_files= +--enable_result_log + +--echo # MYSQLD_DATADIR/aria_log_control after --backup +--let ARIA_DATADIR=$MYSQLD_DATADIR +--source include/aria_log_control_load.inc +CALL display_aria_log_control(@aria_log_control); + +--echo # targetdir/aria_log_control after --backup +--let ARIA_DATADIR=$targetdir +--source include/aria_log_control_load.inc +CALL display_aria_log_control(@aria_log_control); + + +--echo # Running --prepare +--disable_result_log +--exec $XTRABACKUP --prepare --target-dir=$targetdir +--enable_result_log + +--echo # targetdir/aria_log_control after --prepare +--let ARIA_DATADIR=$targetdir +--source include/aria_log_control_load.inc +CALL display_aria_log_control(@aria_log_control); + + +--disable_result_log +--source include/restart_and_restore.inc +--enable_result_log + +--echo # MYSQLD_DATADIR/aria_log_control after --copy-back +--let ARIA_DATADIR=$MYSQLD_DATADIR +--source include/aria_log_control_load.inc +CALL display_aria_log_control(@aria_log_control); + +--echo # Checking that after --restore all t1 data is there +SELECT id, LENGTH(txt) FROM t1 ORDER BY id; +DROP TABLE t1; +rmdir $targetdir; + +DROP PROCEDURE populate_t1; +DROP PROCEDURE display_aria_log_control; diff --git a/mysql-test/suite/mariabackup/auth_plugin_win.test b/mysql-test/suite/mariabackup/auth_plugin_win.test index 70ae74b7028..7c0ba047014 100644 --- a/mysql-test/suite/mariabackup/auth_plugin_win.test +++ b/mysql-test/suite/mariabackup/auth_plugin_win.test @@ -22,7 +22,7 @@ eval GRANT ALL PRIVILEGES ON *.* to '$USERNAME'; let $targetdir=$MYSQLTEST_VARDIR/tmp/backup; --disable_result_log -exec $XTRABACKUP --defaults-file=$MYSQLTEST_VARDIR/my.cnf -u $USERNAME --backup --protocol=pipe --target-dir=$targetdir; +exec $XTRABACKUP --defaults-file=$MYSQLTEST_VARDIR/my.cnf -u $USERNAME --backup --parallel=10 --protocol=pipe --target-dir=$targetdir; --enable_result_log --replace_result $USERNAME USERNAME eval DROP USER '$USERNAME'; diff --git a/mysql-test/suite/mariabackup/backup_grants.result b/mysql-test/suite/mariabackup/backup_grants.result index 6bd6c9f42cd..ae5b42700d1 100644 --- a/mysql-test/suite/mariabackup/backup_grants.result +++ b/mysql-test/suite/mariabackup/backup_grants.result @@ -6,8 +6,6 @@ GRANT RELOAD, PROCESS on *.* to backup@localhost; FOUND 1 /missing required privilege REPLICA MONITOR/ in backup.log GRANT REPLICA MONITOR ON *.* TO backup@localhost; REVOKE REPLICA MONITOR ON *.* FROM backup@localhost; -FOUND 1 /missing required privilege CONNECTION ADMIN/ in backup.log -GRANT CONNECTION ADMIN ON *.* TO backup@localhost; FOUND 1 /missing required privilege REPLICATION SLAVE ADMIN/ in backup.log FOUND 1 /missing required privilege REPLICA MONITOR/ in backup.log GRANT REPLICATION SLAVE ADMIN ON *.* TO backup@localhost; diff --git a/mysql-test/suite/mariabackup/backup_grants.test b/mysql-test/suite/mariabackup/backup_grants.test index 18db3489a94..b4713d231be 100644 --- a/mysql-test/suite/mariabackup/backup_grants.test +++ b/mysql-test/suite/mariabackup/backup_grants.test @@ -3,14 +3,14 @@ CREATE user backup@localhost; # backup possible for unprivileges user, with --no-lock --disable_result_log -exec $XTRABACKUP --defaults-file=$MYSQLTEST_VARDIR/my.cnf --backup -ubackup --no-lock --target-dir=$targetdir; +exec $XTRABACKUP --defaults-file=$MYSQLTEST_VARDIR/my.cnf --backup --parallel=10 -ubackup --no-lock --target-dir=$targetdir; --enable_result_log rmdir $targetdir; # backup fails without --no-lock, because of FTWRL --disable_result_log error 1; -exec $XTRABACKUP --defaults-file=$MYSQLTEST_VARDIR/my.cnf --backup -ubackup --target-dir=$targetdir > $MYSQLTEST_VARDIR/tmp/backup.log 2>&1; +exec $XTRABACKUP --defaults-file=$MYSQLTEST_VARDIR/my.cnf --backup --parallel=10 -ubackup --target-dir=$targetdir > $MYSQLTEST_VARDIR/tmp/backup.log 2>&1; --enable_result_log let SEARCH_FILE=$MYSQLTEST_VARDIR/tmp/backup.log; @@ -23,7 +23,7 @@ let SEARCH_FILE=$MYSQLTEST_VARDIR/tmp/backup.log; # backup succeeds with RELOAD privilege GRANT RELOAD, PROCESS on *.* to backup@localhost; --disable_result_log -exec $XTRABACKUP --defaults-file=$MYSQLTEST_VARDIR/my.cnf --backup -ubackup --target-dir=$targetdir; +exec $XTRABACKUP --defaults-file=$MYSQLTEST_VARDIR/my.cnf --backup --parallel=10 -ubackup --target-dir=$targetdir; --enable_result_log rmdir $targetdir; @@ -45,24 +45,6 @@ exec $XTRABACKUP --defaults-file=$MYSQLTEST_VARDIR/my.cnf --backup -ubackup --sl rmdir $targetdir; REVOKE REPLICA MONITOR ON *.* FROM backup@localhost; -# TODO need a query that would delay a BACKUP STAGE START/ BACKUP STAGE BLOCK_COMMIT longer than the kill-long-queries-timeout -#--send SELECT SLEEP(9) kill_me - -# kill-long-query-type=(not empty) requires CONNECTION ADMIN ---disable_result_log ---exec $XTRABACKUP --defaults-file=$MYSQLTEST_VARDIR/my.cnf --backup -ubackup --kill-long-query-type=ALL --kill-long-queries-timeout=4 --target-dir=$targetdir > $MYSQLTEST_VARDIR/tmp/backup.log 2>&1; ---enable_result_log -rmdir $targetdir; - ---let SEARCH_PATTERN= missing required privilege CONNECTION ADMIN ---source include/search_pattern_in_file.inc - -GRANT CONNECTION ADMIN ON *.* TO backup@localhost; ---disable_result_log -exec $XTRABACKUP --defaults-file=$MYSQLTEST_VARDIR/my.cnf --backup -ubackup --kill-long-query-type=all --kill-long-queries-timeout=1 --target-dir=$targetdir; ---enable_result_log -rmdir $targetdir; - # --safe-slave-backup requires REPLICATION SLAVE ADMIN, and REPLICA MONITOR --disable_result_log error 1; diff --git a/mysql-test/suite/mariabackup/backup_ssl.test b/mysql-test/suite/mariabackup/backup_ssl.test index e858c834d29..b38073cf19f 100644 --- a/mysql-test/suite/mariabackup/backup_ssl.test +++ b/mysql-test/suite/mariabackup/backup_ssl.test @@ -3,7 +3,7 @@ FLUSH PRIVILEGES; echo # xtrabackup backup; let $targetdir=$MYSQLTEST_VARDIR/tmp/backup; --disable_result_log -exec $XTRABACKUP --defaults-file=$MYSQLTEST_VARDIR/my.cnf --user=backup_user --password=x --ssl --backup --target-dir=$targetdir; +exec $XTRABACKUP --defaults-file=$MYSQLTEST_VARDIR/my.cnf --user=backup_user --password=x --ssl --backup --parallel=10 --target-dir=$targetdir; --enable_result_log echo # xtrabackup prepare; diff --git a/mysql-test/suite/mariabackup/binlog.test b/mysql-test/suite/mariabackup/binlog.test index 9d62e5f8d6b..d02d135ebbd 100644 --- a/mysql-test/suite/mariabackup/binlog.test +++ b/mysql-test/suite/mariabackup/binlog.test @@ -9,7 +9,7 @@ INSERT INTO t VALUES(1); SHOW VARIABLES like 'log_bin'; --disable_result_log -exec $XTRABACKUP --defaults-file=$MYSQLTEST_VARDIR/my.cnf --backup --target-dir=$basedir; +exec $XTRABACKUP --defaults-file=$MYSQLTEST_VARDIR/my.cnf --backup --parallel=10 --target-dir=$basedir; --enable_result_log exec $XTRABACKUP --prepare --binlog-info=1 --target-dir=$basedir ; diff --git a/mysql-test/suite/mariabackup/compress_qpress.test b/mysql-test/suite/mariabackup/compress_qpress.test index c7762f8e55e..263fc55eb67 100644 --- a/mysql-test/suite/mariabackup/compress_qpress.test +++ b/mysql-test/suite/mariabackup/compress_qpress.test @@ -4,7 +4,7 @@ echo # xtrabackup backup; let $targetdir=$MYSQLTEST_VARDIR/tmp/backup; --disable_result_log -exec $XTRABACKUP --defaults-file=$MYSQLTEST_VARDIR/my.cnf --backup --compress --target-dir=$targetdir; +exec $XTRABACKUP --defaults-file=$MYSQLTEST_VARDIR/my.cnf --backup --parallel=10 --compress --target-dir=$targetdir; --enable_result_log INSERT INTO t VALUES(2); diff --git a/mysql-test/suite/mariabackup/create_during_backup.test b/mysql-test/suite/mariabackup/create_during_backup.test index 985a5a3e53a..16d47a648f8 100644 --- a/mysql-test/suite/mariabackup/create_during_backup.test +++ b/mysql-test/suite/mariabackup/create_during_backup.test @@ -7,7 +7,7 @@ mkdir $targetdir; echo # xtrabackup backup; --disable_result_log -exec $XTRABACKUP --defaults-file=$MYSQLTEST_VARDIR/my.cnf --backup --target-dir=$targetdir --dbug=+d,mariabackup_events; +exec $XTRABACKUP --defaults-file=$MYSQLTEST_VARDIR/my.cnf --backup --parallel=10 --target-dir=$targetdir --dbug=+d,mariabackup_events; --enable_result_log --let after_load_tables= diff --git a/mysql-test/suite/mariabackup/create_with_data_directory_during_backup.test b/mysql-test/suite/mariabackup/create_with_data_directory_during_backup.test index f01028b6494..aa7d6de2739 100644 --- a/mysql-test/suite/mariabackup/create_with_data_directory_during_backup.test +++ b/mysql-test/suite/mariabackup/create_with_data_directory_during_backup.test @@ -8,7 +8,7 @@ mkdir $table_data_dir; echo # xtrabackup backup; let $targetdir=$MYSQLTEST_VARDIR/tmp/backup; --disable_result_log -exec $XTRABACKUP --defaults-file=$MYSQLTEST_VARDIR/my.cnf --backup --target-dir=$targetdir --dbug=+d,mariabackup_events; +exec $XTRABACKUP --defaults-file=$MYSQLTEST_VARDIR/my.cnf --backup --parallel=10 --target-dir=$targetdir --dbug=+d,mariabackup_events; --enable_result_log --source include/shutdown_mysqld.inc echo # xtrabackup prepare; diff --git a/mysql-test/suite/mariabackup/data_directory.test b/mysql-test/suite/mariabackup/data_directory.test index ffb3ab3073c..96d76ba0253 100644 --- a/mysql-test/suite/mariabackup/data_directory.test +++ b/mysql-test/suite/mariabackup/data_directory.test @@ -7,7 +7,7 @@ INSERT INTO t VALUES(1); echo # xtrabackup backup; let $targetdir=$MYSQLTEST_VARDIR/tmp/backup; --disable_result_log -exec $XTRABACKUP --defaults-file=$MYSQLTEST_VARDIR/my.cnf --backup --target-dir=$targetdir; +exec $XTRABACKUP --defaults-file=$MYSQLTEST_VARDIR/my.cnf --backup --parallel=10 --target-dir=$targetdir; --enable_result_log --source include/shutdown_mysqld.inc echo # xtrabackup prepare; @@ -21,6 +21,7 @@ rmdir $table_data_dir; SELECT * FROM t; DROP TABLE t; rmdir $targetdir; +rmdir $table_data_dir; --echo # --echo # MDEV-18200 MariaBackup full backup failed with InnoDB: Failing assertion: success @@ -32,8 +33,8 @@ chmod 0000 $DATADIR/ibdata1; exec $XTRABACKUP --defaults-file=$MYSQLTEST_VARDIR/my.cnf --backup --target-dir=$targetdir; --enable_result_log chmod 0755 $DATADIR/ibdata1; -rmdir $table_data_dir; rmdir $targetdir; + --echo # --echo # End of 10.4 tests --echo # diff --git a/mysql-test/suite/mariabackup/ddl_for_common_engine.result b/mysql-test/suite/mariabackup/ddl_for_common_engine.result new file mode 100644 index 00000000000..27a2f288107 --- /dev/null +++ b/mysql-test/suite/mariabackup/ddl_for_common_engine.result @@ -0,0 +1,67 @@ +CREATE TABLE t1 (a INT NOT NULL) ENGINE=CSV; +CREATE TABLE t2 (a INT NOT NULL) ENGINE=CSV; +CREATE TABLE t3 (a INT NOT NULL) ENGINE=CSV; +### Backup to dir +# xtrabackup prepare +# shutdown server +# remove datadir +# xtrabackup move back +# restart +SELECT * FROM t4; +a +SELECT * FROM t2; +ERROR 42S02: Table 'test.t2' doesn't exist +SELECT * FROM t3; +ERROR 42S02: Table 'test.t3' doesn't exist +SELECT * FROM t5; +a +SELECT * FROM t1; +a +DROP TABLE t4, t5, t1; +CREATE TABLE t1_m1 (a INT NOT NULL) ENGINE=MyISAM; +CREATE TABLE t1_m2 (a INT NOT NULL) ENGINE=MyISAM; +CREATE TABLE t1 (a INT NOT NULL) ENGINE=MERGE UNION=(t1_m1, t1_m2) INSERT_METHOD=LAST; +CREATE TABLE t2_m1 (a INT NOT NULL) ENGINE=MyISAM; +CREATE TABLE t2_m2 (a INT NOT NULL) ENGINE=MyISAM; +CREATE TABLE t2 (a INT NOT NULL) ENGINE=MERGE UNION=(t2_m1, t2_m2) INSERT_METHOD=LAST; +CREATE TABLE t3_m1 (a INT NOT NULL) ENGINE=MyISAM; +CREATE TABLE t3_m2 (a INT NOT NULL) ENGINE=MyISAM; +CREATE TABLE t3 (a INT NOT NULL) ENGINE=MERGE UNION=(t3_m1, t3_m2) INSERT_METHOD=LAST; +### Backup to dir +# xtrabackup prepare +# shutdown server +# remove datadir +# xtrabackup move back +# restart +SELECT * FROM t4; +a +SELECT * FROM t2; +ERROR 42S02: Table 'test.t2' doesn't exist +SELECT * FROM t3; +ERROR 42S02: Table 'test.t3' doesn't exist +SELECT * FROM t5; +a +SELECT * FROM t1; +a +DROP TABLE t4, t5, t1; +DROP TABLE t1_m1, t1_m2, t2_m1, t2_m2, t3_m1, t3_m2; +CREATE TABLE t1 (a INT NOT NULL) ENGINE=MyISAM; +CREATE TABLE t2 (a INT NOT NULL) ENGINE=MyISAM; +CREATE TABLE t3 (a INT NOT NULL) ENGINE=MyISAM; +### Backup to dir +# xtrabackup prepare +# shutdown server +# remove datadir +# xtrabackup move back +# restart +SELECT * FROM t4; +a +SELECT * FROM t2; +ERROR 42S02: Table 'test.t2' doesn't exist +SELECT * FROM t3; +ERROR 42S02: Table 'test.t3' doesn't exist +SELECT * FROM t5; +a +SELECT * FROM t1; +a +DROP TABLE t4, t5, t1; diff --git a/mysql-test/suite/mariabackup/ddl_for_common_engine.test b/mysql-test/suite/mariabackup/ddl_for_common_engine.test new file mode 100644 index 00000000000..045c2320edb --- /dev/null +++ b/mysql-test/suite/mariabackup/ddl_for_common_engine.test @@ -0,0 +1,79 @@ +# This test is just to ensure the DDL processing works for common engines like +# MyISAM, ARCHIVE, CSV etc. The more complex test for different cases is +# implemented in aria_backup.test. +--source include/have_archive.inc +--source include/have_csv.inc +--source include/have_debug.inc + +--let $targetdir=$MYSQLTEST_VARDIR/tmp/backup + +--let $e_myisam = 1 +--let $e_merge = 2 +--let $e_csv = 3 +--let $e_archive = 4 +# 'rename' is not logged in $e_archive, return when fix +--let $e_var = $e_csv + +while ($e_var) { +if ($e_var == $e_csv) { +--let $engine = CSV +} +if ($e_var == $e_archive) { +--let $engine = ARCHIVE +} +if ($e_var == $e_merge) { +--let $engine = MERGE +} +if ($e_var == $e_myisam) { +--let $engine = MyISAM +} + +if ($e_var == $e_merge) { +CREATE TABLE t1_m1 (a INT NOT NULL) ENGINE=MyISAM; +CREATE TABLE t1_m2 (a INT NOT NULL) ENGINE=MyISAM; +CREATE TABLE t1 (a INT NOT NULL) ENGINE=MERGE UNION=(t1_m1, t1_m2) INSERT_METHOD=LAST; +CREATE TABLE t2_m1 (a INT NOT NULL) ENGINE=MyISAM; +CREATE TABLE t2_m2 (a INT NOT NULL) ENGINE=MyISAM; +CREATE TABLE t2 (a INT NOT NULL) ENGINE=MERGE UNION=(t2_m1, t2_m2) INSERT_METHOD=LAST; +CREATE TABLE t3_m1 (a INT NOT NULL) ENGINE=MyISAM; +CREATE TABLE t3_m2 (a INT NOT NULL) ENGINE=MyISAM; +CREATE TABLE t3 (a INT NOT NULL) ENGINE=MERGE UNION=(t3_m1, t3_m2) INSERT_METHOD=LAST; +} +if ($e_var != $e_merge) { +eval CREATE TABLE t1 (a INT NOT NULL) ENGINE=$engine; +eval CREATE TABLE t2 (a INT NOT NULL) ENGINE=$engine; +eval CREATE TABLE t3 (a INT NOT NULL) ENGINE=$engine; +} + +--let after_ce_table_copy_test_t1=begin not atomic CREATE TABLE test.t4 LIKE test.t1; DROP TABLE test.t2; RENAME TABLE test.t3 TO test.t5; end + +--mkdir $targetdir +--echo ### Backup to dir +--disable_result_log +--exec $XTRABACKUP --defaults-file=$MYSQLTEST_VARDIR/my.cnf --backup --parallel=10 --target-dir=$targetdir --dbug=+d,mariabackup_events +--enable_result_log + +--echo # xtrabackup prepare +--disable_result_log +--exec $XTRABACKUP --prepare --target-dir=$targetdir +--source include/restart_and_restore.inc +--enable_result_log +--rmdir $targetdir + +SELECT * FROM t4; +--error ER_NO_SUCH_TABLE +SELECT * FROM t2; +--error ER_NO_SUCH_TABLE +SELECT * FROM t3; +SELECT * FROM t5; +SELECT * FROM t1; + +DROP TABLE t4, t5, t1; + +if ($e_var == $e_merge) { +DROP TABLE t1_m1, t1_m2, t2_m1, t2_m2, t3_m1, t3_m2; +} +--let after_ce_table_copy_test_t1= +--dec $e_var +} + diff --git a/mysql-test/suite/mariabackup/disabled.def b/mysql-test/suite/mariabackup/disabled.def index d272540cec8..f8a341814da 100644 --- a/mysql-test/suite/mariabackup/disabled.def +++ b/mysql-test/suite/mariabackup/disabled.def @@ -1 +1,3 @@ log_page_corruption : MDEV-26210 +mariabackup.xb_compressed_encrypted : MDEV-26154 (error 194 "Tablespace is missing for a table") +innodb_ddl_on_intermediate_table : MENT-1213 diff --git a/mysql-test/suite/mariabackup/encrypted_page_compressed.test b/mysql-test/suite/mariabackup/encrypted_page_compressed.test index 54fffb7d08f..245fcc31c0d 100644 --- a/mysql-test/suite/mariabackup/encrypted_page_compressed.test +++ b/mysql-test/suite/mariabackup/encrypted_page_compressed.test @@ -37,7 +37,7 @@ echo # xtrabackup backup; let $targetdir=$MYSQLTEST_VARDIR/tmp/backup; let $backuplog=$MYSQLTEST_VARDIR/tmp/backup.log; --error 1 -exec $XTRABACKUP --defaults-file=$MYSQLTEST_VARDIR/my.cnf --backup --target-dir=$targetdir --core-file > $backuplog; +exec $XTRABACKUP --defaults-file=$MYSQLTEST_VARDIR/my.cnf --backup --parallel=10 --target-dir=$targetdir --core-file > $backuplog; --enable_result_log --let SEARCH_PATTERN=Database page corruption detected.* diff --git a/mysql-test/suite/mariabackup/encrypted_page_corruption.test b/mysql-test/suite/mariabackup/encrypted_page_corruption.test index 1beb020b463..9ba958c68a0 100644 --- a/mysql-test/suite/mariabackup/encrypted_page_corruption.test +++ b/mysql-test/suite/mariabackup/encrypted_page_corruption.test @@ -65,7 +65,7 @@ if (`select @@innodb_checksum_algorithm LIKE '%full_crc32'`) } --disable_result_log --error $expect_error -exec $XTRABACKUP --defaults-file=$MYSQLTEST_VARDIR/my.cnf --backup --extended-validation --target-dir=$targetdir --core-file > $backuplog; +exec $XTRABACKUP --defaults-file=$MYSQLTEST_VARDIR/my.cnf --backup --parallel=10 --extended-validation --target-dir=$targetdir --core-file > $backuplog; --enable_result_log @@ -77,7 +77,7 @@ rmdir $targetdir; # Due to very constructed nature of the "corruption" (faking checksums), the "corruption" won't be found without --extended-validation --disable_result_log -exec $XTRABACKUP --defaults-file=$MYSQLTEST_VARDIR/my.cnf --backup --target-dir=$targetdir; +exec $XTRABACKUP --defaults-file=$MYSQLTEST_VARDIR/my.cnf --backup --parallel=10 --target-dir=$targetdir; --enable_result_log drop table t1; diff --git a/mysql-test/suite/mariabackup/extra_lsndir.test b/mysql-test/suite/mariabackup/extra_lsndir.test index 092ee34c6cc..f880edbef9e 100644 --- a/mysql-test/suite/mariabackup/extra_lsndir.test +++ b/mysql-test/suite/mariabackup/extra_lsndir.test @@ -2,7 +2,7 @@ let $targetdir=$MYSQLTEST_VARDIR/tmp/backup; let $extra_lsndir=$MYSQLTEST_VARDIR/tmp/extra_lsndir; mkdir $extra_lsndir; --disable_result_log -exec $XTRABACKUP --defaults-file=$MYSQLTEST_VARDIR/my.cnf --backup --target-dir=$targetdir --extra-lsndir=$extra_lsndir; +exec $XTRABACKUP --defaults-file=$MYSQLTEST_VARDIR/my.cnf --backup --parallel=10 --target-dir=$targetdir --extra-lsndir=$extra_lsndir; --enable_result_log list_files $extra_lsndir; rmdir $extra_lsndir; diff --git a/mysql-test/suite/mariabackup/full_backup.result.orig b/mysql-test/suite/mariabackup/full_backup.result.orig new file mode 100644 index 00000000000..71525c22080 --- /dev/null +++ b/mysql-test/suite/mariabackup/full_backup.result.orig @@ -0,0 +1,38 @@ +CREATE TABLE t(i INT) ENGINE INNODB; +INSERT INTO t VALUES(1); +# xtrabackup backup +NOT FOUND /InnoDB: Allocated tablespace ID/ in backup.log +INSERT INTO t VALUES(2); +# xtrabackup prepare +# shutdown server +# remove datadir +# xtrabackup move back +# restart +SELECT * FROM t; +i +1 +DROP TABLE t; +# +# MDEV-27121 mariabackup incompatible with disabled dedicated +# undo log tablespaces +# +call mtr.add_suppression("InnoDB: innodb_undo_tablespaces=0 disables dedicated undo log tablespaces"); +call mtr.add_suppression("InnoDB: Cannot change innodb_undo_tablespaces=0 because previous shutdown was not with innodb_fast_shutdown=0"); +call mtr.add_suppression("Found 1 prepared XA transactions"); +CREATE TABLE t(f1 INT NOT NULL)ENGINE=InnoDB; +XA START 'zombie'; +INSERT INTO t VALUES(1); +XA END 'zombie'; +XA PREPARE 'zombie'; +# restart: --innodb_undo_tablespaces=0 +# xtrabackup backup +# xtrabackup prepare +# shutdown server +# remove datadir +# xtrabackup move back +# restart: --innodb_undo_tablespaces=0 +# Display undo log files from target directory +undo001 +undo002 +XA COMMIT 'zombie'; +DROP TABLE t; diff --git a/mysql-test/suite/mariabackup/full_backup.result.rej b/mysql-test/suite/mariabackup/full_backup.result.rej new file mode 100644 index 00000000000..a83f265e136 --- /dev/null +++ b/mysql-test/suite/mariabackup/full_backup.result.rej @@ -0,0 +1,10 @@ +--- mysql-test/suite/mariabackup/full_backup.result ++++ mysql-test/suite/mariabackup/full_backup.result +@@ -17,6 +17,7 @@ DROP TABLE t; + # undo log tablespaces + # + call mtr.add_suppression("InnoDB: innodb_undo_tablespaces=0 disables dedicated undo log tablespaces"); ++call mtr.add_suppression("InnoDB: Cannot change innodb_undo_tablespaces=0 because previous shutdown was not with innodb_fast_shutdown=0"); + # restart: --innodb_undo_tablespaces=0 + # xtrabackup backup + # xtrabackup prepare diff --git a/mysql-test/suite/mariabackup/full_backup.test b/mysql-test/suite/mariabackup/full_backup.test index c6a21112b60..385f3b8785d 100644 --- a/mysql-test/suite/mariabackup/full_backup.test +++ b/mysql-test/suite/mariabackup/full_backup.test @@ -7,7 +7,7 @@ let $targetdir=$MYSQLTEST_VARDIR/tmp/backup; --let $backup_log=$MYSQLTEST_VARDIR/tmp/backup.log --disable_result_log -exec $XTRABACKUP --defaults-file=$MYSQLTEST_VARDIR/my.cnf --backup --target-dir=$targetdir > $backup_log 2>&1; +exec $XTRABACKUP --defaults-file=$MYSQLTEST_VARDIR/my.cnf --backup --target-dir=$targetdir --parallel=10 > $backup_log 2>&1; --enable_result_log # The following warning must not appear after MDEV-27343 fix diff --git a/mysql-test/suite/mariabackup/full_backup.test.orig b/mysql-test/suite/mariabackup/full_backup.test.orig new file mode 100644 index 00000000000..c6a21112b60 --- /dev/null +++ b/mysql-test/suite/mariabackup/full_backup.test.orig @@ -0,0 +1,65 @@ +--source include/innodb_page_size.inc + +CREATE TABLE t(i INT) ENGINE INNODB; +INSERT INTO t VALUES(1); +echo # xtrabackup backup; +let $targetdir=$MYSQLTEST_VARDIR/tmp/backup; +--let $backup_log=$MYSQLTEST_VARDIR/tmp/backup.log + +--disable_result_log +exec $XTRABACKUP --defaults-file=$MYSQLTEST_VARDIR/my.cnf --backup --target-dir=$targetdir > $backup_log 2>&1; +--enable_result_log + +# The following warning must not appear after MDEV-27343 fix +--let SEARCH_PATTERN=InnoDB: Allocated tablespace ID +--let SEARCH_FILE=$backup_log +--source include/search_pattern_in_file.inc +--remove_file $backup_log + +INSERT INTO t VALUES(2); + + +echo # xtrabackup prepare; +--disable_result_log +exec $XTRABACKUP --prepare --target-dir=$targetdir; +-- source include/restart_and_restore.inc +--enable_result_log + +SELECT * FROM t; +DROP TABLE t; +rmdir $targetdir; + +--echo # +--echo # MDEV-27121 mariabackup incompatible with disabled dedicated +--echo # undo log tablespaces +--echo # +call mtr.add_suppression("InnoDB: innodb_undo_tablespaces=0 disables dedicated undo log tablespaces"); +call mtr.add_suppression("InnoDB: Cannot change innodb_undo_tablespaces=0 because previous shutdown was not with innodb_fast_shutdown=0"); +call mtr.add_suppression("Found 1 prepared XA transactions"); + +CREATE TABLE t(f1 INT NOT NULL)ENGINE=InnoDB; +XA START 'zombie'; +INSERT INTO t VALUES(1); +XA END 'zombie'; +XA PREPARE 'zombie'; + +let $restart_parameters=--innodb_undo_tablespaces=0; +--source include/restart_mysqld.inc + +echo # xtrabackup backup; +--disable_result_log +exec $XTRABACKUP --defaults-file=$MYSQLTEST_VARDIR/my.cnf --backup --target-dir=$targetdir; +--enable_result_log + +echo # xtrabackup prepare; +--disable_result_log +exec $XTRABACKUP --prepare --target-dir=$targetdir; +-- source include/restart_and_restore.inc +--enable_result_log + +--echo # Display undo log files from target directory +list_files $targetdir undo*; + +XA COMMIT 'zombie'; +DROP TABLE t; +rmdir $targetdir; diff --git a/mysql-test/suite/mariabackup/full_backup.test.rej b/mysql-test/suite/mariabackup/full_backup.test.rej new file mode 100644 index 00000000000..56996b8d849 --- /dev/null +++ b/mysql-test/suite/mariabackup/full_backup.test.rej @@ -0,0 +1,10 @@ +--- mysql-test/suite/mariabackup/full_backup.test ++++ mysql-test/suite/mariabackup/full_backup.test +@@ -34,6 +34,7 @@ rmdir $targetdir; + --echo # undo log tablespaces + --echo # + call mtr.add_suppression("InnoDB: innodb_undo_tablespaces=0 disables dedicated undo log tablespaces"); ++call mtr.add_suppression("InnoDB: Cannot change innodb_undo_tablespaces=0 because previous shutdown was not with innodb_fast_shutdown=0"); + + let $restart_parameters=--innodb_undo_tablespaces=0; + --source include/restart_mysqld.inc diff --git a/mysql-test/suite/mariabackup/huge_lsn.test b/mysql-test/suite/mariabackup/huge_lsn.test index 8850e9d8954..0da67744457 100644 --- a/mysql-test/suite/mariabackup/huge_lsn.test +++ b/mysql-test/suite/mariabackup/huge_lsn.test @@ -79,7 +79,7 @@ INSERT INTO t VALUES(1); echo # xtrabackup backup; let $targetdir=$MYSQLTEST_VARDIR/tmp/backup; --disable_result_log -exec $XTRABACKUP --defaults-file=$MYSQLTEST_VARDIR/my.cnf --backup --target-dir=$targetdir; +exec $XTRABACKUP --defaults-file=$MYSQLTEST_VARDIR/my.cnf --backup --parallel=10 --target-dir=$targetdir; --enable_result_log SET GLOBAL innodb_flush_log_at_trx_commit=1; INSERT INTO t VALUES(2); diff --git a/mysql-test/suite/mariabackup/huge_lsn.test.orig b/mysql-test/suite/mariabackup/huge_lsn.test.orig new file mode 100644 index 00000000000..8850e9d8954 --- /dev/null +++ b/mysql-test/suite/mariabackup/huge_lsn.test.orig @@ -0,0 +1,114 @@ +--source include/not_embedded.inc +--source include/have_file_key_management.inc + +--echo # +--echo # MDEV-13416 mariabackup fails with EFAULT "Bad Address" +--echo # + +let INNODB_PAGE_SIZE=`select @@innodb_page_size`; +let MYSQLD_DATADIR=`select @@datadir`; + +let $targetdir_old=$MYSQLTEST_VARDIR/tmp/backup_1; +--disable_result_log +exec $XTRABACKUP --defaults-file=$MYSQLTEST_VARDIR/my.cnf --backup --target-dir=$targetdir_old; +--enable_result_log +--source include/shutdown_mysqld.inc + +if ($MTR_COMBINATION_STRICT_CRC32) { +perl; +my $file= "$ENV{MYSQLD_DATADIR}/ibdata1"; +open(FILE, "+<", $file) or die "Unable to open $file\n"; +binmode FILE; +my $ps= $ENV{INNODB_PAGE_SIZE}; +my $page; +die "Unable to read $file" unless sysread(FILE, $page, $ps) == $ps; +substr($page,26,8) = pack("NN", 4096, ~1024); +sysseek(FILE, 0, 0) || die "Unable to rewind $file\n"; +syswrite(FILE, $page, $ps)==$ps || die "Unable to write $file\n"; +close(FILE) || die "Unable to close $file\n"; + +$file= "$ENV{MYSQLD_DATADIR}/ib_logfile0"; +open(FILE, ">", $file) || die "Unable to truncate $file\n"; +close(FILE) || "Unable to close $file\n"; +EOF +--let SEARCH_PATTERN= redo log: [0-9.]*[KMGT]iB; LSN=17596481010687\\b +} + +if (!$MTR_COMBINATION_STRICT_CRC32) { +perl; +do "$ENV{MTR_SUITE_DIR}/../innodb/include/crc32.pl"; +my $file= "$ENV{MYSQLD_DATADIR}/ib_logfile0"; +open(FILE, ">", $file) or die "Unable to open $file\n"; +binmode FILE; +my $extra_repeat = 139820; +# The desired log sequence number is 17596481011216 (0x1000fffffe10). +my $file_size=4<<20; +my $lsn_hi=4096,$lsn_lo=0xfffffe00 - $extra_repeat * 15; +my $polynomial = 0x82f63b78; # CRC-32C +my ($header, $checkpoint, $log); +$header = "Phys" . pack("x[4]NN", $lsn_hi, $lsn_lo - $file_size + 0x300f) . + "some Perl code" . pack("x[478]"); +$header .= pack("Nx[3584]", mycrc32($header, 0, $polynomial)); +$checkpoint = pack("NNNNx[44]", $lsn_hi, $lsn_lo, $lsn_hi, $lsn_lo); +$checkpoint .= pack("Nx[8128]", mycrc32($checkpoint, 0, $polynomial)); +$log = pack("CxxNN", 0xfa, $lsn_hi, $lsn_lo); +$log .= pack("CN", 0, mycrc32($log, 0, $polynomial)); + +# Write more than 2MiB of FILE_MODIFY mini-transactions to exercise the parser. +my $extra = pack("CCxa*", 0xb9, 127, "a/b.ibd"); +$extra .= pack("CN", 0, mycrc32($extra, 0, $polynomial)); + +print FILE $header, $checkpoint, $extra x ($extra_repeat - 1), $log; +seek(FILE, $file_size - 15, 0); +$extra = pack("CCxa*", 0xb9, 127, "c/d.ibd"); +$extra .= pack("CN", 1, mycrc32($extra, 0, $polynomial)); +print FILE $extra; +close(FILE) or die "Unable to close $file\n"; +EOF +--let SEARCH_PATTERN= InnoDB: log sequence number 17596481011216 +--let $restart_parameters=--innodb-log-file-size=4M --innodb-encrypt-log=0 +} + +--source include/start_mysqld.inc +let SEARCH_FILE= $MYSQLTEST_VARDIR/log/mysqld.1.err; +--source include/search_pattern_in_file.inc + +CREATE TABLE t(i INT) ENGINE=INNODB ENCRYPTED=YES; +INSERT INTO t VALUES(1); + +echo # xtrabackup backup; +let $targetdir=$MYSQLTEST_VARDIR/tmp/backup; +--disable_result_log +exec $XTRABACKUP --defaults-file=$MYSQLTEST_VARDIR/my.cnf --backup --target-dir=$targetdir; +--enable_result_log +SET GLOBAL innodb_flush_log_at_trx_commit=1; +INSERT INTO t VALUES(2); +echo # xtrabackup prepare; +--disable_result_log +exec $XTRABACKUP --prepare --target-dir=$targetdir; +--let $restart_parameters= +--source include/restart_and_restore.inc +--enable_result_log +SELECT * FROM t; +FLUSH TABLE t FOR EXPORT; +copy_file $_datadir/test/t.ibd $_datadir/test/t_copy.ibd; +copy_file $_datadir/test/t.cfg $_datadir/test/t_copy.cfg; +UNLOCK TABLES; +ALTER TABLE t DISCARD TABLESPACE; +move_file $_datadir/test/t_copy.ibd $_datadir/test/t.ibd; +move_file $_datadir/test/t_copy.cfg $_datadir/test/t.cfg; +ALTER TABLE t IMPORT TABLESPACE; +FLUSH TABLE t FOR EXPORT; +copy_file $_datadir/test/t.ibd $_datadir/test/t_copy.ibd; +copy_file $_datadir/test/t.cfg $_datadir/test/t_copy.cfg; +UNLOCK TABLES; +ALTER TABLE t DISCARD TABLESPACE; +move_file $_datadir/test/t_copy.ibd $_datadir/test/t.ibd; +move_file $_datadir/test/t_copy.cfg $_datadir/test/t.cfg; +ALTER TABLE t IMPORT TABLESPACE; +DROP TABLE t; +rmdir $targetdir; +let $targetdir= $targetdir_old; +exec $XTRABACKUP --prepare --target-dir=$targetdir; +--source include/restart_and_restore.inc +rmdir $targetdir_old; diff --git a/mysql-test/suite/mariabackup/incremental_encrypted.test b/mysql-test/suite/mariabackup/incremental_encrypted.test index d5570f20006..ddf7e492a56 100644 --- a/mysql-test/suite/mariabackup/incremental_encrypted.test +++ b/mysql-test/suite/mariabackup/incremental_encrypted.test @@ -18,7 +18,7 @@ INSERT INTO t VALUES(1); echo # Create full backup , modify table, then create incremental/differential backup; --disable_result_log -exec $XTRABACKUP --defaults-file=$MYSQLTEST_VARDIR/my.cnf --backup --target-dir=$basedir; +exec $XTRABACKUP --defaults-file=$MYSQLTEST_VARDIR/my.cnf --backup --parallel=10 --target-dir=$basedir; --enable_result_log SET GLOBAL innodb_flush_log_at_trx_commit = 1; @@ -26,7 +26,7 @@ INSERT INTO t VALUES(2); SELECT * FROM t; --disable_result_log -exec $XTRABACKUP --defaults-file=$MYSQLTEST_VARDIR/my.cnf --backup --target-dir=$incremental_dir --incremental-basedir=$basedir; +exec $XTRABACKUP --defaults-file=$MYSQLTEST_VARDIR/my.cnf --backup --parallel=10 --target-dir=$incremental_dir --incremental-basedir=$basedir; echo # Prepare full backup, apply incremental one; exec $XTRABACKUP --prepare --target-dir=$basedir; exec $XTRABACKUP --prepare --target-dir=$basedir --incremental-dir=$incremental_dir; diff --git a/mysql-test/suite/mariabackup/innodb_ddl_on_intermediate_table.result b/mysql-test/suite/mariabackup/innodb_ddl_on_intermediate_table.result new file mode 100644 index 00000000000..46fdfe77145 --- /dev/null +++ b/mysql-test/suite/mariabackup/innodb_ddl_on_intermediate_table.result @@ -0,0 +1,5 @@ +CREATE TABLE IF NOT EXISTS t1 ( col1 INT, col_text TEXT ) ENGINE = InnoDB; +ALTER TABLE t1 ADD FULLTEXT KEY `ftidx1` ( col_text ); +# xtrabackup backup +SET debug_sync='RESET'; +DROP TABLE t1; diff --git a/mysql-test/suite/mariabackup/innodb_ddl_on_intermediate_table.test b/mysql-test/suite/mariabackup/innodb_ddl_on_intermediate_table.test new file mode 100644 index 00000000000..d4c4d70d1a5 --- /dev/null +++ b/mysql-test/suite/mariabackup/innodb_ddl_on_intermediate_table.test @@ -0,0 +1,18 @@ +--source include/have_debug.inc +--source include/have_innodb.inc +--source include/have_debug_sync.inc + +--let $targetdir=$MYSQLTEST_VARDIR/tmp/backup +--mkdir $targetdir + +CREATE TABLE IF NOT EXISTS t1 ( col1 INT, col_text TEXT ) ENGINE = InnoDB; +ALTER TABLE t1 ADD FULLTEXT KEY `ftidx1` ( col_text ); + +echo # xtrabackup backup; +--disable_result_log +exec $XTRABACKUP --defaults-file=$MYSQLTEST_VARDIR/my.cnf --backup --parallel=10 --target-dir=$targetdir --dbug=+d,mariabackup_events,emulate_ddl_on_intermediate_table; +--enable_result_log + +SET debug_sync='RESET'; +rmdir $targetdir; +DROP TABLE t1; diff --git a/mysql-test/suite/mariabackup/lock_ddl_per_table.test b/mysql-test/suite/mariabackup/lock_ddl_per_table.test index 18c207718b5..98e7c5eaf6f 100644 --- a/mysql-test/suite/mariabackup/lock_ddl_per_table.test +++ b/mysql-test/suite/mariabackup/lock_ddl_per_table.test @@ -16,7 +16,7 @@ CREATE TABLE `bobby``tables` (id INT, name VARCHAR(50), purchased DATE) ENGINE I set global innodb_log_checkpoint_now = 1; --disable_result_log -exec $XTRABACKUP --defaults-file=$MYSQLTEST_VARDIR/my.cnf --backup --target-dir=$targetdir --lock-ddl-per-table=1 --dbug=+d,check_mdl_lock_works; +exec $XTRABACKUP --defaults-file=$MYSQLTEST_VARDIR/my.cnf --backup --parallel=10 --target-dir=$targetdir --lock-ddl-per-table=1 --dbug=+d,check_mdl_lock_works; --enable_result_log DROP TABLE t; DROP TABLE `bobby``tables`; diff --git a/mysql-test/suite/mariabackup/log_checksum_mismatch.test b/mysql-test/suite/mariabackup/log_checksum_mismatch.test index c8baf66e917..6cf4b3547e5 100644 --- a/mysql-test/suite/mariabackup/log_checksum_mismatch.test +++ b/mysql-test/suite/mariabackup/log_checksum_mismatch.test @@ -7,7 +7,7 @@ let $targetdir=$MYSQLTEST_VARDIR/tmp/backup; let $backuplog=$MYSQLTEST_VARDIR/tmp/backup.log; --disable_result_log -exec $XTRABACKUP --defaults-file=$MYSQLTEST_VARDIR/my.cnf --backup --target-dir=$targetdir --dbug=+d,log_intermittent_checksum_mismatch --core-file > $backuplog; +exec $XTRABACKUP --defaults-file=$MYSQLTEST_VARDIR/my.cnf --backup --parallel=10 --target-dir=$targetdir --dbug=+d,log_intermittent_checksum_mismatch --core-file > $backuplog; --enable_result_log --let SEARCH_RANGE = 10000000 diff --git a/mysql-test/suite/mariabackup/log_file_unexpected_large_number_in_name.result b/mysql-test/suite/mariabackup/log_file_unexpected_large_number_in_name.result new file mode 100644 index 00000000000..51b4dfc5536 --- /dev/null +++ b/mysql-test/suite/mariabackup/log_file_unexpected_large_number_in_name.result @@ -0,0 +1,20 @@ +# +# Start of 10.5 tests +# +# +# MENT-1587 mariabackup failing due to aria log file copy +# +CREATE TABLE t1(i INT PRIMARY KEY) ENGINE=ARIA; +INSERT INTO t1 VALUES (10); +# Prepare full backup +# shutdown server +# remove datadir +# xtrabackup move back +# restart +SELECT * FROM t1; +i +10 +DROP TABLE t1; +# +# End of 10.5 tests +# diff --git a/mysql-test/suite/mariabackup/log_file_unexpected_large_number_in_name.test b/mysql-test/suite/mariabackup/log_file_unexpected_large_number_in_name.test new file mode 100644 index 00000000000..7fef9d61549 --- /dev/null +++ b/mysql-test/suite/mariabackup/log_file_unexpected_large_number_in_name.test @@ -0,0 +1,47 @@ +--let $MYSQLD_DATADIR=`select @@datadir` + +--echo # +--echo # Start of 10.5 tests +--echo # + +--echo # +--echo # MENT-1587 mariabackup failing due to aria log file copy +--echo # + + +--let $basedir=$MYSQLTEST_VARDIR/tmp/backup +--let $incremental_dir=$MYSQLTEST_VARDIR/tmp/backup_inc1 + +CREATE TABLE t1(i INT PRIMARY KEY) ENGINE=ARIA; +INSERT INTO t1 VALUES (10); + +# +# Add a log file with a number outside of last_log_number +# specified in aria_log_control. +# The actual file number written in the header is 4. +# Let's rename it to 100 for test purposes. +# Hopefully 100 should be enough. +# +--copy_file suite/mariabackup/std_data/ment1587_aria_log.00000004 $MYSQLD_DATADIR/aria_log.00000100 + +--disable_result_log +--exec $XTRABACKUP --defaults-file=$MYSQLTEST_VARDIR/my.cnf --backup --target-dir=$basedir +--enable_result_log + +--disable_result_log +--echo # Prepare full backup +--exec $XTRABACKUP --prepare --target-dir=$basedir +--enable_result_log + +--let $targetdir=$basedir +--source include/restart_and_restore.inc +--enable_result_log +--rmdir $basedir + +SELECT * FROM t1; +DROP TABLE t1; + + +--echo # +--echo # End of 10.5 tests +--echo # diff --git a/mysql-test/suite/mariabackup/log_tables.result b/mysql-test/suite/mariabackup/log_tables.result new file mode 100644 index 00000000000..840efc718e9 --- /dev/null +++ b/mysql-test/suite/mariabackup/log_tables.result @@ -0,0 +1,24 @@ +CREATE TABLE t(i INT) +ENGINE ARIA TRANSACTIONAL=1 ROW_FORMAT=PAGE PAGE_CHECKSUM=1; +SET GLOBAL general_log = 1; +SET GLOBAL log_output = 'TABLE'; +INSERT INTO t VALUES (1); +SELECT * FROM mysql.general_log +WHERE argument LIKE "INSERT INTO %" AND +(command_type = "Query" OR command_type = "Execute") ; +event_time user_host thread_id server_id command_type argument +TIMESTAMP USER_HOST THREAD_ID 1 Query INSERT INTO t VALUES (1) +# Insert new row into general_log table after it has been copied on BLOCK_DDL. +# Backup to dir. +# Xtrabackup prepare. +# shutdown server +# remove datadir +# xtrabackup move back +# restart +SELECT * FROM mysql.general_log +WHERE argument LIKE "INSERT INTO %" AND +(command_type = "Query" OR command_type = "Execute") ; +event_time user_host thread_id server_id command_type argument +TIMESTAMP USER_HOST THREAD_ID 1 Query INSERT INTO t VALUES (1) +TIMESTAMP USER_HOST THREAD_ID 1 Query INSERT INTO test.t VALUES (2) +DROP TABLE t; diff --git a/mysql-test/suite/mariabackup/log_tables.test b/mysql-test/suite/mariabackup/log_tables.test new file mode 100644 index 00000000000..fe540a1ca91 --- /dev/null +++ b/mysql-test/suite/mariabackup/log_tables.test @@ -0,0 +1,49 @@ +# Test for copying log tables tail +--source include/have_aria.inc +--source include/have_debug.inc + +--let $targetdir=$MYSQLTEST_VARDIR/tmp/backup + +CREATE TABLE t(i INT) + ENGINE ARIA TRANSACTIONAL=1 ROW_FORMAT=PAGE PAGE_CHECKSUM=1; + +--let $general_log_old = `SELECT @@global.general_log` +--let $log_output_old = `SELECT @@global.log_output` + +SET GLOBAL general_log = 1; +SET GLOBAL log_output = 'TABLE'; + +INSERT INTO t VALUES (1); + +--replace_column 1 TIMESTAMP 2 USER_HOST 3 THREAD_ID 5 Query +--sorted_result +SELECT * FROM mysql.general_log + WHERE argument LIKE "INSERT INTO %" AND + (command_type = "Query" OR command_type = "Execute") ; + +--echo # Insert new row into general_log table after it has been copied on BLOCK_DDL. +--let after_stage_block_ddl=INSERT INTO test.t VALUES (2) + +--echo # Backup to dir. +--disable_result_log +--exec $XTRABACKUP --defaults-file=$MYSQLTEST_VARDIR/my.cnf --backup --parallel=10 --target-dir=$targetdir --dbug=+d,mariabackup_events +--enable_result_log + +--echo # Xtrabackup prepare. +--disable_result_log +--exec $XTRABACKUP --prepare --target-dir=$targetdir +--source include/restart_and_restore.inc +--enable_result_log + +--replace_column 1 TIMESTAMP 2 USER_HOST 3 THREAD_ID 5 Query +--sorted_result +SELECT * FROM mysql.general_log + WHERE argument LIKE "INSERT INTO %" AND + (command_type = "Query" OR command_type = "Execute") ; + +--rmdir $targetdir +DROP TABLE t; +--disable_query_log +--eval SET GLOBAL general_log = $general_log_old +--eval SET GLOBAL log_output = $log_output_old +--enable_query_log diff --git a/mysql-test/suite/mariabackup/mdev-14447.test b/mysql-test/suite/mariabackup/mdev-14447.test index 79a0d075897..74ae1378ac6 100644 --- a/mysql-test/suite/mariabackup/mdev-14447.test +++ b/mysql-test/suite/mariabackup/mdev-14447.test @@ -11,7 +11,7 @@ CREATE TABLE t(a varchar(40) PRIMARY KEY, b varchar(40), c varchar(40), d varcha echo # Create full backup , modify table, then create incremental/differential backup; --disable_result_log -exec $XTRABACKUP --defaults-file=$MYSQLTEST_VARDIR/my.cnf --backup --target-dir=$basedir; +exec $XTRABACKUP --defaults-file=$MYSQLTEST_VARDIR/my.cnf --backup --parallel=10 --target-dir=$basedir; --enable_result_log SET debug_dbug='+d,skip_page_checksum',foreign_key_checks=0,unique_checks=0; diff --git a/mysql-test/suite/mariabackup/missing_ibd.test b/mysql-test/suite/mariabackup/missing_ibd.test index f406a555b4a..76d5a4ff281 100644 --- a/mysql-test/suite/mariabackup/missing_ibd.test +++ b/mysql-test/suite/mariabackup/missing_ibd.test @@ -24,7 +24,7 @@ call mtr.add_suppression('InnoDB: Ignoring tablespace for test/t1 because it cou echo # xtrabackup backup; let $targetdir=$MYSQLTEST_VARDIR/tmp/backup; --disable_result_log -exec $XTRABACKUP --defaults-file=$MYSQLTEST_VARDIR/my.cnf --backup --target-dir=$targetdir; +exec $XTRABACKUP --defaults-file=$MYSQLTEST_VARDIR/my.cnf --backup --parallel=10 --target-dir=$targetdir; --enable_result_log rmdir $targetdir; diff --git a/mysql-test/suite/mariabackup/nolock_ddl_during_backup_end.test b/mysql-test/suite/mariabackup/nolock_ddl_during_backup_end.test index f6bc51bd9a6..c75c063ab2a 100644 --- a/mysql-test/suite/mariabackup/nolock_ddl_during_backup_end.test +++ b/mysql-test/suite/mariabackup/nolock_ddl_during_backup_end.test @@ -9,6 +9,6 @@ CREATE TABLE t1(i int) ENGINE=INNODB; echo # xtrabackup backup; --disable_result_log error 1; -exec $XTRABACKUP --defaults-file=$MYSQLTEST_VARDIR/my.cnf --backup --target-dir=$targetdir --no-lock --dbug=+d,mariabackup_events; +exec $XTRABACKUP --defaults-file=$MYSQLTEST_VARDIR/my.cnf --backup --parallel=10 --target-dir=$targetdir --no-lock --dbug=+d,mariabackup_events; --enable_result_log rmdir $targetdir; diff --git a/mysql-test/suite/mariabackup/partial.test b/mysql-test/suite/mariabackup/partial.test index 85808749b62..af6da274102 100644 --- a/mysql-test/suite/mariabackup/partial.test +++ b/mysql-test/suite/mariabackup/partial.test @@ -14,7 +14,7 @@ echo # xtrabackup backup; let targetdir=$MYSQLTEST_VARDIR/tmp/backup; --disable_result_log -exec $XTRABACKUP --defaults-file=$MYSQLTEST_VARDIR/my.cnf --backup "--tables=test.*1" --target-dir=$targetdir; +exec $XTRABACKUP --defaults-file=$MYSQLTEST_VARDIR/my.cnf --backup --parallel=10 "--tables=test.*1" --target-dir=$targetdir; --enable_result_log list_files $targetdir/test *.ibd; list_files $targetdir/test *.new; diff --git a/mysql-test/suite/mariabackup/partial_exclude.test b/mysql-test/suite/mariabackup/partial_exclude.test index 6a1ae13b512..973e7a4f328 100644 --- a/mysql-test/suite/mariabackup/partial_exclude.test +++ b/mysql-test/suite/mariabackup/partial_exclude.test @@ -28,7 +28,7 @@ echo # xtrabackup backup; let $targetdir=$MYSQLTEST_VARDIR/tmp/backup; --disable_result_log -exec $XTRABACKUP --defaults-file=$MYSQLTEST_VARDIR/my.cnf --backup "--tables-exclude=test.*2" "--databases-exclude=db2" --target-dir=$targetdir; +exec $XTRABACKUP --defaults-file=$MYSQLTEST_VARDIR/my.cnf --backup --parallel=10 "--tables-exclude=test.*2" "--databases-exclude=db2" --target-dir=$targetdir; --enable_result_log COMMIT; diff --git a/mysql-test/suite/mariabackup/partition_datadir.test b/mysql-test/suite/mariabackup/partition_datadir.test index 36520d331bf..078055a5a1a 100644 --- a/mysql-test/suite/mariabackup/partition_datadir.test +++ b/mysql-test/suite/mariabackup/partition_datadir.test @@ -14,7 +14,7 @@ PARTITION BY RANGE (i) PARTITION p3 VALUES LESS THAN (400) DATA DIRECTORY = '$MYSQLTEST_VARDIR/partitdata', PARTITION p4 VALUES LESS THAN MAXVALUE); INSERT INTO t VALUES (1), (101), (201), (301), (401); -exec $XTRABACKUP --defaults-file=$MYSQLTEST_VARDIR/my.cnf --backup --target-dir=$targetdir; +exec $XTRABACKUP --defaults-file=$MYSQLTEST_VARDIR/my.cnf --backup --parallel=10 --target-dir=$targetdir; exec $XTRABACKUP --prepare --target-dir=$targetdir; DROP TABLE t; rmdir $MYSQLTEST_VARDIR/partitdata; diff --git a/mysql-test/suite/mariabackup/partition_partial.test b/mysql-test/suite/mariabackup/partition_partial.test index 7ccc42c036c..30e31a9d43e 100644 --- a/mysql-test/suite/mariabackup/partition_partial.test +++ b/mysql-test/suite/mariabackup/partition_partial.test @@ -16,7 +16,7 @@ echo # xtrabackup backup; let $targetdir=$MYSQLTEST_VARDIR/tmp/backup; --disable_result_log -exec $XTRABACKUP --defaults-file=$MYSQLTEST_VARDIR/my.cnf --backup "--tables=test.t1" --target-dir=$targetdir; +exec $XTRABACKUP --defaults-file=$MYSQLTEST_VARDIR/my.cnf --backup --parallel=10 "--tables=test.t1" --target-dir=$targetdir; --enable_result_log INSERT INTO t1 VALUES (1), (101), (201), (301); diff --git a/mysql-test/suite/mariabackup/rename_during_mdl_lock.test b/mysql-test/suite/mariabackup/rename_during_mdl_lock.test index 212b7aabd69..b14b04a5e26 100644 --- a/mysql-test/suite/mariabackup/rename_during_mdl_lock.test +++ b/mysql-test/suite/mariabackup/rename_during_mdl_lock.test @@ -3,7 +3,7 @@ let $targetdir=$MYSQLTEST_VARDIR/tmp/backup; mkdir $targetdir; CREATE TABLE t1(i int) ENGINE INNODB; set global innodb_log_checkpoint_now = 1; -exec $XTRABACKUP --defaults-file=$MYSQLTEST_VARDIR/my.cnf --backup --target-dir=$targetdir --lock-ddl-per-table --dbug=+d,rename_during_mdl_lock_table; +exec $XTRABACKUP --defaults-file=$MYSQLTEST_VARDIR/my.cnf --backup --parallel=10 --target-dir=$targetdir --lock-ddl-per-table --dbug=+d,rename_during_mdl_lock_table; echo # xtrabackup prepare; --disable_result_log diff --git a/mysql-test/suite/mariabackup/skip_innodb.test.orig b/mysql-test/suite/mariabackup/skip_innodb.test.orig new file mode 100644 index 00000000000..25495b308b0 --- /dev/null +++ b/mysql-test/suite/mariabackup/skip_innodb.test.orig @@ -0,0 +1,13 @@ +--source include/innodb_undo_tablespaces.inc +let $targetdir=$MYSQLTEST_VARDIR/tmp/backup; +CREATE TABLE t(i int); +INSERT INTO t VALUES(1); +--disable_result_log +exec $XTRABACKUP --defaults-file=$MYSQLTEST_VARDIR/my.cnf --backup --target-dir=$targetdir; +exec $XTRABACKUP --prepare --target-dir=$targetdir; +-- source include/restart_and_restore.inc +--enable_result_log +SELECT * from t; +DROP TABLE t; + +rmdir $targetdir; diff --git a/mysql-test/suite/mariabackup/skip_innodb.test.rej b/mysql-test/suite/mariabackup/skip_innodb.test.rej new file mode 100644 index 00000000000..2be862e891c --- /dev/null +++ b/mysql-test/suite/mariabackup/skip_innodb.test.rej @@ -0,0 +1,17 @@ +--- mysql-test/suite/mariabackup/skip_innodb.test ++++ mysql-test/suite/mariabackup/skip_innodb.test +@@ -1,12 +1,13 @@ ++--source include/innodb_undo_tablespaces.inc + let $targetdir=$MYSQLTEST_VARDIR/tmp/backup; + CREATE TABLE t(i int); + INSERT INTO t VALUES(1); + --disable_result_log +-exec $XTRABACKUP --defaults-file=$MYSQLTEST_VARDIR/my.cnf --backup --target-dir=$targetdir; ++exec $XTRABACKUP --defaults-file=$MYSQLTEST_VARDIR/my.cnf --backup --parallel=10 --target-dir=$targetdir; + exec $XTRABACKUP --prepare --target-dir=$targetdir; + -- source include/restart_and_restore.inc + --enable_result_log + SELECT * from t; + DROP TABLE t; + +-rmdir $targetdir;+rmdir $targetdir; diff --git a/mysql-test/suite/mariabackup/small_ibd.test b/mysql-test/suite/mariabackup/small_ibd.test index e8175fce7c9..bb476b8771e 100644 --- a/mysql-test/suite/mariabackup/small_ibd.test +++ b/mysql-test/suite/mariabackup/small_ibd.test @@ -13,7 +13,7 @@ echo #backup; let $targetdir=$MYSQLTEST_VARDIR/tmp/backup; --disable_result_log -exec $XTRABACKUP --defaults-file=$MYSQLTEST_VARDIR/my.cnf --backup --target-dir=$targetdir; +exec $XTRABACKUP --defaults-file=$MYSQLTEST_VARDIR/my.cnf --backup --parallel=10 --target-dir=$targetdir; --enable_result_log remove_file $_datadir/test/small.ibd; rmdir $targetdir; diff --git a/mysql-test/suite/mariabackup/std_data/ment1587_aria_log.00000004 b/mysql-test/suite/mariabackup/std_data/ment1587_aria_log.00000004 new file mode 100644 index 00000000000..e69de29bb2d diff --git a/mysql-test/suite/mariabackup/system_versioning.test b/mysql-test/suite/mariabackup/system_versioning.test index 1ced00b4588..04a5f72ac2c 100644 --- a/mysql-test/suite/mariabackup/system_versioning.test +++ b/mysql-test/suite/mariabackup/system_versioning.test @@ -5,7 +5,7 @@ update t set a=2; let $targetdir=$MYSQLTEST_VARDIR/tmp/backup; --disable_result_log -exec $XTRABACKUP --defaults-file=$MYSQLTEST_VARDIR/my.cnf --backup --target-dir=$targetdir; +exec $XTRABACKUP --defaults-file=$MYSQLTEST_VARDIR/my.cnf --backup --parallel=10 --target-dir=$targetdir; --enable_result_log insert into t values (3); @@ -32,7 +32,7 @@ insert into t values (1); update t set a=2; --disable_result_log -exec $XTRABACKUP --defaults-file=$MYSQLTEST_VARDIR/my.cnf --backup --target-dir=$targetdir; +exec $XTRABACKUP --defaults-file=$MYSQLTEST_VARDIR/my.cnf --backup --parallel=10 --target-dir=$targetdir; --enable_result_log insert into t values (3); diff --git a/mysql-test/suite/mariabackup/truncate_during_backup.test b/mysql-test/suite/mariabackup/truncate_during_backup.test index 46ee244dfb0..8928fc4eb07 100644 --- a/mysql-test/suite/mariabackup/truncate_during_backup.test +++ b/mysql-test/suite/mariabackup/truncate_during_backup.test @@ -7,7 +7,7 @@ CREATE TABLE t1 ENGINE=InnoDB SELECT 1; --let after_load_tablespaces=TRUNCATE test.t1 --disable_result_log -exec $XTRABACKUP --defaults-file=$MYSQLTEST_VARDIR/my.cnf --backup --target-dir=$targetdir --dbug=+d,mariabackup_events; +exec $XTRABACKUP --defaults-file=$MYSQLTEST_VARDIR/my.cnf --backup --parallel=10 --target-dir=$targetdir --dbug=+d,mariabackup_events; --enable_result_log --let after_load_tablespaces= diff --git a/mysql-test/suite/mariabackup/undo_space_id.test b/mysql-test/suite/mariabackup/undo_space_id.test index 2c56492fd8e..ea762608eb6 100644 --- a/mysql-test/suite/mariabackup/undo_space_id.test +++ b/mysql-test/suite/mariabackup/undo_space_id.test @@ -11,7 +11,7 @@ INSERT INTO t1 VALUES(1); --echo # xtrabackup backup --disable_result_log -exec $XTRABACKUP --defaults-file=$MYSQLTEST_VARDIR/my.cnf --backup --target-dir=$basedir; +exec $XTRABACKUP --defaults-file=$MYSQLTEST_VARDIR/my.cnf --backup --parallel=10 --target-dir=$basedir; --enable_result_log --echo # Display undo log files from target directory list_files $basedir undo*; diff --git a/mysql-test/suite/mariabackup/undo_upgrade.result.orig b/mysql-test/suite/mariabackup/undo_upgrade.result.orig new file mode 100644 index 00000000000..7b77e9ff6e6 --- /dev/null +++ b/mysql-test/suite/mariabackup/undo_upgrade.result.orig @@ -0,0 +1,19 @@ +set global innodb_fast_shutdown=0; +# restart: --innodb_undo_tablespaces=2 +CREATE TABLE t1(a varchar(60)) ENGINE INNODB; +start transaction; +INSERT INTO t1 VALUES(1); +# xtrabackup backup +# Restart the server with 4 undo tablespaces +set global innodb_fast_shutdown=0; +# restart: --innodb_undo_tablespaces=4 +# incremental backup should fail +FOUND 1 /--incremental backup is impossible if the server had been restarted with different innodb_undo_tablespaces/ in backup.log +# Take full backup again +# Prepare full backup +# Display 4 undo log files from target directory +undo001 +undo002 +undo003 +undo004 +DROP TABLE t1; diff --git a/mysql-test/suite/mariabackup/undo_upgrade.result.rej b/mysql-test/suite/mariabackup/undo_upgrade.result.rej new file mode 100644 index 00000000000..8565acdf45b --- /dev/null +++ b/mysql-test/suite/mariabackup/undo_upgrade.result.rej @@ -0,0 +1,22 @@ +--- mysql-test/suite/mariabackup/undo_upgrade.result ++++ mysql-test/suite/mariabackup/undo_upgrade.result +@@ -0,0 +1,19 @@ ++set global innodb_fast_shutdown=0; ++# restart: --innodb_undo_tablespaces=2 ++CREATE TABLE t1(a varchar(60)) ENGINE INNODB; ++start transaction; ++INSERT INTO t1 VALUES(1); ++# xtrabackup backup ++# Restart the server with 4 undo tablespaces ++set global innodb_fast_shutdown=0; ++# restart: --innodb_undo_tablespaces=4 ++# incremental backup should fail ++FOUND 1 /--incremental backup is impossible if the server had been restarted with different innodb_undo_tablespaces./ in backup.log ++# Take full backup again ++# Prepare full backup ++# Display 4 undo log files from target directory ++undo001 ++undo002 ++undo003 ++undo004 ++DROP TABLE t1; diff --git a/mysql-test/suite/mariabackup/undo_upgrade.test.orig b/mysql-test/suite/mariabackup/undo_upgrade.test.orig new file mode 100644 index 00000000000..3d599269782 --- /dev/null +++ b/mysql-test/suite/mariabackup/undo_upgrade.test.orig @@ -0,0 +1,50 @@ +--source include/have_innodb.inc +--source include/innodb_page_size.inc + +let basedir=$MYSQLTEST_VARDIR/tmp/backup; +let incremental_dir=$MYSQLTEST_VARDIR/tmp/backup_inc1; + +set global innodb_fast_shutdown=0; +let $restart_parameters=--innodb_undo_tablespaces=2; +--source include/restart_mysqld.inc + +CREATE TABLE t1(a varchar(60)) ENGINE INNODB; +start transaction; +INSERT INTO t1 VALUES(1); + +--echo # xtrabackup backup +--disable_result_log +exec $XTRABACKUP --defaults-file=$MYSQLTEST_VARDIR/my.cnf --backup --target-dir=$basedir; +--enable_result_log + +--echo # Restart the server with 4 undo tablespaces +let $restart_parameters=--innodb_undo_tablespaces=4; +set global innodb_fast_shutdown=0; +--source include/restart_mysqld.inc + +let $backuplog=$MYSQLTEST_VARDIR/tmp/backup.log; +--echo # incremental backup should fail +--error 1 +exec $XTRABACKUP --defaults-file=$MYSQLTEST_VARDIR/my.cnf --backup --target-dir=$incremental_dir --incremental-basedir=$basedir 2> $backuplog; + +--let SEARCH_PATTERN=--incremental backup is impossible if the server had been restarted with different innodb_undo_tablespaces +--let SEARCH_FILE=$backuplog +--source include/search_pattern_in_file.inc +remove_file $backuplog; +rmdir $incremental_dir; +rmdir $basedir; + +--echo # Take full backup again +--disable_result_log +exec $XTRABACKUP --defaults-file=$MYSQLTEST_VARDIR/my.cnf --backup --target-dir=$basedir; +--enable_result_log +--disable_result_log + +echo # Prepare full backup; +exec $XTRABACKUP --prepare --target-dir=$basedir; + +--echo # Display 4 undo log files from target directory +list_files $basedir undo*; + +DROP TABLE t1; +rmdir $basedir; diff --git a/mysql-test/suite/mariabackup/undo_upgrade.test.rej b/mysql-test/suite/mariabackup/undo_upgrade.test.rej new file mode 100644 index 00000000000..71716d551fa --- /dev/null +++ b/mysql-test/suite/mariabackup/undo_upgrade.test.rej @@ -0,0 +1,53 @@ +--- mysql-test/suite/mariabackup/undo_upgrade.test ++++ mysql-test/suite/mariabackup/undo_upgrade.test +@@ -0,0 +1,50 @@ ++--source include/have_innodb.inc ++--source include/innodb_page_size.inc ++ ++let basedir=$MYSQLTEST_VARDIR/tmp/backup; ++let incremental_dir=$MYSQLTEST_VARDIR/tmp/backup_inc1; ++ ++set global innodb_fast_shutdown=0; ++let $restart_parameters=--innodb_undo_tablespaces=2; ++--source include/restart_mysqld.inc ++ ++CREATE TABLE t1(a varchar(60)) ENGINE INNODB; ++start transaction; ++INSERT INTO t1 VALUES(1); ++ ++--echo # xtrabackup backup ++--disable_result_log ++exec $XTRABACKUP --defaults-file=$MYSQLTEST_VARDIR/my.cnf --backup --target-dir=$basedir; ++--enable_result_log ++ ++--echo # Restart the server with 4 undo tablespaces ++let $restart_parameters=--innodb_undo_tablespaces=4; ++set global innodb_fast_shutdown=0; ++--source include/restart_mysqld.inc ++ ++let $backuplog=$MYSQLTEST_VARDIR/tmp/backup.log; ++--echo # incremental backup should fail ++--error 1 ++exec $XTRABACKUP --defaults-file=$MYSQLTEST_VARDIR/my.cnf --backup --target-dir=$incremental_dir --incremental-basedir=$basedir 2> $backuplog; ++ ++--let SEARCH_PATTERN=--incremental backup is impossible if the server had been restarted with different innodb_undo_tablespaces. ++--let SEARCH_FILE=$backuplog ++--source include/search_pattern_in_file.inc ++remove_file $backuplog; ++rmdir $incremental_dir; ++rmdir $basedir; ++ ++--echo # Take full backup again ++--disable_result_log ++exec $XTRABACKUP --defaults-file=$MYSQLTEST_VARDIR/my.cnf --backup --target-dir=$basedir; ++--enable_result_log ++--disable_result_log ++ ++echo # Prepare full backup; ++exec $XTRABACKUP --prepare --target-dir=$basedir; ++ ++--echo # Display 4 undo log files from target directory ++list_files $basedir undo*; ++ ++DROP TABLE t1; ++rmdir $basedir; diff --git a/mysql-test/suite/mariabackup/unencrypted_page_compressed.result.orig b/mysql-test/suite/mariabackup/unencrypted_page_compressed.result.orig new file mode 100644 index 00000000000..dfcf19b6c2b --- /dev/null +++ b/mysql-test/suite/mariabackup/unencrypted_page_compressed.result.orig @@ -0,0 +1,10 @@ +call mtr.add_suppression("InnoDB: Table `test`.`t1` has an unreadable root page"); +CREATE TABLE t1 (a INT AUTO_INCREMENT PRIMARY KEY, b TEXT, c char(200)) +ENGINE=InnoDB PAGE_COMPRESSED=YES STATS_PERSISTENT=0; +insert into t1(b, c) values("mariadb", "mariabackup"); +InnoDB 0 transactions not purged +# Corrupt the table +# restart: --skip-innodb-buffer-pool-load-at-startup +# xtrabackup backup +FOUND 1 /Database page corruption detected.*/ in backup.log +drop table t1; diff --git a/mysql-test/suite/mariabackup/unencrypted_page_compressed.result.rej b/mysql-test/suite/mariabackup/unencrypted_page_compressed.result.rej new file mode 100644 index 00000000000..cbd53159811 --- /dev/null +++ b/mysql-test/suite/mariabackup/unencrypted_page_compressed.result.rej @@ -0,0 +1,8 @@ +--- mysql-test/suite/mariabackup/unencrypted_page_compressed.result ++++ mysql-test/suite/mariabackup/unencrypted_page_compressed.result +@@ -1,4 +1,5 @@ + call mtr.add_suppression("InnoDB: Table `test`.`t1` has an unreadable root page"); ++SET GLOBAL innodb_purge_rseg_truncate_frequency = 1; + CREATE TABLE t1 (a INT AUTO_INCREMENT PRIMARY KEY, b TEXT, c char(200)) ENGINE=InnoDB page_compressed=yes; + insert into t1(b, c) values("mariadb", "mariabackup"); + InnoDB 0 transactions not purged diff --git a/mysql-test/suite/mariabackup/unencrypted_page_compressed.test b/mysql-test/suite/mariabackup/unencrypted_page_compressed.test index 700c4dd2034..31e8323b8b6 100644 --- a/mysql-test/suite/mariabackup/unencrypted_page_compressed.test +++ b/mysql-test/suite/mariabackup/unencrypted_page_compressed.test @@ -38,7 +38,7 @@ echo # xtrabackup backup; let $targetdir=$MYSQLTEST_VARDIR/tmp/backup; let $backuplog=$MYSQLTEST_VARDIR/tmp/backup.log; --error 1 -exec $XTRABACKUP --defaults-file=$MYSQLTEST_VARDIR/my.cnf --backup --target-dir=$targetdir --core-file > $backuplog; +exec $XTRABACKUP --defaults-file=$MYSQLTEST_VARDIR/my.cnf --backup --parallel=10 --target-dir=$targetdir --core-file > $backuplog; --enable_result_log --let SEARCH_PATTERN=Database page corruption detected.* diff --git a/mysql-test/suite/mariabackup/unencrypted_page_compressed.test.orig b/mysql-test/suite/mariabackup/unencrypted_page_compressed.test.orig new file mode 100644 index 00000000000..700c4dd2034 --- /dev/null +++ b/mysql-test/suite/mariabackup/unencrypted_page_compressed.test.orig @@ -0,0 +1,50 @@ +call mtr.add_suppression("InnoDB: Table `test`.`t1` has an unreadable root page"); +CREATE TABLE t1 (a INT AUTO_INCREMENT PRIMARY KEY, b TEXT, c char(200)) +ENGINE=InnoDB PAGE_COMPRESSED=YES STATS_PERSISTENT=0; +insert into t1(b, c) values("mariadb", "mariabackup"); +--source ../innodb/include/wait_all_purged.inc + +let $MYSQLD_DATADIR=`select @@datadir`; +let t1_IBD = $MYSQLD_DATADIR/test/t1.ibd; +let INNODB_PAGE_SIZE=`select @@innodb_page_size`; + +--source include/shutdown_mysqld.inc + +--echo # Corrupt the table + +perl; +use strict; +use warnings; +use Fcntl qw(:DEFAULT :seek); + +my $ibd_file = $ENV{'t1_IBD'}; + +my $chunk; +my $page_size = $ENV{'INNODB_PAGE_SIZE'}; + +sysopen IBD_FILE, $ibd_file, O_RDWR || die "Unable to open $ibd_file"; +sysseek IBD_FILE, 16384 * 3 + 75, SEEK_CUR; +$chunk = '\xAA\xAA\xAA\xAA'; +syswrite IBD_FILE, $chunk, 4; + +close IBD_FILE; +EOF + +--let $restart_parameters= --skip-innodb-buffer-pool-load-at-startup +--source include/start_mysqld.inc + +echo # xtrabackup backup; +--disable_result_log +let $targetdir=$MYSQLTEST_VARDIR/tmp/backup; +let $backuplog=$MYSQLTEST_VARDIR/tmp/backup.log; +--error 1 +exec $XTRABACKUP --defaults-file=$MYSQLTEST_VARDIR/my.cnf --backup --target-dir=$targetdir --core-file > $backuplog; +--enable_result_log + +--let SEARCH_PATTERN=Database page corruption detected.* +--let SEARCH_FILE=$backuplog +--source include/search_pattern_in_file.inc +remove_file $backuplog; + +drop table t1; +rmdir $targetdir; diff --git a/mysql-test/suite/mariabackup/unencrypted_page_compressed.test.rej b/mysql-test/suite/mariabackup/unencrypted_page_compressed.test.rej new file mode 100644 index 00000000000..dc39536e162 --- /dev/null +++ b/mysql-test/suite/mariabackup/unencrypted_page_compressed.test.rej @@ -0,0 +1,8 @@ +--- mysql-test/suite/mariabackup/unencrypted_page_compressed.test ++++ mysql-test/suite/mariabackup/unencrypted_page_compressed.test +@@ -1,4 +1,5 @@ + call mtr.add_suppression("InnoDB: Table `test`.`t1` has an unreadable root page"); ++SET GLOBAL innodb_purge_rseg_truncate_frequency = 1; + CREATE TABLE t1 (a INT AUTO_INCREMENT PRIMARY KEY, b TEXT, c char(200)) ENGINE=InnoDB page_compressed=yes; + insert into t1(b, c) values("mariadb", "mariabackup"); + --source ../innodb/include/wait_all_purged.inc diff --git a/mysql-test/suite/mariabackup/unsupported_redo.test b/mysql-test/suite/mariabackup/unsupported_redo.test index 97e1cad222d..38cceb2f687 100644 --- a/mysql-test/suite/mariabackup/unsupported_redo.test +++ b/mysql-test/suite/mariabackup/unsupported_redo.test @@ -15,7 +15,7 @@ ALTER TABLE t1 FORCE, ALGORITHM=INPLACE; echo # No longer fails during full backup; --disable_result_log -exec $XTRABACKUP --defaults-file=$MYSQLTEST_VARDIR/my.cnf --backup --target-dir=$basedir; +exec $XTRABACKUP --defaults-file=$MYSQLTEST_VARDIR/my.cnf --backup --parallel=10 --target-dir=$basedir; --enable_result_log DROP TABLE t1; @@ -29,13 +29,13 @@ INSERT INTO t1(a) select 1 union select 2 union select 3; --echo # Create full backup , modify table, then fails during creation of --echo # incremental/differential backup --disable_result_log -exec $XTRABACKUP --defaults-file=$MYSQLTEST_VARDIR/my.cnf --backup --target-dir=$basedir; +exec $XTRABACKUP --defaults-file=$MYSQLTEST_VARDIR/my.cnf --backup --parallel=10 --target-dir=$basedir; --enable_result_log ALTER TABLE t1 FORCE, ALGORITHM=INPLACE; --disable_result_log -exec $XTRABACKUP --defaults-file=$MYSQLTEST_VARDIR/my.cnf --backup --target-dir=$incremental_dir --incremental-basedir=$basedir; +exec $XTRABACKUP --defaults-file=$MYSQLTEST_VARDIR/my.cnf --backup --parallel=10 --target-dir=$incremental_dir --incremental-basedir=$basedir; --enable_result_log DROP TABLE t1; @@ -58,7 +58,7 @@ ALTER TABLE t21 FORCE, ALGORITHM=INPLACE; --echo # unsupported redo log for the table t21. --disable_result_log -exec $XTRABACKUP --defaults-file=$MYSQLTEST_VARDIR/my.cnf --backup "--tables-exclude=test.t21" --target-dir=$targetdir; +exec $XTRABACKUP --defaults-file=$MYSQLTEST_VARDIR/my.cnf --backup --parallel=10 "--tables-exclude=test.t21" --target-dir=$targetdir; --enable_result_log --list_files $targetdir/test *.ibd --list_files $targetdir/test *.new diff --git a/mysql-test/suite/mariabackup/xb_aws_key_management.test b/mysql-test/suite/mariabackup/xb_aws_key_management.test index c8a12f6ed08..a2e407d3b22 100644 --- a/mysql-test/suite/mariabackup/xb_aws_key_management.test +++ b/mysql-test/suite/mariabackup/xb_aws_key_management.test @@ -10,7 +10,7 @@ INSERT INTO t VALUES('foobar1'); echo # xtrabackup backup; let $targetdir=$MYSQLTEST_VARDIR/tmp/backup; --disable_result_log -exec $XTRABACKUP --defaults-file=$MYSQLTEST_VARDIR/my.cnf --backup --target-dir=$targetdir; +exec $XTRABACKUP --defaults-file=$MYSQLTEST_VARDIR/my.cnf --backup --parallel=10 --target-dir=$targetdir; exec $XTRABACKUP --prepare --target-dir=$targetdir; -- source include/restart_and_restore.inc --enable_result_log diff --git a/mysql-test/suite/mariabackup/xb_file_key_management.test b/mysql-test/suite/mariabackup/xb_file_key_management.test index 4d27b2dfa95..eca69c976ff 100644 --- a/mysql-test/suite/mariabackup/xb_file_key_management.test +++ b/mysql-test/suite/mariabackup/xb_file_key_management.test @@ -8,7 +8,7 @@ DELETE FROM t LIMIT 1; echo # xtrabackup backup; let $targetdir=$MYSQLTEST_VARDIR/tmp/backup; --disable_result_log -exec $XTRABACKUP --defaults-file=$MYSQLTEST_VARDIR/my.cnf --backup --target-dir=$targetdir; +exec $XTRABACKUP --defaults-file=$MYSQLTEST_VARDIR/my.cnf --backup --parallel=10 --target-dir=$targetdir; --enable_result_log --let SEARCH_RANGE = 10000000 diff --git a/mysql-test/suite/mariabackup/xb_file_key_management.test.orig b/mysql-test/suite/mariabackup/xb_file_key_management.test.orig new file mode 100644 index 00000000000..4d27b2dfa95 --- /dev/null +++ b/mysql-test/suite/mariabackup/xb_file_key_management.test.orig @@ -0,0 +1,33 @@ +#--source include/innodb_page_size.inc +--source include/have_file_key_management.inc + +CREATE TABLE t(c TEXT) ENGINE INNODB encrypted=yes; +INSERT INTO t VALUES(REPEAT('fubar',100)); +INSERT INTO t VALUES('foobar1'); +DELETE FROM t LIMIT 1; +echo # xtrabackup backup; +let $targetdir=$MYSQLTEST_VARDIR/tmp/backup; +--disable_result_log +exec $XTRABACKUP --defaults-file=$MYSQLTEST_VARDIR/my.cnf --backup --target-dir=$targetdir; +--enable_result_log + +--let SEARCH_RANGE = 10000000 +--let SEARCH_PATTERN=foobar1 +--let SEARCH_FILE=$targetdir/ib_logfile0 +--source include/search_pattern_in_file.inc +--echo # expect NOT FOUND + +INSERT INTO t VALUES('foobar2'); +echo # xtrabackup prepare; + +--disable_result_log +exec $XTRABACKUP --prepare --target-dir=$targetdir; +-- source include/restart_and_restore.inc +--enable_result_log + +--list_files $targetdir ib_logfile* +--remove_file $targetdir/ib_logfile0 + +SELECT * FROM t; +DROP TABLE t; +rmdir $targetdir; diff --git a/mysql-test/suite/mariabackup/xb_history.test b/mysql-test/suite/mariabackup/xb_history.test index f9374a1aaab..e05b7721dc9 100644 --- a/mysql-test/suite/mariabackup/xb_history.test +++ b/mysql-test/suite/mariabackup/xb_history.test @@ -6,7 +6,7 @@ DROP TABLE IF EXISTS mysql.mariadb_backup_history; let $targetdir=$MYSQLTEST_VARDIR/tmp/backup; --disable_result_log -exec $XTRABACKUP --defaults-file=$MYSQLTEST_VARDIR/my.cnf --history=foo --backup --target-dir=$targetdir; +exec $XTRABACKUP --defaults-file=$MYSQLTEST_VARDIR/my.cnf --history=foo --backup --parallel=10 --target-dir=$targetdir; --enable_result_log rmdir $targetdir; diff --git a/mysql-test/suite/mariabackup/xb_history.test.orig b/mysql-test/suite/mariabackup/xb_history.test.orig new file mode 100644 index 00000000000..f9374a1aaab --- /dev/null +++ b/mysql-test/suite/mariabackup/xb_history.test.orig @@ -0,0 +1,27 @@ +#--source include/innodb_page_size.inc + +--disable_warnings +DROP TABLE IF EXISTS mysql.mariadb_backup_history; +--enable_warnings + +let $targetdir=$MYSQLTEST_VARDIR/tmp/backup; +--disable_result_log +exec $XTRABACKUP --defaults-file=$MYSQLTEST_VARDIR/my.cnf --history=foo --backup --target-dir=$targetdir; +--enable_result_log +rmdir $targetdir; + +SHOW CREATE TABLE mysql.mariadb_backup_history; +SELECT COUNT(*) FROM mysql.mariadb_backup_history; +SELECT name FROM mysql.mariadb_backup_history; + +let $targetdir=$MYSQLTEST_VARDIR/tmp/backup; +--disable_result_log +exec $XTRABACKUP --defaults-file=$MYSQLTEST_VARDIR/my.cnf --history --backup --target-dir=$targetdir; +--enable_result_log +rmdir $targetdir; + +SELECT COUNT(*) FROM mysql.mariadb_backup_history; +--sorted_result +SELECT name FROM mysql.mariadb_backup_history; + +DROP TABLE mysql.mariadb_backup_history; diff --git a/mysql-test/suite/mariabackup/xb_page_compress.test b/mysql-test/suite/mariabackup/xb_page_compress.test index 7e806e6de22..e2819e264c1 100644 --- a/mysql-test/suite/mariabackup/xb_page_compress.test +++ b/mysql-test/suite/mariabackup/xb_page_compress.test @@ -27,7 +27,7 @@ echo # xtrabackup backup; let $targetdir=$MYSQLTEST_VARDIR/tmp/backup; --disable_result_log -exec $XTRABACKUP --defaults-file=$MYSQLTEST_VARDIR/my.cnf --backup "--tables=test.*1" --target-dir=$targetdir; +exec $XTRABACKUP --defaults-file=$MYSQLTEST_VARDIR/my.cnf --backup --parallel=10 "--tables=test.*1" --target-dir=$targetdir; echo # xtrabackup prepare; exec $XTRABACKUP --defaults-file=$MYSQLTEST_VARDIR/my.cnf --defaults-group-suffix=.1 --prepare --export --target-dir=$targetdir; --enable_result_log diff --git a/mysql-test/suite/mariabackup/xb_partition.test b/mysql-test/suite/mariabackup/xb_partition.test index 1c8eeaa19e6..13ce8fa2b39 100644 --- a/mysql-test/suite/mariabackup/xb_partition.test +++ b/mysql-test/suite/mariabackup/xb_partition.test @@ -39,7 +39,7 @@ INSERT INTO isam_p VALUES (1), (101), (201), (301); let $targetdir=$MYSQLTEST_VARDIR/tmp; --disable_result_log -exec $XTRABACKUP --defaults-file=$MYSQLTEST_VARDIR/my.cnf --backup --target-dir=$targetdir/full; +exec $XTRABACKUP --defaults-file=$MYSQLTEST_VARDIR/my.cnf --backup --parallel=10 --target-dir=$targetdir/full; --enable_result_log DROP TABLE t1; diff --git a/mysql-test/suite/mariabackup/xb_rocksdb.test b/mysql-test/suite/mariabackup/xb_rocksdb.test index e41f3b2bf7e..6c23fe3d1ea 100644 --- a/mysql-test/suite/mariabackup/xb_rocksdb.test +++ b/mysql-test/suite/mariabackup/xb_rocksdb.test @@ -8,9 +8,9 @@ echo # xtrabackup backup; let $targetdir=$MYSQLTEST_VARDIR/tmp/backup; let $stream=$MYSQLTEST_VARDIR/tmp/backup.xb; --disable_result_log -exec $XTRABACKUP --defaults-file=$MYSQLTEST_VARDIR/my.cnf --backup --target-dir=$targetdir $backup_extra_param; +exec $XTRABACKUP --defaults-file=$MYSQLTEST_VARDIR/my.cnf --backup --parallel=10 --target-dir=$targetdir $backup_extra_param; --enable_result_log -exec $XTRABACKUP --defaults-file=$MYSQLTEST_VARDIR/my.cnf --backup --stream=xbstream > $stream 2>$MYSQLTEST_VARDIR/tmp/backup_stream.log; +exec $XTRABACKUP --defaults-file=$MYSQLTEST_VARDIR/my.cnf --backup --parallel=10 --stream=xbstream > $stream 2>$MYSQLTEST_VARDIR/tmp/backup_stream.log; INSERT INTO t VALUES(2); diff --git a/mysql-test/suite/mariabackup/xb_rocksdb_datadir.test b/mysql-test/suite/mariabackup/xb_rocksdb_datadir.test index c2e90d9075b..2a0b2a4666e 100644 --- a/mysql-test/suite/mariabackup/xb_rocksdb_datadir.test +++ b/mysql-test/suite/mariabackup/xb_rocksdb_datadir.test @@ -9,7 +9,7 @@ INSERT INTO t VALUES(1); echo # xtrabackup backup; let $targetdir=$MYSQLTEST_VARDIR/tmp/backup; --disable_result_log -exec $XTRABACKUP --defaults-file=$MYSQLTEST_VARDIR/my.cnf --backup --target-dir=$targetdir; +exec $XTRABACKUP --defaults-file=$MYSQLTEST_VARDIR/my.cnf --backup --parallel=10 --target-dir=$targetdir; --enable_result_log INSERT INTO t VALUES(2); diff --git a/mysql-test/suite/mariabackup/xbstream.test b/mysql-test/suite/mariabackup/xbstream.test index 212ac598064..8429a3b587d 100644 --- a/mysql-test/suite/mariabackup/xbstream.test +++ b/mysql-test/suite/mariabackup/xbstream.test @@ -8,7 +8,7 @@ mkdir $targetdir; let $streamfile=$MYSQLTEST_VARDIR/tmp/backup.xb; echo # xtrabackup backup to stream; -exec $XTRABACKUP --defaults-file=$MYSQLTEST_VARDIR/my.cnf --backup --databases-exclude=foobar --stream=xbstream > $streamfile 2>$targetdir/backup_stream.log; +exec $XTRABACKUP --defaults-file=$MYSQLTEST_VARDIR/my.cnf --backup --parallel=10 --databases-exclude=foobar --stream=xbstream > $streamfile 2>$targetdir/backup_stream.log; echo # xbstream extract; --disable_result_log exec $XBSTREAM -x -C $targetdir < $streamfile;