diff --git a/.travis.yml b/.travis.yml index 454bd351627..56db0736607 100644 --- a/.travis.yml +++ b/.travis.yml @@ -40,15 +40,50 @@ matrix: include: - os: linux compiler: gcc - # disable cache - was out of disk space - cache: false + addons: + apt: + packages: # make sure these match debian/control contents + - bison + - chrpath + - cmake + - debhelper + - dh-apparmor + - dpatch + - gdb + - libaio-dev + - libboost-dev + - libcurl3-dev + - libdbd-mysql + - libjudy-dev + - libncurses5-dev + - libpam0g-dev + - libpcre3-dev + - libreadline-gplv2-dev + - libstemmer-dev + - libssl-dev + - libnuma-dev + - libxml2-dev + - lsb-release + - perl + - po-debconf + - psmisc + - zlib1g-dev + - libcrack2-dev + - libjemalloc-dev + - libsnappy-dev + - liblzma-dev + - libzmq-dev + - libdistro-info-perl + - uuid-dev + - devscripts + - fakeroot script: - ${CC} --version ; ${CXX} --version - source .travis.compiler.sh # https://github.com/travis-ci/travis-ci/issues/7062 - /run/shm isn't writable or executable # in trusty containers - export MTR_MEM=/tmp - - env DEB_BUILD_OPTIONS="parallel=6" MYSQL_BUILD_PATH=/usr/local/bin:/usr/bin:/bin debian/autobake-deb.sh; + - env DEB_BUILD_OPTIONS="parallel=6" debian/autobake-deb.sh; - ccache --show-stats # Until OSX becomes a bit more stable: MDEV-12435 allow_failures: @@ -103,7 +138,7 @@ addons: - llvm-toolchain-trusty - llvm-toolchain-trusty-3.9 - llvm-toolchain-trusty-4.0 - packages: # make sure these match debian/control contents + packages: # make sure these match the build requirements - gcc-5 - g++-5 - gcc-6 @@ -116,9 +151,6 @@ addons: - bison - chrpath - cmake - - debhelper - - dh-apparmor - - dpatch - gdb - libaio-dev - libboost-dev @@ -135,7 +167,6 @@ addons: - libxml2-dev - lsb-release - perl - - po-debconf - psmisc - zlib1g-dev - libcrack2-dev @@ -143,10 +174,7 @@ addons: - libsnappy-dev - liblzma-dev - libzmq-dev - - libdistro-info-perl - uuid-dev - - devscripts # implicit for any build on Ubuntu - - fakeroot # libsystemd-daemon-dev # https://github.com/travis-ci/apt-package-whitelist/issues/3882 diff --git a/CREDITS b/CREDITS index d352232ad2e..6288c2cdea4 100644 --- a/CREDITS +++ b/CREDITS @@ -4,17 +4,19 @@ organization registered in the USA. The current main sponsors of the MariaDB Foundation are: Alibaba Cloud https://intl.aliyun.com (2017) -Booking.com https://www.booking.com (2013 - 2017) -Development Bank of Singapore https://dbs.com (2016 - 2017) -MariaDB Corporation https://www.mariadb.com (2013 - 2017) -Visma https://visma.com (2015 - 2017) -Acronis http://acronis.com (2016 - 2017) -Nexedi https://www.nexedi.com (2016 - 2017) -Automattic https://automattic.com (2014 - 2017) -Tencent Game DBA http://tencentdba.com/about (2016 - 2017) -Tencent TDSQL http://tdsql.org/ (2016 - 2017) -Verkkokauppa.com https://www.verkkokauppa.com (2015 - 2017) -Virtuozzo https://virtuozzo.com (2016 - 2017) +Booking.com https://www.booking.com (2013) +Tencent Cloud https://cloud.tencent.com (2017) +Development Bank of Singapore https://dbs.com (2016) +IBM https://www.ibm.com (2017) +MariaDB Corporation https://www.mariadb.com (2013) +Visma https://visma.com (2015) +Acronis http://acronis.com (2016) +Nexedi https://www.nexedi.com (2016) +Automattic https://automattic.com (2014) +Tencent Game DBA http://tencentdba.com/about (2016) +Tencent TDSQL http://tdsql.org (2016) +Verkkokauppa.com https://www.verkkokauppa.com (2015) +Virtuozzo https://virtuozzo.com (2016) For a full list of sponsors, see https://mariadb.org/about/supporters/ diff --git a/client/mysql.cc b/client/mysql.cc index b2e83eb8967..8252d9af9e5 100644 --- a/client/mysql.cc +++ b/client/mysql.cc @@ -1143,8 +1143,6 @@ int main(int argc,char *argv[]) outfile[0]=0; // no (default) outfile strmov(pager, "stdout"); // the default, if --pager wasn't given - mysql_init(&mysql); - { char *tmp=getenv("PAGER"); if (tmp && strlen(tmp)) @@ -1183,7 +1181,11 @@ int main(int argc,char *argv[]) } defaults_argv=argv; if ((status.exit_status= get_options(argc, (char **) argv))) - mysql_end(-1); + { + free_defaults(defaults_argv); + my_end(0); + exit(status.exit_status); + } if (status.batch && !status.line_buff && !(status.line_buff= batch_readline_init(MAX_BATCH_BUFFER_SIZE, stdin))) @@ -2319,8 +2321,10 @@ static bool add_line(String &buffer, char *line, ulong line_length, continue; } #endif - if (!*ml_comment && inchar == '\\' && - !(*in_string && + if (!*ml_comment && inchar == '\\' && *in_string != '`' && + !(*in_string == '"' && + (mysql.server_status & SERVER_STATUS_ANSI_QUOTES)) && + !(*in_string && (mysql.server_status & SERVER_STATUS_NO_BACKSLASH_ESCAPES))) { // Found possbile one character command like \c diff --git a/client/mysqlbinlog.cc b/client/mysqlbinlog.cc index 34e810f7b6b..584f6955453 100644 --- a/client/mysqlbinlog.cc +++ b/client/mysqlbinlog.cc @@ -68,6 +68,7 @@ CHARSET_INFO* system_charset_info= &my_charset_utf8_general_ci; /* Needed for Flashback */ DYNAMIC_ARRAY binlog_events; // Storing the events output string +DYNAMIC_ARRAY events_in_stmt; // Storing the events that in one statement String stop_event_string; // Storing the STOP_EVENT output string char server_version[SERVER_VERSION_LENGTH]; @@ -894,6 +895,25 @@ static bool print_row_event(PRINT_EVENT_INFO *print_event_info, Log_event *ev, print_event_info->m_table_map_ignored.get_table(table_id); bool skip_event= (ignored_map != NULL); + if (opt_flashback) + { + Rows_log_event *e= (Rows_log_event*) ev; + // The last Row_log_event will be the first event in Flashback + if (is_stmt_end) + e->clear_flags(Rows_log_event::STMT_END_F); + // The first Row_log_event will be the last event in Flashback + if (events_in_stmt.elements == 0) + e->set_flags(Rows_log_event::STMT_END_F); + // Update the temp_buf + e->update_flags(); + + if (insert_dynamic(&events_in_stmt, (uchar *) &ev)) + { + error("Out of memory: can't allocate memory to store the flashback events."); + exit(1); + } + } + /* end of statement check: i) destroy/free ignored maps @@ -945,7 +965,36 @@ static bool print_row_event(PRINT_EVENT_INFO *print_event_info, Log_event *ev, if (skip_event) return 0; - return print_base64(print_event_info, ev); + if (!opt_flashback) + return print_base64(print_event_info, ev); + else + { + if (is_stmt_end) + { + bool res= false; + Log_event *e= NULL; + + // Print the row_event from the last one to the first one + for (uint i= events_in_stmt.elements; i > 0; --i) + { + e= *(dynamic_element(&events_in_stmt, i - 1, Log_event**)); + res= res || print_base64(print_event_info, e); + } + // Copy all output into the Log_event + ev->output_buf.copy(e->output_buf); + // Delete Log_event + for (uint i= 0; i < events_in_stmt.elements-1; ++i) + { + e= *(dynamic_element(&events_in_stmt, i, Log_event**)); + delete e; + } + reset_dynamic(&events_in_stmt); + + return res; + } + } + + return 0; } @@ -1386,6 +1435,8 @@ Exit_status process_event(PRINT_EVENT_INFO *print_event_info, Log_event *ev, } if (print_base64(print_event_info, ev)) goto err; + if (opt_flashback) + reset_dynamic(&events_in_stmt); break; } case WRITE_ROWS_EVENT: @@ -1402,9 +1453,12 @@ Exit_status process_event(PRINT_EVENT_INFO *print_event_info, Log_event *ev, case DELETE_ROWS_COMPRESSED_EVENT_V1: { Rows_log_event *e= (Rows_log_event*) ev; + bool is_stmt_end= e->get_flags(Rows_log_event::STMT_END_F); if (print_row_event(print_event_info, ev, e->get_table_id(), e->get_flags(Rows_log_event::STMT_END_F))) goto err; + if (!is_stmt_end) + destroy_evt= FALSE; break; } case PRE_GA_WRITE_ROWS_EVENT: @@ -1412,9 +1466,12 @@ Exit_status process_event(PRINT_EVENT_INFO *print_event_info, Log_event *ev, case PRE_GA_UPDATE_ROWS_EVENT: { Old_rows_log_event *e= (Old_rows_log_event*) ev; + bool is_stmt_end= e->get_flags(Rows_log_event::STMT_END_F); if (print_row_event(print_event_info, ev, e->get_table_id(), e->get_flags(Old_rows_log_event::STMT_END_F))) goto err; + if (!is_stmt_end) + destroy_evt= FALSE; break; } case START_ENCRYPTION_EVENT: @@ -1459,7 +1516,7 @@ end: &my_charset_bin); else { - if (push_dynamic(&binlog_events, (uchar *) &tmp_str)) + if (insert_dynamic(&binlog_events, (uchar *) &tmp_str)) { error("Out of memory: can't allocate memory to store the flashback events."); exit(1); @@ -2915,9 +2972,12 @@ int main(int argc, char** argv) my_set_max_open_files(open_files_limit); if (opt_flashback) + { my_init_dynamic_array(&binlog_events, sizeof(LEX_STRING), 1024, 1024, MYF(0)); - + my_init_dynamic_array(&events_in_stmt, sizeof(Rows_log_event*), 1024, 1024, + MYF(0)); + } if (opt_stop_never) to_last_remote_log= TRUE; @@ -3031,6 +3091,7 @@ int main(int argc, char** argv) } fprintf(result_file, "COMMIT\n/*!*/;\n"); delete_dynamic(&binlog_events); + delete_dynamic(&events_in_stmt); } /* Set delimiter back to semicolon */ diff --git a/client/mysqltest.cc b/client/mysqltest.cc index 4c14234174e..915727adedb 100644 --- a/client/mysqltest.cc +++ b/client/mysqltest.cc @@ -82,7 +82,7 @@ static my_bool non_blocking_api_enabled= 0; #define MAX_DELIMITER_LENGTH 16 #define DEFAULT_MAX_CONN 64 -#define DIE_BUFF_SIZE 8192 +#define DIE_BUFF_SIZE 256*1024 /* Flags controlling send and reap */ #define QUERY_SEND_FLAG 1 @@ -1720,12 +1720,23 @@ void log_msg(const char *fmt, ...) int cat_file(DYNAMIC_STRING* ds, const char* filename) { int fd; - int len; - char buff[16384]; + size_t len; + char *buff; if ((fd= my_open(filename, O_RDONLY, MYF(0))) < 0) return 1; - while((len= (int)my_read(fd, (uchar*)&buff, sizeof(buff)-1, MYF(0))) > 0) + + len= (size_t) my_seek(fd, 0, SEEK_END, MYF(0)); + my_seek(fd, 0, SEEK_SET, MYF(0)); + if (len == (size_t)MY_FILEPOS_ERROR || + !(buff= (char*)my_malloc(len + 1, MYF(0)))) + { + my_close(fd, MYF(0)); + return 1; + } + len= my_read(fd, (uchar*)buff, len, MYF(0)); + my_close(fd, MYF(0)); + { char *p= buff, *start= buff,*end=buff+len; while (p < end) @@ -1748,7 +1759,7 @@ int cat_file(DYNAMIC_STRING* ds, const char* filename) *p= 0; replace_dynstr_append_mem(ds, start, p-start); } - my_close(fd, MYF(0)); + my_free(buff); return 0; } @@ -6490,6 +6501,16 @@ my_bool end_of_query(int c) } +static inline bool is_escape_char(char c, char in_string) +{ + if (c != '\\' || in_string == '`') return false; + if (!cur_con) return true; + uint server_status= cur_con->mysql->server_status; + if (server_status & SERVER_STATUS_NO_BACKSLASH_ESCAPES) return false; + return !(server_status & SERVER_STATUS_ANSI_QUOTES && in_string == '"'); +} + + /* Read one "line" from the file @@ -6516,7 +6537,7 @@ my_bool end_of_query(int c) int read_line(char *buf, int size) { - char c, UNINIT_VAR(last_quote), last_char= 0; + char c, last_quote=0, last_char= 0; char *p= buf, *buf_end= buf + size - 1; int skip_char= 0; my_bool have_slash= FALSE; @@ -6598,7 +6619,7 @@ int read_line(char *buf, int size) state= R_Q; } } - have_slash= (c == '\\'); + have_slash= is_escape_char(c, last_quote); break; case R_COMMENT: @@ -6668,7 +6689,7 @@ int read_line(char *buf, int size) case R_Q: if (c == last_quote) state= R_NORMAL; - else if (c == '\\') + else if (is_escape_char(c, last_quote)) state= R_SLASH_IN_Q; break; diff --git a/cmake/cpack_rpm.cmake b/cmake/cpack_rpm.cmake index 7a037d7ce43..47daa88d0a2 100644 --- a/cmake/cpack_rpm.cmake +++ b/cmake/cpack_rpm.cmake @@ -33,7 +33,12 @@ SET(CPACK_COMPONENTS_ALL Server ManPagesServer IniFiles Server_Scripts ) SET(CPACK_RPM_PACKAGE_NAME ${CPACK_PACKAGE_NAME}) -SET(CPACK_PACKAGE_FILE_NAME "${CPACK_RPM_PACKAGE_NAME}-${VERSION}-${RPM}-${CMAKE_SYSTEM_PROCESSOR}") +IF(CMAKE_VERSION VERSION_LESS "3.6.0") + SET(CPACK_PACKAGE_FILE_NAME "${CPACK_RPM_PACKAGE_NAME}-${VERSION}-${RPM}-${CMAKE_SYSTEM_PROCESSOR}") +ELSE() + SET(CPACK_RPM_FILE_NAME "RPM-DEFAULT") + SET(CPACK_RPM_DEBUGINFO_PACKAGE ON) +ENDIF() SET(CPACK_RPM_PACKAGE_RELEASE "1%{?dist}") SET(CPACK_RPM_PACKAGE_LICENSE "GPLv2") diff --git a/cmake/for_clients.cmake b/cmake/for_clients.cmake index 636610e7fba..e5916c56ddc 100644 --- a/cmake/for_clients.cmake +++ b/cmake/for_clients.cmake @@ -45,7 +45,7 @@ MACRO(EXTRACT_LINK_LIBRARIES target var) ENDIF() ENDMACRO() -EXTRACT_LINK_LIBRARIES(mariadb LIBS) +EXTRACT_LINK_LIBRARIES(libmariadb LIBS) EXTRACT_LINK_LIBRARIES(mysqlserver EMB_LIBS) SET(LIBS "-lmariadb ${ZLIB_DEPS} ${LIBS} ${openssl_libs}") @@ -72,6 +72,6 @@ REPLACE_FOR_CLIENTS(CFLAGS "[DU]DBUG_OFF" "[DU]SAFE_MUTEX" "[DU]NDEBUG" "xstrconst" "xc99=none" "AC99" "restrict" "W[-A-Za-z]*=[-A-Za-z0-9]*") # Same for --libs -REPLACE_FOR_CLIENTS(LIBS lmtmalloc static-libcxa i-static static-intel) +REPLACE_FOR_CLIENTS(LIBS "Wl,[^ ]*" lmtmalloc static-libcxa i-static static-intel) REPLACE_FOR_CLIENTS(EMB_LIBS lmtmalloc static-libcxa i-static static-intel) diff --git a/cmake/ssl.cmake b/cmake/ssl.cmake index d407ea3d26f..be25c3bbd28 100644 --- a/cmake/ssl.cmake +++ b/cmake/ssl.cmake @@ -124,7 +124,7 @@ MACRO (MYSQL_CHECK_SSL) SET(OPENSSL_ROOT_DIR ${WITH_SSL_PATH}) ENDIF() ENDIF() - FIND_PACKAGE(OpenSSL) + FIND_PACKAGE(OpenSSL 1.0.0) IF(OPENSSL_FOUND) SET(OPENSSL_LIBRARY ${OPENSSL_SSL_LIBRARY}) INCLUDE(CheckSymbolExists) diff --git a/debian/control b/debian/control index 3ee57b2e6a4..e8134dfcd65 100644 --- a/debian/control +++ b/debian/control @@ -595,6 +595,16 @@ Replaces: mariadb-gssapi-client-10.1, mariadb-gssapi-client-10.3 Description: GSSAPI authentication plugin for MariaDB client +Package: mariadb-backup-10.2 +Section: database +Architecture: any +Breaks: mariadb-backup-10.1 +Replaces: mariadb-backup-10.1 +Depends: mariadb-server-10.2, + ${misc:Depends}, + ${shlibs:Depends} +Description: Backup tool for MariaDB server + Package: mariadb-plugin-cracklib-password-check Architecture: any Depends: libcrack2 (>= 2.9.0), diff --git a/debian/mariadb-backup-10.1.files b/debian/mariadb-backup-10.2.files similarity index 100% rename from debian/mariadb-backup-10.1.files rename to debian/mariadb-backup-10.2.files diff --git a/extra/mariabackup/CMakeLists.txt b/extra/mariabackup/CMakeLists.txt index 0dbcbc2889a..78188d2f8e8 100644 --- a/extra/mariabackup/CMakeLists.txt +++ b/extra/mariabackup/CMakeLists.txt @@ -30,7 +30,6 @@ ENDIF() INCLUDE_DIRECTORIES( ${CMAKE_SOURCE_DIR}/include - ${CMAKE_SOURCE_DIR}/storage/innobase/include ${CMAKE_SOURCE_DIR}/sql ${CMAKE_CURRENT_SOURCE_DIR}/quicklz ${CMAKE_CURRENT_SOURCE_DIR} @@ -74,9 +73,9 @@ MYSQL_ADD_EXECUTABLE(mariabackup backup_mysql.cc backup_copy.cc encryption_plugin.cc - ${PROJECT_SOURCE_DIR}/libmysql/libmysql.c ${PROJECT_SOURCE_DIR}/sql/net_serv.cc ${NT_SERVICE_SOURCE} + ${PROJECT_SOURCE_DIR}/libmysqld/libmysql.c COMPONENT backup ) diff --git a/extra/mariabackup/backup_copy.cc b/extra/mariabackup/backup_copy.cc index 156e0b20e7c..19168ff991b 100644 --- a/extra/mariabackup/backup_copy.cc +++ b/extra/mariabackup/backup_copy.cc @@ -89,7 +89,7 @@ struct datadir_iter_t { ulint filepath_len; char *filepath_rel; ulint filepath_rel_len; - os_ib_mutex_t mutex; + pthread_mutex_t mutex; os_file_dir_t dir; os_file_dir_t dbdir; os_file_stat_t dbinfo; @@ -107,7 +107,7 @@ struct datadir_thread_ctxt_t { datadir_iter_t *it; uint n_thread; uint *count; - os_ib_mutex_t count_mutex; + pthread_mutex_t count_mutex; os_thread_id_t id; bool ret; }; @@ -134,12 +134,12 @@ datadir_node_fill(datadir_node_t *node, datadir_iter_t *it) { if (node->filepath_len < it->filepath_len) { free(node->filepath); - node->filepath = (char*)(ut_malloc(it->filepath_len)); + node->filepath = (char*)(malloc(it->filepath_len)); node->filepath_len = it->filepath_len; } if (node->filepath_rel_len < it->filepath_rel_len) { free(node->filepath_rel); - node->filepath_rel = (char*)(ut_malloc(it->filepath_rel_len)); + node->filepath_rel = (char*)(malloc(it->filepath_rel_len)); node->filepath_rel_len = it->filepath_rel_len; } @@ -153,8 +153,8 @@ static void datadir_node_free(datadir_node_t *node) { - ut_free(node->filepath); - ut_free(node->filepath_rel); + free(node->filepath); + free(node->filepath_rel); memset(node, 0, sizeof(datadir_node_t)); } @@ -178,10 +178,10 @@ datadir_iter_new(const char *path, bool skip_first_level = true) { datadir_iter_t *it; - it = static_cast(ut_malloc(sizeof(datadir_iter_t))); + it = static_cast(malloc(sizeof(datadir_iter_t))); memset(it, 0, sizeof(datadir_iter_t)); - it->mutex = os_mutex_create(); + pthread_mutex_init(&it->mutex, NULL); it->datadir_path = strdup(path); it->dir = os_file_opendir(it->datadir_path, TRUE); @@ -194,20 +194,20 @@ datadir_iter_new(const char *path, bool skip_first_level = true) it->err = DB_SUCCESS; it->dbpath_len = FN_REFLEN; - it->dbpath = static_cast(ut_malloc(it->dbpath_len)); + it->dbpath = static_cast(malloc(it->dbpath_len)); it->filepath_len = FN_REFLEN; - it->filepath = static_cast(ut_malloc(it->filepath_len)); + it->filepath = static_cast(malloc(it->filepath_len)); it->filepath_rel_len = FN_REFLEN; - it->filepath_rel = static_cast(ut_malloc(it->filepath_rel_len)); + it->filepath_rel = static_cast(malloc(it->filepath_rel_len)); it->skip_first_level = skip_first_level; return(it); error: - ut_free(it); + free(it); return(NULL); } @@ -246,19 +246,15 @@ datadir_iter_next_database(datadir_iter_t *it) + strlen (it->dbinfo.name) + 2; if (len > it->dbpath_len) { it->dbpath_len = len; + free(it->dbpath); - if (it->dbpath) { - - ut_free(it->dbpath); - } - - it->dbpath = static_cast - (ut_malloc(it->dbpath_len)); + it->dbpath = static_cast( + malloc(it->dbpath_len)); } ut_snprintf(it->dbpath, it->dbpath_len, "%s/%s", it->datadir_path, it->dbinfo.name); - srv_normalize_path_for_win(it->dbpath); + os_normalize_path(it->dbpath); if (it->dbinfo.type == OS_FILE_TYPE_FILE) { it->is_file = true; @@ -306,8 +302,8 @@ make_path_n(int n, char **path, ulint *path_len, ...) va_end(vl); if (len_needed < *path_len) { - ut_free(*path); - *path = static_cast(ut_malloc(len_needed)); + free(*path); + *path = static_cast(malloc(len_needed)); } va_start(vl, path_len); @@ -378,7 +374,7 @@ datadir_iter_next(datadir_iter_t *it, datadir_node_t *node) { bool ret = true; - os_mutex_enter(it->mutex); + pthread_mutex_lock(&it->mutex); if (datadir_iter_next_file(it)) { @@ -413,7 +409,7 @@ datadir_iter_next(datadir_iter_t *it, datadir_node_t *node) ret = false; done: - os_mutex_exit(it->mutex); + pthread_mutex_unlock(&it->mutex); return(ret); } @@ -427,7 +423,7 @@ static void datadir_iter_free(datadir_iter_t *it) { - os_mutex_free(it->mutex); + pthread_mutex_destroy(&it->mutex); if (it->dbdir) { @@ -439,11 +435,11 @@ datadir_iter_free(datadir_iter_t *it) os_file_closedir(it->dir); } - ut_free(it->dbpath); - ut_free(it->filepath); - ut_free(it->filepath_rel); + free(it->dbpath); + free(it->filepath); + free(it->filepath_rel); free(it->datadir_path); - ut_free(it); + free(it); } @@ -466,17 +462,17 @@ static void datafile_close(datafile_cur_t *cursor) { - if (cursor->file != 0) { + if (cursor->file != OS_FILE_CLOSED) { os_file_close(cursor->file); } - ut_free(cursor->buf); + free(cursor->buf); } static bool datafile_open(const char *file, datafile_cur_t *cursor, uint thread_n) { - ulint success; + bool success; memset(cursor, 0, sizeof(datafile_cur_t)); @@ -490,11 +486,9 @@ datafile_open(const char *file, datafile_cur_t *cursor, uint thread_n) xb_get_relative_path(cursor->abs_path, FALSE), sizeof(cursor->rel_path)); - cursor->file = os_file_create_simple_no_error_handling(0, - cursor->abs_path, - OS_FILE_OPEN, - OS_FILE_READ_ONLY, - &success, 0); + cursor->file = os_file_create_simple_no_error_handling( + 0, cursor->abs_path, + OS_FILE_OPEN, OS_FILE_READ_ALLOW_DELETE, true, &success); if (!success) { /* The following call prints an error message */ os_file_get_last_error(TRUE); @@ -518,7 +512,7 @@ datafile_open(const char *file, datafile_cur_t *cursor, uint thread_n) posix_fadvise(cursor->file, 0, 0, POSIX_FADV_SEQUENTIAL); cursor->buf_size = 10 * 1024 * 1024; - cursor->buf = static_cast(ut_malloc((ulint)cursor->buf_size)); + cursor->buf = static_cast(malloc((ulint)cursor->buf_size)); return(true); } @@ -528,7 +522,6 @@ static xb_fil_cur_result_t datafile_read(datafile_cur_t *cursor) { - ulint success; ulint to_read; xtrabackup_io_throttling(); @@ -540,14 +533,14 @@ datafile_read(datafile_cur_t *cursor) return(XB_FIL_CUR_EOF); } - success = os_file_read(cursor->file, cursor->buf, cursor->buf_offset, - to_read); - if (!success) { + if (!os_file_read(IORequestRead, + cursor->file, cursor->buf, cursor->buf_offset, + to_read)) { return(XB_FIL_CUR_ERROR); } posix_fadvise(cursor->file, cursor->buf_offset, to_read, - POSIX_FADV_DONTNEED); + POSIX_FADV_DONTNEED); cursor->buf_read = to_read; cursor->buf_offset += to_read; @@ -917,13 +910,13 @@ run_data_threads(datadir_iter_t *it, os_thread_func_t func, uint n) { datadir_thread_ctxt_t *data_threads; uint i, count; - os_ib_mutex_t count_mutex; + pthread_mutex_t count_mutex; bool ret; data_threads = (datadir_thread_ctxt_t*) - (ut_malloc(sizeof(datadir_thread_ctxt_t) * n)); + malloc(sizeof(datadir_thread_ctxt_t) * n); - count_mutex = os_mutex_create(); + pthread_mutex_init(&count_mutex, NULL); count = n; for (i = 0; i < n; i++) { @@ -937,15 +930,15 @@ run_data_threads(datadir_iter_t *it, os_thread_func_t func, uint n) /* Wait for threads to exit */ while (1) { os_thread_sleep(100000); - os_mutex_enter(count_mutex); + pthread_mutex_lock(&count_mutex); if (count == 0) { - os_mutex_exit(count_mutex); + pthread_mutex_unlock(&count_mutex); break; } - os_mutex_exit(count_mutex); + pthread_mutex_unlock(&count_mutex); } - os_mutex_free(count_mutex); + pthread_mutex_destroy(&count_mutex); ret = true; for (i = 0; i < n; i++) { @@ -955,7 +948,7 @@ run_data_threads(datadir_iter_t *it, os_thread_func_t func, uint n) } } - ut_free(data_threads); + free(data_threads); return(ret); } @@ -974,7 +967,6 @@ copy_file(ds_ctxt_t *datasink, ds_file_t *dstfile = NULL; datafile_cur_t cursor; xb_fil_cur_result_t res; - const char *action; if (!datafile_open(src_file_path, &cursor, thread_n)) { goto error_close; @@ -991,9 +983,8 @@ copy_file(ds_ctxt_t *datasink, goto error; } - action = xb_get_copy_action(); msg_ts("[%02u] %s %s to %s\n", - thread_n, action, src_file_path, dstfile->path); + thread_n, xb_get_copy_action(), src_file_path, dstfile->path); /* The main copy loop */ while ((res = datafile_read(&cursor)) == XB_FIL_CUR_SUCCESS) { @@ -1111,7 +1102,7 @@ read_link_file(const char *ibd_filepath, const char *link_filepath) while (lastch > 4 && filepath[lastch] <= 0x20) { filepath[lastch--] = 0x00; } - srv_normalize_path_for_win(filepath); + os_normalize_path(filepath); } tablespace_locations[ibd_filepath] = filepath; @@ -1629,14 +1620,9 @@ apply_log_finish() return(true); } -extern void -os_io_init_simple(void); - bool copy_back() { - char *innobase_data_file_path_copy; - ulint i; bool ret; datadir_iter_t *it = NULL; datadir_node_t node; @@ -1679,24 +1665,16 @@ copy_back() if (!innobase_data_file_path) { innobase_data_file_path = (char*) "ibdata1:10M:autoextend"; } - innobase_data_file_path_copy = strdup(innobase_data_file_path); - if (!(ret = srv_parse_data_file_paths_and_sizes( - innobase_data_file_path_copy))) { + srv_sys_space.set_path("."); + + if (!srv_sys_space.parse_params(innobase_data_file_path, true)) { msg("syntax error in innodb_data_file_path\n"); return(false); } srv_max_n_threads = 1000; - //os_sync_mutex = NULL; - ut_mem_init(); - /* temporally dummy value to avoid crash */ - srv_page_size_shift = 14; - srv_page_size = (1 << srv_page_size_shift); - os_sync_init(); - sync_init(); - os_io_init_simple(); - mem_init(srv_mem_pool_size); + sync_check_init(); ut_crc32_init(); /* copy undo tablespaces */ @@ -1707,9 +1685,9 @@ copy_back() ds_data = ds_create(dst_dir, DS_TYPE_LOCAL); - for (i = 1; i <= srv_undo_tablespaces; i++) { + for (ulong i = 1; i <= srv_undo_tablespaces; i++) { char filename[20]; - sprintf(filename, "undo%03u", (uint)i); + sprintf(filename, "undo%03lu", i); if (!(ret = copy_or_move_file(filename, filename, dst_dir, 1))) { goto cleanup; @@ -1720,29 +1698,30 @@ copy_back() ds_data = NULL; } - /* copy redo logs */ - dst_dir = (srv_log_group_home_dir && *srv_log_group_home_dir) - ? srv_log_group_home_dir : mysql_data_home; + ? srv_log_group_home_dir : mysql_data_home; + + /* --backup generates a single ib_logfile0, which we must copy + if it exists. */ ds_data = ds_create(dst_dir, DS_TYPE_LOCAL); + if (!file_exists("ib_logfile0")) { + /* After completed --prepare, redo log files are redundant. + We must delete any redo logs at the destination, so that + the database will not jump to a different log sequence number + (LSN). */ - for (i = 0; i < (ulong)innobase_log_files_in_group; i++) { - char filename[20]; - sprintf(filename, "ib_logfile%lu", i); - - if (!file_exists(filename)) { - continue; - } - - if (!(ret = copy_or_move_file(filename, filename, - dst_dir, 1))) { - goto cleanup; + for (uint i = 0; i <= SRV_N_LOG_FILES_MAX + 1; i++) { + char filename[FN_REFLEN]; + snprintf(filename, sizeof filename, "%s/ib_logfile%u", + dst_dir, i); + unlink(filename); } + } else if (!(ret = copy_or_move_file("ib_logfile0", "ib_logfile0", + dst_dir, 1))) { + goto cleanup; } - ds_destroy(ds_data); - ds_data = NULL; /* copy innodb system tablespace(s) */ @@ -1751,17 +1730,19 @@ copy_back() ds_data = ds_create(dst_dir, DS_TYPE_LOCAL); - for (i = 0; i < srv_n_data_files; i++) { - const char *filename = base_name(srv_data_file_names[i]); + for (Tablespace::const_iterator iter(srv_sys_space.begin()), + end(srv_sys_space.end()); + iter != end; + ++iter) { + const char *filename = base_name(iter->name()); - if (!(ret = copy_or_move_file(filename, srv_data_file_names[i], + if (!(ret = copy_or_move_file(filename, iter->name(), dst_dir, 1))) { goto cleanup; } } ds_destroy(ds_data); - ds_data = NULL; /* copy the rest of tablespaces */ ds_data = ds_create(mysql_data_home, DS_TYPE_LOCAL); @@ -1771,7 +1752,7 @@ copy_back() datadir_node_init(&node); while (datadir_iter_next(it, &node)) { - const char *ext_list[] = {"backup-my.cnf", "xtrabackup_logfile", + const char *ext_list[] = {"backup-my.cnf", "xtrabackup_binary", "xtrabackup_binlog_info", "xtrabackup_checkpoints", ".qp", ".pmap", ".tmp", NULL}; @@ -1817,21 +1798,18 @@ copy_back() continue; } - /* skip redo logs */ - if (sscanf(filename, "ib_logfile%d%c", &i_tmp, &c_tmp) == 1) { + /* skip the redo log (it was already copied) */ + if (!strcmp(filename, "ib_logfile0")) { continue; } /* skip innodb data files */ is_ibdata_file = false; - for (i = 0; i < srv_n_data_files; i++) { - const char *ibfile; - - ibfile = base_name(srv_data_file_names[i]); - - if (strcmp(ibfile, filename) == 0) { + for (Tablespace::const_iterator iter(srv_sys_space.begin()), + end(srv_sys_space.end()); iter != end; ++iter) { + if (strcmp(iter->name(), filename) == 0) { is_ibdata_file = true; - continue; + break; } } if (is_ibdata_file) { @@ -1873,20 +1851,13 @@ cleanup: datadir_node_free(&node); - free(innobase_data_file_path_copy); - if (ds_data != NULL) { ds_destroy(ds_data); } ds_data = NULL; - //os_sync_free(); - mem_close(); - //os_sync_mutex = NULL; - ut_free_all_mem(); - sync_close(); - sync_initialized = FALSE; + sync_check_close(); return(ret); } @@ -1922,13 +1893,6 @@ decrypt_decompress_file(const char *filepath, uint thread_n) if (system(cmd.str().c_str()) != 0) { return(false); } - - if (opt_remove_original) { - msg_ts("[%02u] removing %s\n", thread_n, filepath); - if (my_delete(filepath, MYF(MY_WME)) != 0) { - return(false); - } - } } return(true); @@ -1965,9 +1929,9 @@ cleanup: datadir_node_free(&node); - os_mutex_enter(ctxt->count_mutex); + pthread_mutex_lock(&ctxt->count_mutex); --(*ctxt->count); - os_mutex_exit(ctxt->count_mutex); + pthread_mutex_unlock(&ctxt->count_mutex); ctxt->ret = ret; @@ -1982,10 +1946,7 @@ decrypt_decompress() datadir_iter_t *it = NULL; srv_max_n_threads = 1000; - //os_sync_mutex = NULL; - ut_mem_init(); - os_sync_init(); - sync_init(); + sync_check_init(); /* cd to backup directory */ if (my_setwd(xtrabackup_target_dir, MYF(MY_WME))) @@ -2014,11 +1975,7 @@ decrypt_decompress() ds_data = NULL; - sync_close(); - sync_initialized = FALSE; - //os_sync_free(); - //os_sync_mutex = NULL; - ut_free_all_mem(); + sync_check_close(); return(ret); } diff --git a/extra/mariabackup/backup_mysql.cc b/extra/mariabackup/backup_mysql.cc index 2353c7692cb..5cf63ea94c1 100644 --- a/extra/mariabackup/backup_mysql.cc +++ b/extra/mariabackup/backup_mysql.cc @@ -54,6 +54,7 @@ Place, Suite 330, Boston, MA 02111-1307 USA #include "mysqld.h" #include "encryption_plugin.h" #include +#include char *tool_name; @@ -344,7 +345,8 @@ get_mysql_vars(MYSQL *connection) char *innodb_data_home_dir_var = NULL; char *innodb_undo_directory_var = NULL; char *innodb_page_size_var = NULL; - + char *innodb_undo_tablespaces_var = NULL; + char *endptr; unsigned long server_version = mysql_get_server_version(connection); bool ret = true; @@ -372,6 +374,7 @@ get_mysql_vars(MYSQL *connection) {"innodb_data_home_dir", &innodb_data_home_dir_var}, {"innodb_undo_directory", &innodb_undo_directory_var}, {"innodb_page_size", &innodb_page_size_var}, + {"innodb_undo_tablespaces", &innodb_undo_tablespaces_var}, {NULL, NULL} }; @@ -460,62 +463,55 @@ get_mysql_vars(MYSQL *connection) } /* get some default values is they are missing from my.cnf */ - if (!check_if_param_set("datadir") && datadir_var && *datadir_var) { + if (datadir_var && *datadir_var) { strmake(mysql_real_data_home, datadir_var, FN_REFLEN - 1); mysql_data_home= mysql_real_data_home; } - if (!check_if_param_set("innodb_data_file_path") - && innodb_data_file_path_var && *innodb_data_file_path_var) { + if (innodb_data_file_path_var && *innodb_data_file_path_var) { innobase_data_file_path = my_strdup( innodb_data_file_path_var, MYF(MY_FAE)); } - if (!check_if_param_set("innodb_data_home_dir") - && innodb_data_home_dir_var && *innodb_data_home_dir_var) { + if (innodb_data_home_dir_var && *innodb_data_home_dir_var) { innobase_data_home_dir = my_strdup( innodb_data_home_dir_var, MYF(MY_FAE)); } - if (!check_if_param_set("innodb_log_group_home_dir") - && innodb_log_group_home_dir_var + if (innodb_log_group_home_dir_var && *innodb_log_group_home_dir_var) { srv_log_group_home_dir = my_strdup( innodb_log_group_home_dir_var, MYF(MY_FAE)); } - if (!check_if_param_set("innodb_undo_directory") - && innodb_undo_directory_var && *innodb_undo_directory_var) { + if (innodb_undo_directory_var && *innodb_undo_directory_var) { srv_undo_dir = my_strdup( innodb_undo_directory_var, MYF(MY_FAE)); } - if (!check_if_param_set("innodb_log_files_in_group") - && innodb_log_files_in_group_var) { - char *endptr; - - innobase_log_files_in_group = strtol( + if (innodb_log_files_in_group_var) { + srv_n_log_files = strtol( innodb_log_files_in_group_var, &endptr, 10); ut_ad(*endptr == 0); } - if (!check_if_param_set("innodb_log_file_size") - && innodb_log_file_size_var) { - char *endptr; - - innobase_log_file_size = strtoll( + if (innodb_log_file_size_var) { + srv_log_file_size = strtoll( innodb_log_file_size_var, &endptr, 10); ut_ad(*endptr == 0); } - if (!check_if_param_set("innodb_page_size") && innodb_page_size_var) { - char *endptr; - + if (innodb_page_size_var) { innobase_page_size = strtoll( innodb_page_size_var, &endptr, 10); ut_ad(*endptr == 0); } + if (innodb_undo_tablespaces_var) { + srv_undo_tablespaces = strtoul(innodb_undo_tablespaces_var, &endptr, 10); + ut_ad(*endptr == 0); + } + out: free_mysql_variables(mysql_vars); @@ -850,9 +846,9 @@ static void start_query_killer() { - kill_query_thread_stop = os_event_create(); - kill_query_thread_started = os_event_create(); - kill_query_thread_stopped = os_event_create(); + kill_query_thread_stop = os_event_create(0); + kill_query_thread_started = os_event_create(0); + kill_query_thread_stopped = os_event_create(0); os_thread_create(kill_query_thread, NULL, &kill_query_thread_id); @@ -1368,17 +1364,27 @@ cleanup: return(result); } -static string escape_and_quote(MYSQL *mysql,const char *str) +struct escape_and_quote { - if (!str) - return "NULL"; - size_t len = strlen(str); - char* escaped = (char *)alloca(2 * len + 3); - escaped[0] = '\''; - size_t new_len = mysql_real_escape_string(mysql, escaped+1, str, len); - escaped[new_len + 1] = '\''; - escaped[new_len + 2] = 0; - return string(escaped); + escape_and_quote(MYSQL *mysql, const char *str) + : mysql(mysql), str(str) {} + MYSQL * const mysql; + const char * const str; +}; + +static +std::ostream& +operator<<(std::ostream& s, const escape_and_quote& eq) +{ + if (!eq.str) + return s << "NULL"; + s << '\''; + size_t len = strlen(eq.str); + char* escaped = (char *)alloca(2 * len + 1); + len = mysql_real_escape_string(eq.mysql, escaped, eq.str, len); + s << std::string(escaped, len); + s << '\''; + return s; } /*********************************************************************//** @@ -1395,12 +1401,9 @@ write_xtrabackup_info(MYSQL *connection) char buf_start_time[100]; char buf_end_time[100]; tm tm; - ostringstream oss; + std::ostringstream oss; const char *xb_stream_name[] = {"file", "tar", "xbstream"}; - - ut_ad(xtrabackup_stream_fmt < 3); - uuid = read_mysql_one_value(connection, "SELECT UUID()"); server_version = read_mysql_one_value(connection, "SELECT VERSION()"); localtime_r(&history_start_time, &tm); @@ -1528,28 +1531,21 @@ bool write_backup_config_file() "# The MySQL server\n" "[mysqld]\n" "innodb_checksum_algorithm=%s\n" - "innodb_log_checksum_algorithm=%s\n" "innodb_data_file_path=%s\n" "innodb_log_files_in_group=%lu\n" - "innodb_log_file_size=%lld\n" + "innodb_log_file_size=%llu\n" "innodb_page_size=%lu\n" - "innodb_log_block_size=%lu\n" "innodb_undo_directory=%s\n" "innodb_undo_tablespaces=%lu\n" "%s%s\n" - "%s%s\n" "%s\n", innodb_checksum_algorithm_names[srv_checksum_algorithm], - innodb_checksum_algorithm_names[srv_log_checksum_algorithm], innobase_data_file_path, srv_n_log_files, - innobase_log_file_size, + srv_log_file_size, srv_page_size, - srv_log_block_size, srv_undo_dir, srv_undo_tablespaces, - innobase_doublewrite_file ? "innodb_doublewrite_file=" : "", - innobase_doublewrite_file ? innobase_doublewrite_file : "", innobase_buffer_pool_filename ? "innodb_buffer_pool_filename=" : "", innobase_buffer_pool_filename ? diff --git a/extra/mariabackup/wsrep.h b/extra/mariabackup/backup_wsrep.h similarity index 96% rename from extra/mariabackup/wsrep.h rename to extra/mariabackup/backup_wsrep.h index 7638d1f2b54..6537b304e12 100644 --- a/extra/mariabackup/wsrep.h +++ b/extra/mariabackup/backup_wsrep.h @@ -19,8 +19,8 @@ along with this program; if not, write to the Free Software Foundation, Inc., 51 Franklin Street, Fifth Floor, Boston, MA 02110-1301, USA *******************************************************/ -#ifndef WSREP_H -#define WSREP_H +#ifndef BACKUP_WSREP_H +#define BACKUP_WSREP_H /*********************************************************************** Store Galera checkpoint info in the 'xtrabackup_galera_info' file, if that diff --git a/extra/mariabackup/changed_page_bitmap.cc b/extra/mariabackup/changed_page_bitmap.cc index 86a873ef69c..ce769375a16 100644 --- a/extra/mariabackup/changed_page_bitmap.cc +++ b/extra/mariabackup/changed_page_bitmap.cc @@ -193,9 +193,9 @@ log_online_read_bitmap_page( ut_a(bitmap_file->offset <= bitmap_file->size - MODIFIED_PAGE_BLOCK_SIZE); ut_a(bitmap_file->offset % MODIFIED_PAGE_BLOCK_SIZE == 0); - - success = os_file_read(bitmap_file->file, page, bitmap_file->offset, - MODIFIED_PAGE_BLOCK_SIZE); + success = os_file_read(IORequestRead, + bitmap_file->file, page, bitmap_file->offset, + MODIFIED_PAGE_BLOCK_SIZE); if (UNIV_UNLIKELY(!success)) { @@ -355,8 +355,7 @@ log_online_setup_bitmap_file_range( bitmap_files->files = static_cast - (ut_malloc(bitmap_files->count - * sizeof(bitmap_files->files[0]))); + (malloc(bitmap_files->count * sizeof(bitmap_files->files[0]))); memset(bitmap_files->files, 0, bitmap_files->count * sizeof(bitmap_files->files[0])); @@ -426,9 +425,9 @@ log_online_setup_bitmap_file_range( /****************************************************************//** Open a bitmap file for reading. -@return TRUE if opened successfully */ +@return whether opened successfully */ static -ibool +bool log_online_open_bitmap_file_read_only( /*==================================*/ const char* name, /*!name, FN_REFLEN, "%s%s", srv_data_home, name); - bitmap_file->file - = os_file_create_simple_no_error_handling(0, bitmap_file->name, - OS_FILE_OPEN, - OS_FILE_READ_ONLY, - &success,0); + bitmap_file->file = os_file_create_simple_no_error_handling( + 0, bitmap_file->name, + OS_FILE_OPEN, OS_FILE_READ_ONLY, true, &success); if (UNIV_UNLIKELY(!success)) { /* Here and below assume that bitmap file names do not contain apostrophes, thus no need for ut_print_filename(). */ msg("InnoDB: Warning: error opening the changed page " "bitmap \'%s\'\n", bitmap_file->name); - return FALSE; + return success; } bitmap_file->size = os_file_get_size(bitmap_file->file); @@ -465,7 +462,7 @@ log_online_open_bitmap_file_read_only( posix_fadvise(bitmap_file->file, 0, 0, POSIX_FADV_NOREUSE); #endif - return TRUE; + return success; } /****************************************************************//** @@ -909,6 +906,37 @@ xb_page_bitmap_setup_next_page( return TRUE; } +/** Find the node with the smallest key that greater than equal to search key. +@param[in] tree red-black tree +@param[in] key search key +@return node with the smallest greater-than-or-equal key +@retval NULL if none was found */ +static +const ib_rbt_node_t* +rbt_lower_bound(const ib_rbt_t* tree, const void* key) +{ + ut_ad(!tree->cmp_arg); + const ib_rbt_node_t* ge = NULL; + + for (const ib_rbt_node_t *node = tree->root->left; + node != tree->nil; ) { + int result = tree->compare(node->value, key); + + if (result < 0) { + node = node->right; + } else { + ge = node; + if (result == 0) { + break; + } + + node = node->left; + } + } + + return(ge); +} + /****************************************************************//** Set up a new bitmap range iterator over a given space id changed pages in a given bitmap. @@ -922,8 +950,7 @@ xb_page_bitmap_range_init( { byte search_page[MODIFIED_PAGE_BLOCK_SIZE]; xb_page_bitmap_range *result - = static_cast - (ut_malloc(sizeof(*result))); + = static_cast(malloc(sizeof(*result))); memset(result, 0, sizeof(*result)); result->bitmap = bitmap; @@ -1014,5 +1041,5 @@ xb_page_bitmap_range_deinit( /*========================*/ xb_page_bitmap_range* bitmap_range) /*! in/out: bitmap range */ { - ut_free(bitmap_range); + free(bitmap_range); } diff --git a/extra/mariabackup/encryption_plugin.cc b/extra/mariabackup/encryption_plugin.cc index 8f7741b057a..76512c185e2 100644 --- a/extra/mariabackup/encryption_plugin.cc +++ b/extra/mariabackup/encryption_plugin.cc @@ -8,6 +8,7 @@ #include #include #include +#include extern struct st_maria_plugin *mysql_optional_plugins[]; @@ -18,14 +19,14 @@ extern char *xb_plugin_load; extern char *xb_plugin_dir; const int PLUGIN_MAX_ARGS = 1024; -vector backup_plugins_args; +std::vector backup_plugins_args; const char *QUERY_PLUGIN = "SELECT plugin_name, plugin_library, @@plugin_dir" " FROM information_schema.plugins WHERE plugin_type='ENCRYPTION'" " AND plugin_status='ACTIVE'"; -string encryption_plugin_config; +std::string encryption_plugin_config; static void add_to_plugin_load_list(const char *plugin_def) { @@ -38,7 +39,7 @@ void encryption_plugin_backup_init(MYSQL *mysql) { MYSQL_RES *result; MYSQL_ROW row; - ostringstream oss; + std::ostringstream oss; char *argv[PLUGIN_MAX_ARGS]; int argc; @@ -59,17 +60,17 @@ void encryption_plugin_backup_init(MYSQL *mysql) if (*p == '\\') *p = '/'; #endif - string plugin_load(name); + std::string plugin_load(name); if (library) - plugin_load += string("=") + library; + plugin_load += std::string("=") + library; - oss << "plugin_load=" << plugin_load << endl; + oss << "plugin_load=" << plugin_load << std::endl; /* Required to load the plugin later.*/ add_to_plugin_load_list(plugin_load.c_str()); strncpy(opt_plugin_dir, dir, FN_REFLEN); - oss << "plugin_dir=" << '"' << dir << '"' << endl; + oss << "plugin_dir=" << '"' << dir << '"' << std::endl; /* Read plugin variables. */ @@ -80,12 +81,12 @@ void encryption_plugin_backup_init(MYSQL *mysql) result = xb_mysql_query(mysql, query, true, true); while ((row = mysql_fetch_row(result))) { - string arg("--"); + std::string arg("--"); arg += row[0]; arg += "="; arg += row[1]; backup_plugins_args.push_back(arg); - oss << row[0] << "=" << row[1] << endl; + oss << row[0] << "=" << row[1] << std::endl; } mysql_free_result(result); @@ -94,7 +95,7 @@ void encryption_plugin_backup_init(MYSQL *mysql) result = xb_mysql_query(mysql, "select @@innodb_encrypt_log", true, true); row = mysql_fetch_row(result); srv_encrypt_log = (row != 0 && row[0][0] == '1'); - oss << "innodb_encrypt_log=" << row[0] << endl; + oss << "innodb_encrypt_log=" << row[0] << std::endl; mysql_free_result(result); diff --git a/extra/mariabackup/fil_cur.cc b/extra/mariabackup/fil_cur.cc index 820d8e10c29..b733f98457e 100644 --- a/extra/mariabackup/fil_cur.cc +++ b/extra/mariabackup/fil_cur.cc @@ -24,8 +24,8 @@ Foundation, Inc., 51 Franklin Street, Fifth Floor, Boston, MA 02110-1301, USA #include -#include #include +#include #include #include @@ -63,7 +63,7 @@ xb_get_relative_path( prev = NULL; cur = path; - while ((next = strchr(cur, SRV_PATH_SEPARATOR)) != NULL) { + while ((next = strchr(cur, OS_PATH_SEPARATOR)) != NULL) { prev = cur; cur = next + 1; @@ -96,7 +96,7 @@ xb_fil_node_close_file( ut_a(node->n_pending_flushes == 0); ut_a(!node->being_extended); - if (!node->open) { + if (!node->is_open()) { mutex_exit(&fil_system->mutex); @@ -106,19 +106,19 @@ xb_fil_node_close_file( ret = os_file_close(node->handle); ut_a(ret); - node->open = FALSE; + node->handle = OS_FILE_CLOSED; ut_a(fil_system->n_open > 0); fil_system->n_open--; fil_n_file_opened--; - if (node->space->purpose == FIL_TABLESPACE && + if (node->space->purpose == FIL_TYPE_TABLESPACE && fil_is_user_tablespace_id(node->space->id)) { ut_a(UT_LIST_GET_LEN(fil_system->LRU) > 0); /* The node is in the LRU list, remove it */ - UT_LIST_REMOVE(LRU, fil_system->LRU, node); + UT_LIST_REMOVE(fil_system->LRU, node); } mutex_exit(&fil_system->mutex); @@ -137,10 +137,7 @@ xb_fil_cur_open( fil_node_t* node, /*!< in: source tablespace node */ uint thread_n) /*!< thread number for diagnostics */ { - ulint page_size; - ulint page_size_shift; - ulint zip_size; - ibool success; + bool success; /* Initialize these first so xb_fil_cur_close() handles them correctly in case of error */ @@ -148,27 +145,26 @@ xb_fil_cur_open( cursor->node = NULL; cursor->space_id = node->space->id; - cursor->is_system = !fil_is_user_tablespace_id(node->space->id); strncpy(cursor->abs_path, node->name, sizeof(cursor->abs_path)); /* Get the relative path for the destination tablespace name, i.e. the one that can be appended to the backup root directory. Non-system - tablespaces may have absolute paths for remote tablespaces in MySQL - 5.6+. We want to make "local" copies for the backup. */ + tablespaces may have absolute paths for DATA DIRECTORY. + We want to make "local" copies for the backup. */ strncpy(cursor->rel_path, - xb_get_relative_path(cursor->abs_path, cursor->is_system), + xb_get_relative_path(cursor->abs_path, cursor->is_system()), sizeof(cursor->rel_path)); /* In the backup mode we should already have a tablespace handle created - by fil_load_single_table_tablespace() unless it is a system + by fil_ibd_load() unless it is a system tablespace. Otherwise we open the file here. */ - if (cursor->is_system || !srv_backup_mode || srv_close_files) { - node->handle = - os_file_create_simple_no_error_handling(0, node->name, - OS_FILE_OPEN, - OS_FILE_READ_ONLY, - &success,0); + if (cursor->is_system() || srv_operation == SRV_OPERATION_RESTORE + || xb_close_files) { + node->handle = os_file_create_simple_no_error_handling( + 0, node->name, + OS_FILE_OPEN, + OS_FILE_READ_ALLOW_DELETE, true, &success); if (!success) { /* The following call prints an error message */ os_file_get_last_error(TRUE); @@ -181,22 +177,20 @@ xb_fil_cur_open( } mutex_enter(&fil_system->mutex); - node->open = TRUE; - fil_system->n_open++; fil_n_file_opened++; - if (node->space->purpose == FIL_TABLESPACE && + if (node->space->purpose == FIL_TYPE_TABLESPACE && fil_is_user_tablespace_id(node->space->id)) { /* Put the node to the LRU list */ - UT_LIST_ADD_FIRST(LRU, fil_system->LRU, node); + UT_LIST_ADD_FIRST(fil_system->LRU, node); } mutex_exit(&fil_system->mutex); } - ut_ad(node->open); + ut_ad(node->is_open()); cursor->node = node; cursor->file = node->handle; @@ -210,8 +204,8 @@ xb_fil_cur_open( return(XB_FIL_CUR_ERROR); } - if (srv_unix_file_flush_method == SRV_UNIX_O_DIRECT - || srv_unix_file_flush_method == SRV_UNIX_O_DIRECT_NO_FSYNC) { + if (srv_file_flush_method == SRV_O_DIRECT + || srv_file_flush_method == SRV_O_DIRECT_NO_FSYNC) { os_file_set_nocache(cursor->file, node->name, "OPEN"); } @@ -219,32 +213,29 @@ xb_fil_cur_open( posix_fadvise(cursor->file, 0, 0, POSIX_FADV_SEQUENTIAL); /* Determine the page size */ - zip_size = xb_get_zip_size(cursor->file); - if (zip_size == ULINT_UNDEFINED) { + ulint flags = xb_get_space_flags(cursor->file); + if (flags == ULINT_UNDEFINED) { xb_fil_cur_close(cursor); return(XB_FIL_CUR_SKIP); - } else if (zip_size) { - page_size = zip_size; - page_size_shift = get_bit_shift(page_size); - msg("[%02u] %s is compressed with page size = " - "%lu bytes\n", thread_n, node->name, page_size); - if (page_size_shift < 10 || page_size_shift > 14) { - msg("[%02u] xtrabackup: Error: Invalid " - "page size: %lu.\n", thread_n, page_size); - ut_error; - } - } else { - page_size = UNIV_PAGE_SIZE; - page_size_shift = UNIV_PAGE_SIZE_SHIFT; } + + if (!fsp_flags_is_valid(flags, cursor->space_id)) { + ulint cflags = fsp_flags_convert_from_101(flags); + if (cflags == ULINT_UNDEFINED) { + msg("[%02u] xtrabackup: Error: Invalid " + "tablespace flags: %x.\n", thread_n, uint(flags)); + return(XB_FIL_CUR_SKIP); + } + flags = cflags; + } + + const page_size_t page_size(flags); cursor->page_size = page_size; - cursor->page_size_shift = page_size_shift; - cursor->zip_size = zip_size; /* Allocate read buffer */ - cursor->buf_size = XB_FIL_CUR_PAGES * page_size; + cursor->buf_size = XB_FIL_CUR_PAGES * page_size.physical(); cursor->orig_buf = static_cast - (ut_malloc(cursor->buf_size + UNIV_PAGE_SIZE)); + (malloc(cursor->buf_size + UNIV_PAGE_SIZE)); cursor->buf = static_cast (ut_align(cursor->orig_buf, UNIV_PAGE_SIZE)); @@ -254,7 +245,8 @@ xb_fil_cur_open( cursor->buf_page_no = 0; cursor->thread_n = thread_n; - cursor->space_size = (ulint)(cursor->statinfo.st_size / page_size); + cursor->space_size = (ulint)(cursor->statinfo.st_size + / page_size.physical()); cursor->read_filter = read_filter; cursor->read_filter->init(&cursor->read_filter_ctxt, cursor, @@ -282,6 +274,8 @@ xb_fil_cur_read( xb_fil_cur_result_t ret; ib_int64_t offset; ib_int64_t to_read; + const ulint page_size = cursor->page_size.physical(); + xb_ad(!cursor->is_system() || page_size == UNIV_PAGE_SIZE); cursor->read_filter->get_next_batch(&cursor->read_filter_ctxt, &offset, &to_read); @@ -296,10 +290,10 @@ xb_fil_cur_read( xb_a(to_read > 0 && to_read <= 0xFFFFFFFFLL); - if (to_read % cursor->page_size != 0 && - offset + to_read == cursor->statinfo.st_size) { + if ((to_read & ~(page_size - 1)) + && offset + to_read == cursor->statinfo.st_size) { - if (to_read < (ib_int64_t) cursor->page_size) { + if (to_read < (ib_int64_t) page_size) { msg("[%02u] xtrabackup: Warning: junk at the end of " "%s:\n", cursor->thread_n, cursor->abs_path); msg("[%02u] xtrabackup: Warning: offset = %llu, " @@ -312,12 +306,12 @@ xb_fil_cur_read( } to_read = (ib_int64_t) (((ulint) to_read) & - ~(cursor->page_size - 1)); + ~(page_size - 1)); } - xb_a(to_read % cursor->page_size == 0); + xb_a((to_read & (page_size - 1)) == 0); - npages = (ulint) (to_read >> cursor->page_size_shift); + npages = (ulint) (to_read / cursor->page_size.physical()); retry_count = 10; ret = XB_FIL_CUR_SUCCESS; @@ -328,59 +322,59 @@ read_retry: cursor->buf_read = 0; cursor->buf_npages = 0; cursor->buf_offset = offset; - cursor->buf_page_no = (ulint)(offset >> cursor->page_size_shift); + cursor->buf_page_no = (ulint)(offset / cursor->page_size.physical()); - success = os_file_read(cursor->file, cursor->buf, offset, - (ulint)to_read); + FilSpace space(cursor->space_id); + + if (!space()) { + return(XB_FIL_CUR_ERROR); + } + + success = os_file_read(IORequestRead, + cursor->file, cursor->buf, offset, + (ulint) to_read); if (!success) { return(XB_FIL_CUR_ERROR); } - fil_system_enter(); - fil_space_t *space = fil_space_get_by_id(cursor->space_id); - fil_system_exit(); - /* check pages for corruption and re-read if necessary. i.e. in case of partially written pages */ for (page = cursor->buf, i = 0; i < npages; - page += cursor->page_size, i++) { - ib_int64_t page_no = cursor->buf_page_no + i; + page += page_size, i++) { + ulint page_no = cursor->buf_page_no + i; - bool checksum_ok = fil_space_verify_crypt_checksum(page, cursor->zip_size,space, (ulint)page_no); - - if (!checksum_ok && - buf_page_is_corrupted(true, page, cursor->zip_size,space)) { - - if (cursor->is_system && - page_no >= (ib_int64_t)FSP_EXTENT_SIZE && - page_no < (ib_int64_t) FSP_EXTENT_SIZE * 3) { - /* skip doublewrite buffer pages */ - xb_a(cursor->page_size == UNIV_PAGE_SIZE); + if (cursor->space_id == TRX_SYS_SPACE && + page_no >= FSP_EXTENT_SIZE && + page_no < FSP_EXTENT_SIZE * 3) { + /* We ignore the doublewrite buffer pages */ + } else if (!fil_space_verify_crypt_checksum( + page, cursor->page_size, space->id, page_no) + && buf_page_is_corrupted(true, page, + cursor->page_size, + space)) { + retry_count--; + if (retry_count == 0) { msg("[%02u] xtrabackup: " - "Page %lu is a doublewrite buffer page, " - "skipping.\n", cursor->thread_n, page_no); - } else { - retry_count--; - if (retry_count == 0) { - msg("[%02u] xtrabackup: " - "Error: failed to read page after " - "10 retries. File %s seems to be " - "corrupted.\n", cursor->thread_n, - cursor->abs_path); - ret = XB_FIL_CUR_ERROR; - break; - } + "Error: failed to read page after " + "10 retries. File %s seems to be " + "corrupted.\n", cursor->thread_n, + cursor->abs_path); + ret = XB_FIL_CUR_ERROR; + break; + } + + if (retry_count == 9) { msg("[%02u] xtrabackup: " "Database page corruption detected at page " - "%lu, retrying...\n", cursor->thread_n, - page_no); - - os_thread_sleep(100000); - - goto read_retry; + ULINTPF ", retrying...\n", + cursor->thread_n, page_no); } + + os_thread_sleep(100000); + + goto read_retry; } - cursor->buf_read += cursor->page_size; + cursor->buf_read += page_size; cursor->buf_npages++; } @@ -399,11 +393,10 @@ xb_fil_cur_close( { cursor->read_filter->deinit(&cursor->read_filter_ctxt); - if (cursor->orig_buf != NULL) { - ut_free(cursor->orig_buf); - } + free(cursor->orig_buf); + if (cursor->node != NULL) { xb_fil_node_close_file(cursor->node); - cursor->file = XB_FILE_UNDEFINED; + cursor->file = OS_FILE_CLOSED; } } diff --git a/extra/mariabackup/fil_cur.h b/extra/mariabackup/fil_cur.h index f3601823a5a..e3f356a346c 100644 --- a/extra/mariabackup/fil_cur.h +++ b/extra/mariabackup/fil_cur.h @@ -27,6 +27,7 @@ Foundation, Inc., 51 Franklin Street, Fifth Floor, Boston, MA 02110-1301, USA #include #include "read_filt.h" +#include "srv0start.h" struct xb_fil_cur_t { pfs_os_file_t file; /*!< source file handle */ @@ -36,14 +37,7 @@ struct xb_fil_cur_t { char abs_path[FN_REFLEN]; /*!< absolute file path */ MY_STAT statinfo; /*!< information about the file */ - ulint zip_size; /*!< compressed page size in bytes or 0 - for uncompressed pages */ - ulint page_size; /*!< = zip_size for compressed pages or - UNIV_PAGE_SIZE for uncompressed ones */ - ulint page_size_shift;/*!< bit shift corresponding to - page_size */ - my_bool is_system; /*!< TRUE for system tablespace, FALSE - otherwise */ + page_size_t page_size; /*!< page size */ xb_read_filt_t* read_filter; /*!< read filter */ xb_read_filt_ctxt_t read_filter_ctxt; /*!< read filter context */ @@ -61,6 +55,17 @@ struct xb_fil_cur_t { uint thread_n; /*!< thread number for diagnostics */ ulint space_id; /*!< ID of tablespace */ ulint space_size; /*!< space size in pages */ + + /** TODO: remove this default constructor */ + xb_fil_cur_t() : page_size(0), read_filter_ctxt() {} + + /** @return whether this is not a file-per-table tablespace */ + bool is_system() const + { + ut_ad(space_id != SRV_TMP_SPACE_ID); + return(space_id == TRX_SYS_SPACE + || srv_is_undo_tablespace(space_id)); + } }; typedef enum { diff --git a/extra/mariabackup/innobackupex.cc b/extra/mariabackup/innobackupex.cc index 7e97b258489..c15e02cdff3 100644 --- a/extra/mariabackup/innobackupex.cc +++ b/extra/mariabackup/innobackupex.cc @@ -45,7 +45,6 @@ Place, Suite 330, Boston, MA 02111-1307 USA #include #include #include -#include #include #include #include @@ -70,7 +69,6 @@ using std::max; my_bool opt_ibx_version = FALSE; my_bool opt_ibx_help = FALSE; my_bool opt_ibx_apply_log = FALSE; -my_bool opt_ibx_redo_only = FALSE; my_bool opt_ibx_incremental = FALSE; my_bool opt_ibx_notimestamp = FALSE; @@ -95,8 +93,6 @@ char *opt_ibx_host = NULL; char *opt_ibx_defaults_group = NULL; char *opt_ibx_socket = NULL; uint opt_ibx_port = 0; -char *opt_ibx_login_path = NULL; - ulong opt_ibx_lock_wait_query_type; ulong opt_ibx_kill_long_query_type; @@ -226,21 +222,11 @@ static struct my_option ibx_long_options[] = GET_BOOL, NO_ARG, 0, 0, 0, 0, 0, 0}, {"apply-log", OPT_APPLY_LOG, "Prepare a backup in BACKUP-DIR by " - "applying the transaction log file named \"xtrabackup_logfile\" " - "located in the same directory. Also, create new transaction logs. " + "applying the redo log 'ib_logfile0' and creating new redo log. " "The InnoDB configuration is read from the file \"backup-my.cnf\".", (uchar*) &opt_ibx_apply_log, (uchar*) &opt_ibx_apply_log, 0, GET_BOOL, NO_ARG, 0, 0, 0, 0, 0, 0}, - {"redo-only", OPT_REDO_ONLY, "This option should be used when " - "preparing the base full backup and when merging all incrementals " - "except the last one. This forces xtrabackup to skip the \"rollback\" " - "phase and do a \"redo\" only. This is necessary if the backup will " - "have incremental changes applied to it later. See the xtrabackup " - "documentation for details.", - (uchar *) &opt_ibx_redo_only, (uchar *) &opt_ibx_redo_only, 0, - GET_BOOL, NO_ARG, 0, 0, 0, 0, 0, 0}, - {"copy-back", OPT_COPY_BACK, "Copy all the files in a previously made " "backup from the backup directory to their original locations.", (uchar *) &opt_ibx_copy_back, (uchar *) &opt_ibx_copy_back, 0, @@ -682,7 +668,7 @@ innobackupex [--compress] [--compress-threads=NUMBER-OF-THREADS] [--compress-chu \n\ innobackupex --apply-log [--use-memory=B]\n\ [--defaults-file=MY.CNF]\n\ - [--export] [--redo-only] [--ibbackup=IBBACKUP-BINARY]\n\ + [--export] [--ibbackup=IBBACKUP-BINARY]\n\ BACKUP-DIR\n\ \n\ innobackupex --copy-back [--defaults-file=MY.CNF] [--defaults-group=GROUP-NAME] BACKUP-DIR\n\ @@ -710,7 +696,7 @@ process.\n\ \n\ The --apply-log command prepares a backup for starting a MySQL\n\ server on the backup. This command recovers InnoDB data files as specified\n\ -in BACKUP-DIR/backup-my.cnf using BACKUP-DIR/xtrabackup_logfile,\n\ +in BACKUP-DIR/backup-my.cnf using BACKUP-DIR/ib_logfile0,\n\ and creates new InnoDB log files as specified in BACKUP-DIR/backup-my.cnf.\n\ The BACKUP-DIR should be the path to a backup directory created by\n\ xtrabackup. This command runs xtrabackup as a child process, but it does not \n\ @@ -909,7 +895,6 @@ ibx_init() opt_defaults_group = opt_ibx_defaults_group; opt_socket = opt_ibx_socket; opt_port = opt_ibx_port; - opt_login_path = opt_ibx_login_path; opt_lock_wait_query_type = opt_ibx_lock_wait_query_type; opt_kill_long_query_type = opt_ibx_kill_long_query_type; @@ -980,9 +965,6 @@ ibx_init() switch (ibx_mode) { case IBX_MODE_APPLY_LOG: xtrabackup_prepare = TRUE; - if (opt_ibx_redo_only) { - xtrabackup_apply_log_only = TRUE; - } xtrabackup_target_dir = ibx_position_arg; run = "apply-log"; break; diff --git a/extra/mariabackup/read_filt.cc b/extra/mariabackup/read_filt.cc index 05e6b7c86c7..a48591abf29 100644 --- a/extra/mariabackup/read_filt.cc +++ b/extra/mariabackup/read_filt.cc @@ -127,10 +127,11 @@ rf_bitmap_get_next_batch( of pages */ { ulint start_page_id; + const ulint page_size = ctxt->page_size.physical(); - start_page_id = (ulint)(ctxt->offset / ctxt->page_size); + start_page_id = (ulint)(ctxt->offset / page_size); - xb_a (ctxt->offset % ctxt->page_size == 0); + xb_a (ctxt->offset % page_size == 0); if (start_page_id == ctxt->filter_batch_end) { @@ -146,7 +147,7 @@ rf_bitmap_get_next_batch( return; } - ctxt->offset = next_page_id * ctxt->page_size; + ctxt->offset = next_page_id * page_size; /* Find the end of the current changed page block by searching for the next cleared bitmap bit */ @@ -162,7 +163,7 @@ rf_bitmap_get_next_batch( remaining pages. */ *read_batch_len = ctxt->data_file_size - ctxt->offset; } else { - *read_batch_len = ctxt->filter_batch_end * ctxt->page_size + *read_batch_len = ctxt->filter_batch_end * page_size - ctxt->offset; } @@ -175,9 +176,9 @@ rf_bitmap_get_next_batch( } ctxt->offset += *read_batch_len; - xb_a (ctxt->offset % ctxt->page_size == 0); - xb_a (*read_batch_start % ctxt->page_size == 0); - xb_a (*read_batch_len % ctxt->page_size == 0); + xb_a (ctxt->offset % page_size == 0); + xb_a (*read_batch_start % page_size == 0); + xb_a (*read_batch_len % page_size == 0); } /****************************************************************//** diff --git a/extra/mariabackup/read_filt.h b/extra/mariabackup/read_filt.h index d16f4e1093d..cebc714eed8 100644 --- a/extra/mariabackup/read_filt.h +++ b/extra/mariabackup/read_filt.h @@ -27,6 +27,8 @@ Foundation, Inc., 51 Franklin Street, Fifth Floor, Boston, MA 02110-1301, USA #include "changed_page_bitmap.h" +typedef ulint space_id_t; + struct xb_fil_cur_t; /* The read filter context */ @@ -34,15 +36,17 @@ struct xb_read_filt_ctxt_t { ib_int64_t offset; /*!< current file offset */ ib_int64_t data_file_size; /*!< data file size */ size_t buffer_capacity;/*!< read buffer capacity */ - ib_int64_t space_id; /*!< space id */ + space_id_t space_id; /*!< space id */ /* The following fields used only in bitmap filter */ /* Move these to union if any other filters are added in future */ xb_page_bitmap_range *bitmap_range; /*!< changed page bitmap range iterator for space_id */ - size_t page_size; /*!< page size */ + page_size_t page_size; /*!< page size */ ulint filter_batch_end;/*!< the ending page id of the current changed page block in the bitmap */ + /** TODO: remove this default constructor */ + xb_read_filt_ctxt_t() : page_size(0) {} }; /* The read filter */ diff --git a/extra/mariabackup/write_filt.cc b/extra/mariabackup/write_filt.cc index cf7753bf380..a0633818405 100644 --- a/extra/mariabackup/write_filt.cc +++ b/extra/mariabackup/write_filt.cc @@ -67,7 +67,6 @@ wf_incremental_init(xb_write_filt_ctxt_t *ctxt, char *dst_name, xb_fil_cur_t *cursor) { char meta_name[FN_REFLEN]; - xb_delta_info_t info; ulint buf_size; xb_wf_incremental_ctxt_t *cp = &(ctxt->u.wf_incremental_ctxt); @@ -75,8 +74,9 @@ wf_incremental_init(xb_write_filt_ctxt_t *ctxt, char *dst_name, ctxt->cursor = cursor; /* allocate buffer for incremental backup (4096 pages) */ - buf_size = (cursor->page_size / 4 + 1) * cursor->page_size; - cp->delta_buf_base = static_cast(ut_malloc(buf_size)); + buf_size = (cursor->page_size.physical() / 4 + 1) + * cursor->page_size.physical(); + cp->delta_buf_base = static_cast(malloc(buf_size)); memset(cp->delta_buf_base, 0, buf_size); cp->delta_buf = static_cast (ut_align(cp->delta_buf_base, UNIV_PAGE_SIZE_MAX)); @@ -84,9 +84,7 @@ wf_incremental_init(xb_write_filt_ctxt_t *ctxt, char *dst_name, /* write delta meta info */ snprintf(meta_name, sizeof(meta_name), "%s%s", dst_name, XB_DELTA_INFO_SUFFIX); - info.page_size = cursor->page_size; - info.zip_size = cursor->zip_size; - info.space_id = cursor->space_id; + const xb_delta_info_t info(cursor->page_size, cursor->space_id); if (!xb_write_delta_metadata(meta_name, &info)) { msg("[%02u] xtrabackup: Error: " "failed to write meta info for %s\n", @@ -113,8 +111,9 @@ wf_incremental_process(xb_write_filt_ctxt_t *ctxt, ds_file_t *dstfile) { ulint i; xb_fil_cur_t *cursor = ctxt->cursor; - ulint page_size = cursor->page_size; byte *page; + const ulint page_size + = cursor->page_size.physical(); xb_wf_incremental_ctxt_t *cp = &(ctxt->u.wf_incremental_ctxt); for (i = 0, page = cursor->buf; i < cursor->buf_npages; @@ -159,7 +158,8 @@ static my_bool wf_incremental_finalize(xb_write_filt_ctxt_t *ctxt, ds_file_t *dstfile) { xb_fil_cur_t *cursor = ctxt->cursor; - ulint page_size = cursor->page_size; + const ulint page_size + = cursor->page_size.physical(); xb_wf_incremental_ctxt_t *cp = &(ctxt->u.wf_incremental_ctxt); if (cp->npages != page_size / 4) { @@ -184,9 +184,7 @@ wf_incremental_deinit(xb_write_filt_ctxt_t *ctxt) { xb_wf_incremental_ctxt_t *cp = &(ctxt->u.wf_incremental_ctxt); - if (cp->delta_buf_base != NULL) { - ut_free(cp->delta_buf_base); - } + free(cp->delta_buf_base); } /************************************************************************ diff --git a/extra/mariabackup/xb0xb.h b/extra/mariabackup/xb0xb.h index a8b17f59579..59938a014c6 100644 --- a/extra/mariabackup/xb0xb.h +++ b/extra/mariabackup/xb0xb.h @@ -21,43 +21,7 @@ Foundation, Inc., 51 Franklin Street, Fifth Floor, Boston, MA 02110-1301, USA #ifndef xb0xb_h #define xb0xb_h - -extern void os_io_init_simple(void); -extern pfs_os_file_t files[1000]; extern const char *innodb_checksum_algorithm_names[]; extern TYPELIB innodb_checksum_algorithm_typelib; -extern dberr_t open_or_create_data_files( - bool* create_new_db, -#ifdef UNIV_LOG_ARCHIVE - lsn_t* min_arch_log_no, - lsn_t* max_arch_log_no, -#endif - lsn_t* flushed_lsn, - ulint* sum_of_new_sizes) - ; -int -fil_file_readdir_next_file( -/*=======================*/ -dberr_t* err, /*!< out: this is set to DB_ERROR if an error - was encountered, otherwise not changed */ - const char* dirname,/*!< in: directory name or path */ - os_file_dir_t dir, /*!< in: directory stream */ - os_file_stat_t* info) /*!< in/out: buffer where the - info is returned */; -fil_space_t* -fil_space_get_by_name(const char *); -ibool -recv_check_cp_is_consistent(const byte* buf); -void -innodb_log_checksum_func_update( -/*============================*/ -ulint algorithm) /*!< in: algorithm */; -dberr_t -srv_undo_tablespaces_init( -/*======================*/ -ibool create_new_db, -ibool backup_mode, -const ulint n_conf_tablespaces, -ulint* n_opened); #endif diff --git a/extra/mariabackup/xbstream.c b/extra/mariabackup/xbstream.c index edfe20a9e3c..f3880f9ec03 100644 --- a/extra/mariabackup/xbstream.c +++ b/extra/mariabackup/xbstream.c @@ -444,7 +444,11 @@ extract_worker_thread_func(void *arg) } if (chunk.type == XB_CHUNK_TYPE_EOF) { + pthread_mutex_lock(ctxt->mutex); pthread_mutex_unlock(&entry->mutex); + my_hash_delete(ctxt->filehash, (uchar *) entry); + pthread_mutex_unlock(ctxt->mutex); + continue; } diff --git a/extra/mariabackup/xtrabackup.cc b/extra/mariabackup/xtrabackup.cc index 1592172e3f1..83c40a14bbd 100644 --- a/extra/mariabackup/xtrabackup.cc +++ b/extra/mariabackup/xtrabackup.cc @@ -4,6 +4,8 @@ XtraBackup: hot backup tool for InnoDB Originally Created 3/3/2009 Yasufumi Kinoshita Written by Alexey Kopytov, Aleksandr Kuzminsky, Stewart Smith, Vadim Tkachenko, Yasufumi Kinoshita, Ignacio Nin and Baron Schwartz. +(c) 2017, MariaDB Corporation. +Portions written by Marko Mäkelä. This program is free software; you can redistribute it and/or modify it under the terms of the GNU General Public License as published by @@ -41,6 +43,8 @@ Place, Suite 330, Boston, MA 02111-1307 USA //#define XTRABACKUP_TARGET_IS_PLUGIN +#include +#include #include #include #include @@ -61,6 +65,7 @@ Place, Suite 330, Boston, MA 02111-1307 USA #include #include #include +#include #include #include #include @@ -85,7 +90,7 @@ Place, Suite 330, Boston, MA 02111-1307 USA #include "xbstream.h" #include "changed_page_bitmap.h" #include "read_filt.h" -#include "wsrep.h" +#include "backup_wsrep.h" #include "innobackupex.h" #include "backup_mysql.h" #include "backup_copy.h" @@ -95,48 +100,39 @@ Place, Suite 330, Boston, MA 02111-1307 USA #include #include #include - -/* TODO: replace with appropriate macros used in InnoDB 5.6 */ -#define PAGE_ZIP_MIN_SIZE_SHIFT 10 -#define DICT_TF_ZSSIZE_SHIFT 1 -#define DICT_TF_FORMAT_ZIP 1 -#define DICT_TF_FORMAT_SHIFT 5 +#include int sys_var_init(); -my_bool innodb_inited= 0; - /* === xtrabackup specific options === */ char xtrabackup_real_target_dir[FN_REFLEN] = "./xtrabackup_backupfiles/"; char *xtrabackup_target_dir= xtrabackup_real_target_dir; -my_bool xtrabackup_version = FALSE; -my_bool xtrabackup_backup = FALSE; -my_bool xtrabackup_prepare = FALSE; -my_bool xtrabackup_copy_back = FALSE; -my_bool xtrabackup_move_back = FALSE; -my_bool xtrabackup_decrypt_decompress = FALSE; -my_bool xtrabackup_print_param = FALSE; +static my_bool xtrabackup_version; +my_bool xtrabackup_backup; +my_bool xtrabackup_prepare; +my_bool xtrabackup_copy_back; +my_bool xtrabackup_move_back; +my_bool xtrabackup_decrypt_decompress; +my_bool xtrabackup_print_param; -my_bool xtrabackup_export = FALSE; -my_bool xtrabackup_apply_log_only = FALSE; +my_bool xtrabackup_export; -longlong xtrabackup_use_memory = 100*1024*1024L; -my_bool xtrabackup_create_ib_logfile = FALSE; +longlong xtrabackup_use_memory; -long xtrabackup_throttle = 0; /* 0:unlimited */ -lint io_ticket; -os_event_t wait_throttle = NULL; -os_event_t log_copying_stop = NULL; +long xtrabackup_throttle; /* 0:unlimited */ +static lint io_ticket; +static os_event_t wait_throttle; +static os_event_t log_copying_stop; -char *xtrabackup_incremental = NULL; +char *xtrabackup_incremental; lsn_t incremental_lsn; lsn_t incremental_to_lsn; lsn_t incremental_last_lsn; -xb_page_bitmap *changed_page_bitmap = NULL; +xb_page_bitmap *changed_page_bitmap; -char *xtrabackup_incremental_basedir = NULL; /* for --backup */ -char *xtrabackup_extra_lsndir = NULL; /* for --backup with --extra-lsndir */ -char *xtrabackup_incremental_dir = NULL; /* for --prepare */ +char *xtrabackup_incremental_basedir; /* for --backup */ +char *xtrabackup_extra_lsndir; /* for --backup with --extra-lsndir */ +char *xtrabackup_incremental_dir; /* for --prepare */ char xtrabackup_real_incremental_basedir[FN_REFLEN]; char xtrabackup_real_extra_lsndir[FN_REFLEN]; @@ -144,9 +140,9 @@ char xtrabackup_real_incremental_dir[FN_REFLEN]; char *xtrabackup_tmpdir; -char *xtrabackup_tables = NULL; -char *xtrabackup_tables_file = NULL; -char *xtrabackup_tables_exclude = NULL; +char *xtrabackup_tables; +char *xtrabackup_tables_file; +char *xtrabackup_tables_exclude; typedef std::list regex_list_t; static regex_list_t regex_include_list; @@ -170,17 +166,12 @@ struct xb_filter_entry_struct{ }; typedef struct xb_filter_entry_struct xb_filter_entry_t; -static ulint thread_nr[SRV_MAX_N_IO_THREADS + 6]; -static os_thread_id_t thread_ids[SRV_MAX_N_IO_THREADS + 6]; - lsn_t checkpoint_lsn_start; lsn_t checkpoint_no_start; -lsn_t log_copy_scanned_lsn; -ibool log_copying = TRUE; -ibool log_copying_running = FALSE; -ibool io_watching_thread_running = FALSE; - -ibool xtrabackup_logfile_is_renamed = FALSE; +static lsn_t log_copy_scanned_lsn; +static bool log_copying; +static bool log_copying_running; +static bool io_watching_thread_running; int xtrabackup_parallel; @@ -199,19 +190,16 @@ ulint xtrabackup_log_copy_interval = 1000; static ulong max_buf_pool_modified_pct; /* Ignored option (--log) for MySQL option compatibility */ -char* log_ignored_opt = NULL; +static char* log_ignored_opt; /* === metadata of backup === */ #define XTRABACKUP_METADATA_FILENAME "xtrabackup_checkpoints" -char metadata_type[30] = ""; /*[full-backuped|log-applied| - full-prepared|incremental]*/ -lsn_t metadata_from_lsn = 0; -lsn_t metadata_to_lsn = 0; -lsn_t metadata_last_lsn = 0; +char metadata_type[30] = ""; /*[full-backuped|log-applied|incremental]*/ +lsn_t metadata_from_lsn; +lsn_t metadata_to_lsn; +lsn_t metadata_last_lsn; -#define XB_LOG_FILENAME "xtrabackup_logfile" - -ds_file_t *dst_log_file = NULL; +static ds_file_t* dst_log_file; static char mysql_data_home_buff[2]; @@ -227,49 +215,34 @@ ulong innobase_large_page_size = 0; /* The default values for the following, type long or longlong, start-up parameters are declared in mysqld.cc: */ -long innobase_additional_mem_pool_size = 1*1024*1024L; long innobase_buffer_pool_awe_mem_mb = 0; long innobase_file_io_threads = 4; long innobase_read_io_threads = 4; long innobase_write_io_threads = 4; -long innobase_force_recovery = 0; long innobase_log_buffer_size = 1024*1024L; -long innobase_log_files_in_group = 2; long innobase_open_files = 300L; longlong innobase_page_size = (1LL << 14); /* 16KB */ -static ulong innobase_log_block_size = 512; -my_bool innobase_fast_checksum = FALSE; -char* innobase_doublewrite_file = NULL; char* innobase_buffer_pool_filename = NULL; longlong innobase_buffer_pool_size = 8*1024*1024L; -longlong innobase_log_file_size = 48*1024*1024L; /* The default values for the following char* start-up parameters are determined in innobase_init below: */ -char* innobase_ignored_opt = NULL; -char* innobase_data_home_dir = NULL; -char* innobase_data_file_path = NULL; +static char* innobase_ignored_opt; +char* innobase_data_home_dir; +char* innobase_data_file_path; /* The following has a misleading name: starting from 4.0.5, this also affects Windows: */ -char* innobase_unix_file_flush_method = NULL; +char* innobase_unix_file_flush_method; -/* Below we have boolean-valued start-up parameters, and their default -values */ - -ulong innobase_fast_shutdown = 1; -my_bool innobase_use_doublewrite = TRUE; -my_bool innobase_use_checksums = TRUE; -my_bool innobase_use_large_pages = FALSE; -my_bool innobase_file_per_table = FALSE; -my_bool innobase_locks_unsafe_for_binlog = FALSE; -my_bool innobase_rollback_on_timeout = FALSE; -my_bool innobase_create_status_file = FALSE; -my_bool innobase_adaptive_hash_index = TRUE; - -static char *internal_innobase_data_file_path = NULL; +my_bool innobase_use_doublewrite; +my_bool innobase_use_large_pages; +my_bool innobase_file_per_table; +my_bool innobase_locks_unsafe_for_binlog; +my_bool innobase_rollback_on_timeout; +my_bool innobase_create_status_file; /* The following counter is used to convey information to InnoDB about server activity: in selects it is not sensible to call @@ -290,7 +263,7 @@ lsn_t flushed_lsn= 0; ulong xb_open_files_limit= 0; char *xb_plugin_dir; char *xb_plugin_load; -my_bool xb_close_files= FALSE; +my_bool xb_close_files; /* Datasinks */ ds_ctxt_t *ds_data = NULL; @@ -299,10 +272,6 @@ ds_ctxt_t *ds_redo = NULL; static bool innobackupex_mode = false; -static long innobase_log_files_in_group_save; -static char *srv_log_group_home_dir_save; -static longlong innobase_log_file_size_save; - /* String buffer used by --print-param to accumulate server options as they are parsed from the defaults file */ static std::ostringstream print_param_str; @@ -313,6 +282,7 @@ std::set param_set; static ulonglong global_max_value; extern "C" sig_handler handle_fatal_signal(int sig); +extern LOGGER logger; my_bool opt_galera_info = FALSE; my_bool opt_slave_info = FALSE; @@ -323,7 +293,6 @@ my_bool opt_force_non_empty_dirs = FALSE; my_bool opt_noversioncheck = FALSE; my_bool opt_no_backup_locks = FALSE; my_bool opt_decompress = FALSE; -my_bool opt_remove_original = FALSE; static const char *binlog_info_values[] = {"off", "lockless", "on", "auto", NullS}; @@ -331,17 +300,16 @@ static TYPELIB binlog_info_typelib = {array_elements(binlog_info_values)-1, "", binlog_info_values, NULL}; ulong opt_binlog_info; -char *opt_incremental_history_name = NULL; -char *opt_incremental_history_uuid = NULL; +char *opt_incremental_history_name; +char *opt_incremental_history_uuid; -char *opt_user = NULL; -char *opt_password = NULL; -char *opt_host = NULL; -char *opt_defaults_group = NULL; -char *opt_socket = NULL; -uint opt_port = 0; -char *opt_login_path = NULL; -char *opt_log_bin = NULL; +char *opt_user; +char *opt_password; +char *opt_host; +char *opt_defaults_group; +char *opt_socket; +uint opt_port; +char *opt_log_bin; const char *query_type_names[] = { "ALL", "UPDATE", "SELECT", NullS}; @@ -361,9 +329,6 @@ const char *opt_history = NULL; #if defined(HAVE_OPENSSL) my_bool opt_ssl_verify_server_cert = FALSE; -#if !defined(HAVE_YASSL) -char *opt_server_public_key = NULL; -#endif #endif /* Whether xtrabackup_binlog_info should be created on recovery */ @@ -383,14 +348,23 @@ xtrabackup_add_datasink(ds_ctxt_t *ds) } /* ======== Datafiles iterator ======== */ +struct datafiles_iter_t { + fil_system_t *system; + fil_space_t *space; + fil_node_t *node; + ibool started; + pthread_mutex_t mutex; +}; + +/* ======== Datafiles iterator ======== */ +static datafiles_iter_t * datafiles_iter_new(fil_system_t *f_system) { datafiles_iter_t *it; - it = static_cast - (ut_malloc(sizeof(datafiles_iter_t))); - it->mutex = os_mutex_create(); + it = static_cast(malloc(sizeof(datafiles_iter_t))); + pthread_mutex_init(&it->mutex, NULL); it->system = f_system; it->space = NULL; @@ -400,12 +374,13 @@ datafiles_iter_new(fil_system_t *f_system) return it; } +static fil_node_t * datafiles_iter_next(datafiles_iter_t *it) { fil_node_t *new_node; - os_mutex_enter(it->mutex); + pthread_mutex_lock(&it->mutex); if (it->node == NULL) { if (it->started) @@ -422,7 +397,7 @@ datafiles_iter_next(datafiles_iter_t *it) UT_LIST_GET_NEXT(space_list, it->space); while (it->space != NULL && - (it->space->purpose != FIL_TABLESPACE || + (it->space->purpose != FIL_TYPE_TABLESPACE || UT_LIST_GET_LEN(it->space->chain) == 0)) it->space = UT_LIST_GET_NEXT(space_list, it->space); if (it->space == NULL) @@ -432,16 +407,17 @@ datafiles_iter_next(datafiles_iter_t *it) end: new_node = it->node; - os_mutex_exit(it->mutex); + pthread_mutex_unlock(&it->mutex); return new_node; } +static void datafiles_iter_free(datafiles_iter_t *it) { - os_mutex_free(it->mutex); - ut_free(it); + pthread_mutex_destroy(&it->mutex); + free(it); } /* ======== Date copying thread context ======== */ @@ -450,7 +426,7 @@ typedef struct { datafiles_iter_t *it; uint num; uint *count; - os_ib_mutex_t count_mutex; + pthread_mutex_t count_mutex; os_thread_id_t id; } data_thread_ctxt_t; @@ -463,7 +439,6 @@ enum options_xtrabackup OPT_XTRA_BACKUP, OPT_XTRA_PREPARE, OPT_XTRA_EXPORT, - OPT_XTRA_APPLY_LOG_ONLY, OPT_XTRA_PRINT_PARAM, OPT_XTRA_USE_MEMORY, OPT_XTRA_THROTTLE, @@ -476,7 +451,6 @@ enum options_xtrabackup OPT_XTRA_TABLES_FILE, OPT_XTRA_DATABASES, OPT_XTRA_DATABASES_FILE, - OPT_XTRA_CREATE_IB_LOGFILE, OPT_XTRA_PARALLEL, OPT_XTRA_STREAM, OPT_XTRA_COMPRESS, @@ -484,12 +458,10 @@ enum options_xtrabackup OPT_XTRA_COMPRESS_CHUNK_SIZE, OPT_LOG, OPT_INNODB, - OPT_INNODB_CHECKSUMS, OPT_INNODB_DATA_FILE_PATH, OPT_INNODB_DATA_HOME_DIR, OPT_INNODB_ADAPTIVE_HASH_INDEX, OPT_INNODB_DOUBLEWRITE, - OPT_INNODB_FAST_SHUTDOWN, OPT_INNODB_FILE_PER_TABLE, OPT_INNODB_FLUSH_LOG_AT_TRX_COMMIT, OPT_INNODB_FLUSH_METHOD, @@ -499,7 +471,6 @@ enum options_xtrabackup OPT_INNODB_MAX_PURGE_LAG, OPT_INNODB_ROLLBACK_ON_TIMEOUT, OPT_INNODB_STATUS_FILE, - OPT_INNODB_ADDITIONAL_MEM_POOL_SIZE, OPT_INNODB_AUTOEXTEND_INCREMENT, OPT_INNODB_BUFFER_POOL_SIZE, OPT_INNODB_COMMIT_CONCURRENCY, @@ -510,25 +481,17 @@ enum options_xtrabackup OPT_INNODB_WRITE_IO_THREADS, OPT_INNODB_USE_NATIVE_AIO, OPT_INNODB_PAGE_SIZE, - OPT_INNODB_LOG_BLOCK_SIZE, - OPT_INNODB_FAST_CHECKSUM, - OPT_INNODB_EXTRA_UNDOSLOTS, - OPT_INNODB_DOUBLEWRITE_FILE, OPT_INNODB_BUFFER_POOL_FILENAME, - OPT_INNODB_FORCE_RECOVERY, OPT_INNODB_LOCK_WAIT_TIMEOUT, OPT_INNODB_LOG_BUFFER_SIZE, OPT_INNODB_LOG_FILE_SIZE, OPT_INNODB_LOG_FILES_IN_GROUP, OPT_INNODB_OPEN_FILES, - OPT_INNODB_SYNC_SPIN_LOOPS, - OPT_INNODB_THREAD_CONCURRENCY, - OPT_INNODB_THREAD_SLEEP_DELAY, OPT_XTRA_DEBUG_SYNC, OPT_INNODB_CHECKSUM_ALGORITHM, OPT_INNODB_UNDO_DIRECTORY, OPT_INNODB_UNDO_TABLESPACES, - OPT_INNODB_LOG_CHECKSUM_ALGORITHM, + OPT_INNODB_LOG_CHECKSUMS, OPT_XTRA_INCREMENTAL_FORCE_SCAN, OPT_DEFAULTS_GROUP, OPT_OPEN_FILES_LIMIT, @@ -551,7 +514,6 @@ enum options_xtrabackup OPT_DECOMPRESS, OPT_INCREMENTAL_HISTORY_NAME, OPT_INCREMENTAL_HISTORY_UUID, - OPT_REMOVE_ORIGINAL, OPT_LOCK_WAIT_QUERY_TYPE, OPT_KILL_LONG_QUERY_TYPE, OPT_HISTORY, @@ -563,10 +525,6 @@ enum options_xtrabackup OPT_BINLOG_INFO, OPT_XB_SECURE_AUTH, - OPT_SSL_SSL, - OPT_SSL_VERIFY_SERVER_CERT, - OPT_SERVER_PUBLIC_KEY, - OPT_XTRA_TABLES_EXCLUDE, OPT_XTRA_DATABASES_EXCLUDE, }; @@ -587,10 +545,6 @@ struct my_option xb_client_options[] = {"export", OPT_XTRA_EXPORT, "create files to import to another database when prepare.", (G_PTR*) &xtrabackup_export, (G_PTR*) &xtrabackup_export, 0, GET_BOOL, NO_ARG, 0, 0, 0, 0, 0, 0}, - {"apply-log-only", OPT_XTRA_APPLY_LOG_ONLY, - "stop recovery process not to progress LSN after applying log when prepare.", - (G_PTR*) &xtrabackup_apply_log_only, (G_PTR*) &xtrabackup_apply_log_only, - 0, GET_BOOL, NO_ARG, 0, 0, 0, 0, 0, 0}, {"print-param", OPT_XTRA_PRINT_PARAM, "print parameter of mysqld needed for copyback.", (G_PTR*) &xtrabackup_print_param, (G_PTR*) &xtrabackup_print_param, 0, GET_BOOL, NO_ARG, 0, 0, 0, 0, 0, 0}, @@ -628,7 +582,7 @@ struct my_option xb_client_options[] = {"databases", OPT_XTRA_DATABASES, "filtering by list of databases.", (G_PTR*) &xtrabackup_databases, (G_PTR*) &xtrabackup_databases, 0, GET_STR, REQUIRED_ARG, 0, 0, 0, 0, 0, 0}, - {"databases_file", OPT_XTRA_TABLES_FILE, + {"databases_file", OPT_XTRA_DATABASES_FILE, "filtering by list of databases in the file.", (G_PTR*) &xtrabackup_databases_file, (G_PTR*) &xtrabackup_databases_file, 0, GET_STR, REQUIRED_ARG, 0, 0, 0, 0, 0, 0}, @@ -642,9 +596,6 @@ struct my_option xb_client_options[] = "Note that this option has a higher priority than --databases.", (G_PTR*) &xtrabackup_databases_exclude, (G_PTR*) &xtrabackup_databases_exclude, 0, GET_STR, REQUIRED_ARG, 0, 0, 0, 0, 0, 0}, - {"create-ib-logfile", OPT_XTRA_CREATE_IB_LOGFILE, "** not work for now** creates ib_logfile* also after '--prepare'. ### If you want create ib_logfile*, only re-execute this command in same options. ###", - (G_PTR*) &xtrabackup_create_ib_logfile, (G_PTR*) &xtrabackup_create_ib_logfile, - 0, GET_BOOL, NO_ARG, 0, 0, 0, 0, 0, 0}, {"stream", OPT_XTRA_STREAM, "Stream all backup files to the standard output " "in the specified format." @@ -922,6 +873,11 @@ struct my_option xb_client_options[] = uint xb_client_options_count = array_elements(xb_client_options); +#ifndef DBUG_OFF +/** Parameters to DBUG */ +static const char *dbug_option; +#endif + struct my_option xb_server_options[] = { {"datadir", 'h', "Path to the database root.", (G_PTR*) &mysql_data_home, @@ -947,45 +903,39 @@ struct my_option xb_server_options[] = GET_STR, OPT_ARG, 0, 0, 0, 0, 0, 0}, {"log_bin", OPT_LOG, "Base name for the log sequence", - &opt_log_bin, &opt_log_bin, 0, GET_STR_ALLOC, OPT_ARG, 0, 0, 0, 0, 0, 0}, + &opt_log_bin, &opt_log_bin, 0, GET_STR, OPT_ARG, 0, 0, 0, 0, 0, 0}, {"innodb", OPT_INNODB, "Ignored option for MySQL option compatibility", (G_PTR*) &innobase_ignored_opt, (G_PTR*) &innobase_ignored_opt, 0, GET_STR, OPT_ARG, 0, 0, 0, 0, 0, 0}, - +#ifdef BTR_CUR_HASH_ADAPT {"innodb_adaptive_hash_index", OPT_INNODB_ADAPTIVE_HASH_INDEX, "Enable InnoDB adaptive hash index (enabled by default). " "Disable with --skip-innodb-adaptive-hash-index.", - (G_PTR*) &innobase_adaptive_hash_index, - (G_PTR*) &innobase_adaptive_hash_index, + &btr_search_enabled, + &btr_search_enabled, 0, GET_BOOL, NO_ARG, 1, 0, 0, 0, 0, 0}, - {"innodb_additional_mem_pool_size", OPT_INNODB_ADDITIONAL_MEM_POOL_SIZE, - "Size of a memory pool InnoDB uses to store data dictionary information and other internal data structures.", - (G_PTR*) &innobase_additional_mem_pool_size, - (G_PTR*) &innobase_additional_mem_pool_size, 0, GET_LONG, REQUIRED_ARG, - 1*1024*1024L, 512*1024L, LONG_MAX, 0, 1024, 0}, +#endif /* BTR_CUR_HASH_ADAPT */ {"innodb_autoextend_increment", OPT_INNODB_AUTOEXTEND_INCREMENT, "Data file autoextend increment in megabytes", - (G_PTR*) &srv_auto_extend_increment, - (G_PTR*) &srv_auto_extend_increment, + (G_PTR*) &sys_tablespace_auto_extend_increment, + (G_PTR*) &sys_tablespace_auto_extend_increment, 0, GET_ULONG, REQUIRED_ARG, 8L, 1L, 1000L, 0, 1L, 0}, {"innodb_buffer_pool_size", OPT_INNODB_BUFFER_POOL_SIZE, "The size of the memory buffer InnoDB uses to cache data and indexes of its tables.", (G_PTR*) &innobase_buffer_pool_size, (G_PTR*) &innobase_buffer_pool_size, 0, GET_LL, REQUIRED_ARG, 8*1024*1024L, 1024*1024L, LONGLONG_MAX, 0, 1024*1024L, 0}, - {"innodb_checksums", OPT_INNODB_CHECKSUMS, "Enable InnoDB checksums validation (enabled by default). \ -Disable with --skip-innodb-checksums.", (G_PTR*) &innobase_use_checksums, - (G_PTR*) &innobase_use_checksums, 0, GET_BOOL, NO_ARG, 1, 0, 0, 0, 0, 0}, {"innodb_data_file_path", OPT_INNODB_DATA_FILE_PATH, "Path to individual files and their sizes.", &innobase_data_file_path, - &innobase_data_file_path, 0, GET_STR_ALLOC, REQUIRED_ARG, 0, 0, 0, 0, 0, 0}, + &innobase_data_file_path, 0, GET_STR, REQUIRED_ARG, 0, 0, 0, 0, 0, 0}, {"innodb_data_home_dir", OPT_INNODB_DATA_HOME_DIR, "The common part for InnoDB table spaces.", &innobase_data_home_dir, - &innobase_data_home_dir, 0, GET_STR_ALLOC, REQUIRED_ARG, 0, 0, 0, 0, 0, 0}, - {"innodb_doublewrite", OPT_INNODB_DOUBLEWRITE, "Enable InnoDB doublewrite buffer (enabled by default). \ -Disable with --skip-innodb-doublewrite.", (G_PTR*) &innobase_use_doublewrite, - (G_PTR*) &innobase_use_doublewrite, 0, GET_BOOL, NO_ARG, 1, 0, 0, 0, 0, 0}, + &innobase_data_home_dir, 0, GET_STR, REQUIRED_ARG, 0, 0, 0, 0, 0, 0}, + {"innodb_doublewrite", OPT_INNODB_DOUBLEWRITE, + "Enable InnoDB doublewrite buffer during --prepare.", + (G_PTR*) &innobase_use_doublewrite, + (G_PTR*) &innobase_use_doublewrite, 0, GET_BOOL, NO_ARG, 0, 0, 0, 0, 0, 0}, {"innodb_io_capacity", OPT_INNODB_IO_CAPACITY, "Number of IOPs the server can do. Tunes the background IO rate", (G_PTR*) &srv_io_capacity, (G_PTR*) &srv_io_capacity, @@ -1013,29 +963,22 @@ Disable with --skip-innodb-doublewrite.", (G_PTR*) &innobase_use_doublewrite, (G_PTR*) &innobase_unix_file_flush_method, 0, GET_STR, REQUIRED_ARG, 0, 0, 0, 0, 0, 0}, -/* ####### Should we use this option? ####### */ - {"innodb_force_recovery", OPT_INNODB_FORCE_RECOVERY, - "Helps to save your data in case the disk image of the database becomes corrupt.", - (G_PTR*) &innobase_force_recovery, (G_PTR*) &innobase_force_recovery, 0, - GET_LONG, REQUIRED_ARG, 0, 0, 6, 0, 1, 0}, - {"innodb_log_buffer_size", OPT_INNODB_LOG_BUFFER_SIZE, "The size of the buffer which InnoDB uses to write log to the log files on disk.", (G_PTR*) &innobase_log_buffer_size, (G_PTR*) &innobase_log_buffer_size, 0, GET_LONG, REQUIRED_ARG, 1024*1024L, 256*1024L, LONG_MAX, 0, 1024, 0}, {"innodb_log_file_size", OPT_INNODB_LOG_FILE_SIZE, - "Size of each log file in a log group.", - (G_PTR*) &innobase_log_file_size, (G_PTR*) &innobase_log_file_size, 0, - GET_LL, REQUIRED_ARG, 48*1024*1024L, 1*1024*1024L, LONGLONG_MAX, 0, - 1024*1024L, 0}, + "Ignored for mysqld option compatibility", + (G_PTR*) &srv_log_file_size, (G_PTR*) &srv_log_file_size, 0, + GET_ULL, REQUIRED_ARG, 48 << 20, 1 << 20, 512ULL << 30, 0, + UNIV_PAGE_SIZE_MAX, 0}, {"innodb_log_files_in_group", OPT_INNODB_LOG_FILES_IN_GROUP, - "Number of log files in the log group. InnoDB writes to the files in a " - "circular fashion. Value 3 is recommended here.", - &innobase_log_files_in_group, &innobase_log_files_in_group, - 0, GET_LONG, REQUIRED_ARG, 2, 2, 100, 0, 1, 0}, + "Ignored for mysqld option compatibility", + &srv_n_log_files, &srv_n_log_files, + 0, GET_LONG, REQUIRED_ARG, 1, 1, 100, 0, 1, 0}, {"innodb_log_group_home_dir", OPT_INNODB_LOG_GROUP_HOME_DIR, "Path to InnoDB log files.", &srv_log_group_home_dir, - &srv_log_group_home_dir, 0, GET_STR_ALLOC, REQUIRED_ARG, 0, 0, 0, 0, 0, 0}, + &srv_log_group_home_dir, 0, GET_STR, REQUIRED_ARG, 0, 0, 0, 0, 0, 0}, {"innodb_max_dirty_pages_pct", OPT_INNODB_MAX_DIRTY_PAGES_PCT, "Percentage of dirty pages allowed in bufferpool.", (G_PTR*) &srv_max_buf_pool_modified_pct, (G_PTR*) &srv_max_buf_pool_modified_pct, 0, GET_ULONG, REQUIRED_ARG, 90, 0, 100, 0, 0, 0}, @@ -1054,25 +997,17 @@ Disable with --skip-innodb-doublewrite.", (G_PTR*) &innobase_use_doublewrite, /* Use GET_LL to support numeric suffixes in 5.6 */ GET_LL, REQUIRED_ARG, (1LL << 14), (1LL << 12), (1LL << UNIV_PAGE_SIZE_SHIFT_MAX), 0, 1L, 0}, - {"innodb_log_block_size", OPT_INNODB_LOG_BLOCK_SIZE, - "The log block size of the transaction log file. " - "Changing for created log file is not supported. Use on your own risk!", - (G_PTR*) &innobase_log_block_size, (G_PTR*) &innobase_log_block_size, 0, - GET_ULONG, REQUIRED_ARG, 512, 512, 1 << UNIV_PAGE_SIZE_SHIFT_MAX, 0, 1L, 0}, - {"innodb_fast_checksum", OPT_INNODB_FAST_CHECKSUM, - "Change the algorithm of checksum for the whole of datapage to 4-bytes word based.", - (G_PTR*) &innobase_fast_checksum, - (G_PTR*) &innobase_fast_checksum, 0, GET_BOOL, NO_ARG, 0, 0, 0, 0, 0, 0}, - {"innodb_doublewrite_file", OPT_INNODB_DOUBLEWRITE_FILE, - "Path to special datafile for doublewrite buffer. (default is "": not used)", - (G_PTR*) &innobase_doublewrite_file, (G_PTR*) &innobase_doublewrite_file, - 0, GET_STR, REQUIRED_ARG, 0, 0, 0, 0, 0, 0}, {"innodb_buffer_pool_filename", OPT_INNODB_BUFFER_POOL_FILENAME, - "Filename to/from which to dump/load the InnoDB buffer pool", + "Ignored for mysqld option compatibility", (G_PTR*) &innobase_buffer_pool_filename, (G_PTR*) &innobase_buffer_pool_filename, 0, GET_STR, REQUIRED_ARG, 0, 0, 0, 0, 0, 0}, +#ifndef DBUG_OFF /* unfortunately "debug" collides with existing options */ + {"dbug", '#', "Built in DBUG debugger.", + &dbug_option, &dbug_option, 0, GET_STR, OPT_ARG, + 0, 0, 0, 0, 0, 0}, +#endif #ifndef __WIN__ {"debug-sync", OPT_XTRA_DEBUG_SYNC, "Debug sync point. This is only used by the xtrabackup test suite", @@ -1086,14 +1021,10 @@ Disable with --skip-innodb-doublewrite.", (G_PTR*) &innobase_use_doublewrite, "INNODB, STRICT_INNODB, NONE, STRICT_NONE]", &srv_checksum_algorithm, &srv_checksum_algorithm, &innodb_checksum_algorithm_typelib, GET_ENUM, REQUIRED_ARG, SRV_CHECKSUM_ALGORITHM_INNODB, 0, 0, 0, 0, 0}, - {"innodb_log_checksum_algorithm", OPT_INNODB_LOG_CHECKSUM_ALGORITHM, - "The algorithm InnoDB uses for log checksumming. [CRC32, STRICT_CRC32, " - "INNODB, STRICT_INNODB, NONE, STRICT_NONE]", &srv_log_checksum_algorithm, - &srv_log_checksum_algorithm, &innodb_checksum_algorithm_typelib, GET_ENUM, - REQUIRED_ARG, SRV_CHECKSUM_ALGORITHM_INNODB, 0, 0, 0, 0, 0}, + {"innodb_undo_directory", OPT_INNODB_UNDO_DIRECTORY, "Directory where undo tablespace files live, this path can be absolute.", - &srv_undo_dir, &srv_undo_dir, 0, GET_STR_ALLOC, REQUIRED_ARG, 0, 0, 0, 0, 0, + &srv_undo_dir, &srv_undo_dir, 0, GET_STR, REQUIRED_ARG, 0, 0, 0, 0, 0, 0}, {"innodb_undo_tablespaces", OPT_INNODB_UNDO_TABLESPACES, @@ -1116,7 +1047,12 @@ Disable with --skip-innodb-doublewrite.", (G_PTR*) &innobase_use_doublewrite, { "innodb-encrypt-log", OPT_INNODB_ENCRYPT_LOG, "encrypton plugin to load", &srv_encrypt_log, &srv_encrypt_log, 0, GET_BOOL, REQUIRED_ARG, 0, 0, 0, 0, 0, 0 }, - + + {"innodb-log-checksums", OPT_INNODB_LOG_CHECKSUMS, + "Whether to require checksums for InnoDB redo log blocks", + &innodb_log_checksums, &innodb_log_checksums, + 0, GET_BOOL, REQUIRED_ARG, 1, 0, 0, 0, 0, 0 }, + {"open_files_limit", OPT_OPEN_FILES_LIMIT, "the maximum number of file " "descriptors to reserve with setrlimit().", (G_PTR*) &xb_open_files_limit, (G_PTR*) &xb_open_files_limit, 0, GET_ULONG, @@ -1270,13 +1206,7 @@ xb_get_one_option(int optid, break; case OPT_INNODB_LOG_FILES_IN_GROUP: - - ADD_PRINT_PARAM_OPT(innobase_log_files_in_group); - break; - case OPT_INNODB_LOG_FILE_SIZE: - - ADD_PRINT_PARAM_OPT(innobase_log_file_size); break; case OPT_INNODB_FLUSH_METHOD: @@ -1289,21 +1219,6 @@ xb_get_one_option(int optid, ADD_PRINT_PARAM_OPT(innobase_page_size); break; - case OPT_INNODB_FAST_CHECKSUM: - - ADD_PRINT_PARAM_OPT(!!innobase_fast_checksum); - break; - - case OPT_INNODB_LOG_BLOCK_SIZE: - - ADD_PRINT_PARAM_OPT(innobase_log_block_size); - break; - - case OPT_INNODB_DOUBLEWRITE_FILE: - - ADD_PRINT_PARAM_OPT(innobase_doublewrite_file); - break; - case OPT_INNODB_UNDO_DIRECTORY: ADD_PRINT_PARAM_OPT(srv_undo_dir); @@ -1321,13 +1236,6 @@ xb_get_one_option(int optid, ADD_PRINT_PARAM_OPT(innodb_checksum_algorithm_names[srv_checksum_algorithm]); break; - case OPT_INNODB_LOG_CHECKSUM_ALGORITHM: - - ut_a(srv_log_checksum_algorithm <= SRV_CHECKSUM_ALGORITHM_STRICT_NONE); - - ADD_PRINT_PARAM_OPT(innodb_checksum_algorithm_names[srv_log_checksum_algorithm]); - break; - case OPT_INNODB_BUFFER_POOL_FILENAME: ADD_PRINT_PARAM_OPT(innobase_buffer_pool_filename); @@ -1400,40 +1308,9 @@ xb_get_one_option(int optid, return 0; } -/*********************************************************************** -Initializes log_block_size */ -static -ibool -xb_init_log_block_size(void) -{ - srv_log_block_size = 0; - if (innobase_log_block_size != 512) { - uint n_shift = (uint)get_bit_shift(innobase_log_block_size);; - - if (n_shift > 0) { - srv_log_block_size = (ulint)(1LL << n_shift); - msg("InnoDB: The log block size is set to %lu.\n", - srv_log_block_size); - } - } else { - srv_log_block_size = 512; - } - if (!srv_log_block_size) { - msg("InnoDB: Error: %lu is not valid value for " - "innodb_log_block_size.\n", innobase_log_block_size); - return FALSE; - } - - return TRUE; -} - static my_bool innodb_init_param(void) { - /* innobase_init */ - static char current_dir[3]; /* Set if using current lib */ - my_bool ret; - char *default_path; srv_is_being_started = TRUE; /* === some variables from mysqld === */ memset((G_PTR) &mysql_tmpdir_list, 0, sizeof(mysql_tmpdir_list)); @@ -1465,10 +1342,6 @@ innodb_init_param(void) srv_page_size = (1 << srv_page_size_shift); } - if (!xb_init_log_block_size()) { - goto error; - } - /* Check that values don't overflow on 32-bit systems. */ if (sizeof(ulint) == 4) { if (xtrabackup_use_memory > UINT_MAX32) { @@ -1482,30 +1355,10 @@ innodb_init_param(void) goto error; } - - if (innobase_log_file_size > UINT_MAX32) { - msg("xtrabackup: innobase_log_file_size can't be " - "over 4GB on 32-bit systemsi\n"); - - goto error; - } } - os_innodb_umask = (ulint)0664; - - /* First calculate the default path for innodb_data_home_dir etc., - in case the user has not given any value. - - Note that when using the embedded server, the datadirectory is not - necessarily the current directory of this program. */ - - /* It's better to use current lib, to keep paths short */ - current_dir[0] = FN_CURLIB; - current_dir[1] = FN_LIBCHAR; - current_dir[2] = 0; - default_path = current_dir; - - ut_a(default_path); + static char default_path[2] = { FN_CURLIB, 0 }; + fil_path_to_mysql_datadir = default_path; /* Set InnoDB initialization parameters according to the values read from MySQL .cnf file */ @@ -1535,37 +1388,20 @@ innodb_init_param(void) msg("xtrabackup: innodb_data_file_path = %s\n", innobase_data_file_path); - /* Since InnoDB edits the argument in the next call, we make another - copy of it: */ + /* This is the first time univ_page_size is used. + It was initialized to 16k pages before srv_page_size was set */ + univ_page_size.copy_from( + page_size_t(srv_page_size, srv_page_size, false)); - internal_innobase_data_file_path = strdup(innobase_data_file_path); + srv_sys_space.set_space_id(TRX_SYS_SPACE); + srv_sys_space.set_name("innodb_system"); + srv_sys_space.set_path(srv_data_home); + srv_sys_space.set_flags(FSP_FLAGS_PAGE_SSIZE()); - ret = (my_bool) srv_parse_data_file_paths_and_sizes( - internal_innobase_data_file_path); - if (ret == FALSE) { - msg("xtrabackup: syntax error in innodb_data_file_path\n"); -mem_free_and_error: - free(internal_innobase_data_file_path); - internal_innobase_data_file_path = NULL; + if (!srv_sys_space.parse_params(innobase_data_file_path, true)) { goto error; } - if (xtrabackup_prepare) { - /* "--prepare" needs filenames only */ - ulint i; - - for (i=0; i < srv_n_data_files; i++) { - char *p; - - p = srv_data_file_names[i]; - while ((p = strchr(p, SRV_PATH_SEPARATOR)) != NULL) - { - p++; - srv_data_file_names[i] = p; - } - } - } - /* -------------- Log files ---------------------------*/ /* The default dir for log files is the datadir of MySQL */ @@ -1579,57 +1415,35 @@ mem_free_and_error: msg("xtrabackup: innodb_log_group_home_dir = %s\n", srv_log_group_home_dir); - srv_normalize_path_for_win(srv_log_group_home_dir); + os_normalize_path(srv_log_group_home_dir); if (strchr(srv_log_group_home_dir, ';')) { msg("syntax error in innodb_log_group_home_dir, "); - - goto mem_free_and_error; + goto error; } srv_adaptive_flushing = FALSE; - srv_use_sys_malloc = TRUE; srv_file_flush_method_str = innobase_unix_file_flush_method; - srv_n_log_files = (ulint) innobase_log_files_in_group; - srv_log_file_size = (ulint) innobase_log_file_size; - msg("xtrabackup: innodb_log_files_in_group = %ld\n", - srv_n_log_files); - msg("xtrabackup: innodb_log_file_size = %lld\n", - (long long int) srv_log_file_size); - srv_log_buffer_size = (ulint) innobase_log_buffer_size; /* We set srv_pool_size here in units of 1 kB. InnoDB internally changes the value so that it becomes the number of database pages. */ - //srv_buf_pool_size = (ulint) innobase_buffer_pool_size; srv_buf_pool_size = (ulint) xtrabackup_use_memory; - - srv_mem_pool_size = (ulint) innobase_additional_mem_pool_size; + srv_buf_pool_chunk_unit = srv_buf_pool_size; + srv_buf_pool_instances = 1; srv_n_file_io_threads = (ulint) innobase_file_io_threads; srv_n_read_io_threads = (ulint) innobase_read_io_threads; srv_n_write_io_threads = (ulint) innobase_write_io_threads; - srv_force_recovery = (ulint) innobase_force_recovery; - srv_use_doublewrite_buf = (ibool) innobase_use_doublewrite; - if (!innobase_use_checksums) { - - srv_checksum_algorithm = SRV_CHECKSUM_ALGORITHM_NONE; - } - - btr_search_enabled = (char) innobase_adaptive_hash_index; - btr_search_index_num = 1; - os_use_large_pages = (ibool) innobase_use_large_pages; os_large_page_size = (ulint) innobase_large_page_size; - static char default_dir[3] = "./"; - srv_arch_dir = default_dir; row_rollback_on_timeout = (ibool) innobase_rollback_on_timeout; srv_file_per_table = (my_bool) innobase_file_per_table; @@ -1647,18 +1461,8 @@ mem_free_and_error: /* We cannot treat characterset here for now!! */ data_mysql_default_charset_coll = (ulint)default_charset_info->number; - ut_a(DATA_MYSQL_LATIN1_SWEDISH_CHARSET_COLL == - my_charset_latin1.number); ut_a(DATA_MYSQL_BINARY_CHARSET_COLL == my_charset_bin.number); - /* Store the latin1_swedish_ci character ordering table to InnoDB. For - non-latin1_swedish_ci charsets we use the MySQL comparison functions, - and consequently we do not need to know the ordering internally in - InnoDB. */ - - ut_a(0 == strcmp(my_charset_latin1.name, "latin1_swedish_ci")); - srv_latin1_ordering = my_charset_latin1.sort_order; - //innobase_commit_concurrency_init_default(); /* Since we in this module access directly the fields of a trx @@ -1672,31 +1476,8 @@ mem_free_and_error: innobase_start_or_create_for_mysql(). As we don't call it in xtrabackup, we have to duplicate checks from that function here. */ -#ifdef __WIN__ - switch (os_get_os_version()) { - case OS_WIN95: - case OS_WIN31: - case OS_WINNT: - /* On Win 95, 98, ME, Win32 subsystem for Windows 3.1, - and NT use simulated aio. In NT Windows provides async i/o, - but when run in conjunction with InnoDB Hot Backup, it seemed - to corrupt the data files. */ - - srv_use_native_aio = FALSE; - break; - - case OS_WIN2000: - case OS_WINXP: - /* On 2000 and XP, async IO is available. */ - srv_use_native_aio = TRUE; - break; - - default: - /* Vista and later have both async IO and condition variables */ - srv_use_native_aio = TRUE; - srv_use_native_conditions = TRUE; - break; - } +#ifdef _WIN32 + srv_use_native_aio = TRUE; #elif defined(LINUX_NATIVE_AIO) @@ -1719,11 +1500,12 @@ mem_free_and_error: directory. */ if (!srv_undo_dir || !xtrabackup_backup) { - my_free(srv_undo_dir); - srv_undo_dir = my_strdup(".", MYF(MY_FAE)); + srv_undo_dir = (char*) "."; } - innodb_log_checksum_func_update(srv_log_checksum_algorithm); + log_checksum_algorithm_ptr = innodb_log_checksums || srv_encrypt_log + ? log_block_calc_checksum_crc32 + : log_block_calc_checksum_none; return(FALSE); @@ -1732,57 +1514,17 @@ error: return(TRUE); } -static my_bool -innodb_init(void) +static bool innodb_init() { - int err; - srv_is_being_started = TRUE; - err = innobase_start_or_create_for_mysql(); - + dberr_t err = innobase_start_or_create_for_mysql(); if (err != DB_SUCCESS) { - free(internal_innobase_data_file_path); - internal_innobase_data_file_path = NULL; - goto error; + msg("xtrabackup: innodb_init() returned %d (%s).\n", + err, ut_strerr(err)); + innodb_shutdown(); + return(TRUE); } - /* They may not be needed for now */ -// (void) hash_init(&innobase_open_tables,system_charset_info, 32, 0, 0, -// (hash_get_key) innobase_get_key, 0, 0); -// pthread_mutex_init(&innobase_share_mutex, MY_MUTEX_INIT_FAST); -// pthread_mutex_init(&prepare_commit_mutex, MY_MUTEX_INIT_FAST); -// pthread_mutex_init(&commit_threads_m, MY_MUTEX_INIT_FAST); -// pthread_mutex_init(&commit_cond_m, MY_MUTEX_INIT_FAST); -// pthread_cond_init(&commit_cond, NULL); - - innodb_inited= 1; - return(FALSE); - -error: - msg("xtrabackup: innodb_init(): Error occured.\n"); - return(TRUE); -} - -static void -innodb_end() -{ - srv_fast_shutdown = (ulint) innobase_fast_shutdown; - innodb_inited = 0; - - msg("xtrabackup: starting shutdown with innodb_fast_shutdown = %lu\n", - srv_fast_shutdown); - - innodb_shutdown(); - free(internal_innobase_data_file_path); - internal_innobase_data_file_path = NULL; - - /* They may not be needed for now */ -// hash_free(&innobase_open_tables); -// pthread_mutex_destroy(&innobase_share_mutex); -// pthread_mutex_destroy(&prepare_commit_mutex); -// pthread_mutex_destroy(&commit_threads_m); -// pthread_mutex_destroy(&commit_cond_m); -// pthread_cond_destroy(&commit_cond); } /* ================= common ================= */ @@ -1937,8 +1679,7 @@ xb_read_delta_metadata(const char *filepath, xb_delta_info_t *info) my_bool r = TRUE; /* set defaults */ - info->page_size = ULINT_UNDEFINED; - info->zip_size = ULINT_UNDEFINED; + ulint page_size = ULINT_UNDEFINED, zip_size = 0; info->space_id = ULINT_UNDEFINED; fp = fopen(filepath, "r"); @@ -1950,9 +1691,9 @@ xb_read_delta_metadata(const char *filepath, xb_delta_info_t *info) while (!feof(fp)) { if (fscanf(fp, "%50s = %50s\n", key, value) == 2) { if (strcmp(key, "page_size") == 0) { - info->page_size = strtoul(value, NULL, 10); + page_size = strtoul(value, NULL, 10); } else if (strcmp(key, "zip_size") == 0) { - info->zip_size = strtoul(value, NULL, 10); + zip_size = strtoul(value, NULL, 10); } else if (strcmp(key, "space_id") == 0) { info->space_id = strtoul(value, NULL, 10); } @@ -1961,10 +1702,14 @@ xb_read_delta_metadata(const char *filepath, xb_delta_info_t *info) fclose(fp); - if (info->page_size == ULINT_UNDEFINED) { + if (page_size == ULINT_UNDEFINED) { msg("xtrabackup: page_size is required in %s\n", filepath); r = FALSE; + } else { + info->page_size = page_size_t(zip_size ? zip_size : page_size, + page_size, zip_size != 0); } + if (info->space_id == ULINT_UNDEFINED) { msg("xtrabackup: Warning: This backup was taken with XtraBackup 2.0.1 " "or earlier, some DDL operations between full and incremental " @@ -1990,7 +1735,10 @@ xb_write_delta_metadata(const char *filename, const xb_delta_info_t *info) "page_size = %lu\n" "zip_size = %lu\n" "space_id = %lu\n", - info->page_size, info->zip_size, info->space_id); + info->page_size.logical(), + info->page_size.is_compressed() + ? info->page_size.physical() : 0, + info->space_id); len = strlen(buf); mystat.st_size = len; @@ -2147,7 +1895,7 @@ check_if_skip_database_by_path( return(FALSE); } - const char* db_name = strrchr(path, SRV_PATH_SEPARATOR); + const char* db_name = strrchr(path, OS_PATH_SEPARATOR); if (db_name == NULL) { db_name = path; } else { @@ -2255,33 +2003,26 @@ check_if_skip_table( return(FALSE); } -/*********************************************************************** -Reads the space flags from a given data file and returns the compressed -page size, or 0 if the space is not compressed. */ -ulint -xb_get_zip_size(pfs_os_file_t file) +/** @return the tablespace flags from a given data file +@retval ULINT_UNDEFINED if the file is not readable */ +ulint xb_get_space_flags(pfs_os_file_t file) { byte *buf; byte *page; - ulint zip_size = ULINT_UNDEFINED; - ibool success; - ulint space; + ulint flags; - buf = static_cast(ut_malloc(2 * UNIV_PAGE_SIZE)); + buf = static_cast(malloc(2 * UNIV_PAGE_SIZE)); page = static_cast(ut_align(buf, UNIV_PAGE_SIZE)); - success = os_file_read(file, page, 0, UNIV_PAGE_SIZE); - if (!success) { - goto end; + if (os_file_read(IORequestRead, file, page, 0, UNIV_PAGE_SIZE)) { + flags = fsp_header_get_flags(page); + } else { + flags = ULINT_UNDEFINED; } - space = mach_read_from_4(page + FIL_PAGE_ARCH_LOG_NO_OR_SPACE_ID); - zip_size = (space == 0 ) ? 0 : - dict_tf_get_zip_size(fsp_header_get_flags(page)); -end: - ut_free(buf); + free(buf); - return(zip_size); + return(flags); } const char* @@ -2320,7 +2061,6 @@ xtrabackup_copy_datafile(fil_node_t* node, uint thread_n) xb_write_filt_ctxt_t write_filt_ctxt; const char *action; xb_read_filt_t *read_filter; - ibool is_system; my_bool rc = FALSE; /* Get the name and the path for the tablespace. node->name always @@ -2335,9 +2075,8 @@ xtrabackup_copy_datafile(fil_node_t* node, uint thread_n) const char* const node_name = node->space->name; const char* const node_path = node->name; - is_system = !fil_is_user_tablespace_id(node->space->id); - - if (!is_system && check_if_skip_table(node_name)) { + if (fil_is_user_tablespace_id(node->space->id) + && check_if_skip_table(node_name)) { msg("[%02u] Skipping %s.\n", thread_n, node_name); return(FALSE); } @@ -2446,190 +2185,91 @@ skip: return(FALSE); } -extern ibool log_block_checksum_is_ok_or_old_format(const byte* block); +/** How to copy a redo log segment in backup */ +enum copy_logfile { + /** Initial copying: copy at least one block */ + COPY_FIRST, + /** Tracking while copying data files */ + COPY_ONLINE, + /** Final copying: copy until the end of the log */ + COPY_LAST +}; -/*******************************************************//** -Scans log from a buffer and writes new log data to the outpud datasinc. -@return true if success */ +/** Copy redo log blocks to the data sink. +@param[in] copy how to copy the log +@param[in] start_lsn buffer start LSN +@param[in] end_lsn buffer end LSN +@return last scanned LSN (equals to last copied LSN if copy=COPY_LAST) +@retval 0 on failure */ static -bool -xtrabackup_scan_log_recs( -/*===============*/ - log_group_t* group, /*!< in: log group */ - bool is_last, /*!< in: whether it is last segment - to copy */ - lsn_t start_lsn, /*!< in: buffer start lsn */ - lsn_t* contiguous_lsn, /*!< in/out: it is known that all log - groups contain contiguous log data up - to this lsn */ - lsn_t* group_scanned_lsn,/*!< out: scanning succeeded up to - this lsn */ - bool* finished) /*!< out: false if is not able to scan - any more in this log group */ +lsn_t +xtrabackup_copy_log(copy_logfile copy, lsn_t start_lsn, lsn_t end_lsn) { - lsn_t scanned_lsn; - ulint data_len; - ulint write_size; - const byte* log_block; + lsn_t scanned_lsn = start_lsn; - ulint scanned_checkpoint_no = 0; + const byte* log_block = log_sys->buf; - *finished = false; - scanned_lsn = start_lsn; - log_block = log_sys->buf; - - while (log_block < log_sys->buf + RECV_SCAN_SIZE && !*finished) { - ulint no = log_block_get_hdr_no(log_block); - ulint scanned_no = log_block_convert_lsn_to_no(scanned_lsn); - ibool checksum_is_ok = - log_block_checksum_is_ok_or_old_format(log_block); - - if (no != scanned_no && checksum_is_ok) { - ulint blocks_in_group; - - blocks_in_group = log_block_convert_lsn_to_no( - log_group_get_capacity(group)) - 1; - - if ((no < scanned_no && - ((scanned_no - no) % blocks_in_group) == 0) || - no == 0 || - /* Log block numbers wrap around at 0x3FFFFFFF */ - ((scanned_no | 0x40000000UL) - no) % - blocks_in_group == 0) { - - /* old log block, do nothing */ - *finished = true; - break; - } - - msg("xtrabackup: error:" - " log block numbers mismatch:\n" - "xtrabackup: error: expected log block no. %lu," - " but got no. %lu from the log file.\n", - (ulong) scanned_no, (ulong) no); - - if ((no - scanned_no) % blocks_in_group == 0) { - msg("xtrabackup: error:" - " it looks like InnoDB log has wrapped" - " around before xtrabackup could" - " process all records due to either" - " log copying being too slow, or " - " log files being too small.\n"); - } - - return(false); - } else if (!checksum_is_ok) { - /* Garbage or an incompletely written log block */ - - msg("xtrabackup: warning: Log block checksum mismatch" - " (block no %lu at lsn " LSN_PF "): \n" - "expected %lu, calculated checksum %lu\n", - (ulong) no, - scanned_lsn, - (ulong) log_block_get_checksum(log_block), - (ulong) log_block_calc_checksum(log_block)); - msg("xtrabackup: warning: this is possible when the " - "log block has not been fully written by the " - "server, will retry later.\n"); - *finished = true; - break; - } - - if (log_block_get_flush_bit(log_block)) { - /* This block was a start of a log flush operation: - we know that the previous flush operation must have - been completed for all log groups before this block - can have been flushed to any of the groups. Therefore, - we know that log data is contiguous up to scanned_lsn - in all non-corrupt log groups. */ - - if (scanned_lsn > *contiguous_lsn) { - - *contiguous_lsn = scanned_lsn; - } - } - - data_len = log_block_get_data_len(log_block); - - if ( - (scanned_checkpoint_no > 0) - && (log_block_get_checkpoint_no(log_block) - < scanned_checkpoint_no) - && (scanned_checkpoint_no - - log_block_get_checkpoint_no(log_block) - > 0x80000000UL)) { + for (ulint scanned_checkpoint = 0; + scanned_lsn < end_lsn; + log_block += OS_FILE_LOG_BLOCK_SIZE) { + ulint checkpoint = log_block_get_checkpoint_no(log_block); + if (scanned_checkpoint > checkpoint + && scanned_checkpoint - checkpoint >= 0x80000000UL) { /* Garbage from a log buffer flush which was made before the most recent database recovery */ - - *finished = true; break; } - scanned_lsn = scanned_lsn + data_len; - scanned_checkpoint_no = log_block_get_checkpoint_no(log_block); + scanned_checkpoint = checkpoint; + ulint data_len = log_block_get_data_len(log_block); + scanned_lsn += data_len; - if (data_len < OS_FILE_LOG_BLOCK_SIZE) { - /* Log data for this group ends here */ - - *finished = true; - } else { - log_block += OS_FILE_LOG_BLOCK_SIZE; + if (data_len != OS_FILE_LOG_BLOCK_SIZE) { + /* The current end of the log was reached. */ + break; } } - *group_scanned_lsn = scanned_lsn; + log_sys->log.scanned_lsn = scanned_lsn; - /* ===== write log to 'xtrabackup_logfile' ====== */ - if (!*finished) { - write_size = RECV_SCAN_SIZE; - } else { - write_size = (ulint)(ut_uint64_align_up(scanned_lsn, - OS_FILE_LOG_BLOCK_SIZE) - start_lsn); - if (!is_last && scanned_lsn % OS_FILE_LOG_BLOCK_SIZE) { - write_size -= OS_FILE_LOG_BLOCK_SIZE; + end_lsn = copy == COPY_LAST + ? ut_uint64_align_up(scanned_lsn, OS_FILE_LOG_BLOCK_SIZE) + : scanned_lsn & ~(OS_FILE_LOG_BLOCK_SIZE - 1); + + if (ulint write_size = ulint(end_lsn - start_lsn)) { + if (srv_encrypt_log) { + log_crypt(log_sys->buf, write_size); + } + + if (ds_write(dst_log_file, log_sys->buf, write_size)) { + msg("xtrabackup: Error: " + "write to logfile failed\n"); + return(0); } } - if (write_size == 0) { - return(true); - } - - if (srv_encrypt_log) { - log_encrypt_before_write(scanned_checkpoint_no, - log_sys->buf, write_size); - } - - if (ds_write(dst_log_file, log_sys->buf, write_size)) { - msg("xtrabackup: Error: " - "write to logfile failed\n"); - return(false); - } - - return(true); + return(scanned_lsn); } -static my_bool -xtrabackup_copy_logfile(lsn_t from_lsn, my_bool is_last) +/** Copy redo log until the current end of the log is reached +@param copy how to copy the log +@return whether the operation failed */ +static bool +xtrabackup_copy_logfile(copy_logfile copy) { - /* definition from recv_recovery_from_checkpoint_start() */ - lsn_t contiguous_lsn; - ut_a(dst_log_file != NULL); + ut_ad(recv_sys != NULL); - /* read from checkpoint_lsn_start to current */ - contiguous_lsn = ut_uint64_align_down(from_lsn, OS_FILE_LOG_BLOCK_SIZE); - - /* TODO: We must check the contiguous_lsn still exists in log file.. */ - - bool finished; lsn_t start_lsn; lsn_t end_lsn; - /* reference recv_group_scan_log_recs() */ - - start_lsn = contiguous_lsn; + start_lsn = ut_uint64_align_down(log_copy_scanned_lsn, + OS_FILE_LOG_BLOCK_SIZE); + /* When copying the first or last part of the log, retry a few + times to ensure that all log up to the last checkpoint will be + read. */ do { end_lsn = start_lsn + RECV_SCAN_SIZE; @@ -2637,45 +2277,34 @@ xtrabackup_copy_logfile(lsn_t from_lsn, my_bool is_last) log_mutex_enter(); - log_group_read_log_seg(LOG_RECOVER, log_sys->buf, - &log_sys->log, start_lsn, end_lsn); + lsn_t lsn = log_group_read_log_seg(log_sys->buf, &log_sys->log, + start_lsn, end_lsn); - bool success = xtrabackup_scan_log_recs( - &log_sys->log, is_last, - start_lsn, &contiguous_lsn, - &log_sys->log.scanned_lsn, - &finished); + start_lsn = xtrabackup_copy_log(copy, start_lsn, lsn); log_mutex_exit(); - if (!success) { + if (!start_lsn) { ds_close(dst_log_file); + dst_log_file = NULL; msg("xtrabackup: Error: xtrabackup_copy_logfile()" " failed.\n"); - return(TRUE); + return(true); } + } while (start_lsn == end_lsn); - start_lsn = end_lsn; - } while (!finished); + ut_ad(start_lsn == log_sys->log.scanned_lsn); - msg_ts(">> log scanned up to (" LSN_PF ")\n", - log_sys->log.scanned_lsn); + msg_ts(">> log scanned up to (" LSN_PF ")\n", start_lsn); /* update global variable*/ - log_copy_scanned_lsn = log_sys->log.scanned_lsn; + log_copy_scanned_lsn = start_lsn; debug_sync_point("xtrabackup_copy_logfile_pause"); - return(FALSE); + return(false); } -static -#ifndef __WIN__ -void* -#else -ulint -#endif -log_copying_thread( - void* arg __attribute__((unused))) +static os_thread_ret_t log_copying_thread(void*) { /* Initialize mysys thread-specific memory so we can @@ -2683,31 +2312,14 @@ log_copying_thread( */ my_thread_init(); - ut_a(dst_log_file != NULL); - - log_copying_running = TRUE; - - while(log_copying) { + do { os_event_reset(log_copying_stop); os_event_wait_time_low(log_copying_stop, xtrabackup_log_copy_interval * 1000ULL, 0); - if (log_copying) { - if(xtrabackup_copy_logfile(log_copy_scanned_lsn, - FALSE)) { + } while (log_copying && xtrabackup_copy_logfile(COPY_ONLINE)); - exit(EXIT_FAILURE); - } - } - } - - /* last copying */ - if(xtrabackup_copy_logfile(log_copy_scanned_lsn, TRUE)) { - - exit(EXIT_FAILURE); - } - - log_copying_running = FALSE; + log_copying_running = false; my_thread_end(); os_thread_exit(NULL); @@ -2715,21 +2327,11 @@ log_copying_thread( } /* io throttle watching (rough) */ -static -#ifndef __WIN__ -void* -#else -ulint -#endif -io_watching_thread( - void* arg) +static os_thread_ret_t io_watching_thread(void*) { - (void)arg; /* currently, for --backup only */ ut_a(xtrabackup_backup); - io_watching_thread_running = TRUE; - while (log_copying) { os_thread_sleep(1000000); /*1 sec*/ io_ticket = xtrabackup_throttle; @@ -2740,49 +2342,13 @@ io_watching_thread( xtrabackup_throttle = 0; os_event_set(wait_throttle); - io_watching_thread_running = FALSE; + io_watching_thread_running = false; os_thread_exit(NULL); return(0); } -/************************************************************************ -I/o-handler thread function. */ -static - -#ifndef __WIN__ -void* -#else -ulint -#endif -io_handler_thread( -/*==============*/ - void* arg) -{ - ulint segment; - - - segment = *((ulint*)arg); - - while (srv_shutdown_state != SRV_SHUTDOWN_EXIT_THREADS) { - fil_aio_wait(segment); - } - - /* We count the number of threads in os_thread_exit(). A created - thread should always use that to exit and not use return() to exit. - The thread actually never comes here because it is exited in an - os_event_wait(). */ - - os_thread_exit(NULL); - -#ifndef __WIN__ - return(NULL); /* Not reached */ -#else - return(0); -#endif -} - /************************************************************************** Datafiles copying thread.*/ static @@ -2813,9 +2379,9 @@ data_copy_thread_func( } } - os_mutex_enter(ctxt->count_mutex); + pthread_mutex_lock(&ctxt->count_mutex); (*ctxt->count)--; - os_mutex_exit(ctxt->count_mutex); + pthread_mutex_unlock(&ctxt->count_mutex); my_thread_end(); os_thread_exit(NULL); @@ -2828,7 +2394,7 @@ Initialize the appropriate datasink(s). Both local backups and streaming in the Otherwise (i.e. when streaming in the 'tar' format) we need 2 separate datasinks for the data stream (and don't allow parallel data copying) and for metainfo -files (including xtrabackup_logfile). The second datasink writes to temporary +files (including ib_logfile0). The second datasink writes to temporary files first, and then streams them in a serialized way when closed. */ static void xtrabackup_init_datasinks(void) @@ -2914,43 +2480,263 @@ static void xtrabackup_destroy_datasinks(void) ds_redo = NULL; } -#define SRV_N_PENDING_IOS_PER_THREAD OS_AIO_N_PENDING_IOS_PER_THREAD #define SRV_MAX_N_PENDING_SYNC_IOS 100 -/************************************************************************ -@return TRUE if table should be opened. */ -static -ibool -xb_check_if_open_tablespace( - const char* db, - const char* table) -{ - char buf[FN_REFLEN]; - - snprintf(buf, sizeof(buf), "%s/%s", db, table); - - return !check_if_skip_table(buf); -} - -/************************************************************************ -Initializes the I/O and tablespace cache subsystems. */ +/** Initialize the tablespace cache subsystem. */ static void -xb_fil_io_init(void) -/*================*/ +xb_fil_io_init() { - srv_n_file_io_threads = srv_n_read_io_threads; - - os_aio_init(8 * SRV_N_PENDING_IOS_PER_THREAD, - srv_n_read_io_threads, - srv_n_write_io_threads, - SRV_MAX_N_PENDING_SYNC_IOS); - fil_init(srv_file_per_table ? 50000 : 5000, LONG_MAX); - fsp_init(); } +static +Datafile* +xb_new_datafile(const char *name, bool is_remote) +{ + if (is_remote) { + RemoteDatafile *remote_file = new RemoteDatafile(); + remote_file->set_name(name); + return(remote_file); + } else { + Datafile *file = new Datafile(); + file->set_name(name); + file->make_filepath(".", name, IBD); + return(file); + } +} + +static +void +xb_load_single_table_tablespace( + const char *dirname, + const char *filname, + bool is_remote) +{ + /* Ignore .isl files on XtraBackup recovery. All tablespaces must be + local. */ + if (is_remote && srv_operation == SRV_OPERATION_RESTORE) { + return; + } + if (check_if_skip_table(filname)) { + return; + } + + /* The name ends in .ibd or .isl; + try opening the file */ + char* name; + size_t dirlen = dirname == NULL ? 0 : strlen(dirname); + size_t namelen = strlen(filname); + ulint pathlen = dirname == NULL ? namelen + 1: dirlen + namelen + 2; + lsn_t flush_lsn; + dberr_t err; + fil_space_t *space; + + name = static_cast(ut_malloc_nokey(pathlen)); + + if (dirname != NULL) { + ut_snprintf(name, pathlen, "%s/%s", dirname, filname); + name[pathlen - 5] = 0; + } else { + ut_snprintf(name, pathlen, "%s", filname); + name[pathlen - 5] = 0; + } + + Datafile *file = xb_new_datafile(name, is_remote); + + if (file->open_read_only(true) != DB_SUCCESS) { + ut_free(name); + exit(EXIT_FAILURE); + } + + err = file->validate_first_page(&flush_lsn); + + if (err == DB_SUCCESS && file->space_id() != SRV_TMP_SPACE_ID) { + os_offset_t node_size = os_file_get_size(file->handle()); + os_offset_t n_pages; + + ut_a(node_size != (os_offset_t) -1); + + n_pages = node_size / page_size_t(file->flags()).physical(); + + space = fil_space_create( + name, file->space_id(), file->flags(), + FIL_TYPE_TABLESPACE, NULL/* TODO: crypt_data */); + + ut_a(space != NULL); + + if (!fil_node_create(file->filepath(), n_pages, space, + false, false)) { + ut_error; + } + + /* by opening the tablespace we forcing node and space objects + in the cache to be populated with fields from space header */ + fil_space_open(space->name); + + if (srv_operation == SRV_OPERATION_RESTORE || xb_close_files) { + fil_space_close(space->name); + } + } + + ut_free(name); + + if (fil_space_crypt_t* crypt_info = file->get_crypt_info()) { + fil_space_destroy_crypt_data(&crypt_info); + } + + delete file; + + if (err != DB_SUCCESS && err != DB_CORRUPTION && xtrabackup_backup) { + /* allow corrupted first page for xtrabackup, it could be just + zero-filled page, which we restore from redo log later */ + exit(EXIT_FAILURE); + } +} + +/** Scan the database directories under the MySQL datadir, looking for +.ibd files and determining the space id in each of them. +@return DB_SUCCESS or error number */ +static +dberr_t +xb_load_single_table_tablespaces() +{ + int ret; + char* dbpath = NULL; + ulint dbpath_len = 100; + os_file_dir_t dir; + os_file_dir_t dbdir; + os_file_stat_t dbinfo; + os_file_stat_t fileinfo; + dberr_t err = DB_SUCCESS; + + /* The datadir of MySQL is always the default directory of mysqld */ + + dir = os_file_opendir(fil_path_to_mysql_datadir, true); + + if (dir == NULL) { + + return(DB_ERROR); + } + + dbpath = static_cast(ut_malloc_nokey(dbpath_len)); + + /* Scan all directories under the datadir. They are the database + directories of MySQL. */ + + ret = fil_file_readdir_next_file(&err, fil_path_to_mysql_datadir, dir, + &dbinfo); + while (ret == 0) { + size_t len = strlen(dbinfo.name); + + /* General tablespaces are always at the first level of the + data home dir */ + if (dbinfo.type == OS_FILE_TYPE_FILE && len > 4) { + bool is_isl = !strcmp(dbinfo.name + len - 4, ".isl"); + bool is_ibd = !is_isl + && !strcmp(dbinfo.name + len - 4, ".ibd"); + + if (is_isl || is_ibd) { + xb_load_single_table_tablespace( + NULL, dbinfo.name, is_isl); + } + } + + if (dbinfo.type == OS_FILE_TYPE_FILE + || dbinfo.type == OS_FILE_TYPE_UNKNOWN) { + + goto next_datadir_item; + } + + /* We found a symlink or a directory; try opening it to see + if a symlink is a directory */ + + len = strlen(fil_path_to_mysql_datadir) + + strlen (dbinfo.name) + 2; + if (len > dbpath_len) { + dbpath_len = len; + + if (dbpath) { + ut_free(dbpath); + } + + dbpath = static_cast(ut_malloc_nokey(dbpath_len)); + } + ut_snprintf(dbpath, dbpath_len, + "%s/%s", fil_path_to_mysql_datadir, dbinfo.name); + os_normalize_path(dbpath); + + if (check_if_skip_database_by_path(dbpath)) { + fprintf(stderr, "Skipping db: %s\n", dbpath); + goto next_datadir_item; + } + + /* We want wrong directory permissions to be a fatal error for + XtraBackup. */ + dbdir = os_file_opendir(dbpath, true); + + if (dbdir != NULL) { + + /* We found a database directory; loop through it, + looking for possible .ibd files in it */ + + for (ret = fil_file_readdir_next_file(&err, dbpath, + dbdir, + &fileinfo); + ret == 0; + ret = fil_file_readdir_next_file(&err, dbpath, + dbdir, + &fileinfo)) { + if (fileinfo.type == OS_FILE_TYPE_DIR) { + continue; + } + + size_t len = strlen(fileinfo.name); + + /* We found a symlink or a file */ + if (len > 4 + && !strcmp(fileinfo.name + len - 4, + ".ibd")) { + xb_load_single_table_tablespace( + dbinfo.name, fileinfo.name, + false); + } + } + + if (0 != os_file_closedir(dbdir)) { + fprintf(stderr, "InnoDB: Warning: could not" + " close database directory %s\n", + dbpath); + + err = DB_ERROR; + } + + } else { + + err = DB_ERROR; + break; + + } + +next_datadir_item: + ret = fil_file_readdir_next_file(&err, + fil_path_to_mysql_datadir, + dir, &dbinfo); + } + + ut_free(dbpath); + + if (0 != os_file_closedir(dir)) { + fprintf(stderr, + "InnoDB: Error: could not close MySQL datadir\n"); + + return(DB_ERROR); + } + + return(err); +} + /**************************************************************************** Populates the tablespace memory cache by scanning for and opening data files. @returns DB_SUCCESS or error code.*/ @@ -2958,25 +2744,26 @@ static dberr_t xb_load_tablespaces() { - ulint i; bool create_new_db; dberr_t err; ulint sum_of_new_sizes; - lsn_t min_arch_logno, max_arch_logno; + lsn_t flush_lsn; - for (i = 0; i < srv_n_file_io_threads; i++) { - thread_nr[i] = i; + ut_ad(srv_operation == SRV_OPERATION_BACKUP + || srv_operation == SRV_OPERATION_RESTORE); - os_thread_create(io_handler_thread, thread_nr + i, - thread_ids + i); - } + err = srv_sys_space.check_file_spec(&create_new_db, 0); - os_thread_sleep(200000); /*0.2 sec*/ + /* create_new_db must not be true. */ + if (err != DB_SUCCESS || create_new_db) { + msg("xtrabackup: could not find data files at the " + "specified datadir\n"); + return(DB_ERROR); + } + + err = srv_sys_space.open_or_create(false, false, &sum_of_new_sizes, + &flush_lsn); - err = open_or_create_data_files(&create_new_db, - &min_arch_logno, &max_arch_logno, - &flushed_lsn, - &sum_of_new_sizes); if (err != DB_SUCCESS) { msg("xtrabackup: Could not open or create data files.\n" "xtrabackup: If you tried to add new data files, and it " @@ -2994,30 +2781,21 @@ xb_load_tablespaces() return(err); } - /* create_new_db must not be TRUE.. */ - if (create_new_db) { - msg("xtrabackup: could not find data files at the " - "specified datadir\n"); - return(DB_ERROR); - } - /* Add separate undo tablespaces to fil_system */ - err = srv_undo_tablespaces_init(FALSE, - TRUE, - srv_undo_tablespaces, - &srv_undo_tablespaces_open); + err = srv_undo_tablespaces_init(false); + if (err != DB_SUCCESS) { return(err); } - /* It is important to call fil_load_single_table_tablespace() after + /* It is important to call xb_load_single_table_tablespaces() after srv_undo_tablespaces_init(), because fil_is_user_tablespace_id() * relies on srv_undo_tablespaces_open to be properly initialized */ msg("xtrabackup: Generating a list of tablespaces\n"); - err = fil_load_single_table_tablespaces(xb_check_if_open_tablespace); + err = xb_load_single_table_tablespaces(); if (err != DB_SUCCESS) { return(err); } @@ -3046,42 +2824,11 @@ static void xb_data_files_close() { - ulint i; - - /* Shutdown the aio threads. This has been copied from - innobase_shutdown_for_mysql(). */ - - srv_shutdown_state = SRV_SHUTDOWN_EXIT_THREADS; - - for (i = 0; i < 1000; i++) { - os_aio_wake_all_threads_at_shutdown(); - - if (os_thread_count == 0) { - break; - } - os_thread_sleep(10000); - } - - if (i == 1000) { - msg("xtrabackup: Warning: %lu threads created by InnoDB" - " had not exited at shutdown!\n", - (ulong) os_thread_count); - } - - os_aio_free(); - + ut_ad(!os_thread_count); fil_close_all_files(); - - /* Free the double write data structures. */ if (buf_dblwr) { buf_dblwr_free(); } - - /* Reset srv_file_io_threads to its default value to avoid confusing - warning on --prepare in innobase_start_or_create_for_mysql()*/ - srv_n_file_io_threads = 4; - - srv_shutdown_state = SRV_SHUTDOWN_NONE; } /*********************************************************************** @@ -3100,7 +2847,7 @@ xb_new_filter_entry( ut_a(namelen <= NAME_LEN * 2 + 1); entry = static_cast - (ut_malloc(sizeof(xb_filter_entry_t) + namelen + 1)); + (malloc(sizeof(xb_filter_entry_t) + namelen + 1)); memset(entry, '\0', sizeof(xb_filter_entry_t) + namelen + 1); entry->name = ((char*)entry) + sizeof(xb_filter_entry_t); strcpy(entry->name, name); @@ -3399,7 +3146,7 @@ xb_filter_hash_free(hash_table_t* hash) HASH_DELETE(xb_filter_entry_t, name_hash, hash, ut_fold_string(prev_table->name), prev_table); - ut_free(prev_table); + free(prev_table); } } @@ -3448,77 +3195,33 @@ static ulint open_or_create_log_file( /*====================*/ - ibool create_new_db, /*!< in: TRUE if we should create a - new database */ + fil_space_t* space, ibool* log_file_created, /*!< out: TRUE if new log file created */ - ibool log_file_has_been_opened,/*!< in: TRUE if a log file has been - opened before: then it is an error - to try to create another log file */ ulint i) /*!< in: log file number in group */ { - ibool ret; - os_offset_t size; char name[10000]; ulint dirnamelen; - UT_NOT_USED(create_new_db); - UT_NOT_USED(log_file_has_been_opened); - *log_file_created = FALSE; - srv_normalize_path_for_win(srv_log_group_home_dir); + os_normalize_path(srv_log_group_home_dir); dirnamelen = strlen(srv_log_group_home_dir); ut_a(dirnamelen < (sizeof name) - 10 - sizeof "ib_logfile"); memcpy(name, srv_log_group_home_dir, dirnamelen); /* Add a path separator if needed. */ - if (dirnamelen && name[dirnamelen - 1] != SRV_PATH_SEPARATOR) { - name[dirnamelen++] = SRV_PATH_SEPARATOR; + if (dirnamelen && name[dirnamelen - 1] != OS_PATH_SEPARATOR) { + name[dirnamelen++] = OS_PATH_SEPARATOR; } sprintf(name + dirnamelen, "%s%lu", "ib_logfile", (ulong) i); - files[i] = os_file_create(innodb_file_log_key, name, - OS_FILE_OPEN, OS_FILE_NORMAL, - OS_LOG_FILE, &ret,0); - if (ret == FALSE) { - fprintf(stderr, "InnoDB: Error in opening %s\n", name); - - return(DB_ERROR); - } - - size = os_file_get_size(files[i]); - - if (size != srv_log_file_size * UNIV_PAGE_SIZE) { - - fprintf(stderr, - "InnoDB: Error: log file %s is" - " of different size " UINT64PF " bytes\n" - "InnoDB: than specified in the .cnf" - " file " UINT64PF " bytes!\n", - name, size, srv_log_file_size * UNIV_PAGE_SIZE); - - return(DB_ERROR); - } - - ret = os_file_close(files[i]); - ut_a(ret); - - if (i == 0) { - /* Create in memory the file space object - which is for this log group */ - - fil_space_create(name, - SRV_LOG_SPACE_FIRST_ID, 0, FIL_TYPE_LOG, 0, 0); - log_init(srv_n_log_files, srv_log_file_size * UNIV_PAGE_SIZE); - } - ut_a(fil_validate()); - ut_a(fil_node_create(name, (ulint)srv_log_file_size, - SRV_LOG_SPACE_FIRST_ID, FALSE)); + ut_a(fil_node_create(name, srv_log_file_size >> srv_page_size_shift, + space, false, false)); return(DB_SUCCESS); } @@ -3531,20 +3234,8 @@ void xb_normalize_init_values(void) /*==========================*/ { - ulint i; - - for (i = 0; i < srv_n_data_files; i++) { - srv_data_file_sizes[i] = srv_data_file_sizes[i] - * ((1024 * 1024) / UNIV_PAGE_SIZE); - } - - srv_last_file_size_max = srv_last_file_size_max - * ((1024 * 1024) / UNIV_PAGE_SIZE); - - srv_log_file_size = srv_log_file_size / UNIV_PAGE_SIZE; - - srv_log_buffer_size = srv_log_buffer_size / UNIV_PAGE_SIZE; - + srv_sys_space.normalize(); + srv_log_buffer_size /= UNIV_PAGE_SIZE; srv_lock_table_size = 5 * (srv_buf_pool_size / UNIV_PAGE_SIZE); } @@ -3605,8 +3296,35 @@ end: #endif } -void -xtrabackup_backup_func(void) +static void stop_backup_threads() +{ + log_copying = false; + + if (log_copying_stop) { + os_event_set(log_copying_stop); + msg("xtrabackup: Stopping log copying thread.\n"); + while (log_copying_running) { + msg("."); + os_thread_sleep(200000); /*0.2 sec*/ + } + msg("\n"); + os_event_destroy(log_copying_stop); + } + + if (wait_throttle) { + /* wait for io_watching_thread completion */ + while (io_watching_thread_running) { + os_thread_sleep(1000000); + } + os_event_destroy(wait_throttle); + } +} + +/** Implement --backup +@return whether the operation succeeded */ +static +bool +xtrabackup_backup_func() { MY_STAT stat_info; lsn_t latest_cp; @@ -3624,7 +3342,7 @@ xtrabackup_backup_func(void) if (my_setwd(mysql_real_data_home,MYF(MY_WME))) { msg("xtrabackup: cannot my_setwd %s\n", mysql_real_data_home); - exit(EXIT_FAILURE); + return(false); } msg("xtrabackup: cd to %s\n", mysql_real_data_home); @@ -3639,49 +3357,51 @@ xtrabackup_backup_func(void) srv_n_purge_threads = 1; srv_read_only_mode = TRUE; - srv_backup_mode = TRUE; - srv_close_files = (bool)xb_close_files; + srv_operation = SRV_OPERATION_BACKUP; - if (srv_close_files) + if (xb_close_files) msg("xtrabackup: warning: close-files specified. Use it " "at your own risk. If there are DDL operations like table DROP TABLE " "or RENAME TABLE during the backup, inconsistent backup will be " "produced.\n"); /* initialize components */ - if(innodb_init_param()) - exit(EXIT_FAILURE); + if(innodb_init_param()) { +fail: + stop_backup_threads(); + innodb_shutdown(); + return(false); + } xb_normalize_init_values(); if (srv_file_flush_method_str == NULL) { - /* These are the default options */ - srv_unix_file_flush_method = SRV_UNIX_FSYNC; + /* These are the default options */ + srv_file_flush_method = SRV_FSYNC; } else if (0 == ut_strcmp(srv_file_flush_method_str, "fsync")) { - srv_unix_file_flush_method = SRV_UNIX_FSYNC; + srv_file_flush_method = SRV_FSYNC; } else if (0 == ut_strcmp(srv_file_flush_method_str, "O_DSYNC")) { - srv_unix_file_flush_method = SRV_UNIX_O_DSYNC; + srv_file_flush_method = SRV_O_DSYNC; } else if (0 == ut_strcmp(srv_file_flush_method_str, "O_DIRECT")) { - srv_unix_file_flush_method = SRV_UNIX_O_DIRECT; + srv_file_flush_method = SRV_O_DIRECT; msg("xtrabackup: using O_DIRECT\n"); } else if (0 == ut_strcmp(srv_file_flush_method_str, "littlesync")) { - srv_unix_file_flush_method = SRV_UNIX_LITTLESYNC; - + srv_file_flush_method = SRV_LITTLESYNC; } else if (0 == ut_strcmp(srv_file_flush_method_str, "nosync")) { - srv_unix_file_flush_method = SRV_UNIX_NOSYNC; + srv_file_flush_method = SRV_NOSYNC; } else if (0 == ut_strcmp(srv_file_flush_method_str, "ALL_O_DIRECT")) { - srv_unix_file_flush_method = SRV_UNIX_ALL_O_DIRECT; + srv_file_flush_method = SRV_ALL_O_DIRECT_FSYNC; msg("xtrabackup: using ALL_O_DIRECT\n"); } else if (0 == ut_strcmp(srv_file_flush_method_str, "O_DIRECT_NO_FSYNC")) { - srv_unix_file_flush_method = SRV_UNIX_O_DIRECT_NO_FSYNC; + srv_file_flush_method = SRV_O_DIRECT_NO_FSYNC; msg("xtrabackup: using O_DIRECT_NO_FSYNC\n"); } else { - msg("xtrabackup: Unrecognized value %s for " + msg("xtrabackup: Unrecognized value %s for " "innodb_flush_method\n", srv_file_flush_method_str); - exit(EXIT_FAILURE); + goto fail; } /* We can only use synchronous unbuffered IO on Windows for now */ @@ -3691,8 +3411,8 @@ xtrabackup_backup_func(void) } #ifdef _WIN32 - srv_win_file_flush_method = SRV_WIN_IO_UNBUFFERED; - srv_use_native_aio = FALSE; + srv_file_flush_method = SRV_ALL_O_DIRECT_FSYNC; + srv_use_native_aio = TRUE; #endif if (srv_buf_pool_size >= 1000 * 1024 * 1024) { @@ -3712,12 +3432,19 @@ xtrabackup_backup_func(void) computers */ } - srv_general_init(); + sync_check_init(); + ut_d(sync_check_enable()); + /* Reset the system variables in the recovery module. */ + recv_sys_var_init(); + trx_pool_init(); + row_mysql_init(); + ut_crc32_init(); crc_init(); + recv_sys_init(); #ifdef WITH_INNODB_DISALLOW_WRITES - srv_allow_writes_event = os_event_create(); + srv_allow_writes_event = os_event_create(0); os_event_set(srv_allow_writes_event); #endif @@ -3731,18 +3458,23 @@ xtrabackup_backup_func(void) ulint i; xb_fil_io_init(); + srv_n_file_io_threads = srv_n_read_io_threads; + + os_aio_init(srv_n_read_io_threads, srv_n_write_io_threads, + SRV_MAX_N_PENDING_SYNC_IOS); log_sys_init(); + log_init(srv_n_log_files); + fil_space_t* space = fil_space_create( + "innodb_redo_log", SRV_LOG_SPACE_FIRST_ID, 0, + FIL_TYPE_LOG, NULL); lock_sys_create(srv_lock_table_size); for (i = 0; i < srv_n_log_files; i++) { - err = open_or_create_log_file(FALSE, &log_file_created, - log_opened, i); + err = open_or_create_log_file(space, &log_file_created, i); if (err != DB_SUCCESS) { - - //return((int) err); - exit(EXIT_FAILURE); + goto fail; } if (log_file_created) { @@ -3759,15 +3491,14 @@ xtrabackup_backup_func(void) "xtrabackup: Then delete the existing log files. Edit the .cnf file\n" "xtrabackup: and start the database again.\n"); - //return(DB_ERROR); - exit(EXIT_FAILURE); + goto fail; } } /* log_file_created must not be TRUE, if online */ if (log_file_created) { msg("xtrabackup: Something wrong with source files...\n"); - exit(EXIT_FAILURE); + goto fail; } } @@ -3778,7 +3509,7 @@ xtrabackup_backup_func(void) && (my_mkdir(xtrabackup_extra_lsndir,0777,MYF(0)) < 0)) { msg("xtrabackup: Error: cannot mkdir %d: %s\n", my_errno, xtrabackup_extra_lsndir); - exit(EXIT_FAILURE); + goto fail; } /* create target dir if not exist */ @@ -3786,12 +3517,10 @@ xtrabackup_backup_func(void) && (my_mkdir(xtrabackup_target_dir,0777,MYF(0)) < 0)){ msg("xtrabackup: Error: cannot mkdir %d: %s\n", my_errno, xtrabackup_target_dir); - exit(EXIT_FAILURE); + goto fail; } { - fil_system_t* f_system = fil_system; - /* definition from recv_recovery_from_checkpoint_start() */ ulint max_cp_field; @@ -3807,14 +3536,17 @@ xtrabackup_backup_func(void) dberr_t err = recv_find_max_checkpoint(&max_cp_field); if (err != DB_SUCCESS) { - exit(EXIT_FAILURE); +log_fail: + log_mutex_exit(); + goto fail; } if (log_sys->log.format == 0) { old_format: msg("xtrabackup: Error: cannot process redo log" " before MariaDB 10.2.2\n"); - exit(EXIT_FAILURE); + log_mutex_exit(); + goto log_fail; } ut_ad(!((log_sys->log.format ^ LOG_HEADER_FORMAT_CURRENT) @@ -3829,7 +3561,7 @@ reread_log_header: err = recv_find_max_checkpoint(&max_cp_field); if (err != DB_SUCCESS) { - exit(EXIT_FAILURE); + goto log_fail; } if (log_sys->log.format == 0) { @@ -3839,7 +3571,7 @@ reread_log_header: ut_ad(!((log_sys->log.format ^ LOG_HEADER_FORMAT_CURRENT) & ~LOG_HEADER_FORMAT_ENCRYPTED)); - if(checkpoint_no_start != mach_read_from_8(buf + LOG_CHECKPOINT_NO)) { + if (checkpoint_no_start != mach_read_from_8(buf + LOG_CHECKPOINT_NO)) { checkpoint_lsn_start = mach_read_from_8(buf + LOG_CHECKPOINT_LSN); checkpoint_no_start = mach_read_from_8(buf + LOG_CHECKPOINT_NO); @@ -3851,33 +3583,49 @@ reread_log_header: xtrabackup_init_datasinks(); if (!select_history()) { - exit(EXIT_FAILURE); + goto fail; } /* open the log file */ memset(&stat_info, 0, sizeof(MY_STAT)); - dst_log_file = ds_open(ds_redo, XB_LOG_FILENAME, &stat_info); + dst_log_file = ds_open(ds_redo, "ib_logfile0", &stat_info); if (dst_log_file == NULL) { msg("xtrabackup: error: failed to open the target stream for " - "'%s'.\n", XB_LOG_FILENAME); - ut_free(log_hdr_buf_); - exit(EXIT_FAILURE); + "'ib_logfile0'.\n"); + goto fail; } /* label it */ - strcpy((char*) log_hdr_buf + LOG_FILE_WAS_CREATED_BY_HOT_BACKUP, - "xtrabkup "); - ut_sprintf_timestamp( - (char*) log_hdr_buf + (LOG_FILE_WAS_CREATED_BY_HOT_BACKUP - + (sizeof "xtrabkup ") - 1)); + byte MY_ALIGNED(OS_FILE_LOG_BLOCK_SIZE) log_hdr[OS_FILE_LOG_BLOCK_SIZE]; + memset(log_hdr, 0, sizeof log_hdr); + mach_write_to_4(LOG_HEADER_FORMAT + log_hdr, log_sys->log.format); + mach_write_to_8(LOG_HEADER_START_LSN + log_hdr, checkpoint_lsn_start); + strcpy(reinterpret_cast(LOG_HEADER_CREATOR + log_hdr), + "Backup " MYSQL_SERVER_VERSION); + log_block_set_checksum(log_hdr, + log_block_calc_checksum_crc32(log_hdr)); - if (ds_write(dst_log_file, log_hdr_buf, LOG_FILE_HDR_SIZE)) { + /* Write the log header. */ + if (ds_write(dst_log_file, log_hdr, sizeof log_hdr)) { + log_write_fail: msg("xtrabackup: error: write to logfile failed\n"); - ut_free(log_hdr_buf_); - exit(EXIT_FAILURE); + goto fail; + } + /* Adjust the checkpoint page. */ + memcpy(log_hdr, buf, OS_FILE_LOG_BLOCK_SIZE); + mach_write_to_8(log_hdr + LOG_CHECKPOINT_OFFSET, + (checkpoint_lsn_start & (OS_FILE_LOG_BLOCK_SIZE - 1)) + | LOG_FILE_HDR_SIZE); + log_block_set_checksum(log_hdr, + log_block_calc_checksum_crc32(log_hdr)); + /* Write checkpoint page 1 and two empty log pages before the + payload. */ + if (ds_write(dst_log_file, log_hdr, OS_FILE_LOG_BLOCK_SIZE) + || !memset(log_hdr, 0, sizeof log_hdr) + || ds_write(dst_log_file, log_hdr, sizeof log_hdr) + || ds_write(dst_log_file, log_hdr, sizeof log_hdr)) { + goto log_write_fail; } - - ut_free(log_hdr_buf_); /* start flag */ log_copying = TRUE; @@ -3887,18 +3635,20 @@ reread_log_header: os_thread_id_t io_watching_thread_id; io_ticket = xtrabackup_throttle; - wait_throttle = os_event_create(); + wait_throttle = os_event_create(0); + io_watching_thread_running = true; os_thread_create(io_watching_thread, NULL, &io_watching_thread_id); } /* copy log file by current position */ - if(xtrabackup_copy_logfile(checkpoint_lsn_start, FALSE)) - exit(EXIT_FAILURE); + log_copy_scanned_lsn = checkpoint_lsn_start; + if (xtrabackup_copy_logfile(COPY_FIRST)) + goto fail; - - log_copying_stop = os_event_create(); + log_copying_stop = os_event_create(0); + log_copying_running = true; os_thread_create(log_copying_thread, NULL, &log_copying_thread_id); /* Populate fil_system with tablespaces to copy */ @@ -3906,12 +3656,12 @@ reread_log_header: if (err != DB_SUCCESS) { msg("xtrabackup: error: xb_load_tablespaces() failed with" "error code %u\n", err); - exit(EXIT_FAILURE); + goto fail; } /* FLUSH CHANGED_PAGE_BITMAPS call */ if (!flush_changed_page_bitmaps()) { - exit(EXIT_FAILURE); + goto fail; } debug_sync_point("xtrabackup_suspend_at_start"); @@ -3936,17 +3686,17 @@ reread_log_header: "files transfer\n", xtrabackup_parallel); } - it = datafiles_iter_new(f_system); + it = datafiles_iter_new(fil_system); if (it == NULL) { msg("xtrabackup: Error: datafiles_iter_new() failed.\n"); - exit(EXIT_FAILURE); + goto fail; } /* Create data copying threads */ data_threads = (data_thread_ctxt_t *) - ut_malloc(sizeof(data_thread_ctxt_t) * xtrabackup_parallel); + malloc(sizeof(data_thread_ctxt_t) * xtrabackup_parallel); count = xtrabackup_parallel; - count_mutex = os_mutex_create(); + pthread_mutex_init(&count_mutex, NULL); for (i = 0; i < (uint) xtrabackup_parallel; i++) { data_threads[i].it = it; @@ -3960,16 +3710,16 @@ reread_log_header: /* Wait for threads to exit */ while (1) { os_thread_sleep(1000000); - os_mutex_enter(count_mutex); - if (count == 0) { - os_mutex_exit(count_mutex); + pthread_mutex_lock(&count_mutex); + bool stop = count == 0; + pthread_mutex_unlock(&count_mutex); + if (stop) { break; } - os_mutex_exit(count_mutex); } - os_mutex_free(count_mutex); - ut_free(data_threads); + pthread_mutex_destroy(&count_mutex); + free(data_threads); datafiles_iter_free(it); if (changed_page_bitmap) { @@ -3978,7 +3728,7 @@ reread_log_header: } if (!backup_start()) { - exit(EXIT_FAILURE); + goto fail; } /* read the latest checkpoint lsn */ @@ -4000,21 +3750,19 @@ reread_log_header: log_mutex_exit(); } - /* stop log_copying_thread */ - log_copying = FALSE; - os_event_set(log_copying_stop); - msg("xtrabackup: Stopping log copying thread.\n"); - while (log_copying_running) { - msg("."); - os_thread_sleep(200000); /*0.2 sec*/ - } - msg("\n"); + stop_backup_threads(); - os_event_free(log_copying_stop); - if (ds_close(dst_log_file)) { - exit(EXIT_FAILURE); + if (!dst_log_file || xtrabackup_copy_logfile(COPY_LAST)) { + goto fail; } + if (ds_close(dst_log_file)) { + dst_log_file = NULL; + goto fail; + } + + dst_log_file = NULL; + if(!xtrabackup_incremental) { strcpy(metadata_type, "full-backuped"); metadata_from_lsn = 0; @@ -4027,7 +3775,7 @@ reread_log_header: if (!xtrabackup_stream_metadata(ds_meta)) { msg("xtrabackup: Error: failed to stream metadata.\n"); - exit(EXIT_FAILURE); + goto fail; } if (xtrabackup_extra_lsndir) { char filename[FN_REFLEN]; @@ -4037,335 +3785,37 @@ reread_log_header: if (!xtrabackup_write_metadata(filename)) { msg("xtrabackup: Error: failed to write metadata " "to '%s'.\n", filename); - exit(EXIT_FAILURE); + goto fail; } } if (!backup_finish()) { - exit(EXIT_FAILURE); + goto fail; } xtrabackup_destroy_datasinks(); - if (wait_throttle) { - /* wait for io_watching_thread completion */ - while (io_watching_thread_running) { - os_thread_sleep(1000000); - } - os_event_free(wait_throttle); - wait_throttle = NULL; - } - - msg("xtrabackup: Transaction log of lsn (" LSN_PF ") to (" LSN_PF + msg("xtrabackup: Redo log (from LSN " LSN_PF " to " LSN_PF ") was copied.\n", checkpoint_lsn_start, log_copy_scanned_lsn); xb_filters_free(); xb_data_files_close(); - /* Make sure that the latest checkpoint made it to xtrabackup_logfile */ + /* Make sure that the latest checkpoint was included */ if (latest_cp > log_copy_scanned_lsn) { - msg("xtrabackup: error: last checkpoint LSN (" LSN_PF - ") is larger than last copied LSN (" LSN_PF ").\n", - latest_cp, log_copy_scanned_lsn); - exit(EXIT_FAILURE); + msg("xtrabackup: error: failed to copy enough redo log (" + "LSN=" LSN_PF "; checkpoint LSN=" LSN_PF ").\n", + log_copy_scanned_lsn, latest_cp); + goto fail; } + + innodb_shutdown(); + return(true); } /* ================= prepare ================= */ -static my_bool -xtrabackup_init_temp_log(void) -{ - pfs_os_file_t src_file; - char src_path[FN_REFLEN]; - char dst_path[FN_REFLEN]; - ibool success; - - ulint field; - byte* log_buf= (byte *)malloc(UNIV_PAGE_SIZE_MAX * 128); /* 2 MB */ - - ib_int64_t file_size; - - lsn_t max_no; - lsn_t max_lsn; - lsn_t checkpoint_no; - - ulint fold; - - bool checkpoint_found; - - max_no = 0; - - if (!log_buf) { - goto error; - } - - if (!xb_init_log_block_size()) { - goto error; - } - - if(!xtrabackup_incremental_dir) { - sprintf(dst_path, "%s/ib_logfile0", xtrabackup_target_dir); - sprintf(src_path, "%s/%s", xtrabackup_target_dir, - XB_LOG_FILENAME); - } else { - sprintf(dst_path, "%s/ib_logfile0", xtrabackup_incremental_dir); - sprintf(src_path, "%s/%s", xtrabackup_incremental_dir, - XB_LOG_FILENAME); - } - - srv_normalize_path_for_win(dst_path); - srv_normalize_path_for_win(src_path); -retry: - src_file = os_file_create_simple_no_error_handling(0, src_path, - OS_FILE_OPEN, - OS_FILE_READ_WRITE, - &success,0); - if (!success) { - /* The following call prints an error message */ - os_file_get_last_error(TRUE); - - msg("xtrabackup: Warning: cannot open %s. will try to find.\n", - src_path); - - /* check if ib_logfile0 may be xtrabackup_logfile */ - src_file = os_file_create_simple_no_error_handling(0, dst_path, - OS_FILE_OPEN, - OS_FILE_READ_WRITE, - &success,0); - if (!success) { - os_file_get_last_error(TRUE); - msg(" xtrabackup: Fatal error: cannot find %s.\n", - src_path); - - goto error; - } - - success = os_file_read(src_file, log_buf, 0, - LOG_FILE_HDR_SIZE); - if (!success) { - goto error; - } - - if ( ut_memcmp(log_buf + LOG_FILE_WAS_CREATED_BY_HOT_BACKUP, - (byte*)"xtrabkup", (sizeof "xtrabkup") - 1) == 0) { - msg(" xtrabackup: 'ib_logfile0' seems to be " - "'xtrabackup_logfile'. will retry.\n"); - - os_file_close(src_file); - src_file = XB_FILE_UNDEFINED; - - /* rename and try again */ - success = os_file_rename(0, dst_path, src_path); - if (!success) { - goto error; - } - - goto retry; - } - - msg(" xtrabackup: Fatal error: cannot find %s.\n", - src_path); - - os_file_close(src_file); - src_file = XB_FILE_UNDEFINED; - - goto error; - } - - file_size = os_file_get_size(src_file); - - - /* TODO: We should skip the following modifies, if it is not the first time. */ - - /* read log file header */ - success = os_file_read(src_file, log_buf, 0, LOG_FILE_HDR_SIZE); - if (!success) { - goto error; - } - - if ( ut_memcmp(log_buf + LOG_FILE_WAS_CREATED_BY_HOT_BACKUP, - (byte*)"xtrabkup", (sizeof "xtrabkup") - 1) != 0 ) { - msg("xtrabackup: notice: xtrabackup_logfile was already used " - "to '--prepare'.\n"); - goto skip_modify; - } else { - /* clear it later */ - //memset(log_buf + LOG_FILE_WAS_CREATED_BY_HOT_BACKUP, - // ' ', 4); - } - - checkpoint_found = false; - - /* read last checkpoint lsn */ - for (field = LOG_CHECKPOINT_1; field <= LOG_CHECKPOINT_2; - field += LOG_CHECKPOINT_2 - LOG_CHECKPOINT_1) { - if (!recv_check_cp_is_consistent(const_cast - (log_buf + field))) - goto not_consistent; - - checkpoint_no = mach_read_from_8(log_buf + field + - LOG_CHECKPOINT_NO); - - if (checkpoint_no >= max_no) { - - max_no = checkpoint_no; - max_lsn = mach_read_from_8(log_buf + field + - LOG_CHECKPOINT_LSN); - checkpoint_found = true; - } -not_consistent: - ; - } - - if (!checkpoint_found) { - msg("xtrabackup: No valid checkpoint found.\n"); - goto error; - } - - - /* It seems to be needed to overwrite the both checkpoint area. */ - mach_write_to_8(log_buf + LOG_CHECKPOINT_1 + LOG_CHECKPOINT_LSN, - max_lsn); - mach_write_to_4(log_buf + LOG_CHECKPOINT_1 - + LOG_CHECKPOINT_OFFSET_LOW32, - LOG_FILE_HDR_SIZE + - (ulint)(max_lsn - - ut_uint64_align_down(max_lsn, - OS_FILE_LOG_BLOCK_SIZE))); - mach_write_to_4(log_buf + LOG_CHECKPOINT_1 - + LOG_CHECKPOINT_OFFSET_HIGH32, 0); - fold = ut_fold_binary(log_buf + LOG_CHECKPOINT_1, LOG_CHECKPOINT_CHECKSUM_1); - mach_write_to_4(log_buf + LOG_CHECKPOINT_1 + LOG_CHECKPOINT_CHECKSUM_1, fold); - - fold = ut_fold_binary(log_buf + LOG_CHECKPOINT_1 + LOG_CHECKPOINT_LSN, - LOG_CHECKPOINT_CHECKSUM_2 - LOG_CHECKPOINT_LSN); - mach_write_to_4(log_buf + LOG_CHECKPOINT_1 + LOG_CHECKPOINT_CHECKSUM_2, fold); - - mach_write_to_8(log_buf + LOG_CHECKPOINT_2 + LOG_CHECKPOINT_LSN, - max_lsn); - mach_write_to_4(log_buf + LOG_CHECKPOINT_2 - + LOG_CHECKPOINT_OFFSET_LOW32, - LOG_FILE_HDR_SIZE + - (ulint)(max_lsn - - ut_uint64_align_down(max_lsn, - OS_FILE_LOG_BLOCK_SIZE))); - mach_write_to_4(log_buf + LOG_CHECKPOINT_2 - + LOG_CHECKPOINT_OFFSET_HIGH32, 0); - fold = ut_fold_binary(log_buf + LOG_CHECKPOINT_2, LOG_CHECKPOINT_CHECKSUM_1); - mach_write_to_4(log_buf + LOG_CHECKPOINT_2 + LOG_CHECKPOINT_CHECKSUM_1, fold); - - fold = ut_fold_binary(log_buf + LOG_CHECKPOINT_2 + LOG_CHECKPOINT_LSN, - LOG_CHECKPOINT_CHECKSUM_2 - LOG_CHECKPOINT_LSN); - mach_write_to_4(log_buf + LOG_CHECKPOINT_2 + LOG_CHECKPOINT_CHECKSUM_2, fold); - - - success = os_file_write(src_path, src_file, log_buf, 0, - LOG_FILE_HDR_SIZE); - if (!success) { - goto error; - } - - /* expand file size (9/8) and align to UNIV_PAGE_SIZE_MAX */ - - if (file_size % UNIV_PAGE_SIZE_MAX) { - memset(log_buf, 0, UNIV_PAGE_SIZE_MAX); - success = os_file_write(src_path, src_file, log_buf, - file_size, - UNIV_PAGE_SIZE_MAX - - (ulint) (file_size - % UNIV_PAGE_SIZE_MAX)); - if (!success) { - goto error; - } - - file_size = os_file_get_size(src_file); - } - - /* TODO: We should judge whether the file is already expanded or not... */ - { - ulint expand; - - memset(log_buf, 0, UNIV_PAGE_SIZE_MAX * 128); - expand = (ulint) (file_size / UNIV_PAGE_SIZE_MAX / 8); - - for (; expand > 128; expand -= 128) { - success = os_file_write(src_path, src_file, log_buf, - file_size, - UNIV_PAGE_SIZE_MAX * 128); - if (!success) { - goto error; - } - file_size += UNIV_PAGE_SIZE_MAX * 128; - } - - if (expand) { - success = os_file_write(src_path, src_file, log_buf, - file_size, - expand * UNIV_PAGE_SIZE_MAX); - if (!success) { - goto error; - } - file_size += UNIV_PAGE_SIZE_MAX * expand; - } - } - - /* make larger than 2MB */ - if (file_size < 2*1024*1024L) { - memset(log_buf, 0, UNIV_PAGE_SIZE_MAX); - while (file_size < 2*1024*1024L) { - success = os_file_write(src_path, src_file, log_buf, - file_size, - UNIV_PAGE_SIZE_MAX); - if (!success) { - goto error; - } - file_size += UNIV_PAGE_SIZE_MAX; - } - file_size = os_file_get_size(src_file); - } - - msg("xtrabackup: xtrabackup_logfile detected: size=" INT64PF ", " - "start_lsn=(" LSN_PF ")\n", file_size, max_lsn); - - os_file_close(src_file); - src_file = XB_FILE_UNDEFINED; - - /* fake InnoDB */ - innobase_log_files_in_group_save = innobase_log_files_in_group; - srv_log_group_home_dir_save = srv_log_group_home_dir; - innobase_log_file_size_save = innobase_log_file_size; - - srv_log_group_home_dir = NULL; - innobase_log_file_size = file_size; - innobase_log_files_in_group = 1; - - srv_thread_concurrency = 0; - - /* rename 'xtrabackup_logfile' to 'ib_logfile0' */ - success = os_file_rename(0, src_path, dst_path); - if (!success) { - goto error; - } - xtrabackup_logfile_is_renamed = TRUE; - free(log_buf); - return(FALSE); - -skip_modify: - free(log_buf); - os_file_close(src_file); - src_file = XB_FILE_UNDEFINED; - return(FALSE); - -error: - free(log_buf); - if (src_file != XB_FILE_UNDEFINED) - os_file_close(src_file); - msg("xtrabackup: Error: xtrabackup_init_temp_log() failed.\n"); - return(TRUE); /*ERROR*/ -} - /*********************************************************************** Generates path to the meta file path from a given path to an incremental .delta by replacing trailing ".delta" with ".meta", or returns error if 'delta_path' @@ -4395,31 +3845,30 @@ file. Code adopted from fil_create_new_single_table_tablespace with the main difference that only disk file is created without updating the InnoDB in-memory dictionary data structures. -@return TRUE on success, FALSE on error. */ +@return true on success, false on error. */ static -ibool +bool xb_space_create_file( /*==================*/ const char* path, /*!(ut_malloc(3 * UNIV_PAGE_SIZE)); + buf = static_cast(malloc(3 * UNIV_PAGE_SIZE)); /* Align the memory for file i/o if we might have O_DIRECT set */ page = static_cast(ut_align(buf, UNIV_PAGE_SIZE)); @@ -4436,19 +3885,19 @@ xb_space_create_file( fsp_header_init_fields(page, space_id, flags); mach_write_to_4(page + FIL_PAGE_ARCH_LOG_NO_OR_SPACE_ID, space_id); - if (!fsp_flags_is_compressed(flags)) { - buf_flush_init_for_writing(page, NULL, 0); + const page_size_t page_size(flags); - ret = os_file_write(path, *file, page, 0, UNIV_PAGE_SIZE); - } - else { + if (!page_size.is_compressed()) { + buf_flush_init_for_writing(NULL, page, NULL, 0, false); + + ret = os_file_write(IORequestWrite, path, *file, page, 0, + UNIV_PAGE_SIZE); + } else { page_zip_des_t page_zip; - ulint zip_size; - - zip_size = fsp_flags_get_zip_size(flags); + ulint zip_size = page_size.physical(); page_zip_set_size(&page_zip, zip_size); page_zip.data = page + UNIV_PAGE_SIZE; - fprintf(stderr, "zip_size = %lu\n", zip_size); + fprintf(stderr, "zip_size = " ULINTPF "\n", zip_size); #ifdef UNIV_DEBUG page_zip.m_start = @@ -4456,13 +3905,13 @@ xb_space_create_file( page_zip.m_end = page_zip.m_nonempty = page_zip.n_blobs = 0; - buf_flush_init_for_writing(page, &page_zip, 0); + buf_flush_init_for_writing(NULL, page, &page_zip, 0, false); - ret = os_file_write(path, *file, page_zip.data, 0, - zip_size); + ret = os_file_write(IORequestWrite, path, *file, + page_zip.data, 0, zip_size); } - ut_free(buf); + free(buf); if (!ret) { msg("xtrabackup: could not write the first page to %s\n", @@ -4487,35 +3936,32 @@ pfs_os_file_t xb_delta_open_matching_space( const char* dbname, /* in: path to destination database dir */ const char* name, /* in: name of delta file (without .delta) */ - ulint space_id, /* in: space id of delta file */ - ulint zip_size, /* in: zip_size of tablespace */ + const xb_delta_info_t& info, char* real_name, /* out: full path of destination file */ size_t real_name_len, /* out: buffer size for real_name */ - ibool* success) /* out: indicates error. TRUE = success */ + bool* success) /* out: indicates error. true = success */ { char dest_dir[FN_REFLEN]; char dest_space_name[FN_REFLEN]; - ibool ok; fil_space_t* fil_space; pfs_os_file_t file; - ulint tablespace_flags; xb_filter_entry_t* table; ut_a(dbname != NULL || - !fil_is_user_tablespace_id(space_id) || - space_id == ULINT_UNDEFINED); + !fil_is_user_tablespace_id(info.space_id) || + info.space_id == ULINT_UNDEFINED); - *success = FALSE; + *success = false; if (dbname) { snprintf(dest_dir, FN_REFLEN, "%s/%s", xtrabackup_target_dir, dbname); - srv_normalize_path_for_win(dest_dir); + os_normalize_path(dest_dir); snprintf(dest_space_name, FN_REFLEN, "%s/%s", dbname, name); } else { snprintf(dest_dir, FN_REFLEN, "%s", xtrabackup_target_dir); - srv_normalize_path_for_win(dest_dir); + os_normalize_path(dest_dir); snprintf(dest_space_name, FN_REFLEN, "%s", name); } @@ -4523,7 +3969,7 @@ xb_delta_open_matching_space( snprintf(real_name, real_name_len, "%s/%s", xtrabackup_target_dir, dest_space_name); - srv_normalize_path_for_win(real_name); + os_normalize_path(real_name); /* Truncate ".ibd" */ dest_space_name[strlen(dest_space_name) - 4] = '\0'; @@ -4533,13 +3979,26 @@ xb_delta_open_matching_space( return file; } - if (!fil_is_user_tablespace_id(space_id)) { - goto found; + log_mutex_enter(); + if (!fil_is_user_tablespace_id(info.space_id)) { +found: + /* open the file and return its handle */ + + file = os_file_create_simple_no_error_handling( + 0, real_name, + OS_FILE_OPEN, OS_FILE_READ_WRITE, false, success); + + if (!*success) { + msg("xtrabackup: Cannot open file %s\n", real_name); + } +exit: + log_mutex_exit(); + return file; } /* remember space name for further reference */ table = static_cast - (ut_malloc(sizeof(xb_filter_entry_t) + + (malloc(sizeof(xb_filter_entry_t) + strlen(dest_space_name) + 1)); table->name = ((char*)table) + sizeof(xb_filter_entry_t); @@ -4552,7 +4011,8 @@ xb_delta_open_matching_space( mutex_exit(&fil_system->mutex); if (fil_space != NULL) { - if (fil_space->id == space_id || space_id == ULINT_UNDEFINED) { + if (fil_space->id == info.space_id + || info.space_id == ULINT_UNDEFINED) { /* we found matching space */ goto found; } else { @@ -4565,8 +4025,10 @@ xb_delta_open_matching_space( msg("xtrabackup: Renaming %s to %s.ibd\n", fil_space->name, tmpname); - if (!fil_rename_tablespace(NULL, fil_space->id, - tmpname, NULL)) + if (!fil_rename_tablespace( + fil_space->id, + fil_space->chain.start->name, + tmpname, NULL)) { msg("xtrabackup: Cannot rename %s to %s\n", fil_space->name, tmpname); @@ -4575,14 +4037,14 @@ xb_delta_open_matching_space( } } - if (space_id == ULINT_UNDEFINED) + if (info.space_id == ULINT_UNDEFINED) { msg("xtrabackup: Error: Cannot handle DDL operation on tablespace " "%s\n", dest_space_name); exit(EXIT_FAILURE); } mutex_enter(&fil_system->mutex); - fil_space = fil_space_get_by_id(space_id); + fil_space = fil_space_get_by_id(info.space_id); mutex_exit(&fil_system->mutex); if (fil_space != NULL) { char tmpname[FN_REFLEN]; @@ -4592,7 +4054,9 @@ xb_delta_open_matching_space( msg("xtrabackup: Renaming %s to %s\n", fil_space->name, dest_space_name); - if (!fil_rename_tablespace(NULL, fil_space->id, tmpname, + if (!fil_rename_tablespace(fil_space->id, + fil_space->chain.start->name, + tmpname, NULL)) { msg("xtrabackup: Cannot rename %s to %s\n", @@ -4604,49 +4068,30 @@ xb_delta_open_matching_space( } /* No matching space found. create the new one. */ + const ulint flags = info.page_size.is_compressed() + ? get_bit_shift(info.page_size.physical() + >> (UNIV_ZIP_SIZE_SHIFT_MIN - 1)) + << FSP_FLAGS_POS_ZIP_SSIZE + | FSP_FLAGS_MASK_POST_ANTELOPE + | FSP_FLAGS_MASK_ATOMIC_BLOBS + | (info.page_size.logical() == UNIV_PAGE_SIZE_ORIG + ? 0 + : get_bit_shift(info.page_size.logical() + >> (UNIV_ZIP_SIZE_SHIFT_MIN - 1)) + << FSP_FLAGS_POS_PAGE_SSIZE) + : FSP_FLAGS_PAGE_SSIZE(); + ut_ad(page_size_t(flags).equals_to(info.page_size)); - if (!fil_space_create(dest_space_name, space_id, 0, - FIL_TABLESPACE, 0, false)) { - msg("xtrabackup: Cannot create tablespace %s\n", - dest_space_name); - goto exit; - } - - /* Calculate correct tablespace flags for compressed tablespaces. */ - if (!zip_size || zip_size == ULINT_UNDEFINED) { - tablespace_flags = 0; - } - else { - tablespace_flags - = (get_bit_shift(zip_size >> PAGE_ZIP_MIN_SIZE_SHIFT - << 1) - << DICT_TF_ZSSIZE_SHIFT) - | DICT_TF_COMPACT - | (DICT_TF_FORMAT_ZIP << DICT_TF_FORMAT_SHIFT); - ut_a(dict_tf_get_zip_size(tablespace_flags) - == zip_size); - } - *success = xb_space_create_file(real_name, space_id, tablespace_flags, - &file); - goto exit; - -found: - /* open the file and return it's handle */ - - file = os_file_create_simple_no_error_handling(0, real_name, - OS_FILE_OPEN, - OS_FILE_READ_WRITE, - &ok,0); - - if (ok) { - *success = TRUE; + if (fil_space_create(dest_space_name, info.space_id, flags, + FIL_TYPE_TABLESPACE, 0)) { + *success = xb_space_create_file(real_name, info.space_id, + flags, &file); } else { - msg("xtrabackup: Cannot open file %s\n", real_name); + msg("xtrabackup: Cannot create tablespace %s\n", + dest_space_name); } -exit: - - return file; + goto exit; } /************************************************************************ @@ -4667,13 +4112,13 @@ xtrabackup_apply_delta( char dst_path[FN_REFLEN]; char meta_path[FN_REFLEN]; char space_name[FN_REFLEN]; - ibool success; + bool success; ibool last_buffer = FALSE; ulint page_in_buffer; ulint incremental_buffers = 0; - xb_delta_info_t info; + xb_delta_info_t info(univ_page_size, SRV_TMP_SPACE_ID); ulint page_size; ulint page_size_shift; byte* incremental_buffer_base = NULL; @@ -4703,15 +4148,15 @@ xtrabackup_apply_delta( goto error; } - srv_normalize_path_for_win(dst_path); - srv_normalize_path_for_win(src_path); - srv_normalize_path_for_win(meta_path); + os_normalize_path(dst_path); + os_normalize_path(src_path); + os_normalize_path(meta_path); if (!xb_read_delta_metadata(meta_path, &info)) { goto error; } - page_size = info.page_size; + page_size = info.page_size.physical(); page_size_shift = get_bit_shift(page_size); msg("xtrabackup: page size for %s is %lu bytes\n", src_path, page_size); @@ -4722,10 +4167,9 @@ xtrabackup_apply_delta( goto error; } - src_file = os_file_create_simple_no_error_handling(0, src_path, - OS_FILE_OPEN, - OS_FILE_READ_WRITE, - &success,0); + src_file = os_file_create_simple_no_error_handling( + 0, src_path, + OS_FILE_OPEN, OS_FILE_READ_WRITE, false, &success); if (!success) { os_file_get_last_error(TRUE); msg("xtrabackup: error: cannot open %s\n", src_path); @@ -4737,7 +4181,7 @@ xtrabackup_apply_delta( os_file_set_nocache(src_file, src_path, "OPEN"); dst_file = xb_delta_open_matching_space( - dbname, space_name, info.space_id, info.zip_size, + dbname, space_name, info, dst_path, sizeof(dst_path), &success); if (!success) { msg("xtrabackup: error: cannot open %s\n", dst_path); @@ -4750,8 +4194,7 @@ xtrabackup_apply_delta( /* allocate buffer for incremental backup (4096 pages) */ incremental_buffer_base = static_cast - (ut_malloc((page_size / 4 + 1) * - page_size)); + (malloc((page_size / 4 + 1) * page_size)); incremental_buffer = static_cast (ut_align(incremental_buffer_base, page_size)); @@ -4765,8 +4208,8 @@ xtrabackup_apply_delta( /* first block of block cluster */ offset = ((incremental_buffers * (page_size / 4)) << page_size_shift); - success = os_file_read(src_file, incremental_buffer, - offset, page_size); + success = os_file_read(IORequestRead, src_file, + incremental_buffer, offset, page_size); if (!success) { goto error; } @@ -4784,6 +4227,9 @@ xtrabackup_apply_delta( goto error; } + /* FIXME: If the .delta modifies FSP_SIZE on page 0, + extend the file to that size. */ + for (page_in_buffer = 1; page_in_buffer < page_size / 4; page_in_buffer++) { if (mach_read_from_4(incremental_buffer + page_in_buffer * 4) @@ -4794,8 +4240,9 @@ xtrabackup_apply_delta( ut_a(last_buffer || page_in_buffer == page_size / 4); /* read whole of the cluster */ - success = os_file_read(src_file, incremental_buffer, - offset, page_in_buffer * page_size); + success = os_file_read(IORequestRead, src_file, + incremental_buffer, + offset, page_in_buffer * page_size); if (!success) { goto error; } @@ -4812,7 +4259,8 @@ xtrabackup_apply_delta( if (offset_on_page == 0xFFFFFFFFUL) break; - success = os_file_write(dst_path, dst_file, + success = os_file_write(IORequestWrite, + dst_path, dst_file, incremental_buffer + page_in_buffer * page_size, (offset_on_page << @@ -4826,20 +4274,18 @@ xtrabackup_apply_delta( incremental_buffers++; } - if (incremental_buffer_base) - ut_free(incremental_buffer_base); - if (src_file != XB_FILE_UNDEFINED) + free(incremental_buffer_base); + if (src_file != OS_FILE_CLOSED) os_file_close(src_file); - if (dst_file != XB_FILE_UNDEFINED) + if (dst_file != OS_FILE_CLOSED) os_file_close(dst_file); return TRUE; error: - if (incremental_buffer_base) - ut_free(incremental_buffer_base); - if (src_file != XB_FILE_UNDEFINED) + free(incremental_buffer_base); + if (src_file != OS_FILE_CLOSED) os_file_close(src_file); - if (dst_file != XB_FILE_UNDEFINED) + if (dst_file != OS_FILE_CLOSED) os_file_close(dst_file); msg("xtrabackup: Error: xtrabackup_apply_delta(): " "failed to apply %s to %s.\n", src_path, dst_path); @@ -4900,9 +4346,7 @@ xb_process_datadir( const char* path, /*!name != 0); - len = (ib_uint32_t)strlen(table->name) + 1; + const char* table_name = table->name.m_name; + ut_a(table_name != 0); + len = (ib_uint32_t)strlen(table_name) + 1; /* Write the table name. */ mach_write_to_4(value, len); if (fwrite(&value, 1, sizeof(value), file) != sizeof(value) - || fwrite(table->name, 1, len, file) != len) { + || fwrite(table_name, 1, len, file) != len) { msg("xtrabackup: Error: writing table name."); @@ -5453,48 +4825,38 @@ static void innodb_free_param() { - srv_free_paths_and_sizes(); - free(internal_innobase_data_file_path); - internal_innobase_data_file_path = NULL; + srv_sys_space.shutdown(); free_tmpdir(&mysql_tmpdir_list); } -/************************************************************************** -Store the current binary log coordinates in a specified file. -@return 'false' on error. */ +/** Store the current binary log coordinates in a specified file. +@param[in] filename file name +@param[in] name binary log file name +@param[in] pos binary log file position +@return whether the operation succeeded */ static bool -store_binlog_info( -/*==============*/ - const char *filename) /*!< in: output file name */ +store_binlog_info(const char* filename, const char* name, ulonglong pos) { - FILE *fp; - - if (trx_sys_mysql_bin_log_name[0] == '\0') { - return(true); - } - - fp = fopen(filename, "w"); + FILE *fp = fopen(filename, "w"); if (!fp) { msg("xtrabackup: failed to open '%s'\n", filename); return(false); } - fprintf(fp, "%s\t" UINT64PF "\n", - trx_sys_mysql_bin_log_name, trx_sys_mysql_bin_log_pos); + fprintf(fp, "%s\t%llu\n", name, pos); fclose(fp); return(true); } -static void -xtrabackup_prepare_func(int argc, char ** argv) +/** Implement --prepare +@return whether the operation succeeded */ +static bool +xtrabackup_prepare_func(char** argv) { - ulint err; datafiles_iter_t *it; - fil_node_t *node; - fil_space_t *space; char metadata_path[FN_REFLEN]; /* cd to target-dir */ @@ -5503,10 +4865,11 @@ xtrabackup_prepare_func(int argc, char ** argv) { msg("xtrabackup: cannot my_setwd %s\n", xtrabackup_real_target_dir); - exit(EXIT_FAILURE); + return(false); } msg("xtrabackup: cd to %s\n", xtrabackup_real_target_dir); + int argc; for (argc = 0; argv[argc]; argc++) {} encryption_plugin_prepare_init(argc, argv); xtrabackup_target_dir= mysql_data_home_buff; @@ -5522,116 +4885,100 @@ xtrabackup_prepare_func(int argc, char ** argv) if (!xtrabackup_read_metadata(metadata_path)) { msg("xtrabackup: Error: failed to read metadata from '%s'\n", metadata_path); - exit(EXIT_FAILURE); + return(false); } if (!strcmp(metadata_type, "full-backuped")) { + if (xtrabackup_incremental) { + msg("xtrabackup: error: applying incremental backup " + "needs a prepared target.\n"); + return(false); + } msg("xtrabackup: This target seems to be not prepared yet.\n"); } else if (!strcmp(metadata_type, "log-applied")) { - msg("xtrabackup: This target seems to be already " - "prepared with --apply-log-only.\n"); - goto skip_check; - } else if (!strcmp(metadata_type, "full-prepared")) { msg("xtrabackup: This target seems to be already prepared.\n"); } else { - msg("xtrabackup: This target seems not to have correct " - "metadata...\n"); - exit(EXIT_FAILURE); + msg("xtrabackup: This target does not have correct metadata.\n"); + return(false); } - if (xtrabackup_incremental) { - msg("xtrabackup: error: applying incremental backup " - "needs target prepared with --apply-log-only.\n"); - exit(EXIT_FAILURE); - } -skip_check: - if (xtrabackup_incremental - && metadata_to_lsn != incremental_lsn) { + bool ok = !xtrabackup_incremental + || metadata_to_lsn == incremental_lsn; + if (!ok) { msg("xtrabackup: error: This incremental backup seems " "not to be proper for the target.\n" "xtrabackup: Check 'to_lsn' of the target and " "'from_lsn' of the incremental.\n"); - exit(EXIT_FAILURE); + return(false); } - /* Create logfiles for recovery from 'xtrabackup_logfile', before start InnoDB */ srv_max_n_threads = 1000; + srv_undo_logs = 1; srv_n_purge_threads = 1; - ut_mem_init(); - /* temporally dummy value to avoid crash */ - srv_page_size_shift = 14; - srv_page_size = (1 << srv_page_size_shift); - os_sync_init(); - sync_init(); - os_io_init_simple(); - mem_init(srv_mem_pool_size); - ut_crc32_init(); - -#ifdef WITH_INNODB_DISALLOW_WRITES - srv_allow_writes_event = os_event_create(); - os_event_set(srv_allow_writes_event); -#endif xb_filters_init(); - if (xtrabackup_init_temp_log()) - goto error_cleanup; - - if(innodb_init_param()) { - goto error_cleanup; - } - - xb_normalize_init_values(); + srv_log_group_home_dir = NULL; + srv_thread_concurrency = 1; if (xtrabackup_incremental) { - err = xb_data_files_init(); + if (innodb_init_param()) { +error_cleanup: + xb_filters_free(); + return(false); + } + + xb_normalize_init_values(); + sync_check_init(); + ut_d(sync_check_enable()); + ut_crc32_init(); + recv_sys_init(); + log_sys_init(); + recv_recovery_on = true; + +#ifdef WITH_INNODB_DISALLOW_WRITES + srv_allow_writes_event = os_event_create(0); + os_event_set(srv_allow_writes_event); +#endif + + dberr_t err = xb_data_files_init(); if (err != DB_SUCCESS) { msg("xtrabackup: error: xb_data_files_init() failed " - "with error code %lu\n", err); + "with error %s\n", ut_strerr(err)); goto error_cleanup; } - } - if (xtrabackup_incremental) { + inc_dir_tables_hash = hash_create(1000); - if(!xtrabackup_apply_deltas()) { - xb_data_files_close(); - xb_filter_hash_free(inc_dir_tables_hash); - goto error_cleanup; - } - } - if (xtrabackup_incremental) { - xb_data_files_close(); - } - if (xtrabackup_incremental) { - /* Cleanup datadir from tablespaces deleted between full and - incremental backups */ + ok = xtrabackup_apply_deltas(); - xb_process_datadir("./", ".ibd", rm_if_not_found, NULL); + xb_data_files_close(); + + if (ok) { + /* Cleanup datadir from tablespaces deleted + between full and incremental backups */ + + xb_process_datadir("./", ".ibd", rm_if_not_found); + } xb_filter_hash_free(inc_dir_tables_hash); - } - if (fil_system) { + fil_close(); +#ifdef WITH_INNODB_DISALLOW_WRITES + os_event_destroy(srv_allow_writes_event); +#endif + innodb_free_param(); + log_shutdown(); + sync_check_close(); + if (!ok) goto error_cleanup; } - mem_close(); - ut_free_all_mem(); - - innodb_free_param(); - sync_close(); - sync_initialized = FALSE; - - /* Reset the configuration as it might have been changed by - xb_data_files_init(). */ - if(innodb_init_param()) { + if (innodb_init_param()) { goto error_cleanup; } - srv_apply_log_only = (bool) xtrabackup_apply_log_only; - /* increase IO threads */ - if(srv_n_file_io_threads < 10) { + if (srv_n_file_io_threads < 10) { srv_n_read_io_threads = 4; srv_n_write_io_threads = 4; } @@ -5646,85 +4993,51 @@ skip_check: srv_max_dirty_pages_pct_lwm = srv_max_buf_pool_modified_pct; } - if(innodb_init()) + if (innodb_init()) { goto error_cleanup; - - if (xtrabackup_incremental) { - - it = datafiles_iter_new(fil_system); - if (it == NULL) { - msg("xtrabackup: Error: datafiles_iter_new() failed.\n"); - exit(EXIT_FAILURE); } - while ((node = datafiles_iter_next(it)) != NULL) { - byte *header; - ulint size; - ulint actual_size; - mtr_t mtr; - buf_block_t *block; - ulint flags; - - space = node->space; - - /* Align space sizes along with fsp header. We want to process - each space once, so skip all nodes except the first one in a - multi-node space. */ - if (UT_LIST_GET_PREV(chain, node) != NULL) { - continue; - } - - mtr_start(&mtr); - - mtr_s_lock(fil_space_get_latch(space->id, &flags), &mtr); - - block = buf_page_get(space->id, - dict_tf_get_zip_size(flags), - 0, RW_S_LATCH, &mtr); - header = FSP_HEADER_OFFSET + buf_block_get_frame(block); - - size = mtr_read_ulint(header + FSP_SIZE, MLOG_4BYTES, - &mtr); - - mtr_commit(&mtr); - - fil_extend_space_to_desired_size(&actual_size, space->id, size); - } - - datafiles_iter_free(it); - - } /* if (xtrabackup_incremental) */ - if (xtrabackup_export) { +#if 1 // FIXME: remove the option or fix the logic + /* In MariaDB 10.2, undo log processing would need the + ability to evaluate indexed virtual columns, and we + have not initialized the necessary infrastructure. */ + msg("xtrabackup: --export does not work!\n"); + ok = false; + } else if (xtrabackup_export) { +#endif msg("xtrabackup: export option is specified.\n"); + + /* To allow subsequent MariaDB server startup independent + of the value of --innodb-log-checksums, + unconditionally enable redo log checksums. */ + log_checksum_algorithm_ptr = log_block_calc_checksum_crc32; + pfs_os_file_t info_file; char info_file_path[FN_REFLEN]; - ibool success; + bool success; char table_name[FN_REFLEN]; byte* page; byte* buf = NULL; - buf = static_cast(ut_malloc(UNIV_PAGE_SIZE * 2)); + buf = static_cast(malloc(UNIV_PAGE_SIZE * 2)); page = static_cast(ut_align(buf, UNIV_PAGE_SIZE)); - /* flush insert buffer at shutdwon */ - innobase_fast_shutdown = 0; - it = datafiles_iter_new(fil_system); if (it == NULL) { msg("xtrabackup: Error: datafiles_iter_new() " "failed.\n"); - exit(EXIT_FAILURE); - } - while ((node = datafiles_iter_next(it)) != NULL) { + ok = false; + } else + while (fil_node_t *node = datafiles_iter_next(it)) { int len; char *next, *prev, *p; dict_table_t* table; dict_index_t* index; ulint n_index; - space = node->space; + const fil_space_t* space = node->space; /* treat file_per_table only */ if (!fil_is_user_tablespace_id(space->id)) { @@ -5779,15 +5092,14 @@ skip_check: "than 31 indexes, .exp file was not " "generated. Table will fail to import " "on server version prior to 5.6.\n", - table->name); + table->name.m_name); goto next_node; } /* init exp file */ - memset(page, 0, UNIV_PAGE_SIZE); - mach_write_to_4(page , 0x78706f72UL); - mach_write_to_4(page + 4, 0x74696e66UL);/*"xportinf"*/ + memcpy(page, "xportinf", 8); mach_write_to_4(page + 8, n_index); + memset(page + 12, 0, UNIV_PAGE_SIZE - 12); strncpy((char *) page + 12, table_name, 500); @@ -5807,7 +5119,7 @@ skip_check: msg("xtrabackup: name=%s, " "id.low=%lu, page=%lu\n", - index->name, + index->name(), (ulint)(index->id & 0xFFFFFFFFUL), (ulint) index->page); @@ -5815,18 +5127,18 @@ skip_check: n_index++; } - srv_normalize_path_for_win(info_file_path); + os_normalize_path(info_file_path); info_file = os_file_create( 0, info_file_path, OS_FILE_OVERWRITE, OS_FILE_NORMAL, OS_DATA_FILE, - &success,0); + false, &success); if (!success) { os_file_get_last_error(TRUE); goto next_node; } - success = os_file_write(info_file_path, + success = os_file_write(IORequestWrite, info_file_path, info_file, page, 0, UNIV_PAGE_SIZE); if (!success) { @@ -5839,76 +5151,73 @@ skip_check: goto next_node; } next_node: - if (info_file != XB_FILE_UNDEFINED) { + if (info_file != OS_FILE_CLOSED) { os_file_close(info_file); - info_file = XB_FILE_UNDEFINED; + info_file = OS_FILE_CLOSED; } mutex_exit(&(dict_sys->mutex)); } - ut_free(buf); + free(buf); } - /* print the binary log position */ - trx_sys_print_mysql_binlog_offset(); - msg("\n"); + if (ok) { + mtr_t mtr; + mtr.start(); + const trx_sysf_t* sys_header = trx_sysf_get(&mtr); - /* output to xtrabackup_binlog_pos_innodb and (if - backup_safe_binlog_info was available on the server) to - xtrabackup_binlog_info. In the latter case xtrabackup_binlog_pos_innodb - becomes redundant and is created only for compatibility. */ - if (!store_binlog_info("xtrabackup_binlog_pos_innodb") || - (recover_binlog_info && - !store_binlog_info(XTRABACKUP_BINLOG_INFO))) { + if (mach_read_from_4(TRX_SYS_MYSQL_LOG_INFO + + TRX_SYS_MYSQL_LOG_MAGIC_N_FLD + + sys_header) + == TRX_SYS_MYSQL_LOG_MAGIC_N) { + ulonglong pos = mach_read_from_8( + TRX_SYS_MYSQL_LOG_INFO + + TRX_SYS_MYSQL_LOG_OFFSET + + sys_header); + const char* name = reinterpret_cast( + TRX_SYS_MYSQL_LOG_INFO + TRX_SYS_MYSQL_LOG_NAME + + sys_header); + msg("Last binlog file %s, position %llu\n", name, pos); - exit(EXIT_FAILURE); + /* output to xtrabackup_binlog_pos_innodb and + (if backup_safe_binlog_info was available on + the server) to xtrabackup_binlog_info. In the + latter case xtrabackup_binlog_pos_innodb + becomes redundant and is created only for + compatibility. */ + ok = store_binlog_info( + "xtrabackup_binlog_pos_innodb", name, pos) + && (!recover_binlog_info || store_binlog_info( + XTRABACKUP_BINLOG_INFO, + name, pos)); + } + + mtr.commit(); } /* Check whether the log is applied enough or not. */ - if ((xtrabackup_incremental - && srv_start_lsn < incremental_to_lsn) - ||(!xtrabackup_incremental - && srv_start_lsn < metadata_to_lsn)) { + const lsn_t target_lsn = xtrabackup_incremental + ? incremental_to_lsn : metadata_to_lsn; + if ((srv_start_lsn || fil_space_get(SRV_LOG_SPACE_FIRST_ID)) + && srv_start_lsn < target_lsn) { msg("xtrabackup: error: " - "The transaction log file is corrupted.\n" - "xtrabackup: error: " - "The log was not applied to the intended LSN!\n"); - msg("xtrabackup: Log applied to lsn " LSN_PF "\n", - srv_start_lsn); - if (xtrabackup_incremental) { - msg("xtrabackup: The intended lsn is " LSN_PF "\n", - incremental_to_lsn); - } else { - msg("xtrabackup: The intended lsn is " LSN_PF "\n", - metadata_to_lsn); - } - exit(EXIT_FAILURE); + "The log was only applied up to LSN " LSN_PF + ", instead of " LSN_PF "\n", + srv_start_lsn, target_lsn); + ok = false; } #ifdef WITH_WSREP - xb_write_galera_info(xtrabackup_incremental); + else if (ok) xb_write_galera_info(xtrabackup_incremental); #endif - innodb_end(); - - innodb_free_param(); - - sync_initialized = FALSE; - - /* re-init necessary components */ - ut_mem_init(); - os_sync_init(); - sync_init(); - os_io_init_simple(); - - if(xtrabackup_close_temp_log(TRUE)) - exit(EXIT_FAILURE); + innodb_shutdown(); + innodb_free_param(); /* output to metadata file */ - { + if (ok) { char filename[FN_REFLEN]; - strcpy(metadata_type, srv_apply_log_only ? - "log-applied" : "full-prepared"); + strcpy(metadata_type, "log-applied"); if(xtrabackup_incremental && metadata_to_lsn < incremental_to_lsn) @@ -5922,73 +5231,21 @@ next_node: msg("xtrabackup: Error: failed to write metadata " "to '%s'\n", filename); - exit(EXIT_FAILURE); - } - - if(xtrabackup_extra_lsndir) { + ok = false; + } else if (xtrabackup_extra_lsndir) { sprintf(filename, "%s/%s", xtrabackup_extra_lsndir, XTRABACKUP_METADATA_FILENAME); if (!xtrabackup_write_metadata(filename)) { msg("xtrabackup: Error: failed to write " "metadata to '%s'\n", filename); - exit(EXIT_FAILURE); + ok = false; } } } - if (!apply_log_finish()) { - exit(EXIT_FAILURE); - } - - sync_close(); - sync_initialized = FALSE; - if (fil_system) { - fil_close(); - } - - ut_free_all_mem(); - - /* start InnoDB once again to create log files */ - - if (!xtrabackup_apply_log_only) { - - /* xtrabackup_incremental_dir is used to indicate that - we are going to apply incremental backup. Here we already - applied incremental backup and are about to do final prepare - of the full backup */ - xtrabackup_incremental_dir = NULL; - - if(innodb_init_param()) { - goto error; - } - - srv_apply_log_only = false; - - /* increase IO threads */ - if(srv_n_file_io_threads < 10) { - srv_n_read_io_threads = 4; - srv_n_write_io_threads = 4; - } - - srv_shutdown_state = SRV_SHUTDOWN_NONE; - - if(innodb_init()) - goto error; - - innodb_end(); - innodb_free_param(); - - } + if (ok) ok = apply_log_finish(); xb_filters_free(); - - return; - -error_cleanup: - xtrabackup_close_temp_log(FALSE); - xb_filters_free(); - -error: - exit(EXIT_FAILURE); + return ok; } /************************************************************************** @@ -6081,7 +5338,8 @@ extern void init_signals(void); /* Messages . Avoid loading errmsg.sys file */ void setup_error_messages() { - static const char *all_msgs[ER_ERROR_LAST - ER_ERROR_FIRST +1]; + static const char *my_msgs[ERRORS_PER_RANGE]; + static const char **all_msgs[] = { my_msgs, my_msgs, my_msgs, my_msgs }; my_default_lc_messages = &my_locale_en_US; my_default_lc_messages->errmsgs->errmsgs = all_msgs; @@ -6108,24 +5366,20 @@ void setup_error_messages() }; for (int i = 0; i < (int)array_elements(all_msgs); i++) - all_msgs[i] = "Unknown error"; + all_msgs[0][i] = "Unknown error"; for (int i = 0; i < (int)array_elements(xb_msgs); i++) - all_msgs[xb_msgs[i].id - ER_ERROR_FIRST] = xb_msgs[i].fmt; + all_msgs[0][xb_msgs[i].id - ER_ERROR_FIRST] = xb_msgs[i].fmt; } -extern my_bool(*dict_check_if_skip_table)(const char* name) ; - void handle_options(int argc, char **argv, char ***argv_client, char ***argv_server) { /* Setup some variables for Innodb.*/ - srv_xtrabackup = true; - + srv_operation = SRV_OPERATION_RESTORE; files_charset_info = &my_charset_utf8_general_ci; - dict_check_if_skip_table = check_if_skip_table; setup_error_messages(); sys_var_init(); @@ -6309,25 +5563,21 @@ handle_options(int argc, char **argv, char ***argv_client, char ***argv_server) } } -/* ================= main =================== */ -extern my_bool(*fil_check_if_skip_database_by_path)(const char* name); +static int main_low(char** argv); +/* ================= main =================== */ int main(int argc, char **argv) { char **client_defaults, **server_defaults; - char cwd[FN_REFLEN]; - static char INNOBACKUPEX_EXE[]= "innobackupex"; + static char INNOBACKUPEX_EXE[]= "innobackupex"; if (argc > 1 && (strcmp(argv[1], "--innobackupex") == 0)) { argv++; argc--; - argv[0] = INNOBACKUPEX_EXE; + argv[0] = INNOBACKUPEX_EXE; innobackupex_mode = true; } - /* Setup skip fil_load_single_tablespaces callback.*/ - fil_check_if_skip_database_by_path = check_if_skip_database_by_path; - init_signals(); MY_INIT(argv[0]); @@ -6346,30 +5596,69 @@ int main(int argc, char **argv) system_charset_info = &my_charset_utf8_general_ci; key_map_full.set_all(); + logger.init_base(); + logger.set_handlers(LOG_FILE, LOG_NONE, LOG_NONE); + mysql_mutex_init(key_LOCK_error_log, &LOCK_error_log, + MY_MUTEX_INIT_FAST); + handle_options(argc, argv, &client_defaults, &server_defaults); - int argc_server; - for (argc_server = 0; server_defaults[argc_server]; argc_server++) {} +#ifndef DBUG_OFF + if (dbug_option) { + DBUG_SET_INITIAL(dbug_option); + DBUG_SET(dbug_option); + } +#endif - int argc_client; - for (argc_client = 0; client_defaults[argc_client]; argc_client++) {} + int status = main_low(server_defaults); + backup_cleanup(); + if (innobackupex_mode) { + ibx_cleanup(); + } + + free_defaults(client_defaults); + free_defaults(server_defaults); + +#ifndef DBUG_OFF + if (dbug_option) { + DBUG_END(); + } +#endif + + if (THR_THD) + (void) pthread_key_delete(THR_THD); + + logger.cleanup_base(); + mysql_mutex_destroy(&LOCK_error_log); + + if (status == EXIT_SUCCESS) { + msg_ts("completed OK!\n"); + } + + return status; +} + +static int main_low(char** argv) +{ if (innobackupex_mode) { if (!ibx_init()) { - exit(EXIT_FAILURE); + return(EXIT_FAILURE); } } - if ((!xtrabackup_print_param) && (!xtrabackup_prepare) && (strcmp(mysql_data_home, "./") == 0)) { + if (!xtrabackup_print_param && !xtrabackup_prepare + && !strcmp(mysql_data_home, "./")) { if (!xtrabackup_print_param) usage(); msg("\nxtrabackup: Error: Please set parameter 'datadir'\n"); - exit(EXIT_FAILURE); + return(EXIT_FAILURE); } /* Expand target-dir, incremental-basedir, etc. */ + char cwd[FN_REFLEN]; my_getwd(cwd, sizeof(cwd), MYF(0)); my_load_path(xtrabackup_real_target_dir, @@ -6435,7 +5724,7 @@ int main(int argc, char **argv) if (error) { msg("xtrabackup: value '%s' may be wrong format for " "incremental option.\n", xtrabackup_incremental); - exit(EXIT_FAILURE); + return(EXIT_FAILURE); } } else if (xtrabackup_backup && xtrabackup_incremental_basedir) { char filename[FN_REFLEN]; @@ -6445,7 +5734,7 @@ int main(int argc, char **argv) if (!xtrabackup_read_metadata(filename)) { msg("xtrabackup: error: failed to read metadata from " "%s\n", filename); - exit(EXIT_FAILURE); + return(EXIT_FAILURE); } incremental_lsn = metadata_to_lsn; @@ -6458,7 +5747,7 @@ int main(int argc, char **argv) if (!xtrabackup_read_metadata(filename)) { msg("xtrabackup: error: failed to read metadata from " "%s\n", filename); - exit(EXIT_FAILURE); + return(EXIT_FAILURE); } incremental_lsn = metadata_from_lsn; @@ -6475,15 +5764,13 @@ int main(int argc, char **argv) } if (!xb_init()) { - exit(EXIT_FAILURE); + return(EXIT_FAILURE); } /* --print-param */ if (xtrabackup_print_param) { - printf("%s", print_param_str.str().c_str()); - - exit(EXIT_SUCCESS); + return(EXIT_SUCCESS); } print_version(); @@ -6509,7 +5796,7 @@ int main(int argc, char **argv) if (xtrabackup_decrypt_decompress) num++; if (num != 1) { /* !XOR (for now) */ usage(); - exit(EXIT_FAILURE); + return(EXIT_FAILURE); } } @@ -6520,41 +5807,28 @@ int main(int argc, char **argv) #endif /* --backup */ - if (xtrabackup_backup) - xtrabackup_backup_func(); + if (xtrabackup_backup && !xtrabackup_backup_func()) { + return(EXIT_FAILURE); + } /* --prepare */ - if (xtrabackup_prepare) { - xtrabackup_prepare_func(argc_server, server_defaults); + if (xtrabackup_prepare + && !xtrabackup_prepare_func(argv)) { + return(EXIT_FAILURE); } if (xtrabackup_copy_back || xtrabackup_move_back) { if (!check_if_param_set("datadir")) { msg("Error: datadir must be specified.\n"); - exit(EXIT_FAILURE); + return(EXIT_FAILURE); } if (!copy_back()) - exit(EXIT_FAILURE); + return(EXIT_FAILURE); } if (xtrabackup_decrypt_decompress && !decrypt_decompress()) { - exit(EXIT_FAILURE); + return(EXIT_FAILURE); } - backup_cleanup(); - - if (innobackupex_mode) { - ibx_cleanup(); - } - - - free_defaults(client_defaults); - free_defaults(server_defaults); - - if (THR_THD) - (void) pthread_key_delete(THR_THD); - - msg_ts("completed OK!\n"); - - exit(EXIT_SUCCESS); + return(EXIT_SUCCESS); } diff --git a/extra/mariabackup/xtrabackup.h b/extra/mariabackup/xtrabackup.h index eafc848fd43..d1d1e8df29a 100644 --- a/extra/mariabackup/xtrabackup.h +++ b/extra/mariabackup/xtrabackup.h @@ -26,26 +26,14 @@ Foundation, Inc., 51 Franklin Street, Fifth Floor, Boston, MA 02110-1301, USA #include "xbstream.h" #include "changed_page_bitmap.h" -#ifdef __WIN__ -#define XB_FILE_UNDEFINED INVALID_HANDLE_VALUE -#else -#define XB_FILE_UNDEFINED (-1) -#endif +struct xb_delta_info_t +{ + xb_delta_info_t(page_size_t page_size, ulint space_id) + : page_size(page_size), space_id(space_id) {} -typedef struct { - ulint page_size; - ulint zip_size; - ulint space_id; -} xb_delta_info_t; - -/* ======== Datafiles iterator ======== */ -typedef struct { - fil_system_t *system; - fil_space_t *space; - fil_node_t *node; - ibool started; - os_ib_mutex_t mutex; -} datafiles_iter_t; + page_size_t page_size; + ulint space_id; +}; /* value of the --incremental option */ extern lsn_t incremental_lsn; @@ -84,15 +72,11 @@ extern ibool xtrabackup_compress; extern my_bool xtrabackup_backup; extern my_bool xtrabackup_prepare; -extern my_bool xtrabackup_apply_log_only; extern my_bool xtrabackup_copy_back; extern my_bool xtrabackup_move_back; extern my_bool xtrabackup_decrypt_decompress; extern char *innobase_data_file_path; -extern char *innobase_doublewrite_file; -extern longlong innobase_log_file_size; -extern long innobase_log_files_in_group; extern longlong innobase_page_size; extern int xtrabackup_parallel; @@ -108,9 +92,7 @@ extern "C"{ } #endif extern my_bool xtrabackup_export; -extern char *xtrabackup_incremental_basedir; extern char *xtrabackup_extra_lsndir; -extern char *xtrabackup_incremental_dir; extern ulint xtrabackup_log_copy_interval; extern char *xtrabackup_stream_str; extern long xtrabackup_throttle; @@ -125,7 +107,6 @@ extern my_bool opt_force_non_empty_dirs; extern my_bool opt_noversioncheck; extern my_bool opt_no_backup_locks; extern my_bool opt_decompress; -extern my_bool opt_remove_original; extern char *opt_incremental_history_name; extern char *opt_incremental_history_uuid; @@ -136,7 +117,6 @@ extern char *opt_host; extern char *opt_defaults_group; extern char *opt_socket; extern uint opt_port; -extern char *opt_login_path; extern char *opt_log_bin; extern const char *query_type_names[]; @@ -166,14 +146,9 @@ void xtrabackup_io_throttling(void); my_bool xb_write_delta_metadata(const char *filename, const xb_delta_info_t *info); -datafiles_iter_t *datafiles_iter_new(fil_system_t *f_system); -fil_node_t *datafiles_iter_next(datafiles_iter_t *it); -void datafiles_iter_free(datafiles_iter_t *it); - -/*********************************************************************** -Reads the space flags from a given data file and returns the compressed -page size, or 0 if the space is not compressed. */ -ulint xb_get_zip_size(pfs_os_file_t file); +/** @return the tablespace flags from a given data file +@retval ULINT_UNDEFINED if the file is not readable */ +ulint xb_get_space_flags(pfs_os_file_t file); /************************************************************************ Checks if a table specified as a name in the form "database/name" (InnoDB 5.6) @@ -204,17 +179,10 @@ bool check_if_param_set(const char *param); #if defined(HAVE_OPENSSL) -extern my_bool opt_use_ssl; extern my_bool opt_ssl_verify_server_cert; -#if !defined(HAVE_YASSL) -extern char *opt_server_public_key; -#endif #endif -void -xtrabackup_backup_func(void); - my_bool xb_get_one_option(int optid, const struct my_option *opt __attribute__((unused)), diff --git a/include/my_dir.h b/include/my_dir.h index be988ae790c..949a9a4d354 100644 --- a/include/my_dir.h +++ b/include/my_dir.h @@ -35,9 +35,16 @@ extern "C" { #define MY_S_ISUID S_ISUID /* set user id on execution */ #define MY_S_ISGID S_ISGID /* set group id on execution */ #define MY_S_ISVTX S_ISVTX /* save swapped text even after use */ -#define MY_S_IREAD S_IREAD /* read permission, owner */ -#define MY_S_IWRITE S_IWRITE /* write permission, owner */ -#define MY_S_IEXEC S_IEXEC /* execute/search permission, owner */ + +#ifndef S_IREAD +#define MY_S_IREAD S_IRUSR /* read permission, owner */ +#define MY_S_IWRITE S_IWUSR /* write permission, owner */ +#define MY_S_IEXEC S_IXUSR /* execute/search permission, owner */ +#else +#define MY_S_IREAD S_IREAD /* read permission, owner */ +#define MY_S_IWRITE S_IWRITE /* write permission, owner */ +#define MY_S_IEXEC S_IEXEC /* execute/search permission, owner */ +#endif #define MY_S_ISDIR(m) (((m) & MY_S_IFMT) == MY_S_IFDIR) #define MY_S_ISCHR(m) (((m) & MY_S_IFMT) == MY_S_IFCHR) diff --git a/include/mysql_com.h b/include/mysql_com.h index 6f0a2097f13..1deaa4c771c 100644 --- a/include/mysql_com.h +++ b/include/mysql_com.h @@ -387,7 +387,9 @@ enum enum_indicator_type This status flag, when on, implies that one of the state information has changed on the server because of the execution of the last statement. */ -#define SERVER_SESSION_STATE_CHANGED (1UL << 14) +#define SERVER_SESSION_STATE_CHANGED 16384U + +#define SERVER_STATUS_ANSI_QUOTES 32768U /** Server status flags that must be cleared when starting diff --git a/libmariadb b/libmariadb index ff4bfdf1caf..eb058204233 160000 --- a/libmariadb +++ b/libmariadb @@ -1 +1 @@ -Subproject commit ff4bfdf1caf2e0a47382fdbe6d796d0e63ec1062 +Subproject commit eb058204233863f1b949209e5c4649489174d299 diff --git a/mysql-test/collections/buildbot_suites.bat b/mysql-test/collections/buildbot_suites.bat index f91692d2918..89e68919c7f 100644 --- a/mysql-test/collections/buildbot_suites.bat +++ b/mysql-test/collections/buildbot_suites.bat @@ -2,4 +2,5 @@ perl mysql-test-run.pl --verbose-restart --force --testcase-timeout=45 --suite-t main,^ innodb,^ plugins,^ +mariabackup,^ rocksdb diff --git a/mysql-test/include/binlog_parallel_replication_marks.test b/mysql-test/include/binlog_parallel_replication_marks.test new file mode 100644 index 00000000000..b915d26ce99 --- /dev/null +++ b/mysql-test/include/binlog_parallel_replication_marks.test @@ -0,0 +1,81 @@ +# Test the markings on GTID events (ddl, waited, trans, +# @@skip_parallel_replication) that are used to control parallel +# replication on the slave. + +--source include/have_innodb.inc + +RESET MASTER; +--source include/wait_for_binlog_checkpoint.inc + +CREATE TABLE t1 (a INT PRIMARY KEY, b INT) ENGINE=InnoDB; +--let $binlog_pos1=query_get_value(SHOW MASTER STATUS, Position, 1) +/* GTID */ INSERT INTO t1 VALUES (1,0); +/* GTID */ BEGIN; +/* GTID */ INSERT INTO t1 VALUES (2,0); +/* GTID */ ALTER TABLE t1 ADD c INT; +/* GTID */ INSERT INTO t1 VALUES (3,0,0); +/* GTID */ COMMIT; +/* GTID */ BEGIN; +/* GTID */ UPDATE t1 SET b=1, c=1 WHERE a=2; +/* GTID */ CREATE TEMPORARY TABLE t2 (a INT PRIMARY KEY, b INT) ENGINE=InnoDB; +/* GTID */ INSERT INTO t2 VALUES (4,10), (5,20); +/* GTID */ INSERT INTO t1 SELECT a, 2, b FROM t2; +/* GTID */ DROP TEMPORARY TABLE t2; +/* GTID */ INSERT INTO t1 VALUES (6, 3, 0); +/* GTID */ COMMIT; +/* GTID */ CREATE TEMPORARY TABLE t3 (a INT PRIMARY KEY) ENGINE=InnoDB; +/* GTID */ BEGIN; +/* GTID */ DELETE FROM t1 WHERE a=5; +/* GTID */ INSERT INTO t3 VALUES (7); +/* GTID */ INSERT INTO t1 SELECT a, 4, 0 FROM t3; +/* GTID */ UPDATE t1 SET c=1 WHERE a=7; +/* GTID */ DROP TEMPORARY TABLE t3; +/* GTID */ COMMIT; +/* GTID */ CREATE TEMPORARY TABLE t4 (a INT PRIMARY KEY) ENGINE=InnoDB; +/* GTID */ BEGIN; +/* GTID */ INSERT INTO t1 VALUES (8, 5, 0); +/* GTID */ ALTER TABLE t4 ADD b INT; +/* GTID */ INSERT INTO t1 VALUES (9, 5, 1); +/* GTID */ COMMIT; +connect (tmp_con,localhost,root,,); +/* GTID */ INSERT INTO t1 VALUES (10, 6, 0); +/* GTID */ BEGIN; +/* GTID */ CREATE TEMPORARY TABLE t5 (a INT PRIMARY KEY) ENGINE=InnoDB; +/* GTID */ INSERT INTO t1 VALUES (11, 7, 0); +/* GTID */ COMMIT; +--let $before_drop_pos=query_get_value(SHOW MASTER STATUS, Position, 1) +disconnect tmp_con; +connection default; + +# We need to wait for the implicit DROP TEMPORARY TABLE to be logged after +# tmp_con disconnect, otherwise we get sporadic test failures. +--let $wait_condition= SELECT variable_value > $before_drop_pos FROM information_schema.global_status WHERE variable_name = 'binlog_snapshot_position' +--source include/wait_condition.inc + +--let $binlog_pos2=query_get_value(SHOW MASTER STATUS, Position, 1) + +--let $binlog_file= query_get_value(SHOW MASTER STATUS, File, 1) +FLUSH LOGS; + +--let $MYSQLD_DATADIR= `select @@datadir` +--let $file= $MYSQLTEST_VARDIR/tmp/binlog_parallel_replication_marks.out +--let OUTPUT_FILE=$file +exec $MYSQL_BINLOG --start_position=$binlog_pos1 --stop_position=$binlog_pos2 $MYSQLD_DATADIR/$binlog_file > $file; + +perl; +my $file= $ENV{'OUTPUT_FILE'}; +open F, "<", $file + or die "Unable to open file '$file': $!\n"; +while () { + s/^#\d+ +\d+:\d+:\d+ /# /; + s/GTID \d+-\d+-\d+/GTID #-#-#/; + s/end_log_pos \d+/end_log_pos #/; + s/table id \d+/table id #/; + s/mapped to number \d+/mapped to number #/; + s/CRC32 0x[0-9a-f]+/CRC32 0x########/; + print if /GTID|BEGIN|COMMIT|Table_map|Write_rows|Update_rows|Delete_rows|generated by server|40005 TEMPORARY/; +} +close F; +EOF + +DROP TABLE t1; diff --git a/mysql-test/mysql-test-run.pl b/mysql-test/mysql-test-run.pl index 9e1c72e32ca..0f720c804ab 100755 --- a/mysql-test/mysql-test-run.pl +++ b/mysql-test/mysql-test-run.pl @@ -3074,6 +3074,7 @@ sub mysql_install_db { my $args; mtr_init_args(\$args); mtr_add_arg($args, "--no-defaults"); + mtr_add_arg($args, "--disable-getopt-prefix-matching"); mtr_add_arg($args, "--bootstrap"); mtr_add_arg($args, "--basedir=%s", $install_basedir); mtr_add_arg($args, "--datadir=%s", $install_datadir); diff --git a/mysql-test/r/alter_table.result b/mysql-test/r/alter_table.result index c88abd9867e..2455bc4f997 100644 --- a/mysql-test/r/alter_table.result +++ b/mysql-test/r/alter_table.result @@ -1925,8 +1925,8 @@ ALTER TABLE ti1 FORCE; affected rows: 0 info: Records: 0 Duplicates: 0 Warnings: 0 ALTER TABLE tm1 FORCE; -affected rows: 0 -info: Records: 0 Duplicates: 0 Warnings: 0 +affected rows: 2 +info: Records: 2 Duplicates: 0 Warnings: 0 ALTER TABLE ti1 AUTO_INCREMENT 3; affected rows: 0 info: Records: 0 Duplicates: 0 Warnings: 0 diff --git a/mysql-test/r/client_xml.result b/mysql-test/r/client_xml.result index 7f74a092af7..6f91c18318c 100644 --- a/mysql-test/r/client_xml.result +++ b/mysql-test/r/client_xml.result @@ -3,7 +3,7 @@ set @@global.concurrent_insert= 0; drop table if exists t1; create table t1 ( `a&b` int, -`ab` text ); insert into t1 values (1, 2, 'a&b ab'); @@ -21,9 +21,9 @@ insert into t1 values (1, 2, 'a&b ab'); - - - + + + diff --git a/mysql-test/r/contributors.result b/mysql-test/r/contributors.result index 4a26d0f19dd..5d92184f191 100644 --- a/mysql-test/r/contributors.result +++ b/mysql-test/r/contributors.result @@ -2,9 +2,11 @@ SHOW CONTRIBUTORS; Name Location Comment Booking.com https://www.booking.com Founding member, Platinum Sponsor of the MariaDB Foundation Alibaba Cloud https://intl.aliyun.com Platinum Sponsor of the MariaDB Foundation +Tencent Cloud https://cloud.tencent.com Platinum Sponsor of the MariaDB Foundation MariaDB Corporation https://mariadb.com Founding member, Gold Sponsor of the MariaDB Foundation Visma https://visma.com Gold Sponsor of the MariaDB Foundation DBS https://dbs.com Gold Sponsor of the MariaDB Foundation +IBM https://www.ibm.com Gold Sponsor of the MariaDB Foundation Nexedi https://www.nexedi.com Silver Sponsor of the MariaDB Foundation Acronis http://www.acronis.com Silver Sponsor of the MariaDB Foundation Auttomattic https://automattic.com Bronze Sponsor of the MariaDB Foundation diff --git a/mysql-test/r/ctype_uca_partitions.result b/mysql-test/r/ctype_uca_partitions.result index 508893522e4..154de148ee0 100644 --- a/mysql-test/r/ctype_uca_partitions.result +++ b/mysql-test/r/ctype_uca_partitions.result @@ -6,7 +6,7 @@ Table Create Table t1 CREATE TABLE `t1` ( `c1` varchar(10) CHARACTER SET utf8 COLLATE utf8_thai_520_w2 DEFAULT NULL ) ENGINE=MyISAM DEFAULT CHARSET=latin1 - PARTITION BY KEY (c1) + PARTITION BY KEY (`c1`) PARTITIONS 3 INSERT INTO t1 VALUES ('A'),('À'),('Á'),('Â'),('Ã'),('Ä'),('Å'); INSERT INTO t1 VALUES ('B'); diff --git a/mysql-test/r/ctype_ucs.result b/mysql-test/r/ctype_ucs.result index 32095ff599b..5b914258515 100644 --- a/mysql-test/r/ctype_ucs.result +++ b/mysql-test/r/ctype_ucs.result @@ -5606,6 +5606,23 @@ SELECT 'a','aa'; a aa a aa # +# MDEV-10306 Wrong results with combination of CONCAT, SUBSTR and CONVERT in subquery +# +SET NAMES utf8, character_set_connection=ucs2; +SET @save_optimizer_switch=@@optimizer_switch; +SET optimizer_switch=_utf8'derived_merge=on'; +CREATE TABLE t1 (t VARCHAR(10) CHARSET latin1); +INSERT INTO t1 VALUES('abcdefghi'); +SET NAMES utf8, character_set_connection=ucs2; +SELECT CONCAT(t2,'-',t2) c2 FROM (SELECT HEX(t) t2 FROM t1) sub; +c2 +616263646566676869-616263646566676869 +SELECT CONCAT(t2,'-',t2) c2 FROM (SELECT TO_BASE64(t) t2 FROM t1) sub; +c2 +YWJjZGVmZ2hp-YWJjZGVmZ2hp +DROP TABLE t1; +SET optimizer_switch=@save_optimizer_switch; +# # End of 10.0 tests # select collation(cast("a" as char(10) unicode binary)); diff --git a/mysql-test/r/drop_debug.result b/mysql-test/r/drop_debug.result deleted file mode 100644 index 852b3ac2163..00000000000 --- a/mysql-test/r/drop_debug.result +++ /dev/null @@ -1,25 +0,0 @@ - -# -- -# -- Bug#43138: DROP DATABASE failure does not clean up message list. -# -- - -DROP DATABASE IF EXISTS mysql_test; - -CREATE DATABASE mysql_test; -CREATE TABLE mysql_test.t1(a INT); -CREATE TABLE mysql_test.t2(b INT); -CREATE TABLE mysql_test.t3(c INT); - -SET SESSION debug_dbug= "+d,bug43138"; - -DROP DATABASE mysql_test; -Warnings: -Error 1051 Unknown table 't1' -Error 1051 Unknown table 't2' -Error 1051 Unknown table 't3' - -SET SESSION debug_dbug= "-d,bug43138"; - -# -- -# -- End of Bug#43138. -# -- diff --git a/mysql-test/r/errors.result b/mysql-test/r/errors.result index a909366a89b..e7a315f4314 100644 --- a/mysql-test/r/errors.result +++ b/mysql-test/r/errors.result @@ -170,3 +170,7 @@ UPDATE t1 SET a = 'new' WHERE COLUMN_CREATE( 1, 'v', 1, 'w' ) IS NULL; ERROR 22007: Illegal value used as argument of dynamic column function drop table t1; +set max_session_mem_used = 8192; +select * from seq_1_to_1000; +Got one of the listed errors +set global max_session_mem_used = default; diff --git a/mysql-test/r/func_concat.result b/mysql-test/r/func_concat.result index 925158ab129..b87ee7bfc52 100644 --- a/mysql-test/r/func_concat.result +++ b/mysql-test/r/func_concat.result @@ -149,3 +149,116 @@ CALL p1(); ########################################40100.000 DROP PROCEDURE p1; # End of 5.1 tests +# +# Start of 10.0 tests +# +# +# MDEV-10306 Wrong results with combination of CONCAT, SUBSTR and CONVERT in subquery +# +SET @save_optimizer_switch=@@optimizer_switch; +SET optimizer_switch='derived_merge=on'; +CREATE TABLE t1 (t VARCHAR(10) CHARSET latin1); +INSERT INTO t1 VALUES('1234567'); +SELECT CONCAT(SUBSTR(t2, 1, 3), SUBSTR(t2, 5)) c1, +CONCAT(SUBSTR(t2,1,3),'---',SUBSTR(t2,5)) c2 +FROM (SELECT CONVERT(t USING latin1) t2 FROM t1) sub; +c1 c2 +123567 123---567 +SELECT CONCAT(t2,'-',t2) c2 FROM (SELECT CONVERT(t USING latin1) t2 FROM t1) sub; +c2 +1234567-1234567 +DROP TABLE t1; +CREATE TABLE t1 (t VARCHAR(10) CHARSET latin1); +INSERT INTO t1 VALUES('1234567'); +SELECT CONCAT(t2,'-',t2) c2 FROM (SELECT CONVERT(t USING latin1) t2 FROM t1) sub; +c2 +1234567-1234567 +SELECT CONCAT(t2,'-',t2) c2 FROM (SELECT REVERSE(t) t2 FROM t1) sub; +c2 +7654321-7654321 +SELECT CONCAT(t2,'-',t2) c2 FROM (SELECT SOUNDEX(t) t2 FROM t1) sub; +c2 +- +SELECT CONCAT(t2,'-',t2) c2 FROM (SELECT TO_BASE64(t) t2 FROM t1) sub; +c2 +MTIzNDU2Nw==-MTIzNDU2Nw== +SELECT CONCAT(t2,'-',t2) c2 FROM (SELECT WEIGHT_STRING(t) t2 FROM t1) sub; +c2 +1234567-1234567 +SELECT CONCAT(t2,'-',t2) c2 FROM (SELECT HEX(t) t2 FROM t1) sub; +c2 +31323334353637-31323334353637 +SELECT CONCAT(t2,'-',t2) c2 FROM (SELECT QUOTE(t) t2 FROM t1) sub; +c2 +'1234567'-'1234567' +DROP TABLE t1; +CREATE TABLE t1 (t VARCHAR(32) CHARSET latin1); +INSERT INTO t1 VALUES(TO_BASE64('abcdefghi')); +SELECT CONCAT(t2,'-',t2) c2 FROM (SELECT FROM_BASE64(t) t2 FROM t1) sub; +c2 +abcdefghi-abcdefghi +DROP TABLE t1; +CREATE TABLE t1 (t VARCHAR(32) CHARSET latin1); +INSERT INTO t1 VALUES(HEX('abcdefghi')); +SELECT CONCAT(t2,'-',t2) c2 FROM (SELECT UNHEX(t) t2 FROM t1) sub; +c2 +abcdefghi-abcdefghi +DROP TABLE t1; +CREATE TABLE t1 (t VARCHAR(30) CHARSET latin1); +INSERT INTO t1 VALUES('test'); +SELECT LENGTH(CONCAT(t2)) c2 FROM (SELECT AES_ENCRYPT(t,'x') t2 FROM t1) sub; +c2 +16 +SELECT LENGTH(CONCAT(t2,'-',t2)) c2 FROM (SELECT AES_ENCRYPT(t,'x') t2 FROM t1) sub; +c2 +33 +SELECT LENGTH(CONCAT(t2,'--',t2)) c2 FROM (SELECT AES_ENCRYPT(t,'x') t2 FROM t1) sub; +c2 +34 +SELECT LENGTH(CONCAT(t2)) c2 FROM (SELECT AES_DECRYPT(AES_ENCRYPT(t,'x'),'x') t2 FROM t1) sub; +c2 +4 +SELECT LENGTH(CONCAT(t2,'-',t2)) c2 FROM (SELECT AES_DECRYPT(AES_ENCRYPT(t,'x'),'x') t2 FROM t1) sub; +c2 +9 +SELECT LENGTH(CONCAT(t2,'--',t2)) c2 FROM (SELECT AES_DECRYPT(AES_ENCRYPT(t,'x'),'x') t2 FROM t1) sub; +c2 +10 +DROP TABLE t1; +CREATE TABLE t1 (t VARCHAR(64) CHARSET latin1); +INSERT INTO t1 VALUES('123456789'); +SELECT CONCAT(t2,'-',t2) c2 FROM (SELECT MD5(t) t2 FROM t1) sub; +c2 +25f9e794323b453885f5181f1b624d0b-25f9e794323b453885f5181f1b624d0b +SELECT CONCAT(t2,'-',t2) c2 FROM (SELECT FORMAT(t,2) t2 FROM t1) sub; +c2 +123,456,789.00-123,456,789.00 +DROP TABLE t1; +CREATE TABLE t1 (t VARCHAR(32) CHARSET latin1); +INSERT INTO t1 VALUES('abcdefghi'); +SELECT CONCAT(t2,'-',t2) c2 FROM (SELECT INSERT(t,3,4,'xxx') t2 FROM t1) sub; +c2 +abxxxghi-abxxxghi +DROP TABLE t1; +CREATE TABLE t1 (t VARCHAR(10) CHARSET latin1); +INSERT INTO t1 VALUES('abcdefghi'); +SELECT CONCAT(t2,'-',t2) c2 FROM (SELECT LEFT(t,10) t2 FROM t1) sub; +c2 +abcdefghi-abcdefghi +SELECT CONCAT(t2,'-',t2) c2 FROM (SELECT RIGHT(t,10) t2 FROM t1) sub; +c2 +abcdefghi-abcdefghi +SELECT CONCAT(t2,'-',t2) c2 FROM (SELECT SUBSTR(t,1,10) t2 FROM t1) sub; +c2 +abcdefghi-abcdefghi +SELECT CONCAT(t2,'-',t2) c2 FROM (SELECT LTRIM(t) t2 FROM t1) sub; +c2 +abcdefghi-abcdefghi +SELECT CONCAT(t2,'-',t2) c2 FROM (SELECT RTRIM(t) t2 FROM t1) sub; +c2 +abcdefghi-abcdefghi +SELECT CONCAT(t2,'-',t2) c2 FROM (SELECT TRIM(t) t2 FROM t1) sub; +c2 +abcdefghi-abcdefghi +DROP TABLE t1; +SET optimizer_switch=@save_optimizer_switch; diff --git a/mysql-test/r/func_crypt.result b/mysql-test/r/func_crypt.result index c8a621e2fd3..aaa6aa61eae 100644 --- a/mysql-test/r/func_crypt.result +++ b/mysql-test/r/func_crypt.result @@ -107,6 +107,24 @@ OLD_PASSWORD(c1) PASSWORD(c1) DROP TABLE t1; # End of 5.0 tests # +# Start of 10.0 tests +# +# +# MDEV-10306 Wrong results with combination of CONCAT, SUBSTR and CONVERT in subquery +# +SET @save_optimizer_switch=@@optimizer_switch; +SET optimizer_switch='derived_merge=on'; +CREATE TABLE t1 (t VARCHAR(32) CHARSET latin1); +INSERT INTO t1 VALUES('abcdefghi'); +SELECT CONCAT(t2,'-',t2) c2 FROM (SELECT ENCRYPT(t,'aa') t2 FROM t1) sub; +c2 +aaHHlPHAM4sjs-aaHHlPHAM4sjs +DROP TABLE t1; +SET optimizer_switch=@save_optimizer_switch; +# +# End of 10.0 tests +# +# # Start of 10.1 tests # # Start of func_str_ascii_checksum.inc diff --git a/mysql-test/r/func_regexp_pcre.result b/mysql-test/r/func_regexp_pcre.result index 266ea6c5eef..0030fa53381 100644 --- a/mysql-test/r/func_regexp_pcre.result +++ b/mysql-test/r/func_regexp_pcre.result @@ -894,3 +894,5 @@ REGEXP_INSTR('a_kollision', '(oll)') SELECT REGEXP_INSTR('a_kollision', 'o([lm])\\1'); REGEXP_INSTR('a_kollision', 'o([lm])\\1') 4 +SELECT a FROM (SELECT "aa" a) t WHERE a REGEXP '[0-9]'; +a diff --git a/mysql-test/r/gis.result b/mysql-test/r/gis.result index daa8c20ddd2..73ec9d3049d 100644 --- a/mysql-test/r/gis.result +++ b/mysql-test/r/gis.result @@ -1703,6 +1703,24 @@ AsText(g) NULL POINT(1 1) # +# MDEV-10306 Wrong results with combination of CONCAT, SUBSTR and CONVERT in subquery +# +SET @save_optimizer_switch=@@optimizer_switch; +SET optimizer_switch='derived_merge=on'; +CREATE TABLE t1 (x INT, y INT); +INSERT INTO t1 VALUES(0,0); +SELECT LENGTH(t2) c2 FROM (SELECT ST_BUFFER(POINT(x,y), 0) t2 FROM t1) sub; +c2 +25 +SELECT LENGTH(CONCAT(t2,'-',t2)) c2 FROM (SELECT ST_BUFFER(POINT(x,y), 0) t2 FROM t1) sub; +c2 +51 +SELECT LENGTH(CONCAT(t2,'--',t2)) c2 FROM (SELECT ST_BUFFER(POINT(x,y), 0) t2 FROM t1) sub; +c2 +52 +DROP TABLE t1; +SET optimizer_switch=@save_optimizer_switch; +# # End 10.0 tests # SHOW CREATE TABLE information_schema.geometry_columns; diff --git a/mysql-test/r/information_schema.result b/mysql-test/r/information_schema.result index dad6d5d8c8a..00ec9d9f39c 100644 --- a/mysql-test/r/information_schema.result +++ b/mysql-test/r/information_schema.result @@ -1344,13 +1344,13 @@ DROP TABLE IF EXISTS bug23037; DROP FUNCTION IF EXISTS get_value; SELECT COLUMN_NAME, MD5(COLUMN_DEFAULT), LENGTH(COLUMN_DEFAULT) FROM INFORMATION_SCHEMA.COLUMNS WHERE TABLE_NAME='bug23037'; COLUMN_NAME MD5(COLUMN_DEFAULT) LENGTH(COLUMN_DEFAULT) -fld1 7cf7a6782be951a1f2464a350da926a5 65532 +fld1 85ea6a55b8f0058e640b3de141a3a9d9 65534 SELECT MD5(get_value()); MD5(get_value()) -7cf7a6782be951a1f2464a350da926a5 +76176d2daa20c582375b8dcfc18033cd SELECT COLUMN_NAME, MD5(COLUMN_DEFAULT), LENGTH(COLUMN_DEFAULT), COLUMN_DEFAULT=get_value() FROM INFORMATION_SCHEMA.COLUMNS WHERE TABLE_NAME='bug23037'; COLUMN_NAME MD5(COLUMN_DEFAULT) LENGTH(COLUMN_DEFAULT) COLUMN_DEFAULT=get_value() -fld1 7cf7a6782be951a1f2464a350da926a5 65532 1 +fld1 85ea6a55b8f0058e640b3de141a3a9d9 65534 0 DROP TABLE bug23037; DROP FUNCTION get_value; set @tmp_optimizer_switch=@@optimizer_switch; @@ -1437,12 +1437,12 @@ select column_default from information_schema.columns where table_name= 't1'; column_default NULL NULL - +'' NULL NULL 10 NULL -2006-01-01 00:00:00 +'2006-01-01 00:00:00' show columns from t1; Field Type Null Key Default Extra f1 varchar(50) YES NULL diff --git a/mysql-test/r/information_schema_parameters.result b/mysql-test/r/information_schema_parameters.result index 67527fbfe7f..1d974851133 100644 --- a/mysql-test/r/information_schema_parameters.result +++ b/mysql-test/r/information_schema_parameters.result @@ -29,7 +29,7 @@ TABLE_SCHEMA information_schema TABLE_NAME PARAMETERS COLUMN_NAME SPECIFIC_CATALOG ORDINAL_POSITION 1 -COLUMN_DEFAULT +COLUMN_DEFAULT '' IS_NULLABLE NO DATA_TYPE varchar CHARACTER_MAXIMUM_LENGTH 512 @@ -51,7 +51,7 @@ TABLE_SCHEMA information_schema TABLE_NAME PARAMETERS COLUMN_NAME SPECIFIC_SCHEMA ORDINAL_POSITION 2 -COLUMN_DEFAULT +COLUMN_DEFAULT '' IS_NULLABLE NO DATA_TYPE varchar CHARACTER_MAXIMUM_LENGTH 64 @@ -73,7 +73,7 @@ TABLE_SCHEMA information_schema TABLE_NAME PARAMETERS COLUMN_NAME SPECIFIC_NAME ORDINAL_POSITION 3 -COLUMN_DEFAULT +COLUMN_DEFAULT '' IS_NULLABLE NO DATA_TYPE varchar CHARACTER_MAXIMUM_LENGTH 64 @@ -161,7 +161,7 @@ TABLE_SCHEMA information_schema TABLE_NAME PARAMETERS COLUMN_NAME DATA_TYPE ORDINAL_POSITION 7 -COLUMN_DEFAULT +COLUMN_DEFAULT '' IS_NULLABLE NO DATA_TYPE varchar CHARACTER_MAXIMUM_LENGTH 64 @@ -337,7 +337,7 @@ TABLE_SCHEMA information_schema TABLE_NAME PARAMETERS COLUMN_NAME DTD_IDENTIFIER ORDINAL_POSITION 15 -COLUMN_DEFAULT +COLUMN_DEFAULT '' IS_NULLABLE NO DATA_TYPE longtext CHARACTER_MAXIMUM_LENGTH 4294967295 @@ -359,7 +359,7 @@ TABLE_SCHEMA information_schema TABLE_NAME PARAMETERS COLUMN_NAME ROUTINE_TYPE ORDINAL_POSITION 16 -COLUMN_DEFAULT +COLUMN_DEFAULT '' IS_NULLABLE NO DATA_TYPE varchar CHARACTER_MAXIMUM_LENGTH 9 diff --git a/mysql-test/r/information_schema_part.result b/mysql-test/r/information_schema_part.result index 91720d12ac4..081631a512f 100644 --- a/mysql-test/r/information_schema_part.result +++ b/mysql-test/r/information_schema_part.result @@ -7,9 +7,9 @@ partition x3 values in (16, 8, 5+19, 70-43) tablespace ts3); select * from information_schema.partitions where table_schema="test" and table_name="t1"; TABLE_CATALOG TABLE_SCHEMA TABLE_NAME PARTITION_NAME SUBPARTITION_NAME PARTITION_ORDINAL_POSITION SUBPARTITION_ORDINAL_POSITION PARTITION_METHOD SUBPARTITION_METHOD PARTITION_EXPRESSION SUBPARTITION_EXPRESSION PARTITION_DESCRIPTION TABLE_ROWS AVG_ROW_LENGTH DATA_LENGTH MAX_DATA_LENGTH INDEX_LENGTH DATA_FREE CREATE_TIME UPDATE_TIME CHECK_TIME CHECKSUM PARTITION_COMMENT NODEGROUP TABLESPACE_NAME -def test t1 x1 NULL 1 NULL LIST NULL b*a NULL 1 0 0 0 # 1024 0 # # NULL NULL default ts1 -def test t1 x2 NULL 2 NULL LIST NULL b*a NULL 3,11,5,7 0 0 0 # 1024 0 # # NULL NULL default ts2 -def test t1 x3 NULL 3 NULL LIST NULL b*a NULL 16,8,24,27 0 0 0 # 1024 0 # # NULL NULL default ts3 +def test t1 x1 NULL 1 NULL LIST NULL `b` * `a` NULL 1 0 0 0 # 1024 0 # # NULL NULL default ts1 +def test t1 x2 NULL 2 NULL LIST NULL `b` * `a` NULL 3,11,5,7 0 0 0 # 1024 0 # # NULL NULL default ts2 +def test t1 x3 NULL 3 NULL LIST NULL `b` * `a` NULL 16,8,24,27 0 0 0 # 1024 0 # # NULL NULL default ts3 create table t2 (a int not null,b int not null,c int not null, primary key(a,b)) partition by range (a) partitions 3 @@ -19,18 +19,18 @@ partition x3 values less than maxvalue tablespace ts3); select * from information_schema.partitions where table_schema="test" and table_name="t2"; TABLE_CATALOG TABLE_SCHEMA TABLE_NAME PARTITION_NAME SUBPARTITION_NAME PARTITION_ORDINAL_POSITION SUBPARTITION_ORDINAL_POSITION PARTITION_METHOD SUBPARTITION_METHOD PARTITION_EXPRESSION SUBPARTITION_EXPRESSION PARTITION_DESCRIPTION TABLE_ROWS AVG_ROW_LENGTH DATA_LENGTH MAX_DATA_LENGTH INDEX_LENGTH DATA_FREE CREATE_TIME UPDATE_TIME CHECK_TIME CHECKSUM PARTITION_COMMENT NODEGROUP TABLESPACE_NAME -def test t2 x1 NULL 1 NULL RANGE NULL a NULL 5 0 0 0 # 1024 0 # # NULL NULL default ts1 -def test t2 x2 NULL 2 NULL RANGE NULL a NULL 10 0 0 0 # 1024 0 # # NULL NULL default ts2 -def test t2 x3 NULL 3 NULL RANGE NULL a NULL MAXVALUE 0 0 0 # 1024 0 # # NULL NULL default ts3 +def test t2 x1 NULL 1 NULL RANGE NULL `a` NULL 5 0 0 0 # 1024 0 # # NULL NULL default ts1 +def test t2 x2 NULL 2 NULL RANGE NULL `a` NULL 10 0 0 0 # 1024 0 # # NULL NULL default ts2 +def test t2 x3 NULL 3 NULL RANGE NULL `a` NULL MAXVALUE 0 0 0 # 1024 0 # # NULL NULL default ts3 create table t3 (f1 date) partition by hash(month(f1)) partitions 3; select * from information_schema.partitions where table_schema="test" and table_name="t3"; TABLE_CATALOG TABLE_SCHEMA TABLE_NAME PARTITION_NAME SUBPARTITION_NAME PARTITION_ORDINAL_POSITION SUBPARTITION_ORDINAL_POSITION PARTITION_METHOD SUBPARTITION_METHOD PARTITION_EXPRESSION SUBPARTITION_EXPRESSION PARTITION_DESCRIPTION TABLE_ROWS AVG_ROW_LENGTH DATA_LENGTH MAX_DATA_LENGTH INDEX_LENGTH DATA_FREE CREATE_TIME UPDATE_TIME CHECK_TIME CHECKSUM PARTITION_COMMENT NODEGROUP TABLESPACE_NAME -def test t3 p0 NULL 1 NULL HASH NULL month(f1) NULL NULL 0 0 0 # 1024 0 # # NULL NULL default NULL -def test t3 p1 NULL 2 NULL HASH NULL month(f1) NULL NULL 0 0 0 # 1024 0 # # NULL NULL default NULL -def test t3 p2 NULL 3 NULL HASH NULL month(f1) NULL NULL 0 0 0 # 1024 0 # # NULL NULL default NULL +def test t3 p0 NULL 1 NULL HASH NULL month(`f1`) NULL NULL 0 0 0 # 1024 0 # # NULL NULL default NULL +def test t3 p1 NULL 2 NULL HASH NULL month(`f1`) NULL NULL 0 0 0 # 1024 0 # # NULL NULL default NULL +def test t3 p2 NULL 3 NULL HASH NULL month(`f1`) NULL NULL 0 0 0 # 1024 0 # # NULL NULL default NULL create table t4 (f1 date, f2 int) partition by key(f1,f2) partitions 3; @@ -63,14 +63,14 @@ subpartition x22 tablespace t2) ); select * from information_schema.partitions where table_schema="test"; TABLE_CATALOG TABLE_SCHEMA TABLE_NAME PARTITION_NAME SUBPARTITION_NAME PARTITION_ORDINAL_POSITION SUBPARTITION_ORDINAL_POSITION PARTITION_METHOD SUBPARTITION_METHOD PARTITION_EXPRESSION SUBPARTITION_EXPRESSION PARTITION_DESCRIPTION TABLE_ROWS AVG_ROW_LENGTH DATA_LENGTH MAX_DATA_LENGTH INDEX_LENGTH DATA_FREE CREATE_TIME UPDATE_TIME CHECK_TIME CHECKSUM PARTITION_COMMENT NODEGROUP TABLESPACE_NAME -def test t1 x1 x11 1 1 RANGE HASH a a+b 1 0 0 0 # 1024 0 # # NULL NULL default t1 -def test t1 x1 x12 1 2 RANGE HASH a a+b 1 0 0 0 # 1024 0 # # NULL NULL default t2 -def test t1 x2 x21 2 1 RANGE HASH a a+b 5 0 0 0 # 1024 0 # # NULL NULL default t1 -def test t1 x2 x22 2 2 RANGE HASH a a+b 5 0 0 0 # 1024 0 # # NULL NULL default t2 -def test t2 x1 x11 1 1 RANGE KEY a `a` 1 0 0 0 # 1024 0 # # NULL NULL default t1 -def test t2 x1 x12 1 2 RANGE KEY a `a` 1 0 0 0 # 1024 0 # # NULL NULL default t2 -def test t2 x2 x21 2 1 RANGE KEY a `a` 5 0 0 0 # 1024 0 # # NULL NULL default t1 -def test t2 x2 x22 2 2 RANGE KEY a `a` 5 0 0 0 # 1024 0 # # NULL NULL default t2 +def test t1 x1 x11 1 1 RANGE HASH `a` `a` + `b` 1 0 0 0 # 1024 0 # # NULL NULL default t1 +def test t1 x1 x12 1 2 RANGE HASH `a` `a` + `b` 1 0 0 0 # 1024 0 # # NULL NULL default t2 +def test t1 x2 x21 2 1 RANGE HASH `a` `a` + `b` 5 0 0 0 # 1024 0 # # NULL NULL default t1 +def test t1 x2 x22 2 2 RANGE HASH `a` `a` + `b` 5 0 0 0 # 1024 0 # # NULL NULL default t2 +def test t2 x1 x11 1 1 RANGE KEY `a` `a` 1 0 0 0 # 1024 0 # # NULL NULL default t1 +def test t2 x1 x12 1 2 RANGE KEY `a` `a` 1 0 0 0 # 1024 0 # # NULL NULL default t2 +def test t2 x2 x21 2 1 RANGE KEY `a` `a` 5 0 0 0 # 1024 0 # # NULL NULL default t1 +def test t2 x2 x22 2 2 RANGE KEY `a` `a` 5 0 0 0 # 1024 0 # # NULL NULL default t2 drop table t1,t2; create table t1 ( a int not null, @@ -88,10 +88,10 @@ subpartition x22 tablespace t2 nodegroup 1) ); select * from information_schema.partitions where table_schema="test"; TABLE_CATALOG TABLE_SCHEMA TABLE_NAME PARTITION_NAME SUBPARTITION_NAME PARTITION_ORDINAL_POSITION SUBPARTITION_ORDINAL_POSITION PARTITION_METHOD SUBPARTITION_METHOD PARTITION_EXPRESSION SUBPARTITION_EXPRESSION PARTITION_DESCRIPTION TABLE_ROWS AVG_ROW_LENGTH DATA_LENGTH MAX_DATA_LENGTH INDEX_LENGTH DATA_FREE CREATE_TIME UPDATE_TIME CHECK_TIME CHECKSUM PARTITION_COMMENT NODEGROUP TABLESPACE_NAME -def test t1 x1 x11 1 1 RANGE HASH a a+b 1 0 0 0 # 1024 0 # # NULL NULL 0 t1 -def test t1 x1 x12 1 2 RANGE HASH a a+b 1 0 0 0 # 1024 0 # # NULL NULL 1 t2 -def test t1 x2 x21 2 1 RANGE HASH a a+b 5 0 0 0 # 1024 0 # # NULL NULL 0 t1 -def test t1 x2 x22 2 2 RANGE HASH a a+b 5 0 0 0 # 1024 0 # # NULL NULL 1 t2 +def test t1 x1 x11 1 1 RANGE HASH `a` `a` + `b` 1 0 0 0 # 1024 0 # # NULL NULL 0 t1 +def test t1 x1 x12 1 2 RANGE HASH `a` `a` + `b` 1 0 0 0 # 1024 0 # # NULL NULL 1 t2 +def test t1 x2 x21 2 1 RANGE HASH `a` `a` + `b` 5 0 0 0 # 1024 0 # # NULL NULL 0 t1 +def test t1 x2 x22 2 2 RANGE HASH `a` `a` + `b` 5 0 0 0 # 1024 0 # # NULL NULL 1 t2 show tables; Tables_in_test t1 @@ -107,9 +107,9 @@ partitions 3; select * from information_schema.partitions where table_schema="test" and table_name="t1"; TABLE_CATALOG TABLE_SCHEMA TABLE_NAME PARTITION_NAME SUBPARTITION_NAME PARTITION_ORDINAL_POSITION SUBPARTITION_ORDINAL_POSITION PARTITION_METHOD SUBPARTITION_METHOD PARTITION_EXPRESSION SUBPARTITION_EXPRESSION PARTITION_DESCRIPTION TABLE_ROWS AVG_ROW_LENGTH DATA_LENGTH MAX_DATA_LENGTH INDEX_LENGTH DATA_FREE CREATE_TIME UPDATE_TIME CHECK_TIME CHECKSUM PARTITION_COMMENT NODEGROUP TABLESPACE_NAME -def test t1 p0 NULL 1 NULL LINEAR HASH NULL month(f1) NULL NULL 0 0 0 # 1024 0 # # NULL NULL default NULL -def test t1 p1 NULL 2 NULL LINEAR HASH NULL month(f1) NULL NULL 0 0 0 # 1024 0 # # NULL NULL default NULL -def test t1 p2 NULL 3 NULL LINEAR HASH NULL month(f1) NULL NULL 0 0 0 # 1024 0 # # NULL NULL default NULL +def test t1 p0 NULL 1 NULL LINEAR HASH NULL month(`f1`) NULL NULL 0 0 0 # 1024 0 # # NULL NULL default NULL +def test t1 p1 NULL 2 NULL LINEAR HASH NULL month(`f1`) NULL NULL 0 0 0 # 1024 0 # # NULL NULL default NULL +def test t1 p2 NULL 3 NULL LINEAR HASH NULL month(`f1`) NULL NULL 0 0 0 # 1024 0 # # NULL NULL default NULL drop table t1; create table t1 (a int) PARTITION BY RANGE (a) @@ -120,9 +120,9 @@ Table Create Table t1 CREATE TABLE `t1` ( `a` int(11) DEFAULT NULL ) ENGINE=MyISAM DEFAULT CHARSET=latin1 - PARTITION BY RANGE (a) -SUBPARTITION BY LINEAR HASH (a) -(PARTITION p0 VALUES LESS THAN (10) ENGINE = MyISAM) + PARTITION BY RANGE (`a`) +SUBPARTITION BY LINEAR HASH (`a`) +(PARTITION `p0` VALUES LESS THAN (10) ENGINE = MyISAM) select SUBPARTITION_METHOD FROM information_schema.partitions WHERE table_schema="test" AND table_name="t1"; SUBPARTITION_METHOD @@ -138,8 +138,8 @@ Table Create Table t1 CREATE TABLE `t1` ( `a` int(11) DEFAULT NULL ) ENGINE=MyISAM DEFAULT CHARSET=latin1 - PARTITION BY LIST (a) -(PARTITION p0 VALUES IN (10,11,12,13,14,15,16,17,18,19,20,21,22,23,24,25,26,27,28,29,30,31,32,33,34,35,36,37,38,39,40,41,42,43,44,45,46,47,48,49,50,51,52,53) ENGINE = MyISAM) + PARTITION BY LIST (`a`) +(PARTITION `p0` VALUES IN (10,11,12,13,14,15,16,17,18,19,20,21,22,23,24,25,26,27,28,29,30,31,32,33,34,35,36,37,38,39,40,41,42,43,44,45,46,47,48,49,50,51,52,53) ENGINE = MyISAM) SELECT PARTITION_DESCRIPTION FROM information_schema.partitions WHERE table_schema = "test" AND table_name = "t1"; PARTITION_DESCRIPTION diff --git a/mysql-test/r/information_schema_routines.result b/mysql-test/r/information_schema_routines.result index bfbd5941834..0d83f3a2bda 100644 --- a/mysql-test/r/information_schema_routines.result +++ b/mysql-test/r/information_schema_routines.result @@ -46,7 +46,7 @@ TABLE_SCHEMA information_schema TABLE_NAME ROUTINES COLUMN_NAME SPECIFIC_NAME ORDINAL_POSITION 1 -COLUMN_DEFAULT +COLUMN_DEFAULT '' IS_NULLABLE NO DATA_TYPE varchar CHARACTER_MAXIMUM_LENGTH 64 @@ -68,7 +68,7 @@ TABLE_SCHEMA information_schema TABLE_NAME ROUTINES COLUMN_NAME ROUTINE_CATALOG ORDINAL_POSITION 2 -COLUMN_DEFAULT +COLUMN_DEFAULT '' IS_NULLABLE NO DATA_TYPE varchar CHARACTER_MAXIMUM_LENGTH 512 @@ -90,7 +90,7 @@ TABLE_SCHEMA information_schema TABLE_NAME ROUTINES COLUMN_NAME ROUTINE_SCHEMA ORDINAL_POSITION 3 -COLUMN_DEFAULT +COLUMN_DEFAULT '' IS_NULLABLE NO DATA_TYPE varchar CHARACTER_MAXIMUM_LENGTH 64 @@ -112,7 +112,7 @@ TABLE_SCHEMA information_schema TABLE_NAME ROUTINES COLUMN_NAME ROUTINE_NAME ORDINAL_POSITION 4 -COLUMN_DEFAULT +COLUMN_DEFAULT '' IS_NULLABLE NO DATA_TYPE varchar CHARACTER_MAXIMUM_LENGTH 64 @@ -134,7 +134,7 @@ TABLE_SCHEMA information_schema TABLE_NAME ROUTINES COLUMN_NAME ROUTINE_TYPE ORDINAL_POSITION 5 -COLUMN_DEFAULT +COLUMN_DEFAULT '' IS_NULLABLE NO DATA_TYPE varchar CHARACTER_MAXIMUM_LENGTH 9 @@ -156,7 +156,7 @@ TABLE_SCHEMA information_schema TABLE_NAME ROUTINES COLUMN_NAME DATA_TYPE ORDINAL_POSITION 6 -COLUMN_DEFAULT +COLUMN_DEFAULT '' IS_NULLABLE NO DATA_TYPE varchar CHARACTER_MAXIMUM_LENGTH 64 @@ -354,7 +354,7 @@ TABLE_SCHEMA information_schema TABLE_NAME ROUTINES COLUMN_NAME ROUTINE_BODY ORDINAL_POSITION 15 -COLUMN_DEFAULT +COLUMN_DEFAULT '' IS_NULLABLE NO DATA_TYPE varchar CHARACTER_MAXIMUM_LENGTH 8 @@ -442,7 +442,7 @@ TABLE_SCHEMA information_schema TABLE_NAME ROUTINES COLUMN_NAME PARAMETER_STYLE ORDINAL_POSITION 19 -COLUMN_DEFAULT +COLUMN_DEFAULT '' IS_NULLABLE NO DATA_TYPE varchar CHARACTER_MAXIMUM_LENGTH 8 @@ -464,7 +464,7 @@ TABLE_SCHEMA information_schema TABLE_NAME ROUTINES COLUMN_NAME IS_DETERMINISTIC ORDINAL_POSITION 20 -COLUMN_DEFAULT +COLUMN_DEFAULT '' IS_NULLABLE NO DATA_TYPE varchar CHARACTER_MAXIMUM_LENGTH 3 @@ -486,7 +486,7 @@ TABLE_SCHEMA information_schema TABLE_NAME ROUTINES COLUMN_NAME SQL_DATA_ACCESS ORDINAL_POSITION 21 -COLUMN_DEFAULT +COLUMN_DEFAULT '' IS_NULLABLE NO DATA_TYPE varchar CHARACTER_MAXIMUM_LENGTH 64 @@ -530,7 +530,7 @@ TABLE_SCHEMA information_schema TABLE_NAME ROUTINES COLUMN_NAME SECURITY_TYPE ORDINAL_POSITION 23 -COLUMN_DEFAULT +COLUMN_DEFAULT '' IS_NULLABLE NO DATA_TYPE varchar CHARACTER_MAXIMUM_LENGTH 7 @@ -552,7 +552,7 @@ TABLE_SCHEMA information_schema TABLE_NAME ROUTINES COLUMN_NAME CREATED ORDINAL_POSITION 24 -COLUMN_DEFAULT 0000-00-00 00:00:00 +COLUMN_DEFAULT '0000-00-00 00:00:00' IS_NULLABLE NO DATA_TYPE datetime CHARACTER_MAXIMUM_LENGTH NULL @@ -574,7 +574,7 @@ TABLE_SCHEMA information_schema TABLE_NAME ROUTINES COLUMN_NAME LAST_ALTERED ORDINAL_POSITION 25 -COLUMN_DEFAULT 0000-00-00 00:00:00 +COLUMN_DEFAULT '0000-00-00 00:00:00' IS_NULLABLE NO DATA_TYPE datetime CHARACTER_MAXIMUM_LENGTH NULL @@ -596,7 +596,7 @@ TABLE_SCHEMA information_schema TABLE_NAME ROUTINES COLUMN_NAME SQL_MODE ORDINAL_POSITION 26 -COLUMN_DEFAULT +COLUMN_DEFAULT '' IS_NULLABLE NO DATA_TYPE varchar CHARACTER_MAXIMUM_LENGTH 8192 @@ -618,7 +618,7 @@ TABLE_SCHEMA information_schema TABLE_NAME ROUTINES COLUMN_NAME ROUTINE_COMMENT ORDINAL_POSITION 27 -COLUMN_DEFAULT +COLUMN_DEFAULT '' IS_NULLABLE NO DATA_TYPE longtext CHARACTER_MAXIMUM_LENGTH 4294967295 @@ -640,7 +640,7 @@ TABLE_SCHEMA information_schema TABLE_NAME ROUTINES COLUMN_NAME DEFINER ORDINAL_POSITION 28 -COLUMN_DEFAULT +COLUMN_DEFAULT '' IS_NULLABLE NO DATA_TYPE varchar CHARACTER_MAXIMUM_LENGTH 189 @@ -662,7 +662,7 @@ TABLE_SCHEMA information_schema TABLE_NAME ROUTINES COLUMN_NAME CHARACTER_SET_CLIENT ORDINAL_POSITION 29 -COLUMN_DEFAULT +COLUMN_DEFAULT '' IS_NULLABLE NO DATA_TYPE varchar CHARACTER_MAXIMUM_LENGTH 32 @@ -684,7 +684,7 @@ TABLE_SCHEMA information_schema TABLE_NAME ROUTINES COLUMN_NAME COLLATION_CONNECTION ORDINAL_POSITION 30 -COLUMN_DEFAULT +COLUMN_DEFAULT '' IS_NULLABLE NO DATA_TYPE varchar CHARACTER_MAXIMUM_LENGTH 32 @@ -706,7 +706,7 @@ TABLE_SCHEMA information_schema TABLE_NAME ROUTINES COLUMN_NAME DATABASE_COLLATION ORDINAL_POSITION 31 -COLUMN_DEFAULT +COLUMN_DEFAULT '' IS_NULLABLE NO DATA_TYPE varchar CHARACTER_MAXIMUM_LENGTH 32 diff --git a/mysql-test/r/join.result b/mysql-test/r/join.result index be497475f4d..df48dbba605 100644 --- a/mysql-test/r/join.result +++ b/mysql-test/r/join.result @@ -757,8 +757,8 @@ statistics.TABLE_NAME, statistics.COLUMN_NAME, statistics.TABLE_CATALOG, statist columns.TABLE_CATALOG, columns.TABLE_SCHEMA, columns.COLUMN_DEFAULT, columns.IS_NULLABLE, columns.DATA_TYPE, columns.CHARACTER_MAXIMUM_LENGTH, columns.CHARACTER_OCTET_LENGTH, columns.NUMERIC_PRECISION, columns.NUMERIC_SCALE, columns.CHARACTER_SET_NAME, columns.COLLATION_NAME, columns.COLUMN_TYPE, columns.COLUMN_KEY, columns.EXTRA, columns.COLUMN_COMMENT from information_schema.statistics join information_schema.columns using(table_name,column_name) where table_name='user'; TABLE_NAME COLUMN_NAME TABLE_CATALOG TABLE_SCHEMA NON_UNIQUE INDEX_SCHEMA INDEX_NAME SEQ_IN_INDEX COLLATION SUB_PART PACKED NULLABLE INDEX_TYPE COMMENT TABLE_CATALOG TABLE_SCHEMA COLUMN_DEFAULT IS_NULLABLE DATA_TYPE CHARACTER_MAXIMUM_LENGTH CHARACTER_OCTET_LENGTH NUMERIC_PRECISION NUMERIC_SCALE CHARACTER_SET_NAME COLLATION_NAME COLUMN_TYPE COLUMN_KEY EXTRA COLUMN_COMMENT -user Host def mysql 0 mysql PRIMARY 1 A NULL NULL BTREE def mysql NO char 60 180 NULL NULL utf8 utf8_bin char(60) PRI -user User def mysql 0 mysql PRIMARY 2 A NULL NULL BTREE def mysql NO char 80 240 NULL NULL utf8 utf8_bin char(80) PRI +user Host def mysql 0 mysql PRIMARY 1 A NULL NULL BTREE def mysql '' NO char 60 180 NULL NULL utf8 utf8_bin char(60) PRI +user User def mysql 0 mysql PRIMARY 2 A NULL NULL BTREE def mysql '' NO char 80 240 NULL NULL utf8 utf8_bin char(80) PRI Warnings: Warning 1286 Unknown storage engine 'InnoDB' Warning 1286 Unknown storage engine 'InnoDB' diff --git a/mysql-test/r/mdl.result b/mysql-test/r/mdl.result index 1a7291d922b..d93bfd5c729 100644 --- a/mysql-test/r/mdl.result +++ b/mysql-test/r/mdl.result @@ -7,16 +7,16 @@ # CREATE TABLE t1(a INT) ENGINE=InnoDB; LOCK TABLES t1 WRITE CONCURRENT, t1 AS t2 READ; -SELECT * FROM information_schema.metadata_lock_info; -THREAD_ID LOCK_MODE LOCK_DURATION LOCK_TYPE TABLE_SCHEMA TABLE_NAME -9 MDL_INTENTION_EXCLUSIVE NULL Global read lock -9 MDL_SHARED_NO_READ_WRITE NULL Table metadata lock test t1 +SELECT LOCK_MODE, LOCK_TYPE, TABLE_SCHEMA, TABLE_NAME FROM information_schema.metadata_lock_info; +LOCK_MODE LOCK_TYPE TABLE_SCHEMA TABLE_NAME +MDL_INTENTION_EXCLUSIVE Global read lock +MDL_SHARED_NO_READ_WRITE Table metadata lock test t1 UNLOCK TABLES; LOCK TABLES t1 AS t2 READ, t1 WRITE CONCURRENT; -SELECT * FROM information_schema.metadata_lock_info; -THREAD_ID LOCK_MODE LOCK_DURATION LOCK_TYPE TABLE_SCHEMA TABLE_NAME -9 MDL_INTENTION_EXCLUSIVE NULL Global read lock -9 MDL_SHARED_WRITE NULL Table metadata lock test t1 -9 MDL_SHARED_READ_ONLY NULL Table metadata lock test t1 +SELECT LOCK_MODE, LOCK_TYPE, TABLE_SCHEMA, TABLE_NAME FROM information_schema.metadata_lock_info; +LOCK_MODE LOCK_TYPE TABLE_SCHEMA TABLE_NAME +MDL_INTENTION_EXCLUSIVE Global read lock +MDL_SHARED_WRITE Table metadata lock test t1 +MDL_SHARED_READ_ONLY Table metadata lock test t1 UNLOCK TABLES; DROP TABLE t1; diff --git a/mysql-test/r/mysql.result b/mysql-test/r/mysql.result index 4097a22ea43..09f014da627 100644 --- a/mysql-test/r/mysql.result +++ b/mysql-test/r/mysql.result @@ -544,3 +544,61 @@ ERROR 1300 (HY000) at line 2: Invalid utf8 character string: 'test\xF0\x9F\x98\x set GLOBAL sql_mode=default; End of tests +create table `a1\``b1` (a int); +show tables; +Tables_in_test +a1\`b1 +insert `a1\``b1` values (1),(2); +show create table `a1\``b1`; +Table Create Table +a1\`b1 CREATE TABLE `a1\``b1` ( + `a` int(11) DEFAULT NULL +) ENGINE=MyISAM DEFAULT CHARSET=latin1 +/*!40101 SET @saved_cs_client = @@character_set_client */; +/*!40101 SET character_set_client = utf8 */; +CREATE TABLE `a1\``b1` ( + `a` int(11) DEFAULT NULL +) ENGINE=MyISAM DEFAULT CHARSET=latin1; +/*!40101 SET character_set_client = @saved_cs_client */; +INSERT INTO `a1\``b1` VALUES (1),(2); +insert `a1\``b1` values (4),(5); +show create table `a1\``b1`; +Table Create Table +a1\`b1 CREATE TABLE `a1\``b1` ( + `a` int(11) DEFAULT NULL +) ENGINE=MyISAM DEFAULT CHARSET=latin1 +select * from `a1\``b1`; +a +1 +2 +drop table `a1\``b1`; +set sql_mode=ansi_quotes; +create table "a1\""b1" (a int); +show tables; +Tables_in_test +a1\"b1 +insert "a1\""b1" values (1),(2); +show create table "a1\""b1"; +Table Create Table +a1\"b1 CREATE TABLE "a1\""b1" ( + "a" int(11) DEFAULT NULL +) ENGINE=MyISAM DEFAULT CHARSET=latin1 +/*!40101 SET @saved_cs_client = @@character_set_client */; +/*!40101 SET character_set_client = utf8 */; +CREATE TABLE "a1\""b1" ( + "a" int(11) DEFAULT NULL +); +/*!40101 SET character_set_client = @saved_cs_client */; +INSERT INTO "a1\""b1" VALUES (1),(2); +insert "a1\""b1" values (4),(5); +show create table "a1\""b1"; +Table Create Table +a1\"b1 CREATE TABLE "a1\""b1" ( + "a" int(11) DEFAULT NULL +) ENGINE=MyISAM DEFAULT CHARSET=latin1 +select * from "a1\""b1"; +a +1 +2 +drop table "a1\""b1"; +set sql_mode=default; diff --git a/mysql-test/r/mysqld--help.result b/mysql-test/r/mysqld--help.result index 3bbf149c0ee..9f748953521 100644 --- a/mysql-test/r/mysqld--help.result +++ b/mysql-test/r/mysqld--help.result @@ -1,3 +1,4 @@ +Windows bug: happens when a new line is exactly at the right offset The following options may be given as the first argument: --print-defaults Print the program argument list and exit. --no-defaults Don't read default options from any option file. @@ -1270,7 +1271,7 @@ ft-query-expansion-limit 20 ft-stopword-file (No default value) gdb FALSE general-log FALSE -getopt-prefix-matching TRUE +getopt-prefix-matching FALSE group-concat-max-len 1048576 gtid-domain-id 0 gtid-ignore-duplicates FALSE diff --git a/mysql-test/r/mysqldump.result b/mysql-test/r/mysqldump.result index 5a8c1e7a8ae..7f6107db5e4 100644 --- a/mysql-test/r/mysqldump.result +++ b/mysql-test/r/mysqldump.result @@ -15,7 +15,7 @@ INSERT INTO t1 VALUES (1), (2); - + @@ -151,9 +151,9 @@ INSERT INTO t1 VALUES (1, "test", "tes"), (2, "TEST", "TES"); - - - + + + @@ -179,7 +179,7 @@ INSERT INTO t1 VALUES ("1\""), ("\"2"); - + @@ -1613,10 +1613,10 @@ CREATE TABLE `t2` ( - + - + @@ -1624,10 +1624,10 @@ CREATE TABLE `t2` ( - + - + @@ -3739,8 +3739,8 @@ INSERT INTO t1 VALUES(1,0xff00fef0); - - + + @@ -4681,7 +4681,7 @@ CREATE TABLE `comment_table` (i INT COMMENT 'FIELD COMMENT') COMMENT = 'TABLE CO - + @@ -4778,8 +4778,8 @@ CREATE VIEW v2 AS SELECT * FROM t2; - - + + @@ -4797,7 +4797,7 @@ CREATE VIEW v2 AS SELECT * FROM t2; - + @@ -4833,12 +4833,12 @@ END - - + + - + @@ -4901,8 +4901,8 @@ END --> - - + + - + - - + + - +