From 1cfaafafee1d3e25470508c0479210c615fa3f18 Mon Sep 17 00:00:00 2001 From: Alexander Barkov Date: Thu, 5 Oct 2017 13:41:16 +0400 Subject: [PATCH 1/7] MDEV-13242 Wrong results for queries with row constructors and information_schema --- mysql-test/r/information_schema.result | 32 ++++++++++++++++++++++++++ mysql-test/t/information_schema.test | 26 +++++++++++++++++++++ sql/item.h | 2 ++ sql/sql_show.cc | 14 +++++++++++ 4 files changed, 74 insertions(+) diff --git a/mysql-test/r/information_schema.result b/mysql-test/r/information_schema.result index cee4797617f..2b5a536308a 100644 --- a/mysql-test/r/information_schema.result +++ b/mysql-test/r/information_schema.result @@ -2069,3 +2069,35 @@ Opened_tables 3 drop database mysqltest; drop database db1; set global sql_mode=default; +USE test; +# +# End of 10.0 tests +# +# +# Start of 10.1 tests +# +# +# MDEV-13242 Wrong results for queries with row constructors and information_schema +# +CREATE TABLE tt1(c1 INT); +CREATE TABLE tt2(c2 INT); +SELECT count(*) FROM information_schema.columns WHERE table_schema='test' AND (table_name, column_name) IN (('tt1', 'c1')); +count(*) +1 +SELECT count(*) FROM information_schema.columns WHERE table_schema='test' AND (table_name, column_name) IN (('tt2', 'c2')); +count(*) +1 +SELECT count(*) FROM information_schema.columns WHERE table_schema='test' AND (table_name, column_name) IN (('tt1','c1'),('tt2', 'c2')); +count(*) +2 +SELECT count(*) FROM information_schema.columns WHERE table_schema='test' AND (table_name, column_name) IN (SELECT 'tt1','c1' FROM dual UNION SELECT 'tt2', 'c2' FROM dual); +count(*) +2 +SELECT count(*) FROM information_schema.columns WHERE table_schema='test' AND (table_name='tt1' AND column_name='c1') OR (table_name='tt2' AND column_name='c2'); +count(*) +2 +SELECT column_name FROM information_schema.columns WHERE (table_name, column_name) IN (('tt1','c1'),('tt2', 'c2')) ORDER BY column_name; +column_name +c1 +c2 +DROP TABLE tt1, tt2; diff --git a/mysql-test/t/information_schema.test b/mysql-test/t/information_schema.test index 5b0e2910889..5b3fa7b653c 100644 --- a/mysql-test/t/information_schema.test +++ b/mysql-test/t/information_schema.test @@ -1910,3 +1910,29 @@ disconnect con1; --source include/wait_until_count_sessions.inc set global sql_mode=default; + +USE test; + +--echo # +--echo # End of 10.0 tests +--echo # + + +--echo # +--echo # Start of 10.1 tests +--echo # + + +--echo # +--echo # MDEV-13242 Wrong results for queries with row constructors and information_schema +--echo # + +CREATE TABLE tt1(c1 INT); +CREATE TABLE tt2(c2 INT); +SELECT count(*) FROM information_schema.columns WHERE table_schema='test' AND (table_name, column_name) IN (('tt1', 'c1')); +SELECT count(*) FROM information_schema.columns WHERE table_schema='test' AND (table_name, column_name) IN (('tt2', 'c2')); +SELECT count(*) FROM information_schema.columns WHERE table_schema='test' AND (table_name, column_name) IN (('tt1','c1'),('tt2', 'c2')); +SELECT count(*) FROM information_schema.columns WHERE table_schema='test' AND (table_name, column_name) IN (SELECT 'tt1','c1' FROM dual UNION SELECT 'tt2', 'c2' FROM dual); +SELECT count(*) FROM information_schema.columns WHERE table_schema='test' AND (table_name='tt1' AND column_name='c1') OR (table_name='tt2' AND column_name='c2'); +SELECT column_name FROM information_schema.columns WHERE (table_name, column_name) IN (('tt1','c1'),('tt2', 'c2')) ORDER BY column_name; +DROP TABLE tt1, tt2; diff --git a/sql/item.h b/sql/item.h index c338b14e340..0ca2eaae97c 100644 --- a/sql/item.h +++ b/sql/item.h @@ -4131,6 +4131,8 @@ public: bool fix_fields(THD *thd, Item **it); void cleanup(); + Item *get_orig_item() const { return orig_item; } + /* Methods of getting value which should be cached in the cache */ void save_val(Field *to); double val_real(); diff --git a/sql/sql_show.cc b/sql/sql_show.cc index 953842e29d1..1fdab201158 100644 --- a/sql/sql_show.cc +++ b/sql/sql_show.cc @@ -3640,6 +3640,15 @@ bool uses_only_table_name_fields(Item *item, TABLE_LIST *table) return 0; } } + else if (item->type() == Item::ROW_ITEM) + { + Item_row *item_row= static_cast(item); + for (uint i= 0; i < item_row->cols(); i++) + { + if (!uses_only_table_name_fields(item_row->element_index(i), table)) + return 0; + } + } else if (item->type() == Item::FIELD_ITEM) { Item_field *item_field= (Item_field*)item; @@ -3659,6 +3668,11 @@ bool uses_only_table_name_fields(Item *item, TABLE_LIST *table) strlen(item_field->field_name), 0))) return 0; } + else if (item->type() == Item::EXPR_CACHE_ITEM) + { + Item_cache_wrapper *tmp= static_cast(item); + return uses_only_table_name_fields(tmp->get_orig_item(), table); + } else if (item->type() == Item::REF_ITEM) return uses_only_table_name_fields(item->real_item(), table); From 0373e05a59afd14444337dfc9f9d3bc98a815e18 Mon Sep 17 00:00:00 2001 From: Vladislav Vaintroub Date: Thu, 5 Oct 2017 18:38:29 +0000 Subject: [PATCH 2/7] Refactor os_file_set_size() so it can be used to extend existing file, not just for creating new files. Use os_file_set_size() in fil_space_extend_must_retry() --- storage/xtradb/fil/fil0fil.cc | 166 ++++------------------------------ storage/xtradb/os/os0file.cc | 20 +++- 2 files changed, 35 insertions(+), 151 deletions(-) diff --git a/storage/xtradb/fil/fil0fil.cc b/storage/xtradb/fil/fil0fil.cc index 0cd67f5b19d..abaffc48135 100644 --- a/storage/xtradb/fil/fil0fil.cc +++ b/storage/xtradb/fil/fil0fil.cc @@ -1045,156 +1045,26 @@ fil_space_extend_must_retry( page_size = UNIV_PAGE_SIZE; } -#ifdef _WIN32 - const ulint io_completion_type = OS_FILE_READ; - /* Logically or physically extend the file with zero bytes, - depending on whether it is sparse. */ + /* fil_read_first_page() expects UNIV_PAGE_SIZE bytes. + fil_node_open_file() expects at least 4 * UNIV_PAGE_SIZE bytes.*/ - /* FIXME: Call DeviceIoControl(node->handle, FSCTL_SET_SPARSE, ...) - when opening a file when FSP_FLAGS_HAS_PAGE_COMPRESSION(). */ - { - FILE_END_OF_FILE_INFO feof; - /* fil_read_first_page() expects UNIV_PAGE_SIZE bytes. - fil_node_open_file() expects at least 4 * UNIV_PAGE_SIZE bytes. - Do not shrink short ROW_FORMAT=COMPRESSED files. */ - feof.EndOfFile.QuadPart = std::max( - os_offset_t(size - file_start_page_no) * page_size, - os_offset_t(FIL_IBD_FILE_INITIAL_SIZE - * UNIV_PAGE_SIZE)); - *success = SetFileInformationByHandle(node->handle, - FileEndOfFileInfo, - &feof, sizeof feof); - if (!*success) { - ib_logf(IB_LOG_LEVEL_ERROR, "extending file %s" - " from " INT64PF - " to " INT64PF " bytes failed with %u", - node->name, - os_offset_t(node->size) * page_size, - feof.EndOfFile.QuadPart, GetLastError()); - } else { - start_page_no = size; - } + os_offset_t new_size = std::max( + os_offset_t(size - file_start_page_no) * page_size, + os_offset_t(FIL_IBD_FILE_INITIAL_SIZE * UNIV_PAGE_SIZE)); + + *success = os_file_set_size(node->name, node->handle, new_size, + FSP_FLAGS_HAS_PAGE_COMPRESSION(space->flags)); + + + DBUG_EXECUTE_IF("ib_os_aio_func_io_failure_28", + *success = FALSE; + os_has_said_disk_full = TRUE;); + + if (*success) { + os_has_said_disk_full = FALSE; + start_page_no = size; } -#else - /* We will logically extend the file with ftruncate() if - page_compression is enabled, because the file is expected to - be sparse in that case. Make sure that ftruncate() can deal - with large files. */ - const bool is_sparse = sizeof(off_t) >= 8 - && FSP_FLAGS_HAS_PAGE_COMPRESSION(space->flags); -# ifdef HAVE_POSIX_FALLOCATE - /* We must complete the I/O request after invoking - posix_fallocate() to avoid an assertion failure at shutdown. - Because no actual writes were dispatched, a read operation - will suffice. */ - const ulint io_completion_type = srv_use_posix_fallocate - || is_sparse ? OS_FILE_READ : OS_FILE_WRITE; - - if (srv_use_posix_fallocate && !is_sparse) { - const os_offset_t start_offset - = os_offset_t(start_page_no - file_start_page_no) - * page_size; - const ulint n_pages = size - start_page_no; - const os_offset_t len = os_offset_t(n_pages) * page_size; - - int err; - do { - err = posix_fallocate(node->handle, start_offset, len); - } while (err == EINTR - && srv_shutdown_state == SRV_SHUTDOWN_NONE); - - *success = !err; - if (!*success) { - ib_logf(IB_LOG_LEVEL_ERROR, "extending file %s" - " from " INT64PF " to " INT64PF " bytes" - " failed with error %d", - node->name, start_offset, len + start_offset, - err); - } - - DBUG_EXECUTE_IF("ib_os_aio_func_io_failure_28", - *success = FALSE; - os_has_said_disk_full = TRUE;); - - if (*success) { - os_has_said_disk_full = FALSE; - start_page_no = size; - } - } else -# else - const ulint io_completion_type = is_sparse - ? OS_FILE_READ : OS_FILE_WRITE; -# endif - if (is_sparse) { - /* fil_read_first_page() expects UNIV_PAGE_SIZE bytes. - fil_node_open_file() expects at least 4 * UNIV_PAGE_SIZE bytes. - Do not shrink short ROW_FORMAT=COMPRESSED files. */ - off_t s = std::max(off_t(size - file_start_page_no) - * off_t(page_size), - off_t(FIL_IBD_FILE_INITIAL_SIZE - * UNIV_PAGE_SIZE)); - *success = !ftruncate(node->handle, s); - if (!*success) { - ib_logf(IB_LOG_LEVEL_ERROR, "ftruncate of file %s" - " from " INT64PF " to " INT64PF " bytes" - " failed with error %d", - node->name, - os_offset_t(start_page_no - file_start_page_no) - * page_size, os_offset_t(s), errno); - } else { - start_page_no = size; - } - } else { - /* Extend at most 64 pages at a time */ - ulint buf_size = ut_min(64, size - start_page_no) - * page_size; - byte* buf2 = static_cast( - calloc(1, buf_size + page_size)); - *success = buf2 != NULL; - if (!buf2) { - ib_logf(IB_LOG_LEVEL_ERROR, "Cannot allocate " ULINTPF - " bytes to extend file", - buf_size + page_size); - } - byte* const buf = static_cast( - ut_align(buf2, page_size)); - - while (*success && start_page_no < size) { - ulint n_pages - = ut_min(buf_size / page_size, - size - start_page_no); - - os_offset_t offset = static_cast( - start_page_no - file_start_page_no) - * page_size; - - *success = os_aio(OS_FILE_WRITE, 0, OS_AIO_SYNC, - node->name, node->handle, buf, - offset, page_size * n_pages, - page_size, node, NULL, - space->id, NULL, 0); - - DBUG_EXECUTE_IF("ib_os_aio_func_io_failure_28", - *success = FALSE; - os_has_said_disk_full = TRUE;); - - if (*success) { - os_has_said_disk_full = FALSE; - } - /* Let us measure the size of the file - to determine how much we were able to - extend it */ - os_offset_t fsize = os_file_get_size(node->handle); - ut_a(fsize != os_offset_t(-1)); - - start_page_no = ulint(fsize / page_size) - + file_start_page_no; - } - - free(buf2); - } -#endif mutex_enter(&fil_system->mutex); ut_a(node->being_extended); @@ -1204,7 +1074,7 @@ fil_space_extend_must_retry( space->size += file_size - node->size; node->size = file_size; - fil_node_complete_io(node, fil_system, io_completion_type); + fil_node_complete_io(node, fil_system, OS_FILE_READ); node->being_extended = FALSE; diff --git a/storage/xtradb/os/os0file.cc b/storage/xtradb/os/os0file.cc index f3c72576501..a3f1e860657 100644 --- a/storage/xtradb/os/os0file.cc +++ b/storage/xtradb/os/os0file.cc @@ -2575,7 +2575,16 @@ os_file_get_size( #endif /* __WIN__ */ } -/** Set the size of a newly created file. +/** Extend a file. + +On Windows, extending a file allocates blocks for the file, +unless the file is sparse. + +On Unix, we will extend the file with ftruncate(), if +file needs to be sparse. Otherwise posix_fallocate() is used +when available, and if not, binary zeroes are added to the end +of file. + @param[in] name file name @param[in] file file handle @param[in] size desired file size @@ -2626,15 +2635,20 @@ os_file_set_size( "file %s failed with error %d", size, name, err); } + /* Set errno because posix_fallocate() does not do it.*/ return(!err); } # endif + os_offset_t current_size = os_file_get_size(file); + + if (current_size >= size) { + return true; + } + /* Write up to 1 megabyte at a time. */ ulint buf_size = ut_min(64, (ulint) (size / UNIV_PAGE_SIZE)) * UNIV_PAGE_SIZE; - os_offset_t current_size = 0; - byte* buf2 = static_cast(calloc(1, buf_size + UNIV_PAGE_SIZE)); if (!buf2) { From f9b50c065718a237cb3474dda45062e4c7926240 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Marko=20M=C3=A4kel=C3=A4?= Date: Fri, 6 Oct 2017 17:51:29 +0300 Subject: [PATCH 3/7] MDEV-13512 buf_flush_update_zip_checksum() corrupts SPATIAL INDEX in ROW_FORMAT=COMPRESSED tables In MariaDB Server 10.1, this problem manifests itself only as a debug assertion failure in page_zip_decompress() when an insert requires a page to be decompressed. In MariaDB 10.1, the encryption of InnoDB data files repurposes the previously unused field FILE_FLUSH_LSN for an encryption key version. This field was only used in the first page of each file of the system tablespace. For ROW_FORMAT=COMPRESSED tables, the field was always written as 0 until encryption was implemented. There is no bug in the encryption, because the buffer pool blocks will not be written to files. Instead, copies of the blocks will be encrypted. In these encrypted copies, the key version field will be updated before the buffer is written to the file. The field in the buffer pool is basically garbage that does not really matter. Already in MariaDB 10.0, the memset() calls to reset this unused field in buf_flush_update_zip_checksum() and buf_flush_write_block_low() are unnecessary, because fsp_init_file_page_low() would guarantee that the field is always 0 in the buffer pool (unless 10.1 encryption is used). Removing the unnecessary memset() calls makes page_zip_decompress() happy and will prevent a SPATIAL INDEX corruption bug in MariaDB Server 10.2. In MySQL 5.7.5, as part of WL#6968, the same field was repurposed for an R-tree split sequence number (SSN) and these memset() were removed. (Because of the repurposing, MariaDB encryption is not available for tables that contain SPATIAL INDEX.) --- storage/innobase/buf/buf0flu.cc | 3 --- storage/xtradb/buf/buf0flu.cc | 3 --- 2 files changed, 6 deletions(-) diff --git a/storage/innobase/buf/buf0flu.cc b/storage/innobase/buf/buf0flu.cc index 31feb322826..dacddfca385 100644 --- a/storage/innobase/buf/buf0flu.cc +++ b/storage/innobase/buf/buf0flu.cc @@ -715,7 +715,6 @@ buf_flush_update_zip_checksum( srv_checksum_algorithm))); mach_write_to_8(page + FIL_PAGE_LSN, lsn); - memset(page + FIL_PAGE_FILE_FLUSH_LSN_OR_KEY_VERSION, 0, 8); mach_write_to_4(page + FIL_PAGE_SPACE_OR_CHKSUM, checksum); } @@ -894,8 +893,6 @@ buf_flush_write_block_low( bpage->newest_modification); ut_a(page_zip_verify_checksum(frame, zip_size)); - - memset(frame + FIL_PAGE_FILE_FLUSH_LSN_OR_KEY_VERSION, 0, 8); break; case BUF_BLOCK_FILE_PAGE: frame = bpage->zip.data; diff --git a/storage/xtradb/buf/buf0flu.cc b/storage/xtradb/buf/buf0flu.cc index b1a0f2daa21..abbcd5141cf 100644 --- a/storage/xtradb/buf/buf0flu.cc +++ b/storage/xtradb/buf/buf0flu.cc @@ -757,7 +757,6 @@ buf_flush_update_zip_checksum( srv_checksum_algorithm))); mach_write_to_8(page + FIL_PAGE_LSN, lsn); - memset(page + FIL_PAGE_FILE_FLUSH_LSN_OR_KEY_VERSION, 0, 8); mach_write_to_4(page + FIL_PAGE_SPACE_OR_CHKSUM, checksum); } @@ -935,8 +934,6 @@ buf_flush_write_block_low( bpage->newest_modification); ut_a(page_zip_verify_checksum(frame, zip_size)); - - memset(frame + FIL_PAGE_FILE_FLUSH_LSN_OR_KEY_VERSION, 0, 8); break; case BUF_BLOCK_FILE_PAGE: frame = bpage->zip.data; From 420798a81ac9a81d20629535fac3032e025e7733 Mon Sep 17 00:00:00 2001 From: Vladislav Vaintroub Date: Fri, 6 Oct 2017 16:36:40 +0000 Subject: [PATCH 4/7] Refactor os_file_set_size to extend already existing files. Change fil_space_extend_must_retry() to use this function. --- storage/innobase/fil/fil0fil.cc | 165 ++++---------------------------- storage/innobase/os/os0file.cc | 21 +++- storage/xtradb/fil/fil0fil.cc | 2 - storage/xtradb/os/os0file.cc | 1 + 4 files changed, 37 insertions(+), 152 deletions(-) diff --git a/storage/innobase/fil/fil0fil.cc b/storage/innobase/fil/fil0fil.cc index 7200c3c32e2..b2576f6bd81 100644 --- a/storage/innobase/fil/fil0fil.cc +++ b/storage/innobase/fil/fil0fil.cc @@ -1037,155 +1037,26 @@ fil_space_extend_must_retry( page_size = UNIV_PAGE_SIZE; } -#ifdef _WIN32 - const ulint io_completion_type = OS_FILE_READ; - /* Logically or physically extend the file with zero bytes, - depending on whether it is sparse. */ + /* fil_read_first_page() expects UNIV_PAGE_SIZE bytes. + fil_node_open_file() expects at least 4 * UNIV_PAGE_SIZE bytes.*/ - /* FIXME: Call DeviceIoControl(node->handle, FSCTL_SET_SPARSE, ...) - when opening a file when FSP_FLAGS_HAS_PAGE_COMPRESSION(). */ - { - FILE_END_OF_FILE_INFO feof; - /* fil_read_first_page() expects UNIV_PAGE_SIZE bytes. - fil_node_open_file() expects at least 4 * UNIV_PAGE_SIZE bytes. - Do not shrink short ROW_FORMAT=COMPRESSED files. */ - feof.EndOfFile.QuadPart = std::max( - os_offset_t(size - file_start_page_no) * page_size, - os_offset_t(FIL_IBD_FILE_INITIAL_SIZE - * UNIV_PAGE_SIZE)); - *success = SetFileInformationByHandle(node->handle, - FileEndOfFileInfo, - &feof, sizeof feof); - if (!*success) { - ib_logf(IB_LOG_LEVEL_ERROR, "extending file %s" - " from " INT64PF - " to " INT64PF " bytes failed with %u", - node->name, - os_offset_t(node->size) * page_size, - feof.EndOfFile.QuadPart, GetLastError()); - } else { - start_page_no = size; - } + os_offset_t new_size = std::max( + os_offset_t(size - file_start_page_no) * page_size, + os_offset_t(FIL_IBD_FILE_INITIAL_SIZE * UNIV_PAGE_SIZE)); + + *success = os_file_set_size(node->name, node->handle, new_size, + FSP_FLAGS_HAS_PAGE_COMPRESSION(space->flags)); + + + DBUG_EXECUTE_IF("ib_os_aio_func_io_failure_28", + *success = FALSE; + os_has_said_disk_full = TRUE;); + + if (*success) { + os_has_said_disk_full = FALSE; + start_page_no = size; } -#else - /* We will logically extend the file with ftruncate() if - page_compression is enabled, because the file is expected to - be sparse in that case. Make sure that ftruncate() can deal - with large files. */ - const bool is_sparse = sizeof(off_t) >= 8 - && FSP_FLAGS_HAS_PAGE_COMPRESSION(space->flags); -# ifdef HAVE_POSIX_FALLOCATE - /* We must complete the I/O request after invoking - posix_fallocate() to avoid an assertion failure at shutdown. - Because no actual writes were dispatched, a read operation - will suffice. */ - const ulint io_completion_type = srv_use_posix_fallocate - || is_sparse ? OS_FILE_READ : OS_FILE_WRITE; - - if (srv_use_posix_fallocate && !is_sparse) { - const os_offset_t start_offset - = os_offset_t(start_page_no - file_start_page_no) - * page_size; - const ulint n_pages = size - start_page_no; - const os_offset_t len = os_offset_t(n_pages) * page_size; - - int err; - do { - err = posix_fallocate(node->handle, start_offset, len); - } while (err == EINTR - && srv_shutdown_state == SRV_SHUTDOWN_NONE); - - *success = !err; - if (!*success) { - ib_logf(IB_LOG_LEVEL_ERROR, "extending file %s" - " from " INT64PF " to " INT64PF " bytes" - " failed with error %d", - node->name, start_offset, len + start_offset, - err); - } - - DBUG_EXECUTE_IF("ib_os_aio_func_io_failure_28", - *success = FALSE; - os_has_said_disk_full = TRUE;); - - if (*success) { - os_has_said_disk_full = FALSE; - start_page_no = size; - } - } else -# else - const ulint io_completion_type = is_sparse - ? OS_FILE_READ : OS_FILE_WRITE; -# endif - if (is_sparse) { - /* fil_read_first_page() expects UNIV_PAGE_SIZE bytes. - fil_node_open_file() expects at least 4 * UNIV_PAGE_SIZE bytes. - Do not shrink short ROW_FORMAT=COMPRESSED files. */ - off_t s = std::max(off_t(size - file_start_page_no) - * off_t(page_size), - off_t(FIL_IBD_FILE_INITIAL_SIZE - * UNIV_PAGE_SIZE)); - *success = !ftruncate(node->handle, s); - if (!*success) { - ib_logf(IB_LOG_LEVEL_ERROR, "ftruncate of file %s" - " from " INT64PF " to " INT64PF " bytes" - " failed with error %d", - node->name, - os_offset_t(start_page_no - file_start_page_no) - * page_size, os_offset_t(s), errno); - } else { - start_page_no = size; - } - } else { - /* Extend at most 64 pages at a time */ - ulint buf_size = ut_min(64, size - start_page_no) - * page_size; - byte* buf2 = static_cast( - calloc(1, buf_size + page_size)); - *success = buf2 != NULL; - if (!buf2) { - ib_logf(IB_LOG_LEVEL_ERROR, "Cannot allocate " ULINTPF - " bytes to extend file", - buf_size + page_size); - } - byte* const buf = static_cast( - ut_align(buf2, page_size)); - - while (*success && start_page_no < size) { - ulint n_pages - = ut_min(buf_size / page_size, - size - start_page_no); - - os_offset_t offset = static_cast( - start_page_no - file_start_page_no) - * page_size; - - *success = os_aio(OS_FILE_WRITE, 0, OS_AIO_SYNC, - node->name, node->handle, buf, - offset, page_size * n_pages, - page_size, node, NULL, 0); - - DBUG_EXECUTE_IF("ib_os_aio_func_io_failure_28", - *success = FALSE; - os_has_said_disk_full = TRUE;); - - if (*success) { - os_has_said_disk_full = FALSE; - } - /* Let us measure the size of the file - to determine how much we were able to - extend it */ - os_offset_t fsize = os_file_get_size(node->handle); - ut_a(fsize != os_offset_t(-1)); - - start_page_no = ulint(fsize / page_size) - + file_start_page_no; - } - - free(buf2); - } -#endif mutex_enter(&fil_system->mutex); ut_a(node->being_extended); @@ -1195,7 +1066,7 @@ fil_space_extend_must_retry( space->size += file_size - node->size; node->size = file_size; - fil_node_complete_io(node, fil_system, io_completion_type); + fil_node_complete_io(node, fil_system, OS_FILE_READ); node->being_extended = FALSE; diff --git a/storage/innobase/os/os0file.cc b/storage/innobase/os/os0file.cc index 44093935bc7..2a13746516f 100644 --- a/storage/innobase/os/os0file.cc +++ b/storage/innobase/os/os0file.cc @@ -2340,7 +2340,16 @@ os_file_get_size( #endif /* __WIN__ */ } -/** Set the size of a newly created file. +/** Extend a file. + +On Windows, extending a file allocates blocks for the file, +unless the file is sparse. + +On Unix, we will extend the file with ftruncate(), if +file needs to be sparse. Otherwise posix_fallocate() is used +when available, and if not, binary zeroes are added to the end +of file. + @param[in] name file name @param[in] file file handle @param[in] size desired file size @@ -2391,15 +2400,21 @@ os_file_set_size( "file %s failed with error %d", size, name, err); } + /* Set errno because posix_fallocate() does not do it.*/ + errno = err; return(!err); } # endif + os_offset_t current_size = os_file_get_size(file); + + if (current_size >= size) { + return true; + } + /* Write up to 1 megabyte at a time. */ ulint buf_size = ut_min(64, (ulint) (size / UNIV_PAGE_SIZE)) * UNIV_PAGE_SIZE; - os_offset_t current_size = 0; - byte* buf2 = static_cast(calloc(1, buf_size + UNIV_PAGE_SIZE)); if (!buf2) { diff --git a/storage/xtradb/fil/fil0fil.cc b/storage/xtradb/fil/fil0fil.cc index abaffc48135..38cfc13383c 100644 --- a/storage/xtradb/fil/fil0fil.cc +++ b/storage/xtradb/fil/fil0fil.cc @@ -1047,7 +1047,6 @@ fil_space_extend_must_retry( /* fil_read_first_page() expects UNIV_PAGE_SIZE bytes. fil_node_open_file() expects at least 4 * UNIV_PAGE_SIZE bytes.*/ - os_offset_t new_size = std::max( os_offset_t(size - file_start_page_no) * page_size, os_offset_t(FIL_IBD_FILE_INITIAL_SIZE * UNIV_PAGE_SIZE)); @@ -1064,7 +1063,6 @@ fil_space_extend_must_retry( os_has_said_disk_full = FALSE; start_page_no = size; } - mutex_enter(&fil_system->mutex); ut_a(node->being_extended); diff --git a/storage/xtradb/os/os0file.cc b/storage/xtradb/os/os0file.cc index a3f1e860657..183f65bcbd8 100644 --- a/storage/xtradb/os/os0file.cc +++ b/storage/xtradb/os/os0file.cc @@ -2636,6 +2636,7 @@ os_file_set_size( size, name, err); } /* Set errno because posix_fallocate() does not do it.*/ + errno = err; return(!err); } # endif From 0f8295d7d5ec3ea1ff2076dc1966e23c5a880f80 Mon Sep 17 00:00:00 2001 From: Vladislav Vaintroub Date: Fri, 6 Oct 2017 22:34:51 +0000 Subject: [PATCH 5/7] MDEV-13822 mariabackup incremental prepare incorrectly sets file size. Fix incremental prepare to change file size while applying delta file, if delta file contains page 0 with the new size. --- extra/mariabackup/xtrabackup.cc | 63 ++++++--------------------------- 1 file changed, 11 insertions(+), 52 deletions(-) diff --git a/extra/mariabackup/xtrabackup.cc b/extra/mariabackup/xtrabackup.cc index f07b1c6d3e3..02c19379961 100644 --- a/extra/mariabackup/xtrabackup.cc +++ b/extra/mariabackup/xtrabackup.cc @@ -4953,12 +4953,17 @@ xtrabackup_apply_delta( if (offset_on_page == 0xFFFFFFFFUL) break; - success = os_file_write(dst_path, dst_file, - incremental_buffer + - page_in_buffer * page_size, - (offset_on_page << - page_size_shift), - page_size); + uchar *buf = incremental_buffer + page_in_buffer * page_size; + const os_offset_t off = os_offset_t(offset_on_page)*page_size; + + if (off == 0) { + /* Fix tablespace size. */ + os_offset_t n_pages = fsp_get_size_low(static_cast(buf)); + if (!os_file_set_size(dst_path, dst_file, n_pages*page_size)) + goto error; + } + + success = os_file_write(dst_path, dst_file, buf, off, page_size); if (!success) { goto error; } @@ -5790,52 +5795,6 @@ skip_check: if(innodb_init()) goto error_cleanup; - if (xtrabackup_incremental) { - - it = datafiles_iter_new(fil_system); - if (it == NULL) { - msg("xtrabackup: Error: datafiles_iter_new() failed.\n"); - exit(EXIT_FAILURE); - } - - while ((node = datafiles_iter_next(it)) != NULL) { - byte *header; - ulint size; - ulint actual_size; - mtr_t mtr; - buf_block_t *block; - ulint flags; - - space = node->space; - - /* Align space sizes along with fsp header. We want to process - each space once, so skip all nodes except the first one in a - multi-node space. */ - if (UT_LIST_GET_PREV(chain, node) != NULL) { - continue; - } - - mtr_start(&mtr); - - mtr_s_lock(fil_space_get_latch(space->id, &flags), &mtr); - - block = buf_page_get(space->id, - dict_tf_get_zip_size(flags), - 0, RW_S_LATCH, &mtr); - header = FSP_HEADER_OFFSET + buf_block_get_frame(block); - - size = mtr_read_ulint(header + FSP_SIZE, MLOG_4BYTES, - &mtr); - - mtr_commit(&mtr); - - fil_extend_space_to_desired_size(&actual_size, space->id, size); - } - - datafiles_iter_free(it); - - } /* if (xtrabackup_incremental) */ - if (xtrabackup_export) { msg("xtrabackup: export option is specified.\n"); pfs_os_file_t info_file; From 8d1fb47e1d7403f1e6b61c2400399666f51067c2 Mon Sep 17 00:00:00 2001 From: Vladislav Vaintroub Date: Fri, 6 Oct 2017 22:40:28 +0000 Subject: [PATCH 6/7] MDEV-13798 - fix incorrect alignment of the buffer in incremental backup This incorrect alignment can later lead to memcpy over buffer boundaries, and to a crash. --- extra/mariabackup/write_filt.cc | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/extra/mariabackup/write_filt.cc b/extra/mariabackup/write_filt.cc index cf7753bf380..97529fb1726 100644 --- a/extra/mariabackup/write_filt.cc +++ b/extra/mariabackup/write_filt.cc @@ -79,7 +79,7 @@ wf_incremental_init(xb_write_filt_ctxt_t *ctxt, char *dst_name, cp->delta_buf_base = static_cast(ut_malloc(buf_size)); memset(cp->delta_buf_base, 0, buf_size); cp->delta_buf = static_cast - (ut_align(cp->delta_buf_base, UNIV_PAGE_SIZE_MAX)); + (ut_align(cp->delta_buf_base, cursor->page_size)); /* write delta meta info */ snprintf(meta_name, sizeof(meta_name), "%s%s", dst_name, From bb3f4fbb59c89f985f3649f276bdb3122886a5d9 Mon Sep 17 00:00:00 2001 From: Vladislav Vaintroub Date: Sat, 7 Oct 2017 07:36:17 +0000 Subject: [PATCH 7/7] MDEV-13310 Preparing an incremental backup twice can corrupt data Remove .delta file after it was successfully applied --- extra/mariabackup/xtrabackup.cc | 5 ++++- 1 file changed, 4 insertions(+), 1 deletion(-) diff --git a/extra/mariabackup/xtrabackup.cc b/extra/mariabackup/xtrabackup.cc index 02c19379961..14d7c5821c7 100644 --- a/extra/mariabackup/xtrabackup.cc +++ b/extra/mariabackup/xtrabackup.cc @@ -4974,8 +4974,11 @@ xtrabackup_apply_delta( if (incremental_buffer_base) ut_free(incremental_buffer_base); - if (src_file != XB_FILE_UNDEFINED) + if (src_file != XB_FILE_UNDEFINED) { os_file_close(src_file); + /* Remove .delta file after it was successfully applied.*/ + os_file_delete(0,src_path); + } if (dst_file != XB_FILE_UNDEFINED) os_file_close(dst_file); return TRUE;