diff --git a/cmake/install_macros.cmake b/cmake/install_macros.cmake index 7814f1dbeac..fba0bd03dac 100644 --- a/cmake/install_macros.cmake +++ b/cmake/install_macros.cmake @@ -266,7 +266,7 @@ SET(DEBUGBUILDDIR "${BINARY_PARENTDIR}/debug" CACHE INTERNAL "Directory of debug FUNCTION(INSTALL_MYSQL_TEST from to) IF(INSTALL_MYSQLTESTDIR) IF(NOT WITH_WSREP) - SET(EXCL_GALERA "(suite/(galera|wsrep|sys_vars/[rt]/(sysvars_)?wsrep).*|include/((w.*)?wsrep.*|.*galera.*)\\.inc|std_data/(galera|wsrep).*)") + SET(EXCL_GALERA "(suite/(galera|wsrep|sys_vars/[rt]/(sysvars_)?wsrep).*|std_data/(galera|wsrep).*)") ELSE() SET(EXCL_GALERA "^DOES_NOT_EXIST$") ENDIF() diff --git a/extra/mariabackup/fil_cur.cc b/extra/mariabackup/fil_cur.cc index d7aca1362ed..8820ce40c2b 100644 --- a/extra/mariabackup/fil_cur.cc +++ b/extra/mariabackup/fil_cur.cc @@ -129,6 +129,7 @@ xb_fil_cur_open( in case of error */ cursor->buf = NULL; cursor->node = NULL; + cursor->n_process_batch = 0; cursor->space_id = node->space->id; @@ -374,6 +375,8 @@ xb_fil_cur_result_t xb_fil_cur_read(xb_fil_cur_t* cursor, return(XB_FIL_CUR_EOF); } +reinit_buf: + cursor->n_process_batch++; if (to_read > (ib_int64_t) cursor->buf_size) { to_read = (ib_int64_t) cursor->buf_size; } @@ -416,8 +419,26 @@ read_retry: if (os_file_read(IORequestRead, cursor->file, cursor->buf, offset, (ulint) to_read, nullptr) != DB_SUCCESS) { - ret = XB_FIL_CUR_ERROR; - goto func_exit; + if (!srv_is_undo_tablespace(cursor->space_id)) { + ret = XB_FIL_CUR_ERROR; + goto func_exit; + } + + if (cursor->buf_page_no + >= SRV_UNDO_TABLESPACE_SIZE_IN_PAGES) { + ret = XB_FIL_CUR_SKIP; + goto func_exit; + } + + to_read = SRV_UNDO_TABLESPACE_SIZE_IN_PAGES * page_size; + + if (cursor->n_process_batch > 1) { + ret = XB_FIL_CUR_ERROR; + goto func_exit; + } + + space->release(); + goto reinit_buf; } defer = UT_LIST_GET_FIRST(space->chain)->deferred; diff --git a/extra/mariabackup/fil_cur.h b/extra/mariabackup/fil_cur.h index 0027b7768e9..ac561f71060 100644 --- a/extra/mariabackup/fil_cur.h +++ b/extra/mariabackup/fil_cur.h @@ -58,6 +58,7 @@ struct xb_fil_cur_t { uint thread_n; /*!< thread number for diagnostics */ ulint space_id; /*!< ID of tablespace */ ulint space_size; /*!< space size in pages */ + uint32_t n_process_batch;/*!< Number of batch processed */ /** @return whether this is not a file-per-table tablespace */ bool is_system() const diff --git a/extra/mariabackup/xtrabackup.cc b/extra/mariabackup/xtrabackup.cc index 408d3e5da56..dcc4884ac1c 100644 --- a/extra/mariabackup/xtrabackup.cc +++ b/extra/mariabackup/xtrabackup.cc @@ -245,6 +245,10 @@ long innobase_file_io_threads = 4; ulong innobase_read_io_threads = 4; ulong innobase_write_io_threads = 4; +/** Store the failed read of undo tablespace ids. Protected by +backup mutex */ +static std::set fail_undo_ids; + longlong innobase_page_size = (1LL << 14); /* 16KB */ char* innobase_buffer_pool_filename = NULL; @@ -406,6 +410,10 @@ struct ddl_tracker_t { static ddl_tracker_t ddl_tracker; +/** Store the space ids of truncated undo log tablespaces. Protected +by recv_sys.mutex */ +static std::set undo_trunc_ids; + /** Stores the space ids of page0 INIT_PAGE redo records. It is used to indicate whether the given deferred tablespace can be reconstructed. */ @@ -920,6 +928,11 @@ static void backup_file_op_fail(ulint space_id, int type, } } +static void backup_undo_trunc(uint32_t space_id) +{ + undo_trunc_ids.insert(space_id); +} + /* Function to store the space id of page0 INIT_PAGE @param space_id space id which has page0 init page */ static void backup_first_page_op(ulint space_id) @@ -2851,15 +2864,27 @@ static my_bool xtrabackup_copy_datafile(fil_node_t *node, uint thread_n, } /* The main copy loop */ - while ((res = xb_fil_cur_read(&cursor, corrupted_pages)) == - XB_FIL_CUR_SUCCESS) { + while (1) { + res = xb_fil_cur_read(&cursor, corrupted_pages); + if (res == XB_FIL_CUR_ERROR) { + goto error; + } + + if (res == XB_FIL_CUR_EOF) { + break; + } + if (!write_filter.process(&write_filt_ctxt, dstfile)) { goto error; } - } - if (res == XB_FIL_CUR_ERROR) { - goto error; + if (res == XB_FIL_CUR_SKIP) { + pthread_mutex_lock(&backup_mutex); + fail_undo_ids.insert( + static_cast(cursor.space_id)); + pthread_mutex_unlock(&backup_mutex); + break; + } } if (write_filter.finalize @@ -4450,6 +4475,23 @@ static bool xtrabackup_backup_low() dst_log_file = NULL; + std::vector failed_ids; + std::set_difference( + fail_undo_ids.begin(), fail_undo_ids.end(), + undo_trunc_ids.begin(), undo_trunc_ids.end(), + std::inserter(failed_ids, failed_ids.begin())); + + for (uint32_t id : failed_ids) { + msg("mariabackup: Failed to read undo log " + "tablespace space id %d and there is no undo " + "tablespace truncation redo record.", + id); + } + + if (failed_ids.size() > 0) { + return false; + } + if(!xtrabackup_incremental) { strcpy(metadata_type, "full-backuped"); metadata_from_lsn = 0; @@ -4524,6 +4566,7 @@ static bool xtrabackup_backup_func() srv_operation = SRV_OPERATION_BACKUP; log_file_op = backup_file_op; + undo_space_trunc = backup_undo_trunc; first_page_init = backup_first_page_op; metadata_to_lsn = 0; @@ -4538,6 +4581,7 @@ fail: } log_file_op = NULL; + undo_space_trunc = NULL; first_page_init = NULL; if (dst_log_file) { ds_close(dst_log_file); @@ -4827,6 +4871,7 @@ fail_before_log_copying_thread_start: innodb_shutdown(); log_file_op = NULL; + undo_space_trunc = NULL; first_page_init = NULL; pthread_mutex_destroy(&backup_mutex); pthread_cond_destroy(&scanned_lsn_cond); diff --git a/include/CMakeLists.txt b/include/CMakeLists.txt index e094a7b7a0c..1024821e569 100644 --- a/include/CMakeLists.txt +++ b/include/CMakeLists.txt @@ -77,7 +77,6 @@ FOREACH(f ${HEADERS_GEN_CONFIGURE}) ENDFOREACH(f) IF(NOT WITH_WSREP) SET(EXCL_SERVICE_WSREP "service_wsrep.h") - SET(EXCL_WSREP "wsrep.h") ENDIF() INSTALL(DIRECTORY mysql/ DESTINATION ${INSTALL_INCLUDEDIR}/server/mysql COMPONENT Development @@ -94,7 +93,6 @@ MACRO(INSTALL_PRIVATE DIR) FILES_MATCHING PATTERN "*.h" PATTERN CMakeFiles EXCLUDE PATTERN mysql EXCLUDE - PATTERN "${EXCL_WSREP}" EXCLUDE REGEX "\\./(${EXCL_RE}$)" EXCLUDE) ENDMACRO() diff --git a/mysql-test/main/group_min_max.result b/mysql-test/main/group_min_max.result index 083b3eabeee..af9f2767a6c 100644 --- a/mysql-test/main/group_min_max.result +++ b/mysql-test/main/group_min_max.result @@ -4066,6 +4066,26 @@ owner_id 1 DROP TABLE t1; # +# MDEV-30240 Wrong result upon aggregate function with SQL_BUFFER_RESULT +# +drop table if exists t1,t2; +Warnings: +Note 1051 Unknown table 'test.t1,test.t2' +CREATE TABLE t1 (pk INT PRIMARY KEY); +INSERT INTO t1 VALUES (1),(2); +CREATE TABLE t2 (a INT); +INSERT INTO t2 VALUES (1),(2); +SELECT SQL_BUFFER_RESULT MIN(pk) FROM t1, t2; +MIN(pk) +1 +SELECT MIN(pk) FROM t1, t2; +MIN(pk) +1 +DROP TABLE t1, t2; +# +# End of 10.5 tests +# +# # MDEV-24353: Adding GROUP BY slows down a query # CREATE TABLE t1 (p int NOT NULL, a int NOT NULL, PRIMARY KEY (p,a)); diff --git a/mysql-test/main/group_min_max.test b/mysql-test/main/group_min_max.test index cf9952e817d..0fa91e4d72b 100644 --- a/mysql-test/main/group_min_max.test +++ b/mysql-test/main/group_min_max.test @@ -1725,6 +1725,23 @@ SELECT DISTINCT owner_id FROM t1 WHERE foo = true GROUP BY owner_id HAVING (COUN SELECT DISTINCT owner_id FROM t1 WHERE foo = true GROUP BY owner_id HAVING (COUNT(*) = 1); DROP TABLE t1; +--echo # +--echo # MDEV-30240 Wrong result upon aggregate function with SQL_BUFFER_RESULT +--echo # + +drop table if exists t1,t2; +CREATE TABLE t1 (pk INT PRIMARY KEY); +INSERT INTO t1 VALUES (1),(2); +CREATE TABLE t2 (a INT); +INSERT INTO t2 VALUES (1),(2); +SELECT SQL_BUFFER_RESULT MIN(pk) FROM t1, t2; +SELECT MIN(pk) FROM t1, t2; +DROP TABLE t1, t2; + +--echo # +--echo # End of 10.5 tests +--echo # + --echo # --echo # MDEV-24353: Adding GROUP BY slows down a query --echo # diff --git a/mysql-test/main/range_aria_dbt3.result b/mysql-test/main/range_aria_dbt3.result new file mode 100644 index 00000000000..ae5a2e1329f --- /dev/null +++ b/mysql-test/main/range_aria_dbt3.result @@ -0,0 +1,13 @@ +set default_storage_engine=Aria; +CREATE DATABASE dbt3_s001; +use dbt3_s001; +# +# MDEV-30325 Wrong result upon range query using index condition +# +SELECT COUNT(*) FROM lineitem force index (i_l_orderkey_quantity,i_l_shipdate) WHERE l_shipdate < '1994-01-01' AND l_orderkey < 800 OR l_quantity > 3 AND l_orderkey NOT IN ( 157, 1444 ); +COUNT(*) +5056 +# +# End of 10.5 tests +# +DROP DATABASE dbt3_s001; diff --git a/mysql-test/main/range_aria_dbt3.test b/mysql-test/main/range_aria_dbt3.test new file mode 100644 index 00000000000..89328280987 --- /dev/null +++ b/mysql-test/main/range_aria_dbt3.test @@ -0,0 +1,24 @@ +# +# This is generic tests using dbt3_s001 tables +# This file uses the Aria storage engine +# + +set default_storage_engine=Aria; + +CREATE DATABASE dbt3_s001; +use dbt3_s001; +--disable_query_log +--source include/dbt3_s001.inc +--enable_query_log + +--echo # +--echo # MDEV-30325 Wrong result upon range query using index condition +--echo # + +SELECT COUNT(*) FROM lineitem force index (i_l_orderkey_quantity,i_l_shipdate) WHERE l_shipdate < '1994-01-01' AND l_orderkey < 800 OR l_quantity > 3 AND l_orderkey NOT IN ( 157, 1444 ); + +--echo # +--echo # End of 10.5 tests +--echo # + +DROP DATABASE dbt3_s001; diff --git a/mysys/array.c b/mysys/array.c index 32606cafb23..8eda279a492 100644 --- a/mysys/array.c +++ b/mysys/array.c @@ -23,6 +23,7 @@ SYNOPSIS init_dynamic_array2() + ps_key Key to register instrumented memory array Pointer to an array element_size Size of element init_buffer Initial buffer pointer diff --git a/mysys/hash.c b/mysys/hash.c index abc11b42500..c86b0d92cb6 100644 --- a/mysys/hash.c +++ b/mysys/hash.c @@ -60,8 +60,9 @@ my_hash_value_type my_hash_sort(CHARSET_INFO *cs, const uchar *key, dynamic array that is part of the hash will allocate memory as required during insertion. + @param[in] psi_key The key to register instrumented memory @param[in,out] hash The hash that is initialized - @param[in[ growth_size size incrememnt for the underlying dynarray + @param[in] growth_size size incrememnt for the underlying dynarray @param[in] charset The character set information @param[in] size The hash size @param[in] key_offest The key offset for the hash diff --git a/mysys/my_alloc.c b/mysys/my_alloc.c index 3e0b774b0c7..c3205eac6f0 100644 --- a/mysys/my_alloc.c +++ b/mysys/my_alloc.c @@ -34,8 +34,8 @@ SYNOPSIS init_alloc_root() + key - key to register instrumented memory mem_root - memory root to initialize - name - name of memroot (for debugging) block_size - size of chunks (blocks) used for memory allocation (It is external size of chunk i.e. it should include memory required for internal structures, thus it diff --git a/mysys/my_malloc.c b/mysys/my_malloc.c index befdcb0e5c3..c32831ed20d 100644 --- a/mysys/my_malloc.c +++ b/mysys/my_malloc.c @@ -59,6 +59,7 @@ void set_malloc_size_cb(MALLOC_SIZE_CB func) /** Allocate a sized block of memory. + @param key Key to register instrumented memory @param size The size of the memory block in bytes. @param flags Failure action modifiers (bitmasks). @@ -120,7 +121,8 @@ void *my_malloc(PSI_memory_key key, size_t size, myf my_flags) /** @brief wrapper around realloc() - @param old_point pointer to currently allocated area + @param key key to register instrumented memory + @param old_point pointer to currently allocated area @param size new size requested, must be >0 @param my_flags flags diff --git a/sql/CMakeLists.txt b/sql/CMakeLists.txt index 3633d9d8fba..7b99e2bdb0b 100644 --- a/sql/CMakeLists.txt +++ b/sql/CMakeLists.txt @@ -503,7 +503,7 @@ IF(WIN32) ENDIF(WIN32) IF(NOT WITH_WSREP) - SET(EXCL_WSREP "wsrep*.h") + SET(EXCL_WSREP "wsrep_[a-np-z]*.h") ENDIF() INSTALL(DIRECTORY . DESTINATION ${INSTALL_INCLUDEDIR}/server/private COMPONENT Development FILES_MATCHING PATTERN "*.h" diff --git a/sql/item_sum.cc b/sql/item_sum.cc index d33202d0a42..6fd33fe84b1 100644 --- a/sql/item_sum.cc +++ b/sql/item_sum.cc @@ -2373,8 +2373,15 @@ Item *Item_sum_variance::result_item(THD *thd, Field *field) void Item_sum_min_max::clear() { DBUG_ENTER("Item_sum_min_max::clear"); - value->clear(); - null_value= 1; + /* + We should not clear const items (from SELECT MIN(key) from t1) as then we would loose the + value cached in opt_sum_query() where we replace MIN/MAX/COUNT with constants. + */ + if (!const_item()) + { + value->clear(); + null_value= 1; + } DBUG_VOID_RETURN; } @@ -2487,9 +2494,12 @@ void Item_sum_min_max::no_rows_in_result() /* We may be called here twice in case of ref field in function */ if (was_values) { + bool org_const_item_cache= const_item_cache; was_values= FALSE; was_null_value= value->null_value; + const_item_cache= 0; // Ensure that clear works on const items clear(); + const_item_cache= org_const_item_cache; } DBUG_VOID_RETURN; } diff --git a/sql/opt_range.cc b/sql/opt_range.cc index 44e8e2c878c..06334263220 100644 --- a/sql/opt_range.cc +++ b/sql/opt_range.cc @@ -1890,7 +1890,7 @@ SEL_ARG::SEL_ARG(SEL_ARG &arg) :Sql_alloc() next= 0; if (next_key_part) { - ++next_key_part->use_count; + next_key_part->increment_use_count(1); weight += next_key_part->weight; } } @@ -10607,8 +10607,7 @@ key_or(RANGE_OPT_PARAM *param, SEL_ARG *key1,SEL_ARG *key2) Move on to next range in key2 */ key2->increment_use_count(-1); // Free not used tree - key2=key2_next; - continue; + key2= key2_next; } else { @@ -10622,8 +10621,9 @@ key_or(RANGE_OPT_PARAM *param, SEL_ARG *key1,SEL_ARG *key2) tmp: [---------] */ key2->copy_max_to_min(tmp); - continue; + key2= key2_next; } + continue; } /* diff --git a/sql/opt_sum.cc b/sql/opt_sum.cc index b3871254e6e..627ddc86abd 100644 --- a/sql/opt_sum.cc +++ b/sql/opt_sum.cc @@ -438,7 +438,7 @@ int opt_sum_query(THD *thd, The optimization is not applicable in both cases: (a) 'expr' is a non-constant expression. Then we can't replace 'expr' by a constant. - (b) 'expr' is a costant. According to ANSI, MIN/MAX must return + (b) 'expr' is a constant. According to ANSI, MIN/MAX must return NULL if the query does not return any rows. Thus, if we are not able to determine if the query returns any rows, we can't apply the optimization and replace MIN/MAX with a constant. diff --git a/sql/sql_select.cc b/sql/sql_select.cc index e4ac24df0b5..d8073d320f8 100644 --- a/sql/sql_select.cc +++ b/sql/sql_select.cc @@ -26267,7 +26267,6 @@ bool JOIN::alloc_func_list() @param field_list All items @param send_result_set_metadata Items in select list @param before_group_by Set to 1 if this is called before GROUP BY handling - @param recompute Set to TRUE if sum_funcs must be recomputed @retval 0 ok diff --git a/storage/innobase/include/log0recv.h b/storage/innobase/include/log0recv.h index b1c09cfa2bc..5e8dc1c0160 100644 --- a/storage/innobase/include/log0recv.h +++ b/storage/innobase/include/log0recv.h @@ -92,6 +92,11 @@ extern void (*log_file_op)(ulint space_id, int type, const byte* name, ulint len, const byte* new_name, ulint new_len); +/** Report an operation which does undo log tablespace truncation +during backup +@param space_id undo tablespace identifier */ +extern void (*undo_space_trunc)(uint32_t space_id); + /** Report an operation which does INIT_PAGE for page0 during backup. @param space_id tablespace identifier */ extern void (*first_page_init)(ulint space_id); diff --git a/storage/innobase/log/log0recv.cc b/storage/innobase/log/log0recv.cc index b0651d8b8da..2becd00cec6 100644 --- a/storage/innobase/log/log0recv.cc +++ b/storage/innobase/log/log0recv.cc @@ -1037,6 +1037,8 @@ void (*log_file_op)(ulint space_id, int type, const byte* name, ulint len, const byte* new_name, ulint new_len); +void (*undo_space_trunc)(uint32_t space_id); + void (*first_page_init)(ulint space_id); /** Information about initializing page contents during redo log processing. @@ -2424,6 +2426,8 @@ same_page: TRX_SYS_MAX_UNDO_SPACES, "compatibility"); truncated_undo_spaces[space_id - srv_undo_space_id_start]= { recovered_lsn, page_no }; + if (undo_space_trunc) + undo_space_trunc(space_id); #endif last_offset= 1; /* the next record must not be same_page */ continue;