Merge 10.4 into 10.5

mysql-test/suite/encryption/r/innodb-redo-badkey.result

@@ -1,9 +1,10 @@
 call mtr.add_suppression("Plugin 'file_key_management'");
 call mtr.add_suppression("Plugin 'InnoDB' init function returned error.");
-call mtr.add_suppression("InnoDB: The page \\[page id: space=[1-9][0-9]*, page number=[0-9]+\\] in file .*test/t[1-4]\\.ibd cannot be decrypted");
+call mtr.add_suppression("InnoDB: The page \\[page id: space=[1-9][0-9]*, page number=[0-9]+\\] in file '.*test/t[1-4]\\.ibd' cannot be decrypted");
 call mtr.add_suppression("failed to read or decrypt \\[page id: space=[1-9][0-9]*, page number=[1-9][0-9]*\\]");
-call mtr.add_suppression("InnoDB: Unable to decompress .*.test.t1\\.ibd\\[page id: space=[1-9][0-9]*, page number=[0-9]+\\]");
+call mtr.add_suppression("InnoDB: Unable to decompress .*.test.t[12]\\.ibd\\[page id: space=[1-9][0-9]*, page number=[0-9]+\\]");
-call mtr.add_suppression("InnoDB: Database page corruption on disk or a failed file read of tablespace test/t1 page \\[page id: space=[1-9][0-9]*, page number=[0-9]*\\]");
+call mtr.add_suppression("InnoDB: Database page corruption on disk or a failed file read of tablespace test/t[12] page \\[page id: space=[1-9][0-9]*, page number=[0-9]*\\]");
+call mtr.add_suppression("InnoDB: Failed to read file '.*' at offset .*");
 call mtr.add_suppression("InnoDB: Plugin initialization aborted");
 call mtr.add_suppression("Plugin 'InnoDB' registration as a STORAGE ENGINE failed");
 # restart: --file-key-management-filename=MYSQL_TEST_DIR/std_data/keys2.txt

mysql-test/suite/encryption/t/innodb-redo-badkey.test

@@ -5,10 +5,11 @@
 
 call mtr.add_suppression("Plugin 'file_key_management'");
 call mtr.add_suppression("Plugin 'InnoDB' init function returned error.");
-call mtr.add_suppression("InnoDB: The page \\[page id: space=[1-9][0-9]*, page number=[0-9]+\\] in file .*test/t[1-4]\\.ibd cannot be decrypted");
+call mtr.add_suppression("InnoDB: The page \\[page id: space=[1-9][0-9]*, page number=[0-9]+\\] in file '.*test/t[1-4]\\.ibd' cannot be decrypted");
 call mtr.add_suppression("failed to read or decrypt \\[page id: space=[1-9][0-9]*, page number=[1-9][0-9]*\\]");
-call mtr.add_suppression("InnoDB: Unable to decompress .*.test.t1\\.ibd\\[page id: space=[1-9][0-9]*, page number=[0-9]+\\]");
+call mtr.add_suppression("InnoDB: Unable to decompress .*.test.t[12]\\.ibd\\[page id: space=[1-9][0-9]*, page number=[0-9]+\\]");
-call mtr.add_suppression("InnoDB: Database page corruption on disk or a failed file read of tablespace test/t1 page \\[page id: space=[1-9][0-9]*, page number=[0-9]*\\]");
+call mtr.add_suppression("InnoDB: Database page corruption on disk or a failed file read of tablespace test/t[12] page \\[page id: space=[1-9][0-9]*, page number=[0-9]*\\]");
+call mtr.add_suppression("InnoDB: Failed to read file '.*' at offset .*");
 call mtr.add_suppression("InnoDB: Plugin initialization aborted");
 call mtr.add_suppression("Plugin 'InnoDB' registration as a STORAGE ENGINE failed");
 

mysql-test/suite/innodb/r/instant_alter_bugs.result

@@ -1,3 +1,5 @@
+SET @save_frequency= @@GLOBAL.innodb_purge_rseg_truncate_frequency;
+SET GLOBAL innodb_purge_rseg_truncate_frequency=1;
 #
 # MDEV-17821 Assertion `!page_rec_is_supremum(rec)' failed
 # in btr_pcur_store_position
@@ -295,3 +297,16 @@ ALTER TABLE t DROP COLUMN c, ALGORITHM=INSTANT;
 SELECT * FROM t;
 a	b
 DROP TABLE t;
+CREATE TABLE t1 (a INT PRIMARY KEY, b INT, c INT, d INT, e INT) ENGINE=InnoDB;
+INSERT INTO t1 SET a=1;
+INSERT INTO t1 SET a=2;
+BEGIN;
+UPDATE t1 SET b=1;
+DELETE FROM t1;
+COMMIT;
+ALTER TABLE t1 DROP b, DROP c, DROP d, DROP e;
+InnoDB		0 transactions not purged
+SELECT * FROM t1;
+a
+DROP TABLE t1;
+SET GLOBAL innodb_purge_rseg_truncate_frequency=@save_frequency;

mysql-test/suite/innodb/t/instant_alter_bugs.test

@@ -1,5 +1,8 @@
 --source include/have_innodb.inc
 
+SET @save_frequency= @@GLOBAL.innodb_purge_rseg_truncate_frequency;
+SET GLOBAL innodb_purge_rseg_truncate_frequency=1;
+
 --echo #
 --echo # MDEV-17821 Assertion `!page_rec_is_supremum(rec)' failed
 --echo # in btr_pcur_store_position
@@ -313,3 +316,21 @@ ALTER TABLE t ADD COLUMN c INT, ALGORITHM=INSTANT;
 ALTER TABLE t DROP COLUMN c, ALGORITHM=INSTANT;
 SELECT * FROM t;
 DROP TABLE t;
+
+# The following is nondeterministically repeating the bug in a
+# different scenario: the table is empty at the time the ALTER TABLE
+# is invoked, apparently because purge already processed the records
+# for the DELETE, but not the record for the UPDATE.
+CREATE TABLE t1 (a INT PRIMARY KEY, b INT, c INT, d INT, e INT) ENGINE=InnoDB;
+INSERT INTO t1 SET a=1;
+INSERT INTO t1 SET a=2;
+BEGIN;
+UPDATE t1 SET b=1;
+DELETE FROM t1;
+COMMIT;
+
+ALTER TABLE t1 DROP b, DROP c, DROP d, DROP e;
+--source include/wait_all_purged.inc
+SELECT * FROM t1;
+DROP TABLE t1;
+SET GLOBAL innodb_purge_rseg_truncate_frequency=@save_frequency;

mysql-test/suite/innodb_fts/r/fulltext_table_evict.result

@@ -1,19 +0,0 @@
-#
-# Bug Bug #27304661 MYSQL CRASH DOING SYNC INDEX ]
-# [FATAL] INNODB: SEMAPHORE WAIT HAS LASTED > 600
-#
-CREATE TABLE t1 (
-id INT UNSIGNED AUTO_INCREMENT NOT NULL PRIMARY KEY,
-f1 TEXT(500),
-FULLTEXT idx (f1)
-) ENGINE=InnoDB;
-insert into t1 (f1) values ('fjdhfsjhf'),('dhjfhjshfj'),('dhjafjhfj');
-set @save_table_definition_cache=@@global.table_definition_cache;
-set @save_table_open_cache=@@global.table_open_cache;
-set global table_definition_cache=400;
-set global table_open_cache= 1024;
-SET @save_dbug = @@GLOBAL.debug_dbug;
-SET GLOBAL DEBUG_DBUG="+d,crash_if_fts_table_is_evicted";
-set @@global.table_definition_cache=@save_table_definition_cache;
-set @@global.table_open_cache=@save_table_open_cache;
-drop table t1;

mysql-test/suite/innodb_fts/t/fulltext_table_evict.test

@@ -1,50 +0,0 @@
---source include/have_innodb.inc
---source include/have_debug.inc
---source include/big_test.inc
---source include/have_64bit.inc
-let $restart_noprint=2;
-
---echo #
---echo # Bug Bug #27304661 MYSQL CRASH DOING SYNC INDEX ]
---echo # [FATAL] INNODB: SEMAPHORE WAIT HAS LASTED > 600
---echo #
-
-CREATE TABLE t1 (
-       id INT UNSIGNED AUTO_INCREMENT NOT NULL PRIMARY KEY,
-       f1 TEXT(500),
-       FULLTEXT idx (f1)
-       ) ENGINE=InnoDB;
-insert into t1 (f1) values ('fjdhfsjhf'),('dhjfhjshfj'),('dhjafjhfj');
-
---source include/restart_mysqld.inc
-
-set @save_table_definition_cache=@@global.table_definition_cache;
-set @save_table_open_cache=@@global.table_open_cache;
-
-set global table_definition_cache=400;
-set global table_open_cache= 1024;
-
-SET @save_dbug = @@GLOBAL.debug_dbug;
-SET GLOBAL DEBUG_DBUG="+d,crash_if_fts_table_is_evicted";
-#Create 1000 tables, try the best to evict t1 .
-
---disable_query_log
-let $loop=1000;
-while($loop)
-{
-  eval create table t_$loop(id int, name text(100), fulltext idxt_$loop(name) )engine=innodb;
-  dec $loop;
-}
-
-let $loop=1000;
-while($loop)
-{
-  eval drop table t_$loop;
-  dec $loop;
-}
-
-SET GLOBAL DEBUG_DBUG = @save_dbug;
---enable_query_log
-set @@global.table_definition_cache=@save_table_definition_cache;
-set @@global.table_open_cache=@save_table_open_cache;
-drop table t1;

mysql-test/unstable-tests

@@ -623,7 +623,6 @@ innodb.xa_recovery                           : MDEV-15279 - mysqld got exception
 #-----------------------------------------------------------------------
 
 innodb_fts.fulltext2                   : Modified in 10.4.7
-innodb_fts.fulltext_table_evict        : Modified in 10.4.8
 innodb_fts.innodb_fts_misc             : Modified in 10.4.8
 innodb_fts.innodb_fts_misc_debug       : MDEV-14156 - Unexpected warning
 innodb_fts.innodb_fts_plugin           : MDEV-13888 - Errors in server log

sql/item_cmpfunc.cc

@@ -321,11 +321,9 @@ static bool convert_const_to_int(THD *thd, Item_field *field_item,
     TABLE *table= field->table;
     sql_mode_t orig_sql_mode= thd->variables.sql_mode;
     enum_check_fields orig_count_cuted_fields= thd->count_cuted_fields;
-    my_bitmap_map *old_maps[2];
+    my_bitmap_map *old_maps[2] = { NULL, NULL };
     ulonglong UNINIT_VAR(orig_field_val); /* original field value if valid */
 
-    LINT_INIT_STRUCT(old_maps);
-
     /* table->read_set may not be set if we come here from a CREATE TABLE */
     if (table && table->read_set)
       dbug_tmp_use_all_columns(table, old_maps, 

sql/opt_subselect.h

@@ -95,7 +95,7 @@ class Loose_scan_opt
 
 public:
   Loose_scan_opt():
-    try_loosescan(FALSE),
+    try_loosescan(false),
     bound_sj_equalities(0),
     quick_uses_applicable_index(0),
     quick_max_loose_keypart(0),
@@ -103,10 +103,11 @@ public:
     best_loose_scan_cost(0),
     best_loose_scan_records(0),
     best_loose_scan_start_key(NULL),
-    best_max_loose_keypart(0)
+    best_max_loose_keypart(0),
+    best_ref_depend_map(0)
   {
   }
-  
+
   void init(JOIN *join, JOIN_TAB *s, table_map remaining_tables)
   {
     /*

storage/innobase/dict/dict0dict.cc

@@ -888,7 +888,7 @@ dict_table_open_on_name(
 	table = dict_table_check_if_in_cache_low(table_name);
 
 	if (table == NULL) {
-		table = dict_load_table(table_name, true, ignore_err);
+		table = dict_load_table(table_name, ignore_err);
 	}
 
 	ut_ad(!table || table->cached);
@@ -1137,14 +1137,7 @@ dict_make_room_in_cache(
 	        prev_table = UT_LIST_GET_PREV(table_LRU, table);
 
 		if (dict_table_can_be_evicted(table)) {
-
+			ut_ad(!table->fts);
-			DBUG_EXECUTE_IF("crash_if_fts_table_is_evicted",
-			{
-				  if (table->fts &&
-				      dict_table_has_fts_index(table)) {
-					ut_ad(0);
-				  }
-			};);
 			dict_sys.remove(table, true);
 
 			++n_evicted;

storage/innobase/dict/dict0load.cc

@@ -69,7 +69,6 @@ NULL.  These tables must be subsequently loaded so that all the foreign
 key constraints are loaded into memory.
 
 @param[in]	name		Table name in the db/tablename format
-@param[in]	cached		true=add to cache, false=do not
 @param[in]	ignore_err	Error to be ignored when loading table
 				and its index definition
 @param[out]	fk_tables	Related table names that must also be
@@ -82,7 +81,6 @@ static
 dict_table_t*
 dict_load_table_one(
 	const table_name_t&	name,
-	bool			cached,
 	dict_err_ignore_t	ignore_err,
 	dict_names_t&		fk_tables);
 
@@ -2723,17 +2721,12 @@ the cluster definition if the table is a member in a cluster. Also loads
 all foreign key constraints where the foreign key is in the table or where
 a foreign key references columns in this table.
 @param[in]	name		Table name in the dbname/tablename format
-@param[in]	cached		true=add to cache, false=do not
 @param[in]	ignore_err	Error to be ignored when loading
 				table and its index definition
 @return table, NULL if does not exist; if the table is stored in an
 .ibd file, but the file does not exist, then we set the file_unreadable
 flag in the table object we return. */
-dict_table_t*
+dict_table_t* dict_load_table(const char* name, dict_err_ignore_t ignore_err)
-dict_load_table(
-	const char*	name,
-	bool		cached,
-	dict_err_ignore_t ignore_err)
 {
 	dict_names_t			fk_list;
 	dict_table_t*			result;
@@ -2748,12 +2741,12 @@ dict_load_table(
 
 	if (!result) {
 		result = dict_load_table_one(const_cast<char*>(name),
-					     cached, ignore_err, fk_list);
+					     ignore_err, fk_list);
 		while (!fk_list.empty()) {
 			if (!dict_table_check_if_in_cache_low(fk_list.front()))
 				dict_load_table_one(
 					const_cast<char*>(fk_list.front()),
-					cached, ignore_err, fk_list);
+					ignore_err, fk_list);
 			fk_list.pop_front();
 		}
 	}
@@ -2846,7 +2839,6 @@ NULL.  These tables must be subsequently loaded so that all the foreign
 key constraints are loaded into memory.
 
 @param[in]	name		Table name in the db/tablename format
-@param[in]	cached		true=add to cache, false=do not
 @param[in]	ignore_err	Error to be ignored when loading table
 				and its index definition
 @param[out]	fk_tables	Related table names that must also be
@@ -2859,7 +2851,6 @@ static
 dict_table_t*
 dict_load_table_one(
 	const table_name_t&	name,
-	bool			cached,
 	dict_err_ignore_t	ignore_err,
 	dict_names_t&		fk_tables)
 {
@@ -2948,10 +2939,8 @@ err_exit:
 
 	dict_table_add_system_columns(table, heap);
 
-	if (cached) {
+	table->can_be_evicted = true;
-		table->can_be_evicted = true;
+	table->add_to_cache();
-		table->add_to_cache();
-	}
 
 	mem_heap_empty(heap);
 
@@ -2989,7 +2978,7 @@ err_exit:
 		}
 	}
 
-	if (err == DB_SUCCESS && cached && table->is_readable()) {
+	if (err == DB_SUCCESS && table->is_readable()) {
 		if (table->space && !fil_space_get_size(table->space_id)) {
 corrupted:
 			table->corrupted = true;
@@ -3033,7 +3022,7 @@ corrupted:
 	of the error condition, since the user may want to dump data from the
 	clustered index. However we load the foreign key information only if
 	all indexes were loaded. */
-	if (!cached || !table->is_readable()) {
+	if (!table->is_readable()) {
 		/* Don't attempt to load the indexes from disk. */
 	} else if (err == DB_SUCCESS) {
 		err = dict_load_foreigns(table->name.m_name, NULL,
@@ -3185,7 +3174,7 @@ check_rec:
 				/* Load the table definition to memory */
 				char*	table_name = mem_heap_strdupl(
 					heap, (char*) field, len);
-				table = dict_load_table(table_name, true, ignore_err);
+				table = dict_load_table(table_name, ignore_err);
 			}
 		}
 	}

storage/innobase/handler/ha_innodb.cc

@@ -12299,7 +12299,7 @@ int create_table_info_t::create_table(bool create_fk)
 						 DICT_ERR_IGNORE_NONE,
 						 fk_tables);
 			while (err == DB_SUCCESS && !fk_tables.empty()) {
-				dict_load_table(fk_tables.front(), true,
+				dict_load_table(fk_tables.front(),
 						DICT_ERR_IGNORE_NONE);
 				fk_tables.pop_front();
 			}

storage/innobase/handler/handler0alter.cc

@@ -5630,7 +5630,7 @@ static bool innobase_instant_try(
 	dict_index_t* index = dict_table_get_first_index(user_table);
 	mtr_t mtr;
 	mtr.start();
-	/* Prevent purge from calling dict_index_t::clear_instant_alter(),
+	/* Prevent purge from calling dict_index_t::clear_instant_add(),
 	to protect index->n_core_fields, index->table->instant and others
 	from changing during ctx->instant_column(). */
 	instant_metadata_lock(*index, mtr);
@@ -5821,6 +5821,7 @@ add_all_virtual:
 	if (rec_is_metadata(rec, *index)) {
 		ut_ad(page_rec_is_user_rec(rec));
 		if (!rec_is_alter_metadata(rec, *index)
+		    && !index->table->instant
 		    && !page_has_next(block->frame)
 		    && page_rec_is_last(rec, block->frame)) {
 			goto empty_table;
@@ -5902,7 +5903,7 @@ add_all_virtual:
 		}
 		btr_pcur_close(&pcur);
 		goto func_exit;
-	} else if (page_rec_is_supremum(rec)) {
+	} else if (page_rec_is_supremum(rec) && !index->table->instant) {
 empty_table:
 		/* The table is empty. */
 		ut_ad(fil_page_index_page_check(block->frame));
@@ -5910,7 +5911,9 @@ empty_table:
 		ut_ad(block->page.id.page_no() == index->page);
 		/* MDEV-17383: free metadata BLOBs! */
 		btr_page_empty(block, NULL, index, 0, &mtr);
-		index->clear_instant_alter();
+		if (index->is_instant()) {
+			index->clear_instant_add();
+		}
 		goto func_exit;
 	} else if (!user_table->is_instant()) {
 		ut_ad(!user_table->not_redundant());
@@ -9590,7 +9593,7 @@ innobase_update_foreign_cache(
 	also be loaded. */
 	while (err == DB_SUCCESS && !fk_tables.empty()) {
 		dict_table_t*	table = dict_load_table(
-			fk_tables.front(), true, DICT_ERR_IGNORE_NONE);
+			fk_tables.front(), DICT_ERR_IGNORE_NONE);
 
 		if (table == NULL) {
 			err = DB_TABLE_NOT_FOUND;

storage/innobase/include/dict0load.h

@@ -101,17 +101,12 @@ the cluster definition if the table is a member in a cluster. Also loads
 all foreign key constraints where the foreign key is in the table or where
 a foreign key references columns in this table.
 @param[in]	name		Table name in the dbname/tablename format
-@param[in]	cached		true=add to cache, false=do not
 @param[in]	ignore_err	Error to be ignored when loading
 				table and its index definition
 @return table, NULL if does not exist; if the table is stored in an
 .ibd file, but the file does not exist, then we set the file_unreadable
 flag in the table object we return. */
-dict_table_t*
+dict_table_t* dict_load_table(const char* name, dict_err_ignore_t ignore_err);
-dict_load_table(
-	const char*	name,
-	bool		cached,
-	dict_err_ignore_t ignore_err);
 
 /***********************************************************************//**
 Loads a table object based on the table id.

storage/innobase/include/dict0priv.ic

@@ -54,7 +54,7 @@ dict_table_get_low(
 	}
 
 	if (table == NULL) {
-		table = dict_load_table(table_name, true, DICT_ERR_IGNORE_NONE);
+		table = dict_load_table(table_name, DICT_ERR_IGNORE_NONE);
 	}
 
 	ut_ad(!table || table->cached);

storage/innobase/row/row0mysql.cc

@@ -2754,7 +2754,7 @@ row_mysql_drop_garbage_tables()
 			btr_pcur_store_position(&pcur, &mtr);
 			btr_pcur_commit_specify_mtr(&pcur, &mtr);
 
-			if (dict_load_table(table_name, true,
+			if (dict_load_table(table_name,
 					    DICT_ERR_IGNORE_DROP)) {
 				row_drop_table_for_mysql(table_name, trx,
 							 SQLCOM_DROP_TABLE);
@@ -3265,7 +3265,7 @@ row_drop_table_from_cache(
 
 	dict_sys.remove(table);
 
-	if (dict_load_table(tablename, true, DICT_ERR_IGNORE_FK_NOKEY)) {
+	if (dict_load_table(tablename, DICT_ERR_IGNORE_FK_NOKEY)) {
 		ib::error() << "Not able to remove table "
 			<< ut_get_name(trx, tablename)
 			<< " from the dictionary cache!";
@@ -4584,7 +4584,7 @@ end:
 		dict_mem_table_fill_foreign_vcol_set(table);
 
 		while (!fk_tables.empty()) {
-			dict_load_table(fk_tables.front(), true,
+			dict_load_table(fk_tables.front(),
 					DICT_ERR_IGNORE_NONE);
 			fk_tables.pop_front();
 		}

storage/innobase/sync/sync0rw.cc

@@ -986,13 +986,22 @@ rw_lock_own(
 	ut_ad(lock);
 	ut_ad(rw_lock_validate(lock));
 
+	const os_thread_id_t thread_id = os_thread_get_curr_id();
+
+	if (!os_thread_eq(lock->writer_thread, thread_id)) {
+	} else if (lock_type == RW_LOCK_X && rw_lock_get_x_lock_count(lock)) {
+		return TRUE;
+	} else if (lock_type == RW_LOCK_SX && rw_lock_get_sx_lock_count(lock)) {
+		return TRUE;
+	}
+
 	rw_lock_debug_mutex_enter();
 
 	for (const rw_lock_debug_t* info = UT_LIST_GET_FIRST(lock->debug_list);
 	     info != NULL;
 	     info = UT_LIST_GET_NEXT(list, info)) {
 
-		if (os_thread_eq(info->thread_id, os_thread_get_curr_id())
+		if (os_thread_eq(info->thread_id, thread_id)
 		    && info->pass == 0
 		    && info->lock_type == lock_type) {
 
@@ -1017,12 +1026,23 @@ bool rw_lock_own_flagged(const rw_lock_t* lock, rw_lock_flags_t flags)
 {
 	ut_ad(rw_lock_validate(lock));
 
+	const os_thread_id_t thread_id = os_thread_get_curr_id();
+
+	if (!os_thread_eq(lock->writer_thread, thread_id)) {
+	} else if ((flags & RW_LOCK_FLAG_X)
+		   && rw_lock_get_x_lock_count(lock)) {
+		return true;
+	} else if ((flags & RW_LOCK_FLAG_SX)
+		   && rw_lock_get_sx_lock_count(lock)) {
+		return true;
+	}
+
 	rw_lock_debug_mutex_enter();
 
 	for (rw_lock_debug_t* info = UT_LIST_GET_FIRST(lock->debug_list);
 	     info != NULL;
 	     info = UT_LIST_GET_NEXT(list, info)) {
-		if (!os_thread_eq(info->thread_id, os_thread_get_curr_id())
+		if (!os_thread_eq(info->thread_id, thread_id)
 		    || info->pass) {
 			continue;
 		}