revive innodb_debug_sync

innodb_debug_sync was introduced in commit
b393e2cb0c and reverted in
commit fc58c17216 due to memory leak reported
by valgrind, see MDEV-21336.

The leak is now fixed by adding `rw_lock_free(&slot->debug_sync_lock)`
after background thread working loop is finished, and the patch is
reapplied, with respect to c++98 fixes by Marko.

The missing DEBUG_SYNC for MDEV-18546 in row0vers.cc is also reapplied.

mysql-test/suite/gcol/r/innodb_virtual_debug_purge.result

@@ -233,3 +233,47 @@ set global debug_dbug= @saved_dbug;
 drop table t1;
 set debug_sync=reset;
 SET GLOBAL innodb_purge_rseg_truncate_frequency = @saved_frequency;
+#
+# MDEV-18546 ASAN heap-use-after-free
+# in innobase_get_computed_value / row_purge
+#
+CREATE TABLE t1 (
+pk INT AUTO_INCREMENT,
+b BIT(15),
+v BIT(15) AS (b) VIRTUAL,
+PRIMARY KEY(pk),
+UNIQUE(v)
+) ENGINE=InnoDB;
+INSERT IGNORE INTO t1 (b) VALUES
+(NULL),(b'011'),(b'000110100'),
+(b'01101101010'),(b'01111001001011'),(NULL);
+SET GLOBAL innodb_debug_sync = "ib_clust_v_col_before_row_allocated "
+                               "SIGNAL before_row_allocated "
+                               "WAIT_FOR flush_unlock";
+SET GLOBAL innodb_debug_sync = "ib_open_after_dict_open "
+                               "SIGNAL purge_open "
+                               "WAIT_FOR select_open";
+set global debug_dbug= "d,ib_purge_virtual_index_callback";
+connect purge_waiter,localhost,root;
+SET debug_sync= "now WAIT_FOR before_row_allocated";
+connection default;
+REPLACE INTO t1 (pk, b) SELECT pk, b FROM t1;
+connection purge_waiter;
+connection default;
+disconnect purge_waiter;
+FLUSH TABLES;
+SET GLOBAL innodb_debug_sync = reset;
+SET debug_sync= "now SIGNAL flush_unlock WAIT_FOR purge_open";
+SET GLOBAL innodb_debug_sync = reset;
+SET debug_sync= "ib_open_after_dict_open SIGNAL select_open";
+SELECT * FROM t1;
+pk	b	v
+1	NULL	NULL
+2		
+3	4	4
+4	j	j
+5	K	K
+6	NULL	NULL
+DROP TABLE t1;
+SET debug_sync= reset;
+set global debug_dbug= @old_dbug;

mysql-test/suite/gcol/t/innodb_virtual_debug_purge.test

@@ -323,3 +323,66 @@ drop table t1;
 --source include/wait_until_count_sessions.inc
 set debug_sync=reset;
 SET GLOBAL innodb_purge_rseg_truncate_frequency = @saved_frequency;
+
+--echo #
+--echo # MDEV-18546 ASAN heap-use-after-free
+--echo # in innobase_get_computed_value / row_purge
+--echo #
+
+CREATE TABLE t1 (
+  pk INT AUTO_INCREMENT,
+  b BIT(15),
+  v BIT(15) AS (b) VIRTUAL,
+  PRIMARY KEY(pk),
+  UNIQUE(v)
+) ENGINE=InnoDB;
+INSERT IGNORE INTO t1 (b) VALUES
+  (NULL),(b'011'),(b'000110100'),
+  (b'01101101010'),(b'01111001001011'),(NULL);
+
+SET GLOBAL innodb_debug_sync = "ib_clust_v_col_before_row_allocated "
+                               "SIGNAL before_row_allocated "
+                               "WAIT_FOR flush_unlock";
+SET GLOBAL innodb_debug_sync = "ib_open_after_dict_open "
+                               "SIGNAL purge_open "
+                               "WAIT_FOR select_open";
+
+# In 10.2 trx_undo_roll_ptr_is_insert(t_roll_ptr) condition never pass in purge,
+# so this condition is forced to pass in row_vers_old_has_index_entry
+set global debug_dbug= "d,ib_purge_virtual_index_callback";
+
+# The purge starts from REPLACE command. To avoid possible race, separate
+# connection is used.
+--connect(purge_waiter,localhost,root)
+--send
+SET debug_sync= "now WAIT_FOR before_row_allocated";
+
+--connection default
+REPLACE INTO t1 (pk, b) SELECT pk, b FROM t1;
+
+--connection purge_waiter
+# Now we will definitely catch ib_clust_v_col_before_row_allocated
+--reap
+--connection default
+--disconnect purge_waiter
+
+# purge hangs on the sync point. table is purged, ref_count is set to 0
+FLUSH TABLES;
+
+# Avoid hang on repeating purge.
+# Reset Will be applied after first record is purged
+SET GLOBAL innodb_debug_sync = reset;
+
+SET debug_sync= "now SIGNAL flush_unlock WAIT_FOR purge_open";
+
+# Avoid hang on repeating purge
+SET GLOBAL innodb_debug_sync = reset;
+
+# select unblocks purge thread
+SET debug_sync= "ib_open_after_dict_open SIGNAL select_open";
+SELECT * FROM t1;
+
+# Cleanup
+DROP TABLE t1;
+SET debug_sync= reset;
+set global debug_dbug= @old_dbug;

mysql-test/suite/sys_vars/r/sysvars_innodb.result

@@ -654,6 +654,18 @@ NUMERIC_BLOCK_SIZE	NULL
 ENUM_VALUE_LIST	OFF,ON
 READ_ONLY	NO
 COMMAND_LINE_ARGUMENT	REQUIRED
+VARIABLE_NAME	INNODB_DEBUG_SYNC
+SESSION_VALUE	NULL
+DEFAULT_VALUE	
+VARIABLE_SCOPE	GLOBAL
+VARIABLE_TYPE	VARCHAR
+VARIABLE_COMMENT	debug_sync for innodb purge threads. Use it to set up sync points for all purge threads at once. The commands will be applied sequentially at the beginning of purging the next undo record.
+NUMERIC_MIN_VALUE	NULL
+NUMERIC_MAX_VALUE	NULL
+NUMERIC_BLOCK_SIZE	NULL
+ENUM_VALUE_LIST	NULL
+READ_ONLY	NO
+COMMAND_LINE_ARGUMENT	NONE
 VARIABLE_NAME	INNODB_DEFAULT_ENCRYPTION_KEY_ID
 SESSION_VALUE	1
 DEFAULT_VALUE	1

storage/innobase/handler/ha_innodb.cc

@@ -19559,6 +19559,33 @@ innodb_log_checksums_update(
 		thd, *static_cast<const my_bool*>(save));
 }
 
+#ifdef UNIV_DEBUG
+static
+void
+innobase_debug_sync_callback(srv_slot_t *slot, const void *value)
+{
+	const char *value_str = *static_cast<const char* const*>(value);
+	size_t len = strlen(value_str) + 1;
+
+
+	// One allocation for list node object and value.
+	void *buf = ut_malloc_nokey(sizeof(srv_slot_t::debug_sync_t) + len-1);
+	srv_slot_t::debug_sync_t *sync = new(buf) srv_slot_t::debug_sync_t();
+	strcpy(sync->str, value_str);
+
+	rw_lock_x_lock(&slot->debug_sync_lock);
+	UT_LIST_ADD_LAST(slot->debug_sync, sync);
+	rw_lock_x_unlock(&slot->debug_sync_lock);
+}
+static
+void
+innobase_debug_sync_set(THD *thd, st_mysql_sys_var*, void *, const void *value)
+{
+	srv_for_each_thread(SRV_WORKER, innobase_debug_sync_callback, value);
+	srv_for_each_thread(SRV_PURGE, innobase_debug_sync_callback, value);
+}
+#endif
+
 static SHOW_VAR innodb_status_variables_export[]= {
 	{"Innodb", (char*) &show_innodb_vars, SHOW_FUNC},
 	{NullS, NullS, SHOW_LONG}
@@ -21205,6 +21232,16 @@ static MYSQL_SYSVAR_BOOL(debug_force_scrubbing,
 			 0,
 			 "Perform extra scrubbing to increase test exposure",
 			 NULL, NULL, FALSE);
+
+char *innobase_debug_sync;
+static MYSQL_SYSVAR_STR(debug_sync, innobase_debug_sync,
+			PLUGIN_VAR_NOCMDARG,
+			"debug_sync for innodb purge threads. "
+			"Use it to set up sync points for all purge threads "
+			"at once. The commands will be applied sequentially at"
+			" the beginning of purging the next undo record.",
+			NULL,
+			innobase_debug_sync_set, NULL);
 #endif /* UNIV_DEBUG */
 
 static MYSQL_SYSVAR_BOOL(instrument_semaphores, innodb_instrument_semaphores,
@@ -21428,6 +21465,7 @@ static struct st_mysql_sys_var* innobase_system_variables[]= {
   MYSQL_SYSVAR(background_scrub_data_check_interval),
 #ifdef UNIV_DEBUG
   MYSQL_SYSVAR(debug_force_scrubbing),
+  MYSQL_SYSVAR(debug_sync),
 #endif
   MYSQL_SYSVAR(instrument_semaphores),
   MYSQL_SYSVAR(buf_dump_status_frequency),

storage/innobase/include/srv0srv.h

@@ -1142,6 +1142,16 @@ struct srv_slot_t{
 						to do */
 	que_thr_t*	thr;			/*!< suspended query thread
 						(only used for user threads) */
+#ifdef UNIV_DEBUG
+	struct debug_sync_t {
+		UT_LIST_NODE_T(debug_sync_t)
+			debug_sync_list;
+		char str[1];
+	};
+	UT_LIST_BASE_NODE_T(debug_sync_t)
+		debug_sync;
+	rw_lock_t debug_sync_lock;
+#endif
 };
 
 #ifdef UNIV_DEBUG

storage/innobase/row/row0purge.cc

@@ -46,6 +46,7 @@ Created 3/14/1997 Heikki Tuuri
 #include "handler.h"
 #include "ha_innodb.h"
 #include "fil0fil.h"
+#include "debug_sync.h"
 
 /*************************************************************************
 IMPORTANT NOTE: Any operation that generates redo MUST check that there
@@ -1208,6 +1209,25 @@ row_purge_step(
 
 	node->start();
 
+#ifdef UNIV_DEBUG
+	srv_slot_t *slot = thr->thread_slot;
+	ut_ad(slot);
+
+	rw_lock_x_lock(&slot->debug_sync_lock);
+	while (UT_LIST_GET_LEN(slot->debug_sync)) {
+		srv_slot_t::debug_sync_t *sync =
+					UT_LIST_GET_FIRST(slot->debug_sync);
+		bool result = debug_sync_set_action(current_thd,
+						    sync->str,
+						    strlen(sync->str));
+		ut_a(!result);
+
+		UT_LIST_REMOVE(slot->debug_sync, sync);
+		ut_free(sync);
+	}
+	rw_lock_x_unlock(&slot->debug_sync_lock);
+#endif
+
 	if (!(node->undo_recs == NULL || ib_vector_is_empty(node->undo_recs))) {
 		trx_purge_rec_t*purge_rec;
 

storage/innobase/row/row0vers.cc

@@ -467,6 +467,7 @@ row_vers_build_clust_v_col(
 		vcol_info->set_used();
 		maria_table = vcol_info->table();
 	}
+	DEBUG_SYNC(current_thd, "ib_clust_v_col_before_row_allocated");
 
 	ib_vcol_row vc(NULL);
 	byte *record = vc.record(thd, index, &maria_table);

storage/innobase/srv/srv0srv.cc

@@ -2601,6 +2601,13 @@ DECLARE_THREAD(srv_worker_thread)(
 
 	slot = srv_reserve_slot(SRV_WORKER);
 
+#ifdef UNIV_DEBUG
+	UT_LIST_INIT(slot->debug_sync,
+		     &srv_slot_t::debug_sync_t::debug_sync_list);
+	rw_lock_create(PFS_NOT_INSTRUMENTED, &slot->debug_sync_lock,
+		       SYNC_NO_ORDER_CHECK);
+#endif
+
 	ut_a(srv_n_purge_threads > 1);
 	ut_a(ulong(my_atomic_loadlint(&srv_sys.n_threads_active[SRV_WORKER]))
 	     < srv_n_purge_threads);
@@ -2625,6 +2632,8 @@ DECLARE_THREAD(srv_worker_thread)(
 		purge_sys->latch here. */
 	} while (purge_sys->state != PURGE_STATE_EXIT);
 
+	ut_d(rw_lock_free(&slot->debug_sync_lock));
+
 	srv_free_slot(slot);
 
 	rw_lock_x_lock(&purge_sys->latch);
@@ -2848,6 +2857,12 @@ DECLARE_THREAD(srv_purge_coordinator_thread)(
 
 	slot = srv_reserve_slot(SRV_PURGE);
 
+#ifdef UNIV_DEBUG
+	UT_LIST_INIT(slot->debug_sync,
+		     &srv_slot_t::debug_sync_t::debug_sync_list);
+	rw_lock_create(PFS_NOT_INSTRUMENTED, &slot->debug_sync_lock,
+		       SYNC_NO_ORDER_CHECK);
+#endif
 	ulint	rseg_history_len = trx_sys->rseg_history_len;
 
 	do {
@@ -2881,6 +2896,8 @@ DECLARE_THREAD(srv_purge_coordinator_thread)(
 	shutdown state. */
 	ut_a(srv_get_task_queue_length() == 0);
 
+	ut_d(rw_lock_free(&slot->debug_sync_lock));
+
 	srv_free_slot(slot);
 
 	/* Note that we are shutting down. */