Part 1 of MDEV-8139 Fix scrubbing tests

Port a bug fix from MySQL 5.7, so that all undo log pages will be freed
during a slow shutdown. We cannot scrub pages that are left allocated.

commit 173e171c6fb55f064eea278c76fbb28e2b1c757b
Author: Thirunarayanan Balathandayuthapani <thirunarayanan.balathandayuth@oracle.com>
Date:   Fri Sep 9 18:01:27 2016 +0530

    Bug #24450908   UNDO LOG EXISTS AFTER SLOW SHUTDOWN

    Problem:
    ========

    1) cached undo segment is not removed from rollback segment history
    (RSEG_HISTORY) during slow shutdown. In other words, If the segment is
    not completely free, we are failing to remove an entry from the history
    list. While starting the server, we traverse all rollback segment slots
    history list and make it as list of undo logs to be purged in purge
    queue.
    In that case, purge queue will never be empty after slow shutdown.

    2) Freeing of undo log segment is linked with removing undo log header
    from history.

    Fix:
    ====
    1) Have separate logic of removing the undo log header from
    history list from rollback segment slots and remove it from
    rollback segment history even though it is not completely free.

    Reviewed-by: Debarun Banerjee <debarun.banerjee@oracle.com>
    Reviewed-by: Marko Mäkelä <marko.makela@oracle.com>
    RB:13672

storage/innobase/fut/fut0lst.cc

@@ -338,104 +338,6 @@ flst_remove(
 	mlog_write_ulint(base + FLST_LEN, len - 1, MLOG_4BYTES, mtr);
 }
 
-/********************************************************************//**
-Cuts off the tail of the list, including the node given. The number of
-nodes which will be removed must be provided by the caller, as this function
-does not measure the length of the tail. */
-UNIV_INTERN
-void
-flst_cut_end(
-/*=========*/
-	flst_base_node_t*	base,	/*!< in: pointer to base node of list */
-	flst_node_t*		node2,	/*!< in: first node to remove */
-	ulint			n_nodes,/*!< in: number of nodes to remove,
-					must be >= 1 */
-	mtr_t*			mtr)	/*!< in: mini-transaction handle */
-{
-	ulint		space;
-	flst_node_t*	node1;
-	fil_addr_t	node1_addr;
-	fil_addr_t	node2_addr;
-	ulint		len;
-
-	ut_ad(mtr && node2 && base);
-	ut_ad(mtr_memo_contains_page(mtr, base, MTR_MEMO_PAGE_X_FIX));
-	ut_ad(mtr_memo_contains_page(mtr, node2, MTR_MEMO_PAGE_X_FIX));
-	ut_ad(n_nodes > 0);
-
-	buf_ptr_get_fsp_addr(node2, &space, &node2_addr);
-
-	node1_addr = flst_get_prev_addr(node2, mtr);
-
-	if (!fil_addr_is_null(node1_addr)) {
-
-		/* Update next field of node1 */
-
-		if (node1_addr.page == node2_addr.page) {
-
-			node1 = page_align(node2) + node1_addr.boffset;
-		} else {
-			node1 = fut_get_ptr(space,
-					    fil_space_get_zip_size(space),
-					    node1_addr, RW_X_LATCH, mtr);
-		}
-
-		flst_write_addr(node1 + FLST_NEXT, fil_addr_null, mtr);
-	} else {
-		/* node2 was first in list: update the field in base */
-		flst_write_addr(base + FLST_FIRST, fil_addr_null, mtr);
-	}
-
-	flst_write_addr(base + FLST_LAST, node1_addr, mtr);
-
-	/* Update len of base node */
-	len = flst_get_len(base, mtr);
-	ut_ad(len >= n_nodes);
-
-	mlog_write_ulint(base + FLST_LEN, len - n_nodes, MLOG_4BYTES, mtr);
-}
-
-/********************************************************************//**
-Cuts off the tail of the list, not including the given node. The number of
-nodes which will be removed must be provided by the caller, as this function
-does not measure the length of the tail. */
-UNIV_INTERN
-void
-flst_truncate_end(
-/*==============*/
-	flst_base_node_t*	base,	/*!< in: pointer to base node of list */
-	flst_node_t*		node2,	/*!< in: first node not to remove */
-	ulint			n_nodes,/*!< in: number of nodes to remove */
-	mtr_t*			mtr)	/*!< in: mini-transaction handle */
-{
-	fil_addr_t	node2_addr;
-	ulint		len;
-	ulint		space;
-
-	ut_ad(mtr && node2 && base);
-	ut_ad(mtr_memo_contains_page(mtr, base, MTR_MEMO_PAGE_X_FIX));
-	ut_ad(mtr_memo_contains_page(mtr, node2, MTR_MEMO_PAGE_X_FIX));
-	if (n_nodes == 0) {
-
-		ut_ad(fil_addr_is_null(flst_get_next_addr(node2, mtr)));
-
-		return;
-	}
-
-	buf_ptr_get_fsp_addr(node2, &space, &node2_addr);
-
-	/* Update next field of node2 */
-	flst_write_addr(node2 + FLST_NEXT, fil_addr_null, mtr);
-
-	flst_write_addr(base + FLST_LAST, node2_addr, mtr);
-
-	/* Update len of base node */
-	len = flst_get_len(base, mtr);
-	ut_ad(len >= n_nodes);
-
-	mlog_write_ulint(base + FLST_LEN, len - n_nodes, MLOG_4BYTES, mtr);
-}
-
 /********************************************************************//**
 Validates a file-based list.
 @return	TRUE if ok */

storage/innobase/include/fut0lst.h

@@ -102,31 +102,6 @@ flst_remove(
 	flst_node_t*		node2,	/*!< in: node to remove */
 	mtr_t*			mtr);	/*!< in: mini-transaction handle */
 /********************************************************************//**
-Cuts off the tail of the list, including the node given. The number of
-nodes which will be removed must be provided by the caller, as this function
-does not measure the length of the tail. */
-UNIV_INTERN
-void
-flst_cut_end(
-/*=========*/
-	flst_base_node_t*	base,	/*!< in: pointer to base node of list */
-	flst_node_t*		node2,	/*!< in: first node to remove */
-	ulint			n_nodes,/*!< in: number of nodes to remove,
-					must be >= 1 */
-	mtr_t*			mtr);	/*!< in: mini-transaction handle */
-/********************************************************************//**
-Cuts off the tail of the list, not including the given node. The number of
-nodes which will be removed must be provided by the caller, as this function
-does not measure the length of the tail. */
-UNIV_INTERN
-void
-flst_truncate_end(
-/*==============*/
-	flst_base_node_t*	base,	/*!< in: pointer to base node of list */
-	flst_node_t*		node2,	/*!< in: first node not to remove */
-	ulint			n_nodes,/*!< in: number of nodes to remove */
-	mtr_t*			mtr);	/*!< in: mini-transaction handle */
-/********************************************************************//**
 Gets list length.
 @return	length */
 UNIV_INLINE

storage/innobase/trx/trx0purge.cc

@@ -281,18 +281,33 @@ trx_purge_add_update_undo_to_history(
 	}
 }
 
-/**********************************************************************//**
-Frees an undo log segment which is in the history list. Cuts the end of the
-history list at the youngest undo log in this segment. */
+/** Remove undo log header from the history list.
+@param[in,out]	rseg_hdr	rollback segment header
+@param[in]	log_hdr		undo log segment header
+@param[in,out]	mtr		mini transaction. */
+static
+void
+trx_purge_remove_log_hdr(
+	trx_rsegf_t*	rseg_hdr,
+	trx_ulogf_t*	log_hdr,
+	mtr_t*		mtr)
+{
+	flst_remove(rseg_hdr + TRX_RSEG_HISTORY,
+		    log_hdr + TRX_UNDO_HISTORY_NODE, mtr);
+
+	os_atomic_decrement_ulint(&trx_sys->rseg_history_len, 1);
+}
+
+/** Frees an undo log segment which is in the history list. Removes the
+undo log hdr from the history list.
+@param[in,out]	rseg		rollback segment
+@param[in]	hdr_addr	file address of log_hdr
+@param[in]	noredo		skip redo logging. */
 static
 void
 trx_purge_free_segment(
-/*===================*/
-	trx_rseg_t*	rseg,		/*!< in: rollback segment */
-	fil_addr_t	hdr_addr,	/*!< in: the file address of log_hdr */
-	ulint		n_removed_logs)	/*!< in: count of how many undo logs we
-					will cut off from the end of the
-					history list */
+	trx_rseg_t*	rseg,
+	fil_addr_t	hdr_addr)
 {
 	mtr_t		mtr;
 	trx_rsegf_t*	rseg_hdr;
@@ -356,16 +371,7 @@ trx_purge_free_segment(
 	history list: otherwise, in case of a database crash, the segment
 	could become inaccessible garbage in the file space. */
 
-	flst_cut_end(rseg_hdr + TRX_RSEG_HISTORY,
-		     log_hdr + TRX_UNDO_HISTORY_NODE, n_removed_logs, &mtr);
-
-#ifdef HAVE_ATOMIC_BUILTINS
-	os_atomic_decrement_ulint(&trx_sys->rseg_history_len, n_removed_logs);
-#else
-	mutex_enter(&trx_sys->mutex);
-	trx_sys->rseg_history_len -= n_removed_logs;
-	mutex_exit(&trx_sys->mutex);
-#endif /* HAVE_ATOMIC_BUILTINS */
+	trx_purge_remove_log_hdr(rseg_hdr, log_hdr, &mtr);
 
 	do {
 
@@ -407,7 +413,6 @@ trx_purge_truncate_rseg_history(
 	page_t*		undo_page;
 	trx_ulogf_t*	log_hdr;
 	trx_usegf_t*	seg_hdr;
-	ulint		n_removed_logs	= 0;
 	mtr_t		mtr;
 	trx_id_t	undo_trx_no;
 
@@ -445,19 +450,6 @@ loop:
 				hdr_addr.boffset, limit->undo_no);
 		}
 
-#ifdef HAVE_ATOMIC_BUILTINS
-		os_atomic_decrement_ulint(
-			&trx_sys->rseg_history_len, n_removed_logs);
-#else
-		mutex_enter(&trx_sys->mutex);
-		trx_sys->rseg_history_len -= n_removed_logs;
-		mutex_exit(&trx_sys->mutex);
-#endif /* HAVE_ATOMIC_BUILTINS */
-
-		flst_truncate_end(rseg_hdr + TRX_RSEG_HISTORY,
-				  log_hdr + TRX_UNDO_HISTORY_NODE,
-				  n_removed_logs, &mtr);
-
 		mutex_exit(&(rseg->mutex));
 		mtr_commit(&mtr);
 
@@ -466,7 +458,6 @@ loop:
 
 	prev_hdr_addr = trx_purge_get_log_from_hist(
 		flst_get_prev_addr(log_hdr + TRX_UNDO_HISTORY_NODE, &mtr));
-	n_removed_logs++;
 
 	seg_hdr = undo_page + TRX_UNDO_SEG_HDR;
 
@@ -478,10 +469,14 @@ loop:
 		mutex_exit(&(rseg->mutex));
 		mtr_commit(&mtr);
 
-		trx_purge_free_segment(rseg, hdr_addr, n_removed_logs);
+		/* calls the trx_purge_remove_log_hdr()
+		inside trx_purge_free_segment(). */
+		trx_purge_free_segment(rseg, hdr_addr);
 
-		n_removed_logs = 0;
 	} else {
+		/* Remove the log hdr from the rseg history. */
+		trx_purge_remove_log_hdr(rseg_hdr, log_hdr, &mtr);
+
 		mutex_exit(&(rseg->mutex));
 		mtr_commit(&mtr);
 	}

storage/xtradb/fut/fut0lst.cc

@@ -338,104 +338,6 @@ flst_remove(
 	mlog_write_ulint(base + FLST_LEN, len - 1, MLOG_4BYTES, mtr);
 }
 
-/********************************************************************//**
-Cuts off the tail of the list, including the node given. The number of
-nodes which will be removed must be provided by the caller, as this function
-does not measure the length of the tail. */
-UNIV_INTERN
-void
-flst_cut_end(
-/*=========*/
-	flst_base_node_t*	base,	/*!< in: pointer to base node of list */
-	flst_node_t*		node2,	/*!< in: first node to remove */
-	ulint			n_nodes,/*!< in: number of nodes to remove,
-					must be >= 1 */
-	mtr_t*			mtr)	/*!< in: mini-transaction handle */
-{
-	ulint		space;
-	flst_node_t*	node1;
-	fil_addr_t	node1_addr;
-	fil_addr_t	node2_addr;
-	ulint		len;
-
-	ut_ad(mtr && node2 && base);
-	ut_ad(mtr_memo_contains_page(mtr, base, MTR_MEMO_PAGE_X_FIX));
-	ut_ad(mtr_memo_contains_page(mtr, node2, MTR_MEMO_PAGE_X_FIX));
-	ut_ad(n_nodes > 0);
-
-	buf_ptr_get_fsp_addr(node2, &space, &node2_addr);
-
-	node1_addr = flst_get_prev_addr(node2, mtr);
-
-	if (!fil_addr_is_null(node1_addr)) {
-
-		/* Update next field of node1 */
-
-		if (node1_addr.page == node2_addr.page) {
-
-			node1 = page_align(node2) + node1_addr.boffset;
-		} else {
-			node1 = fut_get_ptr(space,
-					    fil_space_get_zip_size(space),
-					    node1_addr, RW_X_LATCH, mtr);
-		}
-
-		flst_write_addr(node1 + FLST_NEXT, fil_addr_null, mtr);
-	} else {
-		/* node2 was first in list: update the field in base */
-		flst_write_addr(base + FLST_FIRST, fil_addr_null, mtr);
-	}
-
-	flst_write_addr(base + FLST_LAST, node1_addr, mtr);
-
-	/* Update len of base node */
-	len = flst_get_len(base, mtr);
-	ut_ad(len >= n_nodes);
-
-	mlog_write_ulint(base + FLST_LEN, len - n_nodes, MLOG_4BYTES, mtr);
-}
-
-/********************************************************************//**
-Cuts off the tail of the list, not including the given node. The number of
-nodes which will be removed must be provided by the caller, as this function
-does not measure the length of the tail. */
-UNIV_INTERN
-void
-flst_truncate_end(
-/*==============*/
-	flst_base_node_t*	base,	/*!< in: pointer to base node of list */
-	flst_node_t*		node2,	/*!< in: first node not to remove */
-	ulint			n_nodes,/*!< in: number of nodes to remove */
-	mtr_t*			mtr)	/*!< in: mini-transaction handle */
-{
-	fil_addr_t	node2_addr;
-	ulint		len;
-	ulint		space;
-
-	ut_ad(mtr && node2 && base);
-	ut_ad(mtr_memo_contains_page(mtr, base, MTR_MEMO_PAGE_X_FIX));
-	ut_ad(mtr_memo_contains_page(mtr, node2, MTR_MEMO_PAGE_X_FIX));
-	if (n_nodes == 0) {
-
-		ut_ad(fil_addr_is_null(flst_get_next_addr(node2, mtr)));
-
-		return;
-	}
-
-	buf_ptr_get_fsp_addr(node2, &space, &node2_addr);
-
-	/* Update next field of node2 */
-	flst_write_addr(node2 + FLST_NEXT, fil_addr_null, mtr);
-
-	flst_write_addr(base + FLST_LAST, node2_addr, mtr);
-
-	/* Update len of base node */
-	len = flst_get_len(base, mtr);
-	ut_ad(len >= n_nodes);
-
-	mlog_write_ulint(base + FLST_LEN, len - n_nodes, MLOG_4BYTES, mtr);
-}
-
 /********************************************************************//**
 Validates a file-based list.
 @return	TRUE if ok */

storage/xtradb/include/fut0lst.h

@@ -102,31 +102,6 @@ flst_remove(
 	flst_node_t*		node2,	/*!< in: node to remove */
 	mtr_t*			mtr);	/*!< in: mini-transaction handle */
 /********************************************************************//**
-Cuts off the tail of the list, including the node given. The number of
-nodes which will be removed must be provided by the caller, as this function
-does not measure the length of the tail. */
-UNIV_INTERN
-void
-flst_cut_end(
-/*=========*/
-	flst_base_node_t*	base,	/*!< in: pointer to base node of list */
-	flst_node_t*		node2,	/*!< in: first node to remove */
-	ulint			n_nodes,/*!< in: number of nodes to remove,
-					must be >= 1 */
-	mtr_t*			mtr);	/*!< in: mini-transaction handle */
-/********************************************************************//**
-Cuts off the tail of the list, not including the given node. The number of
-nodes which will be removed must be provided by the caller, as this function
-does not measure the length of the tail. */
-UNIV_INTERN
-void
-flst_truncate_end(
-/*==============*/
-	flst_base_node_t*	base,	/*!< in: pointer to base node of list */
-	flst_node_t*		node2,	/*!< in: first node not to remove */
-	ulint			n_nodes,/*!< in: number of nodes to remove */
-	mtr_t*			mtr);	/*!< in: mini-transaction handle */
-/********************************************************************//**
 Gets list length.
 @return	length */
 UNIV_INLINE

storage/xtradb/trx/trx0purge.cc

@@ -285,18 +285,33 @@ trx_purge_add_update_undo_to_history(
 	}
 }
 
-/**********************************************************************//**
-Frees an undo log segment which is in the history list. Cuts the end of the
-history list at the youngest undo log in this segment. */
+/** Remove undo log header from the history list.
+@param[in,out]	rseg_hdr	rollback segment header
+@param[in]	log_hdr		undo log segment header
+@param[in,out]	mtr		mini transaction. */
+static
+void
+trx_purge_remove_log_hdr(
+	trx_rsegf_t*	rseg_hdr,
+	trx_ulogf_t*	log_hdr,
+	mtr_t*		mtr)
+{
+	flst_remove(rseg_hdr + TRX_RSEG_HISTORY,
+		    log_hdr + TRX_UNDO_HISTORY_NODE, mtr);
+
+	os_atomic_decrement_ulint(&trx_sys->rseg_history_len, 1);
+}
+
+/** Frees an undo log segment which is in the history list. Removes the
+undo log hdr from the history list.
+@param[in,out]	rseg		rollback segment
+@param[in]	hdr_addr	file address of log_hdr
+@param[in]	noredo		skip redo logging. */
 static
 void
 trx_purge_free_segment(
-/*===================*/
-	trx_rseg_t*	rseg,		/*!< in: rollback segment */
-	fil_addr_t	hdr_addr,	/*!< in: the file address of log_hdr */
-	ulint		n_removed_logs)	/*!< in: count of how many undo logs we
-					will cut off from the end of the
-					history list */
+	trx_rseg_t*	rseg,
+	fil_addr_t	hdr_addr)
 {
 	mtr_t		mtr;
 	trx_rsegf_t*	rseg_hdr;
@@ -360,16 +375,7 @@ trx_purge_free_segment(
 	history list: otherwise, in case of a database crash, the segment
 	could become inaccessible garbage in the file space. */
 
-	flst_cut_end(rseg_hdr + TRX_RSEG_HISTORY,
-		     log_hdr + TRX_UNDO_HISTORY_NODE, n_removed_logs, &mtr);
-
-#ifdef HAVE_ATOMIC_BUILTINS
-	os_atomic_decrement_ulint(&trx_sys->rseg_history_len, n_removed_logs);
-#else
-	mutex_enter(&trx_sys->mutex);
-	trx_sys->rseg_history_len -= n_removed_logs;
-	mutex_exit(&trx_sys->mutex);
-#endif /* HAVE_ATOMIC_BUILTINS */
+	trx_purge_remove_log_hdr(rseg_hdr, log_hdr, &mtr);
 
 	do {
 
@@ -411,7 +417,6 @@ trx_purge_truncate_rseg_history(
 	page_t*		undo_page;
 	trx_ulogf_t*	log_hdr;
 	trx_usegf_t*	seg_hdr;
-	ulint		n_removed_logs	= 0;
 	mtr_t		mtr;
 	trx_id_t	undo_trx_no;
 
@@ -449,19 +454,6 @@ loop:
 				hdr_addr.boffset, limit->undo_no);
 		}
 
-#ifdef HAVE_ATOMIC_BUILTINS
-		os_atomic_decrement_ulint(
-			&trx_sys->rseg_history_len, n_removed_logs);
-#else
-		mutex_enter(&trx_sys->mutex);
-		trx_sys->rseg_history_len -= n_removed_logs;
-		mutex_exit(&trx_sys->mutex);
-#endif /* HAVE_ATOMIC_BUILTINS */
-
-		flst_truncate_end(rseg_hdr + TRX_RSEG_HISTORY,
-				  log_hdr + TRX_UNDO_HISTORY_NODE,
-				  n_removed_logs, &mtr);
-
 		mutex_exit(&(rseg->mutex));
 		mtr_commit(&mtr);
 
@@ -470,7 +462,6 @@ loop:
 
 	prev_hdr_addr = trx_purge_get_log_from_hist(
 		flst_get_prev_addr(log_hdr + TRX_UNDO_HISTORY_NODE, &mtr));
-	n_removed_logs++;
 
 	seg_hdr = undo_page + TRX_UNDO_SEG_HDR;
 
@@ -482,10 +473,14 @@ loop:
 		mutex_exit(&(rseg->mutex));
 		mtr_commit(&mtr);
 
-		trx_purge_free_segment(rseg, hdr_addr, n_removed_logs);
+		/* calls the trx_purge_remove_log_hdr()
+		inside trx_purge_free_segment(). */
+		trx_purge_free_segment(rseg, hdr_addr);
 
-		n_removed_logs = 0;
 	} else {
+		/* Remove the log hdr from the rseg history. */
+		trx_purge_remove_log_hdr(rseg_hdr, log_hdr, &mtr);
+
 		mutex_exit(&(rseg->mutex));
 		mtr_commit(&mtr);
 	}