buf0buf.c, srv0srv.h, buf0buf.h, srv0srv.c:

Clean up the working of the main thread; add a tunable parameter srv_max_buf_pool_modified_pct which can be used to make the flush phase in shutdown quicker


innobase/srv/srv0srv.c:
  Clean up the working of the main thread; add a tunable parameter srv_max_buf_pool_modified_pct which can be used to make the flush phase in shutdown quicker
innobase/include/buf0buf.h:
  Clean up the working of the main thread; add a tunable parameter srv_max_buf_pool_modified_pct which can be used to make the flush phase in shutdown quicker
innobase/include/srv0srv.h:
  Clean up the working of the main thread; add a tunable parameter srv_max_buf_pool_modified_pct which can be used to make the flush phase in shutdown quicker
innobase/buf/buf0buf.c:
  Clean up the working of the main thread; add a tunable parameter srv_max_buf_pool_modified_pct which can be used to make the flush phase in shutdown quicker

innobase/buf/buf0buf.c

@@ -1832,6 +1832,28 @@ buf_get_n_pending_ios(void)
 		+ buf_pool->n_flush[BUF_FLUSH_SINGLE_PAGE]);
 }
 
+/*************************************************************************
+Returns the ratio in percents of modified pages in the buffer pool /
+database pages in the buffer pool. */
+
+ulint
+buf_get_modified_ratio_pct(void)
+/*============================*/
+{
+	ulint	ratio;
+
+	mutex_enter(&(buf_pool->mutex));
+
+	ratio = (100 * UT_LIST_GET_LEN(buf_pool->flush_list))
+		     / (1 + UT_LIST_GET_LEN(buf_pool->LRU));
+
+		       /* 1 + is there to avoid division by zero */   
+
+	mutex_exit(&(buf_pool->mutex));
+
+	return(ratio);
+}
+
 /*************************************************************************
 Prints info of the buffer i/o. */
 

innobase/include/buf0buf.h

@@ -472,6 +472,13 @@ buf_print_io(
 /*=========*/
 	char*	buf,	/* in/out: buffer where to print */
 	char*	buf_end);/* in: buffer end */
+/*************************************************************************
+Returns the ratio in percents of modified pages in the buffer pool /
+database pages in the buffer pool. */
+
+ulint
+buf_get_modified_ratio_pct(void);
+/*============================*/
 /**************************************************************************
 Refreshes the statistics used to print per-second averages. */
 

innobase/include/srv0srv.h

@@ -74,6 +74,9 @@ extern ulint	srv_lock_wait_timeout;
 extern char*    srv_file_flush_method_str;
 extern ulint    srv_unix_file_flush_method;
 extern ulint   	srv_win_file_flush_method;
+
+extern ulint	srv_max_dirty_pages_pct;
+
 extern ulint	srv_force_recovery;
 extern ulint	srv_thread_concurrency;
 

innobase/srv/srv0srv.c

@@ -157,6 +157,13 @@ char*   srv_file_flush_method_str = NULL;
 ulint   srv_unix_file_flush_method = SRV_UNIX_FDATASYNC;
 ulint   srv_win_file_flush_method = SRV_WIN_IO_UNBUFFERED;
 
+/* The InnoDB main thread tries to keep the ratio of modified pages
+in the buffer pool to all database pages in the buffer pool smaller than
+the following number. But it is not guaranteed that the value stays below
+that during a time of heavy update/insert activity. */
+
+ulint	srv_max_buf_pool_modified_pct	= 30;
+
 /* If the following is != 0 we do not allow inserts etc. This protects
 the user from forgetting the innodb_force_recovery keyword to my.cnf */
 
@@ -2770,16 +2777,29 @@ srv_master_thread(
 
 	os_event_set(srv_sys->operational);
 loop:
+	/*****************************************************************/
+	/* ---- When there is database activity by users, we cycle in this
+	loop */
+
 	srv_main_thread_op_info = (char*) "reserving kernel mutex";
 
 	n_ios_very_old = log_sys->n_log_ios + buf_pool->n_pages_read
 						+ buf_pool->n_pages_written;
 	mutex_enter(&kernel_mutex);
 
+	/* Store the user activity counter at the start of this loop */
 	old_activity_count = srv_activity_count;
 
 	mutex_exit(&kernel_mutex);
 
+	if (srv_force_recovery >= SRV_FORCE_NO_BACKGROUND) {
+
+		goto suspend_thread;
+	}
+
+	/* ---- We run the following loop approximately once per second
+	when there is database activity */
+
 	for (i = 0; i < 10; i++) {
 		n_ios_old = log_sys->n_log_ios + buf_pool->n_pages_read
 						+ buf_pool->n_pages_written;
@@ -2797,11 +2817,6 @@ loop:
 
 		srv_main_thread_op_info = (char*)"";
 
-		if (srv_force_recovery >= SRV_FORCE_NO_BACKGROUND) {
-
-			goto suspend_thread;
-		}
-
 		if (srv_fast_shutdown && srv_shutdown_state > 0) {
 
 			goto background_loop;
@@ -2814,7 +2829,7 @@ loop:
 		srv_main_thread_op_info = (char*)"flushing log";
 		log_write_up_to(ut_dulint_max, LOG_WAIT_ONE_GROUP, TRUE);
 
-		/* If there were less than 10 i/os during the
+		/* If there were less than 5 i/os during the
 		one second sleep, we assume that there is free
 		disk i/o capacity available, and it makes sense to
 		do an insert buffer merge. */
@@ -2823,7 +2838,7 @@ loop:
 						+ log_sys->n_pending_writes;
 		n_ios = log_sys->n_log_ios + buf_pool->n_pages_read
 						+ buf_pool->n_pages_written;
-		if (n_pend_ios < 3 && (n_ios - n_ios_old < 10)) {
+		if (n_pend_ios < 3 && (n_ios - n_ios_old < 5)) {
 			srv_main_thread_op_info =
 					(char*)"doing insert buffer merge";
 			ibuf_contract_for_n_pages(TRUE, 5);
@@ -2834,19 +2849,27 @@ loop:
 									TRUE);
 		}
 
+		if (buf_get_modified_ratio_pct() >
+				             srv_max_buf_pool_modified_pct) {
+
+			/* Try to keep the number of modified pages in the
+			buffer pool under the limit wished by the user */
+			
+			n_pages_flushed = buf_flush_batch(BUF_FLUSH_LIST, 100,
+							  ut_dulint_max);
+		}
+
 		if (srv_activity_count == old_activity_count) {
 
-			if (srv_print_thread_releases) {
+			/* There is no user activity at the moment, go to
-				printf("Master thread wakes up!\n");
+			the background loop */
-			}
 
 			goto background_loop;
 		}
 	}
 
-	if (srv_print_thread_releases) {
+	/* ---- We perform the following code approximately once per
-		printf("Master thread wakes up!\n");
+	10 seconds when there is database activity */
-	}
 
 #ifdef MEM_PERIODIC_CHECK
 	/* Check magic numbers of every allocated mem block once in 10
@@ -2855,7 +2878,7 @@ loop:
 #endif	
 	/* If there were less than 200 i/os during the 10 second period,
 	we assume that there is free disk i/o capacity available, and it
-	makes sense to do a buffer pool flush. */
+	makes sense to flush 100 pages. */
 
 	n_pend_ios = buf_get_n_pending_ios() + log_sys->n_pending_writes;
 	n_ios = log_sys->n_log_ios + buf_pool->n_pages_read
@@ -2905,25 +2928,24 @@ loop:
 			last_flush_time = current_time;
 		}
 	}
-
-background_loop:
-	/* In this loop we run background operations when the server
-	is quiet and we also come here about once in 10 seconds */
-
-	srv_main_thread_op_info = (char*)"doing background drop tables";
-
-	n_tables_to_drop = row_drop_tables_for_mysql_in_background();
-
-	srv_main_thread_op_info = (char*)"";
 	
 	srv_main_thread_op_info = (char*)"flushing buffer pool pages";
 
-	/* Flush a few oldest pages to make the checkpoint younger */
+	/* Flush a few oldest pages to make a new checkpoint younger */
+
+	if (buf_get_modified_ratio_pct() > 70) {
+
+		/* If there are lots of modified pages in the buffer pool
+		(> 70 %), we assume we can afford reserving the disk(s) for
+		the time it requires to flush 100 pages */
 
-	if (srv_fast_shutdown && srv_shutdown_state > 0) {
 	        n_pages_flushed = buf_flush_batch(BUF_FLUSH_LIST, 100,
 							ut_dulint_max);
 	} else {
+	        /* Otherwise, we only flush a small number of pages so that
+		we do not unnecessarily use much disk i/o capacity from
+		other work */
+
 	        n_pages_flushed = buf_flush_batch(BUF_FLUSH_LIST, 10,
 							ut_dulint_max);
 	}
@@ -2937,16 +2959,31 @@ background_loop:
 	srv_main_thread_op_info = (char*)"reserving kernel mutex";
 
 	mutex_enter(&kernel_mutex);
+	
+	/* ---- When there is database activity, we jump from here back to
+	the start of loop */
+
 	if (srv_activity_count != old_activity_count) {
 		mutex_exit(&kernel_mutex);
 		goto loop;
 	}
-	old_activity_count = srv_activity_count;
+	
 	mutex_exit(&kernel_mutex);
 
+	/* If the database is quiet, we enter the background loop */
+
+	/*****************************************************************/
+background_loop:
+	/* ---- In this loop we run background operations when the server
+	is quiet from user activity */
+
 	/* The server has been quiet for a while: start running background
 	operations */
 		
+	srv_main_thread_op_info = (char*)"doing background drop tables";
+
+	n_tables_to_drop = row_drop_tables_for_mysql_in_background();
+
 	srv_main_thread_op_info = (char*)"purging";
 
 	if (srv_fast_shutdown && srv_shutdown_state > 0) {
@@ -2981,6 +3018,7 @@ background_loop:
 	}
 	mutex_exit(&kernel_mutex);
 	
+flush_loop:
 	srv_main_thread_op_info = (char*)"flushing buffer pool pages";
 	n_pages_flushed = buf_flush_batch(BUF_FLUSH_LIST, 100, ut_dulint_max);
 
@@ -3001,6 +3039,14 @@ background_loop:
 
 	log_checkpoint(TRUE, FALSE);
 
+	if (buf_get_modified_ratio_pct() > srv_max_buf_pool_modified_pct) {
+
+		/* Try to keep the number of modified pages in the
+		buffer pool under the limit wished by the user */
+			
+		goto flush_loop;
+	}
+
 	srv_main_thread_op_info = (char*)"reserving kernel mutex";
 
 	mutex_enter(&kernel_mutex);
@@ -3015,15 +3061,24 @@ background_loop:
 	
 	log_archive_do(FALSE, &n_bytes_archived);
 
+	/* Keep looping in the background loop if still work to do */
+
 	if (srv_fast_shutdown && srv_shutdown_state > 0) {
 		if (n_tables_to_drop + n_pages_flushed
 				+ n_bytes_archived != 0) {
 
+			/* If we are doing a fast shutdown (= the default)
+			we do not do purge or insert buffer merge. But we
+			flush the buffer pool completely to disk. */
+
 			goto background_loop;
 		}
 	} else if (n_tables_to_drop +
-		n_pages_purged + n_bytes_merged + n_pages_flushed
+		   n_pages_purged + n_bytes_merged + n_pages_flushed
 						+ n_bytes_archived != 0) {
+		/* In a 'slow' shutdown we run purge and the insert buffer
+		merge to completion */
+
 		goto background_loop;
 	}
 		
@@ -3055,6 +3110,9 @@ suspend_thread:
 
 	os_event_wait(event);
 
+	/* When there is user activity, InnoDB will set the event and the main
+	thread goes back to loop: */
+
 	goto loop;
 
 #ifndef __WIN__