BUG#40611: MySQL cannot make a binary log after sequential number

beyond unsigned long.
BUG#44779: binlog.binlog_max_extension may be causing failure on 
next test in PB
      
NOTE1: this is the backport to next-mr.
NOTE2: already includes patch for BUG#44779.
      
Binlog file extensions would turn into negative numbers once the
variable used to hold the value reached maximum for signed
long. Consequently, incrementing value to the next (negative) number
would lead to .000000 extension, causing the server to fail.
                  
This patch addresses this issue by not allowing negative extensions
and by returning an error on find_uniq_filename, when the limit is
reached. Additionally, warnings are printed to the error log when the
limit is approaching. FLUSH LOGS will also report warnings to the
user, if the extension number has reached the limit. The limit has been
set to 0x7FFFFFFF as the maximum.

mysql-test/suite/binlog/r/binlog_max_extension.result

@@ -0,0 +1,8 @@
+call mtr.add_suppression("Next log extension: 2147483647. Remaining log filename extensions: 0.");
+call mtr.add_suppression("Log filename extension number exhausted:");
+call mtr.add_suppression("Can't generate a unique log-filename");
+RESET MASTER;
+FLUSH LOGS;
+Warnings:
+Warning	1098	Can't generate a unique log-filename master-bin.(1-999)
+

mysql-test/suite/binlog/t/binlog_max_extension.test

@@ -0,0 +1,92 @@
+#  BUG#40611: MySQL cannot make a binary log after sequential number beyond 
+#             unsigned long.
+# 
+#  Problem statement
+#  =================
+#
+#    Extension for log file names might be created with negative 
+#    numbers (when counter used would wrap around), causing server 
+#    failure when incrementing -00001 (reaching number 000000 
+#    extension).
+#
+#  Test
+#  ====
+#    This tests aims at testing the a patch that removes negatives 
+#    numbers from log name extensions and checks that the server 
+#    reports gracefully that the limit has been reached.
+#
+#    It instruments index file to point to a log file close to
+#    the new maximum and calls flush logs to get warning.
+#
+
+call mtr.add_suppression("Next log extension: 2147483647. Remaining log filename extensions: 0.");
+call mtr.add_suppression("Log filename extension number exhausted:");
+call mtr.add_suppression("Can't generate a unique log-filename");
+
+
+-- source include/have_log_bin.inc
+RESET MASTER;
+
+-- let $MYSQLD_DATADIR= `select @@datadir`
+
+###############################################
+# check hitting maximum file name extension:
+###############################################
+
+##########
+# Prepare
+##########
+
+# 1. Stop master server
+-- write_file $MYSQLTEST_VARDIR/tmp/mysqld.1.expect
+wait
+EOF
+-- shutdown_server 10
+-- source include/wait_until_disconnected.inc
+
+# 2. Prepare log and index file
+-- copy_file $MYSQLD_DATADIR/master-bin.index $MYSQLD_DATADIR/master-bin.index.orig
+-- copy_file $MYSQLD_DATADIR/master-bin.000001 $MYSQLD_DATADIR/master-bin.2147483646
+-- append_file $MYSQLD_DATADIR/master-bin.index
+master-bin.2147483646
+EOF
+
+# 3. Restart the server
+-- append_file $MYSQLTEST_VARDIR/tmp/mysqld.1.expect
+restart
+EOF
+-- enable_reconnect
+-- source include/wait_until_connected_again.inc
+
+###########
+# Assertion
+###########
+
+# assertion: should throw warning
+FLUSH LOGS;
+
+##############
+# Clean up
+##############
+
+# 1. Stop the server
+-- write_file $MYSQLTEST_VARDIR/tmp/mysqld.1.expect
+wait
+EOF
+-- shutdown_server 10
+-- source include/wait_until_disconnected.inc
+
+# 2. Undo changes to index and log files
+-- remove_file $MYSQLD_DATADIR/master-bin.index
+-- copy_file $MYSQLD_DATADIR/master-bin.index.orig $MYSQLD_DATADIR/master-bin.index
+-- remove_file $MYSQLD_DATADIR/master-bin.index.orig
+
+-- remove_file $MYSQLD_DATADIR/master-bin.2147483646
+-- remove_file $MYSQLD_DATADIR/master-bin.2147483647
+
+# 3. Restart the server
+-- append_file $MYSQLTEST_VARDIR/tmp/mysqld.1.expect
+restart
+EOF
+-- enable_reconnect
+-- source include/wait_until_connected_again.inc

sql/log.cc

@@ -53,7 +53,7 @@ MYSQL_BIN_LOG mysql_bin_log;
 ulong sync_binlog_counter= 0;
 
 static bool test_if_number(const char *str,
-			   long *res, bool allow_wildcards);
+			   ulong *res, bool allow_wildcards);
 static int binlog_init(void *p);
 static int binlog_close_connection(handlerton *hton, THD *thd);
 static int binlog_savepoint_set(handlerton *hton, THD *thd, void *sv);
@@ -1835,22 +1835,27 @@ static void setup_windows_event_source()
 /**
   Find a unique filename for 'filename.#'.
 
-  Set '#' to a number as low as possible.
+  Set '#' to the number next to the maximum found in the most
+  recent log file extension.
+
+  This function will return nonzero if: (i) the generated name
+  exceeds FN_REFLEN; (ii) if the number of extensions is exhausted;
+  or (iii) some other error happened while examining the filesystem.
 
   @return
-    nonzero if not possible to get unique filename
+    nonzero if not possible to get unique filename.
 */
 
 static int find_uniq_filename(char *name)
 {
-  long                  number;
   uint                  i;
-  char                  buff[FN_REFLEN];
+  char                  buff[FN_REFLEN], ext_buf[FN_REFLEN];
   struct st_my_dir     *dir_info;
   reg1 struct fileinfo *file_info;
-  ulong                 max_found=0;
+  ulong                 max_found= 0, next= 0, number= 0;
   size_t		buf_length, length;
   char			*start, *end;
+  int                   error= 0;
   DBUG_ENTER("find_uniq_filename");
 
   length= dirname_part(buff, name, &buf_length);
@@ -1858,15 +1863,15 @@ static int find_uniq_filename(char *name)
   end=    strend(start);
 
   *end='.';
-  length= (size_t) (end-start+1);
+  length= (size_t) (end - start + 1);
 
-  if (!(dir_info = my_dir(buff,MYF(MY_DONT_SORT))))
+  if (!(dir_info= my_dir(buff,MYF(MY_DONT_SORT))))
   {						// This shouldn't happen
     strmov(end,".1");				// use name+1
-    DBUG_RETURN(0);
+    DBUG_RETURN(1);
   }
   file_info= dir_info->dir_entry;
-  for (i=dir_info->number_off_files ; i-- ; file_info++)
+  for (i= dir_info->number_off_files ; i-- ; file_info++)
   {
     if (bcmp((uchar*) file_info->name, (uchar*) start, length) == 0 &&
 	test_if_number(file_info->name+length, &number,0))
@@ -1876,9 +1881,44 @@ static int find_uniq_filename(char *name)
   }
   my_dirend(dir_info);
 
+  /* check if reached the maximum possible extension number */
+  if ((max_found == MAX_LOG_UNIQUE_FN_EXT))
+  {
+    sql_print_error("Log filename extension number exhausted: %06lu. \
+Please fix this by archiving old logs and \
+updating the index files.", max_found);
+    error= 1;
+    goto end;
+  }
+
+  next= max_found + 1;
+  sprintf(ext_buf, "%06lu", next);
   *end++='.';
-  sprintf(end,"%06ld",max_found+1);
-  DBUG_RETURN(0);
+
+  /* 
+    Check if the generated extension size + the file name exceeds the
+    buffer size used. If one did not check this, then the filename might be
+    truncated, resulting in error.
+   */
+  if (((strlen(ext_buf) + (end - name)) >= FN_REFLEN))
+  {
+    sql_print_error("Log filename too large: %s%s (%d). \
+Please fix this by archiving old logs and updating the \
+index files.", name, ext_buf, (strlen(ext_buf) + (end - name)));
+    error= 1;
+    goto end;
+  }
+
+  sprintf(end, "%06lu", next);
+
+  /* print warning if reaching the end of available extensions. */
+  if ((next > (MAX_LOG_UNIQUE_FN_EXT - LOG_WARN_UNIQUE_FN_EXT_LEFT)))
+    sql_print_warning("Next log extension: %lu. \
+Remaining log filename extensions: %lu. \
+Please consider archiving some logs.", next, (MAX_LOG_UNIQUE_FN_EXT - next));
+
+end:
+  DBUG_RETURN(error);
 }
 
 
@@ -2077,6 +2117,13 @@ int MYSQL_LOG::generate_new_name(char *new_name, const char *log_name)
     {
       if (find_uniq_filename(new_name))
       {
+        /* 
+          This should be treated as error once propagation of error further
+          up in the stack gets proper handling.
+        */
+        push_warning_printf(current_thd, MYSQL_ERROR::WARN_LEVEL_WARN, 
+                            ER_NO_UNIQUE_LOGFILE, ER(ER_NO_UNIQUE_LOGFILE),
+                            log_name);
 	sql_print_error(ER(ER_NO_UNIQUE_LOGFILE), log_name);
 	return 1;
       }
@@ -4739,11 +4786,11 @@ void MYSQL_BIN_LOG::set_max_size(ulong max_size_arg)
   @retval
     1	String is a number
   @retval
-    0	Error
+    0	String is not a number
 */
 
 static bool test_if_number(register const char *str,
-			   long *res, bool allow_wildcards)
+			   ulong *res, bool allow_wildcards)
 {
   reg2 int flag;
   const char *start;

sql/log.h

@@ -121,6 +121,19 @@ extern TC_LOG_DUMMY tc_log_dummy;
 #define LOG_CLOSE_TO_BE_OPENED	2
 #define LOG_CLOSE_STOP_EVENT	4
 
+/* 
+  Maximum unique log filename extension.
+  Note: setting to 0x7FFFFFFF due to atol windows 
+        overflow/truncate.
+ */
+#define MAX_LOG_UNIQUE_FN_EXT 0x7FFFFFFF
+
+/* 
+   Number of warnings that will be printed to error log
+   before extension number is exhausted.
+*/
+#define LOG_WARN_UNIQUE_FN_EXT_LEFT 1000
+
 class Relay_log_info;
 
 typedef struct st_log_info