InnoDB: Avoid test suite failures caused by a locking conflict

between two server instances at server shutdown/startup.
This conflict on advisory locks appears to be the result of a bug
in the operating system; these locks should be released when the
files are closed, but somehow that does not always happen
immediately in Linux.  (Bug #9381)


innobase/include/os0file.h:
  Add OS_FILE_OPEN_RETRY for os_file_create()ing ibdata1
innobase/os/os0file.c:
  os_file_lock(): Do not close the file on failure, but let the
  callers do that.
  os_file_create(): If create_mode==OS_FILE_OPEN_RETRY and
  os_file_lock() fails, keep retrying for 100 seconds.
innobase/srv/srv0start.c:
  open_or_create_data_files(): Open the first data file with
  OS_FILE_OPEN_RETRY, to resolve a conflict with a shutting-down
  instance of the MySQL server.

innobase/include/os0file.h

@@ -65,6 +65,8 @@ log. */
 #define OS_FILE_OVERWRITE		53
 #define OS_FILE_OPEN_RAW		54
 #define	OS_FILE_CREATE_PATH		55
+#define	OS_FILE_OPEN_RETRY		56	/* for os_file_create() on
+						the first ibdata file */
 
 #define OS_FILE_READ_ONLY 		333
 #define	OS_FILE_READ_WRITE		444

innobase/os/os0file.c

@@ -402,8 +402,6 @@ os_file_lock(
 "InnoDB: using the same InnoDB data or log files.\n");
 		}
 
-		close(fd);
-
 		return(-1);
 	}
 
@@ -978,6 +976,7 @@ try_again:
 	} else if (access_type == OS_FILE_READ_WRITE
 			&& os_file_lock(file, name)) {
 		*success = FALSE;
+		close(file);
 		file = -1;
 #endif
 	} else {
@@ -1090,6 +1089,7 @@ os_file_create_simple_no_error_handling(
 	} else if (access_type == OS_FILE_READ_WRITE
 			&& os_file_lock(file, name)) {
 		*success = FALSE;
+		close(file);
 		file = -1;
 #endif
 	} else {
@@ -1141,7 +1141,8 @@ try_again:
 	if (create_mode == OS_FILE_OPEN_RAW) {
 		create_flag = OPEN_EXISTING;
 		share_mode = FILE_SHARE_WRITE;
-	} else if (create_mode == OS_FILE_OPEN) {
+	} else if (create_mode == OS_FILE_OPEN
+			|| create_mode == OS_FILE_OPEN_RETRY) {
 		create_flag = OPEN_EXISTING;
 	} else if (create_mode == OS_FILE_CREATE) {
 		create_flag = CREATE_NEW;
@@ -1232,7 +1233,8 @@ try_again:
 try_again:	
 	ut_a(name);
 
-	if (create_mode == OS_FILE_OPEN || create_mode == OS_FILE_OPEN_RAW) {
+	if (create_mode == OS_FILE_OPEN || create_mode == OS_FILE_OPEN_RAW
+			|| create_mode == OS_FILE_OPEN_RETRY) {
 		mode_str = "OPEN";
 		create_flag = O_RDWR;
 	} else if (create_mode == OS_FILE_CREATE) {
@@ -1305,6 +1307,23 @@ try_again:
 	} else if (create_mode != OS_FILE_OPEN_RAW
 			&& os_file_lock(file, name)) {
 		*success = FALSE;
+		if (create_mode == OS_FILE_OPEN_RETRY) {
+			int i;
+			ut_print_timestamp(stderr);
+			fputs("  InnoDB: Retrying to lock the first data file\n",
+				stderr);
+			for (i = 0; i < 100; i++) {
+				os_thread_sleep(1000000);
+				if (!os_file_lock(file, name)) {
+					*success = TRUE;
+					return(file);
+				}
+			}
+			ut_print_timestamp(stderr);
+			fputs("  InnoDB: Unable to open the first data file\n",
+				stderr);
+		}
+		close(file);
 		file = -1;
 #endif
 	} else {

innobase/srv/srv0start.c

@@ -789,6 +789,11 @@ open_or_create_data_files(
 				files[i] = os_file_create(
 					name, OS_FILE_OPEN_RAW, OS_FILE_NORMAL,
 							 OS_DATA_FILE, &ret);
+			} else if (i == 0) {
+				files[i] = os_file_create(
+					name, OS_FILE_OPEN_RETRY,
+							OS_FILE_NORMAL,
+							OS_DATA_FILE, &ret);
 			} else {
 				files[i] = os_file_create(
 					name, OS_FILE_OPEN, OS_FILE_NORMAL,