Merge 10.2 into bb-10.2-ext

extra/mariabackup/xtrabackup.cc

@@ -2908,19 +2908,7 @@ xb_load_tablespaces()
 					   &flush_lsn);
 
 	if (err != DB_SUCCESS) {
-		msg("mariabackup: Could not open or create data files.\n"
+		msg("mariabackup: Could not open data files.\n");
-		    "mariabackup: If you tried to add new data files, and it "
-		    "failed here,\n"
-		    "mariabackup: you should now edit innodb_data_file_path in "
-		    "my.cnf back\n"
-		    "mariabackup: to what it was, and remove the new ibdata "
-		    "files InnoDB created\n"
-		    "mariabackup: in this failed attempt. InnoDB only wrote "
-		    "those files full of\n"
-		    "mariabackup: zeros, but did not yet use them in any way. "
-		    "But be careful: do not\n"
-		    "mariabackup: remove old data files which contain your "
-		    "precious data!\n");
 		return(err);
 	}
 
@@ -3859,7 +3847,7 @@ reread_log_header:
 	err = xb_load_tablespaces();
 	if (err != DB_SUCCESS) {
 		msg("mariabackup: error: xb_load_tablespaces() failed with"
-		    "error code %u\n", err);
+		    " error code %u\n", err);
 		goto fail;
 	}
 

mysql-test/suite/innodb/r/read_only_recover_committed.result

@@ -0,0 +1,47 @@
+connect con1, localhost, root;
+CREATE TABLE t(a INT PRIMARY KEY) ENGINE=InnoDB;
+INSERT INTO t VALUES(1);
+BEGIN;
+INSERT INTO t VALUES(2);
+DELETE FROM t WHERE a=2;
+connect con2, localhost, root;
+# Normal MariaDB shutdown would roll back the above transaction.
+# We want the transaction to remain open, so we will kill the server
+# after ensuring that any non-transactional files are clean.
+FLUSH TABLES;
+# Create another transaction that will be recovered as COMMITTED.
+BEGIN;
+SET DEBUG_SYNC='after_trx_committed_in_memory SIGNAL committed WAIT_FOR ever';
+COMMIT;
+connection default;
+SET DEBUG_SYNC='now WAIT_FOR committed';
+# Ensure that the above incomplete transactions become durable.
+SET GLOBAL innodb_flush_log_at_trx_commit=1;
+BEGIN;
+INSERT INTO t VALUES(-10000);
+ROLLBACK;
+disconnect con1;
+disconnect con2;
+SELECT * FROM t;
+a
+1
+SET TRANSACTION ISOLATION LEVEL READ UNCOMMITTED;
+SELECT * FROM t;
+a
+1
+UPDATE t SET a=3 WHERE a=1;
+# Starting with MariaDB 10.2, innodb_read_only implies READ UNCOMMITTED.
+# In earlier versions, this would return the last committed version
+# (empty table)!
+SELECT * FROM t;
+a
+3
+SET TRANSACTION ISOLATION LEVEL READ UNCOMMITTED;
+SELECT * FROM t;
+a
+3
+SELECT * FROM t;
+a
+3
+DROP TABLE t;
+FOUND 1 /Rolled back recovered transaction [^0]/ in mysqld.1.err

mysql-test/suite/innodb/r/table_flags.result

@@ -116,8 +116,16 @@ SHOW CREATE TABLE tr;
 ERROR 42S02: Table 'test.tr' doesn't exist in engine
 SHOW CREATE TABLE tc;
 ERROR 42S02: Table 'test.tc' doesn't exist in engine
+SELECT * FROM tc;
+ERROR 42S02: Table 'test.tc' doesn't exist in engine
 SHOW CREATE TABLE td;
-ERROR 42S02: Table 'test.td' doesn't exist in engine
+Table	Create Table
+td	CREATE TABLE `td` (
+  `a` int(11) NOT NULL,
+  PRIMARY KEY (`a`)
+) ENGINE=InnoDB DEFAULT CHARSET=latin1 ROW_FORMAT=DYNAMIC
+SELECT * FROM td;
+a
 SHOW CREATE TABLE tz;
 Table	Create Table
 tz	CREATE TABLE `tz` (
@@ -132,8 +140,7 @@ a
 42
 SHOW CREATE TABLE tp;
 ERROR 42S02: Table 'test.tp' doesn't exist in engine
-FOUND 4 /InnoDB: Table `test`.`t[cp]` in InnoDB data dictionary contains invalid flags\. SYS_TABLES\.TYPE=(129|289|3873|1232[13]) SYS_TABLES\.N_COLS=2147483649/ in mysqld.1.err
+FOUND 5 /InnoDB: Table `test`.`t[cp]` in InnoDB data dictionary contains invalid flags\. SYS_TABLES\.TYPE=(129|289|3873|1232[13]) SYS_TABLES\.N_COLS=2147483649/ in mysqld.1.err
-FOUND 2 /InnoDB: Refusing to load '\..test.td\.ibd' \(id=3, flags=0x1?[2ae]1\); dictionary contains id=3, flags=0x10[01][2ae]1\b/ in mysqld.1.err
 FOUND 2 /InnoDB: Table `test`\.`tr` in InnoDB data dictionary contains invalid flags\. SYS_TABLES\.TYPE=65 SYS_TABLES\.MIX_LEN=4294967295\b/ in mysqld.1.err
 Restoring SYS_TABLES clustered index root page (8)
 SHOW CREATE TABLE tr;

mysql-test/suite/innodb/t/log_file_name.test

@@ -163,6 +163,7 @@ call mtr.add_suppression("InnoDB: Cannot rename.*because the target file exists"
 call mtr.add_suppression("InnoDB: Log scan aborted at LSN");
 # The following are for the --innodb-force-recovery=1 with broken u* tables:
 call mtr.add_suppression("InnoDB: Header page consists of zero bytes in datafile: .*u1.ibd");
+call mtr.add_suppression("InnoDB: The size of the file .*u1\\.ibd is only 16384 bytes, should be at least 65536");
 call mtr.add_suppression("InnoDB: The error means the system cannot find the path specified");
 call mtr.add_suppression("InnoDB: .*you must create directories");
 call mtr.add_suppression("InnoDB: Cannot open datafile for read-only: '.*u[1-5]\.ibd'");

mysql-test/suite/innodb/t/read_only_recover_committed.test

@@ -0,0 +1,67 @@
+--source include/have_innodb.inc
+--source include/have_debug.inc
+--source include/have_debug_sync.inc
+# need to restart server
+--source include/not_embedded.inc
+
+--connect(con1, localhost, root)
+CREATE TABLE t(a INT PRIMARY KEY) ENGINE=InnoDB;
+INSERT INTO t VALUES(1);
+BEGIN;
+# Generate insert_undo log.
+INSERT INTO t VALUES(2);
+# Generate update_undo log.
+DELETE FROM t WHERE a=2;
+--connect(con2, localhost, root)
+--echo # Normal MariaDB shutdown would roll back the above transaction.
+--echo # We want the transaction to remain open, so we will kill the server
+--echo # after ensuring that any non-transactional files are clean.
+FLUSH TABLES;
+--echo # Create another transaction that will be recovered as COMMITTED.
+BEGIN;
+# Generate multiple pages of both insert_undo and update_undo, so that
+# the state TRX_UNDO_CACHE will not be chosen.
+--disable_query_log
+let $n= 10000;
+while ($n) {
+dec $n;
+eval INSERT INTO t VALUES(-$n);
+eval DELETE FROM t WHERE a=-$n;
+}
+--enable_query_log
+SET DEBUG_SYNC='after_trx_committed_in_memory SIGNAL committed WAIT_FOR ever';
+send COMMIT;
+
+connection default;
+SET DEBUG_SYNC='now WAIT_FOR committed';
+--echo # Ensure that the above incomplete transactions become durable.
+SET GLOBAL innodb_flush_log_at_trx_commit=1;
+BEGIN;
+INSERT INTO t VALUES(-10000);
+ROLLBACK;
+--let $restart_parameters= --innodb-force-recovery=3
+--let $shutdown_timeout= 0
+--source include/restart_mysqld.inc
+--let $shutdown_timeout= 30
+--disconnect con1
+--disconnect con2
+SELECT * FROM t;
+SET TRANSACTION ISOLATION LEVEL READ UNCOMMITTED;
+SELECT * FROM t;
+# refused on MySQL 5.6, MariaDB 10.0, 10.1, but not MariaDB 10.2+
+UPDATE t SET a=3 WHERE a=1;
+--let $restart_parameters= --innodb-read-only
+--source include/restart_mysqld.inc
+--echo # Starting with MariaDB 10.2, innodb_read_only implies READ UNCOMMITTED.
+--echo # In earlier versions, this would return the last committed version
+--echo # (empty table)!
+SELECT * FROM t;
+SET TRANSACTION ISOLATION LEVEL READ UNCOMMITTED;
+SELECT * FROM t;
+--let $restart_parameters=
+--source include/restart_mysqld.inc
+SELECT * FROM t;
+DROP TABLE t;
+let SEARCH_FILE= $MYSQLTEST_VARDIR/log/mysqld.1.err;
+--let SEARCH_PATTERN= Rolled back recovered transaction [^0]
+--source include/search_pattern_in_file.inc

mysql-test/suite/innodb/t/table_flags.test

@@ -14,6 +14,7 @@ call mtr.add_suppression("InnoDB: Ignoring tablespace for `test`.`td` because it
 call mtr.add_suppression("InnoDB: Operating system error number .* in a file operation");
 call mtr.add_suppression("InnoDB: The error means the system cannot find the path specified");
 call mtr.add_suppression("InnoDB: If you are installing InnoDB, remember that you must create directories yourself");
+call mtr.add_suppression("InnoDB: adjusting FSP_SPACE_FLAGS of tablespace");
 FLUSH TABLES;
 --enable_query_log
 
@@ -133,7 +134,9 @@ SHOW CREATE TABLE tr;
 --error ER_NO_SUCH_TABLE_IN_ENGINE
 SHOW CREATE TABLE tc;
 --error ER_NO_SUCH_TABLE_IN_ENGINE
+SELECT * FROM tc;
 SHOW CREATE TABLE td;
+SELECT * FROM td;
 # This table was converted to NO_ROLLBACK due to the SYS_TABLES.TYPE change.
 SHOW CREATE TABLE tz;
 BEGIN;
@@ -148,8 +151,6 @@ SHOW CREATE TABLE tp;
 let SEARCH_FILE= $MYSQLTEST_VARDIR/log/mysqld.1.err;
 --let SEARCH_PATTERN= InnoDB: Table `test`.`t[cp]` in InnoDB data dictionary contains invalid flags\. SYS_TABLES\.TYPE=(129|289|3873|1232[13]) SYS_TABLES\.N_COLS=2147483649
 --source include/search_pattern_in_file.inc
---let SEARCH_PATTERN= InnoDB: Refusing to load '\..test.td\.ibd' \(id=3, flags=0x1?[2ae]1\); dictionary contains id=3, flags=0x10[01][2ae]1\b
---source include/search_pattern_in_file.inc
 --let SEARCH_PATTERN= InnoDB: Table `test`\.`tr` in InnoDB data dictionary contains invalid flags\. SYS_TABLES\.TYPE=65 SYS_TABLES\.MIX_LEN=4294967295\b
 --source include/search_pattern_in_file.inc
 

mysql-test/suite/mariabackup/huge_lsn.test

@@ -28,7 +28,6 @@ EOF
 --remove_files_wildcard $MYSQLD_DATADIR ib_logfile*
 
 --source include/start_mysqld.inc
-let SEARCH_RANGE= -50000;
 let SEARCH_FILE= $MYSQLTEST_VARDIR/log/mysqld.1.err;
 --let SEARCH_PATTERN= InnoDB: New log files created, LSN=175964\d{8}
 --source include/search_pattern_in_file.inc

storage/innobase/dict/dict0load.cc

@@ -1,7 +1,7 @@
 /*****************************************************************************
 
 Copyright (c) 1996, 2016, Oracle and/or its affiliates. All Rights Reserved.
-Copyright (c) 2016, 2017, MariaDB Corporation.
+Copyright (c) 2016, 2018, MariaDB Corporation.
 
 This program is free software; you can redistribute it and/or modify it under
 the terms of the GNU General Public License as published by the Free Software
@@ -1468,8 +1468,6 @@ dict_check_sys_tables(
 		char*	filepath = dict_get_first_path(space_id);
 
 		/* Check that the .ibd file exists. */
-		validate = true; /* Encryption */
-
 		dberr_t	err = fil_ibd_open(
 			validate,
 			!srv_read_only_mode && srv_log_file_size != 0,
@@ -3074,6 +3072,12 @@ err_exit:
 		} else {
 			dict_mem_table_fill_foreign_vcol_set(table);
 			table->fk_max_recusive_level = 0;
+
+			if (table->space
+			    && !fil_space_get_size(table->space)) {
+				table->corrupted = true;
+				table->file_unreadable = true;
+			}
 		}
 	} else {
 		dict_index_t*   index;

storage/innobase/fil/fil0fil.cc

@@ -624,6 +624,7 @@ retry:
 				<< " is only " << size_bytes
 				<< " bytes, should be at least " << min_size;
 			os_file_close(node->handle);
+			node->handle = OS_FILE_CLOSED;
 			return(false);
 		}
 
@@ -661,10 +662,12 @@ retry:
 
 		ut_free(buf2);
 		os_file_close(node->handle);
+		node->handle = OS_FILE_CLOSED;
 
 		if (!fsp_flags_is_valid(flags, space->id)) {
 			ulint cflags = fsp_flags_convert_from_101(flags);
-			if (cflags == ULINT_UNDEFINED) {
+			if (cflags == ULINT_UNDEFINED
+			    || (cflags ^ space->flags) & ~FSP_FLAGS_MEM_MASK) {
 				ib::error()
 					<< "Expected tablespace flags "
 					<< ib::hex(space->flags)
@@ -4638,7 +4641,9 @@ fsp_flags_try_adjust(ulint space_id, ulint flags)
 {
 	ut_ad(!srv_read_only_mode);
 	ut_ad(fsp_flags_is_valid(flags, space_id));
-
+	if (!fil_space_get_size(space_id)) {
+		return;
+	}
 	mtr_t	mtr;
 	mtr.start();
 	if (buf_block_t* b = buf_page_get(

storage/innobase/fsp/fsp0file.cc

@@ -1,7 +1,7 @@
 /*****************************************************************************
 
 Copyright (c) 2013, 2016, Oracle and/or its affiliates. All Rights Reserved.
-Copyright (c) 2017, MariaDB Corporation.
+Copyright (c) 2017, 2018, MariaDB Corporation.
 
 This program is free software; you can redistribute it and/or modify it under
 the terms of the GNU General Public License as published by the Free Software
@@ -770,6 +770,10 @@ the double write buffer.
 bool
 Datafile::restore_from_doublewrite()
 {
+	if (srv_operation != SRV_OPERATION_NORMAL) {
+		return true;
+	}
+
 	/* Find if double write buffer contains page_no of given space id. */
 	const byte*	page = recv_sys->dblwr.find_page(m_space_id, 0);
 	const page_id_t	page_id(m_space_id, 0);

storage/innobase/fsp/fsp0sysspace.cc

@@ -567,8 +567,9 @@ SysTablespace::read_lsn_and_check_flags(lsn_t* flushed_lsn)
 
 	ut_a(it->order() == 0);
 
-
+	if (srv_operation == SRV_OPERATION_NORMAL) {
-	buf_dblwr_init_or_load_pages(it->handle(), it->filepath());
+		buf_dblwr_init_or_load_pages(it->handle(), it->filepath());
+	}
 
 	/* Check the contents of the first page of the
 	first datafile. */

storage/innobase/handler/ha_innodb.cc

@@ -18579,7 +18579,7 @@ innodb_make_page_dirty(
 		return;
 	}
 
-	if (srv_saved_page_number_debug > space->size) {
+	if (srv_saved_page_number_debug >= space->size) {
 		fil_space_release(space);
 		return;
 	}

storage/innobase/include/fsp0file.h

@@ -1,6 +1,7 @@
 /*****************************************************************************
 
 Copyright (c) 2013, 2016, Oracle and/or its affiliates. All Rights Reserved.
+Copyright (c) 2018, MariaDB Corporation.
 
 This program is free software; you can redistribute it and/or modify it under
 the terms of the GNU General Public License as published by the Free Software
@@ -363,7 +364,7 @@ private:
 	@param[in]	read_only_mode	if true, then readonly mode checks
 					are enforced.
 	@return DB_SUCCESS or DB_IO_ERROR if page cannot be read */
-	dberr_t read_first_page(bool read_first_page)
+	dberr_t read_first_page(bool read_only_mode)
 		MY_ATTRIBUTE((warn_unused_result));
 
 	/** Free the first page from memory when it is no longer needed. */

storage/innobase/lock/lock0lock.cc

@@ -7484,7 +7484,10 @@ lock_trx_release_locks(
 
 		mutex_exit(&trx_sys->mutex);
 	} else {
-		ut_ad(trx_state_eq(trx, TRX_STATE_ACTIVE));
+		ut_ad(trx_state_eq(trx, TRX_STATE_ACTIVE)
+		      || (trx_state_eq(trx, TRX_STATE_COMMITTED_IN_MEMORY)
+			  && trx->is_recovered
+			  && !UT_LIST_GET_LEN(trx->lock.trx_locks)));
 	}
 
 	bool	release_lock;

storage/innobase/log/log0recv.cc

@@ -3147,7 +3147,9 @@ recv_init_crash_recovery_spaces()
 			<< "', but there were no modifications either.";
 	}
 
-	buf_dblwr_process();
+	if (srv_operation == SRV_OPERATION_NORMAL) {
+		buf_dblwr_process();
+	}
 
 	if (srv_force_recovery < SRV_FORCE_NO_LOG_REDO) {
 		/* Spawn the background thread to flush dirty pages

storage/innobase/os/os0file.cc

@@ -2,7 +2,7 @@
 
 Copyright (c) 1995, 2017, Oracle and/or its affiliates. All Rights Reserved.
 Copyright (c) 2009, Percona Inc.
-Copyright (c) 2013, 2017, MariaDB Corporation.
+Copyright (c) 2013, 2018, MariaDB Corporation.
 
 Portions of this file contain modifications contributed and copyrighted
 by Percona Inc.. Those modifications are
@@ -4994,7 +4994,7 @@ os_file_write_func(
 	if ((ulint) n_bytes != n && !os_has_said_disk_full) {
 
 		ib::error()
-			<< "Write to file " << name << "failed at offset "
+			<< "Write to file " << name << " failed at offset "
 			<< offset << ", " << n
 			<< " bytes should have been written,"
 			" only " << n_bytes << " were written."

storage/innobase/trx/trx0trx.cc

@@ -1,7 +1,7 @@
 /*****************************************************************************
 
 Copyright (c) 1996, 2016, Oracle and/or its affiliates. All Rights Reserved.
-Copyright (c) 2015, 2017, MariaDB Corporation.
+Copyright (c) 2015, 2018, MariaDB Corporation.
 
 This program is free software; you can redistribute it and/or modify it under
 the terms of the GNU General Public License as published by the Free Software

storage/innobase/trx/trx0undo.cc

@@ -1793,7 +1793,9 @@ trx_undo_commit_cleanup(trx_undo_t* undo, bool is_temp)
 
 		/* Delete first the undo log segment in the file */
 		mutex_exit(&rseg->mutex);
-		trx_undo_seg_free(undo, is_temp);
+		if (!srv_read_only_mode) {
+			trx_undo_seg_free(undo, is_temp);
+		}
 		mutex_enter(&rseg->mutex);
 
 		ut_ad(rseg->curr_size > undo->size);

storage/xtradb/lock/lock0lock.cc

@@ -8042,7 +8042,10 @@ lock_trx_release_locks(
 		}
 		mutex_exit(&trx_sys->mutex);
 	} else {
-		ut_ad(trx_state_eq(trx, TRX_STATE_ACTIVE));
+		ut_ad(trx_state_eq(trx, TRX_STATE_ACTIVE)
+		      || (trx_state_eq(trx, TRX_STATE_COMMITTED_IN_MEMORY)
+			  && trx->is_recovered
+			  && !UT_LIST_GET_LEN(trx->lock.trx_locks)));
 	}
 
 	/* The transition of trx->state to TRX_STATE_COMMITTED_IN_MEMORY

storage/xtradb/trx/trx0trx.cc

@@ -1,7 +1,7 @@
 /*****************************************************************************
 
 Copyright (c) 1996, 2016, Oracle and/or its affiliates. All Rights Reserved.
-Copyright (c) 2015, 2017, MariaDB Corporation.
+Copyright (c) 2015, 2018, MariaDB Corporation.
 
 This program is free software; you can redistribute it and/or modify it under
 the terms of the GNU General Public License as published by the Free Software
@@ -1472,6 +1472,8 @@ trx_commit_in_memory(
 	if (lsn) {
 		ulint	flush_log_at_trx_commit;
 
+		DEBUG_SYNC_C("after_trx_committed_in_memory");
+
 		if (trx->insert_undo != NULL) {
 
 			trx_undo_insert_cleanup(trx);

storage/xtradb/trx/trx0undo.cc

@@ -1,7 +1,7 @@
 /*****************************************************************************
 
 Copyright (c) 1996, 2016, Oracle and/or its affiliates. All Rights Reserved.
-Copyright (c) 2014, 2017, MariaDB Corporation.
+Copyright (c) 2014, 2018, MariaDB Corporation.
 
 This program is free software; you can redistribute it and/or modify it under
 the terms of the GNU General Public License as published by the Free Software
@@ -1989,7 +1989,9 @@ trx_undo_insert_cleanup(
 
 		mutex_exit(&(rseg->mutex));
 
-		trx_undo_seg_free(undo);
+		if (!srv_read_only_mode) {
+			trx_undo_seg_free(undo);
+		}
 
 		mutex_enter(&(rseg->mutex));