MDEV-17816 Crash in TRUNCATE TABLE when table creation fails

The error handling in the MDEV-13564 TRUNCATE TABLE was broken
when an error occurred during table creation.

row_create_index_for_mysql(): Do not drop the table on error.

fts_create_one_common_table(), fts_create_one_index_table():
Do drop the table on error.

create_index(), create_table_info_t::create_table():
Let the caller handle the index creation errors.

ha_innobase::create(): If create_table_info_t::create_table()
fails, drop the incomplete table, roll back the transaction,
and finally return an error to the caller.

mysql-test/suite/innodb/r/truncate.result

@@ -6,3 +6,15 @@ connection default;
 TRUNCATE TABLE t;
 disconnect dml;
 DROP TABLE t;
+#
+# MDEV-17816 Crash in TRUNCATE TABLE when table creation fails
+#
+CREATE TABLE t1 (c VARCHAR(1024), KEY(c)) ENGINE=InnoDB ROW_FORMAT=DYNAMIC;
+INSERT INTO t1 SET c='character';
+ALTER TABLE t1 ROW_FORMAT=REDUNDANT;
+TRUNCATE TABLE t1;
+ERROR HY000: Index column size too large. The maximum column size is 767 bytes
+SELECT * FROM t1;
+c
+character
+DROP TABLE t1;

mysql-test/suite/innodb/t/truncate.test

@@ -15,3 +15,15 @@ TRUNCATE TABLE t;
 disconnect dml;
 
 DROP TABLE t;
+
+--echo #
+--echo # MDEV-17816 Crash in TRUNCATE TABLE when table creation fails
+--echo #
+CREATE TABLE t1 (c VARCHAR(1024), KEY(c)) ENGINE=InnoDB ROW_FORMAT=DYNAMIC;
+INSERT INTO t1 SET c='character';
+# FIXME: MDEV-17833 ALTER TABLE is not enforcing prefix index size limit
+ALTER TABLE t1 ROW_FORMAT=REDUNDANT;
+--error ER_INDEX_COLUMN_TOO_LONG
+TRUNCATE TABLE t1;
+SELECT * FROM t1;
+DROP TABLE t1;

storage/innobase/fts/fts0fts.cc

@@ -1771,7 +1771,7 @@ fts_create_one_common_table(
 	const char*		fts_suffix,
 	mem_heap_t*		heap)
 {
-	dict_table_t*		new_table = NULL;
+	dict_table_t*		new_table;
 	dberr_t			error;
 	bool			is_config = strcmp(fts_suffix, "CONFIG") == 0;
 
@@ -1823,11 +1823,13 @@ fts_create_one_common_table(
 	}
 
 	if (error != DB_SUCCESS) {
-		trx->error_state = error;
 		dict_mem_table_free(new_table);
 		new_table = NULL;
 		ib::warn() << "Failed to create FTS common table "
 			<< fts_table_name;
+		trx->error_state = DB_SUCCESS;
+		row_drop_table_for_mysql(fts_table_name, trx, SQLCOM_DROP_DB);
+		trx->error_state = error;
 	}
 	return(new_table);
 }
@@ -1969,7 +1971,7 @@ fts_create_one_index_table(
 	mem_heap_t*		heap)
 {
 	dict_field_t*		field;
-	dict_table_t*		new_table = NULL;
+	dict_table_t*		new_table;
 	char			table_name[MAX_FULL_NAME_LEN];
 	dberr_t			error;
 	CHARSET_INFO*		charset;
@@ -2032,11 +2034,13 @@ fts_create_one_index_table(
 	}
 
 	if (error != DB_SUCCESS) {
-		trx->error_state = error;
 		dict_mem_table_free(new_table);
 		new_table = NULL;
 		ib::warn() << "Failed to create FTS index table "
 			<< table_name;
+		trx->error_state = DB_SUCCESS;
+		row_drop_table_for_mysql(table_name, trx, SQLCOM_DROP_DB);
+		trx->error_state = error;
 	}
 
 	return(new_table);

storage/innobase/handler/ha_innodb.cc

@@ -12589,11 +12589,14 @@ int create_table_info_t::create_table(bool create_fk)
 		dict_table_close(innobase_table, TRUE, FALSE);
 
 		if (error) {
-			trx_rollback_to_savepoint(m_trx, NULL);
+			/* Drop the being-created table before rollback,
+			so that rollback can possibly rename back a table
+			that could have been renamed before
+			the failed creation. */
 			m_trx->error_state = DB_SUCCESS;
-
 			row_drop_table_for_mysql(m_table_name, m_trx,
 						 SQLCOM_DROP_DB);
+			trx_rollback_to_savepoint(m_trx, NULL);
 
 			m_trx->error_state = DB_SUCCESS;
 			DBUG_RETURN(error);
@@ -12835,12 +12838,18 @@ ha_innobase::create(
 	}
 
 	if ((error = info.create_table(own_trx))) {
+		/* Drop the being-created table before rollback,
+		so that rollback can possibly rename back a table
+		that could have been renamed before the failed creation. */
+		trx->error_state = DB_SUCCESS;
+		row_drop_table_for_mysql(info.table_name(), trx,
+					 SQLCOM_DROP_DB, true);
 		trx_rollback_for_mysql(trx);
 		row_mysql_unlock_data_dictionary(trx);
 		if (own_trx) {
 			trx_free_for_mysql(trx);
-			DBUG_RETURN(error);
 		}
+		DBUG_RETURN(error);
 	}
 
 	innobase_commit_low(trx);

storage/innobase/include/row0mysql.h

@@ -359,9 +359,8 @@ row_create_table_for_mysql(
 	MY_ATTRIBUTE((warn_unused_result));
 
 /*********************************************************************//**
-Does an index creation operation for MySQL. TODO: currently failure
+Create an index when creating a table.
-to create an index results in dropping the whole table! This is no problem
+On failure, the caller must drop the table!
-currently as all indexes must be created at the same time as the table.
 @return error number or DB_SUCCESS */
 dberr_t
 row_create_index_for_mysql(

storage/innobase/row/row0mysql.cc

@@ -2354,9 +2354,8 @@ err_exit:
 }
 
 /*********************************************************************//**
-Does an index creation operation for MySQL. TODO: currently failure
+Create an index when creating a table.
-to create an index results in dropping the whole table! This is no problem
+On failure, the caller must drop the table!
-currently as all indexes must be created at the same time as the table.
 @return error number or DB_SUCCESS */
 dberr_t
 row_create_index_for_mysql(
@@ -2490,27 +2489,6 @@ row_create_index_for_mysql(
 error_handling:
 	dict_table_close(table, TRUE, FALSE);
 
-	if (err != DB_SUCCESS) {
-		/* We have special error handling here */
-
-		trx->error_state = DB_SUCCESS;
-
-		if (trx_is_started(trx)) {
-
-			trx_rollback_to_savepoint(trx, NULL);
-		}
-
-		row_drop_table_for_mysql(table_name, trx, SQLCOM_DROP_TABLE,
-					 true);
-
-		if (trx_is_started(trx)) {
-
-			trx_commit_for_mysql(trx);
-		}
-
-		trx->error_state = DB_SUCCESS;
-	}
-
 	trx->op_info = "";
 
 	ut_free(table_name);