MDEV-8577: With enforce-storage-engine mysql_upgrade corrupts the schema:

ALTER TABLE should either bypass enforce-storage-engine, or mysql_upgrade
should refuse to run

Allow user to alter contents of existing table without enforcing
storage engine. However, enforce storage engine on ALTER TABLE
x ENGINE=y;

mysql-test/r/enforce_storage_engine.result

@@ -122,3 +122,33 @@ DROP TABLE t1;
 set global sql_mode=default;
 SET SESSION enforce_storage_engine=NULL;
 SET GLOBAL enforce_storage_engine=NULL;
+CREATE TABLE t3 (c1 INT PRIMARY KEY AUTO_INCREMENT, c2 VARCHAR(10)) ENGINE=MyISAM;
+INSERT INTO t3 values (NULL, 'test');
+SET SESSION enforce_storage_engine=Memory;
+ALTER TABLE t3 ENGINE=MyISAM;
+Warnings:
+Note	1266	Using storage engine MEMORY for table 't3'
+SHOW CREATE TABLE t3;
+Table	Create Table
+t3	CREATE TABLE `t3` (
+  `c1` int(11) NOT NULL AUTO_INCREMENT,
+  `c2` varchar(10) DEFAULT NULL,
+  PRIMARY KEY (`c1`)
+) ENGINE=MEMORY AUTO_INCREMENT=2 DEFAULT CHARSET=latin1
+DROP TABLE t3;
+SET SESSION enforce_storage_engine=NULL;
+CREATE TABLE t3 (c1 INT PRIMARY KEY AUTO_INCREMENT, c2 VARCHAR(10)) ENGINE=MyISAM;
+INSERT INTO t3 values (NULL, 'test');
+SET SESSION enforce_storage_engine=Memory;
+ALTER TABLE t3 ADD COLUMN c3 INT;
+SHOW CREATE TABLE t3;
+Table	Create Table
+t3	CREATE TABLE `t3` (
+  `c1` int(11) NOT NULL AUTO_INCREMENT,
+  `c2` varchar(10) DEFAULT NULL,
+  `c3` int(11) DEFAULT NULL,
+  PRIMARY KEY (`c1`)
+) ENGINE=MyISAM AUTO_INCREMENT=2 DEFAULT CHARSET=latin1
+DROP TABLE t3;
+SET SESSION enforce_storage_engine=NULL;
+SET GLOBAL enforce_storage_engine=NULL;

mysql-test/r/mysql_upgrade.result

@@ -459,4 +459,64 @@ even_longer_user_name_number_3_to_test_the_grantor_and_definer_field_length@loca
 DROP USER very_long_user_name_number_1, very_long_user_name_number_2, even_longer_user_name_number_3_to_test_the_grantor_and_definer_field_length@localhost;
 DROP PROCEDURE test.pr;
 set sql_mode=default;
+# Droping the previously created mysql_upgrade_info file..
+create table test.t1(a int) engine=MyISAM;
+# Trying to enforce InnoDB for all tables
+SET GLOBAL enforce_storage_engine=InnoDB;
+Phase 1/6: Checking and upgrading mysql database
+Processing databases
+mysql
+mysql.column_stats                                 OK
+mysql.columns_priv                                 OK
+mysql.db                                           OK
+mysql.event                                        OK
+mysql.func                                         OK
+mysql.gtid_slave_pos                               OK
+mysql.help_category                                OK
+mysql.help_keyword                                 OK
+mysql.help_relation                                OK
+mysql.help_topic                                   OK
+mysql.host                                         OK
+mysql.index_stats                                  OK
+mysql.innodb_index_stats                           OK
+mysql.innodb_table_stats                           OK
+mysql.plugin                                       OK
+mysql.proc                                         OK
+mysql.procs_priv                                   OK
+mysql.proxies_priv                                 OK
+mysql.roles_mapping                                OK
+mysql.servers                                      OK
+mysql.table_stats                                  OK
+mysql.tables_priv                                  OK
+mysql.time_zone                                    OK
+mysql.time_zone_leap_second                        OK
+mysql.time_zone_name                               OK
+mysql.time_zone_transition                         OK
+mysql.time_zone_transition_type                    OK
+mysql.user                                         OK
+Phase 2/6: Fixing views
+Phase 3/6: Running 'mysql_fix_privilege_tables'
+Phase 4/6: Fixing table and database names
+Phase 5/6: Checking and upgrading tables
+Processing databases
+information_schema
+mtr
+mtr.global_suppressions                            OK
+mtr.test_suppressions                              OK
+performance_schema
+test
+test.t1                                            OK
+Phase 6/6: Running 'FLUSH PRIVILEGES'
+OK
+# Should return 2
+SELECT count(*) FROM information_schema.tables where ENGINE="InnoDB";
+count(*)
+2
+SHOW CREATE TABLE test.t1;
+Table	Create Table
+t1	CREATE TABLE `t1` (
+  `a` int(11) DEFAULT NULL
+) ENGINE=MyISAM DEFAULT CHARSET=latin1
+DROP TABLE test.t1;
+SET GLOBAL enforce_storage_engine=NULL;
 End of tests

mysql-test/t/enforce_storage_engine.test

@@ -87,3 +87,25 @@ disconnect con1;
 set global sql_mode=default;
 SET SESSION enforce_storage_engine=NULL;
 SET GLOBAL enforce_storage_engine=NULL;
+
+#
+# MDEV-8577: With enforce-storage-engine mysql_upgrade corrupts the schema:
+#   ALTER TABLE should either bypass enforce-storage-engine, or mysql_upgrade
+#   should refuse to run
+#
+CREATE TABLE t3 (c1 INT PRIMARY KEY AUTO_INCREMENT, c2 VARCHAR(10)) ENGINE=MyISAM;
+INSERT INTO t3 values (NULL, 'test');
+SET SESSION enforce_storage_engine=Memory;
+ALTER TABLE t3 ENGINE=MyISAM;
+SHOW CREATE TABLE t3;
+DROP TABLE t3;
+SET SESSION enforce_storage_engine=NULL;
+CREATE TABLE t3 (c1 INT PRIMARY KEY AUTO_INCREMENT, c2 VARCHAR(10)) ENGINE=MyISAM;
+INSERT INTO t3 values (NULL, 'test');
+SET SESSION enforce_storage_engine=Memory;
+ALTER TABLE t3 ADD COLUMN c3 INT;
+SHOW CREATE TABLE t3;
+DROP TABLE t3;
+
+SET SESSION enforce_storage_engine=NULL;
+SET GLOBAL enforce_storage_engine=NULL;

mysql-test/t/mysql_upgrade.test

@@ -161,4 +161,27 @@ DROP USER very_long_user_name_number_1, very_long_user_name_number_2, even_longe
 DROP PROCEDURE test.pr;
 
 set sql_mode=default;
+
+#
+# Enforce storage engine option should not effect mysql_upgrade
+#
+--echo # Droping the previously created mysql_upgrade_info file..
+--remove_file $MYSQLD_DATADIR/mysql_upgrade_info
+
+create table test.t1(a int) engine=MyISAM;
+--echo # Trying to enforce InnoDB for all tables
+SET GLOBAL enforce_storage_engine=InnoDB;
+
+--replace_result $MYSQLTEST_VARDIR var
+--exec $MYSQL_UPGRADE --force 2>&1
+
+--echo # Should return 2
+SELECT count(*) FROM information_schema.tables where ENGINE="InnoDB";
+SHOW CREATE TABLE test.t1;
+DROP TABLE test.t1;
+# mysql_upgrade must have created mysql_upgrade_info file,
+# so the following command should never fail.
+--remove_file $MYSQLD_DATADIR/mysql_upgrade_info
+SET GLOBAL enforce_storage_engine=NULL;
+
 --echo End of tests

sql/sql_table.cc

@@ -9818,14 +9818,21 @@ static bool check_engine(THD *thd, const char *db_name,
   DBUG_ENTER("check_engine");
   handlerton **new_engine= &create_info->db_type;
   handlerton *req_engine= *new_engine;
-  handlerton *enf_engine= thd->variables.enforced_table_plugin ?
+  handlerton *enf_engine= NULL;
-    plugin_hton(thd->variables.enforced_table_plugin) : NULL;
   bool no_substitution= thd->variables.sql_mode & MODE_NO_ENGINE_SUBSTITUTION;
   *new_engine= ha_checktype(thd, req_engine, no_substitution);
   DBUG_ASSERT(*new_engine);
   if (!*new_engine)
     DBUG_RETURN(true);
 
+  /* Enforced storage engine should not be used in
+  ALTER TABLE that does not use explicit ENGINE = x to
+  avoid unwanted unrelated changes.*/
+  if (!(thd->lex->sql_command == SQLCOM_ALTER_TABLE &&
+        !(create_info->used_fields & HA_CREATE_USED_ENGINE)))
+    enf_engine= thd->variables.enforced_table_plugin ?
+       plugin_hton(thd->variables.enforced_table_plugin) : NULL;
+
   if (enf_engine && enf_engine != *new_engine)
   {
     if (no_substitution)