Fixed bug in lock tables + alter table with Aria tables.

MDEV-18451 Server crashes in maria_create_trn_for_mysql
upon ALTER TABLE

Problem was that when table was locked many times, not all
instances where removed from the transaction by
_ma_remove_table_from_trnman()

mysql-test/r/kill.result

@@ -412,7 +412,6 @@ LOCK TABLES v1 READ, t2 WRITE, t1 WRITE;
 ALTER TABLE t1 CHANGE f1 f2 DOUBLE;
 Got one of the listed errors
 ALTER TABLE t2 DROP c;
-ERROR 70100: Query execution was interrupted (max_statement_time exceeded)
 UNLOCK TABLES;
 DROP VIEW v1;
 DROP TABLE t1, t2;

mysql-test/suite/maria/lock.result

@@ -121,4 +121,25 @@ ALTER TABLE t1 CHANGE b a INT;
 ERROR 42S21: Duplicate column name 'a'
 UNLOCK TABLES;
 DROP TABLE t1, t2;
+#
+# MDEV-10748 Server crashes in ha_maria::implicit_commit upon ALTER TABLE
+#
+CREATE TABLE t1 (a INT) ENGINE=Aria;
+CREATE TABLE t2 (b INT) ENGINE=Aria;
+LOCK TABLES t1 WRITE, t2 AS t2a WRITE, t2 WRITE;
+ALTER TABLE t2 CHANGE b c VARBINARY(30000), ALGORITHM=COPY;
+UNLOCK TABLES;
+DROP TABLE t1, t2;
+# More complex test, from RQG
+CREATE TABLE t1 (a INT) ENGINE=Aria;
+CREATE TABLE t2 (b INT) ENGINE=Aria;
+CREATE OR REPLACE VIEW v2 AS SELECT * FROM t2 ;
+LOCK TABLES t1 WRITE, t2 AS t2a WRITE, v2 WRITE CONCURRENT, t2 WRITE;
+ALTER TABLE t1 FORCE;
+ALTER TABLE t2 CHANGE b c VARBINARY(30000), ALGORITHM=INPLACE;
+ERROR 0A000: ALGORITHM=INPLACE is not supported for this operation. Try ALGORITHM=COPY
+ALTER TABLE t2 CHANGE b c VARBINARY(30000), ALGORITHM=COPY;
+UNLOCK TABLES;
+DROP VIEW v2;
+DROP TABLE t1, t2;
 # End of 10.2 tests

mysql-test/suite/maria/lock.test

@@ -135,4 +135,33 @@ ALTER TABLE t1 CHANGE b a INT;
 UNLOCK TABLES;
 DROP TABLE t1, t2;
 
+--echo #
+--echo # MDEV-10748 Server crashes in ha_maria::implicit_commit upon ALTER TABLE
+--echo #
+
+CREATE TABLE t1 (a INT) ENGINE=Aria;
+CREATE TABLE t2 (b INT) ENGINE=Aria;
+
+LOCK TABLES t1 WRITE, t2 AS t2a WRITE, t2 WRITE;
+ALTER TABLE t2 CHANGE b c VARBINARY(30000), ALGORITHM=COPY;
+UNLOCK TABLES;
+DROP TABLE t1, t2;
+
+--echo # More complex test, from RQG
+
+CREATE TABLE t1 (a INT) ENGINE=Aria;
+CREATE TABLE t2 (b INT) ENGINE=Aria;
+CREATE OR REPLACE VIEW v2 AS SELECT * FROM t2 ;
+
+LOCK TABLES t1 WRITE, t2 AS t2a WRITE, v2 WRITE CONCURRENT, t2 WRITE;
+
+ALTER TABLE t1 FORCE;
+--error ER_ALTER_OPERATION_NOT_SUPPORTED
+ALTER TABLE t2 CHANGE b c VARBINARY(30000), ALGORITHM=INPLACE;
+ALTER TABLE t2 CHANGE b c VARBINARY(30000), ALGORITHM=COPY;
+
+UNLOCK TABLES;
+DROP VIEW v2;
+DROP TABLE t1, t2;
+
 --echo # End of 10.2 tests

mysql-test/t/kill.test

@@ -658,7 +658,7 @@ CREATE TABLE t2 (b INT, c INT);
 LOCK TABLES v1 READ, t2 WRITE, t1 WRITE;
 --error ER_BAD_FIELD_ERROR,ER_STATEMENT_TIMEOUT
 ALTER TABLE t1 CHANGE f1 f2 DOUBLE;
---error ER_STATEMENT_TIMEOUT
+--error 0,ER_STATEMENT_TIMEOUT
 ALTER TABLE t2 DROP c;
 UNLOCK TABLES;
 DROP VIEW v1;

storage/maria/ma_state.c

@@ -455,7 +455,7 @@ my_bool _ma_trnman_end_trans_hook(TRN *trn, my_bool commit,
   MARIA_USED_TABLES *tables, *next;
   DBUG_ENTER("_ma_trnman_end_trans_hook");
   DBUG_PRINT("enter", ("trn: %p  used_tables: %p", trn, trn->used_tables));
-  
+
   for (tables= (MARIA_USED_TABLES*) trn->used_tables;
        tables;
        tables= next)
@@ -572,6 +572,7 @@ void _ma_remove_table_from_trnman(MARIA_HA *info)
   TRN *trn= info->trn;
   MARIA_USED_TABLES *tables, **prev;
   MARIA_HA *handler, **prev_file;
+  uint unlinked= 0;
   DBUG_ENTER("_ma_remove_table_from_trnman");
   DBUG_PRINT("enter", ("trn: %p  used_tables: %p  share: %p  in_trans: %d",
                        trn, trn->used_tables, share, share->in_trans));
@@ -580,7 +581,7 @@ void _ma_remove_table_from_trnman(MARIA_HA *info)
 
   if (trn == &dummy_transaction_object)
     DBUG_VOID_RETURN;
-  
+
   /* First remove share from used_tables */
   for (prev= (MARIA_USED_TABLES**) (char*) &trn->used_tables;
        (tables= *prev);
@@ -594,7 +595,7 @@ void _ma_remove_table_from_trnman(MARIA_HA *info)
       break;
     }
   }
-  if (tables != 0)
+  if (!tables)
   {
     /*
       This can only happens in case of rename of intermediate table as
@@ -603,18 +604,21 @@ void _ma_remove_table_from_trnman(MARIA_HA *info)
     DBUG_PRINT("warning", ("share: %p where not in used_tables_list", share));
   }
 
-  /* unlink table from used_instances */
+  /* unlink all instances of the table from used_instances */
-  for (prev_file= (MARIA_HA**) &trn->used_instances;
+  prev_file= (MARIA_HA**) &trn->used_instances;
-       (handler= *prev_file);
+  while ((handler= *prev_file))
-       prev_file= &handler->trn_next)
   {
-    if (handler == info)
+    if (handler->s == share)
     {
-      *prev_file= info->trn_next;
+      unlinked++;
-      break;
+      *prev_file= handler->trn_next;  /* Remove instance */
     }
+    else
+      prev_file= &handler->trn_next;  /* Continue with next instance */
   }
-  if (handler != 0)
+
+  DBUG_PRINT("note", ("unlinked tables: %u", unlinked));
+  if (!unlinked)
   {
     /*
       This can only happens in case of rename of intermediate table as