Fixing bug when using alter table on locked maria table

Reset history when we reenable logging for transactional tables (safety fix)

mysql-test/r/maria2.result:
  New results
mysql-test/t/maria2.test:
  Added test case for alter table on locked maria table
mysql-test/valgrind.supp:
  Added suppression rules for warnings in libc / libld
storage/maria/ma_extra.c:
  Remove table from trnman list if we are going to drop or rename it; We don't want not existing shares in the list when we do commit!
storage/maria/ma_recovery.c:
  Ensure that info->state don't point to history event when we disable logging for a table;  This is needed as alter table will first do commit and then unlock, which would cause us to access a non existing object when we reenable logging.
  Reset history when we reenable logging (safety fix)
storage/maria/ma_state.c:
  Do less work when share->now_transactional is not set. (Safety fix)
  Added function to remove shares to be deleted from trnman->used_tables
  Added function to reset history to current context
storage/maria/ma_state.h:
  Prototypes for new function

mysql-test/r/maria2.result

@@ -1,3 +1,4 @@
+drop table if exists t1,t2;
 CREATE TABLE t1 (
 line BLOB,
 kind ENUM('po', 'pp', 'rr', 'dr', 'rd', 'ts', 'cl') NOT NULL DEFAULT 'po',
@@ -16,3 +17,16 @@ check table t1 extended;
 Table	Op	Msg_type	Msg_text
 test.t1	check	status	OK
 drop table t1;
+create table t1 (i int) engine=maria;
+create table t2 (j int) engine=maria;
+lock table t1 write, t2 read;
+alter table t1 modify i int default 1;
+insert into t1 values (2);
+alter table t1 modify i bigint default 1;
+select count(*) from t1;
+count(*)
+1
+select * from t1;
+i
+2
+drop table t1,t2;

mysql-test/t/maria2.test

@@ -1,5 +1,10 @@
 --source include/have_maria.inc
 
+# Initialise
+--disable_warnings
+drop table if exists t1,t2;
+--enable_warnings
+
 # Test for BUG#36319
 # "Maria: table is not empty but DELETE and SELECT find no rows"
 
@@ -64,3 +69,19 @@ select count(*) from t1;
 select name from t1;
 check table t1 extended;
 drop table t1;
+
+#
+# Testing of ALTER TABLE under lock tables
+#
+
+create table t1 (i int) engine=maria;
+create table t2 (j int) engine=maria;
+lock table t1 write, t2 read;
+alter table t1 modify i int default 1;
+insert into t1 values (2);
+# This caused a core dump
+alter table t1 modify i bigint default 1;
+select count(*) from t1;
+select * from t1;
+drop table t1,t2;
+

mysql-test/valgrind.supp

@@ -27,7 +27,7 @@
    pthread allocate_tls memory loss
    Memcheck:Leak
    fun:calloc
-   obj:/lib64/ld*.so
+   obj:/lib*/ld*.so
    fun:_dl_allocate_tls
    fun:pthread_create*
 }
@@ -388,40 +388,58 @@
 }
 
 {
-   dlopen / ptread_cancel_init memory loss on Suse Linux 10.3 64 bit
+   dlopen / ptread_cancel_init memory loss on Suse Linux 10.3 32/64 bit
    Memcheck:Leak
    fun:*alloc
-   obj:/lib64/ld-*.so
+   obj:/lib*/ld-*.so
-   obj:/lib64/ld-*.so
+   obj:/lib*/ld-*.so
-   obj:/lib64/ld-*.so
+   obj:/lib*/ld-*.so
-   obj:/lib64/ld-*.so
+   obj:/lib*/ld-*.so
-   obj:/lib64/ld-*.so
+   obj:/lib*/ld-*.so
-   obj:/lib64/ld-*.so
+   obj:/lib*/ld-*.so
-   obj:/lib64/libc-*.so
+   obj:/lib*/libc-*.so
-   obj:/lib64/ld-*.so
+   obj:/lib*/ld-*.so
-   obj:/lib64/libc-*.so
+   obj:/lib*/libc-*.so
    fun:__libc_dlopen_mode
    fun:pthread_cancel_init
    fun:_Unwind_ForcedUnwind
-
 }
 
 {
-   dlopen / ptread_cancel_init memory loss on Suse Linux 10.3 64 bit
+   dlopen / ptread_cancel_init memory loss on Suse Linux 10.3 32/64 bit
    Memcheck:Leak
    fun:*alloc
-   obj:/lib64/ld-*.so
+   obj:/lib*/ld-*.so
-   obj:/lib64/ld-*.so
+   obj:/lib*/ld-*.so
-   obj:/lib64/ld-*.so
+   obj:/lib*/ld-*.so
-   obj:/lib64/ld-*.so
+   obj:/lib*/ld-*.so
-   obj:/lib64/libc-*.so
+   obj:/lib*/libc-*.so
-   obj:/lib64/ld-*.so
+   obj:/lib*/ld-*.so
-   obj:/lib64/libc-*.so
+   obj:/lib*/libc-*.so
    fun:__libc_dlopen_mode
    fun:pthread_cancel_init
    fun:_Unwind_ForcedUnwind
 }
 
+#
+# Reading wrong addresses on SuSe Linux 10.3 32 bit
+#
+
+{
+   Reading wrong data in libc_dlopen
+   Memcheck:Addr4
+   obj:/lib*/ld-*.so
+   obj:/lib*/ld-*.so
+   obj:/lib*/ld-*.so
+   obj:/lib*/ld-*.so
+   obj:/lib*/ld-*.so
+   obj:/lib*/ld-*.so
+   obj:/lib*/libc-*.so
+   obj:/lib*/ld-*.so
+   obj:/lib*/libc-*.so
+   fun:__libc_dlopen_mode
+   fun:pthread_cancel_init
+}
 
 #
 # These seem to be libc threading stuff, not related to MySQL code (allocations

storage/maria/ma_extra.c

@@ -322,6 +322,13 @@ int maria_extra(MARIA_HA *info, enum ha_extra_function function,
     if (share->kfile.file >= 0)
       _ma_decrement_open_count(info);
     pthread_mutex_lock(&share->intern_lock);
+    if (info->trn)
+    {
+      _ma_remove_table_from_trnman(share, info->trn);
+      /* Ensure we don't point to the deleted data in trn */
+      info->state= &share->state.state;
+    }
+
     type= do_flush ? FLUSH_RELEASE : FLUSH_IGNORE_CHANGED;
     if (_ma_flush_table_files(info, MARIA_FLUSH_DATA | MARIA_FLUSH_INDEX,
                               type, type))

storage/maria/ma_recovery.c

@@ -3214,6 +3214,16 @@ void _ma_tmp_disable_logging_for_table(MARIA_HA *info,
 
   /* if we disabled before writing the record, record wouldn't reach log */
   share->now_transactional= FALSE;
+
+  /*
+    Reset state pointers. This is needed as in ALTER table we may do
+    commit fllowed by _ma_renable_logging_for_table and then
+    info->state may point to a state that was deleted by
+    _ma_trnman_end_trans_hook()
+   */
+  share->state.common= *info->state;
+  info->state= &share->state.common;
+
   /*
     Some code in ma_blockrec.c assumes a trn even if !now_transactional but in
     this case it only reads trn->rec_lsn, which has to be LSN_IMPOSSIBLE and
@@ -3255,6 +3265,7 @@ my_bool _ma_reenable_logging_for_table(MARIA_HA *info, my_bool flush_pages)
       in not transactional mode
     */
     _ma_copy_nontrans_state_information(info);
+    _ma_reset_history(info->s);
 
     if (flush_pages)
     {

storage/maria/ma_state.c

@@ -371,7 +371,7 @@ my_bool _ma_trnman_end_trans_hook(TRN *trn, my_bool commit,
       MARIA_STATE_HISTORY *history;
 
       pthread_mutex_lock(&share->intern_lock);
-      if (active_transactions &&
+      if (active_transactions && share->now_transactional &&
           trnman_exists_active_transactions(share->state_history->trid,
                                             trn->commit_trid, 1))
       {
@@ -415,6 +415,35 @@ my_bool _ma_trnman_end_trans_hook(TRN *trn, my_bool commit,
 }
 
 
+/**
+   Remove table from trnman_list
+
+   @notes
+     This is used when we unlock a table from a group of locked tables
+     just before doing a rename or drop table.
+*/
+
+void _ma_remove_table_from_trnman(MARIA_SHARE *share, TRN *trn)
+{
+  MARIA_USED_TABLES *tables, **prev;
+  
+  for (prev= (MARIA_USED_TABLES**) &trn->used_tables, tables= *prev;
+       tables;
+       tables= *prev)
+  {
+    if (tables->share == share)
+    {
+      *prev= tables->next;
+      my_free(tables, MYF(0));
+    }
+    else
+      prev= &tables->next;
+  }
+}
+
+
+
+
 /****************************************************************************
   The following functions are called by thr_lock() in threaded applications
   for transactional tables.
@@ -509,6 +538,23 @@ void _ma_copy_nontrans_state_information(MARIA_HA *info)
 }
 
 
+void _ma_reset_history(MARIA_SHARE *share)
+{
+  MARIA_STATE_HISTORY *history, *next;
+
+  share->state_history->trid= 0;          /* Visibly by all */
+  share->state_history->state= share->state.state;
+  history= share->state_history->next;
+  share->state_history->next= 0;
+
+  for (; history; history= next)
+  {
+    next= history->next;
+    my_free(history, MYF(0));
+  }
+}
+
+
 /****************************************************************************
   Virtual functions to check if row is visible
 ****************************************************************************/

storage/maria/ma_state.h

@@ -77,3 +77,5 @@ my_bool _ma_row_visible_always(MARIA_HA *info);
 my_bool _ma_row_visible_non_transactional_table(MARIA_HA *info);
 my_bool _ma_row_visible_transactional_table(MARIA_HA *info);
 void _ma_remove_not_visible_states_with_lock(struct st_maria_share *share);
+void _ma_remove_table_from_trnman(struct st_maria_share *share, TRN *trn);
+void _ma_reset_history(struct st_maria_share *share);