MDEV-26308 : Galera test failure on galera.galera_split_brain

Contains following fixes:

* allow TOI commands to timeout while trying to acquire TOI with
override lock_wait_timeout with a LONG_TIMEOUT only after
succesfully entering TOI
* only ignore lock_wait_timeout on TOI
* fix galera_split_brain test as TOI operation now returns ER_LOCK_WAIT_TIMEOUT after lock_wait_timeout
* explicitly test for TOI

Reviewed-by: Jan Lindström <jan.lindstrom@mariadb.com>

mysql-test/suite/galera/r/galera_split_brain.result

@@ -7,8 +7,9 @@ connection node_1;
 connection node_2;
 Killing server ...
 connection node_1;
+SET SESSION lock_wait_timeout= 3;
 CREATE TABLE t1 (f1 INTEGER) ENGINE=InnoDB;
-ERROR 40001: Deadlock found when trying to get lock; try restarting transaction
+ERROR HY000: Lock wait timeout exceeded; try restarting transaction
 connection node_2;
 connect node_2a, 127.0.0.1, root, , test, $NODE_MYPORT_2;
 disconnect node_2;

mysql-test/suite/galera/t/galera_split_brain.test

@@ -22,7 +22,8 @@ call mtr.add_suppression("WSREP: TO isolation failed for: ");
 --source include/kill_galera.inc
 
 --connection node_1
---error ER_LOCK_DEADLOCK
+SET SESSION lock_wait_timeout= 3;
+--error ER_LOCK_WAIT_TIMEOUT
 CREATE TABLE t1 (f1 INTEGER) ENGINE=InnoDB;
 
 # Reset the master and restart the slave so that post-test checks can run

sql/handler.cc

@@ -1,5 +1,5 @@
 /* Copyright (c) 2000, 2016, Oracle and/or its affiliates.
-   Copyright (c) 2009, 2020, MariaDB Corporation.
+   Copyright (c) 2009, 2021, MariaDB Corporation.
 
    This program is free software; you can redistribute it and/or modify
    it under the terms of the GNU General Public License as published by
@@ -1946,7 +1946,7 @@ int ha_rollback_trans(THD *thd, bool all)
   if (thd->is_error())
   {
     WSREP_DEBUG("ha_rollback_trans(%lld, %s) rolled back: %s: %s; is_real %d",
-                thd->thread_id, all?"TRUE":"FALSE", WSREP_QUERY(thd),
+                thd->thread_id, all?"TRUE":"FALSE", wsrep_thd_query(thd),
                 thd->get_stmt_da()->message(), is_real_trans);
   }
   (void) wsrep_after_rollback(thd, all);

sql/sql_insert.cc

@@ -1,6 +1,6 @@
 /*
    Copyright (c) 2000, 2016, Oracle and/or its affiliates.
-   Copyright (c) 2010, 2019, MariaDB Corporation
+   Copyright (c) 2010, 2021, MariaDB Corporation
 
    This program is free software; you can redistribute it and/or modify
    it under the terms of the GNU General Public License as published by
@@ -4782,7 +4782,7 @@ bool select_create::send_eof()
       {
         WSREP_DEBUG("select_create commit failed, thd: %llu err: %s %s",
                     thd->thread_id,
-                    wsrep_thd_transaction_state_str(thd), WSREP_QUERY(thd));
+                    wsrep_thd_transaction_state_str(thd), wsrep_thd_query(thd));
         mysql_mutex_unlock(&thd->LOCK_thd_data);
         abort_result_set();
         DBUG_RETURN(true);

sql/sql_parse.cc

@@ -7845,7 +7845,7 @@ static bool wsrep_mysql_parse(THD *thd, char *rawbuf, uint length,
                       DBUG_ASSERT(!debug_sync_set_action(thd, STRING_WITH_LEN(act)));
                     });
         WSREP_DEBUG("wsrep retrying AC query: %lu  %s",
-                    thd->wsrep_retry_counter, WSREP_QUERY(thd));
+                    thd->wsrep_retry_counter, wsrep_thd_query(thd));
         wsrep_prepare_for_autocommit_retry(thd, rawbuf, length, parser_state);
         if (thd->lex->explain)
           delete_explain_query(thd->lex);
@@ -7859,7 +7859,7 @@ static bool wsrep_mysql_parse(THD *thd, char *rawbuf, uint length,
                     is_autocommit,
                     thd->wsrep_retry_counter,
                     thd->variables.wsrep_retry_autocommit,
-                    WSREP_QUERY(thd));
+                    wsrep_thd_query(thd));
         my_error(ER_LOCK_DEADLOCK, MYF(0));
         thd->reset_kill_query();
         thd->wsrep_retry_counter= 0;             //  reset

sql/wsrep_client_service.cc

@@ -1,4 +1,4 @@
-/* Copyright 2018 Codership Oy <info@codership.com>
+/* Copyright 2018-2021 Codership Oy <info@codership.com>
 
    This program is free software; you can redistribute it and/or modify
    it under the terms of the GNU General Public License as published by
@@ -109,14 +109,14 @@ int Wsrep_client_service::prepare_data_for_replication()
                   "affected rows: %llu, "
                   "changed tables: %d, "
                   "sql_log_bin: %d",
-                  WSREP_QUERY(m_thd),
+                  wsrep_thd_query(m_thd),
                   m_thd->get_stmt_da()->affected_rows(),
                   stmt_has_updated_trans_table(m_thd),
                   m_thd->variables.sql_log_bin);
     }
     else
     {
-      WSREP_DEBUG("empty rbr buffer, query: %s", WSREP_QUERY(m_thd));
+      WSREP_DEBUG("empty rbr buffer, query: %s", wsrep_thd_query(m_thd));
     }
   }
   DBUG_RETURN(0);

sql/wsrep_high_priority_service.cc

@@ -1,4 +1,4 @@
-/* Copyright 2018 Codership Oy <info@codership.com>
+/* Copyright 2018-2021 Codership Oy <info@codership.com>
 
    This program is free software; you can redistribute it and/or modify
    it under the terms of the GNU General Public License as published by
@@ -658,7 +658,7 @@ Wsrep_replayer_service::~Wsrep_replayer_service()
     DBUG_ASSERT(0);
     WSREP_ERROR("trx_replay failed for: %d, schema: %s, query: %s",
                 m_replay_status,
-                orig_thd->db.str, WSREP_QUERY(orig_thd));
+                orig_thd->db.str, wsrep_thd_query(orig_thd));
     unireg_abort(1);
   }
 }

sql/wsrep_mysqld.cc

@@ -1,4 +1,4 @@
-/* Copyright 2008-2015 Codership Oy <http://www.codership.com>
+/* Copyright 2008-2021 Codership Oy <http://www.codership.com>
 
    This program is free software; you can redistribute it and/or modify
    it under the terms of the GNU General Public License as published by
@@ -2008,10 +2008,10 @@ static int wsrep_TOI_begin(THD *thd, const char *db, const char *table,
 {
   DBUG_ASSERT(thd->variables.wsrep_OSU_method == WSREP_OSU_TOI);
 
-  WSREP_DEBUG("TOI Begin for %s", WSREP_QUERY(thd));
+  WSREP_DEBUG("TOI Begin for %s", wsrep_thd_query(thd));
   if (wsrep_can_run_in_toi(thd, db, table, table_list) == false)
   {
-    WSREP_DEBUG("No TOI for %s", WSREP_QUERY(thd));
+    WSREP_DEBUG("No TOI for %s", wsrep_thd_query(thd));
     return 1;
   }
 
@@ -2039,7 +2039,7 @@ static int wsrep_TOI_begin(THD *thd, const char *db, const char *table,
     /* non replicated DDL, affecting temporary tables only */
     WSREP_DEBUG("TO isolation skipped, sql: %s."
                 "Only temporary tables affected.",
-                WSREP_QUERY(thd));
+                wsrep_thd_query(thd));
     if (buf) my_free(buf);
     return -1;
   }
@@ -2054,7 +2054,7 @@ static int wsrep_TOI_begin(THD *thd, const char *db, const char *table,
   {
     DBUG_ASSERT(cs.current_error());
     WSREP_DEBUG("to_execute_start() failed for %llu: %s, seqno: %lld",
-                thd->thread_id, WSREP_QUERY(thd),
+                thd->thread_id, wsrep_thd_query(thd),
                 (long long)wsrep_thd_trx_seqno(thd));
 
     /* jump to error handler in mysql_execute_command() */
@@ -2065,15 +2065,32 @@ static int wsrep_TOI_begin(THD *thd, const char *db, const char *table,
                  "Maximum size exceeded.",
                  ret,
                  (thd->db.str ? thd->db.str : "(null)"),
-                 WSREP_QUERY(thd));
+                 wsrep_thd_query(thd));
       my_error(ER_ERROR_DURING_COMMIT, MYF(0), WSREP_SIZE_EXCEEDED);
       break;
+    case wsrep::e_deadlock_error:
+      WSREP_WARN("TO isolation failed for: %d, schema: %s, sql: %s. "
+                 "Deadlock error.",
+                 ret,
+                 (thd->db.str ? thd->db.str : "(null)"),
+                 wsrep_thd_query(thd));
+      my_error(ER_LOCK_DEADLOCK, MYF(0));
+      break;
+    case wsrep::e_timeout_error:
+      WSREP_WARN("TO isolation failed for: %d, schema: %s, sql: %s. "
+                 "Operation timed out.",
+                 ret,
+                 (thd->db.str ? thd->db.str : "(null)"),
+                 wsrep_thd_query(thd));
+      my_error(ER_LOCK_WAIT_TIMEOUT, MYF(0));
+      break;
     default:
       WSREP_WARN("TO isolation failed for: %d, schema: %s, sql: %s. "
-                 "Check wsrep connection state and retry the query.",
+                 "Check your wsrep connection state and retry the query.",
                  ret,
                  (thd->db.str ? thd->db.str : "(null)"),
-                 WSREP_QUERY(thd));
+                 wsrep_thd_query(thd));
+
       if (!thd->is_error())
       {
         my_error(ER_LOCK_DEADLOCK, MYF(0), "WSREP replication failed. Check "
@@ -2106,19 +2123,19 @@ static void wsrep_TOI_end(THD *thd) {
   if (!ret)
   {
     WSREP_DEBUG("TO END: %lld: %s",
-                client_state.toi_meta().seqno().get(), WSREP_QUERY(thd));
+                client_state.toi_meta().seqno().get(), wsrep_thd_query(thd));
   }
   else
   {
     WSREP_WARN("TO isolation end failed for: %d, sql: %s",
-                ret, WSREP_QUERY(thd));
+                ret, wsrep_thd_query(thd));
   }
 }
 
 static int wsrep_RSU_begin(THD *thd, const char *db_, const char *table_)
 {
   WSREP_DEBUG("RSU BEGIN: %lld, : %s", wsrep_thd_trx_seqno(thd),
-              WSREP_QUERY(thd));
+              wsrep_thd_query(thd));
   if (thd->wsrep_cs().begin_rsu(5000))
   {
     WSREP_WARN("RSU begin failed");
@@ -2133,7 +2150,7 @@ static int wsrep_RSU_begin(THD *thd, const char *db_, const char *table_)
 static void wsrep_RSU_end(THD *thd)
 {
   WSREP_DEBUG("RSU END: %lld : %s", wsrep_thd_trx_seqno(thd),
-              WSREP_QUERY(thd));
+              wsrep_thd_query(thd));
   if (thd->wsrep_cs().end_rsu())
   {
     WSREP_WARN("Failed to end RSU, server may need to be restarted");
@@ -2175,7 +2192,7 @@ int wsrep_to_isolation_begin(THD *thd, const char *db_, const char *table_,
   if (wsrep_debug && thd->mdl_context.has_locks())
   {
     WSREP_DEBUG("thread holds MDL locks at TI begin: %s %llu",
-                WSREP_QUERY(thd), thd->thread_id);
+                wsrep_thd_query(thd), thd->thread_id);
   }
 
   /*
@@ -2191,13 +2208,6 @@ int wsrep_to_isolation_begin(THD *thd, const char *db_, const char *table_,
     thd->variables.auto_increment_increment= 1;
   }
 
-  /*
-    TOI operations will ignore provided lock_wait_timeout and restore it
-    after operation is done.
-   */
-  thd->variables.saved_lock_wait_timeout= thd->variables.lock_wait_timeout;
-  thd->variables.lock_wait_timeout= LONG_TIMEOUT;
-
   if (thd->variables.wsrep_on && wsrep_thd_is_local(thd))
   {
     switch (thd->variables.wsrep_OSU_method) {
@@ -2213,8 +2223,19 @@ int wsrep_to_isolation_begin(THD *thd, const char *db_, const char *table_,
       ret= -1;
       break;
     }
+
     switch (ret) {
-    case 0: /* wsrep_TOI_begin sould set toi mode */ break;
+    case 0: /* wsrep_TOI_begin sould set toi mode */
+      if (thd->variables.wsrep_OSU_method == WSREP_OSU_TOI)
+      {
+        /*
+          TOI operations ignore the provided lock_wait_timeout once replicated,
+          and restore it after operation is done.
+         */
+        thd->variables.saved_lock_wait_timeout= thd->variables.lock_wait_timeout;
+        thd->variables.lock_wait_timeout= LONG_TIMEOUT;
+      }
+      break;
     case 1:
       /* TOI replication skipped, treat as success */
       ret= 0;
@@ -2233,10 +2254,9 @@ void wsrep_to_isolation_end(THD *thd)
   DBUG_ASSERT(wsrep_thd_is_local_toi(thd) ||
               wsrep_thd_is_in_rsu(thd));
 
-  thd->variables.lock_wait_timeout= thd->variables.saved_lock_wait_timeout;
-
   if (wsrep_thd_is_local_toi(thd))
   {
+    thd->variables.lock_wait_timeout= thd->variables.saved_lock_wait_timeout;
     DBUG_ASSERT(thd->variables.wsrep_OSU_method == WSREP_OSU_TOI);
     wsrep_TOI_end(thd);
   }

sql/wsrep_mysqld.h

@@ -1,4 +1,4 @@
-/* Copyright 2008-2017 Codership Oy <http://www.codership.com>
+/* Copyright 2008-2021 Codership Oy <http://www.codership.com>
 
    This program is free software; you can redistribute it and/or modify
    it under the terms of the GNU General Public License as published by
@@ -289,8 +289,6 @@ static inline bool wsrep_cluster_address_exists()
   return wsrep_cluster_address && wsrep_cluster_address[0];
 }
 
-#define WSREP_QUERY(thd) (thd->query())
-
 extern my_bool wsrep_ready_get();
 extern void wsrep_ready_wait();
 

sql/wsrep_thd.h

@@ -1,4 +1,4 @@
-/* Copyright (C) 2013 Codership Oy <info@codership.com>
+/* Copyright (C) 2013-2021 Codership Oy <info@codership.com>
 
    This program is free software; you can redistribute it and/or modify
    it under the terms of the GNU General Public License as published by
@@ -292,7 +292,7 @@ static inline void wsrep_log_thd(const THD *thd,
               (thd->get_stmt_da()->is_error() ? thd->get_stmt_da()->message() : "")
 #ifdef WSREP_THD_LOG_QUERIES
               , thd->lex->sql_command,
-              WSREP_QUERY(thd)
+              wsrep_thd_query(thd)
 #endif /* WSREP_OBSERVER_LOG_QUERIES */
               );
 }

sql/wsrep_var.cc

@@ -1,4 +1,4 @@
-/* Copyright 2008-2015 Codership Oy <http://www.codership.com>
+/* Copyright 2008-2021 Codership Oy <http://www.codership.com>
 
    This program is free software; you can redistribute it and/or modify
    it under the terms of the GNU General Public License as published by
@@ -794,7 +794,7 @@ bool wsrep_desync_check (sys_var *self, THD* thd, set_var* var)
     ret= Wsrep_server_state::instance().provider().desync();
     if (ret) {
       WSREP_WARN ("SET desync failed %d for schema: %s, query: %s", ret,
-                  thd->db.str, WSREP_QUERY(thd));
+                  thd->db.str, wsrep_thd_query(thd));
       my_error (ER_CANNOT_USER, MYF(0), "'desync'", thd->query());
       return true;
     }