More fixes to variable wsrep_on

* Disallow setting wsrep_on = 1 if wsrep_provider is unset. Also, move
  wsrep_on_basic from sys_vars to wsrep suite: this test now requires
  to run with wsrep_provider set
* Disallow setting @@session.wsrep_on = 1 when @@global.wsrep_on = 0
* Handle the case where a new connection turns @@global.wsrep_on from
  off to on. In this case we would miss a call to wsrep_open, causing
  unexpected states in wsrep::client_state (causing assertions).
* Disable wsrep.MDEV-22443 because it is no longer possible to enable
  wsrep_on, if server is started with wsrep_provider='none'

Reviewed-by: Jan Lindström <jan.lindstrom@mariadb.com>

mysql-test/suite/galera/r/galera_var_wsrep_on_off.result

@@ -22,12 +22,14 @@ SELECT COUNT(*) = 1 FROM t1 WHERE f1 = 3;
 COUNT(*) = 1
 1
 DROP TABLE t1;
+connection node_1;
 START TRANSACTION;
 SET SESSION wsrep_on=OFF;
 ERROR 25000: You are not allowed to execute this command in a transaction
 SET GLOBAL wsrep_on=OFF;
 ERROR 25000: You are not allowed to execute this command in a transaction
 COMMIT;
+connection node_1;
 CREATE TABLE t1 (f1 INTEGER PRIMARY KEY);
 START TRANSACTION;
 INSERT INTO t1 VALUES (1);
@@ -68,3 +70,58 @@ SET GLOBAL wsrep_on = ON;
 SHOW SESSION VARIABLES LIKE 'wsrep_on';
 Variable_name	Value
 wsrep_on	ON
+disconnect node_1b;
+connection node_1;
+SET GLOBAL wsrep_on = OFF;
+SET SESSION wsrep_on = ON;
+ERROR HY000: Can't enable @@session.wsrep_on, while @@global.wsrep_on is disabled
+SET GLOBAL wsrep_on = ON;
+SET SESSION wsrep_on = ON;
+connection node_1;
+CREATE TABLE t1 (f1 INTEGER PRIMARY KEY);
+SET GLOBAL wsrep_on = OFF;
+connect node_1b, 127.0.0.1, root, , test, $NODE_MYPORT_1;;
+connection node_1b;
+SHOW SESSION VARIABLES LIKE 'wsrep_on';
+Variable_name	Value
+wsrep_on	OFF
+SHOW GLOBAL VARIABLES LIKE 'wsrep_on';
+Variable_name	Value
+wsrep_on	OFF
+SET GLOBAL wsrep_on = ON;
+START TRANSACTION;
+INSERT INTO t1 VALUES(1);
+COMMIT;
+SELECT * FROM t1;
+f1
+1
+connection node_2;
+SELECT * FROM t1;
+f1
+1
+DROP TABLE t1;
+connection node_1;
+SET SESSION wsrep_on = OFF;
+CREATE TABLE t1 (f1 INTEGER PRIMARY KEY);
+INSERT INTO t1 VALUES (1);
+START TRANSACTION;
+INSERT INTO t1 VALUES (2);
+COMMIT;
+DROP TABLE t1;
+connection node_2;
+SHOW TABLES;
+Tables_in_test
+connection node_1;
+SET SESSION wsrep_on = ON;
+SET GLOBAL wsrep_on = OFF;
+CREATE TABLE t1 (f1 INTEGER PRIMARY KEY);
+INSERT INTO t1 VALUES (1);
+START TRANSACTION;
+INSERT INTO t1 VALUES (2);
+COMMIT;
+connection node_2;
+SHOW TABLES;
+Tables_in_test
+connection node_1;
+DROP TABLE t1;
+SET GLOBAL wsrep_on = ON;

mysql-test/suite/galera/t/galera_var_wsrep_on_off.test

@@ -36,6 +36,7 @@ DROP TABLE t1;
 # active transaction.
 #
 
+--connection node_1
 START TRANSACTION;
 --error ER_CANT_DO_THIS_DURING_AN_TRANSACTION
 SET SESSION wsrep_on=OFF;
@@ -49,6 +50,7 @@ COMMIT;
 # @@session.wsrep_on of current sessions
 #
 
+--connection node_1
 CREATE TABLE t1 (f1 INTEGER PRIMARY KEY);
 START TRANSACTION;
 INSERT INTO t1 VALUES (1);
@@ -75,6 +77,7 @@ DROP TABLE t1;
 #
 # New connections inherit @@session.wsrep_on from @@global.wsrep_on
 #
+
 --connection node_1
 SET GLOBAL wsrep_on = OFF;
 
@@ -87,3 +90,78 @@ DROP TABLE t2;
 
 SET GLOBAL wsrep_on = ON;
 SHOW SESSION VARIABLES LIKE 'wsrep_on';
+
+--disconnect node_1b
+
+
+#
+# Can't set @@session.wsrep_on = ON, while @@global.wsrep_on = OFF
+#
+
+--connection node_1
+SET GLOBAL wsrep_on = OFF;
+--error ER_WRONG_ARGUMENTS
+SET SESSION wsrep_on = ON;
+
+SET GLOBAL wsrep_on = ON;
+SET SESSION wsrep_on = ON;
+
+
+#
+# @@global.wsrep_on = OFF followed by @@global.wsrep_on = ON
+# in a new connection
+#
+
+--connection node_1
+CREATE TABLE t1 (f1 INTEGER PRIMARY KEY);
+SET GLOBAL wsrep_on = OFF;
+--connect node_1b, 127.0.0.1, root, , test, $NODE_MYPORT_1;
+--connection node_1b
+SHOW SESSION VARIABLES LIKE 'wsrep_on';
+SHOW GLOBAL VARIABLES LIKE 'wsrep_on';
+SET GLOBAL wsrep_on = ON;
+START TRANSACTION;
+INSERT INTO t1 VALUES(1);
+COMMIT;
+
+SELECT * FROM t1;
+
+--connection node_2
+SELECT * FROM t1;
+
+DROP TABLE t1;
+
+
+#
+# Test single statement, multi statement, and
+# TOI tansactions while @@session.wsrep_on = OFF
+# and then same @@global.wsrep_on = OFF.
+# Notice, the combination @@global.wsrep_on = OFF
+# and @@session.wsrep_on = ON is not not possible,
+# (as tested above in this test case)
+#
+
+--connection node_1
+SET SESSION wsrep_on = OFF;
+CREATE TABLE t1 (f1 INTEGER PRIMARY KEY);
+INSERT INTO t1 VALUES (1);
+START TRANSACTION;
+INSERT INTO t1 VALUES (2);
+COMMIT;
+DROP TABLE t1;
+--connection node_2
+SHOW TABLES;
+--connection node_1
+SET SESSION wsrep_on = ON;
+
+SET GLOBAL wsrep_on = OFF;
+CREATE TABLE t1 (f1 INTEGER PRIMARY KEY);
+INSERT INTO t1 VALUES (1);
+START TRANSACTION;
+INSERT INTO t1 VALUES (2);
+COMMIT;
+--connection node_2
+SHOW TABLES;
+--connection node_1
+DROP TABLE t1;
+SET GLOBAL wsrep_on = ON;

mysql-test/suite/sys_vars/r/wsrep_on_without_provider.result

@@ -0,0 +1,5 @@
+SET GLOBAL wsrep_on=ON;
+ERROR HY000: WSREP (galera) can't be enabled if the wsrep_provider is unset or set to 'none'
+SELECT @@global.wsrep_on;
+@@global.wsrep_on
+0

mysql-test/suite/sys_vars/t/wsrep_on_without_provider.test

@@ -0,0 +1,9 @@
+--source include/not_embedded.inc
+
+#
+# @@global.wsrep_on is not allowed if there
+# is no wsrep_provider
+#
+--error ER_WRONG_ARGUMENTS
+SET GLOBAL wsrep_on=ON;
+SELECT @@global.wsrep_on;

mysql-test/suite/wsrep/disabled.def

@@ -14,3 +14,4 @@
 mdev_6832: wsrep_provider is read-only for security reasons
 MDEV-23092: wsrep_provider is read-only for security reasons
 wsrep_variables_no_provider: wsrep_provider is read-only for security reasons
+MDEV-22443: it is no longer allowed enable wsrep_on if wsrep_provider is 'none'

mysql-test/suite/wsrep/r/wsrep_on_basic.result

@@ -7,10 +7,10 @@ SET @wsrep_on_session_saved = @@session.wsrep_on;
 # default
 SELECT @@global.wsrep_on;
 @@global.wsrep_on
-0
+1
 SELECT @@session.wsrep_on;
 @@session.wsrep_on
-0
+1
 
 # scope and valid values
 SET @@global.wsrep_on=OFF;

mysql-test/suite/wsrep/t/wsrep_on_basic.opt

@@ -0,0 +1 @@
+--wsrep-provider=$WSREP_PROVIDER --binlog_format=ROW --wsrep-cluster-address=gcomm://

mysql-test/suite/wsrep/t/wsrep_on_basic.test

@@ -1,4 +1,6 @@
 --source include/have_wsrep.inc
+--source include/have_wsrep_provider.inc
+--source include/have_innodb.inc
 
 --echo #
 --echo # wsrep_on

sql/service_wsrep.cc

@@ -120,14 +120,22 @@ extern "C" my_bool wsrep_get_debug()
   return wsrep_debug;
 }
 
+/*
+  Test if this connection is a true local (user) connection and not
+  a replication or wsrep applier thread.
+
+  Note that this is only usable for galera (as there are other kinds
+  of system threads, and only if WSREP_NNULL() is tested by the caller.
+ */
 extern "C" my_bool wsrep_thd_is_local(const THD *thd)
 {
   /*
-    async replication IO and background threads have nothing to replicate in the cluster,
+    async replication IO and background threads have nothing to
-    marking them as non-local here to prevent write set population and replication
+    replicate in the cluster, marking them as non-local here to
+    prevent write set population and replication
 
-    async replication SQL thread, applies client transactions from mariadb master
+    async replication SQL thread, applies client transactions from
-    and will be replicated into cluster
+    mariadb master and will be replicated into cluster
   */
   return (
           thd->system_thread != SYSTEM_THREAD_SLAVE_BACKGROUND &&

sql/wsrep_trans_observer.h

@@ -433,6 +433,16 @@ static inline void wsrep_close(THD* thd)
   DBUG_VOID_RETURN;
 }
 
+static inline void wsrep_cleanup(THD* thd)
+{
+  DBUG_ENTER("wsrep_cleanup");
+  if (thd->wsrep_cs().state() != wsrep::client_state::s_none)
+  {
+    thd->wsrep_cs().cleanup();
+  }
+  DBUG_VOID_RETURN;
+}
+
 static inline void
 wsrep_wait_rollback_complete_and_acquire_ownership(THD *thd)
 {

sql/wsrep_var.cc

@@ -25,6 +25,7 @@
 #include <my_dir.h>
 #include <cstdio>
 #include <cstdlib>
+#include "wsrep_trans_observer.h"
 
 ulong   wsrep_reject_queries;
 
@@ -103,7 +104,8 @@ struct handlerton* innodb_hton_ptr __attribute__((weak));
 
 bool wsrep_on_update (sys_var *self, THD* thd, enum_var_type var_type)
 {
-  if (var_type == OPT_GLOBAL) {
+  if (var_type == OPT_GLOBAL)
+  {
     my_bool saved_wsrep_on= global_system_variables.wsrep_on;
 
     thd->variables.wsrep_on= global_system_variables.wsrep_on;
@@ -111,15 +113,15 @@ bool wsrep_on_update (sys_var *self, THD* thd, enum_var_type var_type)
     // If wsrep has not been inited we need to do it now
     if (global_system_variables.wsrep_on && wsrep_provider && !wsrep_inited)
     {
-      char* tmp= strdup(wsrep_provider); // wsrep_init() rewrites provider
+      // wsrep_init() rewrites provide if it fails
-                                         //when fails
+      char* tmp= strdup(wsrep_provider);
-
       mysql_mutex_unlock(&LOCK_global_system_variables);
 
       if (wsrep_init())
       {
         my_error(ER_CANT_OPEN_LIBRARY, MYF(0), tmp, my_error, "wsrep_init failed");
         //rcode= true;
+        saved_wsrep_on= false;
       }
 
       free(tmp);
@@ -131,6 +133,16 @@ bool wsrep_on_update (sys_var *self, THD* thd, enum_var_type var_type)
 
   wsrep_set_wsrep_on();
 
+  if (var_type == OPT_GLOBAL)
+  {
+    if (thd->variables.wsrep_on &&
+        thd->wsrep_cs().state() == wsrep::client_state::s_none)
+    {
+      wsrep_open(thd);
+      wsrep_before_command(thd);
+    }
+  }
+
   return false;
 }
 
@@ -141,19 +153,57 @@ bool wsrep_on_check(sys_var *self, THD* thd, set_var* var)
   if (check_has_super(self, thd, var))
     return true;
 
-  if (new_wsrep_on && innodb_hton_ptr && innodb_lock_schedule_algorithm != 0) {
+  if (new_wsrep_on)
+  {
+    if (innodb_hton_ptr && innodb_lock_schedule_algorithm != 0)
+    {
       my_message(ER_WRONG_ARGUMENTS, " WSREP (galera) can't be enabled "
                  "if innodb_lock_schedule_algorithm=VATS. Please configure"
                  " innodb_lock_schedule_algorithm=FCFS and restart.", MYF(0));
       return true;
     }
 
+    if (!WSREP_PROVIDER_EXISTS)
+    {
+      my_message(ER_WRONG_ARGUMENTS, "WSREP (galera) can't be enabled "
+                 "if the wsrep_provider is unset or set to 'none'", MYF(0));
+      return true;
+    }
+
+    if (var->type == OPT_SESSION &&
+        !global_system_variables.wsrep_on)
+    {
+      my_message(ER_WRONG_ARGUMENTS,
+                 "Can't enable @@session.wsrep_on, "
+                 "while @@global.wsrep_on is disabled", MYF(0));
+      return true;
+    }
+  }
+
   if (thd->in_active_multi_stmt_transaction())
   {
     my_error(ER_CANT_DO_THIS_DURING_AN_TRANSACTION, MYF(0));
     return true;
   }
 
+  if (var->type == OPT_GLOBAL)
+  {
+    /*
+      The global value is about to change. Cleanup
+      the transaction state and close the client
+      state. wsrep_on_update() will take care of
+      reopening it should wsrep_on be re-enabled.
+     */
+    if (global_system_variables.wsrep_on && !new_wsrep_on)
+    {
+      wsrep_commit_empty(thd, true);
+      wsrep_after_statement(thd);
+      wsrep_after_command_ignore_result(thd);
+      wsrep_close(thd);
+      wsrep_cleanup(thd);
+    }
+  }
+
   return false;
 }