MDEV-22570 Implement wsrep_provider_options as plugin

- Provider options are read from the provider during
  startup, before plugins are initialized.
- New wsrep_provider plugin for which sysvars are generated
  dynamically from options read from the provider.
- The plugin is enabled by option plugin-wsrep-provider=ON.
  If enabled, wsrep_provider_options can no longer be used,
  (an error is raised on attempts to do so).
- Each option is either string, integer, double or bool
- Options can be dynamic / readonly
- Options can be deprecated

Limitations:

- We do not check that the value of a provider option falls
  within a certain range. This type of validation is still
  done in Galera side.

Reviewed-by: Jan Lindström <jan.lindstrom@mariadb.com>

mysql-test/suite/wsrep/r/wsrep_provider_plugin.result

@@ -0,0 +1,20 @@
+CREATE TABLE t1 (f1 INT PRIMARY KEY) ENGINE=InnoDB;
+SET GLOBAL wsrep_provider_repl_max_ws_size=1;
+SHOW VARIABLES LIKE 'wsrep_provider_repl_max_ws_size';
+Variable_name	Value
+wsrep_provider_repl_max_ws_size	1
+INSERT INTO t1 VALUES (1);
+ERROR HY000: Got error 5 "Input/output error" during COMMIT
+SET GLOBAL wsrep_provider_repl_max_ws_size=DEFAULT;
+SHOW VARIABLES LIKE 'wsrep_provider_repl_max_ws_size';
+Variable_name	Value
+wsrep_provider_repl_max_ws_size	2147483647
+INSERT INTO t1 VALUES (1);
+SET GLOBAL wsrep_provider_options='repl.max_ws_size=1';
+ERROR HY000: Variable 'wsrep_provider_options' is a read only variable
+INSERT INTO t1 VALUES (2);
+SET GLOBAL wsrep_provider='none';
+ERROR HY000: Variable 'wsrep_provider' is a read only variable
+DROP TABLE t1;
+CALL mtr.add_suppression("transaction size limit");
+CALL mtr.add_suppression("rbr write fail");

mysql-test/suite/wsrep/r/wsrep_provider_plugin_basic.result

@@ -0,0 +1,65 @@
+select variable_type, global_value from information_schema.system_variables where variable_name = 'wsrep_provider_socket_recv_buf_size';
+variable_type	global_value
+VARCHAR	auto
+set global wsrep_provider_socket_recv_buf_size = 'foo';
+ERROR 42000: Variable 'socket_recv_buf_size' can't be set to the value of 'foo'
+set global wsrep_provider_socket_recv_buf_size = '1M';
+show global variables like 'wsrep_provider_socket_recv_buf_size';
+Variable_name	Value
+wsrep_provider_socket_recv_buf_size	1M
+set global wsrep_provider_socket_recv_buf_size = default;
+show global variables like 'wsrep_provider_socket_recv_buf_size';
+Variable_name	Value
+wsrep_provider_socket_recv_buf_size	auto
+select variable_type, global_value from information_schema.system_variables where variable_name = 'wsrep_provider_evs_send_window';
+variable_type	global_value
+BIGINT	4
+set global wsrep_provider_evs_send_window = -10;
+ERROR 42000: Variable 'evs_send_window' can't be set to the value of '-10'
+set global wsrep_provider_evs_send_window = 10;
+show global variables like 'wsrep_provider_evs_send_window';
+Variable_name	Value
+wsrep_provider_evs_send_window	10
+set global wsrep_provider_evs_send_window = default;
+show global variables like 'wsrep_provider_evs_send_window';
+Variable_name	Value
+wsrep_provider_evs_send_window	4
+select variable_type from information_schema.system_variables where variable_name = 'wsrep_provider_gcs_max_throttle';
+variable_type
+DOUBLE
+set global wsrep_provider_gcs_max_throttle = 1.1;
+ERROR 42000: Variable 'gcs_max_throttle' can't be set to the value of '1.100000'
+set global wsrep_provider_gcs_max_throttle = 0.5;
+show global variables like 'wsrep_provider_gcs_max_throttle';
+Variable_name	Value
+wsrep_provider_gcs_max_throttle	0.500000
+set global wsrep_provider_gcs_max_throttle = default;
+show global variables like 'wsrep_provider_gcs_max_throttle';
+Variable_name	Value
+wsrep_provider_gcs_max_throttle	0.250000
+select variable_type from information_schema.system_variables where variable_name = 'wsrep_provider_cert_log_conflicts';
+variable_type
+BOOLEAN
+set global wsrep_provider_cert_log_conflicts = on;
+show global variables like 'wsrep_provider_cert_log_conflicts';
+Variable_name	Value
+wsrep_provider_cert_log_conflicts	ON
+set global wsrep_provider_cert_log_conflicts = off;
+show global variables like 'wsrep_provider_cert_log_conflicts';
+Variable_name	Value
+wsrep_provider_cert_log_conflicts	OFF
+set global wsrep_provider_cert_log_conflicts = default;
+show global variables like 'wsrep_provider_cert_log_conflicts';
+Variable_name	Value
+wsrep_provider_cert_log_conflicts	OFF
+select read_only from information_schema.system_variables where variable_name = 'wsrep_provider_evs_auto_evict';
+read_only
+YES
+set global wsrep_provider_evs_auto_evict = on;
+ERROR HY000: Variable 'wsrep_provider_evs_auto_evict' is a read only variable
+set global wsrep_provider_gcs_fc_master_slave = default;
+Warnings:
+Warning	1287	'@@wsrep_provider_gcs_fc_master_slave' is deprecated and will be removed in a future release
+call mtr.add_suppression("error setting param");
+call mtr.add_suppression("Unknown parameter");
+call mtr.add_suppression("Setting parameter");

mysql-test/suite/wsrep/t/wsrep_provider_plugin.cnf

@@ -0,0 +1,8 @@
+!include ../my.cnf
+
+[mysqld.1]
+wsrep-on=ON
+wsrep-cluster-address=gcomm://
+wsrep-provider=@ENV.WSREP_PROVIDER
+binlog-format=ROW
+plugin-wsrep-provider=ON

mysql-test/suite/wsrep/t/wsrep_provider_plugin.test

@@ -0,0 +1,33 @@
+#
+# Verify that system variables can be modified via wsrep_provider
+# plugin and wsrep_provider/wsrep_provider_options cannot be modified.
+#
+
+--source include/have_wsrep.inc
+--source include/have_innodb.inc
+
+CREATE TABLE t1 (f1 INT PRIMARY KEY) ENGINE=InnoDB;
+
+SET GLOBAL wsrep_provider_repl_max_ws_size=1;
+SHOW VARIABLES LIKE 'wsrep_provider_repl_max_ws_size';
+
+--error ER_ERROR_DURING_COMMIT
+INSERT INTO t1 VALUES (1);
+
+SET GLOBAL wsrep_provider_repl_max_ws_size=DEFAULT;
+SHOW VARIABLES LIKE 'wsrep_provider_repl_max_ws_size';
+
+INSERT INTO t1 VALUES (1);
+
+# Variable should be read only, must not take effect
+--error ER_INCORRECT_GLOBAL_LOCAL_VAR
+SET GLOBAL wsrep_provider_options='repl.max_ws_size=1';
+INSERT INTO t1 VALUES (2);
+
+--error ER_INCORRECT_GLOBAL_LOCAL_VAR
+SET GLOBAL wsrep_provider='none';
+
+DROP TABLE t1;
+
+CALL mtr.add_suppression("transaction size limit");
+CALL mtr.add_suppression("rbr write fail");

mysql-test/suite/wsrep/t/wsrep_provider_plugin_basic.cnf

@@ -0,0 +1,8 @@
+!include ../my.cnf
+
+[mysqld.1]
+wsrep-on=ON
+wsrep-cluster-address=gcomm://
+wsrep-provider=@ENV.WSREP_PROVIDER
+binlog-format=ROW
+plugin-wsrep-provider=ON

mysql-test/suite/wsrep/t/wsrep_provider_plugin_basic.test

@@ -0,0 +1,75 @@
+--source include/have_wsrep.inc
+--source include/have_innodb.inc
+
+
+#
+# Test string option
+#
+
+select variable_type, global_value from information_schema.system_variables where variable_name = 'wsrep_provider_socket_recv_buf_size';
+--error ER_WRONG_VALUE_FOR_VAR
+set global wsrep_provider_socket_recv_buf_size = 'foo';
+set global wsrep_provider_socket_recv_buf_size = '1M';
+show global variables like 'wsrep_provider_socket_recv_buf_size';
+set global wsrep_provider_socket_recv_buf_size = default;
+show global variables like 'wsrep_provider_socket_recv_buf_size';
+
+
+#
+# Test integer option
+#
+
+select variable_type, global_value from information_schema.system_variables where variable_name = 'wsrep_provider_evs_send_window';
+--error ER_WRONG_VALUE_FOR_VAR
+set global wsrep_provider_evs_send_window = -10;
+set global wsrep_provider_evs_send_window = 10;
+show global variables like 'wsrep_provider_evs_send_window';
+set global wsrep_provider_evs_send_window = default;
+show global variables like 'wsrep_provider_evs_send_window';
+
+
+#
+# Test double option
+#
+
+select variable_type from information_schema.system_variables where variable_name = 'wsrep_provider_gcs_max_throttle';
+--error ER_WRONG_VALUE_FOR_VAR
+set global wsrep_provider_gcs_max_throttle = 1.1;
+set global wsrep_provider_gcs_max_throttle = 0.5;
+show global variables like 'wsrep_provider_gcs_max_throttle';
+set global wsrep_provider_gcs_max_throttle = default;
+show global variables like 'wsrep_provider_gcs_max_throttle';
+
+
+#
+# Test bool option
+#
+
+select variable_type from information_schema.system_variables where variable_name = 'wsrep_provider_cert_log_conflicts';
+set global wsrep_provider_cert_log_conflicts = on;
+show global variables like 'wsrep_provider_cert_log_conflicts';
+set global wsrep_provider_cert_log_conflicts = off;
+show global variables like 'wsrep_provider_cert_log_conflicts';
+set global wsrep_provider_cert_log_conflicts = default;
+show global variables like 'wsrep_provider_cert_log_conflicts';
+
+
+#
+# Test read-only option
+#
+
+select read_only from information_schema.system_variables where variable_name = 'wsrep_provider_evs_auto_evict';
+--error ER_INCORRECT_GLOBAL_LOCAL_VAR
+set global wsrep_provider_evs_auto_evict = on;
+
+
+#
+# Test deprecated option (expect warning in result file)
+#
+
+set global wsrep_provider_gcs_fc_master_slave = default;
+
+
+call mtr.add_suppression("error setting param");
+call mtr.add_suppression("Unknown parameter");
+call mtr.add_suppression("Setting parameter");

mysql-test/suite/wsrep/t/wsrep_provider_plugin_defaults.cnf

@@ -0,0 +1,8 @@
+!include ../my.cnf
+
+[mysqld.1]
+wsrep-on=ON
+wsrep-cluster-address=gcomm://
+wsrep-provider=@ENV.WSREP_PROVIDER
+binlog-format=ROW
+plugin-wsrep-provider=ON

mysql-test/suite/wsrep/t/wsrep_provider_plugin_defaults.test

@@ -0,0 +1,15 @@
+--source include/have_wsrep.inc
+--source include/have_innodb.inc
+
+SELECT COUNT(*) FROM INFORMATION_SCHEMA.GLOBAL_VARIABLES WHERE VARIABLE_NAME LIKE 'wsrep_provider%';
+
+--vertical_results
+SELECT * FROM INFORMATION_SCHEMA.SYSTEM_VARIABLES
+WHERE VARIABLE_NAME LIKE 'wsrep_provider_%' AND VARIABLE_NAME NOT IN (
+      'wsrep_provider',
+      'wsrep_provider_options',
+      'wsrep_provider_base_dir',
+      'wsrep_provider_base_port',
+      'wsrep_provider_gcache_dir',
+      'wsrep_provider_gmcast_listen_addr')
+ORDER BY VARIABLE_NAME;

sql/wsrep_mysqld.cc

@@ -56,7 +56,6 @@
 #include <sstream>
 
 /* wsrep-lib */
-Wsrep_server_state* Wsrep_server_state::m_instance;
 
 my_bool wsrep_emulate_bin_log= FALSE; // activating parts of binlog interface
 my_bool wsrep_preordered_opt= FALSE;
@@ -873,7 +872,8 @@ int wsrep_init()
   {
     // enable normal operation in case no provider is specified
     global_system_variables.wsrep_on= 0;
-    int err= Wsrep_server_state::instance().load_provider(wsrep_provider, wsrep_provider_options ? wsrep_provider_options : "");
+    int err= Wsrep_server_state::init_provider(
+        wsrep_provider, wsrep_provider_options ? wsrep_provider_options : "");
     if (err)
     {
       DBUG_PRINT("wsrep",("wsrep::init() failed: %d", err));
@@ -915,7 +915,7 @@ int wsrep_init()
                 "wsrep_trx_fragment_size to 0 or use wsrep_provider that "
                 "supports streaming replication.",
                 wsrep_provider, global_system_variables.wsrep_trx_fragment_size);
-    Wsrep_server_state::instance().unload_provider();
+    Wsrep_server_state::instance().deinit_provider();
     Wsrep_server_state::deinit_provider_services();
     return 1;
   }
@@ -931,7 +931,6 @@ int wsrep_init()
 
   WSREP_DEBUG("SR storage init for: %s",
               (wsrep_SR_store_type == WSREP_SR_STORE_TABLE) ? "table" : "void");
-
   return 0;
 }
 
@@ -1006,6 +1005,11 @@ void wsrep_init_startup (bool sst_first)
   wsrep_create_rollbacker();
   wsrep_create_appliers(1);
 
+  if (Wsrep_server_state::init_options())
+  {
+    WSREP_WARN("Failed to initialize provider options");
+  }
+
   Wsrep_server_state& server_state= Wsrep_server_state::instance();
   /*
     If the SST happens before server initialization, wait until the server
@@ -1031,7 +1035,7 @@ void wsrep_deinit(bool free_options)
   DBUG_ASSERT(wsrep_inited == 1);
   WSREP_DEBUG("wsrep_deinit");
 
-  Wsrep_server_state::instance().unload_provider();
+  Wsrep_server_state::deinit_provider();
   Wsrep_server_state::deinit_provider_services();
 
   provider_name[0]=    '\0';

sql/wsrep_plugin.cc

@@ -1,4 +1,4 @@
-/* Copyright 2016 Codership Oy <http://www.codership.com>
+/* Copyright 2016-2021 Codership Oy <http://www.codership.com>
 
    This program is free software; you can redistribute it and/or modify
    it under the terms of the GNU General Public License as published by
@@ -13,11 +13,306 @@
    along with this program; if not, write to the Free Software
    Foundation, Inc., 51 Franklin Street, Fifth Floor, Boston, MA 02111-1301 USA */
 
-#include "wsrep_trans_observer.h"
+/*
-#include "wsrep_mysqld.h"
+  Wsrep plugin comes in two parts, wsrep_plugin and wsrep_provider_plugin.
 
+  If plugin-wsrep-provider=ON, wsrep_provider_options variable is disabled,
+  in favor of single options which are initialized from provider.
+*/
+
+#include "my_global.h"
+#include "mysqld_error.h"
 #include <mysql/plugin.h>
 
+#include "wsrep_mysqld.h"
+#include "wsrep/provider_options.hpp"
+#include "wsrep_server_state.h"
+#include "wsrep_var.h" // wsrep_refresh_provider_options()
+
+static bool provider_plugin_enabled= false;
+
+/* Prototype for provider system variables */
+static char *dummy_str= 0;
+static MYSQL_SYSVAR_STR(proto_string, dummy_str, 0, 0, 0, 0, "");
+
+
+bool wsrep_provider_plugin_enabled()
+{
+  return provider_plugin_enabled;
+}
+
+/* Returns the name of the variable without prefix */
+static const char *sysvar_name(struct st_mysql_sys_var *var)
+{
+  const char *var_name= ((decltype(mysql_sysvar_proto_string) *) var)->name;
+  long unsigned int prefix_len= sizeof("wsrep_provider_") - 1;
+  return &var_name[prefix_len];
+}
+
+/* Returns option corresponding to the given sysvar */
+static const wsrep::provider_options::option *
+sysvar_to_option(struct st_mysql_sys_var *var)
+{
+  auto options= Wsrep_server_state::get_options();
+  if (!options)
+  {
+    return nullptr;
+  }
+  return options->get_option(sysvar_name(var));
+}
+
+/* Make a boolean option value */
+static std::unique_ptr<wsrep::provider_options::option_value>
+make_option_value(my_bool value)
+{
+  return std::unique_ptr<wsrep::provider_options::option_value>(
+      new wsrep::provider_options::option_value_bool(value));
+}
+
+/* Make a string option value */
+static std::unique_ptr<wsrep::provider_options::option_value>
+make_option_value(const char *value)
+{
+  return std::unique_ptr<wsrep::provider_options::option_value>(
+      new wsrep::provider_options::option_value_string(value));
+}
+
+/* Make a integer option value */
+static std::unique_ptr<wsrep::provider_options::option_value>
+make_option_value(long long value)
+{
+  return std::unique_ptr<wsrep::provider_options::option_value>(
+      new wsrep::provider_options::option_value_int(value));
+}
+
+/* Make a double option value */
+static std::unique_ptr<wsrep::provider_options::option_value>
+make_option_value(double value)
+{
+  return std::unique_ptr<wsrep::provider_options::option_value>(
+      new wsrep::provider_options::option_value_double(value));
+}
+
+/* Helper to get the actual value out of option_value */
+template <class T>
+static T get_option_value(wsrep::provider_options::option_value *value)
+{
+  return *((T *) value->get_ptr());
+}
+
+/* Same as above, specialized for strings */
+template <>
+char *get_option_value(wsrep::provider_options::option_value *value)
+{
+  return (char *) value->get_ptr();
+}
+
+/* Update function for sysvars */
+template <class T>
+static void wsrep_provider_sysvar_update(THD *thd,
+                                         struct st_mysql_sys_var *var,
+                                         void *var_ptr, const void *save)
+{
+  auto opt= sysvar_to_option(var);
+  if (!opt)
+  {
+    WSREP_ERROR("Could not match var to option");
+    my_error(ER_UNKNOWN_ERROR, MYF(0));
+    return;
+  }
+
+  T new_value= *((T *) save);
+
+  auto options= Wsrep_server_state::get_options();
+  if (options->set(opt->name(), std::move(make_option_value(new_value))))
+  {
+    my_error(ER_WRONG_VALUE_FOR_VAR, MYF(0), opt->name(),
+             make_option_value(new_value)->as_string());
+    return;
+  }
+
+  *((T *) var_ptr)= get_option_value<T>(opt->value());
+
+  wsrep_refresh_provider_options();
+}
+
+/* Convert option flags to corresponding sysvar flags */
+static int map_option_flags_to_sysvar(wsrep::provider_options::option *opt)
+{
+  int flags= 0;
+  if (opt->flags() & wsrep::provider_options::flag::readonly)
+    flags|= PLUGIN_VAR_READONLY;
+  if (opt->flags() & wsrep::provider_options::flag::deprecated)
+    flags|= PLUGIN_VAR_DEPRECATED;
+  return flags;
+}
+
+/* Helper to construct a sysvar of type string for the given option */
+static struct st_mysql_sys_var *
+make_sysvar_for_string_option(wsrep::provider_options::option *opt)
+{
+  char *dummy= 0;
+  MYSQL_SYSVAR_STR(proto_string,
+                   dummy,
+                   map_option_flags_to_sysvar(opt),
+                   "Wsrep provider option",
+                   0,
+                   wsrep_provider_sysvar_update<char *>,
+                   get_option_value<char *>(opt->default_value()));
+  mysql_sysvar_proto_string.name= opt->name();
+  char **val= (char **) my_malloc(PSI_NOT_INSTRUMENTED, sizeof(char *), MYF(0));
+  *val= get_option_value<char *>(opt->value());
+  mysql_sysvar_proto_string.value= val;
+  struct st_mysql_sys_var *var= (struct st_mysql_sys_var *) my_malloc(
+    PSI_NOT_INSTRUMENTED, sizeof(mysql_sysvar_proto_string), MYF(0));
+  memcpy(var, &mysql_sysvar_proto_string, sizeof(mysql_sysvar_proto_string));
+  return var;
+}
+
+/* Helper to construct a sysvar of type boolean for the given option */
+static struct st_mysql_sys_var *
+make_sysvar_for_bool_option(wsrep::provider_options::option *opt)
+{
+  my_bool dummy= 0;
+  MYSQL_SYSVAR_BOOL(proto_bool,
+                    dummy,
+                    map_option_flags_to_sysvar(opt),
+                    "Wsrep provider option",
+                    0,
+                    wsrep_provider_sysvar_update<my_bool>,
+                    get_option_value<my_bool>(opt->default_value()));
+  mysql_sysvar_proto_bool.name= opt->name();
+  char *val= (char *) my_malloc(PSI_NOT_INSTRUMENTED, sizeof(char), MYF(0));
+  *val= get_option_value<bool>(opt->value());
+  mysql_sysvar_proto_bool.value= val;
+  struct st_mysql_sys_var *var= (struct st_mysql_sys_var *) my_malloc(
+    PSI_NOT_INSTRUMENTED, sizeof(mysql_sysvar_proto_bool), MYF(0));
+  memcpy(var, &mysql_sysvar_proto_bool, sizeof(mysql_sysvar_proto_bool));
+  return var;
+}
+
+/* Helper to construct a integer sysvar for the given option */
+static struct st_mysql_sys_var *
+make_sysvar_for_integer_option(wsrep::provider_options::option *opt)
+{
+  long long dummy= 0;
+  MYSQL_SYSVAR_LONGLONG(proto_longlong,
+                        dummy,
+                        map_option_flags_to_sysvar(opt),
+                        "Wsrep provider option",
+                        0,
+                        wsrep_provider_sysvar_update<long long>,
+                        get_option_value<long long>(opt->default_value()),
+                        std::numeric_limits<long long>::min(),
+                        std::numeric_limits<long long>::max(),
+                        0);
+  mysql_sysvar_proto_longlong.name= opt->name();
+  long long *val= (long long *) my_malloc(PSI_NOT_INSTRUMENTED, sizeof(long long), MYF(0));
+  *val= get_option_value<long long>(opt->value());
+  mysql_sysvar_proto_longlong.value= val;
+  struct st_mysql_sys_var *var= (struct st_mysql_sys_var *) my_malloc(
+    PSI_NOT_INSTRUMENTED, sizeof(mysql_sysvar_proto_longlong), MYF(0));
+  memcpy(var, &mysql_sysvar_proto_longlong, sizeof(mysql_sysvar_proto_longlong));
+  return var;
+}
+
+/* Helper to construct a sysvar of type double for the given option */
+static struct st_mysql_sys_var *
+make_sysvar_for_double_option(wsrep::provider_options::option *opt)
+{
+  double dummy= 0;
+  MYSQL_SYSVAR_DOUBLE(proto_double,
+                      dummy,
+                      map_option_flags_to_sysvar(opt),
+                      "Wsrep provider option",
+                      0,
+                      wsrep_provider_sysvar_update<double>,
+                      get_option_value<double>(opt->default_value()),
+                      std::numeric_limits<double>::min(),
+                      std::numeric_limits<double>::max(),
+                      0);
+  mysql_sysvar_proto_double.name= opt->name();
+  double *val= (double *) my_malloc(PSI_NOT_INSTRUMENTED, sizeof(double), MYF(0));
+  *val= get_option_value<double>(opt->value());
+  mysql_sysvar_proto_double.value= val;
+  struct st_mysql_sys_var *var= (struct st_mysql_sys_var *) my_malloc(
+    PSI_NOT_INSTRUMENTED, sizeof(mysql_sysvar_proto_double), MYF(0));
+  memcpy(var, &mysql_sysvar_proto_double, sizeof(mysql_sysvar_proto_double));
+  return var;
+}
+
+/* Construct a sysvar corresponding to the given provider option */
+struct st_mysql_sys_var *
+wsrep_make_sysvar_for_option(wsrep::provider_options::option *opt)
+{
+  const int type_flag= opt->flags() & wsrep::provider_options::flag_type_mask;
+  switch (type_flag)
+  {
+  case wsrep::provider_options::flag::type_bool:
+    return make_sysvar_for_bool_option(opt);
+  case wsrep::provider_options::flag::type_integer:
+    return make_sysvar_for_integer_option(opt);
+  case wsrep::provider_options::flag::type_double:
+    return make_sysvar_for_double_option(opt);
+  default:
+    assert(type_flag == 0);
+    return make_sysvar_for_string_option(opt);
+  };
+}
+
+/* Free a sysvar */
+void wsrep_destroy_sysvar(struct st_mysql_sys_var *var)
+{
+  char **var_value= ((decltype(mysql_sysvar_proto_string) *) var)->value;
+  my_free(var_value);
+  my_free(var);
+}
+
+static int wsrep_provider_plugin_init(void *p)
+{
+  WSREP_DEBUG("wsrep_provider_plugin_init()");
+  provider_plugin_enabled= true;
+  return 0;
+}
+
+static int wsrep_provider_plugin_deinit(void *p)
+{
+  WSREP_DEBUG("wsrep_provider_plugin_deinit()");
+  return 0;
+}
+
+struct Mysql_replication wsrep_provider_plugin = {
+  MYSQL_REPLICATION_INTERFACE_VERSION
+};
+
+maria_declare_plugin(wsrep_provider)
+{
+  MYSQL_REPLICATION_PLUGIN,
+  &wsrep_provider_plugin,
+  "wsrep_provider",
+  "Codership Oy",
+  "Wsrep provider plugin",
+  PLUGIN_LICENSE_GPL,
+  wsrep_provider_plugin_init,
+  wsrep_provider_plugin_deinit,
+  0x0100,
+  NULL, /* Status variables */
+  /* System variables, this will be assigned by wsrep plugin below. */
+  NULL,
+  "1.0", /* Version (string) */
+  MariaDB_PLUGIN_MATURITY_ALPHA     /* Maturity */
+}
+maria_declare_plugin_end;
+
+void wsrep_provider_plugin_set_sysvars(st_mysql_sys_var** vars)
+{
+  builtin_maria_wsrep_provider_plugin->system_vars= vars;
+}
+
+/*
+  Wsrep plugin
+*/
+
 static int wsrep_plugin_init(void *p)
 {
   WSREP_DEBUG("wsrep_plugin_init()");

sql/wsrep_server_state.cc

@@ -1,4 +1,4 @@
-/* Copyright 2018 Codership Oy <info@codership.com>
+/* Copyright 2018-2022 Codership Oy <info@codership.com>
 
    This program is free software; you can redistribute it and/or modify
    it under the terms of the GNU General Public License as published by
@@ -19,6 +19,7 @@
 #include "wsrep_allowlist_service.h"
 #include "wsrep_event_service.h"
 #include "wsrep_binlog.h" /* init/deinit group commit */
+#include "wsrep_plugin.h" /* make/destroy sysvar helpers */
 
 mysql_mutex_t LOCK_wsrep_server_state;
 mysql_cond_t  COND_wsrep_server_state;
@@ -30,6 +31,10 @@ PSI_cond_key  key_COND_wsrep_server_state;
 
 wsrep::provider::services Wsrep_server_state::m_provider_services;
 
+Wsrep_server_state* Wsrep_server_state::m_instance;
+std::unique_ptr<wsrep::provider_options> Wsrep_server_state::m_options;
+std::vector<st_mysql_sys_var*> Wsrep_server_state::m_sysvars;
+
 Wsrep_server_state::Wsrep_server_state(const std::string& name,
                                        const std::string& incoming_address,
                                        const std::string& address,
@@ -76,6 +81,48 @@ void Wsrep_server_state::init_once(const std::string& name,
   }
 }
 
+int Wsrep_server_state::init_provider(const std::string& provider,
+                                      const std::string& options)
+{
+  DBUG_ASSERT(m_instance);
+  int ret= m_instance->load_provider(provider, options);
+  if (ret)
+  {
+    WSREP_ERROR("Failed to load provider %s with options %s",
+                provider.c_str(), options.c_str());
+    return ret;
+  }
+  return 0;
+}
+
+int Wsrep_server_state::init_options()
+{
+  if (!m_instance) return 1;
+  m_options= std::unique_ptr<wsrep::provider_options>(
+     new wsrep::provider_options(m_instance->provider()));
+  int ret= m_options->initial_options();
+  if (ret)
+  {
+    WSREP_ERROR("Failed to initialize provider options");
+    m_options = nullptr;
+    m_instance->unload_provider();
+    return ret;
+  }
+  m_options->for_each([](wsrep::provider_options::option *opt) {
+    struct st_mysql_sys_var *var= wsrep_make_sysvar_for_option(opt);
+    m_sysvars.push_back(var);
+  });
+  m_sysvars.push_back(nullptr);
+  wsrep_provider_plugin_set_sysvars(&m_sysvars[0]);
+  return 0;
+}
+
+void Wsrep_server_state::deinit_provider()
+{
+  m_options = nullptr;
+  m_instance->unload_provider();
+}
+
 void Wsrep_server_state::destroy()
 {
   if (m_instance)
@@ -84,6 +131,14 @@ void Wsrep_server_state::destroy()
     m_instance= 0;
     mysql_mutex_destroy(&LOCK_wsrep_server_state);
     mysql_cond_destroy(&COND_wsrep_server_state);
+    for (auto var : m_sysvars)
+    {
+      if (var)
+      {
+        wsrep_destroy_sysvar(var);
+      }
+    }
+    m_sysvars.clear();
   }
 }
 

sql/wsrep_server_state.h

@@ -1,4 +1,4 @@
-/* Copyright 2018 Codership Oy <info@codership.com>
+/* Copyright 2018-2021 Codership Oy <info@codership.com>
 
    This program is free software; you can redistribute it and/or modify
    it under the terms of the GNU General Public License as published by
@@ -19,6 +19,7 @@
 /* wsrep-lib */
 #include "wsrep/server_state.hpp"
 #include "wsrep/provider.hpp"
+#include "wsrep/provider_options.hpp"
 
 /* implementation */
 #include "wsrep_server_service.h"
@@ -34,6 +35,10 @@ public:
                         const std::string& working_dir,
                         const wsrep::gtid& initial_position,
                         int max_protocol_version);
+  static int init_provider(const std::string& provider,
+                           const std::string& options);
+  static int init_options();
+  static void deinit_provider();
   static void destroy();
 
   static Wsrep_server_state& instance()
@@ -51,6 +56,11 @@ public:
     return instance().provider();
   }
 
+  static wsrep::provider_options* get_options()
+  {
+    return m_options.get();
+  }
+
   static bool has_capability(int capability)
   {
     return (get_provider().capabilities() & capability);
@@ -77,7 +87,11 @@ private:
   Wsrep_server_service m_service;
   static wsrep::provider::services m_provider_services;
   static Wsrep_server_state* m_instance;
-
+  static std::unique_ptr<wsrep::provider_options> m_options;
+  // Sysvars for provider plugin. We keep these here because
+  // they are allocated dynamically and must be freed at some
+  // point during shutdown (after the plugin is deinitialized).
+  static std::vector<st_mysql_sys_var *> m_sysvars;
 };
 
 #endif // WSREP_SERVER_STATE_H

sql/wsrep_var.cc

@@ -27,6 +27,7 @@
 #include <cstdlib>
 #include "wsrep_trans_observer.h"
 #include "wsrep_server_state.h"
+#include "wsrep_plugin.h" /* wsrep_provider_plugin_is_enabled() */
 
 ulong   wsrep_reject_queries;
 
@@ -93,6 +94,11 @@ static bool refresh_provider_options()
   }
 }
 
+bool wsrep_refresh_provider_options()
+{
+  return refresh_provider_options();
+}
+
 void wsrep_set_wsrep_on(THD* thd)
 {
   if (thd)
@@ -447,6 +453,12 @@ static int wsrep_provider_verify (const char* provider_str)
 
 bool wsrep_provider_check (sys_var *self, THD* thd, set_var* var)
 {
+  if (wsrep_provider_plugin_enabled())
+  {
+    my_error(ER_INCORRECT_GLOBAL_LOCAL_VAR, MYF(0), var->var->name.str, "read only");
+    return true;
+  }
+
   char wsrep_provider_buf[FN_REFLEN];
 
   if ((! var->save_result.string_value.str) ||
@@ -538,6 +550,11 @@ bool wsrep_provider_options_check(sys_var *self, THD* thd, set_var* var)
     my_message(ER_WRONG_ARGUMENTS, "WSREP (galera) not started", MYF(0));
     return true;
   }
+  if (wsrep_provider_plugin_enabled())
+  {
+    my_error(ER_INCORRECT_GLOBAL_LOCAL_VAR, MYF(0), var->var->name.str, "read only");
+    return true;
+  }
   return false;
 }
 

sql/wsrep_var.h

@@ -1,4 +1,4 @@
-/* Copyright (C) 2013-2021 Codership Oy <info@codership.com>
+/* Copyright (C) 2013-2022 Codership Oy <info@codership.com>
 
    This program is free software; you can redistribute it and/or modify
    it under the terms of the GNU General Public License as published by
@@ -37,6 +37,7 @@ class THD;
 
 int wsrep_init_vars();
 void wsrep_set_wsrep_on(THD *thd);
+bool wsrep_refresh_provider_options();
 
 #define CHECK_ARGS   (sys_var *self, THD* thd, set_var *var)
 #define UPDATE_ARGS  (sys_var *self, THD* thd, enum_var_type type)