MDEV-18577: Indexes problem on import dump SQL

Problem was that we skipped background persistent statistics calculation
on applier nodes if thread is marked as high priority (a.k.a BF).
However, on applier nodes all DDL which is replicate will be executed
as high priority i.e BF.

Fixed by allowing background persistent statistics calculation on
applier nodes even when thread is marked as BF. This could lead
BF lock waits but for queries on that node needs that statistics.

include/mysql/service_wsrep.h

@@ -112,6 +112,7 @@ extern struct wsrep_service_st {
   int                         (*wsrep_trx_order_before_func)(MYSQL_THD, MYSQL_THD);
   void                        (*wsrep_unlock_rollback_func)();
   void                        (*wsrep_set_data_home_dir_func)(const char *data_dir);
+  my_bool                     (*wsrep_thd_is_applier_func)(MYSQL_THD);
 } *wsrep_service;
 
 #ifdef MYSQL_DYNAMIC_PLUGIN
@@ -155,6 +156,7 @@ extern struct wsrep_service_st {
 #define wsrep_trx_order_before(T1,T2) wsrep_service->wsrep_trx_order_before_func(T1,T2)
 #define wsrep_unlock_rollback() wsrep_service->wsrep_unlock_rollback_func()
 #define wsrep_set_data_home_dir(A) wsrep_service->wsrep_set_data_home_dir_func(A)
+#define wsrep_thd_is_applier(T) wsrep_service->wsrep_thd_is_applier(T)
 
 #define wsrep_debug get_wsrep_debug()
 #define wsrep_log_conflicts get_wsrep_log_conflicts()
@@ -214,7 +216,7 @@ void wsrep_thd_set_conflict_state(THD *thd, enum wsrep_conflict_state state);
 bool wsrep_thd_ignore_table(THD *thd);
 void wsrep_unlock_rollback();
 void wsrep_set_data_home_dir(const char *data_dir);
-
+my_bool wsrep_thd_is_applier(MYSQL_THD thd);
 #endif
 
 #ifdef __cplusplus

mysql-test/suite/galera/t/galera_load_data.cnf

@@ -0,0 +1,10 @@
+!include ../galera_2nodes.cnf
+
+[mysqld]
+secure-file-priv = ""
+innodb_file_format ='Barracuda'
+innodb_file_per_table = ON
+innodb_stats_persistent=ON
+innodb_stats_auto_recalc=ON
+innodb_stats_persistent_sample_pages=20
+innodb_stats_sample_pages=8

sql/sql_plugin_services.ic

@@ -181,7 +181,8 @@ static struct wsrep_service_st wsrep_handler = {
   wsrep_trx_is_aborting,
   wsrep_trx_order_before,
   wsrep_unlock_rollback,
-  wsrep_set_data_home_dir
+  wsrep_set_data_home_dir,
+  wsrep_thd_is_applier
 };
 
 static struct thd_specifics_service_st thd_specifics_handler=

sql/wsrep_dummy.cc

@@ -141,3 +141,6 @@ void wsrep_unlock_rollback()
 
 void wsrep_set_data_home_dir(const char *)
 { }
+
+my_bool wsrep_thd_is_applier(MYSQL_THD thd)
+{ return false; }

sql/wsrep_thd.cc

@@ -698,3 +698,13 @@ void wsrep_thd_auto_increment_variables(THD* thd,
     *increment= thd->variables.auto_increment_increment;
   }
 }
+
+my_bool wsrep_thd_is_applier(MYSQL_THD thd)
+{
+  my_bool is_applier= false;
+
+  if (thd && thd->wsrep_applier)
+    is_applier= true;
+
+  return (is_applier);
+}

storage/innobase/dict/dict0stats_bg.cc

@@ -164,9 +164,25 @@ void dict_stats_update_if_needed_func(dict_table_t* table)
 		    && dict_stats_auto_recalc_is_enabled(table)) {
 
 #ifdef WITH_WSREP
-			if (thd && wsrep_on(thd) && wsrep_thd_is_BF(thd, 0)) {
+			/* Do not add table to background
+			statistic calculation if this thread is not a
+			applier (as all DDL, which is replicated (i.e
+			is binlogged in master node), will be executed
+			with high priority (a.k.a BF) in slave nodes)
+			and is BF. This could again lead BF lock
+			waits in applier node but it is better than
+			no persistent index/table statistics at
+			applier nodes. TODO: allow BF threads
+			wait for these InnoDB internal SQL-parser
+			generated row locks and allow BF thread
+			lock waits to be enqueued at head of waiting
+			queue. */
+			if (thd
+			    && !wsrep_thd_is_applier(thd)
+			    && wsrep_on(thd)
+			    && wsrep_thd_is_BF(thd, 0)) {
 				WSREP_DEBUG("Avoiding background statistics"
-					    " calculation for table %s",
+					    " calculation for table %s.",
 					table->name.m_name);
 				return;
 			}