From 086c4666374ecbae6ded3af8a5bb8bdbeb55d5e7 Mon Sep 17 00:00:00 2001
From: Teemu Ollakka <teemu.ollakka@galeracluster.com>
Date: Mon, 9 Sep 2019 17:38:39 +0300
Subject: [PATCH] - Added wait-until parameter for begin_nbo_phase_two(). -
 Retry enter_toi() in poll_enter_toi() also for error_connection_failed  
 which means that the connectivity to the cluster has been lost,   a.k.a
 non-prim.

---
 include/wsrep/client_state.hpp |  8 +++++++-
 include/wsrep/provider.hpp     |  3 ++-
 src/client_state.cpp           | 23 ++++++++++++++++-------
 3 files changed, 25 insertions(+), 9 deletions(-)

diff --git a/include/wsrep/client_state.hpp b/include/wsrep/client_state.hpp
index f38768f..6b00a7c 100644
--- a/include/wsrep/client_state.hpp
+++ b/include/wsrep/client_state.hpp
@@ -790,8 +790,13 @@ namespace wsrep
          * passed to begin_nbo_phase_one().
          *
          * @param keys Key array.
+         * @param wait_until Time point to wait until for entering TOI for
+         *                   phase two.
          */
-        int begin_nbo_phase_two(const wsrep::key_array& keys);
+        int begin_nbo_phase_two(const wsrep::key_array& keys,
+                                std::chrono::time_point<wsrep::clock>
+                                wait_until =
+                                std::chrono::time_point<wsrep::clock>());
 
         /**
          * End non-blocking operation phase two. This call will
@@ -1018,6 +1023,7 @@ namespace wsrep
         poll_enter_toi(wsrep::unique_lock<wsrep::mutex>& lock,
                        const wsrep::key_array& keys,
                        const wsrep::const_buffer& buffer,
+                       wsrep::ws_meta& meta,
                        int flags,
                        std::chrono::time_point<wsrep::clock> wait_until);
         void enter_toi_common(wsrep::unique_lock<wsrep::mutex>&);
diff --git a/include/wsrep/provider.hpp b/include/wsrep/provider.hpp
index 970cdf3..2cc30c7 100644
--- a/include/wsrep/provider.hpp
+++ b/include/wsrep/provider.hpp
@@ -237,7 +237,8 @@ namespace wsrep
             error_size_exceeded,
             /** Connectivity to cluster lost */
             error_connection_failed,
-            /** Internal provider failure, provider must be reinitialized */
+            /** Internal provider failure or provider was closed,
+                provider must be reinitialized */
             error_provider_failed,
             /** Fatal error, server must abort */
             error_fatal,
diff --git a/src/client_state.cpp b/src/client_state.cpp
index ad22d34..3655d38 100644
--- a/src/client_state.cpp
+++ b/src/client_state.cpp
@@ -336,6 +336,7 @@ wsrep::client_state::poll_enter_toi(
     wsrep::unique_lock<wsrep::mutex>& lock,
     const wsrep::key_array& keys,
     const wsrep::const_buffer& buffer,
+    wsrep::ws_meta& meta,
     int flags,
     std::chrono::time_point<wsrep::clock> wait_until)
 {
@@ -349,7 +350,7 @@ wsrep::client_state::poll_enter_toi(
     do
     {
         lock.unlock();
-        status = provider().enter_toi(id_, keys, buffer, toi_meta_, flags);
+        status = provider().enter_toi(id_, keys, buffer, meta, flags);
         if (status != wsrep::provider::success &&
             not toi_meta_.gtid().is_undefined())
         {
@@ -364,13 +365,15 @@ wsrep::client_state::poll_enter_toi(
             }
             toi_meta_ = wsrep::ws_meta();
         }
-        if (status == wsrep::provider::error_certification_failed)
+        if (status == wsrep::provider::error_certification_failed ||
+            status == wsrep::provider::error_connection_failed)
         {
             ::usleep(100000);
         }
         lock.lock();
     }
-    while (status == wsrep::provider::error_certification_failed &&
+    while ((status == wsrep::provider::error_certification_failed ||
+            status == wsrep::provider::error_connection_failed) &&
            wait_until.time_since_epoch().count() &&
            wsrep::clock::now() < wait_until &&
            not client_service_.interrupted(lock));
@@ -399,6 +402,7 @@ int wsrep::client_state::enter_toi_local(const wsrep::key_array& keys,
 
     auto const status(poll_enter_toi(
                           lock, keys, buffer,
+                          toi_meta_,
                           wsrep::provider::flag::start_transaction |
                           wsrep::provider::flag::commit,
                           wait_until));
@@ -528,6 +532,7 @@ int wsrep::client_state::begin_nbo_phase_one(
     int ret;
     auto const status(poll_enter_toi(
                           lock, keys, buffer,
+                          toi_meta_,
                           wsrep::provider::flag::start_transaction,
                           wait_until));
     switch (status)
@@ -591,7 +596,9 @@ int wsrep::client_state::enter_nbo_mode(const wsrep::ws_meta& ws_meta)
     return 0;
 }
 
-int wsrep::client_state::begin_nbo_phase_two(const wsrep::key_array& keys)
+int wsrep::client_state::begin_nbo_phase_two(
+    const wsrep::key_array& keys,
+    std::chrono::time_point<wsrep::clock> wait_until)
 {
     debug_log_state("begin_nbo_phase_two: enter");
     debug_log_keys(keys);
@@ -606,9 +613,11 @@ int wsrep::client_state::begin_nbo_phase_two(const wsrep::key_array& keys)
     // Output stored in nbo_meta_ is copied to toi_meta_ for
     // phase two end.
     enum wsrep::provider::status status(
-        provider().enter_toi(id_, keys,
-                             wsrep::const_buffer(), nbo_meta_,
-                             wsrep::provider::flag::commit));
+        poll_enter_toi(lock, keys,
+                       wsrep::const_buffer(),
+                       nbo_meta_,
+                       wsrep::provider::flag::commit,
+                       wait_until));
     int ret;
     switch (status)
     {