codership/wsrep-lib#34 Provided a method to interrupt state waiters

Intruduced server_state::interrupt_state_waiters() to interrupt
all waiters inside server_state::wait_until_state(). This mechanism
is needed when an error is encountered during state change processing
and waiting threads may need to be interrupted to check and handle
the error condition.

Made server_state::wait_until_state() to throw exception if the
wait was interrupted and the new server state is either disconnecting
or disconnected, which usually indicates error condition.

include/wsrep/server_state.hpp

@@ -600,6 +600,8 @@ namespace wsrep
         void resync(wsrep::unique_lock<wsrep::mutex>&);
         void state(wsrep::unique_lock<wsrep::mutex>&, enum state);
         void wait_until_state(wsrep::unique_lock<wsrep::mutex>&, enum state) const;
+        // Interrupt all threads which are waiting for state
+        void interrupt_state_waiters(wsrep::unique_lock<wsrep::mutex>&);
         // Close SR transcations whose origin is outside of current
         // cluster view.
         void close_orphaned_sr_transactions(

src/server_state.cpp

@@ -365,6 +365,7 @@ int wsrep::server_state::disconnect()
     {
         wsrep::unique_lock<wsrep::mutex> lock(mutex_);
         state(lock, s_disconnecting);
+        interrupt_state_waiters(lock);
     }
     return provider().disconnect();
 }
@@ -1084,8 +1085,8 @@ void wsrep::server_state::state(
         {
             /* dis, ing, ized, cted, jer, jed, dor, sed, ding */
             {  0,   1,   0,    1,    0,   0,   0,   0,   0}, /* dis */
-            {  1,   0,   1,    0,    0,   0,   0,   0,   0}, /* ing */
+            {  1,   0,   1,    0,    0,   0,   0,   0,   1}, /* ing */
-            {  1,   0,   0,    1,    0,   1,   0,   0,   0}, /* ized */
+            {  1,   0,   0,    1,    0,   1,   0,   0,   1}, /* ized */
             {  1,   0,   0,    1,    1,   0,   0,   1,   1}, /* cted */
             {  1,   1,   0,    0,    0,   1,   0,   0,   1}, /* jer */
             {  1,   0,   0,    1,    0,   0,   0,   1,   1}, /* jed */
@@ -1127,11 +1128,26 @@ void wsrep::server_state::wait_until_state(
     while (state_ != state)
     {
         cond_.wait(lock);
+        // If the waiter waits for any other state than disconnecting
+        // or disconnected and the state has been changed to disconnecting,
+        // this usually means that some error was encountered 
+        if (state != s_disconnecting && state != s_disconnected
+            && state_ == s_disconnecting)
+        {
+            throw wsrep::runtime_error("State wait was interrupted");
+        }
     }
     --state_waiters_[state];
     cond_.notify_all();
 }
 
+void wsrep::server_state::interrupt_state_waiters(
+    wsrep::unique_lock<wsrep::mutex>& lock WSREP_UNUSED)
+{
+    assert(lock.owns_lock());
+    cond_.notify_all();
+}
+
 void wsrep::server_state::close_orphaned_sr_transactions(
     wsrep::unique_lock<wsrep::mutex>& lock,
     wsrep::high_priority_service& high_priority_service)

src/wsrep_provider_v26.cpp

@@ -364,7 +364,7 @@ namespace
         }
         catch (const wsrep::runtime_error& e)
         {
-            std::cerr << "Exception: " << e.what();
+            wsrep::log_error() << "Exception: " << e.what();
             return WSREP_CB_FAILURE;
         }
     }
@@ -389,7 +389,7 @@ namespace
         }
         catch (const wsrep::runtime_error& e)
         {
-            std::cerr << "Exception: " << e.what();
+            wsrep::log_error() << "Exception: " << e.what();
             return WSREP_CB_FAILURE;
         }
     }