Fixed assertion on server_state connected - disconnecting transition

Transition from server_state connected state to disconnecting must
be allowed to deal with errors during server startup.

Added SST first test cases for server_state transitions:
* Successful join via SST
* Error in connect state
* Error in joiner state

include/wsrep/server_state.hpp

@@ -117,7 +117,7 @@ namespace wsrep
          *
          * @todo Fix UML generation
          *
-         * Server state diagram if the sst_before_init() returns false.
+         * Server state diagram if initialization happens before SST.
          *
          * [*] --> disconnected
          * disconnected --> initializing
@@ -129,7 +129,7 @@ namespace wsrep
          * synced --> donor
          * donor --> joined
          *
-         * Server state diagram if the sst_before_init() returns true.
+         * Server state diagram if SST happens before initialization.
          *
          * [*] --> disconnected
          * disconnected --> connected

src/server_state.cpp

@@ -531,6 +531,9 @@ void wsrep::server_state::sst_received(wsrep::client_service& cs,
         if (init_initialized_ == false)
         {
             state(lock, s_initializing);
+            lock.unlock();
+            server_service_.debug_sync("on_view_wait_initialized");
+            lock.lock();
             wait_until_state(lock, s_initialized);
             assert(init_initialized_);
         }
@@ -1072,7 +1075,7 @@ void wsrep::server_state::state(
             {  0,   1,   0,    1,    0,   0,   0,   0,   0}, /* dis */
             {  1,   0,   1,    0,    0,   0,   0,   0,   0}, /* ing */
             {  1,   0,   0,    1,    0,   1,   0,   0,   0}, /* ized */
-            {  1,   0,   0,    1,    1,   0,   0,   1,   0}, /* cted */
+            {  1,   0,   0,    1,    1,   0,   0,   1,   1}, /* cted */
             {  1,   1,   0,    0,    0,   1,   0,   0,   0}, /* jer */
             {  1,   0,   0,    1,    0,   0,   0,   1,   1}, /* jed */
             {  1,   0,   0,    0,    0,   1,   0,   0,   1}, /* dor */

test/mock_server_state.hpp

@@ -186,6 +186,10 @@ namespace wsrep
         } sync_point_action_;
         bool sst_before_init_;
 
+        void logged_view(const wsrep::view& view)
+        {
+            logged_view_ = view;
+        }
     private:
         wsrep::server_state& server_state_;
         unsigned long long last_client_id_;

test/server_context_test.cpp

@@ -42,10 +42,12 @@ namespace
                       wsrep::provider::flag::commit)
             , cluster_id("1")
             , bootstrap_view()
+            , second_view()
         {
             wsrep::gtid state_id(cluster_id, wsrep::seqno(0));
             std::vector<wsrep::view::member> members;
-            members.push_back(wsrep::view::member(wsrep::id("s1"), "name", ""));
+            members.push_back(wsrep::view::member(
+                                  wsrep::id("s1"), "s1", ""));
             bootstrap_view = wsrep::view(state_id,
                                          wsrep::seqno(1),
                                          wsrep::view::primary,
@@ -54,6 +56,16 @@ namespace
                                          1, // protocol version
                                          members);
 
+            members.push_back(wsrep::view::member(
+                                  wsrep::id("s2"), "s2", ""));
+            second_view = wsrep::view(wsrep::gtid(cluster_id, wsrep::seqno(1)),
+                                      wsrep::seqno(2),
+                                      wsrep::view::primary,
+                                      0, // capabilities
+                                      1, // own index
+                                      1, // protocol version
+                                      members);
+
             cc.open(cc.id());
             BOOST_REQUIRE(cc.before_command() == 0);
         }
@@ -65,6 +77,7 @@ namespace
         wsrep::ws_meta ws_meta;
         wsrep::id cluster_id;
         wsrep::view bootstrap_view;
+        wsrep::view second_view;
 
         void final_view()
         {
@@ -79,6 +92,16 @@ namespace
                 );
             ss.on_view(view, &hps);
         }
+
+        void disconnect()
+        {
+            BOOST_REQUIRE(ss.state() != wsrep::server_state::s_disconnecting);
+            ss.disconnect();
+            BOOST_REQUIRE(ss.state() == wsrep::server_state::s_disconnecting);
+            final_view();
+            BOOST_REQUIRE(ss.state() == wsrep::server_state::s_disconnected);
+        }
+
     };
 
     struct applying_server_fixture : server_fixture_base
@@ -98,16 +121,37 @@ namespace
             server_service.sst_before_init_ = true;
         }
 
+        void connect_in_view(const wsrep::view& view)
+        {
+            BOOST_REQUIRE(ss.connect("cluster", "local", "0", false) == 0);
+            ss.on_connect(view);
+            BOOST_REQUIRE(ss.state() == wsrep::server_state::s_connected);
+        }
+
+        void prepare_for_sst()
+        {
+            ss.prepare_for_sst();
+            BOOST_REQUIRE(ss.state() == wsrep::server_state::s_joiner);
+        }
+
+        void sst_received_action()
+        {
+            server_service.sync_point_enabled_ = "on_view_wait_initialized";
+            server_service.sync_point_action_  = server_service.spa_initialize;
+        }
+        void clear_sst_received_action()
+        {
+            server_service.sync_point_enabled_ = "";
+        }
+
         // Helper method to bootstrap the server with bootstrap view
         void bootstrap()
         {
-            BOOST_REQUIRE(ss.connect("cluster", "local", "0", false) == 0);
+            connect_in_view(bootstrap_view);
-            ss.on_connect(bootstrap_view);
+
-            BOOST_REQUIRE(ss.state() == wsrep::server_state::s_connected);
+            sst_received_action();
-            server_service.sync_point_enabled_ = "on_view_wait_initialized";
-            server_service.sync_point_action_  = server_service.spa_initialize;
             ss.on_view(bootstrap_view, &hps);
-            server_service.sync_point_enabled_ = "";
+            clear_sst_received_action();
             BOOST_REQUIRE(ss.state() == wsrep::server_state::s_joined);
             ss.on_sync();
             BOOST_REQUIRE(ss.state() == wsrep::server_state::s_synced);
@@ -258,6 +302,24 @@ BOOST_FIXTURE_TEST_CASE(server_state_sst_first_boostrap,
     BOOST_REQUIRE(ss.state() == wsrep::server_state::s_synced);
 }
 
+
+BOOST_FIXTURE_TEST_CASE(server_state_sst_first_join_with_sst,
+                        sst_first_server_fixture)
+{
+    connect_in_view(second_view);
+    prepare_for_sst();
+    sst_received_action();
+    // Mock server service get_view() gets view from logged_view_.
+    // Get_view() is called from sst_received(). This emulates the
+    // case where SST contains the view in which SST happens.
+    server_service.logged_view(second_view);
+    ss.sst_received(cc, wsrep::gtid(cluster_id, wsrep::seqno(2)), 0);
+    clear_sst_received_action();
+    BOOST_REQUIRE(ss.state() == wsrep::server_state::s_joined);
+    ss.on_sync();
+    BOOST_REQUIRE(ss.state() == wsrep::server_state::s_synced);
+}
+
 // Cycle from synced state to disconnected and back to synced. Server
 // storage engines remain initialized.
 BOOST_FIXTURE_TEST_CASE(
@@ -296,6 +358,28 @@ BOOST_FIXTURE_TEST_CASE(
     BOOST_REQUIRE(ss.state() == wsrep::server_state::s_synced);
 }
 
+//
+// Error after connecting to cluster. This scenario may happen if SST
+// request preparation fails.
+//
+BOOST_FIXTURE_TEST_CASE(
+    server_state_sst_first_error_on_connect,
+    sst_first_server_fixture)
+{
+    connect_in_view(second_view);
+    BOOST_REQUIRE(ss.state() == wsrep::server_state::s_connected);
+    disconnect();
+}
+
+// Error during SST.q
+BOOST_FIXTURE_TEST_CASE(
+    server_state_sst_first_error_on_joiner,
+    sst_first_server_fixture)
+{
+    connect_in_view(second_view);
+    ss.prepare_for_sst();
+    BOOST_REQUIRE(ss.state() == wsrep::server_state::s_joiner);
+}
 
 ///////////////////////////////////////////////////////////////////////////////
 //                     Test cases for init first                             //