Implementation of client_state NBO operations.

- Implemented calls to enter and leave NBO phase one and two
- Extended client_state mode checking to include m_nbo
- Changed client_state state and mode change sanity checks to
  print a warning and assert() instead of throwing exceptions
  to be more graceful in release builds.

test/CMakeLists.txt

@@ -10,6 +10,7 @@ add_executable(wsrep-lib_test
   buffer_test.cpp
   gtid_test.cpp
   id_test.cpp
+  nbo_test.cpp
   server_context_test.cpp
   toi_test.cpp
   transaction_test.cpp

test/mock_provider.hpp

@@ -53,6 +53,9 @@ namespace wsrep
             , fragments_()
             , commit_fragments_()
             , rollback_fragments_()
+            , toi_write_sets_()
+            , toi_start_transaction_()
+            , toi_commit_()
         { }
 
         enum wsrep::provider::status
@@ -238,19 +241,22 @@ namespace wsrep
                                                const wsrep::key_array&,
                                                const wsrep::const_buffer&,
                                                wsrep::ws_meta& toi_meta,
-                                               int)
+                                               int flags)
             WSREP_OVERRIDE
         {
-                ++group_seqno_;
-                wsrep::gtid gtid(group_id_, wsrep::seqno(group_seqno_));
-                wsrep::stid stid(server_id_,
-                                 wsrep::transaction_id::undefined(),
-                                 client_id);
-                toi_meta = wsrep::ws_meta(gtid, stid,
-                                          wsrep::seqno(group_seqno_ - 1),
-                                          wsrep::provider::flag::start_transaction |
-                                          wsrep::provider::flag::commit);
-
+            ++group_seqno_;
+            wsrep::gtid gtid(group_id_, wsrep::seqno(group_seqno_));
+            wsrep::stid stid(server_id_,
+                             wsrep::transaction_id::undefined(),
+                             client_id);
+            toi_meta = wsrep::ws_meta(gtid, stid,
+                                      wsrep::seqno(group_seqno_ - 1),
+                                      flags);
+            ++toi_write_sets_;
+            if (flags & wsrep::provider::flag::start_transaction)
+                ++toi_start_transaction_;
+            if (flags & wsrep::provider::flag::commit)
+                ++toi_commit_;
             return wsrep::provider::success;
         }
 
@@ -325,7 +331,9 @@ namespace wsrep
         size_t fragments() const { return fragments_; }
         size_t commit_fragments() const { return commit_fragments_; }
         size_t rollback_fragments() const { return rollback_fragments_; }
-
+        size_t toi_write_sets() const { return toi_write_sets_; }
+        size_t toi_start_transaction() const { return toi_start_transaction_; }
+        size_t toi_commit() const { return toi_commit_; }
     private:
         wsrep::id group_id_;
         wsrep::id server_id_;
@@ -335,6 +343,9 @@ namespace wsrep
         size_t fragments_;
         size_t commit_fragments_;
         size_t rollback_fragments_;
+        size_t toi_write_sets_;
+        size_t toi_start_transaction_;
+        size_t toi_commit_;
     };
 }
 

test/nbo_test.cpp

@@ -0,0 +1,68 @@
+/*
+ * Copyright (C) 2019 Codership Oy <info@codership.com>
+ *
+ * This file is part of wsrep-lib.
+ *
+ * Wsrep-lib is free software: you can redistribute it and/or modify
+ * it under the terms of the GNU General Public License as published by
+ * the Free Software Foundation, either version 2 of the License, or
+ * (at your option) any later version.
+ *
+ * Wsrep-lib is distributed in the hope that it will be useful,
+ * but WITHOUT ANY WARRANTY; without even the implied warranty of
+ * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
+ * GNU General Public License for more details.
+ *
+ * You should have received a copy of the GNU General Public License
+ * along with wsrep-lib.  If not, see <https://www.gnu.org/licenses/>.
+ */
+
+#include "wsrep/client_state.hpp"
+
+#include "client_state_fixture.hpp"
+
+#include <boost/test/unit_test.hpp>
+
+BOOST_FIXTURE_TEST_CASE(test_local_nbo,
+                        replicating_client_fixture_sync_rm)
+{
+    // NBO is executed in two consecutive TOI operations
+    BOOST_REQUIRE(cc.mode() == wsrep::client_state::m_local);
+    // First phase certifies the write set and enters TOI
+    wsrep::key key(wsrep::key::exclusive);
+    key.append_key_part("k1", 2);
+    key.append_key_part("k2", 2);
+    wsrep::key_array keys{key};
+    std::string data("data");
+    BOOST_REQUIRE(cc.begin_nbo_phase_one(
+                      keys,
+                      wsrep::const_buffer(data.data(),
+                                          data.size())) == 0);
+    BOOST_REQUIRE(cc.mode() == wsrep::client_state::m_nbo);
+    BOOST_REQUIRE(cc.in_toi());
+    BOOST_REQUIRE(cc.toi_mode() == wsrep::client_state::m_local);
+    // After required resoureces have been grabbed, NBO leave should
+    // end TOI but leave the client state in m_nbo.
+    BOOST_REQUIRE(cc.end_nbo_phase_one() == 0);
+    BOOST_REQUIRE(cc.mode() == wsrep::client_state::m_nbo);
+    BOOST_REQUIRE(cc.in_toi() == false);
+    BOOST_REQUIRE(cc.toi_mode() == wsrep::client_state::m_local);
+    // Second phase replicates the NBO end event and grabs TOI
+    // again for finalizing the NBO.
+    BOOST_REQUIRE(cc.begin_nbo_phase_two(keys) == 0);
+    BOOST_REQUIRE(cc.mode() == wsrep::client_state::m_nbo);
+    BOOST_REQUIRE(cc.in_toi());
+    BOOST_REQUIRE(cc.toi_mode() == wsrep::client_state::m_local);
+    // End of NBO phase will leave TOI and put the client state back to
+    // m_local
+    BOOST_REQUIRE(cc.end_nbo_phase_two() == 0);
+    BOOST_REQUIRE(cc.mode() == wsrep::client_state::m_local);
+    BOOST_REQUIRE(cc.in_toi() == false);
+    BOOST_REQUIRE(cc.toi_mode() == wsrep::client_state::m_local);
+
+    // There must have been two toi write sets, one with
+    // start transaction flag, another with commit flag.
+    BOOST_REQUIRE(sc.provider().toi_write_sets() == 2);
+    BOOST_REQUIRE(sc.provider().toi_start_transaction() == 1);
+    BOOST_REQUIRE(sc.provider().toi_commit() == 1);
+}

test/toi_test.cpp

@@ -27,6 +27,7 @@ BOOST_FIXTURE_TEST_CASE(test_toi_mode,
                         replicating_client_fixture_sync_rm)
 {
     BOOST_REQUIRE(cc.mode() == wsrep::client_state::m_local);
+    BOOST_REQUIRE(cc.toi_mode() == wsrep::client_state::m_local);
     wsrep::key key(wsrep::key::exclusive);
     key.append_key_part("k1", 2);
     key.append_key_part("k2", 2);
@@ -40,6 +41,10 @@ BOOST_FIXTURE_TEST_CASE(test_toi_mode,
     BOOST_REQUIRE(cc.toi_mode() == wsrep::client_state::m_local);
     BOOST_REQUIRE(cc.leave_toi() == 0);
     BOOST_REQUIRE(cc.mode() == wsrep::client_state::m_local);
+    BOOST_REQUIRE(cc.toi_mode() == wsrep::client_state::m_local);
+    BOOST_REQUIRE(sc.provider().toi_write_sets() == 1);
+    BOOST_REQUIRE(sc.provider().toi_start_transaction() == 1);
+    BOOST_REQUIRE(sc.provider().toi_commit() == 1);
 }
 
 BOOST_FIXTURE_TEST_CASE(test_toi_applying,
@@ -50,6 +55,7 @@ BOOST_FIXTURE_TEST_CASE(test_toi_applying,
                    cc.after_commit()) == 0);
     cc.after_applying();
 
+    BOOST_REQUIRE(cc.toi_mode() == wsrep::client_state::m_local);
     wsrep::ws_meta ws_meta(wsrep::gtid(wsrep::id("1"), wsrep::seqno(2)),
                            wsrep::stid(sc.id(),
                                        wsrep::transaction_id::undefined(),
@@ -61,5 +67,6 @@ BOOST_FIXTURE_TEST_CASE(test_toi_applying,
     BOOST_REQUIRE(cc.in_toi());
     BOOST_REQUIRE(cc.toi_mode() == wsrep::client_state::m_high_priority);
     BOOST_REQUIRE(cc.leave_toi() == 0);
+    BOOST_REQUIRE(cc.toi_mode() == wsrep::client_state::m_local);
     cc.after_applying();
 }