Transaction context unit tests for failures in certify_commit()

include/wsrep/client_context.hpp

@@ -56,6 +56,7 @@ namespace wsrep
         e_success,
         e_error_during_commit,
         e_deadlock_error,
+        e_interrupted_error,
         e_append_fragment_error
     };
 
@@ -66,6 +67,7 @@ namespace wsrep
         case e_success:               return "success";
         case e_error_during_commit:   return "error_during_commit";
         case e_deadlock_error:        return "deadlock_error";
+        case e_interrupted_error:     return "interrupted_error";
         case e_append_fragment_error: return "append_fragment_error";
         }
         return "unknown";
@@ -453,15 +455,10 @@ namespace wsrep
         /*!
          * Wait until all of the replaying transactions have been committed.
          */
-        virtual void wait_for_replayers(wsrep::unique_lock<wsrep::mutex>&) const = 0;
+        virtual void wait_for_replayers(wsrep::unique_lock<wsrep::mutex>&) = 0;
 
         virtual int prepare_data_for_replication(
-            const wsrep::transaction_context&, wsrep::data& data)
+            const wsrep::transaction_context&, wsrep::data& data) = 0;
-        {
-            static const char buf[1] = { 1 };
-            data = wsrep::data(buf, 1);
-            return 0;
-        }
 
         /*!
          * Return true if the current client operation was killed.

src/dbms_simulator.cpp

@@ -376,8 +376,14 @@ private:
         ++stats_.replays;
         return provider().replay(txc.ws_handle(), this);
     }
-    void wait_for_replayers(wsrep::unique_lock<wsrep::mutex>&) const override
+    void wait_for_replayers(wsrep::unique_lock<wsrep::mutex>&) override
     { }
+    int prepare_data_for_replication(const wsrep::transaction_context&,
+                                     wsrep::data&)
+        override
+    {
+        return 0;
+    }
     bool killed() const override { return false; }
     void abort() const override { ::abort(); }
 public:

src/mock_client_context.hpp

@@ -9,6 +9,8 @@
 #include "wsrep/mutex.hpp"
 #include "wsrep/compiler.hpp"
 
+#include "test_utils.hpp"
+
 namespace wsrep
 {
     class mock_client_context : public wsrep::client_context
@@ -25,6 +27,9 @@ namespace wsrep
             , is_autocommit_(false)
             , do_2pc_(do_2pc)
             , fail_next_applying_()
+            , bf_abort_during_wait_()
+            , error_during_prepare_data_()
+            , killed_before_certify_()
         { }
         ~mock_client_context()
         {
@@ -47,9 +52,29 @@ namespace wsrep
             tc.state(lock, wsrep::transaction_context::s_committed);
             return 0;
         }
-        void wait_for_replayers(wsrep::unique_lock<wsrep::mutex>&) const
+        void wait_for_replayers(wsrep::unique_lock<wsrep::mutex>& lock)
-            WSREP_OVERRIDE { }
+            WSREP_OVERRIDE
-        bool killed() const WSREP_OVERRIDE { return false; }
+        {
+            lock.unlock();
+            if (bf_abort_during_wait_)
+            {
+                wsrep_test::bf_abort_unordered(*this);
+            }
+            lock.lock();
+        }
+        int prepare_data_for_replication(
+            const wsrep::transaction_context&, wsrep::data& data) WSREP_OVERRIDE
+        {
+            if (error_during_prepare_data_)
+            {
+                return 1;
+            }
+            static const char buf[1] = { 1 };
+            data = wsrep::data(buf, 1);
+            return 0;
+
+        }
+        bool killed() const WSREP_OVERRIDE { return killed_before_certify_; }
         void abort() const WSREP_OVERRIDE { }
         void store_globals() WSREP_OVERRIDE { }
         void debug_sync(const char*) WSREP_OVERRIDE { }
@@ -58,14 +83,17 @@ namespace wsrep
             ::abort();
         }
         void on_error(enum wsrep::client_error) { }
-        // Mock state modifiers
+
-        void fail_next_applying(bool fail_next_applying)
+        //
-        { fail_next_applying_ = fail_next_applying; }
     private:
         wsrep::default_mutex mutex_;
+    public:
         bool is_autocommit_;
         bool do_2pc_;
         bool fail_next_applying_;
+        bool bf_abort_during_wait_;
+        bool error_during_prepare_data_;
+        bool killed_before_certify_;
     };
 }
 

src/server_context_test.cpp

@@ -64,7 +64,7 @@ BOOST_FIXTURE_TEST_CASE(server_context_applying_2pc,
 BOOST_FIXTURE_TEST_CASE(server_context_applying_1pc_rollback,
                         applying_server_fixture)
 {
-    cc.fail_next_applying(true);
+    cc.fail_next_applying_ = true;
     char buf[1] = { 1 };
     BOOST_REQUIRE(sc.on_apply(cc, ws_handle, ws_meta,
                               wsrep::data(buf, 1)) == 1);
@@ -77,7 +77,7 @@ BOOST_FIXTURE_TEST_CASE(server_context_applying_1pc_rollback,
 BOOST_FIXTURE_TEST_CASE(server_context_applying_2pc_rollback,
                         applying_server_fixture)
 {
-    cc.fail_next_applying(true);
+    cc.fail_next_applying_ = true;
     char buf[1] = { 1 };
     BOOST_REQUIRE(sc.on_apply(cc, ws_handle, ws_meta,
                               wsrep::data(buf, 1)) == 1);

src/transaction_context.cpp

@@ -688,7 +688,7 @@ int wsrep::transaction_context::certify_commit(
     wsrep::unique_lock<wsrep::mutex>& lock)
 {
     assert(lock.owns_lock());
-    assert(id_ != wsrep::transaction_id::invalid());
+    assert(active());
 
     client_context_.wait_for_replayers(lock);
 
@@ -710,6 +710,7 @@ int wsrep::transaction_context::certify_commit(
     {
         // Note: Error must be set by prepare_data_for_replication()
         lock.lock();
+        client_context_.override_error(wsrep::e_error_during_commit);
         state(lock, s_must_abort);
         return 1;
     }
@@ -717,7 +718,7 @@ int wsrep::transaction_context::certify_commit(
     if (client_context_.killed())
     {
         lock.lock();
-        client_context_.override_error(wsrep::e_deadlock_error);
+        client_context_.override_error(wsrep::e_interrupted_error);
         state(lock, s_must_abort);
         return 1;
     }

src/transaction_context_test.cpp

@@ -90,7 +90,6 @@ namespace
 }
 
 
-
 //
 // Test a succesful 1PC transaction lifecycle
 //
@@ -356,6 +355,164 @@ BOOST_FIXTURE_TEST_CASE_TEMPLATE(
     BOOST_REQUIRE(cc.current_error());
 }
 
+//
+// Test a 1PC transaction which gets BF aborted during before_commit
+// when waiting for replayers
+//
+BOOST_FIXTURE_TEST_CASE_TEMPLATE(
+    transaction_context_1pc_bf_during_commit_wait_for_replayers, T,
+    replicating_fixtures, T)
+{
+    wsrep::mock_client_context& cc(T::cc);
+    const wsrep::transaction_context& tc(T::tc);
+
+    // Start a new transaction with ID 1
+    cc.start_transaction(1);
+    BOOST_REQUIRE(tc.active());
+    BOOST_REQUIRE(tc.id() == wsrep::transaction_id(1));
+    BOOST_REQUIRE(tc.state() == wsrep::transaction_context::s_executing);
+
+    cc.bf_abort_during_wait_ = true;
+
+    // Run before commit
+    BOOST_REQUIRE(cc.before_commit());
+    BOOST_REQUIRE(tc.state() == wsrep::transaction_context::s_must_abort);
+    BOOST_REQUIRE(tc.certified() == false);
+    BOOST_REQUIRE(tc.ordered() == false);
+
+    // Rollback sequence
+    BOOST_REQUIRE(cc.before_rollback() == 0);
+    BOOST_REQUIRE(tc.state() == wsrep::transaction_context::s_aborting);
+    BOOST_REQUIRE(cc.after_rollback() == 0);
+    BOOST_REQUIRE(tc.state() == wsrep::transaction_context::s_aborted);
+
+    // Cleanup after statement
+    cc.after_statement();
+    BOOST_REQUIRE(tc.active() == false);
+    BOOST_REQUIRE(tc.ordered() == false);
+    BOOST_REQUIRE(tc.certified() == false);
+    BOOST_REQUIRE(cc.current_error());
+}
+
+//
+// Test a 1PC transaction for which prepare data fails
+//
+BOOST_FIXTURE_TEST_CASE_TEMPLATE(
+    transaction_context_1pc_error_during_prepare_data, T,
+    replicating_fixtures, T)
+{
+    wsrep::mock_client_context& cc(T::cc);
+    const wsrep::transaction_context& tc(T::tc);
+
+    // Start a new transaction with ID 1
+    cc.start_transaction(1);
+    BOOST_REQUIRE(tc.active());
+    BOOST_REQUIRE(tc.id() == wsrep::transaction_id(1));
+    BOOST_REQUIRE(tc.state() == wsrep::transaction_context::s_executing);
+
+    cc.error_during_prepare_data_ = true;
+
+    // Run before commit
+    BOOST_REQUIRE(cc.before_commit());
+    BOOST_REQUIRE(tc.state() == wsrep::transaction_context::s_must_abort);
+    BOOST_REQUIRE(cc.current_error() == wsrep::e_error_during_commit);
+    BOOST_REQUIRE(tc.certified() == false);
+    BOOST_REQUIRE(tc.ordered() == false);
+
+    // Rollback sequence
+    BOOST_REQUIRE(cc.before_rollback() == 0);
+    BOOST_REQUIRE(tc.state() == wsrep::transaction_context::s_aborting);
+    BOOST_REQUIRE(cc.after_rollback() == 0);
+    BOOST_REQUIRE(tc.state() == wsrep::transaction_context::s_aborted);
+
+    // Cleanup after statement
+    cc.after_statement();
+    BOOST_REQUIRE(tc.active() == false);
+    BOOST_REQUIRE(tc.ordered() == false);
+    BOOST_REQUIRE(tc.certified() == false);
+    BOOST_REQUIRE(cc.current_error());
+}
+
+//
+// Test a 1PC transaction which gets killed by DBMS before certification
+//
+BOOST_FIXTURE_TEST_CASE_TEMPLATE(
+    transaction_context_1pc_killed_before_certify, T,
+    replicating_fixtures, T)
+{
+    wsrep::mock_client_context& cc(T::cc);
+    const wsrep::transaction_context& tc(T::tc);
+
+    // Start a new transaction with ID 1
+    cc.start_transaction(1);
+    BOOST_REQUIRE(tc.active());
+    BOOST_REQUIRE(tc.id() == wsrep::transaction_id(1));
+    BOOST_REQUIRE(tc.state() == wsrep::transaction_context::s_executing);
+
+    cc.killed_before_certify_ = true;
+
+    // Run before commit
+    BOOST_REQUIRE(cc.before_commit());
+    BOOST_REQUIRE(tc.state() == wsrep::transaction_context::s_must_abort);
+    BOOST_REQUIRE(cc.current_error() == wsrep::e_interrupted_error);
+    BOOST_REQUIRE(tc.certified() == false);
+    BOOST_REQUIRE(tc.ordered() == false);
+
+    // Rollback sequence
+    BOOST_REQUIRE(cc.before_rollback() == 0);
+    BOOST_REQUIRE(tc.state() == wsrep::transaction_context::s_aborting);
+    BOOST_REQUIRE(cc.after_rollback() == 0);
+    BOOST_REQUIRE(tc.state() == wsrep::transaction_context::s_aborted);
+
+    // Cleanup after statement
+    cc.after_statement();
+    BOOST_REQUIRE(tc.active() == false);
+    BOOST_REQUIRE(tc.ordered() == false);
+    BOOST_REQUIRE(tc.certified() == false);
+    BOOST_REQUIRE(cc.current_error());
+}
+
+
+//
+// Test a 1PC transaction which gets BF aborted during before_commit via
+// provider after the write set was ordered and certified.
+//
+BOOST_FIXTURE_TEST_CASE_TEMPLATE(
+    transaction_context_1pc_bf_during_before_commit_certified, T,
+    replicating_fixtures, T)
+{
+    wsrep::mock_server_context& sc(T::sc);
+    wsrep::client_context& cc(T::cc);
+    const wsrep::transaction_context& tc(T::tc);
+
+    // Start a new transaction with ID 1
+    cc.start_transaction(1);
+    BOOST_REQUIRE(tc.active());
+    BOOST_REQUIRE(tc.id() == wsrep::transaction_id(1));
+    BOOST_REQUIRE(tc.state() == wsrep::transaction_context::s_executing);
+
+    wsrep_test::bf_abort_provider(sc, tc, 1);
+
+    // Run before commit
+    BOOST_REQUIRE(cc.before_commit());
+    BOOST_REQUIRE(tc.state() == wsrep::transaction_context::s_must_replay);
+    BOOST_REQUIRE(tc.certified() == false);
+    BOOST_REQUIRE(tc.ordered() == true);
+
+    // Rollback sequence
+    BOOST_REQUIRE(cc.before_rollback() == 0);
+    BOOST_REQUIRE(tc.state() == wsrep::transaction_context::s_must_replay);
+    BOOST_REQUIRE(cc.after_rollback() == 0);
+    BOOST_REQUIRE(tc.state() == wsrep::transaction_context::s_must_replay);
+
+    // Cleanup after statement
+    cc.after_statement();
+    BOOST_REQUIRE(tc.active() == false);
+    BOOST_REQUIRE(tc.ordered() == false);
+    BOOST_REQUIRE(tc.certified() == false);
+    BOOST_REQUIRE(cc.current_error() == wsrep::e_success);
+}
+
 //
 // Test a transaction which gets BF aborted before before_statement.
 //
@@ -413,46 +570,6 @@ BOOST_FIXTURE_TEST_CASE_TEMPLATE(
     BOOST_REQUIRE(cc.current_error());
 }
 
-//
-// Test a 1PC transaction which gets BF aborted during before_commit via
-// provider after the write set was ordered and certified.
-//
-BOOST_FIXTURE_TEST_CASE_TEMPLATE(
-    transaction_context_1pc_bf_during_before_commit_certified, T,
-    replicating_fixtures, T)
-{
-    wsrep::mock_server_context& sc(T::sc);
-    wsrep::client_context& cc(T::cc);
-    const wsrep::transaction_context& tc(T::tc);
-
-    // Start a new transaction with ID 1
-    cc.start_transaction(1);
-    BOOST_REQUIRE(tc.active());
-    BOOST_REQUIRE(tc.id() == wsrep::transaction_id(1));
-    BOOST_REQUIRE(tc.state() == wsrep::transaction_context::s_executing);
-
-    wsrep_test::bf_abort_provider(sc, tc, 1);
-
-    // Run before commit
-    BOOST_REQUIRE(cc.before_commit());
-    BOOST_REQUIRE(tc.state() == wsrep::transaction_context::s_must_replay);
-    BOOST_REQUIRE(tc.certified() == false);
-    BOOST_REQUIRE(tc.ordered() == true);
-
-    // Rollback sequence
-    BOOST_REQUIRE(cc.before_rollback() == 0);
-    BOOST_REQUIRE(tc.state() == wsrep::transaction_context::s_must_replay);
-    BOOST_REQUIRE(cc.after_rollback() == 0);
-    BOOST_REQUIRE(tc.state() == wsrep::transaction_context::s_must_replay);
-
-    // Cleanup after statement
-    cc.after_statement();
-    BOOST_REQUIRE(tc.active() == false);
-    BOOST_REQUIRE(tc.ordered() == false);
-    BOOST_REQUIRE(tc.certified() == false);
-    BOOST_REQUIRE(cc.current_error() == wsrep::e_success);
-}
-
 BOOST_FIXTURE_TEST_CASE_TEMPLATE(
     transaction_context_1pc_bf_abort_after_after_statement, T,
     replicating_fixtures, T)