More unit tests for various error conditions on certification.

include/wsrep/client_context.hpp

@@ -470,7 +470,7 @@ namespace wsrep
          * only for critical conditions which would sacrifice data
          * consistency.
          */
-        virtual void abort() const = 0;
+        virtual void abort() = 0;
 
     public:
         /*!
@@ -485,7 +485,7 @@ namespace wsrep
         /*!
          * Enter debug synchronization point.
          */
-        virtual void debug_sync(const char*) = 0;
+        virtual void debug_sync(wsrep::unique_lock<wsrep::mutex>&, const char*) = 0;
 
         /*!
          *

include/wsrep/logger.hpp

@@ -38,6 +38,13 @@ namespace wsrep
     };
 
 
+    class log_warning : public log
+    {
+    public:
+        log_warning()
+            : log("WARNING") { }
+    };
+
     class log_debug : public log
     {
     public:

src/dbms_simulator.cpp

@@ -385,14 +385,14 @@ private:
         return 0;
     }
     bool killed() const override { return false; }
-    void abort() const override { ::abort(); }
+    void abort() override { ::abort(); }
 public:
     void store_globals() override
     {
         wsrep::client_context::store_globals();
     }
 private:
-    void debug_sync(const char*) override { }
+    void debug_sync(wsrep::unique_lock<wsrep::mutex>&, const char*) override { }
     void debug_suicide(const char*) override { }
     void on_error(enum wsrep::client_error) override { }
 

src/mock_client_context.hpp

@@ -31,6 +31,9 @@ namespace wsrep
             , bf_abort_during_wait_()
             , error_during_prepare_data_()
             , killed_before_certify_()
+            , sync_point_action_()
+            , replays_()
+            , aborts_()
         { }
         ~mock_client_context()
         {
@@ -51,6 +54,7 @@ namespace wsrep
             tc.state(lock, wsrep::transaction_context::s_committing);
             tc.state(lock, wsrep::transaction_context::s_ordered_commit);
             tc.state(lock, wsrep::transaction_context::s_committed);
+            ++replays_;
             return 0;
         }
         void wait_for_replayers(wsrep::unique_lock<wsrep::mutex>& lock)
@@ -76,15 +80,26 @@ namespace wsrep
 
         }
         bool killed() const WSREP_OVERRIDE { return killed_before_certify_; }
-        void abort() const WSREP_OVERRIDE { }
+        void abort() WSREP_OVERRIDE { ++aborts_; }
         void store_globals() WSREP_OVERRIDE { }
-        void debug_sync(const char*) WSREP_OVERRIDE { }
+        void debug_sync(wsrep::unique_lock<wsrep::mutex>& lock,
+                        const char* sync_point) WSREP_OVERRIDE
+        {
+            lock.unlock();
+            if (sync_point_action_ == sync_point)
+            {
+                wsrep_test::bf_abort_ordered(*this);
+            }
+            lock.lock();
+        }
         void debug_suicide(const char*) WSREP_OVERRIDE
         {
             ::abort();
         }
         void on_error(enum wsrep::client_error) { }
 
+        size_t replays() const { return replays_; }
+        size_t aborts() const { return aborts_; }
         //
     private:
         wsrep::default_mutex mutex_;
@@ -95,6 +110,10 @@ namespace wsrep
         bool bf_abort_during_wait_;
         bool error_during_prepare_data_;
         bool killed_before_certify_;
+        std::string sync_point_action_;
+    private:
+        size_t replays_;
+        size_t aborts_;
     };
 }
 

src/mock_provider.hpp

@@ -43,6 +43,10 @@ namespace wsrep
                 wsrep::ws_meta& ws_meta)
         {
             assert(flags & wsrep::provider::flag::start_transaction);
+            if (next_error_)
+            {
+                return next_error_;
+            }
             if ((flags & wsrep::provider::flag::commit) == 0)
             {
                 return wsrep::provider::error_provider_failed;

src/test_utils.cpp

@@ -11,10 +11,16 @@
 void wsrep_test::bf_abort_unordered(wsrep::client_context& cc)
 {
     wsrep::unique_lock<wsrep::mutex> lock(cc.mutex());
-    assert(cc.transaction().seqno().nil());
+    assert(cc.transaction().ordered() == false);
     cc.bf_abort(lock, 1);
 }
 
+void wsrep_test::bf_abort_ordered(wsrep::client_context& cc)
+{
+    wsrep::unique_lock<wsrep::mutex> lock(cc.mutex());
+    assert(cc.transaction().ordered());
+    cc.bf_abort(lock, 0);
+}
 // BF abort method to abort transactions via provider
 void wsrep_test::bf_abort_provider(wsrep::mock_server_context& sc,
                                    const wsrep::transaction_context& tc,

src/test_utils.hpp

@@ -21,6 +21,9 @@ namespace wsrep_test
     // Simple BF abort method to BF abort unordered transasctions
     void bf_abort_unordered(wsrep::client_context& cc);
 
+    // Simple BF abort method to BF abort unordered transasctions
+    void bf_abort_ordered(wsrep::client_context& cc);
+
     // BF abort method to abort transactions via provider
     void bf_abort_provider(wsrep::mock_server_context& sc,
                            const wsrep::transaction_context& tc,

src/transaction_context.cpp

@@ -725,7 +725,7 @@ int wsrep::transaction_context::certify_commit(
     lock.lock();
 
     assert(state() == s_certifying || state() == s_must_abort);
-    client_context_.debug_sync("wsrep_after_replication");
+    client_context_.debug_sync(lock, "wsrep_after_certification");
 
     int ret(1);
     switch (cert_ret)
@@ -761,8 +761,8 @@ int wsrep::transaction_context::certify_commit(
         state(lock, s_must_abort);
         // The execution should never reach this point if the
         // transaction has not generated any keys or data.
+        wsrep::log_warning() << "Transaction was missing in provider";
         client_context_.override_error(wsrep::e_error_during_commit);
-        assert(0);
         break;
     case wsrep::provider::error_bf_abort:
         // Transaction was replicated succesfully and it was either
@@ -797,15 +797,20 @@ int wsrep::transaction_context::certify_commit(
         client_context_.override_error(wsrep::e_error_during_commit);
         break;
     case wsrep::provider::error_fatal:
+        client_context_.override_error(wsrep::e_error_during_commit);
+        state(lock, s_must_abort);
         client_context_.abort();
         break;
     case wsrep::provider::error_not_implemented:
     case wsrep::provider::error_not_allowed:
         client_context_.override_error(wsrep::e_error_during_commit);
         state(lock, s_must_abort);
-        assert(0);
+        wsrep::log_warning() << "Certification operation was not allowed: "
+                             << "id: " << id().get()
+                             << " flags: " << std::hex << flags() << std::dec;
         break;
     default:
+        state(lock, s_must_abort);
         client_context_.override_error(wsrep::e_error_during_commit);
         break;
     }

src/transaction_context_test.cpp

@@ -550,6 +550,334 @@ BOOST_FIXTURE_TEST_CASE_TEMPLATE(
     BOOST_REQUIRE(cc.current_error() == wsrep::e_success);
 }
 
+//
+// Test a 1PC transaction which gets "warning error" from certify call
+//
+BOOST_FIXTURE_TEST_CASE_TEMPLATE(
+    transaction_context_1pc_warning_error_from_certify, T,
+    replicating_fixtures, T)
+{
+    wsrep::mock_server_context& sc(T::sc);
+    wsrep::client_context& cc(T::cc);
+    const wsrep::transaction_context& tc(T::tc);
+
+    // Start a new transaction with ID 1
+    cc.start_transaction(1);
+    BOOST_REQUIRE(tc.active());
+    BOOST_REQUIRE(tc.id() == wsrep::transaction_id(1));
+    BOOST_REQUIRE(tc.state() == wsrep::transaction_context::s_executing);
+
+    sc.provider().inject_error(wsrep::provider::error_warning);
+
+    // Run before commit
+    BOOST_REQUIRE(cc.before_commit());
+    BOOST_REQUIRE(tc.state() == wsrep::transaction_context::s_must_abort);
+    BOOST_REQUIRE(tc.certified() == false);
+    BOOST_REQUIRE(tc.ordered() == false);
+
+    sc.provider().inject_error(wsrep::provider::success);
+
+    // Rollback sequence
+    BOOST_REQUIRE(cc.before_rollback() == 0);
+    BOOST_REQUIRE(tc.state() == wsrep::transaction_context::s_aborting);
+    BOOST_REQUIRE(cc.after_rollback() == 0);
+    BOOST_REQUIRE(tc.state() == wsrep::transaction_context::s_aborted);
+
+    // Cleanup after statement
+    cc.after_statement();
+    BOOST_REQUIRE(tc.active() == false);
+    BOOST_REQUIRE(tc.ordered() == false);
+    BOOST_REQUIRE(tc.certified() == false);
+    BOOST_REQUIRE(cc.current_error() == wsrep::e_error_during_commit);
+}
+
+//
+// Test a 1PC transaction which gets transaction missing from certify call
+//
+BOOST_FIXTURE_TEST_CASE_TEMPLATE(
+    transaction_context_1pc_transaction_missing_from_certify, T,
+    replicating_fixtures, T)
+{
+    wsrep::mock_server_context& sc(T::sc);
+    wsrep::client_context& cc(T::cc);
+    const wsrep::transaction_context& tc(T::tc);
+
+    // Start a new transaction with ID 1
+    cc.start_transaction(1);
+    BOOST_REQUIRE(tc.active());
+    BOOST_REQUIRE(tc.id() == wsrep::transaction_id(1));
+    BOOST_REQUIRE(tc.state() == wsrep::transaction_context::s_executing);
+
+    sc.provider().inject_error(wsrep::provider::error_transaction_missing);
+
+    // Run before commit
+    BOOST_REQUIRE(cc.before_commit());
+    BOOST_REQUIRE(tc.state() == wsrep::transaction_context::s_must_abort);
+    BOOST_REQUIRE(tc.certified() == false);
+    BOOST_REQUIRE(tc.ordered() == false);
+
+    sc.provider().inject_error(wsrep::provider::success);
+
+    // Rollback sequence
+    BOOST_REQUIRE(cc.before_rollback() == 0);
+    BOOST_REQUIRE(tc.state() == wsrep::transaction_context::s_aborting);
+    BOOST_REQUIRE(cc.after_rollback() == 0);
+    BOOST_REQUIRE(tc.state() == wsrep::transaction_context::s_aborted);
+
+    // Cleanup after statement
+    cc.after_statement();
+    BOOST_REQUIRE(tc.active() == false);
+    BOOST_REQUIRE(tc.ordered() == false);
+    BOOST_REQUIRE(tc.certified() == false);
+    BOOST_REQUIRE(cc.current_error() == wsrep::e_error_during_commit);
+}
+
+//
+// Test a 1PC transaction which gets size exceeded error from certify call
+//
+BOOST_FIXTURE_TEST_CASE_TEMPLATE(
+    transaction_context_1pc_size_exceeded_from_certify, T,
+    replicating_fixtures, T)
+{
+    wsrep::mock_server_context& sc(T::sc);
+    wsrep::client_context& cc(T::cc);
+    const wsrep::transaction_context& tc(T::tc);
+
+    // Start a new transaction with ID 1
+    cc.start_transaction(1);
+    BOOST_REQUIRE(tc.active());
+    BOOST_REQUIRE(tc.id() == wsrep::transaction_id(1));
+    BOOST_REQUIRE(tc.state() == wsrep::transaction_context::s_executing);
+
+    sc.provider().inject_error(wsrep::provider::error_size_exceeded);
+
+    // Run before commit
+    BOOST_REQUIRE(cc.before_commit());
+    BOOST_REQUIRE(tc.state() == wsrep::transaction_context::s_must_abort);
+    BOOST_REQUIRE(tc.certified() == false);
+    BOOST_REQUIRE(tc.ordered() == false);
+
+    sc.provider().inject_error(wsrep::provider::success);
+
+    // Rollback sequence
+    BOOST_REQUIRE(cc.before_rollback() == 0);
+    BOOST_REQUIRE(tc.state() == wsrep::transaction_context::s_aborting);
+    BOOST_REQUIRE(cc.after_rollback() == 0);
+    BOOST_REQUIRE(tc.state() == wsrep::transaction_context::s_aborted);
+
+    // Cleanup after statement
+    cc.after_statement();
+    BOOST_REQUIRE(tc.active() == false);
+    BOOST_REQUIRE(tc.ordered() == false);
+    BOOST_REQUIRE(tc.certified() == false);
+    BOOST_REQUIRE(cc.current_error() == wsrep::e_error_during_commit);
+}
+
+//
+// Test a 1PC transaction which gets connection failed error from certify call
+//
+BOOST_FIXTURE_TEST_CASE_TEMPLATE(
+    transaction_context_1pc_connection_failed_from_certify, T,
+    replicating_fixtures, T)
+{
+    wsrep::mock_server_context& sc(T::sc);
+    wsrep::client_context& cc(T::cc);
+    const wsrep::transaction_context& tc(T::tc);
+
+    // Start a new transaction with ID 1
+    cc.start_transaction(1);
+    BOOST_REQUIRE(tc.active());
+    BOOST_REQUIRE(tc.id() == wsrep::transaction_id(1));
+    BOOST_REQUIRE(tc.state() == wsrep::transaction_context::s_executing);
+
+    sc.provider().inject_error(wsrep::provider::error_connection_failed);
+
+    // Run before commit
+    BOOST_REQUIRE(cc.before_commit());
+    BOOST_REQUIRE(tc.state() == wsrep::transaction_context::s_must_abort);
+    BOOST_REQUIRE(tc.certified() == false);
+    BOOST_REQUIRE(tc.ordered() == false);
+
+    sc.provider().inject_error(wsrep::provider::success);
+
+    // Rollback sequence
+    BOOST_REQUIRE(cc.before_rollback() == 0);
+    BOOST_REQUIRE(tc.state() == wsrep::transaction_context::s_aborting);
+    BOOST_REQUIRE(cc.after_rollback() == 0);
+    BOOST_REQUIRE(tc.state() == wsrep::transaction_context::s_aborted);
+
+    // Cleanup after statement
+    cc.after_statement();
+    BOOST_REQUIRE(tc.active() == false);
+    BOOST_REQUIRE(tc.ordered() == false);
+    BOOST_REQUIRE(tc.certified() == false);
+    BOOST_REQUIRE(cc.current_error() == wsrep::e_error_during_commit);
+}
+
+//
+// Test a 1PC transaction which gets not allowed error from certify call
+//
+BOOST_FIXTURE_TEST_CASE_TEMPLATE(
+    transaction_context_1pc_no_allowed_from_certify, T,
+    replicating_fixtures, T)
+{
+    wsrep::mock_server_context& sc(T::sc);
+    wsrep::client_context& cc(T::cc);
+    const wsrep::transaction_context& tc(T::tc);
+
+    // Start a new transaction with ID 1
+    cc.start_transaction(1);
+    BOOST_REQUIRE(tc.active());
+    BOOST_REQUIRE(tc.id() == wsrep::transaction_id(1));
+    BOOST_REQUIRE(tc.state() == wsrep::transaction_context::s_executing);
+
+    sc.provider().inject_error(wsrep::provider::error_not_allowed);
+
+    // Run before commit
+    BOOST_REQUIRE(cc.before_commit());
+    BOOST_REQUIRE(tc.state() == wsrep::transaction_context::s_must_abort);
+    BOOST_REQUIRE(tc.certified() == false);
+    BOOST_REQUIRE(tc.ordered() == false);
+
+    sc.provider().inject_error(wsrep::provider::success);
+
+    // Rollback sequence
+    BOOST_REQUIRE(cc.before_rollback() == 0);
+    BOOST_REQUIRE(tc.state() == wsrep::transaction_context::s_aborting);
+    BOOST_REQUIRE(cc.after_rollback() == 0);
+    BOOST_REQUIRE(tc.state() == wsrep::transaction_context::s_aborted);
+
+    // Cleanup after statement
+    cc.after_statement();
+    BOOST_REQUIRE(tc.active() == false);
+    BOOST_REQUIRE(tc.ordered() == false);
+    BOOST_REQUIRE(tc.certified() == false);
+    BOOST_REQUIRE(cc.current_error() == wsrep::e_error_during_commit);
+}
+
+//
+// Test a 1PC transaction which gets fatal error from certify call
+//
+BOOST_FIXTURE_TEST_CASE_TEMPLATE(
+    transaction_context_1pc_fatal_from_certify, T,
+    replicating_fixtures, T)
+{
+    wsrep::mock_server_context& sc(T::sc);
+    wsrep::mock_client_context& cc(T::cc);
+    const wsrep::transaction_context& tc(T::tc);
+
+    // Start a new transaction with ID 1
+    cc.start_transaction(1);
+    BOOST_REQUIRE(tc.active());
+    BOOST_REQUIRE(tc.id() == wsrep::transaction_id(1));
+    BOOST_REQUIRE(tc.state() == wsrep::transaction_context::s_executing);
+
+    sc.provider().inject_error(wsrep::provider::error_fatal);
+
+    // Run before commit
+    BOOST_REQUIRE(cc.before_commit());
+    BOOST_REQUIRE(tc.state() == wsrep::transaction_context::s_must_abort);
+    BOOST_REQUIRE(tc.certified() == false);
+    BOOST_REQUIRE(tc.ordered() == false);
+
+    sc.provider().inject_error(wsrep::provider::success);
+
+    // Rollback sequence
+    BOOST_REQUIRE(cc.before_rollback() == 0);
+    BOOST_REQUIRE(tc.state() == wsrep::transaction_context::s_aborting);
+    BOOST_REQUIRE(cc.after_rollback() == 0);
+    BOOST_REQUIRE(tc.state() == wsrep::transaction_context::s_aborted);
+
+    // Cleanup after statement
+    cc.after_statement();
+    BOOST_REQUIRE(tc.active() == false);
+    BOOST_REQUIRE(tc.ordered() == false);
+    BOOST_REQUIRE(tc.certified() == false);
+    BOOST_REQUIRE(cc.current_error() == wsrep::e_error_during_commit);
+    BOOST_REQUIRE(cc.aborts() == 1);
+}
+
+//
+// Test a 1PC transaction which gets unknown from certify call
+//
+BOOST_FIXTURE_TEST_CASE_TEMPLATE(
+    transaction_context_1pc_unknown_from_certify, T,
+    replicating_fixtures, T)
+{
+    wsrep::mock_server_context& sc(T::sc);
+    wsrep::mock_client_context& cc(T::cc);
+    const wsrep::transaction_context& tc(T::tc);
+
+    // Start a new transaction with ID 1
+    cc.start_transaction(1);
+    BOOST_REQUIRE(tc.active());
+    BOOST_REQUIRE(tc.id() == wsrep::transaction_id(1));
+    BOOST_REQUIRE(tc.state() == wsrep::transaction_context::s_executing);
+
+    sc.provider().inject_error(wsrep::provider::error_unknown);
+
+    // Run before commit
+    BOOST_REQUIRE(cc.before_commit());
+    BOOST_REQUIRE(tc.state() == wsrep::transaction_context::s_must_abort);
+    BOOST_REQUIRE(tc.certified() == false);
+    BOOST_REQUIRE(tc.ordered() == false);
+
+    sc.provider().inject_error(wsrep::provider::success);
+
+    // Rollback sequence
+    BOOST_REQUIRE(cc.before_rollback() == 0);
+    BOOST_REQUIRE(tc.state() == wsrep::transaction_context::s_aborting);
+    BOOST_REQUIRE(cc.after_rollback() == 0);
+    BOOST_REQUIRE(tc.state() == wsrep::transaction_context::s_aborted);
+
+    // Cleanup after statement
+    cc.after_statement();
+    BOOST_REQUIRE(tc.active() == false);
+    BOOST_REQUIRE(tc.ordered() == false);
+    BOOST_REQUIRE(tc.certified() == false);
+    BOOST_REQUIRE(cc.current_error() == wsrep::e_error_during_commit);
+}
+
+//
+// Test a 1PC transaction which gets BF aborted before grabbing lock
+// after certify call
+//
+BOOST_FIXTURE_TEST_CASE_TEMPLATE(
+    transaction_context_1pc_bf_abort_after_certify_regain_lock, T,
+    replicating_fixtures, T)
+{
+    // wsrep::mock_server_context& sc(T::sc);
+    wsrep::mock_client_context& cc(T::cc);
+    const wsrep::transaction_context& tc(T::tc);
+
+    // Start a new transaction with ID 1
+    cc.start_transaction(1);
+    BOOST_REQUIRE(tc.active());
+    BOOST_REQUIRE(tc.id() == wsrep::transaction_id(1));
+    BOOST_REQUIRE(tc.state() == wsrep::transaction_context::s_executing);
+
+    cc.sync_point_action_ = "wsrep_after_certification";
+    // Run before commit
+    BOOST_REQUIRE(cc.before_commit());
+    BOOST_REQUIRE(tc.state() == wsrep::transaction_context::s_must_replay);
+    BOOST_REQUIRE(tc.certified() == true);
+    BOOST_REQUIRE(tc.ordered() == true);
+
+    // Rollback sequence
+    BOOST_REQUIRE(cc.before_rollback() == 0);
+    BOOST_REQUIRE(tc.state() == wsrep::transaction_context::s_must_replay);
+    BOOST_REQUIRE(cc.after_rollback() == 0);
+    BOOST_REQUIRE(tc.state() == wsrep::transaction_context::s_must_replay);
+
+    // Cleanup after statement
+    cc.after_statement();
+    BOOST_REQUIRE(cc.replays() == 1);
+    BOOST_REQUIRE(tc.active() == false);
+    BOOST_REQUIRE(tc.ordered() == false);
+    BOOST_REQUIRE(tc.certified() == false);
+    BOOST_REQUIRE(cc.current_error() == wsrep::e_success);
+}
+
 //
 // Test a 2PC transaction which gets BF aborted when trying to grab
 // commit order.